From e2c868c4bdd4f83f1e33a50bca0e3d91d4605387 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 18 Jul 2026 03:34:51 +0000 Subject: [PATCH] refactor(domain): split analysis boundaries --- src/contracts/toolOutputSchemaGroups.ts | 324 +++++++++ src/contracts/toolOutputSchemaPrimitives.ts | 380 ++++++++++ src/contracts/toolOutputSchemas.ts | 657 +----------------- src/domain/artifactComparison.ts | 60 +- src/domain/browserObservation.ts | 306 +------- src/domain/browserObservationSchemas.ts | 302 ++++++++ src/domain/functionComparison.ts | 471 +------------ src/domain/functionComparisonDimensions.ts | 462 ++++++++++++ .../javascriptApplicationVersionItems.ts | 42 +- .../javascriptApplicationVersionKeys.ts | 32 +- src/domain/javascriptFeatureTraversal.ts | 28 +- src/domain/javascriptStaticAnalysis.ts | 586 +--------------- src/domain/javascriptStaticAnalysisBundler.ts | 339 +++++++++ src/domain/javascriptStaticAnalysisCalls.ts | 258 +++++++ src/domain/webBundleAnalyzer.ts | 630 +---------------- src/domain/webBundleAnalyzerAst.ts | 105 +++ src/domain/webBundleAnalyzerInspection.ts | 453 ++++++++++++ src/domain/webBundleAnalyzerResult.ts | 164 +++++ 18 files changed, 2920 insertions(+), 2679 deletions(-) create mode 100644 src/contracts/toolOutputSchemaGroups.ts create mode 100644 src/contracts/toolOutputSchemaPrimitives.ts create mode 100644 src/domain/browserObservationSchemas.ts create mode 100644 src/domain/functionComparisonDimensions.ts create mode 100644 src/domain/javascriptStaticAnalysisBundler.ts create mode 100644 src/domain/javascriptStaticAnalysisCalls.ts create mode 100644 src/domain/webBundleAnalyzerAst.ts create mode 100644 src/domain/webBundleAnalyzerInspection.ts create mode 100644 src/domain/webBundleAnalyzerResult.ts diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts new file mode 100644 index 0000000000..a00682c7c9 --- /dev/null +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -0,0 +1,324 @@ +import { z } from "zod"; + +import { + processCaptureComparisonSchema, + processCaptureSchema, +} from "../domain/processCapture.js"; +import { evidenceBundleSchema } from "../domain/evidenceBundle.js"; +import { residualUnknownSchema } from "../domain/residualUnknown.js"; +import { referenceKindSchema } from "../domain/hopperValues.js"; +import { + demangleSwiftSchema, + inspectMachoSchema, + inspectPlistSchema, + inspectSignatureSchema, + listArchitecturesSchema, +} from "../domain/nativeInspection.js"; +import { + artifactExtractionResultSchema, + artifactInventoryResultSchema, +} from "../domain/artifactGraph.js"; +import { + managedArtifactInspectionSchema, + managedMemberInspectionSchema, + managedNativeBoundaryInspectionSchema, +} from "../domain/managedArtifact.js"; +import { managedMemberComparisonResultSchema } from "../domain/managedMemberComparison.js"; +import { managedNativeVerificationResultSchema } from "../domain/managedNativeVerification.js"; +import { managedReconstructionImportResultSchema } from "../domain/managedReconstruction.js"; +import { managedRuntimeCorrelationResultSchema } from "../domain/managedRuntimeCorrelation.js"; +import { managedApplicationGraphResultSchema } from "../domain/managedApplicationGraph.js"; +import { artifactComparisonResultSchema } from "../domain/artifactComparison.js"; +import { functionComparisonResultSchema } from "../domain/functionComparison.js"; +import { bundleComparisonResultSchema } from "../domain/bundleComparison.js"; +import { changedBehaviorResultSchema } from "../domain/changedBehavior.js"; +import { callPathResultSchema } from "../domain/callPath.js"; +import { staticRuntimeCorrelationResultSchema } from "../domain/staticRuntimeCorrelation.js"; +import { reconstructionVerificationResultSchema } from "../domain/reconstructionVerification.js"; +import { analysisErrorProjectionSchema } from "./errorSchemas.js"; +import { + addressList, + addressedEntry, + bounded, + containingProcedureResolution, + functionDossierOutput, + graphNode, + lifecycleResultOf, + nullableText, + pageOutput, + procedureIdentity, + procedureInfoOutput, + resultOf, + searchPageOutput, + segmentOutput, + sessionProvider, + symbolDiscoveryOutput, + targetFormatSchema, + targetKindSchema, +} from "./toolOutputSchemaPrimitives.js"; + +/** Exact structured-content schemas shared by direct analysis providers. */ +export const officialOutputSchemas: Readonly> = { + address_name: resultOf(nullableText), + comment: resultOf(nullableText), + current_address: resultOf(z.string()), + current_procedure: resultOf(z.string()), + current_document: resultOf(z.string()), + goto_address: resultOf(z.string()), + inline_comment: resultOf(nullableText), + list_bookmarks: resultOf(z.array(addressedEntry)), + list_documents: resultOf(z.array(z.string())), + list_names: resultOf(pageOutput), + list_procedures: resultOf(pageOutput), + list_segments: segmentOutput, + list_strings: resultOf(pageOutput), + next_address: resultOf(z.string()), + prev_address: resultOf(z.string()), + procedure_address: resultOf(z.string()), + procedure_assembly: resultOf(z.string()), + procedure_callees: resultOf(addressList), + procedure_callers: resultOf(addressList), + procedure_info: procedureInfoOutput, + procedure_references: resultOf( + z.object({ + procedure: procedureIdentity, + direction: z.enum(["incoming", "outgoing"]), + references: bounded( + z.object({ + source_address: z.string(), + target_address: z.string(), + source_procedure: procedureIdentity.nullable(), + target_procedure: procedureIdentity.nullable(), + kind: referenceKindSchema, + }), + ), + instructions_scanned: z.number().int().min(0), + instruction_scan_truncated: z.boolean(), + }), + ), + procedure_pseudo_code: resultOf(nullableText), + resolve_containing_procedure: resultOf(containingProcedureResolution), + search_procedures: resultOf(searchPageOutput), + search_strings: resultOf(searchPageOutput), + set_address_name: resultOf(z.boolean()), + set_addresses_names: resultOf(z.record(z.string(), z.boolean())), + set_bookmark: resultOf(z.boolean()), + set_comment: resultOf(z.boolean()), + set_current_document: resultOf(z.string()), + set_inline_comment: resultOf(z.boolean()), + unset_bookmark: resultOf(z.boolean()), + xrefs: resultOf(addressList), +}; + +/** Exact structured-content schemas for composed analysis workflows. */ +export const enhancedOutputSchemas: Readonly> = { + swift_classes: symbolDiscoveryOutput("classes"), + get_objc_classes: symbolDiscoveryOutput("classes"), + get_objc_protocols: symbolDiscoveryOutput("protocols"), + batch_decompile: resultOf( + z.object({ + items: z.array( + z.discriminatedUnion("status", [ + z.object({ + address: z.string(), + status: z.literal("ok"), + pseudocode: z.string().min(1), + }), + z.object({ + address: z.string(), + status: z.literal("error"), + error: analysisErrorProjectionSchema, + }), + ]), + ), + total: z.number().int().min(0), + succeeded: z.number().int().min(0), + failed: z.number().int().min(0), + }), + ), + get_call_graph: resultOf(z.record(z.string(), z.array(graphNode))), + analyze_swift_types: resultOf( + z.object({ + total: z.number().int().min(0), + categories: z.record( + z.string(), + z.object({ + count: z.number().int().min(0), + items: z.array(addressedEntry), + }), + ), + }), + ), + find_xrefs_to_name: resultOf( + z.discriminatedUnion("status", [ + z.object({ + status: z.literal("resolved"), + name: z.string(), + address: z.string(), + xrefs: addressList, + }), + z.object({ + status: z.literal("unresolved"), + name: z.string(), + reason: z.literal("name_not_found"), + }), + ]), + ), + binary_overview: resultOf( + z.object({ + document: z.string(), + detail: z.enum(["concise", "detailed"]), + segments: z.array( + z.object({ + name: z.string(), + start: z.string(), + end: z.string(), + length: z.number().min(0).optional(), + }), + ), + segment_count: z.number().int().min(0), + procedure_count: z.number().int().min(0), + string_count: z.number().int().min(0), + }), + ), + analyze_function: functionDossierOutput, + trace_feature: resultOf( + z.object({ + query: z.string(), + search_mode: z.literal("literal"), + operations_used: z.number().int().min(0), + operation_budget: z.number().int().min(1), + matches: z.array( + z.object({ + type: z.enum(["string", "procedure"]), + address: z.string(), + value: z.string(), + }), + ), + references: z.array( + z.object({ + target_address: z.string(), + source_address: z.string(), + containing_procedure: containingProcedureResolution, + }), + ), + truncated: z.boolean(), + residual_unknowns: z.array(z.string()), + }), + ), +}; + +/** Exact Evidence v2 schemas for provider-neutral native inspection. */ +export const nativeOutputSchemas: Readonly> = { + inspect_macho: resultOf(inspectMachoSchema), + inspect_signature: resultOf(inspectSignatureSchema), + inspect_plist: resultOf(inspectPlistSchema), + list_architectures: resultOf(listArchitecturesSchema), + demangle_swift: resultOf(demangleSwiftSchema), +}; + +/** Exact Evidence v2 schemas for provider-neutral artifact graph operations. */ +export const artifactOutputSchemas: Readonly> = { + inventory_artifact: resultOf(artifactInventoryResultSchema), + extract_artifact: resultOf(artifactExtractionResultSchema), +}; + +/** Exact Evidence v2 schema for execution-free managed static analysis. */ +export const managedOutputSchemas: Readonly> = { + inspect_managed_artifact: resultOf(managedArtifactInspectionSchema), + inspect_managed_members: resultOf(managedMemberInspectionSchema), + inspect_managed_native_boundaries: resultOf( + managedNativeBoundaryInspectionSchema, + ), +}; + +/** Exact Evidence v2 schema for provider-neutral managed workflows. */ +export const managedWorkflowOutputSchemas: Readonly< + Record +> = { + compare_managed_members: resultOf(managedMemberComparisonResultSchema), + verify_managed_native_boundaries: resultOf( + managedNativeVerificationResultSchema, + ), + import_managed_reconstruction: resultOf( + managedReconstructionImportResultSchema, + ), + plan_managed_runtime_correlation: resultOf( + managedRuntimeCorrelationResultSchema, + ), + project_managed_application_graph: resultOf( + managedApplicationGraphResultSchema, + ), +}; + +/** Exact structured-content schemas for target lifecycle operations. */ +export const sessionOutputSchemas: Readonly> = { + open_binary: lifecycleResultOf( + z.object({ + path: z.string(), + format: targetFormatSchema, + kind: targetKindSchema, + loaderArgs: z.array(z.string()), + sha256: z.string().regex(/^[a-f0-9]{64}$/u), + architecture: z.enum(["x86", "x86_64", "arm", "arm64"]).nullable(), + }), + ), + close_binary: lifecycleResultOf( + z.union([ + z.null(), + z.object({ + path: z.string(), + bytes: z.number().int().min(0), + entries: z.number().int().min(0), + }), + ]), + ), + binary_session: lifecycleResultOf( + z.union([ + sessionProvider.extend({ open: z.literal(false) }), + sessionProvider.extend({ + open: z.literal(true), + path: z.string(), + format: targetFormatSchema, + kind: targetKindSchema, + sha256: z.string().regex(/^[a-f0-9]{64}$/u), + architecture: z.enum(["x86", "x86_64", "arm", "arm64"]).nullable(), + }), + ]), + ), + export_evidence_bundle: lifecycleResultOf( + z.union([ + evidenceBundleSchema, + z.object({ + path: z.string(), + bytes: z.number().int().min(0), + records: z.number().int().min(0), + }), + ]), + ), + import_evidence_bundle: lifecycleResultOf( + z.object({ + imported: z.number().int().min(0), + total: z.number().int().min(0), + }), + ), + capture_process_scenario: resultOf(processCaptureSchema), + compare_process_captures: resultOf(processCaptureComparisonSchema), + compare_artifacts: resultOf(artifactComparisonResultSchema), + compare_functions: resultOf(functionComparisonResultSchema), + compare_bundles: resultOf(bundleComparisonResultSchema), + find_changed_behavior: resultOf(changedBehaviorResultSchema), + build_call_path: resultOf(callPathResultSchema), + correlate_static_and_runtime: resultOf(staticRuntimeCorrelationResultSchema), + verify_reconstruction: resultOf(reconstructionVerificationResultSchema), + list_unknowns: lifecycleResultOf(z.array(residualUnknownSchema)), + record_unknown: lifecycleResultOf(residualUnknownSchema), + update_unknown: lifecycleResultOf(residualUnknownSchema), + verify_unknown_resolution: lifecycleResultOf( + z.object({ + valid: z.boolean(), + truthVerified: z.boolean(), + unknown: residualUnknownSchema, + }), + ), +}; diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts new file mode 100644 index 0000000000..26dab8913b --- /dev/null +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -0,0 +1,380 @@ +import { z } from "zod"; + +import { + procedureClassificationSchema, + procedureIdentitySchema, + localVariableSchema, + functionDossierSchema, +} from "../domain/hopperValues.js"; +import { analysisProfileSchema } from "../domain/analysisProfile.js"; +import { + PROVIDER_REJECTION_CODES, + type ProviderRejectionCode, +} from "./providerSelection.js"; +import { analysisErrorProjectionSchema } from "./errorSchemas.js"; + +/** Compact MCP result with an immutable link to complete session Evidence. */ +export const evidenceResultOf = (schema: z.ZodType) => + z.strictObject({ + result: schema, + evidence_id: z.string().regex(/^ev_[a-f0-9]{64}$/u), + evidence_uri: z.string().regex(/^rea:\/\/evidence\/ev_[a-f0-9]{64}$/u), + }); + +export const resultOf = evidenceResultOf; +export const lifecycleResultOf = (schema: z.ZodType) => + z.object({ result: schema }); + +/** Resolve a required named output schema or reject contract drift. */ +export const requireOutputSchema = ( + schemas: Readonly>, + name: string, +): z.ZodObject => { + const schema = schemas[name]; + if (schema === undefined) + throw new Error(`Missing output schema for ${name}`); + return schema; +}; + +export const targetFormatSchema = z.enum([ + "hopper", + "analysis-database", + "mach-o", + "elf", + "pe", + "zip", + "ipa", + "apk", + "asar", + "dmg", + "pkg", + "plist", + "javascript", + "source-map", +]); + +export const targetKindSchema = z.enum([ + "executable", + "database", + "archive", + "artifact", +]); + +export const providerCapability = z.object({ + operation: z.string(), + available: z.boolean(), + reason: z.string().nullable(), + input_contract_version: z.number().int().min(1), + output_contract_version: z.number().int().min(1), + pagination: z.enum(["none", "offset", "cursor"]), + exhaustive: z.boolean(), + effects: z.object({ + mutates_artifact: z.boolean(), + launches_process: z.boolean(), + may_show_ui: z.boolean(), + may_access_network: z.boolean(), + may_write_filesystem: z.boolean(), + changes_permissions: z.boolean(), + requires_root: z.boolean(), + }), + limits: z.object({ + max_results: z.number().int().min(0).nullable(), + max_payload_bytes: z.number().int().min(0).nullable(), + timeout_ms: z.number().int().min(0).nullable(), + }), + limitations: z.array(z.string()), +}); + +export const providerIdentity = z.object({ + id: z.string(), + name: z.string(), + version: z.string().nullable(), +}); + +export const providerRejectionCode: z.ZodType = z.enum( + PROVIDER_REJECTION_CODES, +); + +export const providerDiagnostics = z.record(z.string(), z.json()); + +export const providerAvailability = z.discriminatedUnion("status", [ + z.object({ + status: z.literal("available"), + code: z.null(), + reason: z.null(), + diagnostics: providerDiagnostics, + }), + z.object({ + status: z.literal("unavailable"), + code: providerRejectionCode, + reason: z.string().min(1), + diagnostics: providerDiagnostics, + }), +]); + +export const providerTargetSupport = z.discriminatedUnion("status", [ + z.object({ + status: z.literal("unknown"), + code: z.null(), + reason: z.string().min(1), + diagnostics: providerDiagnostics, + }), + z.object({ + status: z.literal("supported"), + code: z.null(), + reason: z.null(), + diagnostics: providerDiagnostics, + }), + z.object({ + status: z.literal("unsupported"), + code: z.enum([ + "target_kind_unsupported", + "target_format_unsupported", + "architecture_unsupported", + "target_role_unsupported", + "managed_target_unsupported", + ]), + reason: z.string().min(1), + diagnostics: providerDiagnostics, + }), +]); + +export const sessionProvider = z.object({ + provider: providerIdentity, + providers: z.array(providerIdentity), + capabilities: z.array(providerCapability), + analysis_provider_binding: z + .object({ + provider: providerIdentity, + selection_source: z.enum([ + "request", + "environment", + "auto-single-candidate", + ]), + analysis_profile: analysisProfileSchema, + }) + .nullable(), + analysis_provider_candidates: z.array( + z.object({ + provider: providerIdentity, + availability: providerAvailability, + target_support: providerTargetSupport, + selected: z.boolean(), + capabilities: z.array(providerCapability), + }), + ), + tool_availability: z.array( + z.object({ + name: z.string(), + surface: z.string(), + available: z.boolean(), + reason: z.enum([ + "available", + "target_required", + "provider_missing", + "provider_unavailable", + "target_unsupported", + "unsupported_host", + "policy_disabled", + ]), + remediation: z.string().nullable(), + effects: z.strictObject({ + mutatesTarget: z.boolean(), + mutatesSession: z.boolean(), + writesFilesystem: z.boolean(), + launchesProcess: z.boolean(), + accessesNetwork: z.boolean(), + changesUiState: z.boolean(), + mayDiscardData: z.boolean(), + idempotent: z.boolean(), + }), + annotations: z.object({ + read_only: z.boolean(), + destructive: z.boolean(), + idempotent: z.boolean(), + open_world: z.boolean(), + }), + }), + ), + client_features: z.object({ + elicitation_form: z.boolean(), + elicitation_url: z.boolean(), + roots: z.boolean(), + sampling: z.boolean(), + }), + server_identity: z.object({ + package: z.object({ + name: z.string(), + version: z.string(), + root_path: z.string(), + build_commit: z.string().nullable(), + }), + server: z.object({ + name: z.string(), + version: z.string(), + started_at: z.string(), + command_path: z.string(), + }), + sdk: z.object({ + server: z.string(), + client_test: z.string(), + core: z.string(), + }), + negotiated_protocol_version: z.string().nullable(), + client: z.object({ name: z.string(), version: z.string() }).nullable(), + skill: z.object({ name: z.string(), expected_version: z.string() }), + catalog: z.record(z.string(), z.json()), + protocol_features: z.object({ + progress: z.boolean(), + cancellation: z.boolean(), + evidence_resources: z.boolean(), + elicitation: z.boolean(), + }), + alignment: z.object({ + state: z.enum(["aligned", "mcp_server_restart_required", "unknown"]), + reasons: z.array(z.string()), + remediation: z.string().nullable(), + }), + }), +}); + +export const nullableText = z.string().nullable(); +export const addressList = z.array(z.string()); +export const addressedEntry = z.object({ address: z.string(), name: z.string() }); +export const procedureIdentity = procedureIdentitySchema; +export const localVariable = localVariableSchema; + +export const containingProcedureResolution = z.discriminatedUnion("found", [ + z.object({ + query_address: z.string(), + found: z.literal(true), + procedure: procedureIdentity, + }), + z.object({ + query_address: z.string(), + found: z.literal(false), + procedure: z.null(), + reason: z.enum(["outside_segments", "not_in_procedure"]), + }), +]); + +export const unavailable = z.object({ + available: z.literal(false), + reason: z.string(), +}); + +export const availableMemoryPermissions = z.object({ + available: z.literal(true), + source: z.literal("ghidra-memory-block"), +}); + +export const bounded = (item: z.ZodType) => + z.object({ + items: z.array(item), + total: z.number().int().min(0).nullable(), + returned: z.number().int().min(0), + truncated: z.boolean(), + next_offset: z.number().int().min(0).nullable(), + }); + +export const addressedValue = z.object({ + address: z.string(), + value: z.string(), + value_truncated: z.boolean().optional(), + symbol: z + .object({ + primary: z.boolean(), + dynamic: z.boolean(), + external: z.boolean(), + type: z.string(), + source: z.enum(["default", "analysis", "ai", "imported", "user_defined"]), + }) + .optional(), + procedure: z + .object({ + external: z.boolean(), + thunk: z.boolean(), + thunk_target: z.string().nullable(), + }) + .optional(), + string: z + .object({ + encoding: z.string(), + termination: z.enum(["missing", "present_or_not_required"]), + byte_length: z.number().int().min(0), + }) + .optional(), +}); + +export const pageOutput = z.object({ + items: z.array(addressedValue), + offset: z.number().int().min(0), + limit: z.number().int().min(1), + total: z.number().int().min(0), + next_offset: z.number().int().min(0).nullable(), + has_more: z.boolean(), +}); + +export const searchPageOutput = pageOutput.extend({ + items: z.array( + z.object({ + address: z.string(), + value: z.string(), + value_truncated: z.boolean(), + }), + ), +}); + +export const memoryRegionOutput = z.object({ + name: z.string(), + start: z.string(), + end: z.string(), + readable: z.boolean().nullable(), + writable: z.boolean().nullable(), + executable: z.boolean().nullable(), + permissions: z.union([unavailable, availableMemoryPermissions]), + provenance: z.enum(["hopper-public-python-api", "ghidra-memory-block"]), + address_space: z.string().optional(), + image_base: z.string().optional(), + initialized: z.boolean().optional(), + overlay: z.boolean().optional(), +}); + +export const segmentOutput = resultOf( + z.array(memoryRegionOutput.extend({ sections: z.array(memoryRegionOutput) })), +); + +export const procedureInfoOutput = resultOf( + z.object({ + name: z.string(), + entrypoint: z.string(), + basicblock_count: z.number().int().min(0), + length: z.number().min(0), + signature: nullableText, + locals: z.array(localVariable), + classification: procedureClassificationSchema.nullable().default(null), + }), +); + +export const symbolDiscoveryOutput = (property: "classes" | "protocols") => + resultOf( + z.object({ + count: z.number().int().min(0), + [property]: z.array(addressedEntry), + }), + ); + +export const graphNode = z.discriminatedUnion("status", [ + z.object({ + address: z.string(), + status: z.literal("ok"), + calls: z.array(z.string()), + }), + z.object({ + address: z.string(), + status: z.literal("error"), + error: analysisErrorProjectionSchema, + }), +]); + +export const functionDossierOutput = resultOf(functionDossierSchema); diff --git a/src/contracts/toolOutputSchemas.ts b/src/contracts/toolOutputSchemas.ts index 32f41a37f3..8d087b3005 100644 --- a/src/contracts/toolOutputSchemas.ts +++ b/src/contracts/toolOutputSchemas.ts @@ -1,655 +1,2 @@ -import { z } from "zod"; -import { - processCaptureComparisonSchema, - processCaptureSchema, -} from "../domain/processCapture.js"; -import { evidenceBundleSchema } from "../domain/evidenceBundle.js"; -import { residualUnknownSchema } from "../domain/residualUnknown.js"; -import { - functionDossierSchema, - localVariableSchema, - procedureClassificationSchema, - procedureIdentitySchema, - referenceKindSchema, -} from "../domain/hopperValues.js"; -import { - demangleSwiftSchema, - inspectMachoSchema, - inspectPlistSchema, - inspectSignatureSchema, - listArchitecturesSchema, -} from "../domain/nativeInspection.js"; -import { - artifactExtractionResultSchema, - artifactInventoryResultSchema, -} from "../domain/artifactGraph.js"; -import { - managedArtifactInspectionSchema, - managedMemberInspectionSchema, - managedNativeBoundaryInspectionSchema, -} from "../domain/managedArtifact.js"; -import { managedMemberComparisonResultSchema } from "../domain/managedMemberComparison.js"; -import { managedNativeVerificationResultSchema } from "../domain/managedNativeVerification.js"; -import { managedReconstructionImportResultSchema } from "../domain/managedReconstruction.js"; -import { managedRuntimeCorrelationResultSchema } from "../domain/managedRuntimeCorrelation.js"; -import { managedApplicationGraphResultSchema } from "../domain/managedApplicationGraph.js"; -import { artifactComparisonResultSchema } from "../domain/artifactComparison.js"; -import { functionComparisonResultSchema } from "../domain/functionComparison.js"; -import { bundleComparisonResultSchema } from "../domain/bundleComparison.js"; -import { changedBehaviorResultSchema } from "../domain/changedBehavior.js"; -import { callPathResultSchema } from "../domain/callPath.js"; -import { staticRuntimeCorrelationResultSchema } from "../domain/staticRuntimeCorrelation.js"; -import { reconstructionVerificationResultSchema } from "../domain/reconstructionVerification.js"; -import { analysisErrorProjectionSchema } from "./errorSchemas.js"; -import { analysisProfileSchema } from "../domain/analysisProfile.js"; -import { - PROVIDER_REJECTION_CODES, - type ProviderRejectionCode, -} from "./providerSelection.js"; - -/** Compact MCP result with an immutable link to complete session Evidence. */ -export const evidenceResultOf = (schema: z.ZodType) => - z.strictObject({ - result: schema, - evidence_id: z.string().regex(/^ev_[a-f0-9]{64}$/u), - evidence_uri: z.string().regex(/^rea:\/\/evidence\/ev_[a-f0-9]{64}$/u), - }); -const resultOf = evidenceResultOf; -const lifecycleResultOf = (schema: z.ZodType) => z.object({ result: schema }); - -/** Resolve a required named output schema or reject contract drift. */ -export const requireOutputSchema = ( - schemas: Readonly>, - name: string, -): z.ZodObject => { - const schema = schemas[name]; - if (schema === undefined) - throw new Error(`Missing output schema for ${name}`); - return schema; -}; -const targetFormatSchema = z.enum([ - "hopper", - "analysis-database", - "mach-o", - "elf", - "pe", - "zip", - "ipa", - "apk", - "asar", - "dmg", - "pkg", - "plist", - "javascript", - "source-map", -]); -const targetKindSchema = z.enum([ - "executable", - "database", - "archive", - "artifact", -]); -const providerCapability = z.object({ - operation: z.string(), - available: z.boolean(), - reason: z.string().nullable(), - input_contract_version: z.number().int().min(1), - output_contract_version: z.number().int().min(1), - pagination: z.enum(["none", "offset", "cursor"]), - exhaustive: z.boolean(), - effects: z.object({ - mutates_artifact: z.boolean(), - launches_process: z.boolean(), - may_show_ui: z.boolean(), - may_access_network: z.boolean(), - may_write_filesystem: z.boolean(), - changes_permissions: z.boolean(), - requires_root: z.boolean(), - }), - limits: z.object({ - max_results: z.number().int().min(0).nullable(), - max_payload_bytes: z.number().int().min(0).nullable(), - timeout_ms: z.number().int().min(0).nullable(), - }), - limitations: z.array(z.string()), -}); -const providerIdentity = z.object({ - id: z.string(), - name: z.string(), - version: z.string().nullable(), -}); -const providerRejectionCode: z.ZodType = z.enum( - PROVIDER_REJECTION_CODES, -); -const providerDiagnostics = z.record(z.string(), z.json()); -const providerAvailability = z.discriminatedUnion("status", [ - z.object({ - status: z.literal("available"), - code: z.null(), - reason: z.null(), - diagnostics: providerDiagnostics, - }), - z.object({ - status: z.literal("unavailable"), - code: providerRejectionCode, - reason: z.string().min(1), - diagnostics: providerDiagnostics, - }), -]); -const providerTargetSupport = z.discriminatedUnion("status", [ - z.object({ - status: z.literal("unknown"), - code: z.null(), - reason: z.string().min(1), - diagnostics: providerDiagnostics, - }), - z.object({ - status: z.literal("supported"), - code: z.null(), - reason: z.null(), - diagnostics: providerDiagnostics, - }), - z.object({ - status: z.literal("unsupported"), - code: z.enum([ - "target_kind_unsupported", - "target_format_unsupported", - "architecture_unsupported", - "target_role_unsupported", - "managed_target_unsupported", - ]), - reason: z.string().min(1), - diagnostics: providerDiagnostics, - }), -]); -const sessionProvider = z.object({ - provider: providerIdentity, - providers: z.array(providerIdentity), - capabilities: z.array(providerCapability), - analysis_provider_binding: z - .object({ - provider: providerIdentity, - selection_source: z.enum([ - "request", - "environment", - "auto-single-candidate", - ]), - analysis_profile: analysisProfileSchema, - }) - .nullable(), - analysis_provider_candidates: z.array( - z.object({ - provider: providerIdentity, - availability: providerAvailability, - target_support: providerTargetSupport, - selected: z.boolean(), - capabilities: z.array(providerCapability), - }), - ), - tool_availability: z.array( - z.object({ - name: z.string(), - surface: z.string(), - available: z.boolean(), - reason: z.enum([ - "available", - "target_required", - "provider_missing", - "provider_unavailable", - "target_unsupported", - "unsupported_host", - "policy_disabled", - ]), - remediation: z.string().nullable(), - effects: z.strictObject({ - mutatesTarget: z.boolean(), - mutatesSession: z.boolean(), - writesFilesystem: z.boolean(), - launchesProcess: z.boolean(), - accessesNetwork: z.boolean(), - changesUiState: z.boolean(), - mayDiscardData: z.boolean(), - idempotent: z.boolean(), - }), - annotations: z.object({ - read_only: z.boolean(), - destructive: z.boolean(), - idempotent: z.boolean(), - open_world: z.boolean(), - }), - }), - ), - client_features: z.object({ - elicitation_form: z.boolean(), - elicitation_url: z.boolean(), - roots: z.boolean(), - sampling: z.boolean(), - }), - server_identity: z.object({ - package: z.object({ - name: z.string(), - version: z.string(), - root_path: z.string(), - build_commit: z.string().nullable(), - }), - server: z.object({ - name: z.string(), - version: z.string(), - started_at: z.string(), - command_path: z.string(), - }), - sdk: z.object({ - server: z.string(), - client_test: z.string(), - core: z.string(), - }), - negotiated_protocol_version: z.string().nullable(), - client: z.object({ name: z.string(), version: z.string() }).nullable(), - skill: z.object({ name: z.string(), expected_version: z.string() }), - catalog: z.record(z.string(), z.json()), - protocol_features: z.object({ - progress: z.boolean(), - cancellation: z.boolean(), - evidence_resources: z.boolean(), - elicitation: z.boolean(), - }), - alignment: z.object({ - state: z.enum(["aligned", "mcp_server_restart_required", "unknown"]), - reasons: z.array(z.string()), - remediation: z.string().nullable(), - }), - }), -}); -const nullableText = z.string().nullable(); -const addressList = z.array(z.string()); -const addressedEntry = z.object({ address: z.string(), name: z.string() }); -const procedureIdentity = procedureIdentitySchema; -const localVariable = localVariableSchema; -const containingProcedureResolution = z.discriminatedUnion("found", [ - z.object({ - query_address: z.string(), - found: z.literal(true), - procedure: procedureIdentity, - }), - z.object({ - query_address: z.string(), - found: z.literal(false), - procedure: z.null(), - reason: z.enum(["outside_segments", "not_in_procedure"]), - }), -]); -const unavailable = z.object({ - available: z.literal(false), - reason: z.string(), -}); -const availableMemoryPermissions = z.object({ - available: z.literal(true), - source: z.literal("ghidra-memory-block"), -}); -const bounded = (item: z.ZodType) => - z.object({ - items: z.array(item), - total: z.number().int().min(0).nullable(), - returned: z.number().int().min(0), - truncated: z.boolean(), - next_offset: z.number().int().min(0).nullable(), - }); -const addressedValue = z.object({ - address: z.string(), - value: z.string(), - value_truncated: z.boolean().optional(), - symbol: z - .object({ - primary: z.boolean(), - dynamic: z.boolean(), - external: z.boolean(), - type: z.string(), - source: z.enum(["default", "analysis", "ai", "imported", "user_defined"]), - }) - .optional(), - procedure: z - .object({ - external: z.boolean(), - thunk: z.boolean(), - thunk_target: z.string().nullable(), - }) - .optional(), - string: z - .object({ - encoding: z.string(), - termination: z.enum(["missing", "present_or_not_required"]), - byte_length: z.number().int().min(0), - }) - .optional(), -}); -const pageOutput = z.object({ - items: z.array(addressedValue), - offset: z.number().int().min(0), - limit: z.number().int().min(1), - total: z.number().int().min(0), - next_offset: z.number().int().min(0).nullable(), - has_more: z.boolean(), -}); -const searchPageOutput = pageOutput.extend({ - items: z.array( - z.object({ - address: z.string(), - value: z.string(), - value_truncated: z.boolean(), - }), - ), -}); -const memoryRegionOutput = z.object({ - name: z.string(), - start: z.string(), - end: z.string(), - readable: z.boolean().nullable(), - writable: z.boolean().nullable(), - executable: z.boolean().nullable(), - permissions: z.union([unavailable, availableMemoryPermissions]), - provenance: z.enum(["hopper-public-python-api", "ghidra-memory-block"]), - address_space: z.string().optional(), - image_base: z.string().optional(), - initialized: z.boolean().optional(), - overlay: z.boolean().optional(), -}); -const segmentOutput = resultOf( - z.array(memoryRegionOutput.extend({ sections: z.array(memoryRegionOutput) })), -); -const procedureInfoOutput = resultOf( - z.object({ - name: z.string(), - entrypoint: z.string(), - basicblock_count: z.number().int().min(0), - length: z.number().min(0), - signature: nullableText, - locals: z.array(localVariable), - classification: procedureClassificationSchema.nullable().default(null), - }), -); -const symbolDiscoveryOutput = (property: "classes" | "protocols") => - resultOf( - z.object({ - count: z.number().int().min(0), - [property]: z.array(addressedEntry), - }), - ); -const graphNode = z.discriminatedUnion("status", [ - z.object({ - address: z.string(), - status: z.literal("ok"), - calls: z.array(z.string()), - }), - z.object({ - address: z.string(), - status: z.literal("error"), - error: analysisErrorProjectionSchema, - }), -]); -const functionDossierOutput = resultOf(functionDossierSchema); - -/** Exact structured-content schemas shared by direct analysis providers. */ -export const officialOutputSchemas: Readonly> = { - address_name: resultOf(nullableText), - comment: resultOf(nullableText), - current_address: resultOf(z.string()), - current_procedure: resultOf(z.string()), - current_document: resultOf(z.string()), - goto_address: resultOf(z.string()), - inline_comment: resultOf(nullableText), - list_bookmarks: resultOf(z.array(addressedEntry)), - list_documents: resultOf(z.array(z.string())), - list_names: resultOf(pageOutput), - list_procedures: resultOf(pageOutput), - list_segments: segmentOutput, - list_strings: resultOf(pageOutput), - next_address: resultOf(z.string()), - prev_address: resultOf(z.string()), - procedure_address: resultOf(z.string()), - procedure_assembly: resultOf(z.string()), - procedure_callees: resultOf(addressList), - procedure_callers: resultOf(addressList), - procedure_info: procedureInfoOutput, - procedure_references: resultOf( - z.object({ - procedure: procedureIdentity, - direction: z.enum(["incoming", "outgoing"]), - references: bounded( - z.object({ - source_address: z.string(), - target_address: z.string(), - source_procedure: procedureIdentity.nullable(), - target_procedure: procedureIdentity.nullable(), - kind: referenceKindSchema, - }), - ), - instructions_scanned: z.number().int().min(0), - instruction_scan_truncated: z.boolean(), - }), - ), - procedure_pseudo_code: resultOf(nullableText), - resolve_containing_procedure: resultOf(containingProcedureResolution), - search_procedures: resultOf(searchPageOutput), - search_strings: resultOf(searchPageOutput), - set_address_name: resultOf(z.boolean()), - set_addresses_names: resultOf(z.record(z.string(), z.boolean())), - set_bookmark: resultOf(z.boolean()), - set_comment: resultOf(z.boolean()), - set_current_document: resultOf(z.string()), - set_inline_comment: resultOf(z.boolean()), - unset_bookmark: resultOf(z.boolean()), - xrefs: resultOf(addressList), -}; - -/** Exact structured-content schemas for composed analysis workflows. */ -export const enhancedOutputSchemas: Readonly> = { - swift_classes: symbolDiscoveryOutput("classes"), - get_objc_classes: symbolDiscoveryOutput("classes"), - get_objc_protocols: symbolDiscoveryOutput("protocols"), - batch_decompile: resultOf( - z.object({ - items: z.array( - z.discriminatedUnion("status", [ - z.object({ - address: z.string(), - status: z.literal("ok"), - pseudocode: z.string().min(1), - }), - z.object({ - address: z.string(), - status: z.literal("error"), - error: analysisErrorProjectionSchema, - }), - ]), - ), - total: z.number().int().min(0), - succeeded: z.number().int().min(0), - failed: z.number().int().min(0), - }), - ), - get_call_graph: resultOf(z.record(z.string(), z.array(graphNode))), - analyze_swift_types: resultOf( - z.object({ - total: z.number().int().min(0), - categories: z.record( - z.string(), - z.object({ - count: z.number().int().min(0), - items: z.array(addressedEntry), - }), - ), - }), - ), - find_xrefs_to_name: resultOf( - z.discriminatedUnion("status", [ - z.object({ - status: z.literal("resolved"), - name: z.string(), - address: z.string(), - xrefs: addressList, - }), - z.object({ - status: z.literal("unresolved"), - name: z.string(), - reason: z.literal("name_not_found"), - }), - ]), - ), - binary_overview: resultOf( - z.object({ - document: z.string(), - detail: z.enum(["concise", "detailed"]), - segments: z.array( - z.object({ - name: z.string(), - start: z.string(), - end: z.string(), - length: z.number().min(0).optional(), - }), - ), - segment_count: z.number().int().min(0), - procedure_count: z.number().int().min(0), - string_count: z.number().int().min(0), - }), - ), - analyze_function: functionDossierOutput, - trace_feature: resultOf( - z.object({ - query: z.string(), - search_mode: z.literal("literal"), - operations_used: z.number().int().min(0), - operation_budget: z.number().int().min(1), - matches: z.array( - z.object({ - type: z.enum(["string", "procedure"]), - address: z.string(), - value: z.string(), - }), - ), - references: z.array( - z.object({ - target_address: z.string(), - source_address: z.string(), - containing_procedure: containingProcedureResolution, - }), - ), - truncated: z.boolean(), - residual_unknowns: z.array(z.string()), - }), - ), -}; - -/** Exact Evidence v2 schemas for provider-neutral native inspection. */ -export const nativeOutputSchemas: Readonly> = { - inspect_macho: resultOf(inspectMachoSchema), - inspect_signature: resultOf(inspectSignatureSchema), - inspect_plist: resultOf(inspectPlistSchema), - list_architectures: resultOf(listArchitecturesSchema), - demangle_swift: resultOf(demangleSwiftSchema), -}; - -/** Exact Evidence v2 schemas for provider-neutral artifact graph operations. */ -export const artifactOutputSchemas: Readonly> = { - inventory_artifact: resultOf(artifactInventoryResultSchema), - extract_artifact: resultOf(artifactExtractionResultSchema), -}; - -/** Exact Evidence v2 schema for execution-free managed static analysis. */ -export const managedOutputSchemas: Readonly> = { - inspect_managed_artifact: resultOf(managedArtifactInspectionSchema), - inspect_managed_members: resultOf(managedMemberInspectionSchema), - inspect_managed_native_boundaries: resultOf( - managedNativeBoundaryInspectionSchema, - ), -}; - -/** Exact Evidence v2 schema for provider-neutral managed workflows. */ -export const managedWorkflowOutputSchemas: Readonly< - Record -> = { - compare_managed_members: resultOf(managedMemberComparisonResultSchema), - verify_managed_native_boundaries: resultOf( - managedNativeVerificationResultSchema, - ), - import_managed_reconstruction: resultOf( - managedReconstructionImportResultSchema, - ), - plan_managed_runtime_correlation: resultOf( - managedRuntimeCorrelationResultSchema, - ), - project_managed_application_graph: resultOf( - managedApplicationGraphResultSchema, - ), -}; - -/** Exact structured-content schemas for target lifecycle operations. */ -export const sessionOutputSchemas: Readonly> = { - open_binary: lifecycleResultOf( - z.object({ - path: z.string(), - format: targetFormatSchema, - kind: targetKindSchema, - loaderArgs: z.array(z.string()), - sha256: z.string().regex(/^[a-f0-9]{64}$/u), - architecture: z.enum(["x86", "x86_64", "arm", "arm64"]).nullable(), - }), - ), - close_binary: lifecycleResultOf( - z.union([ - z.null(), - z.object({ - path: z.string(), - bytes: z.number().int().min(0), - entries: z.number().int().min(0), - }), - ]), - ), - binary_session: lifecycleResultOf( - z.union([ - sessionProvider.extend({ open: z.literal(false) }), - sessionProvider.extend({ - open: z.literal(true), - path: z.string(), - format: targetFormatSchema, - kind: targetKindSchema, - sha256: z.string().regex(/^[a-f0-9]{64}$/u), - architecture: z.enum(["x86", "x86_64", "arm", "arm64"]).nullable(), - }), - ]), - ), - export_evidence_bundle: lifecycleResultOf( - z.union([ - evidenceBundleSchema, - z.object({ - path: z.string(), - bytes: z.number().int().min(0), - records: z.number().int().min(0), - }), - ]), - ), - import_evidence_bundle: lifecycleResultOf( - z.object({ - imported: z.number().int().min(0), - total: z.number().int().min(0), - }), - ), - capture_process_scenario: resultOf(processCaptureSchema), - compare_process_captures: resultOf(processCaptureComparisonSchema), - compare_artifacts: resultOf(artifactComparisonResultSchema), - compare_functions: resultOf(functionComparisonResultSchema), - compare_bundles: resultOf(bundleComparisonResultSchema), - find_changed_behavior: resultOf(changedBehaviorResultSchema), - build_call_path: resultOf(callPathResultSchema), - correlate_static_and_runtime: resultOf(staticRuntimeCorrelationResultSchema), - verify_reconstruction: resultOf(reconstructionVerificationResultSchema), - list_unknowns: lifecycleResultOf(z.array(residualUnknownSchema)), - record_unknown: lifecycleResultOf(residualUnknownSchema), - update_unknown: lifecycleResultOf(residualUnknownSchema), - verify_unknown_resolution: lifecycleResultOf( - z.object({ - valid: z.boolean(), - truthVerified: z.boolean(), - unknown: residualUnknownSchema, - }), - ), -}; +export * from "./toolOutputSchemaPrimitives.js"; +export * from "./toolOutputSchemaGroups.js"; diff --git a/src/domain/artifactComparison.ts b/src/domain/artifactComparison.ts index 922fe55f40..fdbc3a6c82 100644 --- a/src/domain/artifactComparison.ts +++ b/src/domain/artifactComparison.ts @@ -212,7 +212,7 @@ const compareOccurrences = ( return output; }; -const classifyPath = (input: { +interface ClassifyPathInput { readonly path: string; readonly leftOccurrence: ArtifactOccurrence | undefined; readonly rightOccurrence: ArtifactOccurrence | undefined; @@ -222,43 +222,71 @@ const classifyPath = (input: { readonly rightRelations: readonly RelationProjection[]; readonly complete: boolean; readonly evidenceLinks: readonly string[]; -}): ArtifactChange | null => { - const base = { - logical_path: input.path, - left_occurrence_id: input.leftOccurrence?.occurrence_id ?? null, - right_occurrence_id: input.rightOccurrence?.occurrence_id ?? null, - left_artifact_id: input.leftOccurrence?.artifact_id ?? null, - right_artifact_id: input.rightOccurrence?.artifact_id ?? null, - evidence_links: [...input.evidenceLinks], - }; +} + +const buildArtifactChangeBase = (input: ClassifyPathInput) => ({ + logical_path: input.path, + left_occurrence_id: input.leftOccurrence?.occurrence_id ?? null, + right_occurrence_id: input.rightOccurrence?.occurrence_id ?? null, + left_artifact_id: input.leftOccurrence?.artifact_id ?? null, + right_artifact_id: input.rightOccurrence?.artifact_id ?? null, + evidence_links: [...input.evidenceLinks], +}); + +const classifyPresence = (input: ClassifyPathInput): ArtifactChange | null => { if (input.leftOccurrence === undefined) return { - ...base, + ...buildArtifactChangeBase(input), classification: input.complete ? "added" : "unknown", dimensions: [input.complete ? "content" : "availability"], }; if (input.rightOccurrence === undefined) return { - ...base, + ...buildArtifactChangeBase(input), classification: input.complete ? "removed" : "unknown", dimensions: [input.complete ? "content" : "availability"], }; + return null; +}; + +const classifyIntegrity = ( + input: ClassifyPathInput, + base: ReturnType, +): ArtifactChange | null => { if ( - input.leftOccurrence.hash_status === "mismatched" || - input.rightOccurrence.hash_status === "mismatched" + input.leftOccurrence?.hash_status === "mismatched" || + input.rightOccurrence?.hash_status === "mismatched" ) return { ...base, classification: "contradiction", dimensions: ["integrity"], }; + return null; +}; + +const classifyAvailability = ( + input: ClassifyPathInput, + base: ReturnType, +): ArtifactChange | null => { if ( - input.leftOccurrence.hash_status !== "verified" || - input.rightOccurrence.hash_status !== "verified" || + input.leftOccurrence?.hash_status !== "verified" || + input.rightOccurrence?.hash_status !== "verified" || input.leftNode === undefined || input.rightNode === undefined ) return { ...base, classification: "unknown", dimensions: ["availability"] }; + return null; +}; + +const classifyPath = (input: ClassifyPathInput): ArtifactChange | null => { + const presence = classifyPresence(input); + if (presence !== null) return presence; + const base = buildArtifactChangeBase(input); + const integrity = classifyIntegrity(input, base); + if (integrity !== null) return integrity; + const availability = classifyAvailability(input, base); + if (availability !== null) return availability; const dimensions = changedDimensions(input); return dimensions.length === 0 ? null diff --git a/src/domain/browserObservation.ts b/src/domain/browserObservation.ts index 9aa84d4e68..15f38e4409 100644 --- a/src/domain/browserObservation.ts +++ b/src/domain/browserObservation.ts @@ -1,8 +1,11 @@ import { z } from "zod"; -import { browserCompletenessSchema } from "./browserCompleteness.js"; -import { webTextArtifactSchema } from "./webContentArtifact.js"; -import { jsonShapeSchema } from "./jsonShape.js"; +export { + browserTargetListSchema, + webPageInspectionSchema, + type BrowserTargetList, + type WebPageInspection, +} from "./browserObservationSchemas.js"; const parseExactOrigin = (value: string): string | undefined => { let url: URL; @@ -347,300 +350,3 @@ export const sanitizeBrowserUrl = (value: string): SanitizedBrowserUrl => { redacted: wasTruncated || hadCredentials || hadFragment || names.length > 0, }; }; - -const browserVersionSchema = z.object({ - product: z.string(), - protocol_version: z.string(), - revision: z.string(), - user_agent: z.string(), - js_version: z.string(), -}); - -const browserTargetSchema = z.object({ - target_id: z.string(), - type: z.string(), - title: z.string(), - url: z.string(), - origin: z.string(), - attached: z.boolean(), -}); - -/** Bounded and policy-filtered browser target discovery result. */ -export const browserTargetListSchema = z.object({ - schema_version: z.literal(1), - browser: browserVersionSchema, - targets: z.object({ - items: z.array(browserTargetSchema), - offset: z.number().int().min(0), - limit: z.number().int().min(1), - total: z.number().int().min(0), - next_offset: z.number().int().min(0).nullable(), - has_more: z.boolean(), - }), - excluded: z.object({ - disallowed_origin: z.number().int().min(0), - unsupported_url: z.number().int().min(0), - non_page: z.number().int().min(0), - }), - limitations: z.array(z.string()), -}); -export type BrowserTargetList = z.infer; - -const browserFrameSchema = z.object({ - frame_id: z.string(), - parent_frame_id: z.string().nullable(), - url: z.string(), - origin: z.string().nullable(), -}); -const browserDomNodeSchema = z.object({ - index: z.number().int().min(0), - parent_index: z.number().int().min(-1), - node_type: z.number().int().min(0), - node_name: z.string(), - node_value_length: z.number().int().min(0), - attribute_names: z.array(z.string()), -}); -const browserAxNodeSchema = z.object({ - node_id: z.string(), - parent_id: z.string().nullable(), - role: z.string().nullable(), - name: z.string().nullable(), - description: z.string().nullable(), - ignored: z.boolean(), -}); -const browserScriptSourceSchema = z.discriminatedUnion("included", [ - z.object({ included: z.literal(false), reason: z.string() }), - z.object({ - included: z.literal(true), - artifact: webTextArtifactSchema, - }), -]); -const browserResourceReconciliationSchema = z.discriminatedUnion("status", [ - z.object({ status: z.literal("exact"), resource_key: z.string() }), - z.object({ - status: z.literal("ambiguous"), - candidate_resource_keys: z.array(z.string()), - }), - z.object({ - status: z.literal("unmatched"), - reason: z.literal("no_exact_sanitized_url"), - }), -]); -const browserScriptSchema = z.object({ - script_key: z.string().regex(/^scr_[a-f0-9]{64}(?:_[0-9]+)?$/u), - frame_id: z.string().nullable().default(null), - url: z.string(), - origin: z.string().nullable(), - cdp_hash: z.string(), - length: z.number().int().min(0), - is_module: z.boolean(), - language: z.string().nullable(), - source_map_url: z.string().nullable(), - resource_reconciliation: browserResourceReconciliationSchema, - source: browserScriptSourceSchema, -}); -const browserResourceSchema = z.object({ - resource_key: z.string().regex(/^res_[a-f0-9]{64}$/u), - url: z.string(), - origin: z.string().nullable(), - type: z.string(), - mime_type: z.string(), - content_size: z.number().min(0).nullable(), -}); -const browserNetworkRequestSchema = z.object({ - request_id: z.string(), - url: z.string(), - origin: z.string().nullable(), - method: z.string(), - resource_type: z.string().nullable(), - status: z.number().nullable(), - mime_type: z.string().nullable(), - encoded_data_length: z.number().min(0).nullable(), - initiator: z.object({ - type: z.string(), - url: z.string().nullable(), - line: z.number().int().min(0).nullable(), - column: z.number().int().min(0).nullable(), - }), - body_shapes: z.object({ - status: z.enum([ - "not_approved", - "included", - "partial", - "unavailable", - "truncated", - ]), - request: jsonShapeSchema.nullable(), - response: jsonShapeSchema.nullable(), - }), -}); -const browserConsoleEventSchema = z.object({ - type: z.string(), - timestamp: z.number(), - argument_types: z.array(z.string()), - url: z.string().nullable(), - line: z.number().int().min(0).nullable(), - column: z.number().int().min(0).nullable(), - text_capture: z.object({ - status: z.enum(["not_approved", "included", "truncated"]), - values: z.array( - z.object({ - argument_index: z.number().int().min(0), - type: z.string(), - text: z.string(), - }), - ), - retained_bytes: z.number().int().min(0), - truncated_values: z.number().int().min(0), - }), -}); -const browserWebSocketEventSchema = z.object({ - request_id: z.string(), - direction: z.enum(["sent", "received"]), - opcode: z.number().int().min(0), - payload_bytes: z.number().int().min(0), - payload_shape: z - .object({ - format: z.enum(["json", "text", "binary"]), - json_shape: jsonShapeSchema.nullable(), - truncated: z.boolean(), - }) - .nullable(), -}); -const browserWorkerSchema = z.object({ - target_id: z.string(), - type: z.string(), - url: z.string(), - origin: z.string().nullable(), - attached: z.boolean(), - opener_target_id: z.string().nullable().default(null), - parent_frame_id: z.string().nullable().default(null), -}); -const browserCspSourceSchema = z.object({ - kind: z.enum([ - "keyword", - "scheme", - "approved_origin", - "external_origin", - "other", - ]), - value: z.string().nullable(), -}); -const browserLinkMetadataSchema = z.object({ - href: z.string().nullable(), - destination_scope: z.enum(["approved", "outside_policy", "unsupported"]), - rel: z.array(z.string()), - as: z.string().nullable(), - type: z.string().nullable(), - crossorigin: z.string().nullable(), -}); -const browserAgentHintSchema = z.object({ - mechanism: z.enum([ - "link_rel", - "dom_link_rel", - "well_known_resource", - "response_header", - ]), - declaration: z.string(), - url: z.string().nullable(), - trust: z.literal("page-declared-untrusted"), -}); -const browserSafeMetadataSchema = z.object({ - responses: z.array( - z.object({ - request_id: z.string(), - url: z.string(), - mime_type: z.string().nullable(), - content_length: z.number().int().min(0).nullable(), - content_encoding: z.string().nullable(), - csp: z.object({ - directives: z.array( - z.object({ - name: z.string(), - sources: z.array(browserCspSourceSchema), - }), - ), - nonce_count: z.number().int().min(0), - hash_count: z.number().int().min(0), - }), - links: z.array(browserLinkMetadataSchema), - policies: z.object({ - coop: z.string().nullable(), - coep: z.string().nullable(), - corp: z.string().nullable(), - referrer_policy: z.string().nullable(), - x_content_type_options: z.string().nullable(), - permissions_policy_features: z.array(z.string()), - }), - }), - ), - dom_urls: z.array( - z.object({ - node_index: z.number().int().min(0), - attribute: z.enum(["href", "src", "action", "formaction", "poster"]), - url: z.string().nullable(), - destination_scope: z.enum(["approved", "outside_policy", "unsupported"]), - }), - ), - agent_hints: z.array(browserAgentHintSchema), - excluded_dom_urls: z.number().int().min(0), - headers_allowlisted: z.literal(true), -}); - -/** Provider-neutral normalized result for one passive web-page observation. */ -export const webPageInspectionSchema = z.object({ - schema_version: z.literal(2), - browser: browserVersionSchema, - target: browserTargetSchema, - capture_window: z.object({ - started_at: z.iso.datetime(), - ended_at: z.iso.datetime(), - observation_ms: z.number().int().min(0), - }), - completeness: browserCompletenessSchema, - frames: z.array(browserFrameSchema), - dom: z.object({ - total_nodes: z.number().int().min(0), - nodes: z.array(browserDomNodeSchema), - }), - accessibility: z.object({ - total_nodes: z.number().int().min(0), - text_capture: z.object({ - status: z.enum(["not_approved", "included", "truncated", "unavailable"]), - retained_bytes: z.number().int().min(0), - excluded_fields: z.number().int().min(0), - truncated_fields: z.number().int().min(0), - }), - nodes: z.array(browserAxNodeSchema), - }), - scripts: z.object({ - total: z.number().int().min(0), - items: z.array(browserScriptSchema), - }), - resources: z.array(browserResourceSchema), - network: z.object({ - requests: z.array(browserNetworkRequestSchema), - websocket_events: z.array(browserWebSocketEventSchema), - coverage_started_at: z.iso.datetime(), - prior_activity_available: z.literal(false), - }), - console: z.object({ - events: z.array(browserConsoleEventSchema), - coverage_started_at: z.iso.datetime(), - prior_activity_available: z.literal(false), - }), - workers: z.array(browserWorkerSchema), - metadata: browserSafeMetadataSchema, - storage: z.object({ - origin: z.string(), - usage_bytes: z.number().min(0).nullable(), - quota_bytes: z.number().min(0).nullable(), - local_storage_keys: z.array(z.string()), - session_storage_keys: z.array(z.string()), - indexed_db_names: z.array(z.string()), - cache_names: z.array(z.string()), - values_redacted: z.literal(true), - }), - limitations: z.array(z.string()), -}); -export type WebPageInspection = z.infer; diff --git a/src/domain/browserObservationSchemas.ts b/src/domain/browserObservationSchemas.ts new file mode 100644 index 0000000000..c90a9922ac --- /dev/null +++ b/src/domain/browserObservationSchemas.ts @@ -0,0 +1,302 @@ +import { z } from "zod"; + +import { browserCompletenessSchema } from "./browserCompleteness.js"; +import { webTextArtifactSchema } from "./webContentArtifact.js"; +import { jsonShapeSchema } from "./jsonShape.js"; + +const browserVersionSchema = z.object({ + product: z.string(), + protocol_version: z.string(), + revision: z.string(), + user_agent: z.string(), + js_version: z.string(), +}); + +const browserTargetSchema = z.object({ + target_id: z.string(), + type: z.string(), + title: z.string(), + url: z.string(), + origin: z.string(), + attached: z.boolean(), +}); + +/** Bounded and policy-filtered browser target discovery result. */ +export const browserTargetListSchema = z.object({ + schema_version: z.literal(1), + browser: browserVersionSchema, + targets: z.object({ + items: z.array(browserTargetSchema), + offset: z.number().int().min(0), + limit: z.number().int().min(1), + total: z.number().int().min(0), + next_offset: z.number().int().min(0).nullable(), + has_more: z.boolean(), + }), + excluded: z.object({ + disallowed_origin: z.number().int().min(0), + unsupported_url: z.number().int().min(0), + non_page: z.number().int().min(0), + }), + limitations: z.array(z.string()), +}); +export type BrowserTargetList = z.infer; + +const browserFrameSchema = z.object({ + frame_id: z.string(), + parent_frame_id: z.string().nullable(), + url: z.string(), + origin: z.string().nullable(), +}); +const browserDomNodeSchema = z.object({ + index: z.number().int().min(0), + parent_index: z.number().int().min(-1), + node_type: z.number().int().min(0), + node_name: z.string(), + node_value_length: z.number().int().min(0), + attribute_names: z.array(z.string()), +}); +const browserAxNodeSchema = z.object({ + node_id: z.string(), + parent_id: z.string().nullable(), + role: z.string().nullable(), + name: z.string().nullable(), + description: z.string().nullable(), + ignored: z.boolean(), +}); +const browserScriptSourceSchema = z.discriminatedUnion("included", [ + z.object({ included: z.literal(false), reason: z.string() }), + z.object({ + included: z.literal(true), + artifact: webTextArtifactSchema, + }), +]); +const browserResourceReconciliationSchema = z.discriminatedUnion("status", [ + z.object({ status: z.literal("exact"), resource_key: z.string() }), + z.object({ + status: z.literal("ambiguous"), + candidate_resource_keys: z.array(z.string()), + }), + z.object({ + status: z.literal("unmatched"), + reason: z.literal("no_exact_sanitized_url"), + }), +]); +const browserScriptSchema = z.object({ + script_key: z.string().regex(/^scr_[a-f0-9]{64}(?:_[0-9]+)?$/u), + frame_id: z.string().nullable().default(null), + url: z.string(), + origin: z.string().nullable(), + cdp_hash: z.string(), + length: z.number().int().min(0), + is_module: z.boolean(), + language: z.string().nullable(), + source_map_url: z.string().nullable(), + resource_reconciliation: browserResourceReconciliationSchema, + source: browserScriptSourceSchema, +}); +const browserResourceSchema = z.object({ + resource_key: z.string().regex(/^res_[a-f0-9]{64}$/u), + url: z.string(), + origin: z.string().nullable(), + type: z.string(), + mime_type: z.string(), + content_size: z.number().min(0).nullable(), +}); +const browserNetworkRequestSchema = z.object({ + request_id: z.string(), + url: z.string(), + origin: z.string().nullable(), + method: z.string(), + resource_type: z.string().nullable(), + status: z.number().nullable(), + mime_type: z.string().nullable(), + encoded_data_length: z.number().min(0).nullable(), + initiator: z.object({ + type: z.string(), + url: z.string().nullable(), + line: z.number().int().min(0).nullable(), + column: z.number().int().min(0).nullable(), + }), + body_shapes: z.object({ + status: z.enum([ + "not_approved", + "included", + "partial", + "unavailable", + "truncated", + ]), + request: jsonShapeSchema.nullable(), + response: jsonShapeSchema.nullable(), + }), +}); +const browserConsoleEventSchema = z.object({ + type: z.string(), + timestamp: z.number(), + argument_types: z.array(z.string()), + url: z.string().nullable(), + line: z.number().int().min(0).nullable(), + column: z.number().int().min(0).nullable(), + text_capture: z.object({ + status: z.enum(["not_approved", "included", "truncated"]), + values: z.array( + z.object({ + argument_index: z.number().int().min(0), + type: z.string(), + text: z.string(), + }), + ), + retained_bytes: z.number().int().min(0), + truncated_values: z.number().int().min(0), + }), +}); +const browserWebSocketEventSchema = z.object({ + request_id: z.string(), + direction: z.enum(["sent", "received"]), + opcode: z.number().int().min(0), + payload_bytes: z.number().int().min(0), + payload_shape: z + .object({ + format: z.enum(["json", "text", "binary"]), + json_shape: jsonShapeSchema.nullable(), + truncated: z.boolean(), + }) + .nullable(), +}); +const browserWorkerSchema = z.object({ + target_id: z.string(), + type: z.string(), + url: z.string(), + origin: z.string().nullable(), + attached: z.boolean(), + opener_target_id: z.string().nullable().default(null), + parent_frame_id: z.string().nullable().default(null), +}); +const browserCspSourceSchema = z.object({ + kind: z.enum([ + "keyword", + "scheme", + "approved_origin", + "external_origin", + "other", + ]), + value: z.string().nullable(), +}); +const browserLinkMetadataSchema = z.object({ + href: z.string().nullable(), + destination_scope: z.enum(["approved", "outside_policy", "unsupported"]), + rel: z.array(z.string()), + as: z.string().nullable(), + type: z.string().nullable(), + crossorigin: z.string().nullable(), +}); +const browserAgentHintSchema = z.object({ + mechanism: z.enum([ + "link_rel", + "dom_link_rel", + "well_known_resource", + "response_header", + ]), + declaration: z.string(), + url: z.string().nullable(), + trust: z.literal("page-declared-untrusted"), +}); +const browserSafeMetadataSchema = z.object({ + responses: z.array( + z.object({ + request_id: z.string(), + url: z.string(), + mime_type: z.string().nullable(), + content_length: z.number().int().min(0).nullable(), + content_encoding: z.string().nullable(), + csp: z.object({ + directives: z.array( + z.object({ + name: z.string(), + sources: z.array(browserCspSourceSchema), + }), + ), + nonce_count: z.number().int().min(0), + hash_count: z.number().int().min(0), + }), + links: z.array(browserLinkMetadataSchema), + policies: z.object({ + coop: z.string().nullable(), + coep: z.string().nullable(), + corp: z.string().nullable(), + referrer_policy: z.string().nullable(), + x_content_type_options: z.string().nullable(), + permissions_policy_features: z.array(z.string()), + }), + }), + ), + dom_urls: z.array( + z.object({ + node_index: z.number().int().min(0), + attribute: z.enum(["href", "src", "action", "formaction", "poster"]), + url: z.string().nullable(), + destination_scope: z.enum(["approved", "outside_policy", "unsupported"]), + }), + ), + agent_hints: z.array(browserAgentHintSchema), + excluded_dom_urls: z.number().int().min(0), + headers_allowlisted: z.literal(true), +}); + +/** Provider-neutral normalized result for one passive web-page observation. */ +export const webPageInspectionSchema = z.object({ + schema_version: z.literal(2), + browser: browserVersionSchema, + target: browserTargetSchema, + capture_window: z.object({ + started_at: z.iso.datetime(), + ended_at: z.iso.datetime(), + observation_ms: z.number().int().min(0), + }), + completeness: browserCompletenessSchema, + frames: z.array(browserFrameSchema), + dom: z.object({ + total_nodes: z.number().int().min(0), + nodes: z.array(browserDomNodeSchema), + }), + accessibility: z.object({ + total_nodes: z.number().int().min(0), + text_capture: z.object({ + status: z.enum(["not_approved", "included", "truncated", "unavailable"]), + retained_bytes: z.number().int().min(0), + excluded_fields: z.number().int().min(0), + truncated_fields: z.number().int().min(0), + }), + nodes: z.array(browserAxNodeSchema), + }), + scripts: z.object({ + total: z.number().int().min(0), + items: z.array(browserScriptSchema), + }), + resources: z.array(browserResourceSchema), + network: z.object({ + requests: z.array(browserNetworkRequestSchema), + websocket_events: z.array(browserWebSocketEventSchema), + coverage_started_at: z.iso.datetime(), + prior_activity_available: z.literal(false), + }), + console: z.object({ + events: z.array(browserConsoleEventSchema), + coverage_started_at: z.iso.datetime(), + prior_activity_available: z.literal(false), + }), + workers: z.array(browserWorkerSchema), + metadata: browserSafeMetadataSchema, + storage: z.object({ + origin: z.string(), + usage_bytes: z.number().min(0).nullable(), + quota_bytes: z.number().min(0).nullable(), + local_storage_keys: z.array(z.string()), + session_storage_keys: z.array(z.string()), + indexed_db_names: z.array(z.string()), + cache_names: z.array(z.string()), + values_redacted: z.literal(true), + }), + limitations: z.array(z.string()), +}); +export type WebPageInspection = z.infer; diff --git a/src/domain/functionComparison.ts b/src/domain/functionComparison.ts index 9e6ac7e866..168ebfa4f8 100644 --- a/src/domain/functionComparison.ts +++ b/src/domain/functionComparison.ts @@ -1,61 +1,17 @@ -import { diffLines } from "diff"; - -import { - parseFunctionEvidence, - type FunctionCollection, - type FunctionSnapshot, -} from "./functionDossierEvidence.js"; -import { - commentProjection, - combineCoverage, - functionMatch, - identityProjection, - isAutoName, - normalizeCfg, - project, - referenceKindProjection, - referenceProjection, - sorted, - stringAndNameProjection, -} from "./functionComparisonNormalization.js"; +import { parseFunctionEvidence } from "./functionDossierEvidence.js"; +import { functionMatch } from "./functionComparisonNormalization.js"; import { functionComparisonResultSchema, - type DimensionName, type FunctionComparisonResult, - type FunctionDimension, } from "./functionComparisonSchemas.js"; -import { - canonicalJson, - dimensionResult, - overallStatus, - summarize, - unresolvedDimension as buildUnresolvedDimension, -} from "./functionComparisonResults.js"; +import { overallStatus, summarize, canonicalJson } from "./functionComparisonResults.js"; +import { compareDimensions } from "./functionComparisonDimensions.js"; export { functionComparisonInputSchema, functionComparisonResultSchema, } from "./functionComparisonSchemas.js"; -const unresolvedDimension = ( - ...[dimension, status, links, leftCount, rightCount, limitations]: readonly [ - DimensionName, - "truncated" | "unknown", - readonly string[], - number | null, - number | null, - readonly string[], - ] -): FunctionDimension => - buildUnresolvedDimension({ - dimension, - status, - links, - leftCount, - rightCount, - limitations, - }); - /** Compare two explicit function Evidence page sets without fuzzy matching. */ export const compareFunctions = ( leftInput: unknown, @@ -103,422 +59,3 @@ export const compareFunctions = ( ].sort((a, b) => a.localeCompare(b)), }); }; - -const compareDimensions = ( - left: FunctionSnapshot, - right: FunctionSnapshot, - links: readonly string[], - providersDiffer: boolean, -): FunctionDimension[] => [ - compareIdentity(left, right, links, providersDiffer), - compareText( - "pseudocode", - left.pseudocode.text, - right.pseudocode.text, - left.pseudocode.complete, - right.pseudocode.complete, - links, - providersDiffer, - ), - compareText( - "assembly", - left.collections.assembly.items.join("\n"), - right.collections.assembly.items.join("\n"), - left.collections.assembly.complete, - right.collections.assembly.complete, - links, - providersDiffer, - left.collections.assembly.truncated || right.collections.assembly.truncated, - "Assembly is opaque provider text; relocation normalization is unavailable.", - ), - compareComments(left, right, links, providersDiffer), - compareCalls(left, right, links, providersDiffer), - compareReferences(left, right, links, providersDiffer), - compareStringsAndNames(left, right, links, providersDiffer), - compareCfg(left, right, links, providersDiffer), -]; - -const compareText = ( - ...[ - dimension, - left, - right, - leftComplete, - rightComplete, - links, - providersDiffer, - explicitTruncation, - limitation, - ]: readonly [ - "pseudocode" | "assembly", - string, - string, - boolean, - boolean, - readonly string[], - boolean, - boolean?, - string?, - ] -): FunctionDimension => { - const truncated = explicitTruncation ?? (!leftComplete || !rightComplete); - if (!leftComplete || !rightComplete || providersDiffer) - return unresolvedDimension( - dimension, - truncated ? "truncated" : "unknown", - links, - left.length, - right.length, - [ - ...(limitation === undefined ? [] : [limitation]), - ...(providersDiffer - ? ["Exact text comparison requires one provider identity."] - : []), - ], - ); - const changes = diffLines(left, right, { - timeout: 100, - maxEditLength: 10_000, - }); - if (changes === undefined) - return unresolvedDimension( - dimension, - "unknown", - links, - left.length, - right.length, - ["Bounded line diff exceeded its time or edit-distance limit."], - ); - const delta = changes.reduce( - (summary, change) => ({ - added_lines: - summary.added_lines + (change.added ? (change.count ?? 0) : 0), - removed_lines: - summary.removed_lines + (change.removed ? (change.count ?? 0) : 0), - hunks: summary.hunks + (change.added || change.removed ? 1 : 0), - }), - { added_lines: 0, removed_lines: 0, hunks: 0 }, - ); - return dimensionResult({ - dimension, - status: delta.hunks === 0 ? "unchanged" : "changed", - left, - right, - links, - providersDiffer, - leftCount: [...left].length, - rightCount: [...right].length, - textDelta: delta, - limitations: limitation === undefined ? [] : [limitation], - }); -}; - -const compareCollections = ( - ...[ - dimension, - left, - right, - leftCoverage, - rightCoverage, - links, - providersDiffer, - ]: readonly [ - DimensionName, - readonly unknown[], - readonly unknown[], - Pick, - Pick, - readonly string[], - boolean, - ] -): FunctionDimension => { - if (!leftCoverage.complete || !rightCoverage.complete) - return unresolvedDimension( - dimension, - leftCoverage.truncated || rightCoverage.truncated - ? "truncated" - : "unknown", - links, - left.length, - right.length, - [], - ); - return compareValues( - dimension, - sorted(left), - sorted(right), - links, - true, - true, - providersDiffer, - ); -}; - -const compareComments = ( - left: FunctionSnapshot, - right: FunctionSnapshot, - links: readonly string[], - providersDiffer: boolean, -): FunctionDimension => { - const leftValues = commentProjection(left); - const rightValues = commentProjection(right); - if (leftValues === null || rightValues === null) - return unresolvedDimension( - "comments", - "unknown", - links, - left.collections.comments.items.length, - right.collections.comments.items.length, - ["Comment locations could not be normalized relative to the function."], - ); - return compareCollections( - "comments", - leftValues, - rightValues, - left.collections.comments, - right.collections.comments, - links, - providersDiffer, - ); -}; - -const compareStringsAndNames = ( - left: FunctionSnapshot, - right: FunctionSnapshot, - links: readonly string[], - providersDiffer: boolean, -): FunctionDimension => { - const leftValues = stringAndNameProjection(left); - const rightValues = stringAndNameProjection(right); - if (leftValues === null || rightValues === null) - return unresolvedDimension("strings_names", "unknown", links, null, null, [ - "Reference source locations could not be normalized to function offsets.", - ]); - return compareCollections( - "strings_names", - leftValues, - rightValues, - combineCoverage( - left.collections.referenced_strings, - left.collections.referenced_names, - ), - combineCoverage( - right.collections.referenced_strings, - right.collections.referenced_names, - ), - links, - providersDiffer, - ); -}; - -const compareReferences = ( - left: FunctionSnapshot, - right: FunctionSnapshot, - links: readonly string[], - providersDiffer: boolean, -): FunctionDimension => { - const leftCoverage = combineCoverage( - left.collections.incoming_references, - left.collections.outgoing_references, - ); - const rightCoverage = combineCoverage( - right.collections.incoming_references, - right.collections.outgoing_references, - ); - if (!leftCoverage.complete || !rightCoverage.complete) - return unresolvedDimension( - "references", - leftCoverage.truncated || rightCoverage.truncated - ? "truncated" - : "unknown", - links, - leftCoverage.items.length, - rightCoverage.items.length, - [], - ); - const leftProjection = referenceProjection(left); - const rightProjection = referenceProjection(right); - if (leftProjection.length === 0 && rightProjection.length === 0) - return compareValues( - "references", - leftProjection, - rightProjection, - links, - true, - true, - providersDiffer, - ); - if (canonicalJson(leftProjection) !== canonicalJson(rightProjection)) - return dimensionResult({ - dimension: "references", - status: "changed", - left: leftProjection, - right: rightProjection, - links, - providersDiffer, - leftCount: leftProjection.length, - rightCount: rightProjection.length, - textDelta: null, - limitations: [ - "Reference endpoints differ independently of provider reference-kind metadata.", - ], - }); - const leftKinds = referenceKindProjection(left); - const rightKinds = referenceKindProjection(right); - if (leftKinds !== null && rightKinds !== null) - return compareValues( - "references", - leftKinds, - rightKinds, - links, - true, - true, - providersDiffer, - ); - return unresolvedDimension( - "references", - "unknown", - links, - leftProjection.length, - rightProjection.length, - [ - "At least one provider did not expose reference kinds, so equal endpoints do not prove equal edge semantics.", - ], - ); -}; - -const compareIdentity = ( - left: FunctionSnapshot, - right: FunctionSnapshot, - links: readonly string[], - providersDiffer: boolean, -): FunctionDimension => { - if (isAutoName(left.procedure.name) || isAutoName(right.procedure.name)) - return unresolvedDimension("identity", "unknown", links, null, null, [ - "Address-derived function names are not stable cross-version identity.", - ]); - return compareValues( - "identity", - identityProjection(left), - identityProjection(right), - links, - true, - true, - providersDiffer, - ); -}; - -const compareCalls = ( - left: FunctionSnapshot, - right: FunctionSnapshot, - links: readonly string[], - providersDiffer: boolean, -): FunctionDimension => { - const leftValues = callProjection(left); - const rightValues = callProjection(right); - if ([...leftValues, ...rightValues].some(({ name }) => isAutoName(name))) - return unresolvedDimension( - "calls", - "unknown", - links, - leftValues.length, - rightValues.length, - ["Address-derived callee or caller names cannot be matched safely."], - ); - return compareCollections( - "calls", - leftValues, - rightValues, - combineCoverage(left.collections.callers, left.collections.callees), - combineCoverage(right.collections.callers, right.collections.callees), - links, - providersDiffer, - ); -}; - -const callProjection = (snapshot: FunctionSnapshot) => [ - ...project(snapshot.collections.callers, ({ name }) => ({ - direction: "in" as const, - name, - })), - ...project(snapshot.collections.callees, ({ name }) => ({ - direction: "out" as const, - name, - })), -]; - -const compareCfg = ( - left: FunctionSnapshot, - right: FunctionSnapshot, - links: readonly string[], - providersDiffer: boolean, -): FunctionDimension => { - const leftBlocks = left.collections.basic_blocks; - const rightBlocks = right.collections.basic_blocks; - if (!leftBlocks.complete || !rightBlocks.complete) - return unresolvedDimension( - "cfg", - leftBlocks.truncated || rightBlocks.truncated ? "truncated" : "unknown", - links, - leftBlocks.items.length, - rightBlocks.items.length, - [], - ); - const leftGraph = normalizeCfg(leftBlocks.items); - const rightGraph = normalizeCfg(rightBlocks.items); - if (leftGraph === null || rightGraph === null) - return unresolvedDimension( - "cfg", - "unknown", - links, - leftBlocks.items.length, - rightBlocks.items.length, - ["CFG addresses could not be normalized to local block indices."], - ); - return compareValues( - "cfg", - leftGraph, - rightGraph, - links, - true, - true, - providersDiffer, - ); -}; - -const compareValues = ( - ...[ - dimension, - left, - right, - links, - leftComplete, - rightComplete, - providersDiffer, - ]: readonly [ - DimensionName, - unknown, - unknown, - readonly string[], - boolean, - boolean, - boolean, - ] -): FunctionDimension => { - if (!leftComplete || !rightComplete) - return unresolvedDimension(dimension, "unknown", links, null, null, []); - const leftJson = canonicalJson(left); - const rightJson = canonicalJson(right); - return dimensionResult({ - dimension, - status: leftJson === rightJson ? "unchanged" : "changed", - left, - right, - links, - providersDiffer, - leftCount: Array.isArray(left) ? left.length : null, - rightCount: Array.isArray(right) ? right.length : null, - textDelta: null, - limitations: [], - }); -}; diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts new file mode 100644 index 0000000000..fc4e4bc3f5 --- /dev/null +++ b/src/domain/functionComparisonDimensions.ts @@ -0,0 +1,462 @@ +import { diffLines } from "diff"; + +import type { + FunctionCollection, + FunctionSnapshot, +} from "./functionDossierEvidence.js"; +import { + commentProjection, + combineCoverage, + identityProjection, + isAutoName, + normalizeCfg, + project, + referenceKindProjection, + referenceProjection, + sorted, + stringAndNameProjection, +} from "./functionComparisonNormalization.js"; +import type { DimensionName, FunctionDimension } from "./functionComparisonSchemas.js"; +import { + canonicalJson, + dimensionResult, + unresolvedDimension as buildUnresolvedDimension, +} from "./functionComparisonResults.js"; + +const unresolvedDimension = ( + ...[dimension, status, links, leftCount, rightCount, limitations]: readonly [ + DimensionName, + "truncated" | "unknown", + readonly string[], + number | null, + number | null, + readonly string[], + ] +): FunctionDimension => + buildUnresolvedDimension({ + dimension, + status, + links, + leftCount, + rightCount, + limitations, + }); + +export const compareDimensions = ( + left: FunctionSnapshot, + right: FunctionSnapshot, + links: readonly string[], + providersDiffer: boolean, +): FunctionDimension[] => [ + compareIdentity(left, right, links, providersDiffer), + compareText( + "pseudocode", + left.pseudocode.text, + right.pseudocode.text, + left.pseudocode.complete, + right.pseudocode.complete, + links, + providersDiffer, + ), + compareText( + "assembly", + left.collections.assembly.items.join("\n"), + right.collections.assembly.items.join("\n"), + left.collections.assembly.complete, + right.collections.assembly.complete, + links, + providersDiffer, + left.collections.assembly.truncated || right.collections.assembly.truncated, + "Assembly is opaque provider text; relocation normalization is unavailable.", + ), + compareComments(left, right, links, providersDiffer), + compareCalls(left, right, links, providersDiffer), + compareReferences(left, right, links, providersDiffer), + compareStringsAndNames(left, right, links, providersDiffer), + compareCfg(left, right, links, providersDiffer), +]; + +const compareText = ( + ...[ + dimension, + left, + right, + leftComplete, + rightComplete, + links, + providersDiffer, + explicitTruncation, + limitation, + ]: readonly [ + "pseudocode" | "assembly", + string, + string, + boolean, + boolean, + readonly string[], + boolean, + boolean?, + string?, + ] +): FunctionDimension => { + const truncated = explicitTruncation ?? (!leftComplete || !rightComplete); + if (!leftComplete || !rightComplete || providersDiffer) + return unresolvedDimension( + dimension, + truncated ? "truncated" : "unknown", + links, + left.length, + right.length, + [ + ...(limitation === undefined ? [] : [limitation]), + ...(providersDiffer + ? ["Exact text comparison requires one provider identity."] + : []), + ], + ); + const changes = diffLines(left, right, { + timeout: 100, + maxEditLength: 10_000, + }); + if (changes === undefined) + return unresolvedDimension( + dimension, + "unknown", + links, + left.length, + right.length, + ["Bounded line diff exceeded its time or edit-distance limit."], + ); + const delta = changes.reduce( + (summary, change) => ({ + added_lines: + summary.added_lines + (change.added ? (change.count ?? 0) : 0), + removed_lines: + summary.removed_lines + (change.removed ? (change.count ?? 0) : 0), + hunks: summary.hunks + (change.added || change.removed ? 1 : 0), + }), + { added_lines: 0, removed_lines: 0, hunks: 0 }, + ); + return dimensionResult({ + dimension, + status: delta.hunks === 0 ? "unchanged" : "changed", + left, + right, + links, + providersDiffer, + leftCount: [...left].length, + rightCount: [...right].length, + textDelta: delta, + limitations: limitation === undefined ? [] : [limitation], + }); +}; + +const compareCollections = ( + ...[ + dimension, + left, + right, + leftCoverage, + rightCoverage, + links, + providersDiffer, + ]: readonly [ + DimensionName, + readonly unknown[], + readonly unknown[], + Pick, + Pick, + readonly string[], + boolean, + ] +): FunctionDimension => { + if (!leftCoverage.complete || !rightCoverage.complete) + return unresolvedDimension( + dimension, + leftCoverage.truncated || rightCoverage.truncated + ? "truncated" + : "unknown", + links, + left.length, + right.length, + [], + ); + return compareValues( + dimension, + sorted(left), + sorted(right), + links, + true, + true, + providersDiffer, + ); +}; + +const compareComments = ( + left: FunctionSnapshot, + right: FunctionSnapshot, + links: readonly string[], + providersDiffer: boolean, +): FunctionDimension => { + const leftValues = commentProjection(left); + const rightValues = commentProjection(right); + if (leftValues === null || rightValues === null) + return unresolvedDimension( + "comments", + "unknown", + links, + left.collections.comments.items.length, + right.collections.comments.items.length, + ["Comment locations could not be normalized relative to the function."], + ); + return compareCollections( + "comments", + leftValues, + rightValues, + left.collections.comments, + right.collections.comments, + links, + providersDiffer, + ); +}; + +const compareStringsAndNames = ( + left: FunctionSnapshot, + right: FunctionSnapshot, + links: readonly string[], + providersDiffer: boolean, +): FunctionDimension => { + const leftValues = stringAndNameProjection(left); + const rightValues = stringAndNameProjection(right); + if (leftValues === null || rightValues === null) + return unresolvedDimension("strings_names", "unknown", links, null, null, [ + "Reference source locations could not be normalized to function offsets.", + ]); + return compareCollections( + "strings_names", + leftValues, + rightValues, + combineCoverage( + left.collections.referenced_strings, + left.collections.referenced_names, + ), + combineCoverage( + right.collections.referenced_strings, + right.collections.referenced_names, + ), + links, + providersDiffer, + ); +}; + +const compareReferences = ( + left: FunctionSnapshot, + right: FunctionSnapshot, + links: readonly string[], + providersDiffer: boolean, +): FunctionDimension => { + const leftCoverage = combineCoverage( + left.collections.incoming_references, + left.collections.outgoing_references, + ); + const rightCoverage = combineCoverage( + right.collections.incoming_references, + right.collections.outgoing_references, + ); + if (!leftCoverage.complete || !rightCoverage.complete) + return unresolvedDimension( + "references", + leftCoverage.truncated || rightCoverage.truncated + ? "truncated" + : "unknown", + links, + leftCoverage.items.length, + rightCoverage.items.length, + [], + ); + const leftProjection = referenceProjection(left); + const rightProjection = referenceProjection(right); + if (leftProjection.length === 0 && rightProjection.length === 0) + return compareValues( + "references", + leftProjection, + rightProjection, + links, + true, + true, + providersDiffer, + ); + if (canonicalJson(leftProjection) !== canonicalJson(rightProjection)) + return dimensionResult({ + dimension: "references", + status: "changed", + left: leftProjection, + right: rightProjection, + links, + providersDiffer, + leftCount: leftProjection.length, + rightCount: rightProjection.length, + textDelta: null, + limitations: [ + "Reference endpoints differ independently of provider reference-kind metadata.", + ], + }); + const leftKinds = referenceKindProjection(left); + const rightKinds = referenceKindProjection(right); + if (leftKinds !== null && rightKinds !== null) + return compareValues( + "references", + leftKinds, + rightKinds, + links, + true, + true, + providersDiffer, + ); + return unresolvedDimension( + "references", + "unknown", + links, + leftProjection.length, + rightProjection.length, + [ + "At least one provider did not expose reference kinds, so equal endpoints do not prove equal edge semantics.", + ], + ); +}; + +const compareIdentity = ( + left: FunctionSnapshot, + right: FunctionSnapshot, + links: readonly string[], + providersDiffer: boolean, +): FunctionDimension => { + if (isAutoName(left.procedure.name) || isAutoName(right.procedure.name)) + return unresolvedDimension("identity", "unknown", links, null, null, [ + "Address-derived function names are not stable cross-version identity.", + ]); + return compareValues( + "identity", + identityProjection(left), + identityProjection(right), + links, + true, + true, + providersDiffer, + ); +}; + +const compareCalls = ( + left: FunctionSnapshot, + right: FunctionSnapshot, + links: readonly string[], + providersDiffer: boolean, +): FunctionDimension => { + const leftValues = callProjection(left); + const rightValues = callProjection(right); + if ([...leftValues, ...rightValues].some(({ name }) => isAutoName(name))) + return unresolvedDimension( + "calls", + "unknown", + links, + leftValues.length, + rightValues.length, + ["Address-derived callee or caller names cannot be matched safely."], + ); + return compareCollections( + "calls", + leftValues, + rightValues, + combineCoverage(left.collections.callers, left.collections.callees), + combineCoverage(right.collections.callers, right.collections.callees), + links, + providersDiffer, + ); +}; + +const callProjection = (snapshot: FunctionSnapshot) => [ + ...project(snapshot.collections.callers, ({ name }) => ({ + direction: "in" as const, + name, + })), + ...project(snapshot.collections.callees, ({ name }) => ({ + direction: "out" as const, + name, + })), +]; + +const compareCfg = ( + left: FunctionSnapshot, + right: FunctionSnapshot, + links: readonly string[], + providersDiffer: boolean, +): FunctionDimension => { + const leftBlocks = left.collections.basic_blocks; + const rightBlocks = right.collections.basic_blocks; + if (!leftBlocks.complete || !rightBlocks.complete) + return unresolvedDimension( + "cfg", + leftBlocks.truncated || rightBlocks.truncated ? "truncated" : "unknown", + links, + leftBlocks.items.length, + rightBlocks.items.length, + [], + ); + const leftGraph = normalizeCfg(leftBlocks.items); + const rightGraph = normalizeCfg(rightBlocks.items); + if (leftGraph === null || rightGraph === null) + return unresolvedDimension( + "cfg", + "unknown", + links, + leftBlocks.items.length, + rightBlocks.items.length, + ["CFG addresses could not be normalized to local block indices."], + ); + return compareValues( + "cfg", + leftGraph, + rightGraph, + links, + true, + true, + providersDiffer, + ); +}; + +const compareValues = ( + ...[ + dimension, + left, + right, + links, + leftComplete, + rightComplete, + providersDiffer, + ]: readonly [ + DimensionName, + unknown, + unknown, + readonly string[], + boolean, + boolean, + boolean, + ] +): FunctionDimension => { + if (!leftComplete || !rightComplete) + return unresolvedDimension(dimension, "unknown", links, null, null, []); + const leftJson = canonicalJson(left); + const rightJson = canonicalJson(right); + return dimensionResult({ + dimension, + status: leftJson === rightJson ? "unchanged" : "changed", + left, + right, + links, + providersDiffer, + leftCount: Array.isArray(left) ? left.length : null, + rightCount: Array.isArray(right) ? right.length : null, + textDelta: null, + limitations: [], + }); +}; diff --git a/src/domain/javascriptApplicationVersionItems.ts b/src/domain/javascriptApplicationVersionItems.ts index 07db36faa2..e007d457ba 100644 --- a/src/domain/javascriptApplicationVersionItems.ts +++ b/src/domain/javascriptApplicationVersionItems.ts @@ -250,39 +250,41 @@ const relationshipSignature = ( const signatures = graph.edges.flatMap((edge) => { if (edge.source_node_id === node.node_id) return [ - edgeSignature( - "out", - edge.relation, - edge.target_node_id, + edgeSignature({ + direction: "out", + relation: edge.relation, + neighborId: edge.target_node_id, paired, nodeById, - ), + }), ]; if (edge.target_node_id === node.node_id) return [ - edgeSignature( - "in", - edge.relation, - edge.source_node_id, + edgeSignature({ + direction: "in", + relation: edge.relation, + neighborId: edge.source_node_id, paired, nodeById, - ), + }), ]; return []; }); return canonical(signatures.sort(compareCodePoints)); }; -const edgeSignature = ( - direction: "in" | "out", - relation: ApplicationEdge["relation"], - neighborId: string, - paired: ReadonlyMap, - nodeById: ReadonlyMap, -): string => { - const mapped = paired.get(neighborId); - const neighbor = nodeById.get(neighborId); - return `${direction}\0${relation}\0${mapped ?? `unmatched:${neighbor?.kind ?? "unknown"}`}`; +interface EdgeSignatureContext { + readonly direction: "in" | "out"; + readonly relation: ApplicationEdge["relation"]; + readonly neighborId: string; + readonly paired: ReadonlyMap; + readonly nodeById: ReadonlyMap; +} + +const edgeSignature = (context: EdgeSignatureContext): string => { + const mapped = context.paired.get(context.neighborId); + const neighbor = context.nodeById.get(context.neighborId); + return `${context.direction}\0${context.relation}\0${mapped ?? `unmatched:${neighbor?.kind ?? "unknown"}`}`; }; const itemEvidenceLinks = ( diff --git a/src/domain/javascriptApplicationVersionKeys.ts b/src/domain/javascriptApplicationVersionKeys.ts index 6ad5475ac4..2ed05138f3 100644 --- a/src/domain/javascriptApplicationVersionKeys.ts +++ b/src/domain/javascriptApplicationVersionKeys.ts @@ -54,7 +54,7 @@ export const matchJavaScriptApplicationVersions = ( const rightCandidates = new Map>(); const pairs: ApplicationVersionNodePair[] = []; for (const tier of matchTiers()) - applyTier(left, right, pairs, leftCandidates, rightCandidates, tier); + applyTier({ left, right, pairs, leftCandidates, rightCandidates, tier }); const remainingLeft = sortedNodes(left.values()); const remainingRight = sortedNodes(right.values()); const leftIds = new Set(remainingLeft.map(({ node_id: id }) => id)); @@ -73,28 +73,30 @@ export const matchJavaScriptApplicationVersions = ( }; }; -const applyTier = ( - left: Map, - right: Map, - pairs: ApplicationVersionNodePair[], - leftCandidates: Map>, - rightCandidates: Map>, - tier: MatchTier, -): void => { - const leftByKey = nodesByKey(left.values(), tier.key); - const rightByKey = nodesByKey(right.values(), tier.key); +interface TierContext { + readonly left: Map; + readonly right: Map; + readonly pairs: ApplicationVersionNodePair[]; + readonly leftCandidates: Map>; + readonly rightCandidates: Map>; + readonly tier: MatchTier; +} + +const applyTier = (context: TierContext): void => { + const leftByKey = nodesByKey(context.left.values(), context.tier.key); + const rightByKey = nodesByKey(context.right.values(), context.tier.key); for (const key of [...leftByKey.keys()].sort(compareCodePoints)) { const leftGroup = leftByKey.get(key) ?? []; const rightGroup = rightByKey.get(key) ?? []; if (rightGroup.length === 0) continue; - recordCandidates(leftGroup, rightGroup, leftCandidates, rightCandidates); + recordCandidates(leftGroup, rightGroup, context.leftCandidates, context.rightCandidates); if (leftGroup.length !== 1 || rightGroup.length !== 1) continue; const leftNode = leftGroup[0]; const rightNode = rightGroup[0]; if (leftNode === undefined || rightNode === undefined) continue; - pairs.push({ left: leftNode, right: rightNode, ...tier }); - left.delete(leftNode.node_id); - right.delete(rightNode.node_id); + context.pairs.push({ left: leftNode, right: rightNode, ...context.tier }); + context.left.delete(leftNode.node_id); + context.right.delete(rightNode.node_id); } }; diff --git a/src/domain/javascriptFeatureTraversal.ts b/src/domain/javascriptFeatureTraversal.ts index 52304e5fbe..4711f1c36f 100644 --- a/src/domain/javascriptFeatureTraversal.ts +++ b/src/domain/javascriptFeatureTraversal.ts @@ -50,11 +50,11 @@ export const traverseApplicationFeature = ( const depth = depths.get(current) ?? 0; for (const entry of adjacency.get(current) ?? []) { if (depth >= limits.max_depth) { - recordOmission(entry, current, omittedNodes, omittedEdges, frontier); + recordOmission({ entry, current, omittedNodes, omittedEdges, frontier }); continue; } if (!visited.has(entry.nextNodeId) && visited.size >= limits.max_nodes) { - recordOmission(entry, current, omittedNodes, omittedEdges, frontier); + recordOmission({ entry, current, omittedNodes, omittedEdges, frontier }); continue; } if ( @@ -129,15 +129,17 @@ const addAdjacency = ( adjacency.set(nodeId, [...(adjacency.get(nodeId) ?? []), entry]); }; -const recordOmission = ( - entry: AdjacencyEntry, - current: string, - omittedNodes: Set, - omittedEdges: Set, - frontier: Set, -): void => { - omittedNodes.add(entry.nextNodeId); - omittedEdges.add(entry.edge.edge_id); - frontier.add(current); - frontier.add(entry.nextNodeId); +interface OmissionContext { + readonly entry: AdjacencyEntry; + readonly current: string; + readonly omittedNodes: Set; + readonly omittedEdges: Set; + readonly frontier: Set; +} + +const recordOmission = (context: OmissionContext): void => { + context.omittedNodes.add(context.entry.nextNodeId); + context.omittedEdges.add(context.entry.edge.edge_id); + context.frontier.add(context.current); + context.frontier.add(context.entry.nextNodeId); }; diff --git a/src/domain/javascriptStaticAnalysis.ts b/src/domain/javascriptStaticAnalysis.ts index 5b7ba3cf01..96ea22980a 100644 --- a/src/domain/javascriptStaticAnalysis.ts +++ b/src/domain/javascriptStaticAnalysis.ts @@ -1,53 +1,30 @@ import { parse } from "@babel/parser"; import * as t from "@babel/types"; +import { inspectElectronStaticNode } from "./electronStaticAnalysis.js"; import { - collectJavaScriptExports, - fingerprintJavaScriptAst, -} from "./javascriptAstFingerprint.js"; -import { - STATIC_ROUTE_CALL_NAMES, - argumentValue, - calleeName, - chunkRuntime, - compareCodePoints, detectVendors, - endpointArgument, failedJavaScriptStaticAnalysis, - factoryRequireName, - moduleFactory, - modulePropertyName, - prefixedArgument, - propertyName, - range, - rangeForOffsets, registrationKey, - sanitizeCandidate, - sha256Text, sortedUnique, - sourceSlice, - staticArrayValues, - staticPath, - staticPathResolutionContext, - storageKind, stringValue, } from "./javascriptStaticAnalysisHelpers.js"; +import { + addReference, + addSourceMapDirectives, + inspectCall, + inspectRouteProperty, + inspectRoleProperty, +} from "./javascriptStaticAnalysisCalls.js"; +import { inspectBundlerRegistration } from "./javascriptStaticAnalysisBundler.js"; +import { + finalizeLocatedFindings, +} from "./javascriptStaticAnalysisFindings.js"; import { createJavaScriptAnalysisAccumulator, type JavaScriptAnalysisAccumulator as AnalysisAccumulator, - type JavaScriptEndpointInput as EndpointInput, - type JavaScriptFindingContext as FindingContext, - type JavaScriptReferenceInput as ReferenceInput, } from "./javascriptStaticAnalysisState.js"; -import { - addBoundedFinding, - addLocatedFinding, - finalizeLocatedFindings, - moduleAtOffset, -} from "./javascriptStaticAnalysisFindings.js"; -import { inspectElectronStaticNode } from "./electronStaticAnalysis.js"; import type { - JavaScriptBundlerModule, JavaScriptStaticAnalysis, JavaScriptStaticAnalysisLimits, } from "./javascriptStaticAnalysisTypes.js"; @@ -184,7 +161,7 @@ const inspectNode = ( accumulator: AnalysisAccumulator, limits: JavaScriptStaticAnalysisLimits, ): void => { - const findings: FindingContext = { + const findings = { source, accumulator, maximum: limits.maxFindings, @@ -220,540 +197,3 @@ const inspectNode = ( inspectRoleProperty(node, findings); } }; - -const inspectCall = ( - node: t.CallExpression | t.NewExpression, - context: FindingContext, -): void => { - const { accumulator } = context; - const name = calleeName(node.callee); - const first = stringValue(node.arguments[0]); - const module = moduleAtOffset(node.start, accumulator.modules); - const moduleRequireName = module?.requireName ?? null; - const requireCall = requireCallSpecifier(node, name, moduleRequireName); - if (requireCall !== null) - addReference(context, { - node, - kind: "require", - specifier: requireCall.specifier, - }); - const chunkLoad = chunkLoadSpecifier(node, name, moduleRequireName); - if (chunkLoad !== null) - addReference(context, { - node, - kind: "dynamic-import", - specifier: chunkLoad.specifier, - }); - if (name === "Worker" || name.endsWith(".Worker")) - addReference(context, { node, kind: "worker", specifier: first }); - if (name.endsWith("serviceWorker.register")) - addReference(context, { node, kind: "service-worker", specifier: first }); - inspectEndpointCall(node, name, first, context); - inspectStorageCall(node, name, first, context); - if (name.endsWith("loadFile") && first !== undefined) - addLocatedFinding(context, { - collection: accumulator.roles, - key: `role\0renderer\0${first}`, - node, - value: { - role: "renderer", - path: first, - resolution_context: "filesystem-expression", - mechanism: `call:${name}`, - module_key: null, - location: range(node), - }, - }); - if (name.endsWith("loadURL") && first !== undefined) - addLocatedFinding(context, { - collection: accumulator.roles, - key: `role\0renderer\0${first}`, - node, - value: { - role: "renderer", - path: first, - resolution_context: "module-specifier", - mechanism: `call:${name}`, - module_key: null, - location: range(node), - }, - }); -}; - -const requireCallSpecifier = ( - node: t.CallExpression | t.NewExpression, - name: string, - moduleRequireName: string | null, -): { readonly specifier: string | undefined } | null => { - if (name === "require" || name.endsWith(".require")) - return { specifier: argumentValue(node.arguments[0]) }; - if (name === "__webpack_require__" || name === moduleRequireName) - return { specifier: argumentValue(node.arguments[0]) }; - if ( - name === "__webpack_require__.bind" || - (moduleRequireName !== null && name === `${moduleRequireName}.bind`) - ) - return { specifier: argumentValue(node.arguments[1]) }; - return null; -}; - -const chunkLoadSpecifier = ( - node: t.CallExpression | t.NewExpression, - name: string, - moduleRequireName: string | null, -): { readonly specifier: string | undefined } | null => { - if ( - name !== "__webpack_require__.e" && - (moduleRequireName === null || name !== `${moduleRequireName}.e`) - ) - return null; - return { specifier: prefixedArgument("chunk:", node.arguments[0]) }; -}; - -const inspectBundlerRegistration = ( - source: string, - call: t.CallExpression, - accumulator: AnalysisAccumulator, - limits: JavaScriptStaticAnalysisLimits, -): void => { - const runtime = chunkRuntime(call); - const entry = call.arguments[0]; - if (runtime === undefined || !t.isArrayExpression(entry)) return; - const chunkIds = entry.elements[0]; - const table = entry.elements[1]; - const runtimeValue = runtimeMetadata(entry.elements[2], limits.maxFindings); - if (!t.isArrayExpression(chunkIds) || !t.isObjectExpression(table)) return; - const recovered = recoverBundlerModules(source, table, accumulator, limits); - const chunkKeys = staticArrayValues( - chunkIds, - Math.min(256, limits.maxFindings), - ); - if (chunkKeys.omitted > 0) { - accumulator.droppedFindings += chunkKeys.omitted; - accumulator.structuralTruncation = true; - } - accumulator.unknownFindings += chunkKeys.unknown; - if ( - runtimeValue.omittedEntryModuleKeys > 0 || - runtimeValue.omittedAsyncChunkKeys > 0 || - recovered.omittedAsyncChunkKeys > 0 - ) { - accumulator.droppedFindings += - runtimeValue.omittedEntryModuleKeys + - runtimeValue.omittedAsyncChunkKeys + - recovered.omittedAsyncChunkKeys; - accumulator.structuralTruncation = true; - } - accumulator.unknownFindings += - runtimeValue.unknownEntryModuleKeys + - runtimeValue.unknownAsyncChunkKeys + - recovered.unknownAsyncChunkKeys; - const asyncChunkKeys = boundedUniqueValues( - [...runtimeValue.asyncChunkKeys, ...recovered.asyncChunkKeys], - Math.min(256, limits.maxFindings), - ); - if (asyncChunkKeys.omitted > 0) { - accumulator.droppedFindings += asyncChunkKeys.omitted; - accumulator.structuralTruncation = true; - } - const registration = { - bundler: runtime.toLowerCase().includes("rspack") - ? ("rspack" as const) - : ("webpack" as const), - runtime, - chunk_keys: chunkKeys.values, - omitted_chunk_keys: chunkKeys.omitted, - unknown_chunk_keys: chunkKeys.unknown, - runtime_require_name: runtimeValue.requireName, - runtime_module_cache_status: bundlerModuleCacheStatus(source), - entry_module_keys: runtimeValue.entryModuleKeys, - omitted_entry_module_keys: runtimeValue.omittedEntryModuleKeys, - unknown_entry_module_keys: runtimeValue.unknownEntryModuleKeys, - async_chunk_keys: asyncChunkKeys.values, - omitted_async_chunk_keys: - runtimeValue.omittedAsyncChunkKeys + - recovered.omittedAsyncChunkKeys + - asyncChunkKeys.omitted, - unknown_async_chunk_keys: - runtimeValue.unknownAsyncChunkKeys + recovered.unknownAsyncChunkKeys, - modules: recovered.modules.sort((left, right) => - compareCodePoints(left.module_key, right.module_key), - ), - location: range(call), - }; - addBoundedFinding( - accumulator, - `registration\0${registrationKey(registration)}`, - limits.maxFindings, - () => accumulator.registrations.push(registration), - ); -}; - -interface RecoveredBundlerModules { - readonly modules: JavaScriptBundlerModule[]; - readonly asyncChunkKeys: readonly string[]; - readonly omittedAsyncChunkKeys: number; - readonly unknownAsyncChunkKeys: number; -} - -const recoverBundlerModules = ( - source: string, - table: t.ObjectExpression, - accumulator: AnalysisAccumulator, - limits: JavaScriptStaticAnalysisLimits, -): RecoveredBundlerModules => { - const modules: JavaScriptBundlerModule[] = []; - const asyncChunkKeys: string[] = []; - let omittedAsyncChunkKeys = 0; - let unknownAsyncChunkKeys = 0; - for (const property of table.properties) { - const recovered = recoverBundlerModule( - source, - property, - accumulator, - limits, - ); - if (recovered === null) continue; - modules.push(recovered.module); - asyncChunkKeys.push(...recovered.asyncChunkKeys); - omittedAsyncChunkKeys += recovered.omittedAsyncChunkKeys; - unknownAsyncChunkKeys += recovered.unknownAsyncChunkKeys; - } - return { - modules, - asyncChunkKeys, - omittedAsyncChunkKeys, - unknownAsyncChunkKeys, - }; -}; - -const recoverBundlerModule = ( - source: string, - property: t.ObjectMethod | t.ObjectProperty | t.SpreadElement, - accumulator: AnalysisAccumulator, - limits: JavaScriptStaticAnalysisLimits, -): { - readonly module: JavaScriptBundlerModule; - readonly asyncChunkKeys: readonly string[]; - readonly omittedAsyncChunkKeys: number; - readonly unknownAsyncChunkKeys: number; -} | null => { - const factory = moduleFactory(property); - if (factory === undefined) return null; - if (accumulator.moduleCount >= limits.maxModules) { - accumulator.droppedFindings += 1; - return null; - } - accumulator.moduleCount += 1; - const key = modulePropertyName(property); - if (key.startsWith("[computed@") || key.startsWith("[unknown@")) - accumulator.unknownFindings += 1; - const fingerprint = fingerprintJavaScriptAst( - factory, - Math.min(limits.maxAstNodes, 100_000), - ); - const exportsValue = collectJavaScriptExports(factory, 256); - const requireName = factoryRequireName(factory); - const asyncChunks = collectBundlerAsyncChunkKeys( - factory, - requireName, - limits.maxFindings, - ); - if (fingerprint.truncated || exportsValue.truncated) { - accumulator.droppedFindings += 1; - accumulator.structuralTruncation = true; - } - if (typeof factory.start === "number" && typeof factory.end === "number") - accumulator.modules.push({ - start: factory.start, - end: factory.end, - key, - requireName, - }); - return { - module: { - module_key: key, - factory_require_name: requireName, - source_sha256: sha256Text(sourceSlice(source, factory)), - structural_fingerprint_sha256: fingerprint.truncated - ? null - : fingerprint.sha256, - structural_fingerprint_algorithm: fingerprint.truncated - ? null - : ("babel-ast-v1" as const), - structural_fingerprint_status: fingerprint.truncated - ? ("truncated" as const) - : ("complete" as const), - exports: exportsValue.values, - exports_truncated: exportsValue.truncated, - location: range(factory), - }, - asyncChunkKeys: asyncChunks.values, - omittedAsyncChunkKeys: asyncChunks.omitted, - unknownAsyncChunkKeys: asyncChunks.unknown, - }; -}; - -interface RuntimeMetadata { - readonly requireName: string | null; - readonly entryModuleKeys: readonly string[]; - readonly omittedEntryModuleKeys: number; - readonly unknownEntryModuleKeys: number; - readonly asyncChunkKeys: readonly string[]; - readonly omittedAsyncChunkKeys: number; - readonly unknownAsyncChunkKeys: number; -} - -interface BoundedValues { - readonly values: readonly string[]; - readonly omitted: number; - readonly unknown: number; -} - -type BundlerFunction = - | t.FunctionExpression - | t.ArrowFunctionExpression - | t.ObjectMethod; - -const runtimeMetadata = ( - node: t.Node | null | undefined, - maximum: number, -): RuntimeMetadata => { - if (!isBundlerFunction(node)) - return { - requireName: null, - entryModuleKeys: [], - omittedEntryModuleKeys: 0, - unknownEntryModuleKeys: node === null || node === undefined ? 0 : 1, - asyncChunkKeys: [], - omittedAsyncChunkKeys: 0, - unknownAsyncChunkKeys: 0, - }; - const parameter = node.params[0]; - const requireName = t.isIdentifier(parameter) - ? parameter.name.slice(0, 4_096) - : null; - if (requireName === null) - return { - requireName, - entryModuleKeys: [], - omittedEntryModuleKeys: 0, - unknownEntryModuleKeys: 1, - asyncChunkKeys: [], - omittedAsyncChunkKeys: 0, - unknownAsyncChunkKeys: 0, - }; - const entries = collectBundlerEntryModuleKeys(node, requireName, maximum); - const asyncChunks = collectBundlerAsyncChunkKeys(node, requireName, maximum); - return { - requireName, - entryModuleKeys: entries.values, - omittedEntryModuleKeys: entries.omitted, - unknownEntryModuleKeys: entries.unknown, - asyncChunkKeys: asyncChunks.values, - omittedAsyncChunkKeys: asyncChunks.omitted, - unknownAsyncChunkKeys: asyncChunks.unknown, - }; -}; - -const collectBundlerEntryModuleKeys = ( - factory: BundlerFunction, - requireName: string, - maximum: number, -): BoundedValues => - collectBundlerCallArgumentValues(factory, requireName, maximum); - -const collectBundlerAsyncChunkKeys = ( - factory: BundlerFunction, - requireName: string | null, - maximum: number, -): BoundedValues => - requireName === null - ? { values: [], omitted: 0, unknown: 0 } - : collectBundlerCallArgumentValues(factory, `${requireName}.e`, maximum); - -const collectBundlerCallArgumentValues = ( - factory: BundlerFunction, - callee: string, - maximum: number, -): BoundedValues => { - const values: string[] = []; - let unknown = 0; - t.traverseFast(factory, (node) => { - if (!t.isCallExpression(node) && !t.isNewExpression(node)) return undefined; - if (calleeName(node.callee) !== callee) return undefined; - const value = argumentValue(node.arguments[0]); - if (value === undefined) unknown += 1; - else values.push(value); - return undefined; - }); - return boundedUniqueValues(values, Math.min(256, maximum), unknown); -}; - -const isBundlerFunction = ( - node: t.Node | null | undefined, -): node is BundlerFunction => - t.isFunctionExpression(node) || - t.isArrowFunctionExpression(node) || - t.isObjectMethod(node); - -const boundedUniqueValues = ( - values: readonly string[], - maximum: number, - unknown = 0, -): BoundedValues => { - const bounded = [ - ...new Set(values.map((value) => value.slice(0, 4_096))), - ].sort(compareCodePoints); - const retained = bounded.slice(0, Math.max(0, maximum)); - return { - values: retained, - omitted: Math.max(0, bounded.length - retained.length), - unknown, - }; -}; - -const bundlerModuleCacheStatus = ( - source: string, -): "observed" | "not-observed" => - /\b(?:__webpack_module_cache__|__rspack_module_cache__|installedModules)\b/u.test( - source, - ) - ? "observed" - : "not-observed"; - -const inspectEndpointCall = ( - node: t.CallExpression | t.NewExpression, - name: string, - first: string | undefined, - context: FindingContext, -): void => { - const route = STATIC_ROUTE_CALL_NAMES.some( - (candidate) => name === candidate || name.endsWith(`.${candidate}`), - ); - if (route && first !== undefined) - addEndpoint(context, { - node, - kind: "route", - value: first, - mechanism: `call:${name}`, - }); - const endpoint = endpointArgument(name, node.arguments); - if (endpoint !== undefined) - addEndpoint(context, { - node, - kind: "network", - value: endpoint, - mechanism: `call:${name}`, - }); -}; - -const inspectRouteProperty = ( - node: t.ObjectProperty, - context: FindingContext, -): void => { - const key = propertyName(node.key); - if ((key === "path" || key === "route") && t.isStringLiteral(node.value)) - addEndpoint(context, { - node, - kind: "route", - value: node.value.value, - mechanism: `property:${key}`, - }); -}; - -const inspectRoleProperty = ( - node: t.ObjectProperty, - context: FindingContext, -): void => { - if (propertyName(node.key) !== "preload") return; - const path = staticPath(node.value); - if (path === undefined) return; - addLocatedFinding(context, { - collection: context.accumulator.roles, - key: `role\0preload\0${path}`, - node, - value: { - role: "preload", - path, - resolution_context: staticPathResolutionContext(node.value), - mechanism: "property:preload", - module_key: null, - location: range(node), - }, - }); -}; - -const inspectStorageCall = ( - node: t.CallExpression | t.NewExpression, - name: string, - first: string | undefined, - context: FindingContext, -): void => { - const storage = storageKind(name); - if (storage === undefined) return; - addLocatedFinding(context, { - collection: context.accumulator.storage, - key: `storage\0${storage}\0${first ?? ""}`, - node, - value: { - kind: storage, - name: first ?? null, - mechanism: `call:${name}`, - module_key: null, - location: range(node), - }, - }); -}; - -const addReference = (context: FindingContext, input: ReferenceInput): void => { - const boundedSpecifier = input.specifier?.slice(0, 4_096) ?? null; - const expression = - boundedSpecifier === null - ? sourceSlice(context.source, input.node).slice(0, 4_096) - : null; - addLocatedFinding(context, { - collection: context.accumulator.references, - key: `reference\0${input.kind}\0${boundedSpecifier ?? expression}`, - node: input.node, - value: { - kind: input.kind, - specifier: boundedSpecifier, - expression, - module_key: null, - location: range(input.node), - }, - }); -}; - -const addEndpoint = (context: FindingContext, input: EndpointInput): void => - addLocatedFinding(context, { - collection: context.accumulator.endpoints, - key: `endpoint\0${input.kind}\0${input.value}`, - node: input.node, - value: { - kind: input.kind, - value: sanitizeCandidate(input.value), - mechanism: input.mechanism, - module_key: null, - location: range(input.node), - }, - }); - -const addSourceMapDirectives = ( - source: string, - accumulator: AnalysisAccumulator, - maximum: number, -): void => { - for (const match of source.matchAll( - /\/\/[#@]\s*sourceMappingURL\s*=\s*([^\s]+)/gu, - )) { - const declared = match[1]?.slice(0, 4_096); - if (declared === undefined) continue; - const start = match.index; - const location = rangeForOffsets(source, start, start + match[0].length); - addBoundedFinding(accumulator, `source-map\0${declared}`, maximum, () => - accumulator.sourceMaps.push({ declared_url: declared, location }), - ); - } -}; diff --git a/src/domain/javascriptStaticAnalysisBundler.ts b/src/domain/javascriptStaticAnalysisBundler.ts new file mode 100644 index 0000000000..6852b4a0c9 --- /dev/null +++ b/src/domain/javascriptStaticAnalysisBundler.ts @@ -0,0 +1,339 @@ +import * as t from "@babel/types"; + +import { + collectJavaScriptExports, + fingerprintJavaScriptAst, +} from "./javascriptAstFingerprint.js"; +import { addBoundedFinding } from "./javascriptStaticAnalysisFindings.js"; +import { + argumentValue, + calleeName, + chunkRuntime, + compareCodePoints, + factoryRequireName, + moduleFactory, + modulePropertyName, + range, + registrationKey, + sha256Text, + sourceSlice, + staticArrayValues, +} from "./javascriptStaticAnalysisHelpers.js"; +import type { JavaScriptAnalysisAccumulator as AnalysisAccumulator } from "./javascriptStaticAnalysisState.js"; +import type { + JavaScriptBundlerModule, + JavaScriptStaticAnalysisLimits, +} from "./javascriptStaticAnalysisTypes.js"; + +/** Inspect a bundler registration call and recover its module table. */ +export const inspectBundlerRegistration = ( + source: string, + call: t.CallExpression, + accumulator: AnalysisAccumulator, + limits: JavaScriptStaticAnalysisLimits, +): void => { + const runtime = chunkRuntime(call); + const entry = call.arguments[0]; + if (runtime === undefined || !t.isArrayExpression(entry)) return; + const chunkIds = entry.elements[0]; + const table = entry.elements[1]; + const runtimeValue = runtimeMetadata(entry.elements[2], limits.maxFindings); + if (!t.isArrayExpression(chunkIds) || !t.isObjectExpression(table)) return; + const recovered = recoverBundlerModules(source, table, accumulator, limits); + const chunkKeys = staticArrayValues( + chunkIds, + Math.min(256, limits.maxFindings), + ); + if (chunkKeys.omitted > 0) { + accumulator.droppedFindings += chunkKeys.omitted; + accumulator.structuralTruncation = true; + } + accumulator.unknownFindings += chunkKeys.unknown; + if ( + runtimeValue.omittedEntryModuleKeys > 0 || + runtimeValue.omittedAsyncChunkKeys > 0 || + recovered.omittedAsyncChunkKeys > 0 + ) { + accumulator.droppedFindings += + runtimeValue.omittedEntryModuleKeys + + runtimeValue.omittedAsyncChunkKeys + + recovered.omittedAsyncChunkKeys; + accumulator.structuralTruncation = true; + } + accumulator.unknownFindings += + runtimeValue.unknownEntryModuleKeys + + runtimeValue.unknownAsyncChunkKeys + + recovered.unknownAsyncChunkKeys; + const asyncChunkKeys = boundedUniqueValues( + [...runtimeValue.asyncChunkKeys, ...recovered.asyncChunkKeys], + Math.min(256, limits.maxFindings), + ); + if (asyncChunkKeys.omitted > 0) { + accumulator.droppedFindings += asyncChunkKeys.omitted; + accumulator.structuralTruncation = true; + } + const registration = { + bundler: runtime.toLowerCase().includes("rspack") + ? ("rspack" as const) + : ("webpack" as const), + runtime, + chunk_keys: chunkKeys.values, + omitted_chunk_keys: chunkKeys.omitted, + unknown_chunk_keys: chunkKeys.unknown, + runtime_require_name: runtimeValue.requireName, + runtime_module_cache_status: bundlerModuleCacheStatus(source), + entry_module_keys: runtimeValue.entryModuleKeys, + omitted_entry_module_keys: runtimeValue.omittedEntryModuleKeys, + unknown_entry_module_keys: runtimeValue.unknownEntryModuleKeys, + async_chunk_keys: asyncChunkKeys.values, + omitted_async_chunk_keys: + runtimeValue.omittedAsyncChunkKeys + + recovered.omittedAsyncChunkKeys + + asyncChunkKeys.omitted, + unknown_async_chunk_keys: + runtimeValue.unknownAsyncChunkKeys + recovered.unknownAsyncChunkKeys, + modules: recovered.modules.sort((left, right) => + compareCodePoints(left.module_key, right.module_key), + ), + location: range(call), + }; + addBoundedFinding( + accumulator, + `registration\0${registrationKey(registration)}`, + limits.maxFindings, + () => accumulator.registrations.push(registration), + ); +}; + +interface RecoveredBundlerModules { + readonly modules: JavaScriptBundlerModule[]; + readonly asyncChunkKeys: readonly string[]; + readonly omittedAsyncChunkKeys: number; + readonly unknownAsyncChunkKeys: number; +} + +const recoverBundlerModules = ( + source: string, + table: t.ObjectExpression, + accumulator: AnalysisAccumulator, + limits: JavaScriptStaticAnalysisLimits, +): RecoveredBundlerModules => { + const modules: JavaScriptBundlerModule[] = []; + const asyncChunkKeys: string[] = []; + let omittedAsyncChunkKeys = 0; + let unknownAsyncChunkKeys = 0; + for (const property of table.properties) { + const recovered = recoverBundlerModule( + source, + property, + accumulator, + limits, + ); + if (recovered === null) continue; + modules.push(recovered.module); + asyncChunkKeys.push(...recovered.asyncChunkKeys); + omittedAsyncChunkKeys += recovered.omittedAsyncChunkKeys; + unknownAsyncChunkKeys += recovered.unknownAsyncChunkKeys; + } + return { + modules, + asyncChunkKeys, + omittedAsyncChunkKeys, + unknownAsyncChunkKeys, + }; +}; + +const recoverBundlerModule = ( + source: string, + property: t.ObjectMethod | t.ObjectProperty | t.SpreadElement, + accumulator: AnalysisAccumulator, + limits: JavaScriptStaticAnalysisLimits, +): { + readonly module: JavaScriptBundlerModule; + readonly asyncChunkKeys: readonly string[]; + readonly omittedAsyncChunkKeys: number; + readonly unknownAsyncChunkKeys: number; +} | null => { + const factory = moduleFactory(property); + if (factory === undefined) return null; + if (accumulator.moduleCount >= limits.maxModules) { + accumulator.droppedFindings += 1; + return null; + } + accumulator.moduleCount += 1; + const key = modulePropertyName(property); + if (key.startsWith("[computed@") || key.startsWith("[unknown@")) + accumulator.unknownFindings += 1; + const fingerprint = fingerprintJavaScriptAst( + factory, + Math.min(limits.maxAstNodes, 100_000), + ); + const exportsValue = collectJavaScriptExports(factory, 256); + const requireName = factoryRequireName(factory); + const asyncChunks = collectBundlerAsyncChunkKeys( + factory, + requireName, + limits.maxFindings, + ); + if (fingerprint.truncated || exportsValue.truncated) { + accumulator.droppedFindings += 1; + accumulator.structuralTruncation = true; + } + if (typeof factory.start === "number" && typeof factory.end === "number") + accumulator.modules.push({ + start: factory.start, + end: factory.end, + key, + requireName, + }); + return { + module: { + module_key: key, + factory_require_name: requireName, + source_sha256: sha256Text(sourceSlice(source, factory)), + structural_fingerprint_sha256: fingerprint.truncated + ? null + : fingerprint.sha256, + structural_fingerprint_algorithm: fingerprint.truncated + ? null + : ("babel-ast-v1" as const), + structural_fingerprint_status: fingerprint.truncated + ? ("truncated" as const) + : ("complete" as const), + exports: exportsValue.values, + exports_truncated: exportsValue.truncated, + location: range(factory), + }, + asyncChunkKeys: asyncChunks.values, + omittedAsyncChunkKeys: asyncChunks.omitted, + unknownAsyncChunkKeys: asyncChunks.unknown, + }; +}; + +interface RuntimeMetadata { + readonly requireName: string | null; + readonly entryModuleKeys: readonly string[]; + readonly omittedEntryModuleKeys: number; + readonly unknownEntryModuleKeys: number; + readonly asyncChunkKeys: readonly string[]; + readonly omittedAsyncChunkKeys: number; + readonly unknownAsyncChunkKeys: number; +} + +interface BoundedValues { + readonly values: readonly string[]; + readonly omitted: number; + readonly unknown: number; +} + +type BundlerFunction = + | t.FunctionExpression + | t.ArrowFunctionExpression + | t.ObjectMethod; + +const runtimeMetadata = ( + node: t.Node | null | undefined, + maximum: number, +): RuntimeMetadata => { + if (!isBundlerFunction(node)) + return { + requireName: null, + entryModuleKeys: [], + omittedEntryModuleKeys: 0, + unknownEntryModuleKeys: node === null || node === undefined ? 0 : 1, + asyncChunkKeys: [], + omittedAsyncChunkKeys: 0, + unknownAsyncChunkKeys: 0, + }; + const parameter = node.params[0]; + const requireName = t.isIdentifier(parameter) + ? parameter.name.slice(0, 4_096) + : null; + if (requireName === null) + return { + requireName, + entryModuleKeys: [], + omittedEntryModuleKeys: 0, + unknownEntryModuleKeys: 1, + asyncChunkKeys: [], + omittedAsyncChunkKeys: 0, + unknownAsyncChunkKeys: 0, + }; + const entries = collectBundlerEntryModuleKeys(node, requireName, maximum); + const asyncChunks = collectBundlerAsyncChunkKeys(node, requireName, maximum); + return { + requireName, + entryModuleKeys: entries.values, + omittedEntryModuleKeys: entries.omitted, + unknownEntryModuleKeys: entries.unknown, + asyncChunkKeys: asyncChunks.values, + omittedAsyncChunkKeys: asyncChunks.omitted, + unknownAsyncChunkKeys: asyncChunks.unknown, + }; +}; + +const collectBundlerEntryModuleKeys = ( + factory: BundlerFunction, + requireName: string, + maximum: number, +): BoundedValues => + collectBundlerCallArgumentValues(factory, requireName, maximum); + +const collectBundlerAsyncChunkKeys = ( + factory: BundlerFunction, + requireName: string | null, + maximum: number, +): BoundedValues => + requireName === null + ? { values: [], omitted: 0, unknown: 0 } + : collectBundlerCallArgumentValues(factory, `${requireName}.e`, maximum); + +const collectBundlerCallArgumentValues = ( + factory: BundlerFunction, + callee: string, + maximum: number, +): BoundedValues => { + const values: string[] = []; + let unknown = 0; + t.traverseFast(factory, (node) => { + if (!t.isCallExpression(node) && !t.isNewExpression(node)) return undefined; + if (calleeName(node.callee) !== callee) return undefined; + const value = argumentValue(node.arguments[0]); + if (value === undefined) unknown += 1; + else values.push(value); + return undefined; + }); + return boundedUniqueValues(values, Math.min(256, maximum), unknown); +}; + +const isBundlerFunction = ( + node: t.Node | null | undefined, +): node is BundlerFunction => + t.isFunctionExpression(node) || + t.isArrowFunctionExpression(node) || + t.isObjectMethod(node); + +const boundedUniqueValues = ( + values: readonly string[], + maximum: number, + unknown = 0, +): BoundedValues => { + const bounded = [ + ...new Set(values.map((value) => value.slice(0, 4_096))), + ].sort(compareCodePoints); + const retained = bounded.slice(0, Math.max(0, maximum)); + return { + values: retained, + omitted: Math.max(0, bounded.length - retained.length), + unknown, + }; +}; + +const bundlerModuleCacheStatus = ( + source: string, +): "observed" | "not-observed" => + /\b(?:__webpack_module_cache__|__rspack_module_cache__|installedModules)\b/u.test( + source, + ) + ? "observed" + : "not-observed"; diff --git a/src/domain/javascriptStaticAnalysisCalls.ts b/src/domain/javascriptStaticAnalysisCalls.ts new file mode 100644 index 0000000000..6314eac2a6 --- /dev/null +++ b/src/domain/javascriptStaticAnalysisCalls.ts @@ -0,0 +1,258 @@ +import * as t from "@babel/types"; + +import { + addBoundedFinding, + addLocatedFinding, + moduleAtOffset, +} from "./javascriptStaticAnalysisFindings.js"; +import { + STATIC_ROUTE_CALL_NAMES, + argumentValue, + calleeName, + endpointArgument as endpointArgumentHelper, + prefixedArgument, + propertyName, + range, + rangeForOffsets, + sanitizeCandidate, + sourceSlice, + staticPath, + staticPathResolutionContext, + storageKind, + stringValue, +} from "./javascriptStaticAnalysisHelpers.js"; +import type { + JavaScriptAnalysisAccumulator as AnalysisAccumulator, + JavaScriptEndpointInput as EndpointInput, + JavaScriptFindingContext as FindingContext, + JavaScriptReferenceInput as ReferenceInput, +} from "./javascriptStaticAnalysisState.js"; + +/** Inspect one call or new expression for references, endpoints, storage, and roles. */ +export const inspectCall = ( + node: t.CallExpression | t.NewExpression, + context: FindingContext, +): void => { + const { accumulator } = context; + const name = calleeName(node.callee); + const first = stringValue(node.arguments[0]); + const module = moduleAtOffset(node.start, context.accumulator.modules); + const moduleRequireName = module?.requireName ?? null; + const requireCall = requireCallSpecifier(node, name, moduleRequireName); + if (requireCall !== null) + addReference(context, { + node, + kind: "require", + specifier: requireCall.specifier, + }); + const chunkLoad = chunkLoadSpecifier(node, name, moduleRequireName); + if (chunkLoad !== null) + addReference(context, { + node, + kind: "dynamic-import", + specifier: chunkLoad.specifier, + }); + if (name === "Worker" || name.endsWith(".Worker")) + addReference(context, { node, kind: "worker", specifier: first }); + if (name.endsWith("serviceWorker.register")) + addReference(context, { node, kind: "service-worker", specifier: first }); + inspectEndpointCall(node, name, first, context); + inspectStorageCall(node, name, first, context); + if (name.endsWith("loadFile") && first !== undefined) + addLocatedFinding(context, { + collection: accumulator.roles, + key: `role\0renderer\0${first}`, + node, + value: { + role: "renderer", + path: first, + resolution_context: "filesystem-expression", + mechanism: `call:${name}`, + module_key: null, + location: range(node), + }, + }); + if (name.endsWith("loadURL") && first !== undefined) + addLocatedFinding(context, { + collection: accumulator.roles, + key: `role\0renderer\0${first}`, + node, + value: { + role: "renderer", + path: first, + resolution_context: "module-specifier", + mechanism: `call:${name}`, + module_key: null, + location: range(node), + }, + }); +}; + +const requireCallSpecifier = ( + node: t.CallExpression | t.NewExpression, + name: string, + moduleRequireName: string | null, +): { readonly specifier: string | undefined } | null => { + if (name === "require" || name.endsWith(".require")) + return { specifier: argumentValue(node.arguments[0]) }; + if (name === "__webpack_require__" || name === moduleRequireName) + return { specifier: argumentValue(node.arguments[0]) }; + if ( + name === "__webpack_require__.bind" || + (moduleRequireName !== null && name === `${moduleRequireName}.bind`) + ) + return { specifier: argumentValue(node.arguments[1]) }; + return null; +}; + +const chunkLoadSpecifier = ( + node: t.CallExpression | t.NewExpression, + name: string, + moduleRequireName: string | null, +): { readonly specifier: string | undefined } | null => { + if ( + name !== "__webpack_require__.e" && + (moduleRequireName === null || name !== `${moduleRequireName}.e`) + ) + return null; + return { specifier: prefixedArgument("chunk:", node.arguments[0]) }; +}; + +const inspectEndpointCall = ( + node: t.CallExpression | t.NewExpression, + name: string, + first: string | undefined, + context: FindingContext, +): void => { + const route = STATIC_ROUTE_CALL_NAMES.some( + (candidate) => name === candidate || name.endsWith(`.${candidate}`), + ); + if (route && first !== undefined) + addEndpoint(context, { + node, + kind: "route", + value: first, + mechanism: `call:${name}`, + }); + const endpoint = endpointArgumentHelper(name, node.arguments); + if (endpoint !== undefined) + addEndpoint(context, { + node, + kind: "network", + value: endpoint, + mechanism: `call:${name}`, + }); +}; + +export const inspectRouteProperty = ( + node: t.ObjectProperty, + context: FindingContext, +): void => { + const key = propertyName(node.key); + if ((key === "path" || key === "route") && t.isStringLiteral(node.value)) + addEndpoint(context, { + node, + kind: "route", + value: node.value.value, + mechanism: `property:${key}`, + }); +}; + +export const inspectRoleProperty = ( + node: t.ObjectProperty, + context: FindingContext, +): void => { + if (propertyName(node.key) !== "preload") return; + const path = staticPath(node.value); + if (path === undefined) return; + addLocatedFinding(context, { + collection: context.accumulator.roles, + key: `role\0preload\0${path}`, + node, + value: { + role: "preload", + path, + resolution_context: staticPathResolutionContext(node.value), + mechanism: "property:preload", + module_key: null, + location: range(node), + }, + }); +}; + +const inspectStorageCall = ( + node: t.CallExpression | t.NewExpression, + name: string, + first: string | undefined, + context: FindingContext, +): void => { + const storage = storageKind(name); + if (storage === undefined) return; + addLocatedFinding(context, { + collection: context.accumulator.storage, + key: `storage\0${storage}\0${first ?? ""}`, + node, + value: { + kind: storage, + name: first ?? null, + mechanism: `call:${name}`, + module_key: null, + location: range(node), + }, + }); +}; + +export const addReference = ( + context: FindingContext, + input: ReferenceInput, +): void => { + const boundedSpecifier = input.specifier?.slice(0, 4_096) ?? null; + const expression = + boundedSpecifier === null + ? sourceSlice(context.source, input.node).slice(0, 4_096) + : null; + addLocatedFinding(context, { + collection: context.accumulator.references, + key: `reference\0${input.kind}\0${boundedSpecifier ?? expression}`, + node: input.node, + value: { + kind: input.kind, + specifier: boundedSpecifier, + expression, + module_key: null, + location: range(input.node), + }, + }); +}; + +const addEndpoint = (context: FindingContext, input: EndpointInput): void => + addLocatedFinding(context, { + collection: context.accumulator.endpoints, + key: `endpoint\0${input.kind}\0${input.value}`, + node: input.node, + value: { + kind: input.kind, + value: sanitizeCandidate(input.value), + mechanism: input.mechanism, + module_key: null, + location: range(input.node), + }, + }); + +export const addSourceMapDirectives = ( + source: string, + accumulator: AnalysisAccumulator, + maximum: number, +): void => { + for (const match of source.matchAll( + /\/\/[#@]\s*sourceMappingURL\s*=\s*([^\s]+)/gu, + )) { + const declared = match[1]?.slice(0, 4_096); + if (declared === undefined) continue; + const start = match.index; + const location = rangeForOffsets(source, start, start + match[0].length); + addBoundedFinding(accumulator, `source-map\0${declared}`, maximum, () => + accumulator.sourceMaps.push({ declared_url: declared, location }), + ); + } +}; diff --git a/src/domain/webBundleAnalyzer.ts b/src/domain/webBundleAnalyzer.ts index 19881f4921..1da97defe0 100644 --- a/src/domain/webBundleAnalyzer.ts +++ b/src/domain/webBundleAnalyzer.ts @@ -1,44 +1,20 @@ -import { parse } from "@babel/parser"; -import * as t from "@babel/types"; - import type { WebPageInspection } from "./browserObservation.js"; -import { sanitizeBrowserUrl } from "./browserObservation.js"; import { - webBundleAnalysisSchema, - type AnalyzeWebBundleInput, - type WebBundleAnalysis, + analyzeScript, + emptyAccumulator, + isIncludedScript, +} from "./webBundleAnalyzerInspection.js"; +import { buildWebBundleAnalysis } from "./webBundleAnalyzerResult.js"; +import type { + AnalyzeWebBundleInput, + WebBundleAnalysis, } from "./webBundleAnalysis.js"; -type BundleObservations = WebBundleAnalysis["observations"]; -type ChunkEdge = BundleObservations["chunks"]["edges"][number]; -type Finding = BundleObservations["routes"][number]; -type WebMcpDeclaration = BundleObservations["webmcp_declarations"][number]; -type Inference = WebBundleAnalysis["inferences"][number]; -type BrowserScript = WebPageInspection["scripts"]["items"][number]; -type IncludedSource = Extract; -type IncludedScript = Omit & { - readonly source: IncludedSource; -}; - -interface AnalysisAccumulator { - readonly edges: ChunkEdge[]; - readonly routes: Finding[]; - readonly endpoints: Finding[]; - readonly webMcp: WebMcpDeclaration[]; - readonly inferences: Inference[]; - readonly seen: Set; - visitedNodes: number; - parsedScripts: number; - parseFailures: number; - droppedFindings: number; - astLimitReached: boolean; -} - /** Analyze captured, explicitly approved JavaScript source without execution. */ export const analyzeCapturedWebBundle = ( inspection: WebPageInspection, input: AnalyzeWebBundleInput, - sourceMaps: BundleObservations["source_maps"] = { + sourceMaps: WebBundleAnalysis["observations"]["source_maps"] = { status: "not_requested", requested: 0, processed: 0, @@ -50,591 +26,5 @@ export const analyzeCapturedWebBundle = ( const accumulator = emptyAccumulator(); const sourceScripts = inspection.scripts.items.filter(isIncludedScript); for (const script of sourceScripts) analyzeScript(script, input, accumulator); - const unavailable = inspection.scripts.items - .filter((script) => !script.source.included) - .map(({ script_key }) => script_key); - const sourceMapIncomplete = - sourceMaps.status === "partial" || sourceMaps.status === "unavailable"; - const partial = - accumulator.parseFailures > 0 || - unavailable.length > 0 || - sourceMapIncomplete; - const truncated = - accumulator.astLimitReached || - accumulator.droppedFindings > 0 || - sourceMaps.status === "truncated"; - return webBundleAnalysisSchema.parse({ - schema_version: 1, - capture: { - target_url: inspection.target.url, - scripts_observed: inspection.scripts.total, - scripts_analyzed: sourceScripts.length, - source_artifacts: sourceScripts.map((script) => { - if (!script.source.included) - throw new TypeError("Filtered source changed"); - const { text: _text, ...artifact } = script.source.artifact; - return { ...artifact, text_available: true }; - }), - }, - observations: { - chunks: { - nodes: sourceScripts.map((script) => { - if (!script.source.included) - throw new TypeError("Filtered source changed"); - return { - script_key: script.script_key, - url: script.url, - artifact_sha256: script.source.artifact.sha256, - bytes: script.source.artifact.bytes, - }; - }), - edges: accumulator.edges, - }, - routes: accumulator.routes, - endpoints: accumulator.endpoints, - webmcp_declarations: accumulator.webMcp, - source_maps: sourceMaps, - }, - inferences: accumulator.inferences, - unknowns: [ - ...(unavailable.length === 0 - ? [] - : [ - { - dimension: "script_source", - reason: "Source artifact was not captured within approved limits", - affected_script_keys: unavailable, - }, - ]), - ...(accumulator.parseFailures === 0 - ? [] - : [ - { - dimension: "javascript_ast", - reason: "One or more source artifacts could not be parsed", - affected_script_keys: sourceScripts.map( - ({ script_key }) => script_key, - ), - }, - ]), - ...(sourceMaps.status === "not_requested" || - sourceMaps.status === "included" - ? [] - : [ - { - dimension: "source_maps", - reason: - sourceMaps.status === "truncated" - ? "Source-map evidence was truncated by approved limits" - : "One or more requested source maps were unavailable or incomplete", - affected_script_keys: [ - ...new Set([ - ...sourceMaps.items - .filter(({ status }) => status !== "included") - .map(({ script_key }) => script_key), - ...sourceMaps.dropped_script_keys, - ]), - ].sort(), - }, - ]), - ], - completeness: { - status: truncated - ? "truncated" - : partial - ? "partial" - : "complete_within_limits", - parsed_scripts: accumulator.parsedScripts, - parse_failures: accumulator.parseFailures, - visited_ast_nodes: accumulator.visitedNodes, - dropped_findings: accumulator.droppedFindings, - }, - limitations: [ - "Static bundle findings are observations or bounded inferences; REA does not execute captured JavaScript.", - "String-built routes and endpoints, encrypted configuration, and server-side behavior may remain unknown.", - "Page-declared WebMCP metadata is untrusted and is never registered or invoked as an REA tool.", - ], - }); + return buildWebBundleAnalysis(inspection, sourceScripts, sourceMaps, accumulator); }; - -const analyzeScript = ( - script: IncludedScript, - input: AnalyzeWebBundleInput, - accumulator: AnalysisAccumulator, -): void => { - let file: ReturnType; - try { - file = parse(script.source.artifact.text, { - sourceType: "unambiguous", - errorRecovery: true, - plugins: ["jsx", "typescript"], - }); - } catch { - accumulator.parseFailures += 1; - return; - } - accumulator.parsedScripts += 1; - detectVendorFingerprints( - script, - accumulator, - input.analysis_limits.max_findings, - ); - t.traverseFast(file, (node) => { - accumulator.visitedNodes += 1; - if (accumulator.visitedNodes > input.analysis_limits.max_ast_nodes) { - accumulator.astLimitReached = true; - return t.traverseFast.stop; - } - inspectNode(script, node, accumulator, input.analysis_limits.max_findings); - return undefined; - }); -}; - -const inspectNode = ( - script: IncludedScript, - node: t.Node, - accumulator: AnalysisAccumulator, - maximumFindings: number, -): void => { - if ( - (t.isImportDeclaration(node) || t.isExportAllDeclaration(node)) && - node.source !== undefined - ) - addEdge( - script, - node.source.value, - "static_import", - node, - accumulator, - maximumFindings, - ); - else if (t.isExportNamedDeclaration(node) && node.source != null) - addEdge( - script, - node.source.value, - "static_import", - node, - accumulator, - maximumFindings, - ); - else if (t.isImportExpression(node) && t.isStringLiteral(node.source)) - addEdge( - script, - node.source.value, - "dynamic_import", - node, - accumulator, - maximumFindings, - ); - if (t.isObjectProperty(node)) - inspectRouteProperty(script, node, accumulator, maximumFindings); - if (t.isCallExpression(node) || t.isNewExpression(node)) - inspectCall(script, node, accumulator, maximumFindings); -}; - -const inspectCall = ( - script: IncludedScript, - node: t.CallExpression | t.NewExpression, - accumulator: AnalysisAccumulator, - maximumFindings: number, -): void => { - const name = calleeName(node.callee); - const first = stringArgument(node.arguments[0]); - if ((name === "require" || name.endsWith(".require")) && first !== undefined) - addEdge(script, first, "require", node, accumulator, maximumFindings); - if ( - (name === "importScripts" || name.endsWith(".importScripts")) && - first !== undefined - ) - addEdge(script, first, "worker_import", node, accumulator, maximumFindings); - if ( - routeCallNames.some( - (candidate) => name === candidate || name.endsWith(`.${candidate}`), - ) - ) { - if (first !== undefined) - addFinding( - accumulator.routes, - script, - first, - `call:${name}`, - node, - accumulator, - maximumFindings, - ); - addRouteFrameworkInference( - script, - name, - node, - accumulator, - maximumFindings, - ); - } - const endpoint = endpointArgument(name, node.arguments); - if (endpoint !== undefined) - addFinding( - accumulator.endpoints, - script, - endpoint, - `call:${name}`, - node, - accumulator, - maximumFindings, - ); - if ( - name.endsWith("modelContext.registerTool") || - name === "modelContext.registerTool" - ) - addWebMcpDeclaration(script, node, accumulator, maximumFindings); -}; - -const inspectRouteProperty = ( - script: IncludedScript, - node: t.ObjectProperty, - accumulator: AnalysisAccumulator, - maximumFindings: number, -): void => { - const key = propertyName(node.key); - if ((key === "path" || key === "route") && t.isStringLiteral(node.value)) - addFinding( - accumulator.routes, - script, - node.value.value, - `property:${key}`, - node, - accumulator, - maximumFindings, - ); -}; - -const addEdge = ( - script: IncludedScript, - specifier: string, - kind: ChunkEdge["kind"], - node: t.Node, - accumulator: AnalysisAccumulator, - maximumFindings: number, -): void => { - const bounded = specifier.slice(0, 4_096); - const key = `edge\0${script.script_key}\0${kind}\0${bounded}`; - addBounded(accumulator, key, maximumFindings, () => - accumulator.edges.push({ - from_script_key: script.script_key, - kind, - specifier: bounded, - resolved_url: resolveSpecifier(bounded, script.url), - location: location(script.script_key, node), - }), - ); -}; - -const addFinding = ( - collection: Finding[], - script: IncludedScript, - rawValue: string, - mechanism: string, - node: t.Node, - accumulator: AnalysisAccumulator, - maximumFindings: number, -): void => { - const value = sanitizeCandidate(rawValue); - const key = `finding\0${mechanism}\0${script.script_key}\0${value}`; - addBounded(accumulator, key, maximumFindings, () => - collection.push({ - value, - mechanism, - location: location(script.script_key, node), - }), - ); -}; - -const addWebMcpDeclaration = ( - script: IncludedScript, - node: t.CallExpression | t.NewExpression, - accumulator: AnalysisAccumulator, - maximumFindings: number, -): void => { - const declaration = node.arguments[0]; - if (!t.isObjectExpression(declaration)) return; - const name = objectString(declaration, "name"); - const description = objectString(declaration, "description"); - const schema = - objectValue(declaration, "inputSchema") ?? - objectValue(declaration, "input_schema"); - const declaredProperties = t.isObjectExpression(schema) - ? objectValue(schema, "properties") - : undefined; - const propertyObject = t.isObjectExpression(declaredProperties) - ? declaredProperties - : schema; - const schemaPropertyNames = t.isObjectExpression(propertyObject) - ? propertyObject.properties.flatMap((property) => - t.isObjectProperty(property) - ? [propertyName(property.key)].filter(Boolean) - : [], - ) - : []; - const key = `webmcp\0${script.script_key}\0${name ?? ""}\0${schemaPropertyNames.join("\0")}`; - addBounded(accumulator, key, maximumFindings, () => - accumulator.webMcp.push({ - name: name?.slice(0, 256) ?? null, - description: description?.slice(0, 2_048) ?? null, - schema_property_names: [...new Set(schemaPropertyNames)] - .sort() - .slice(0, 256), - trust: "page-declared-untrusted", - location: location(script.script_key, node), - }), - ); -}; - -const detectVendorFingerprints = ( - script: IncludedScript, - accumulator: AnalysisAccumulator, - maximumFindings: number, -): void => { - const text = script.source.artifact.text; - for (const detector of vendorDetectors) { - if (!detector.patterns.some((pattern) => text.includes(pattern))) continue; - const key = `vendor\0${script.script_key}\0${detector.value}`; - addBounded(accumulator, key, maximumFindings, () => - accumulator.inferences.push({ - kind: detector.kind, - value: detector.value, - confidence: detector.confidence, - basis: [basis(script, detector.detector)], - }), - ); - } -}; - -const addRouteFrameworkInference = ( - script: IncludedScript, - name: string, - node: t.Node, - accumulator: AnalysisAccumulator, - maximumFindings: number, -): void => { - const value = - name.endsWith("useRoutes") || name.endsWith("createBrowserRouter") - ? "React Router-compatible route API" - : "Generic route registration API"; - const key = `route-framework\0${script.script_key}\0${value}`; - addBounded(accumulator, key, maximumFindings, () => - accumulator.inferences.push({ - kind: "route_framework", - value, - confidence: "medium", - basis: [{ ...basis(script, `call:${name}`), ...locationFields(node) }], - }), - ); -}; - -const addBounded = ( - accumulator: AnalysisAccumulator, - key: string, - maximum: number, - add: () => void, -): void => { - if (accumulator.seen.has(key)) return; - const findings = - accumulator.edges.length + - accumulator.routes.length + - accumulator.endpoints.length + - accumulator.webMcp.length + - accumulator.inferences.length; - if (findings >= maximum) { - accumulator.droppedFindings += 1; - return; - } - accumulator.seen.add(key); - add(); -}; - -const endpointArgument = ( - name: string, - args: readonly ( - | t.Expression - | t.SpreadElement - | t.JSXNamespacedName - | t.ArgumentPlaceholder - )[], -): string | undefined => { - if (name === "fetch" || name.endsWith(".fetch") || name === "WebSocket") - return stringArgument(args[0]); - if (name.endsWith(".open") && stringArgument(args[1]) !== undefined) - return stringArgument(args[1]); - if ( - ["get", "post", "put", "patch", "delete", "request"].some( - (method) => name === method || name.endsWith(`.${method}`), - ) - ) - return stringArgument(args[0]); - return undefined; -}; - -const calleeName = (callee: t.Node): string => { - if (t.isIdentifier(callee)) return callee.name; - if (t.isImport(callee)) return "import"; - if (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) { - const object = t.isExpression(callee.object) - ? calleeName(callee.object) - : ""; - const property = propertyName(callee.property); - return object === "" ? property : `${object}.${property}`; - } - return ""; -}; - -const stringArgument = ( - value: t.Node | null | undefined, -): string | undefined => (t.isStringLiteral(value) ? value.value : undefined); - -const propertyName = (value: t.Node): string => { - if (t.isIdentifier(value)) return value.name; - if (t.isStringLiteral(value)) return value.value; - return ""; -}; - -const objectValue = ( - object: t.ObjectExpression, - name: string, -): t.ObjectProperty["value"] | undefined => { - for (const property of object.properties) - if (t.isObjectProperty(property) && propertyName(property.key) === name) - return property.value; - return undefined; -}; - -const objectString = ( - object: t.ObjectExpression, - name: string, -): string | undefined => { - const value = objectValue(object, name); - return t.isStringLiteral(value) ? value.value : undefined; -}; - -const resolveSpecifier = (specifier: string, base: string): string | null => { - try { - const resolved = new URL(specifier, base); - if (resolved.protocol !== "http:" && resolved.protocol !== "https:") - return null; - return sanitizeBrowserUrl(resolved.href).url; - } catch { - return null; - } -}; - -const sanitizeCandidate = (value: string): string => { - const bounded = value.slice(0, 4_096); - try { - const parsed = new URL(bounded, "https://rea.invalid"); - const sanitized = sanitizeBrowserUrl(parsed.href).url; - return parsed.origin === "https://rea.invalid" - ? sanitized.replace("https://rea.invalid", "") - : sanitized; - } catch { - return bounded.split("#", 1)[0]?.split("?", 1)[0] ?? ""; - } -}; - -const location = (scriptKey: string, node: t.Node) => ({ - script_key: scriptKey, - ...locationFields(node), -}); - -const locationFields = (node: t.Node) => ({ - line: node.loc?.start.line ?? null, - column: node.loc?.start.column ?? null, -}); - -const basis = (script: IncludedScript, detector: string) => ({ - script_key: script.script_key, - artifact_sha256: script.source.artifact.sha256, - line: null, - column: null, - detector, -}); - -const emptyAccumulator = (): AnalysisAccumulator => ({ - edges: [], - routes: [], - endpoints: [], - webMcp: [], - inferences: [], - seen: new Set(), - visitedNodes: 0, - parsedScripts: 0, - parseFailures: 0, - droppedFindings: 0, - astLimitReached: false, -}); - -const isIncludedScript = (script: BrowserScript): script is IncludedScript => - script.source.included; - -const routeCallNames = [ - "route", - "addRoute", - "useRoutes", - "createBrowserRouter", - "createHashRouter", -] as const; - -const vendorDetectors: readonly { - readonly value: string; - readonly detector: string; - readonly patterns: readonly string[]; - readonly confidence: Inference["confidence"]; - readonly kind: Inference["kind"]; -}[] = [ - { - value: "webpack", - detector: "webpack-runtime", - patterns: ["__webpack_require__"], - confidence: "high", - kind: "bundle_runtime", - }, - { - value: "Vite", - detector: "vite-runtime", - patterns: ["__vite__", "import.meta.hot"], - confidence: "medium", - kind: "bundle_runtime", - }, - { - value: "React", - detector: "react-runtime", - patterns: ["__REACT_DEVTOOLS_GLOBAL_HOOK__", "React.createElement"], - confidence: "medium", - kind: "vendor_fingerprint", - }, - { - value: "Vue", - detector: "vue-runtime", - patterns: ["__VUE__", "createApp("], - confidence: "medium", - kind: "vendor_fingerprint", - }, - { - value: "Next.js", - detector: "next-runtime", - patterns: ["__NEXT_DATA__", "/_next/"], - confidence: "high", - kind: "vendor_fingerprint", - }, - { - value: "Angular", - detector: "angular-runtime", - patterns: ["ɵɵdefineComponent"], - confidence: "high", - kind: "vendor_fingerprint", - }, - { - value: "Svelte", - detector: "svelte-runtime", - patterns: ["svelte/internal"], - confidence: "medium", - kind: "vendor_fingerprint", - }, -]; diff --git a/src/domain/webBundleAnalyzerAst.ts b/src/domain/webBundleAnalyzerAst.ts new file mode 100644 index 0000000000..2da30038e6 --- /dev/null +++ b/src/domain/webBundleAnalyzerAst.ts @@ -0,0 +1,105 @@ +import * as t from "@babel/types"; + +import { sanitizeBrowserUrl } from "./browserObservation.js"; + +/** Name of a call/new callee from its AST node. */ +export const calleeName = (callee: t.Node): string => { + if (t.isIdentifier(callee)) return callee.name; + if (t.isImport(callee)) return "import"; + if (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) { + const object = t.isExpression(callee.object) + ? calleeName(callee.object) + : ""; + const property = propertyName(callee.property); + return object === "" ? property : `${object}.${property}`; + } + return ""; +}; + +export const stringArgument = ( + value: t.Node | null | undefined, +): string | undefined => (t.isStringLiteral(value) ? value.value : undefined); + +export const propertyName = (value: t.Node): string => { + if (t.isIdentifier(value)) return value.name; + if (t.isStringLiteral(value)) return value.value; + return ""; +}; + +export const objectValue = ( + object: t.ObjectExpression, + name: string, +): t.ObjectProperty["value"] | undefined => { + for (const property of object.properties) + if (t.isObjectProperty(property) && propertyName(property.key) === name) + return property.value; + return undefined; +}; + +export const objectString = ( + object: t.ObjectExpression, + name: string, +): string | undefined => { + const value = objectValue(object, name); + return t.isStringLiteral(value) ? value.value : undefined; +}; + +export const resolveSpecifier = ( + specifier: string, + base: string, +): string | null => { + try { + const resolved = new URL(specifier, base); + if (resolved.protocol !== "http:" && resolved.protocol !== "https:") + return null; + return sanitizeBrowserUrl(resolved.href).url; + } catch { + return null; + } +}; + +export const sanitizeCandidate = (value: string): string => { + const bounded = value.slice(0, 4_096); + try { + const parsed = new URL(bounded, "https://rea.invalid"); + const sanitized = sanitizeBrowserUrl(parsed.href).url; + return parsed.origin === "https://rea.invalid" + ? sanitized.replace("https://rea.invalid", "") + : sanitized; + } catch { + return bounded.split("#", 1)[0]?.split("?", 1)[0] ?? ""; + } +}; + +export const location = (scriptKey: string, node: t.Node) => ({ + script_key: scriptKey, + ...locationFields(node), +}); + +export const locationFields = (node: t.Node) => ({ + line: node.loc?.start.line ?? null, + column: node.loc?.start.column ?? null, +}); + +/** Pick the endpoint argument for common network-call patterns. */ +export const endpointArgument = ( + name: string, + args: readonly ( + | t.Expression + | t.SpreadElement + | t.JSXNamespacedName + | t.ArgumentPlaceholder + )[], +): string | undefined => { + if (name === "fetch" || name.endsWith(".fetch") || name === "WebSocket") + return stringArgument(args[0]); + if (name.endsWith(".open") && stringArgument(args[1]) !== undefined) + return stringArgument(args[1]); + if ( + ["get", "post", "put", "patch", "delete", "request"].some( + (method) => name === method || name.endsWith(`.${method}`), + ) + ) + return stringArgument(args[0]); + return undefined; +}; diff --git a/src/domain/webBundleAnalyzerInspection.ts b/src/domain/webBundleAnalyzerInspection.ts new file mode 100644 index 0000000000..00752adfb7 --- /dev/null +++ b/src/domain/webBundleAnalyzerInspection.ts @@ -0,0 +1,453 @@ +import { parse } from "@babel/parser"; +import * as t from "@babel/types"; + +import type { WebPageInspection } from "./browserObservation.js"; +import type { + AnalyzeWebBundleInput, + WebBundleAnalysis, +} from "./webBundleAnalysis.js"; +import { + calleeName, + endpointArgument, + location, + locationFields, + objectString, + objectValue, + propertyName, + resolveSpecifier, + sanitizeCandidate, + stringArgument, +} from "./webBundleAnalyzerAst.js"; + +type BundleObservations = WebBundleAnalysis["observations"]; +type ChunkEdge = BundleObservations["chunks"]["edges"][number]; +type Finding = BundleObservations["routes"][number]; +type WebMcpDeclaration = BundleObservations["webmcp_declarations"][number]; +type Inference = WebBundleAnalysis["inferences"][number]; +type BrowserScript = WebPageInspection["scripts"]["items"][number]; +type IncludedSource = Extract; +export type IncludedScript = Omit & { + readonly source: IncludedSource; +}; + +export interface AnalysisAccumulator { + readonly edges: ChunkEdge[]; + readonly routes: Finding[]; + readonly endpoints: Finding[]; + readonly webMcp: WebMcpDeclaration[]; + readonly inferences: Inference[]; + readonly seen: Set; + visitedNodes: number; + parsedScripts: number; + parseFailures: number; + droppedFindings: number; + astLimitReached: boolean; +} + +export const analyzeScript = ( + script: IncludedScript, + input: AnalyzeWebBundleInput, + accumulator: AnalysisAccumulator, +): void => { + let file: ReturnType; + try { + file = parse(script.source.artifact.text, { + sourceType: "unambiguous", + errorRecovery: true, + plugins: ["jsx", "typescript"], + }); + } catch { + accumulator.parseFailures += 1; + return; + } + accumulator.parsedScripts += 1; + detectVendorFingerprints( + script, + accumulator, + input.analysis_limits.max_findings, + ); + t.traverseFast(file, (node) => { + accumulator.visitedNodes += 1; + if (accumulator.visitedNodes > input.analysis_limits.max_ast_nodes) { + accumulator.astLimitReached = true; + return t.traverseFast.stop; + } + inspectNode(script, node, accumulator, input.analysis_limits.max_findings); + return undefined; + }); +}; + +const inspectNode = ( + script: IncludedScript, + node: t.Node, + accumulator: AnalysisAccumulator, + maximumFindings: number, +): void => { + if ( + (t.isImportDeclaration(node) || t.isExportAllDeclaration(node)) && + node.source !== undefined + ) + addEdge({ + script, + specifier: node.source.value, + kind: "static_import", + node, + accumulator, + maximumFindings, + }); + else if (t.isExportNamedDeclaration(node) && node.source != null) + addEdge({ + script, + specifier: node.source.value, + kind: "static_import", + node, + accumulator, + maximumFindings, + }); + else if (t.isImportExpression(node) && t.isStringLiteral(node.source)) + addEdge({ + script, + specifier: node.source.value, + kind: "dynamic_import", + node, + accumulator, + maximumFindings, + }); + if (t.isObjectProperty(node)) + inspectRouteProperty(script, node, accumulator, maximumFindings); + if (t.isCallExpression(node) || t.isNewExpression(node)) + inspectCall(script, node, accumulator, maximumFindings); +}; + +const inspectCall = ( + script: IncludedScript, + node: t.CallExpression | t.NewExpression, + accumulator: AnalysisAccumulator, + maximumFindings: number, +): void => { + const name = calleeName(node.callee); + const first = stringArgument(node.arguments[0]); + if ((name === "require" || name.endsWith(".require")) && first !== undefined) + addEdge({ + script, + specifier: first, + kind: "require", + node, + accumulator, + maximumFindings, + }); + if ( + (name === "importScripts" || name.endsWith(".importScripts")) && + first !== undefined + ) + addEdge({ + script, + specifier: first, + kind: "worker_import", + node, + accumulator, + maximumFindings, + }); + if ( + routeCallNames.some( + (candidate) => name === candidate || name.endsWith(`.${candidate}`), + ) + ) { + if (first !== undefined) + addFinding({ + collection: accumulator.routes, + script, + rawValue: first, + mechanism: `call:${name}`, + node, + accumulator, + maximumFindings, + }); + addRouteFrameworkInference({ + script, + name, + node, + accumulator, + maximumFindings, + }); + } + const endpoint = endpointArgument(name, node.arguments); + if (endpoint !== undefined) + addFinding({ + collection: accumulator.endpoints, + script, + rawValue: endpoint, + mechanism: `call:${name}`, + node, + accumulator, + maximumFindings, + }); + if ( + name.endsWith("modelContext.registerTool") || + name === "modelContext.registerTool" + ) + addWebMcpDeclaration(script, node, accumulator, maximumFindings); +}; + +const inspectRouteProperty = ( + script: IncludedScript, + node: t.ObjectProperty, + accumulator: AnalysisAccumulator, + maximumFindings: number, +): void => { + const key = propertyName(node.key); + if ((key === "path" || key === "route") && t.isStringLiteral(node.value)) + addFinding({ + collection: accumulator.routes, + script, + rawValue: node.value.value, + mechanism: `property:${key}`, + node, + accumulator, + maximumFindings, + }); +}; + +interface AddEdgeContext { + readonly script: IncludedScript; + readonly specifier: string; + readonly kind: ChunkEdge["kind"]; + readonly node: t.Node; + readonly accumulator: AnalysisAccumulator; + readonly maximumFindings: number; +} + +const addEdge = (context: AddEdgeContext): void => { + const bounded = context.specifier.slice(0, 4_096); + const key = `edge\0${context.script.script_key}\0${context.kind}\0${bounded}`; + addBounded(context.accumulator, key, context.maximumFindings, () => + context.accumulator.edges.push({ + from_script_key: context.script.script_key, + kind: context.kind, + specifier: bounded, + resolved_url: resolveSpecifier(bounded, context.script.url), + location: location(context.script.script_key, context.node), + }), + ); +}; + +interface AddFindingContext { + readonly collection: Finding[]; + readonly script: IncludedScript; + readonly rawValue: string; + readonly mechanism: string; + readonly node: t.Node; + readonly accumulator: AnalysisAccumulator; + readonly maximumFindings: number; +} + +const addFinding = (context: AddFindingContext): void => { + const value = sanitizeCandidate(context.rawValue); + const key = `finding\0${context.mechanism}\0${context.script.script_key}\0${value}`; + addBounded(context.accumulator, key, context.maximumFindings, () => + context.collection.push({ + value, + mechanism: context.mechanism, + location: location(context.script.script_key, context.node), + }), + ); +}; + +const addWebMcpDeclaration = ( + script: IncludedScript, + node: t.CallExpression | t.NewExpression, + accumulator: AnalysisAccumulator, + maximumFindings: number, +): void => { + const declaration = node.arguments[0]; + if (!t.isObjectExpression(declaration)) return; + const name = objectString(declaration, "name"); + const description = objectString(declaration, "description"); + const schema = + objectValue(declaration, "inputSchema") ?? + objectValue(declaration, "input_schema"); + const declaredProperties = t.isObjectExpression(schema) + ? objectValue(schema, "properties") + : undefined; + const propertyObject = t.isObjectExpression(declaredProperties) + ? declaredProperties + : schema; + const schemaPropertyNames = t.isObjectExpression(propertyObject) + ? propertyObject.properties.flatMap((property) => + t.isObjectProperty(property) + ? [propertyName(property.key)].filter(Boolean) + : [], + ) + : []; + const key = `webmcp\0${script.script_key}\0${name ?? ""}\0${schemaPropertyNames.join("\0")}`; + addBounded(accumulator, key, maximumFindings, () => + accumulator.webMcp.push({ + name: name?.slice(0, 256) ?? null, + description: description?.slice(0, 2_048) ?? null, + schema_property_names: [...new Set(schemaPropertyNames)] + .sort() + .slice(0, 256), + trust: "page-declared-untrusted", + location: location(script.script_key, node), + }), + ); +}; + +const detectVendorFingerprints = ( + script: IncludedScript, + accumulator: AnalysisAccumulator, + maximumFindings: number, +): void => { + const text = script.source.artifact.text; + for (const detector of vendorDetectors) { + if (!detector.patterns.some((pattern) => text.includes(pattern))) continue; + const key = `vendor\0${script.script_key}\0${detector.value}`; + addBounded(accumulator, key, maximumFindings, () => + accumulator.inferences.push({ + kind: detector.kind, + value: detector.value, + confidence: detector.confidence, + basis: [basis(script, detector.detector)], + }), + ); + } +}; + +interface RouteFrameworkContext { + readonly script: IncludedScript; + readonly name: string; + readonly node: t.Node; + readonly accumulator: AnalysisAccumulator; + readonly maximumFindings: number; +} + +const addRouteFrameworkInference = (context: RouteFrameworkContext): void => { + const value = + context.name.endsWith("useRoutes") || + context.name.endsWith("createBrowserRouter") + ? "React Router-compatible route API" + : "Generic route registration API"; + const key = `route-framework\0${context.script.script_key}\0${value}`; + addBounded(context.accumulator, key, context.maximumFindings, () => + context.accumulator.inferences.push({ + kind: "route_framework", + value, + confidence: "medium", + basis: [ + { ...basis(context.script, `call:${context.name}`), ...locationFields(context.node) }, + ], + }), + ); +}; + +const addBounded = ( + accumulator: AnalysisAccumulator, + key: string, + maximum: number, + add: () => void, +): void => { + if (accumulator.seen.has(key)) return; + const findings = + accumulator.edges.length + + accumulator.routes.length + + accumulator.endpoints.length + + accumulator.webMcp.length + + accumulator.inferences.length; + if (findings >= maximum) { + accumulator.droppedFindings += 1; + return; + } + accumulator.seen.add(key); + add(); +}; + +const basis = (script: IncludedScript, detector: string) => ({ + script_key: script.script_key, + artifact_sha256: script.source.artifact.sha256, + line: null, + column: null, + detector, +}); + +export const emptyAccumulator = (): AnalysisAccumulator => ({ + edges: [], + routes: [], + endpoints: [], + webMcp: [], + inferences: [], + seen: new Set(), + visitedNodes: 0, + parsedScripts: 0, + parseFailures: 0, + droppedFindings: 0, + astLimitReached: false, +}); + +export const isIncludedScript = (script: BrowserScript): script is IncludedScript => + script.source.included; + +const routeCallNames = [ + "route", + "addRoute", + "useRoutes", + "createBrowserRouter", + "createHashRouter", +] as const; + +const vendorDetectors: readonly { + readonly value: string; + readonly detector: string; + readonly patterns: readonly string[]; + readonly confidence: Inference["confidence"]; + readonly kind: Inference["kind"]; +}[] = [ + { + value: "webpack", + detector: "webpack-runtime", + patterns: ["__webpack_require__"], + confidence: "high", + kind: "bundle_runtime", + }, + { + value: "Vite", + detector: "vite-runtime", + patterns: ["__vite__", "import.meta.hot"], + confidence: "medium", + kind: "bundle_runtime", + }, + { + value: "React", + detector: "react-runtime", + patterns: ["__REACT_DEVTOOLS_GLOBAL_HOOK__", "React.createElement"], + confidence: "medium", + kind: "vendor_fingerprint", + }, + { + value: "Vue", + detector: "vue-runtime", + patterns: ["__VUE__", "createApp("], + confidence: "medium", + kind: "vendor_fingerprint", + }, + { + value: "Next.js", + detector: "next-runtime", + patterns: ["__NEXT_DATA__", "/_next/"], + confidence: "high", + kind: "vendor_fingerprint", + }, + { + value: "Angular", + detector: "angular-runtime", + patterns: ["ɵɵdefineComponent"], + confidence: "high", + kind: "vendor_fingerprint", + }, + { + value: "Svelte", + detector: "svelte-runtime", + patterns: ["svelte/internal"], + confidence: "medium", + kind: "vendor_fingerprint", + }, +]; diff --git a/src/domain/webBundleAnalyzerResult.ts b/src/domain/webBundleAnalyzerResult.ts new file mode 100644 index 0000000000..0ab1ed059f --- /dev/null +++ b/src/domain/webBundleAnalyzerResult.ts @@ -0,0 +1,164 @@ +import type { WebPageInspection } from "./browserObservation.js"; +import { + webBundleAnalysisSchema, + type WebBundleAnalysis, +} from "./webBundleAnalysis.js"; +import type { AnalysisAccumulator, IncludedScript } from "./webBundleAnalyzerInspection.js"; + +export const buildWebBundleAnalysis = ( + inspection: WebPageInspection, + sourceScripts: IncludedScript[], + sourceMaps: WebBundleAnalysis["observations"]["source_maps"], + accumulator: AnalysisAccumulator, +): WebBundleAnalysis => { + const unavailable = inspection.scripts.items + .filter((script) => !script.source.included) + .map(({ script_key }) => script_key); + const sourceMapIncomplete = + sourceMaps.status === "partial" || sourceMaps.status === "unavailable"; + const partial = + accumulator.parseFailures > 0 || + unavailable.length > 0 || + sourceMapIncomplete; + const truncated = + accumulator.astLimitReached || + accumulator.droppedFindings > 0 || + sourceMaps.status === "truncated"; + return webBundleAnalysisSchema.parse({ + schema_version: 1, + capture: buildCaptureObservation(inspection, sourceScripts), + observations: { + chunks: { + nodes: sourceScripts.map(chunkNode), + edges: accumulator.edges, + }, + routes: accumulator.routes, + endpoints: accumulator.endpoints, + webmcp_declarations: accumulator.webMcp, + source_maps: sourceMaps, + }, + inferences: accumulator.inferences, + unknowns: buildUnknowns({ + sourceScripts, + unavailable, + parseFailures: accumulator.parseFailures, + sourceMaps, + }), + completeness: buildCompleteness({ + truncated, + partial, + parsedScripts: accumulator.parsedScripts, + parseFailures: accumulator.parseFailures, + visitedNodes: accumulator.visitedNodes, + droppedFindings: accumulator.droppedFindings, + }), + limitations: bundleLimitations(), + }); +}; + +const buildCaptureObservation = ( + inspection: WebPageInspection, + sourceScripts: IncludedScript[], +) => ({ + target_url: inspection.target.url, + scripts_observed: inspection.scripts.total, + scripts_analyzed: sourceScripts.length, + source_artifacts: sourceScripts.map(sourceArtifact), +}); + +const sourceArtifact = (script: IncludedScript) => { + if (!script.source.included) + throw new TypeError("Filtered source changed"); + const { text: _text, ...artifact } = script.source.artifact; + return { ...artifact, text_available: true }; +}; + +const chunkNode = (script: IncludedScript) => { + if (!script.source.included) + throw new TypeError("Filtered source changed"); + return { + script_key: script.script_key, + url: script.url, + artifact_sha256: script.source.artifact.sha256, + bytes: script.source.artifact.bytes, + }; +}; + +interface UnknownsInput { + readonly sourceScripts: IncludedScript[]; + readonly unavailable: string[]; + readonly parseFailures: number; + readonly sourceMaps: WebBundleAnalysis["observations"]["source_maps"]; +} + +const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [ + ...(input.unavailable.length === 0 + ? [] + : [ + { + dimension: "script_source" as const, + reason: "Source artifact was not captured within approved limits", + affected_script_keys: input.unavailable, + }, + ]), + ...(input.parseFailures === 0 + ? [] + : [ + { + dimension: "javascript_ast" as const, + reason: "One or more source artifacts could not be parsed", + affected_script_keys: input.sourceScripts.map( + ({ script_key }) => script_key, + ), + }, + ]), + ...(input.sourceMaps.status === "not_requested" || + input.sourceMaps.status === "included" + ? [] + : [ + { + dimension: "source_maps" as const, + reason: + input.sourceMaps.status === "truncated" + ? "Source-map evidence was truncated by approved limits" + : "One or more requested source maps were unavailable or incomplete", + affected_script_keys: [ + ...new Set([ + ...input.sourceMaps.items + .filter(({ status }) => status !== "included") + .map(({ script_key }) => script_key), + ...input.sourceMaps.dropped_script_keys, + ]), + ].sort(), + }, + ]), +]; + +interface CompletenessInput { + readonly truncated: boolean; + readonly partial: boolean; + readonly parsedScripts: number; + readonly parseFailures: number; + readonly visitedNodes: number; + readonly droppedFindings: number; +} + +const buildCompleteness = ( + input: CompletenessInput, +): WebBundleAnalysis["completeness"] => ({ + status: input.truncated + ? "truncated" + : input.partial + ? "partial" + : "complete_within_limits", + parsed_scripts: input.parsedScripts, + parse_failures: input.parseFailures, + visited_ast_nodes: input.visitedNodes, + dropped_findings: input.droppedFindings, +}); + +const bundleLimitations = (): WebBundleAnalysis["limitations"] => [ + "Static bundle findings are observations or bounded inferences; REA does not execute captured JavaScript.", + "String-built routes and endpoints, encrypted configuration, and server-side behavior may remain unknown.", + "Page-declared WebMCP metadata is untrusted and is never registered or invoked as an REA tool.", +];