fix(javascript): retain unresolved network targets with provenance

Resolve stable lexical and URL targets using shared effects, and preserve unresolved request expressions as partial facts. Carry their uncertainty through workers, artifact graphs, and the CLI. Scope partial endpoint identity to the source site.

Refs: #3451, #3534

Co-authored-by: Hemang Varshney <218273197+hemv-857@users.noreply.github.com>

src/domain/javascript/javascriptStaticAnalysis.event_source.test.ts

@@ -34,7 +34,15 @@ describe("server-sent event endpoints", () => {
     ]);
   });
 
-  it("keeps a dynamic EventSource URL unreported", () => {
-    expect(endpoints("new EventSource(url);")).toEqual([]);
+  it("retains a dynamic EventSource target as partial source evidence", () => {
+    expect(
+      analyzeJavaScriptStaticSource("new EventSource(url);").endpoints,
+    ).toEqual([
+      expect.objectContaining({
+        value: "url",
+        mechanism: "call:EventSource",
+        resolution: "partial",
+      }),
+    ]);
   });
 });

src/domain/javascript/javascriptStaticAnalysis.partialEndpoints.test.ts

@@ -0,0 +1,265 @@
+import { expect, it } from "vitest";
+
+import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js";
+
+const endpoints = (source: string) =>
+  analyzeJavaScriptStaticSource(source).endpoints.map((endpoint) => ({
+    kind: endpoint.kind,
+    value: endpoint.value,
+    mechanism: endpoint.mechanism,
+    resolution: endpoint.resolution,
+  }));
+
+it("resolves unchanged lexical strings and object properties", () => {
+  expect(
+    endpoints(`
+        const target = "/items";
+        const config = { url: "/accounts" };
+        fetch(target);
+        fetch("/direct");
+        fetch(config.url);
+      `),
+  ).toEqual([
+    {
+      kind: "network",
+      value: "/items",
+      mechanism: "call:fetch",
+      resolution: undefined,
+    },
+    {
+      kind: "network",
+      value: "/direct",
+      mechanism: "call:fetch",
+      resolution: undefined,
+    },
+    {
+      kind: "network",
+      value: "/accounts",
+      mechanism: "call:fetch",
+      resolution: undefined,
+    },
+  ]);
+});
+
+it("resolves unchanged URL objects and aliases using URL serialization", () => {
+  expect(
+    endpoints(`
+        const socket = new URL("HTTPS://EXAMPLE.COM:443");
+        const alias = socket;
+        const relative = new URL("/items", "https://api.example.test/base");
+        fetch(alias);
+        fetch(relative);
+      `),
+  ).toEqual([
+    {
+      kind: "network",
+      value: "https://example.com/",
+      mechanism: "call:fetch",
+      resolution: undefined,
+    },
+    {
+      kind: "network",
+      value: "https://api.example.test/items",
+      mechanism: "call:fetch",
+      resolution: undefined,
+    },
+  ]);
+});
+
+it("keeps an interpolated request expression verbatim", () => {
+  expect(
+    endpoints("fetch(`https://${hosts['a@b']}/items/${id}`);"),
+  ).toContainEqual({
+    kind: "network",
+    value: "`https://${hosts['a@b']}/items/${id}`",
+    mechanism: "call:fetch",
+    resolution: "partial",
+  });
+});
+
+it("keeps writes to a config property partial instead of resolving its initializer", () => {
+  expect(
+    endpoints(`
+        const config = { url: "https://old.example.test" };
+        config.url = "https://new.example.test";
+        fetch(config.url);
+      `),
+  ).toContainEqual({
+    kind: "network",
+    value: "config.url",
+    mechanism: "call:fetch",
+    resolution: "partial",
+  });
+});
+
+it("keeps a reassigned endpoint binding partial", () => {
+  expect(
+    endpoints(`
+        let endpoint = "https://old.example.test";
+        endpoint = "https://new.example.test";
+        fetch(endpoint);
+      `),
+  ).toContainEqual({
+    kind: "network",
+    value: "endpoint",
+    mechanism: "call:fetch",
+    resolution: "partial",
+  });
+});
+
+it("does not resolve an endpoint from a declaration after the request", () => {
+  expect(
+    endpoints(`
+        fetch(endpoint);
+        const endpoint = "https://api.example.test/items";
+      `),
+  ).toEqual([
+    {
+      kind: "network",
+      value: "endpoint",
+      mechanism: "call:fetch",
+      resolution: "partial",
+    },
+  ]);
+});
+
+it("keeps URLSearchParams mutations partial", () => {
+  expect(
+    endpoints(`
+        const socket = new URL("wss://old.example.test/socket");
+        socket.searchParams.set("token", token);
+        new WebSocket(socket);
+      `),
+  ).toContainEqual({
+    kind: "network",
+    value: "socket",
+    mechanism: "call:WebSocket",
+    resolution: "partial",
+  });
+});
+
+it("resolves a URL object against a literal base", () => {
+  expect(
+    endpoints(`
+        const socket = new URL("/socket", "wss://api.example.test");
+        new WebSocket(socket);
+      `),
+  ).toEqual([
+    {
+      kind: "network",
+      value: "wss://api.example.test/socket",
+      mechanism: "call:WebSocket",
+      resolution: undefined,
+    },
+  ]);
+});
+
+it("resolves an unchanged URL object's toString target", () => {
+  expect(
+    endpoints(`
+        const socket = new URL("wss://api.example.test/socket");
+        new WebSocket(socket.toString());
+      `),
+  ).toEqual([
+    {
+      kind: "network",
+      value: "wss://api.example.test/socket",
+      mechanism: "call:WebSocket",
+      resolution: undefined,
+    },
+  ]);
+});
+
+it("keeps a URL object partial when its base is unresolved", () => {
+  expect(
+    endpoints(`
+        const socket = new URL("/socket", apiOrigin);
+        new WebSocket(socket);
+      `),
+  ).toContainEqual({
+    kind: "network",
+    value: "socket",
+    mechanism: "call:WebSocket",
+    resolution: "partial",
+  });
+});
+
+it("keeps URL aliases partial after a member mutation", () => {
+  expect(
+    endpoints(`
+        const socket = new URL("wss://old.example.test/socket");
+        const alias = socket;
+        alias.hostname = "new.example.test";
+        fetch(socket);
+      `),
+  ).toContainEqual({
+    kind: "network",
+    value: "socket",
+    mechanism: "call:fetch",
+    resolution: "partial",
+  });
+});
+
+it("keeps an escaped URL object partial", () => {
+  expect(
+    endpoints(`
+        const socket = new URL("wss://old.example.test/socket");
+        configure(socket);
+        new WebSocket(socket);
+      `),
+  ).toContainEqual({
+    kind: "network",
+    value: "socket",
+    mechanism: "call:WebSocket",
+    resolution: "partial",
+  });
+});
+
+it("retains dynamic targets on known global fetch receivers", () => {
+  expect(
+    endpoints(`
+        window.fetch(endpoint);
+        globalThis.fetch(endpoint);
+        self.fetch(endpoint);
+      `),
+  ).toEqual([
+    {
+      kind: "network",
+      value: "endpoint",
+      mechanism: "call:window.fetch",
+      resolution: "partial",
+    },
+    {
+      kind: "network",
+      value: "endpoint",
+      mechanism: "call:globalThis.fetch",
+      resolution: "partial",
+    },
+    {
+      kind: "network",
+      value: "endpoint",
+      mechanism: "call:self.fetch",
+      resolution: "partial",
+    },
+  ]);
+});
+
+it("does not treat a locally bound fetch function as a network API", () => {
+  expect(
+    endpoints(`
+        function fetch(key) { return records[key]; }
+        fetch(key);
+      `),
+  ).toEqual([]);
+});
+
+it("keeps exact literal targets exact", () => {
+  expect(endpoints('fetch("https://api.example.test/items");')).toEqual([
+    {
+      kind: "network",
+      value: "https://api.example.test/items",
+      mechanism: "call:fetch",
+      resolution: undefined,
+    },
+  ]);
+});

src/domain/javascript/javascriptStaticAnalysis.send_beacon.test.ts

@@ -28,7 +28,16 @@ describe("beacon endpoints", () => {
     ]);
   });
 
-  it("keeps a dynamic beacon URL unreported", () => {
-    expect(endpoints("navigator.sendBeacon(url, payload);")).toEqual([]);
+  it("retains a dynamic beacon target as partial source evidence", () => {
+    expect(
+      analyzeJavaScriptStaticSource("navigator.sendBeacon(url, payload);")
+        .endpoints,
+    ).toEqual([
+      expect.objectContaining({
+        value: "url",
+        mechanism: "call:navigator.sendBeacon",
+        resolution: "partial",
+      }),
+    ]);
   });
 });

src/domain/javascript/javascriptStaticAnalysis.ts

@@ -21,6 +21,7 @@ import {
 import {
   addReference,
   addSourceMapDirectives,
+  finishPartialNetworkEndpoints,
   inspectCall,
   inspectRouteProperty,
   inspectRoleProperty,
@@ -89,18 +90,19 @@ export function* analyzeParsedJavaScriptStaticSourceSteps(
     file.program,
     bindings,
   );
-  yield* traverseStaticSourceSteps(
-    file,
-    {
-      source,
-      accumulator,
-      vitePreloadDependencyTable: indexVitePreloadDependencyTable(file.program),
-      openReceiverFacts: receivers.openReceivers,
-      electronBindings: electronIdentity.bindings,
-      rpcScan: createJavaScriptRpcScan(source, receivers.rpcClientRoots, index),
-    },
-    bindings,
-  );
+  const findings: JavaScriptFindingContext = {
+    source,
+    accumulator,
+    vitePreloadDependencyTable: indexVitePreloadDependencyTable(file.program),
+    openReceiverFacts: receivers.openReceivers,
+    electronBindings: electronIdentity.bindings,
+    rpcScan: createJavaScriptRpcScan(source, receivers.rpcClientRoots, index),
+  };
+  yield* traverseStaticSourceSteps(file, findings, bindings);
+  if (accumulator.pendingNetworkCalls.length > 0) {
+    yield* index.indexMemberMutationFactsSteps();
+    finishPartialNetworkEndpoints(findings, bindings, index);
+  }
   accumulator.electronCallPaths = collectElectronWrappedCallPaths(
     file.program,
     bindings,

src/domain/javascript/javascriptStaticAnalysisCalls.ts

@@ -34,12 +34,16 @@ import type {
 } from "./javascriptStaticAnalysisState.js";
 import {
   resolveSemanticBindingState,
+  semanticStableInitializer,
+  type JavaScriptBindingIndex,
   type JavaScriptSemanticAnalysisState,
 } from "./javascriptSemanticState.js";
 import {
   isUnmodifiedGlobalReference,
   isUnmodifiedImportMetaUrl,
 } from "./javascriptSemanticAmbientWrites.js";
+import { evaluateSemanticExpression } from "./javascriptSemanticValues.js";
+import { semanticTopLevelEffectFollowsCapture } from "./javascriptSemanticMutationInitializers.js";
 import {
   semanticFactoryLoaderMemberAt,
   semanticFactoryRequireLoaderAt,
@@ -119,7 +123,7 @@ export const inspectCall = (
       kind: "service-worker",
       specifier: moduleUrlSpecifier(node, bindings),
     });
-  inspectEndpointCall(node, name, first, context);
+  inspectEndpointCall(node, name, first, context, bindings);
   inspectStorageCall(node, name, first, context);
   if (first === "audio" || first === "video")
     noteMediaElementPreloads(node, accumulator);
@@ -424,6 +428,7 @@ const inspectEndpointCall = (
   name: string,
   first: string | undefined,
   context: FindingContext,
+  bindings: JavaScriptSemanticAnalysisState,
 ): void => {
   const route = STATIC_ROUTE_CALL_NAMES.some(
     (candidate) => name === candidate || name.endsWith(`.${candidate}`),
@@ -435,6 +440,35 @@ const inspectEndpointCall = (
       value: first,
       mechanism: `call:${name}`,
     });
+  if (isKnownNetworkRequest(node, bindings)) {
+    const target = argumentNode(node.arguments[0]);
+    if (target === undefined) return;
+    const literal = stringValue(target);
+    if (literal !== undefined) {
+      addEndpoint(context, {
+        node,
+        kind: "network",
+        value: literal,
+        mechanism: `call:${name}`,
+      });
+      return;
+    }
+    const reference = requestReferenceNode(target);
+    const unwrapped = unwrapJavaScriptExpression(reference).node;
+    if (
+      t.isIdentifier(unwrapped) ||
+      t.isMemberExpression(unwrapped) ||
+      t.isOptionalMemberExpression(unwrapped)
+    ) {
+      context.accumulator.pendingNetworkCalls.push({ node, name });
+      return;
+    }
+    addPartialNetworkEndpoint(context, node, name, target);
+    return;
+  }
+  // A local or shadowed function with a familiar network name is not a known
+  // browser request API. Do not let the literal fallback turn it into one.
+  if (isNetworkRequestSpelling(name)) return;
   const receiverFact = context.openReceiverFacts?.get(node.start ?? -1);
   const endpoint = endpointArgumentHelper(
     name,
@@ -452,6 +486,229 @@ const inspectEndpointCall = (
     });
 };
 
+const isNetworkRequestSpelling = (name: string): boolean =>
+  name === "fetch" ||
+  name === "WebSocket" ||
+  name === "EventSource" ||
+  name === "sendBeacon" ||
+  name === "window.fetch" ||
+  name === "globalThis.fetch" ||
+  name === "self.fetch" ||
+  name === "navigator.sendBeacon";
+
+/** Confirm that the call resolves to a built-in network API at this source site. */
+const isKnownNetworkRequest = (
+  node: JavaScriptCallLike,
+  bindings: JavaScriptSemanticAnalysisState,
+): boolean => {
+  if (
+    (t.isCallExpression(node) || t.isOptionalCallExpression(node)) &&
+    isUnmodifiedGlobalReference(node.callee, bindings, "fetch")
+  )
+    return true;
+  if (
+    t.isNewExpression(node) &&
+    (isUnmodifiedGlobalReference(node.callee, bindings, "WebSocket") ||
+      isUnmodifiedGlobalReference(node.callee, bindings, "EventSource"))
+  )
+    return true;
+  return (
+    (t.isCallExpression(node) || t.isOptionalCallExpression(node)) &&
+    isUnmodifiedGlobalReference(node.callee, bindings, [
+      "navigator",
+      "sendBeacon",
+    ])
+  );
+};
+
+/** Finish queued targets after shared member-mutation facts are indexed. */
+export const finishPartialNetworkEndpoints = (
+  context: FindingContext,
+  bindings: JavaScriptSemanticAnalysisState,
+  index: JavaScriptBindingIndex,
+): void => {
+  for (const { node, name } of context.accumulator.pendingNetworkCalls) {
+    const target = argumentNode(node.arguments[0]);
+    if (target === undefined) continue;
+    const resolved = resolvedRequestTarget(target, node, bindings, index);
+    if (resolved !== undefined)
+      addEndpoint(context, {
+        node,
+        kind: "network",
+        value: resolved,
+        mechanism: `call:${name}`,
+      });
+    else addPartialNetworkEndpoint(context, node, name, target);
+  }
+  context.accumulator.pendingNetworkCalls.length = 0;
+  context.accumulator.endpoints.sort(
+    (left, right) => left.offset - right.offset,
+  );
+};
+
+const addPartialNetworkEndpoint = (
+  context: FindingContext,
+  node: JavaScriptCallLike,
+  name: string,
+  target: t.Node,
+): void =>
+  addEndpoint(context, {
+    node,
+    kind: "network",
+    // Partial values preserve the original expression text, byte for byte.
+    value: sourceSlice(context.source, target),
+    mechanism: `call:${name}`,
+    resolution: "partial",
+  });
+
+const resolvedRequestTarget = (
+  node: t.Node,
+  site: t.Node,
+  bindings: JavaScriptSemanticAnalysisState,
+  index: JavaScriptBindingIndex,
+): string | undefined => {
+  const reference = requestReferenceNode(node);
+  // A toString result is selected before the enclosing request invocation.
+  const expression = unwrapJavaScriptExpression(node).node;
+  const capture = reference === expression ? site : expression;
+  if (referenceMayHaveChangedAtSite(reference, capture, index))
+    return undefined;
+  if (!requestInitializerPrecedesSite(reference, capture, bindings, index))
+    return undefined;
+  const evaluated = evaluateSemanticExpression(node, bindings);
+  if (evaluated.status === "literal" && typeof evaluated.value === "string")
+    return evaluated.value;
+  return urlObjectRequestTarget(node, capture, bindings, index);
+};
+
+const requestInitializerPrecedesSite = (
+  reference: t.Node,
+  site: t.Node,
+  bindings: JavaScriptSemanticAnalysisState,
+  index: JavaScriptBindingIndex,
+): boolean => {
+  const effects = index.effectsForReferenceAt(site, reference);
+  if (effects.bindingId === null) return false;
+  const binding = bindings.bindingsById.get(effects.bindingId);
+  const initializer = semanticStableInitializer(binding, bindings);
+  return (
+    initializer !== undefined &&
+    semanticTopLevelEffectFollowsCapture(
+      initializer.node,
+      site,
+      bindings.parentsByNode,
+    )
+  );
+};
+
+const referenceMayHaveChangedAtSite = (
+  reference: t.Node,
+  site: t.Node,
+  index: JavaScriptBindingIndex,
+): boolean => {
+  const effects = index.effectsForReferenceAt(site, reference);
+  return (
+    !effects.memberEffectsIndexed ||
+    effects.resolution !== "resolved" ||
+    effects.effects.some(({ timing }) => timing !== "after")
+  );
+};
+
+const requestReferenceNode = (node: t.Node): t.Node => {
+  const target = unwrapJavaScriptExpression(node).node;
+  if (
+    (t.isCallExpression(target) || t.isOptionalCallExpression(target)) &&
+    target.arguments.length === 0
+  ) {
+    const callee = unwrapJavaScriptExpression(target.callee).node;
+    if (
+      (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) &&
+      semanticStaticPropertyName(callee.property, callee.computed) ===
+        "toString"
+    )
+      return unwrapJavaScriptExpression(callee.object).node;
+  }
+  return target;
+};
+
+/** Resolve an immutable URL object or alias only while shared effects say it is unchanged. */
+const urlObjectRequestTarget = (
+  node: t.Node,
+  site: t.Node,
+  bindings: JavaScriptSemanticAnalysisState,
+  index: JavaScriptBindingIndex,
+  seen = new Set<string>(),
+): string | undefined => {
+  let target = unwrapJavaScriptExpression(node).node;
+  if (
+    (t.isCallExpression(target) || t.isOptionalCallExpression(target)) &&
+    target.arguments.length === 0
+  ) {
+    const callee = unwrapJavaScriptExpression(target.callee).node;
+    if (
+      (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) &&
+      semanticStaticPropertyName(callee.property, callee.computed) ===
+        "toString"
+    )
+      target = unwrapJavaScriptExpression(callee.object).node;
+  }
+  if (
+    !t.isIdentifier(target) ||
+    referenceMayHaveChangedAtSite(target, site, index)
+  )
+    return undefined;
+  const effects = index.effectsForReferenceAt(site, target);
+  const binding =
+    effects.bindingId === null
+      ? undefined
+      : bindings.bindingsById.get(effects.bindingId);
+  if (binding === undefined || seen.has(binding.bindingId)) return undefined;
+  seen.add(binding.bindingId);
+  const stable = semanticStableInitializer(binding, bindings);
+  if (
+    stable === undefined ||
+    !semanticTopLevelEffectFollowsCapture(
+      stable.node,
+      site,
+      bindings.parentsByNode,
+    )
+  )
+    return undefined;
+  const initializer = stable.node;
+  if (initializer === undefined) return undefined;
+  const initialized = unwrapJavaScriptExpression(initializer).node;
+  if (t.isIdentifier(initialized))
+    return urlObjectRequestTarget(initialized, site, bindings, index, seen);
+  if (!t.isNewExpression(initialized)) return undefined;
+  if (!isUnmodifiedGlobalReference(initialized.callee, bindings, "URL"))
+    return undefined;
+  if (
+    ["URL", "Object"].some((owner) =>
+      ["toString", "valueOf"].some((method) =>
+        bindings.ambientWrites.covers(["global", owner, "prototype", method]),
+      ),
+    )
+  )
+    return undefined;
+  const input = stringValue(initialized.arguments[0]);
+  if (input === undefined) return undefined;
+  if (initialized.arguments.length === 1) {
+    try {
+      return new URL(input).href;
+    } catch {
+      return undefined;
+    }
+  }
+  if (initialized.arguments.length !== 2) return undefined;
+  const base = stringValue(initialized.arguments[1]);
+  if (base === undefined) return undefined;
+  try {
+    return new URL(input, base).href;
+  } catch {
+    return undefined;
+  }
+};
+
 /** Record tRPC procedures, tRPC link URLs and named GraphQL operations. */
 export const inspectRpcNode = (node: t.Node, context: FindingContext): void => {
   const rpc = context.rpcScan.inspect(node);
@@ -567,15 +824,21 @@ export const addReference = (
 const addEndpoint = (context: FindingContext, input: EndpointInput): void =>
   addLocatedFinding(context, {
     collection: context.accumulator.endpoints,
-    key: `endpoint\0${input.kind}\0${input.interpretation ?? "literal"}\0${input.value}`,
+    key: `endpoint\0${input.kind}\0${input.interpretation ?? "literal"}\0${input.resolution ?? "exact"}\0${input.value}`,
     node: input.node,
     value: {
       kind: input.kind,
-      value: sanitizeEndpointCandidate(input.value),
+      value:
+        input.resolution === "partial"
+          ? input.value
+          : sanitizeEndpointCandidate(input.value),
       mechanism: input.mechanism,
       ...(input.interpretation === undefined
         ? {}
         : { interpretation: input.interpretation }),
+      ...(input.resolution === undefined
+        ? {}
+        : { resolution: input.resolution }),
       module_key: null,
       location: range(input.node),
     },

src/domain/javascript/javascriptStaticAnalysisState.ts

@@ -42,6 +42,14 @@ export interface JavaScriptAnalysisAccumulator {
   electronCallPathLimitReached?: boolean;
   readonly references: LocatedJavaScriptFinding<JavaScriptStaticReference>[];
   readonly endpoints: LocatedJavaScriptFinding<JavaScriptStaticEndpoint>[];
+  /** Dynamic targets held until the shared mutation index is requested on demand. */
+  readonly pendingNetworkCalls: {
+    readonly node:
+      | t.CallExpression
+      | t.OptionalCallExpression
+      | t.NewExpression;
+    readonly name: string;
+  }[];
   readonly storage: LocatedJavaScriptFinding<JavaScriptStaticStorage>[];
   readonly roles: LocatedJavaScriptFinding<JavaScriptRolePath>[];
   readonly sourceMaps: JavaScriptStaticAnalysis["source_map_urls"][number][];
@@ -87,6 +95,7 @@ export interface JavaScriptEndpointInput {
   readonly value: string;
   readonly mechanism: string;
   readonly interpretation?: "candidate";
+  readonly resolution?: "partial";
 }
 
 /** Values required to append one module-attributable static finding. */
@@ -105,6 +114,7 @@ export const createJavaScriptAnalysisAccumulator = (
 ): JavaScriptAnalysisAccumulator => ({
   references: [],
   endpoints: [],
+  pendingNetworkCalls: [],
   storage: [],
   roles: [],
   sourceMaps: [],

src/domain/javascript/javascriptStaticAnalysisTypes.ts

@@ -47,6 +47,8 @@ export interface JavaScriptStaticEndpoint {
   readonly mechanism: string;
   /** An overloaded callee name suggests this value; its network receiver is unproven. */
   readonly interpretation?: "candidate";
+  /** The request target is unresolved source text rather than an exact value. */
+  readonly resolution?: "partial";
   readonly module_key: string | null;
   /** Recovered factory runtime; absent in older or unbundled findings. */
   readonly module_runtime?: string;