From 3708538e85775f1c5566ff54b9c9affe125a8875 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:34:26 +0800 Subject: [PATCH] feat(javascript): compare static endpoints with HAR captures (#9090) * feat(javascript): compare static endpoints with HAR captures * test(javascript): preserve original HAR fixture bytes during formatting * test(javascript): load the byte-preserved HAR fixture --- docs/mcp-contracts.md | 1 + docs/web-network-captures.md | 103 +++++ ...cObservedEndpointComparisonService.test.ts | 64 +++ ...StaticObservedEndpointComparisonService.ts | 244 +++++++++++ src/cli/applicationCommands.ts | 2 + src/cli/staticObservedEndpointsCommand.ts | 129 ++++++ src/cliCommandNames.ts | 1 + src/contracts/applicationToolContracts.ts | 26 ++ .../applicationWorkflowInputContracts.ts | 27 ++ .../staticObservedEndpointExample.ts | 32 ++ src/contracts/toolEffects.ts | 1 + .../WebNetworkApiSurfaceAccumulator.ts | 76 +++- .../WebNetworkProjectionAccumulator.ts | 20 +- src/domain/analysisView/analysisView.ts | 9 + .../analysisView/analysisViewRequest.ts | 2 + src/domain/analysisView/analysisViewResult.ts | 22 + .../analysisView/endpointComparisonView.ts | 236 ++++++++++ .../staticObservedEndpointSchemas.ts | 103 +++++ .../staticObservedEndpoints.test.ts | 296 +++++++++++++ .../javascript/staticObservedEndpoints.ts | 355 +++++++++++++++ src/server/registerApplicationTools.ts | 2 + .../compareEndpoints.ts | 60 +++ .../staticObservedEndpoints.test.ts | 409 ++++++++++++++++++ .../javascript-endpoint-capture/README.md | 21 + .../javascript-endpoint-capture/app.js.txt | 6 + .../index.html.txt | 1 + .../recording.har.txt | 1 + 27 files changed, 2231 insertions(+), 18 deletions(-) create mode 100644 src/application/javascript/StaticObservedEndpointComparisonService.test.ts create mode 100644 src/application/javascript/StaticObservedEndpointComparisonService.ts create mode 100644 src/cli/staticObservedEndpointsCommand.ts create mode 100644 src/contracts/javascript/staticObservedEndpointExample.ts create mode 100644 src/domain/analysisView/endpointComparisonView.ts create mode 100644 src/domain/javascript/staticObservedEndpointSchemas.ts create mode 100644 src/domain/javascript/staticObservedEndpoints.test.ts create mode 100644 src/domain/javascript/staticObservedEndpoints.ts create mode 100644 src/server/registerApplicationTools/compareEndpoints.ts create mode 100644 tests/acceptance/applications/staticObservedEndpoints.test.ts create mode 100644 tests/fixtures/javascript-endpoint-capture/README.md create mode 100644 tests/fixtures/javascript-endpoint-capture/app.js.txt create mode 100644 tests/fixtures/javascript-endpoint-capture/index.html.txt create mode 100644 tests/fixtures/javascript-endpoint-capture/recording.har.txt diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 58e4b6b651..07e1afd8f7 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -817,6 +817,7 @@ arrays and nested sources declared by the tool's input schema: | `match_native_functions` | `left`, `right`, `left_observations[]`, `right_observations[]` | | `trace_application_feature`, `trace_javascript_semantics`, `compare_source_to_bundle` | `application`; feature tracing also accepts `native_observations[]` | | `compare_application_versions`, `compare_javascript_export_shapes` | `left`, `right`; version comparison also accepts both native-observation arrays | +| `compare_static_and_observed_endpoints` | `application`, `capture`; summary retains the complete comparison for `endpoint-comparison` pages | | `reconcile_javascript_runtime` | `static_layers[].analysis`, `runtime_observations[]` | | `compare_managed_members`, `verify_managed_native_boundaries`, `import_managed_reconstruction` | Comparison `left`/`right`; verification `managed_boundaries`/`native_observations[]`; reconstruction `static_members` | | `project_managed_application_graph` | `managed_artifact`, `managed_members`, `managed_native_boundaries` | diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index ada3ab10fd..f45f0bc71a 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -475,6 +475,109 @@ parent envelope, capture and examined records are deduplicated onto the view. Portable captures must retain unique ordinals within their declared total; caller-selected subsets can preserve any requested ordinal order. +## Compare static JavaScript endpoints with HAR + +`compare_static_and_observed_endpoints` answers which supplied static network +references correspond to captured request groups. Supply `application` from +`analyze_javascript_application` and `capture` from `inspect_web_network_capture` +with HAR format. Both accept inline Evidence or exact retained references. +For normal MCP investigations, request producer summaries, then pass their +`parent_evidence_id` values; large inline graphs can exceed transport admission. +Existing Evidence bundles can instead be loaded with `import_evidence_bundle`. + +```json +{ + "application": { + "kind": "retained-evidence", + "evidence_id": "ev_" + }, + "capture": { + "kind": "retained-evidence", + "evidence_id": "ev_" + }, + "origin": "https://api.example.test" +} +``` + +MCP defaults to a summary with source identities, counts, source coverage and +the complete comparison's `parent_evidence_id`. Select `detail: "complete"` for +all facts. An optional `view` can return a selected `endpoint-comparison` page +in this initial call and takes precedence over `detail`; it uses the same +projector as later retained queries. This answers a selected question without +an extra call or requiring the client's generic view-output schema preparation. +To read another page, call `inspect_analysis_view` on the retained parent: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_" + }, + "view": { + "kind": "endpoint-comparison", + "collection": "static-endpoints", + "statuses": ["unknown"], + "fields": ["path", "status", "reason"], + "offset": 0, + "limit": 32 + } +} +``` + +Collections are `static-endpoints`, `observed-groups` and `matches`. Exact +`endpoint_ids` or `group_ids` select batches where applicable. Fields are +collection-specific; row identities always remain. Empty filters select none. +Pages classify the complete source pair before filtering, and page exhaustion +never upgrades source coverage. Observed-group pages omit potentially large +`record_ordinals` by default; explicitly select that field when needed. + +Each static network **observation** is matched, not observed in this capture, +or unknown. Each unmatched observed group is reported separately. Match rows +link stable endpoint/group identities, identify `exact` or inferred +`template-compatible` basis, and name one `supporting_record_ordinal`. An exact +witness identifies a real whole path within the group; other group members can +have different literal paths. Segment-value combinations are never evidence +that a whole path occurred. Group templates use the existing API-surface owner. + +`origin` is a caller assertion for root-relative targets, never a captured base +URL. Without it those targets remain unknown. Candidate values, unconfirmed +static observations, unresolved +expressions, non-URL GraphQL/tRPC operation names, non-HTTP targets and spellings +requiring URL normalization remain unknown. Known method spellings and the +EventSource/sendBeacon API definitions constrain matching; unavailable methods +stay null and multiple possible methods remain ambiguous. Fetch options and +XHR method arguments are not recovered by this comparison. +If multiple observed methods prevent pairing, those groups also stay unknown +unless another static observation establishes a match; they do not count as +observed-without-static-match. + +Not-observed requires an exercised origin, every parent record retained, and +no excluded request lines. It never means hidden, unused or inaccessible. +Source-specific limitations remain accessible on the linked source Evidence; +the comparison reports their counts without copying unrelated source text. +Query strings, fragments, headers and bodies are not projected. Sensitive-parent +refusals are the same as API-surface. Sources, grouping, whole-path witnesses +and derived rows share its 96 MiB representation allowance. No URL is contacted, +and this workflow does not reopen targets or rerun static analysis. + +CLI uses the same comparison and standard Evidence publication. Put +`application`, `capture` and optional `origin` in the input JSON; delivery uses +flags. Complete detail is the CLI default. A summary requires a bundle so all +facts and their source closure survive process exit: + +```bash +rea compare-static-and-observed-endpoints /evidence/compare-input.json \ + --detail summary --evidence_output /evidence/comparison.json --json +``` + +Add `--view '{"kind":"endpoint-comparison","collection":"static-endpoints","statuses":["not-observed-in-capture"],"fields":["path","status","reason"]}'` +to return that selection immediately. CLI views require summary detail and a +bundle, preserving complete Evidence after exit. + +CLI input can use retained IDs with `--evidence-bundle`. Later +`inspect-analysis-view` calls can use an `evidence-file` source pointing at the +saved bundle and the summary's complete parent ID. + ## Environment manifest Pass retained HAR Evidence to `inspect_analysis_view` with diff --git a/src/application/javascript/StaticObservedEndpointComparisonService.test.ts b/src/application/javascript/StaticObservedEndpointComparisonService.test.ts new file mode 100644 index 0000000000..3170351bdb --- /dev/null +++ b/src/application/javascript/StaticObservedEndpointComparisonService.test.ts @@ -0,0 +1,64 @@ +import { expect, it } from "vitest"; +import { compareStaticAndObservedEndpointsEvidence } from "./StaticObservedEndpointComparisonService.js"; +import { STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE } from "../../contracts/javascript/staticObservedEndpointExample.js"; +import { createEvidence } from "../../domain/evidence.js"; + +it("supports cancellation at a cooperative boundary before publishing a result", async () => { + const controller = new AbortController(); + const pending = compareStaticAndObservedEndpointsEvidence( + STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE, + undefined, + { signal: controller.signal }, + ); + controller.abort("stop comparison"); + expect(await pending).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); +}); + +it("admits the combined representation before source authentication can clone it", async () => { + const result = await compareStaticAndObservedEndpointsEvidence({ + ...STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE, + application: { + ...STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE.application, + raw_result: "x".repeat(8 * 1024 * 1024), + }, + }); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + operation: "compare_static_and_observed_endpoints", + }, + }); +}); + +it("locates malformed retained capture facts at the named consumer input", async () => { + const capture = createEvidence( + undefined, + STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE.capture.provider, + { + operation: "inspect_web_network_capture", + parameters: { sensitive_value_count: 0 }, + result: {}, + confidence: "observed", + }, + ); + const result = await compareStaticAndObservedEndpointsEvidence({ + ...STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE, + capture, + }); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisInputError", + operation: "compare_static_and_observed_endpoints", + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ["capture", "normalized_result", "format"], + }), + ]), + }, + }); +}); diff --git a/src/application/javascript/StaticObservedEndpointComparisonService.ts b/src/application/javascript/StaticObservedEndpointComparisonService.ts new file mode 100644 index 0000000000..a91d85d96b --- /dev/null +++ b/src/application/javascript/StaticObservedEndpointComparisonService.ts @@ -0,0 +1,244 @@ +import type { WebNetworkCapture } from "../../domain/webNetworkCapture.js"; +import { setImmediate } from "node:timers/promises"; +import { compareStaticObservedEndpointsRequestSchema } from "../../contracts/javascript/applicationWorkflowInputContracts.js"; +import { createEvidence, type Evidence } from "../../domain/evidence.js"; +import { readImmutableEvidence } from "../../domain/evidenceValidation.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { EvidenceWorkflowExecution } from "../investigation/EvidenceWorkflowExecution.js"; +import { + AnalysisCancelledError, + AnalysisInputError, +} from "../../domain/analysisErrorCore.js"; +import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER } from "../../domain/investigationProviders.js"; +import { WebNetworkApiSurfaceAccumulator } from "../../domain/analysisView/WebNetworkApiSurfaceAccumulator.js"; +import { admitHarCaptureParent } from "../../domain/analysisView/WebNetworkProjectionAccumulator.js"; +import { compareStaticObservedEndpoints } from "../../domain/javascript/staticObservedEndpoints.js"; +import type { StaticObservedEndpointComparison } from "../../domain/javascript/staticObservedEndpointSchemas.js"; +import { + resolveEvidenceRequest, + type EvidenceLookup, +} from "../EvidenceInputResolver.js"; +import { + applicationGraphEvidenceInputError, + parseApplicationGraphEvidence, + type ApplicationGraphEvidenceSource, +} from "./JavaScriptApplicationEvidenceGraph.js"; + +const OPERATION = "compare_static_and_observed_endpoints"; +interface EndpointComparisonExecution extends EvidenceWorkflowExecution { + readonly result: StaticObservedEndpointComparison; +} + +interface ComparisonSources { + readonly application: ApplicationGraphEvidenceSource; + readonly capture: WebNetworkCapture; + readonly captureEvidence: Evidence; + readonly origin: string | undefined; +} +type ComparisonOptions = { readonly signal?: AbortSignal }; +type ComparisonFacts = Pick< + StaticObservedEndpointComparison, + "static_endpoints" | "observed_groups" | "matches" | "summary" +>; + +const readSources = ( + rawInput: unknown, + lookup: EvidenceLookup | undefined, + accumulator: WebNetworkApiSurfaceAccumulator, + options: ComparisonOptions, +): Result => { + const request = resolveEvidenceRequest( + OPERATION, + rawInput, + compareStaticObservedEndpointsRequestSchema, + lookup, + { + ...options, + readEvidence: (value) => { + const admitted = accumulator.admitInlineEvidence(value); + return admitted.ok ? readImmutableEvidence(value) : admitted; + }, + }, + ); + if (!request.ok) return request; + const input = request.value.input; + const source = parseApplicationGraphEvidence(input.application, [ + "application", + ]); + if (!source.ok) + return err(applicationGraphEvidenceInputError(OPERATION, source.error)); + if (source.value.kind !== "static-application") + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["application"], + reason: "invalid_value", + message: + "Use analyze_javascript_application Evidence; reconciled, managed and web-bundle graphs are outside this comparison's static scope.", + }, + ]), + ); + const captureEvidence = input.capture; + const admitted = admitHarCaptureParent( + { + evidenceId: captureEvidence.evidence_id, + operation: captureEvidence.operation, + predicateType: captureEvidence.predicate_type, + normalizedResult: captureEvidence.normalized_result, + subject: captureEvidence.subject, + parameters: captureEvidence.parameters, + limitations: captureEvidence.limitations, + }, + "api-surface", + accumulator, + { operation: OPERATION, sourceField: "capture" }, + ); + if (!admitted.ok) return admitted; + return ok({ + application: source.value, + capture: admitted.value, + captureEvidence, + origin: input.origin, + }); +}; + +const sourceCoverage = ( + { application, capture, captureEvidence }: ComparisonSources, + excluded: number, +): StaticObservedEndpointComparison["source_coverage"] => ({ + application: { + status: application.graph.coverage.status, + truncated: application.graph.coverage.truncated, + omitted_count: application.graph.coverage.omitted_count, + limit_count: application.graph.coverage.limits.length, + limitation_count: new Set([ + ...application.evidence.limitations, + ...application.graph.limitations, + ]).size, + }, + capture_limitation_count: new Set([ + ...captureEvidence.limitations, + ...capture.limitations, + ]).size, + observed_records_examined: capture.records.length, + observed_records_total: capture.total_records, + observed_records_excluded: excluded, +}); + +const createComparison = ( + sources: ComparisonSources, + comparison: ComparisonFacts, + accumulator: WebNetworkApiSurfaceAccumulator, +): Result => { + const { application, capture, captureEvidence, origin } = sources; + const evidenceLinks = [ + application.evidence.evidence_id, + captureEvidence.evidence_id, + ].sort(); + const limitations = [ + "Source-specific limitation details remain on the two source Evidence records; source_coverage reports their counts without copying unrelated capture or graph text.", + "Not-observed-in-capture refers only to this capture; it does not establish unused, hidden, inaccessible or absent server behavior.", + "Template-compatible pairs are inferred format matches, not handler identity or runtime execution proof. Unknown static methods are never filled from observed methods.", + "Observed-without-static-match means no confirmed correspondence in the supplied static observations; incomplete static discovery can miss code references.", + ...(origin === undefined + ? [] + : [ + "The origin for root-relative static targets is a caller assertion, not an observed application base URL.", + ]), + ].sort(); + const admittedLimitations = accumulator.addLimitations(limitations); + if (!admittedLimitations.ok) return admittedLimitations; + const parameters = { + application_evidence_id: application.evidence.evidence_id, + capture_evidence_id: captureEvidence.evidence_id, + origin: origin ?? null, + }; + const result: StaticObservedEndpointComparison = { + comparison_id: `endpoint_comparison_${digestCanonicalValue(parameters)}`, + application: { + evidence_id: application.evidence.evidence_id, + graph_id: application.graph.graph_id, + }, + capture: { + evidence_id: captureEvidence.evidence_id, + artifact: { + path: capture.artifact.path, + sha256: capture.artifact.sha256, + }, + }, + declared_origin: origin ?? null, + ...comparison, + source_coverage: sourceCoverage( + sources, + accumulator.comparisonCoverage().excluded_records, + ), + evidence_links: evidenceLinks, + limitations, + }; + const evidence: Evidence = createEvidence( + undefined, + JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER, + { + predicateType: "rea.static-observed-endpoint-comparison", + operation: OPERATION, + parameters, + result, + rawResult: null, + confidence: "inferred", + authority: "analyst-inference", + environment: null, + limitations, + evidenceLinks, + }, + ); + return ok({ + evidence, + result, + sources: [application.evidence, captureEvidence], + }); +}; + +/** Authenticate sources once, group cooperatively, then publish the completed comparison. */ +export const compareStaticAndObservedEndpointsEvidence = async ( + rawInput: unknown, + lookup?: EvidenceLookup, + options: ComparisonOptions = {}, +): Promise> => { + const accumulator = new WebNetworkApiSurfaceAccumulator(undefined, false, { + operation: OPERATION, + }); + const admitted = readSources(rawInput, lookup, accumulator, options); + if (!admitted.ok) return admitted; + const sources = admitted.value; + const { capture, captureEvidence } = sources; + for (const [index, record] of capture.records.entries()) { + if (index % 256 === 0) { + await setImmediate(); + if (options.signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { + cause: options.signal.reason, + }), + ); + } + const added = accumulator.add(record); + if (!added.ok) return added; + } + const compared = compareStaticObservedEndpoints({ + applicationEvidenceId: sources.application.evidence.evidence_id, + captureEvidenceId: captureEvidence.evidence_id, + graph: sources.application.graph, + accumulator, + ...(sources.origin === undefined ? {} : { origin: sources.origin }), + captureComplete: capture.records.length === capture.total_records, + }); + if (!compared.ok) return compared; + await setImmediate(); + if (options.signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), + ); + return createComparison(sources, compared.value, accumulator); +}; diff --git a/src/cli/applicationCommands.ts b/src/cli/applicationCommands.ts index 6a88982ae4..fe14a4f0f3 100644 --- a/src/cli/applicationCommands.ts +++ b/src/cli/applicationCommands.ts @@ -8,6 +8,7 @@ import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; import type { Logger } from "pino"; import { registerSourceToBundleCommand } from "./sourceToBundleCommand.js"; +import { registerStaticObservedEndpointsCommand } from "./staticObservedEndpointsCommand.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; import type { Evidence } from "../domain/evidence.js"; @@ -72,6 +73,7 @@ export const registerApplicationCommands = ( }, }); registerSourceToBundleCommand(cli, logger); + registerStaticObservedEndpointsCommand(cli, logger); registerEvidenceJsonCommand({ cli, logger, diff --git a/src/cli/staticObservedEndpointsCommand.ts b/src/cli/staticObservedEndpointsCommand.ts new file mode 100644 index 0000000000..849d0bb501 --- /dev/null +++ b/src/cli/staticObservedEndpointsCommand.ts @@ -0,0 +1,129 @@ +import { z } from "incur"; +import type { Logger } from "pino"; +import type { Evidence } from "../domain/evidence.js"; +import type { CliInstance } from "./types.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { parseCliJsonInput } from "../cliJsonInput.js"; +import { logCliCommand } from "../cliLogging.js"; +import { compareStaticObservedEndpointsFactsSchema } from "../contracts/javascript/applicationWorkflowInputContracts.js"; +import { getEvidenceInputContract } from "../domain/evidenceInputContract.js"; +import { endpointComparisonViewRequestSchema } from "../domain/analysisView/endpointComparisonView.js"; +import { parseAnalysisInput } from "../application/AnalysisInput.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER } from "../domain/investigationProviders.js"; +import { withCommandCancellation } from "./commandCancellation.js"; +import { + evidenceBundleOption, + runEvidenceInputCommand, +} from "./evidenceInputCommand.js"; +import { + analysisEvidenceDeliveryOptions, + parseCliAnalysisEvidenceDelivery, + publishCliAnalysisEvidence, +} from "./analysisEvidenceDelivery.js"; + +const operation = "compare_static_and_observed_endpoints"; +const deliverySchema = z + .object({ + detail: analysisEvidenceDeliveryOptions.detail, + view: endpointComparisonViewRequestSchema.optional(), + }) + .superRefine((value, context) => { + if (value.view !== undefined && value.detail !== "summary") + context.addIssue({ + code: "custom", + path: ["view"], + message: + "A selected page requires --detail summary and --evidence_output.", + }); + }); +const commandOptions = z.object({ + evidenceBundle: evidenceBundleOption, + ...analysisEvidenceDeliveryOptions, + view: z + .string() + .optional() + .describe( + "JSON endpoint-comparison page; requires --detail summary and --evidence_output", + ), +}); + +const runComparison = async ( + inputJson: string, + options: z.output, + signal: AbortSignal, +) => { + const selected = parseCliAnalysisEvidenceDelivery( + operation, + options, + deliverySchema, + ); + if (!selected.ok) return projectAnalysisError(selected.error); + const delivery = { + providerId: JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER.id, + operation, + signal, + detail: selected.value.detail, + evidenceOutput: selected.value.evidenceOutput, + ...(selected.value.view === undefined ? {} : { view: selected.value.view }), + }; + const raw = await parseCliJsonInput(inputJson, operation, signal); + if (!raw.ok) return raw.error; + // The frame validates transport fields without cloning either Evidence before budget admission. + const input = parseAnalysisInput( + operation, + raw.value, + getEvidenceInputContract(compareStaticObservedEndpointsFactsSchema) + .frameSchema, + signal, + ); + if (!input.ok) return projectAnalysisError(input.error); + let sources: readonly Evidence[] = []; + const result = await runEvidenceInputCommand( + operation, + options.evidenceBundle, + signal, + async (lookup) => { + const { compareStaticAndObservedEndpointsEvidence } = + await import("../application/javascript/StaticObservedEndpointComparisonService.js"); + const compared = await compareStaticAndObservedEndpointsEvidence( + { ...input.value, detail: options.detail }, + lookup, + { signal }, + ); + if (!compared.ok) return compared; + sources = compared.value.sources; + return { ok: true, value: compared.value.evidence }; + }, + ); + return publishCliAnalysisEvidence(result, { + ...delivery, + sourceEvidence: sources, + }); +}; + +/** Reuse standard CLI publication so summaries keep their complete source closure on disk. */ +export const registerStaticObservedEndpointsCommand = ( + cli: CliInstance, + logger: Logger, +): void => { + const name = CLI_COMMANDS.compareStaticAndObservedEndpoints; + cli.command(name, { + description: + "Compare static JavaScript endpoints and HAR Evidence JSON offline; --detail summary with --evidence_output saves full facts for selected inspect-analysis-view pages", + args: z.object({ + inputJson: z + .string() + .describe( + "Inline JSON or file containing application, capture and optional origin", + ), + }), + options: commandOptions, + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand(logger, name, () => + runComparison(args.inputJson, options, signal), + ), + ), + }); +}; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 4c7e12ec6f..60d2be5ad6 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -179,6 +179,7 @@ export const CLI_COMMANDS = Object.freeze({ traceApplicationFeature: "trace-application-feature", traceJavaScriptSemantics: "trace-javascript-semantics", compareApplicationVersions: "compare-application-versions", + compareStaticAndObservedEndpoints: "compare-static-and-observed-endpoints", compareSourceToBundle: "compare-source-to-bundle", compareJavaScriptExportShapes: "compare-javascript-export-shapes", buildReconstructionObligationLedger: "build-reconstruction-obligation-ledger", diff --git a/src/contracts/applicationToolContracts.ts b/src/contracts/applicationToolContracts.ts index fe47b7f5d7..09b0e18959 100644 --- a/src/contracts/applicationToolContracts.ts +++ b/src/contracts/applicationToolContracts.ts @@ -5,7 +5,11 @@ import { featureTraceViewResultSchema, sourceToBundleSummaryViewResultSchema, sourceToBundlePageViewResultSchema, + endpointComparisonSummaryViewResultSchema, + endpointComparisonPageViewResultSchema, } from "../domain/analysisView/analysisView.js"; +import { staticObservedEndpointComparisonSchema } from "../domain/javascript/staticObservedEndpointSchemas.js"; +import { STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE } from "./javascript/staticObservedEndpointExample.js"; import { applicationVersionComparisonResultSchema } from "../domain/javascript/javascriptApplicationVersionComparisonSchemas.js"; import { applicationFeatureTraceResultSchema } from "../domain/javascript/javascriptFeatureTraceSchemas.js"; import { javaScriptSemanticTraceResultSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js"; @@ -13,6 +17,7 @@ import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascrip import { sourceToBundleComparisonResultSchema } from "../domain/javascript/sourceToBundleComparisonSchemas.js"; import { compareApplicationVersionsRequestSchema, + compareStaticObservedEndpointsRequestSchema, compareJavaScriptExportShapesRequestSchema, compareSourceToBundleRequestSchema, traceApplicationFeatureToolRequestSchema, @@ -207,6 +212,27 @@ export const APPLICATION_TOOL_CONTRACTS = [ }, ], }, + { + name: "compare_static_and_observed_endpoints", + ...toolContractMetadata("compare_static_and_observed_endpoints"), + description: + "Compare authenticated analyze_javascript_application and HAR Evidence, inline or by exact retained reference. No files, targets or URLs are acquired. Default summary retains the full comparison; optional view returns an endpoint-comparison page in this call (takes precedence over detail). inspect_analysis_view kind endpoint-comparison pages static-endpoints, observed-groups or matches with filters and fields. detail complete returns all facts. Pairings identify exact or inferred template-compatible basis; candidate/partial targets and ambiguous methods remain unknown. Root-relative targets need a caller-declared origin. Not-observed requires an exercised origin, every parent record retained and no excluded records; it never means hidden, unused or inaccessible. Copies no query strings, fragments, headers or bodies. Sensitive-declaration parents are refused; the shared 96 MiB representation budget includes source admission and derived state.", + kind: "application", + inputSchema: compareStaticObservedEndpointsRequestSchema, + outputSchema: evidenceResultOf( + z.union([ + staticObservedEndpointComparisonSchema, + endpointComparisonSummaryViewResultSchema, + endpointComparisonPageViewResultSchema, + ]), + ), + examples: [ + { + title: "Compare retained static endpoints with an offline HAR", + input: STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE, + }, + ], + }, { name: "compare_source_to_bundle", ...toolContractMetadata("compare_source_to_bundle"), diff --git a/src/contracts/javascript/applicationWorkflowInputContracts.ts b/src/contracts/javascript/applicationWorkflowInputContracts.ts index 08a4294584..c5448d1bc9 100644 --- a/src/contracts/javascript/applicationWorkflowInputContracts.ts +++ b/src/contracts/javascript/applicationWorkflowInputContracts.ts @@ -5,6 +5,8 @@ import { import { sourceToBundleViewRequestSchema } from "../../domain/analysisView/sourceToBundleView.js"; import { featureTraceViewRequestSchema } from "../../domain/analysisView/featureTraceView.js"; import { z } from "zod"; +import { recordedBrowserOriginSchema } from "../../domain/browserObservation.js"; +import { endpointComparisonViewRequestSchema } from "../../domain/analysisView/endpointComparisonView.js"; import { evidenceInputSchema } from "../../domain/evidenceReference.js"; import { compareJavaScriptExportShapesInputSchema } from "../../domain/javascript/javascriptExportShapeComparisonSchemas.js"; @@ -51,6 +53,31 @@ const compareApplicationVersionsFacts = { right_native_observations: z.array(evidenceInputSchema).default([]), } as const; +/** Compare existing static Application Graph and HAR Evidence without acquiring targets. */ +export const compareStaticObservedEndpointsFactsSchema = z.strictObject({ + application: evidenceInputSchema, + capture: evidenceInputSchema, + origin: recordedBrowserOriginSchema + .optional() + .describe( + "Caller-declared canonical HTTP(S) origin for root-relative static targets; omit to keep their origin unknown. This never contacts the origin.", + ), +}); +export const compareStaticObservedEndpointsRequestSchema = + compareStaticObservedEndpointsFactsSchema.extend({ + view: endpointComparisonViewRequestSchema + .optional() + .describe( + "Return this selected page in the initial call, retaining the complete comparison. An explicit view takes precedence over detail.", + ), + detail: z + .enum(["summary", "complete"]) + .default("summary") + .describe( + "Summary retains the complete comparison and returns counts and source coverage. Use inspect_analysis_view kind endpoint-comparison for selected rows; complete returns all facts inline.", + ), + }); + /** Application comparison request accepting inline or same-session application Evidence. */ export const compareApplicationVersionsRequestSchema = z.strictObject({ ...compareApplicationVersionsFacts, diff --git a/src/contracts/javascript/staticObservedEndpointExample.ts b/src/contracts/javascript/staticObservedEndpointExample.ts new file mode 100644 index 0000000000..68107fb587 --- /dev/null +++ b/src/contracts/javascript/staticObservedEndpointExample.ts @@ -0,0 +1,32 @@ +import { createEvidence } from "../../domain/evidence.js"; +import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "./javascriptRuntimeReconciliationExample.js"; + +/** Portable source records demonstrate admission, without claiming a real capture. */ +export const STATIC_OBSERVED_ENDPOINT_COMPARISON_EXAMPLE = { + application: JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE, + capture: createEvidence( + undefined, + { id: "rea-example-har", name: "Example HAR", version: "1" }, + { + operation: "inspect_web_network_capture", + parameters: { sensitive_value_count: 0 }, + result: { + artifact: { path: "/capture.har", sha256: null, bytes: 0 }, + format: "har", + decoder: { id: "rea-example-har", name: "Example HAR", version: "1" }, + container: { + reported: {}, + numeric_literals: [], + redactions: [], + records_pointer: "/log/entries", + }, + total_records: 0, + records: [], + runtime_attribution: "unknown", + limitations: ["Empty illustrative capture, not a runtime measurement."], + }, + confidence: "observed", + }, + ), + origin: "https://example.test", +}; diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 4076200999..4e98ab96a7 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -769,6 +769,7 @@ const TOOL_EFFECT_AUDIT: Readonly> = { trace_application_feature: evidence, trace_javascript_semantics: evidence, compare_application_versions: evidence, + compare_static_and_observed_endpoints: evidence, search_guidance: effects(), read_guidance_section: effects(), guided_workflows: effects(), diff --git a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts index fbbf65426d..41aeb8824d 100644 --- a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts +++ b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts @@ -22,6 +22,7 @@ import { textBytes, type RouteRule, type RouteTemplate, + type RecordedRequestLine, } from "./webNetworkRouteTemplates.js"; type ApiSurfaceFields = { @@ -35,6 +36,8 @@ interface GroupState { readonly method: string; readonly origin: string; readonly path_template: string; + readonly rules: string; + readonly paths?: Map; readonly segments: readonly { readonly segment_index: number; readonly rule: RouteRule; @@ -47,6 +50,20 @@ interface GroupState { } type Buckets = Map>>>; +/** Borrowed group facts; exact membership retains whole paths, never segment products. */ +export interface ApiSurfaceComparisonGroup { + readonly method: string; + readonly origin: string; + readonly path_template: string; + readonly rules: string; + readonly templated_segments: readonly { + readonly segment_index: number; + readonly rule: RouteRule; + }[]; + readonly record_ordinals: readonly number[]; + readonly exactPathOrdinal: (path: string) => number | undefined; +} + /** Own group identity, deduplication and admission for the entire projection. */ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumulator { readonly #buckets: Buckets = new Map(); @@ -57,8 +74,11 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul constructor( maximumBytes = WEB_NETWORK_CAPTURE_LIMITS.outputBytes, readonly includeQueryParameterNames = false, + private readonly comparison?: { + readonly operation: string; + }, ) { - super(maximumBytes, "API-surface"); + super(maximumBytes, "API-surface", comparison?.operation); } #exclude( @@ -87,15 +107,7 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul if (route === null) return this.overBudget(); const queryNames = readQueryParameterNames(line.query, this.scratch); if (queryNames === null) return this.overBudget(); - return this.#append( - record.ordinal, - line.method, - line.origin, - route, - line.status, - mediaTypeEssence(line.rawMime), - queryNames, - ); + return this.#append(record.ordinal, line, route, queryNames); } finally { this.releaseTemporary(); } @@ -103,19 +115,20 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul #append( ordinal: number, - method: string, - origin: string, + line: Extract, route: RouteTemplate, - status: number | null, - mime: string | null, queryNames: ReadonlySet | undefined, ): Result { + const { method, origin, path, status } = line; + const mime = mediaTypeEssence(line.rawMime); let group = this.#buckets .get(method) ?.get(origin) ?.get(route.template) ?.get(route.rules); let bytes = ENTRY_BYTES; // One retained record ordinal, including page output. + if (this.comparison !== undefined && !group?.paths?.has(path)) + bytes += ENTRY_BYTES + textBytes(path.length); if (group === undefined) { bytes += GROUP_BYTES + @@ -150,6 +163,10 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul method, origin, path_template: route.template, + rules: route.rules, + ...(this.comparison === undefined + ? {} + : { paths: new Map() }), segments: route.segments.map((segment) => ({ segment_index: segment.segment_index, rule: segment.rule, @@ -173,6 +190,8 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul group.segments[index]!.values.add(segment.value); } group.ordinals.push(ordinal); + if (group.paths !== undefined && !group.paths.has(path)) + group.paths.set(path, ordinal); if (status === null || mime === null) this.#unavailableResponse = true; if (status !== null) group.statuses.add(status); if (mime !== null) group.contentTypes.add(mime); @@ -186,6 +205,35 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul return ok(undefined); } + /** Comparison-only inventory, borrowing admitted ordinals without page serialization. */ + *comparisonGroups(): IterableIterator { + if (this.comparison === undefined) + throw new Error( + "Exact paths must be retained when admitting comparison records", + ); + for (const group of this.#groups) + yield { + method: group.method, + origin: group.origin, + path_template: group.path_template, + rules: group.rules, + templated_segments: group.segments.map(({ segment_index, rule }) => ({ + segment_index, + rule, + })), + record_ordinals: group.ordinals, + exactPathOrdinal: (path) => group.paths!.get(path), + }; + } + + /** Counts do not allocate the existing view's complete page or exclusion list. */ + comparisonCoverage() { + return { + groups: this.#groups.length, + excluded_records: this.#excluded.length, + }; + } + page(offset: number, limit: number) { this.#groups.sort( (a, b) => diff --git a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts index f57364d9c1..9080df2b2b 100644 --- a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts +++ b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts @@ -13,7 +13,7 @@ import { } from "../webNetworkCapture.js"; import { admitAnalysisViewData } from "./analysisViewDataAdmission.js"; import { analysisViewInputError } from "./analysisViewErrors.js"; -import { analysisViewResultInputError } from "./analysisViewParent.js"; +import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import type { AnalysisViewParent } from "./analysisViewParent.js"; import { ENTRY_BYTES, @@ -32,6 +32,7 @@ export abstract class WebNetworkProjectionAccumulator { protected constructor( readonly maximumBytes: number, private readonly budgetSubject: string, + private readonly operation = "inspect_analysis_view", ) {} protected retain(bytes: number): boolean { @@ -72,7 +73,7 @@ export abstract class WebNetworkProjectionAccumulator { protected overBudget(): Result { return err( new AnalysisResourceConstraintError( - "inspect_analysis_view", + this.operation, "memory", `${this.budgetSubject} retained state and temporary work exceed the projection representation budget.`, { output_bytes: this.maximumBytes }, @@ -123,11 +124,14 @@ export const admitHarCaptureParent = ( parent: AnalysisViewParent, viewKind: "api-surface" | "environment-manifest" | "environment-state", accumulator: WebNetworkProjectionAccumulator, + context: { readonly operation?: string; readonly sourceField?: string } = {}, ): Result => { + const operation = context.operation ?? "inspect_analysis_view"; + const sourceField = context.sourceField ?? "source"; const unsupported = (reason: string, remediationAction?: string) => err( new AnalysisUnsupportedTargetError( - "inspect_analysis_view", + operation, parent.evidenceId, reason, remediationAction === undefined ? undefined : { remediationAction }, @@ -142,7 +146,15 @@ export const admitHarCaptureParent = ( const parsed = safeParseOwnedWebNetworkCapture(parent.normalizedResult); if (!parsed.success) return err( - analysisViewResultInputError(parent.normalizedResult, parsed.error), + analysisInputErrorFromIssues( + operation, + parsed.error.issues.map((issue) => ({ + ...issue, + path: [sourceField, "normalized_result", ...issue.path], + })), + { [sourceField]: { normalized_result: parent.normalizedResult } }, + { cause: parsed.error }, + ), ); if (parsed.data.format !== "har") return unsupported( diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 94026bfcae..cf2d23698c 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -19,6 +19,7 @@ import { import { projectJavaScriptRoutesView } from "./javascriptRoutesView.js"; import { projectWebBundleView } from "./webBundleView.js"; import { projectSourceToBundleView } from "./sourceToBundleView.js"; +import { projectEndpointComparisonView } from "./endpointComparisonView.js"; import { projectFeatureTraceView } from "./featureTraceView.js"; import { projectNativeFeatureTraceView } from "./nativeFeatureTraceView.js"; import { projectBatchDecompileView } from "./batchDecompileView.js"; @@ -155,6 +156,8 @@ import { nativeFunctionSummaryViewResultSchema, sourceToBundleSummaryViewResultSchema, sourceToBundlePageViewResultSchema, + endpointComparisonSummaryViewResultSchema, + endpointComparisonPageViewResultSchema, featureTraceSummaryViewResultSchema, batchDecompileSummaryViewResultSchema, featureTraceViewResultSchema, @@ -224,6 +227,8 @@ export { nativeFunctionSummaryViewResultSchema, sourceToBundleSummaryViewResultSchema, sourceToBundlePageViewResultSchema, + endpointComparisonSummaryViewResultSchema, + endpointComparisonPageViewResultSchema, featureTraceSummaryViewResultSchema, batchDecompileSummaryViewResultSchema, featureTraceViewResultSchema, @@ -351,6 +356,10 @@ const retainedViewProjectors = new Map([ retain("inspect_gpu_code_objects", projectGpuCodeObjectsView), retain("match_native_functions", projectNativeFunctionMatchingView), retain("compare_source_to_bundle", projectSourceToBundleView), + retain( + "compare_static_and_observed_endpoints", + projectEndpointComparisonView, + ), retain("trace_application_feature", projectFeatureTraceView), retain("batch_decompile", projectBatchDecompileView), retain("inspect_artifact", projectArtifactInventoryView), diff --git a/src/domain/analysisView/analysisViewRequest.ts b/src/domain/analysisView/analysisViewRequest.ts index 58426500bb..716562a1b4 100644 --- a/src/domain/analysisView/analysisViewRequest.ts +++ b/src/domain/analysisView/analysisViewRequest.ts @@ -11,6 +11,7 @@ import { nativeExportsViewRequestSchema } from "./nativeExportsView.js"; import { javaScriptRoutesViewRequestSchema } from "./javascriptRoutesView.js"; import { webBundleViewRequestSchema } from "./webBundleView.js"; import { sourceToBundleViewRequestSchema } from "./sourceToBundleView.js"; +import { endpointComparisonViewRequestSchema } from "./endpointComparisonView.js"; import { featureTraceViewRequestSchema } from "./featureTraceView.js"; import { nativeFeatureTraceViewRequestSchema } from "./nativeFeatureTraceView.js"; import { batchDecompileViewRequestSchema } from "./batchDecompileView.js"; @@ -72,6 +73,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ webBundleViewRequestSchema, nativeFunctionMatchingViewRequestSchema, sourceToBundleViewRequestSchema, + endpointComparisonViewRequestSchema, featureTraceViewRequestSchema, nativeFeatureTraceViewRequestSchema, batchDecompileViewRequestSchema, diff --git a/src/domain/analysisView/analysisViewResult.ts b/src/domain/analysisView/analysisViewResult.ts index edea69c003..32acb45c50 100644 --- a/src/domain/analysisView/analysisViewResult.ts +++ b/src/domain/analysisView/analysisViewResult.ts @@ -73,6 +73,11 @@ import { sourceToBundleViewItemsSchema, sourceToBundleViewContextSchema, } from "./sourceToBundleView.js"; +import { + endpointComparisonViewContextSchema, + endpointComparisonViewItemsSchema, + endpointComparisonViewRequestSchema, +} from "./endpointComparisonView.js"; import { featureTraceSummarySchema, featureTraceViewCollections, @@ -1017,6 +1022,21 @@ export const sourceToBundleSummaryViewResultSchema = z.strictObject({ parent_operation: z.literal("compare_source_to_bundle"), summary: sourceToBundleViewContextSchema, }); +export const endpointComparisonSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + artifact: z.null(), + parent_operation: z.literal("compare_static_and_observed_endpoints"), + summary: endpointComparisonViewContextSchema, +}); +export const endpointComparisonPageViewResultSchema = z.strictObject({ + ...viewResultBase, + ...endpointComparisonViewContextSchema.shape, + kind: z.literal("endpoint-comparison"), + artifact: z.null(), + parent_operation: z.literal("compare_static_and_observed_endpoints"), + view: endpointComparisonViewRequestSchema, + items: endpointComparisonViewItemsSchema, +}); export const sourceToBundlePageViewResultSchema = z.strictObject({ ...viewResultBase, ...sourceToBundleViewContextSchema.shape, @@ -1146,6 +1166,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ javaScriptRoutesViewResultSchema, webBundlePageViewResultSchema, sourceToBundlePageViewResultSchema, + endpointComparisonPageViewResultSchema, featureTraceViewResultSchema, batchDecompileViewResultSchema, z.strictObject({ @@ -1241,6 +1262,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ nativeFunctionMatchingSummaryViewResultSchema, nativeFunctionSummaryViewResultSchema, sourceToBundleSummaryViewResultSchema, + endpointComparisonSummaryViewResultSchema, featureTraceSummaryViewResultSchema, batchDecompileSummaryViewResultSchema, webBundleSummaryViewResultSchema, diff --git a/src/domain/analysisView/endpointComparisonView.ts b/src/domain/analysisView/endpointComparisonView.ts new file mode 100644 index 0000000000..c2266ea365 --- /dev/null +++ b/src/domain/analysisView/endpointComparisonView.ts @@ -0,0 +1,236 @@ +import { z } from "zod"; +import type { StaticObservedEndpointComparison } from "../javascript/staticObservedEndpointSchemas.js"; +import type { JsonValue } from "../jsonValue.js"; +import { + comparedStaticEndpointSchema, + comparedObservedGroupSchema, + endpointMatchSchema, + staticObservedEndpointComparisonSchema, +} from "../javascript/staticObservedEndpointSchemas.js"; +import { + createAnalysisViewResultReader, + type AnalysisViewParent, +} from "./analysisViewParent.js"; +import { + ownedView, + type AdmittedView, + type SummaryAnalysisView, +} from "./analysisViewShape.js"; +import { + boundedViewMessages, + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; +import { selectViewRows } from "./selectViewRows.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { err, ok } from "../result.js"; + +const page = { + kind: z.literal("endpoint-comparison"), + offset: z.int().nonnegative().default(0), + limit: z.int().positive().default(64), +}; +export const endpointComparisonViewRequestSchema = z.discriminatedUnion( + "collection", + [ + z.strictObject({ + ...page, + collection: z.literal("static-endpoints"), + endpoint_ids: z + .array(comparedStaticEndpointSchema.shape.endpoint_id) + .optional(), + statuses: z.array(comparedStaticEndpointSchema.shape.status).optional(), + fields: z.array(comparedStaticEndpointSchema.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("observed-groups"), + group_ids: z.array(comparedObservedGroupSchema.shape.group_id).optional(), + statuses: z.array(comparedObservedGroupSchema.shape.status).optional(), + fields: z.array(comparedObservedGroupSchema.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("matches"), + endpoint_ids: z.array(endpointMatchSchema.shape.endpoint_id).optional(), + group_ids: z.array(endpointMatchSchema.shape.group_id).optional(), + fields: z.array(endpointMatchSchema.keyof()).optional(), + }), + ], +); +export const endpointComparisonViewItemsSchema = z.union([ + z.array( + comparedStaticEndpointSchema + .partial() + .required({ endpoint_id: true, node_id: true, observation_id: true }), + ), + z.array(comparedObservedGroupSchema.partial().required({ group_id: true })), + z.array( + endpointMatchSchema + .partial() + .required({ match_id: true, endpoint_id: true, group_id: true }), + ), +]); +export const endpointComparisonViewContextSchema = + staticObservedEndpointComparisonSchema.omit({ + static_endpoints: true, + observed_groups: true, + matches: true, + limitations: true, + }); +const readReport = createAnalysisViewResultReader( + staticObservedEndpointComparisonSchema, +); +const set = (values: readonly string[] | undefined) => + values === undefined ? undefined : new Set(values); +const fieldsOf = (row: object, fields: readonly string[]) => { + const source = row as Record; + return Object.fromEntries(fields.map((field) => [field, source[field]!])); +}; + +const selectComparisonRows = ( + report: StaticObservedEndpointComparison, + view: z.output, +) => { + const { static_endpoints, observed_groups, matches } = report; + let selected: { + rows: z.output; + total: number; + }; + if (view.collection === "static-endpoints") { + const ids = set(view.endpoint_ids), + statuses = set(view.statuses); + selected = selectViewRows( + static_endpoints, + view, + (row) => + (ids === undefined || ids.has(row.endpoint_id)) && + (statuses === undefined || statuses.has(row.status)), + (row) => ({ + ...fieldsOf( + row, + view.fields ?? [ + "method", + "method_basis", + "origin", + "origin_basis", + "path", + "status", + "reason", + ], + ), + endpoint_id: row.endpoint_id, + node_id: row.node_id, + observation_id: row.observation_id, + }), + ); + } else if (view.collection === "observed-groups") { + const ids = set(view.group_ids), + statuses = set(view.statuses); + selected = selectViewRows( + observed_groups, + view, + (row) => + (ids === undefined || ids.has(row.group_id)) && + (statuses === undefined || statuses.has(row.status)), + (row) => ({ + ...fieldsOf( + row, + view.fields ?? [ + "method", + "origin", + "path_template", + "status", + "reason", + ], + ), + group_id: row.group_id, + }), + ); + } else { + const endpoints = set(view.endpoint_ids), + groups = set(view.group_ids); + selected = selectViewRows( + matches, + view, + (row) => + (endpoints === undefined || endpoints.has(row.endpoint_id)) && + (groups === undefined || groups.has(row.group_id)), + (row) => ({ + ...fieldsOf(row, view.fields ?? ["basis", "supporting_record_ordinal"]), + match_id: row.match_id, + endpoint_id: row.endpoint_id, + group_id: row.group_id, + }), + ); + } + return selected; +}; + +/** Page completed classifications; filters never reclassify a subset or upgrade source coverage. */ +export const projectEndpointComparisonView = ( + parent: AnalysisViewParent, + admitted: AdmittedView, +) => { + const view = ownedView< + SummaryAnalysisView | z.output + >(admitted, "summary", "endpoint-comparison"); + if (view === undefined) + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Endpoint comparisons support summary and endpoint-comparison pages", + }, + ]), + ); + const parsed = readReport(parent); + if (!parsed.ok) return parsed; + const { + static_endpoints: _static, + observed_groups: _observed, + matches: _matches, + limitations, + ...context + } = parsed.value; + const shared = { + parent_evidence_id: parent.evidenceId, + parent_operation: "compare_static_and_observed_endpoints" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: null, + limitations: boundedViewMessages( + [...parent.limitations, ...limitations], + parent.evidenceId, + "limitations", + ), + unknowns: + parsed.value.summary.unknown > 0 + ? [ + "Some static endpoint observations cannot be classified; page static-endpoints with status unknown for their reasons.", + ] + : [], + }; + if (view.kind === "summary") + return ok({ + ...shared, + kind: "summary" as const, + view, + summary: context, + coverage: completeWithinViewCoverage(1, 1), + }); + const selected = selectComparisonRows(parsed.value, view); + return ok({ + ...shared, + ...context, + kind: "endpoint-comparison" as const, + view, + items: selected.rows, + coverage: pageViewCoverage( + view.offset, + selected.rows.length, + selected.total, + ), + }); +}; diff --git a/src/domain/javascript/staticObservedEndpointSchemas.ts b/src/domain/javascript/staticObservedEndpointSchemas.ts new file mode 100644 index 0000000000..f2cebd59f6 --- /dev/null +++ b/src/domain/javascript/staticObservedEndpointSchemas.ts @@ -0,0 +1,103 @@ +import { z } from "zod"; +import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { ROUTE_RULES } from "../analysisView/webNetworkRouteTemplates.js"; + +export const comparedStaticEndpointSchema = z.strictObject({ + endpoint_id: prefixedDigestSchema("static_endpoint"), + node_id: prefixedDigestSchema("jag_node"), + observation_id: prefixedDigestSchema("jag_observation"), + method: z.string().nullable(), + method_basis: z.enum(["api-definition", "callee-spelling", "unknown"]), + origin: z.string().nullable(), + origin_basis: z.enum(["literal-url", "caller-origin", "unknown"]), + path: z.string().nullable(), + status: z.enum(["matched", "not-observed-in-capture", "unknown"]), + reason: z + .enum([ + "candidate-endpoint", + "unconfirmed-static-observation", + "partial-target", + "non-http-target", + "unresolved-target", + "unparseable-target", + "normalization-required", + "origin-required", + "ambiguous-method", + "capture-incomplete", + "origin-not-exercised", + "no-observed-match", + ]) + .nullable(), +}); + +export const comparedObservedGroupSchema = z.strictObject({ + group_id: prefixedDigestSchema("observed_endpoint"), + method: z.string(), + origin: z.string(), + path_template: z.string(), + templated_segments: z.array( + z.strictObject({ + segment_index: z.int().nonnegative(), + rule: z.enum(ROUTE_RULES), + }), + ), + record_ordinals: z.array(z.int().nonnegative()), + status: z.enum(["matched", "unknown", "observed-without-static-match"]), + reason: z.literal("ambiguous-static-method").nullable(), +}); + +export const endpointMatchSchema = z.strictObject({ + match_id: prefixedDigestSchema("endpoint_match"), + endpoint_id: comparedStaticEndpointSchema.shape.endpoint_id, + group_id: comparedObservedGroupSchema.shape.group_id, + basis: z.enum(["exact", "template-compatible"]), + supporting_record_ordinal: z.int().nonnegative(), +}); + +export const staticObservedEndpointComparisonSchema = z.strictObject({ + comparison_id: prefixedDigestSchema("endpoint_comparison"), + application: z.strictObject({ + evidence_id: prefixedDigestSchema("ev"), + graph_id: prefixedDigestSchema("jag"), + }), + capture: z.strictObject({ + evidence_id: prefixedDigestSchema("ev"), + artifact: z.strictObject({ + path: z.string(), + sha256: digestSchema.nullable(), + }), + }), + declared_origin: z.string().nullable(), + summary: z.strictObject({ + static_network_endpoint_observations: z.int().nonnegative(), + matched: z.int().nonnegative(), + not_observed_in_capture: z.int().nonnegative(), + unknown: z.int().nonnegative(), + observed_groups: z.int().nonnegative(), + observed_without_static_match: z.int().nonnegative(), + observed_unknown: z.int().nonnegative(), + matches: z.int().nonnegative(), + }), + source_coverage: z.strictObject({ + application: z.strictObject({ + status: z.enum(["complete", "partial", "unknown", "unavailable"]), + truncated: z.boolean(), + omitted_count: z.int().nonnegative().nullable(), + limit_count: z.int().nonnegative(), + limitation_count: z.int().nonnegative(), + }), + capture_limitation_count: z.int().nonnegative(), + observed_records_examined: z.int().nonnegative(), + observed_records_total: z.int().nonnegative(), + observed_records_excluded: z.int().nonnegative(), + }), + static_endpoints: z.array(comparedStaticEndpointSchema), + observed_groups: z.array(comparedObservedGroupSchema), + matches: z.array(endpointMatchSchema), + evidence_links: z.array(prefixedDigestSchema("ev")), + limitations: z.array(z.string()), +}); + +export type StaticObservedEndpointComparison = z.output< + typeof staticObservedEndpointComparisonSchema +>; diff --git a/src/domain/javascript/staticObservedEndpoints.test.ts b/src/domain/javascript/staticObservedEndpoints.test.ts new file mode 100644 index 0000000000..03e218a690 --- /dev/null +++ b/src/domain/javascript/staticObservedEndpoints.test.ts @@ -0,0 +1,296 @@ +import { expect, it } from "vitest"; +import { + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, +} from "./javascriptApplicationGraph.js"; +import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; +import { javascriptApplicationAnalysisResultSchema } from "./javascriptApplicationAnalysis.js"; +import { analysisViewCaptureFixture } from "../../../tests/fixtures/analysisView.js"; +import { WebNetworkApiSurfaceAccumulator } from "../analysisView/WebNetworkApiSurfaceAccumulator.js"; +import { compareStaticObservedEndpoints } from "./staticObservedEndpoints.js"; +import { staticObservedEndpointComparisonSchema } from "./staticObservedEndpointSchemas.js"; + +const original = javascriptApplicationAnalysisResultSchema.parse( + JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result, +).graph; +const evidence = original.nodes[0]!.observations[0]!.evidence; +if (!evidence.artifact.available) + throw new Error("Static fixture must identify its source bytes"); +const artifactSha256 = evidence.artifact.sha256; +const graphFor = ( + targets: readonly { + value: string; + mechanism?: string; + interpretation?: string; + resolution?: string; + }[], + sourceState: "observed" | "unknown" = "observed", +) => + createJavaScriptApplicationGraph({ + schema: original.schema, + root_node_ids: original.root_node_ids, + coverage: original.coverage, + limitations: original.limitations, + nodes: [ + ...original.nodes, + ...targets.map((target, index) => + createJavaScriptApplicationNode({ + kind: "endpoint", + identity: { + strategy: "artifact-local-key", + stability: "artifact-version", + artifact_sha256: artifactSha256, + namespace: "network-endpoint", + key: String(index), + }, + observations: [ + { + label: null, + properties: { + endpoint_kind: "network", + mechanism: "call:fetch", + ...target, + }, + evidence: + sourceState === "observed" + ? evidence + : { + ...evidence, + state: sourceState, + confidence: "unknown", + limitations: [ + "Fixture explicitly leaves network interpretation unknown.", + ], + }, + }, + ], + }), + ), + ], + edges: [], + }); +const run = ( + targets: Parameters[0], + requests: readonly { path: string; method?: string; origin?: string }[], + options: { + complete?: boolean; + origin?: string | null; + budget?: number; + sourceState?: "observed" | "unknown"; + } = {}, +) => { + const accumulator = new WebNetworkApiSurfaceAccumulator( + options.budget, + false, + { operation: "compare_static_and_observed_endpoints" }, + ); + for (const [ordinal, request] of requests.entries()) { + const added = accumulator.add({ + ...analysisViewCaptureFixture().records[0]!, + ordinal, + limitations: [], + reported: { + request: { + method: request.method ?? "GET", + url: `${request.origin ?? "https://api.test"}${request.path}`, + }, + response: { status: 200, content: { mimeType: "application/json" } }, + }, + }); + if (!added.ok) return added; + } + return compareStaticObservedEndpoints({ + applicationEvidenceId: `ev_${"a".repeat(64)}`, + captureEvidenceId: `ev_${"b".repeat(64)}`, + graph: graphFor(targets, options.sourceState), + accumulator, + captureComplete: options.complete ?? true, + ...(options.origin === null + ? {} + : { origin: options.origin ?? "https://api.test" }), + }); +}; + +it("reports all four outcomes, exact/template basis and unknown method without copying URL details", () => { + const result = run( + [ + { value: "/matched?source-query=ordinary#fragment" }, + { value: "/users/789" }, + { value: "/missing" }, + { value: "KEY", interpretation: "candidate" }, + ], + [ + { path: "/matched?capture-query=ordinary" }, + { path: "/users/123" }, + { path: "/observed-only" }, + ], + ); + if (!result.ok) throw result.error; + expect(result.value.summary).toMatchObject({ + matched: 2, + not_observed_in_capture: 1, + unknown: 1, + observed_without_static_match: 1, + }); + expect(result.value.matches.map(({ basis }) => basis).sort()).toEqual([ + "exact", + "template-compatible", + ]); + expect( + result.value.static_endpoints.find(({ path }) => path === "/matched"), + ).toMatchObject({ + method: null, + method_basis: "unknown", + origin_basis: "caller-origin", + }); + expect(JSON.stringify(result.value)).not.toMatch( + /source-query|capture-query|fragment|ordinary/, + ); + expect( + staticObservedEndpointComparisonSchema.shape.static_endpoints.safeParse( + result.value.static_endpoints, + ).success, + ).toBe(true); +}); + +it("does not promote an explicitly unknown static observation to a match", () => { + const result = run([{ value: "/matched" }], [{ path: "/matched" }], { + sourceState: "unknown", + }); + if (!result.ok) throw result.error; + expect(result.value.static_endpoints[0]).toMatchObject({ + status: "unknown", + reason: "unconfirmed-static-observation", + }); + expect(result.value.matches).toEqual([]); +}); + +it("does not manufacture an exact whole path from independent templated segment values", () => { + const result = run( + [{ value: "/a/1/b/2" }], + [{ path: "/a/1/b/3" }, { path: "/a/4/b/2" }], + ); + if (!result.ok) throw result.error; + expect(result.value.matches[0]?.basis).toBe("template-compatible"); + expect(result.value.observed_groups[0]?.record_ordinals).toEqual([0, 1]); +}); + +it("names a concrete exact witness within a group containing different literal paths", () => { + const result = run( + [{ value: "/users/123" }], + [{ path: "/users/456" }, { path: "/users/123" }], + ); + if (!result.ok) throw result.error; + expect(result.value.matches[0]).toMatchObject({ + basis: "exact", + supporting_record_ordinal: 1, + }); + expect(result.value.observed_groups[0]?.record_ordinals).toEqual([0, 1]); +}); + +it.each([{ complete: false }, { origin: null }])( + "keeps missing endpoints unknown with incomplete capture or undeclared origin: %j", + (options) => { + const result = run([{ value: "/missing" }], [{ path: "/other" }], options); + if (!result.ok) throw result.error; + expect(result.value.static_endpoints[0]?.status).toBe("unknown"); + }, +); + +it("requires an exercised origin, even for an exact absolute URL", () => { + const result = run( + [{ value: "https://unseen.test/missing" }], + [{ path: "/other" }], + ); + if (!result.ok) throw result.error; + expect(result.value.static_endpoints[0]).toMatchObject({ + status: "unknown", + reason: "origin-not-exercised", + }); +}); + +it("keeps normalization, operation names and partial expressions unknown", () => { + const result = run( + [ + { value: "/a/%2e%2e/admin" }, + { value: "GetViewer" }, + { value: "trpc.user.byId" }, + { value: "fetch(prefix + path)", resolution: "partial" }, + ], + [{ path: "/admin" }], + ); + if (!result.ok) throw result.error; + expect(result.value.summary.unknown).toBe(4); + expect(result.value.matches).toEqual([]); +}); + +it.each([false, true])( + "keeps ambiguity on both sides and preserves confirmed method matches (known peer: %s)", + (knownPeer) => { + const result = run( + [ + { value: "/ambiguous" }, + ...(knownPeer + ? [{ value: "/ambiguous", mechanism: "new:EventSource" }] + : []), + { value: "/events", mechanism: "new:EventSource" }, + ], + [ + { path: "/ambiguous", method: "GET" }, + { path: "/ambiguous", method: "POST" }, + { path: "/events", method: "POST" }, + ], + ); + if (!result.ok) throw result.error; + expect( + result.value.static_endpoints.find( + ({ path, method }) => path === "/ambiguous" && method === null, + ), + ).toMatchObject({ + status: "unknown", + reason: "ambiguous-method", + method: null, + }); + expect( + result.value.static_endpoints.find(({ path }) => path === "/events"), + ).toMatchObject({ status: "not-observed-in-capture", method: "GET" }); + expect(result.value.summary.observed_unknown).toBe(knownPeer ? 1 : 2); + expect( + result.value.observed_groups.find( + ({ path_template, method }) => + path_template === "/ambiguous" && method === "POST", + ), + ).toMatchObject({ status: "unknown", reason: "ambiguous-static-method" }); + expect( + result.value.observed_groups.find( + ({ path_template, method }) => + path_template === "/ambiguous" && method === "GET", + ), + ).toMatchObject({ + status: knownPeer ? "matched" : "unknown", + reason: knownPeer ? null : "ambiguous-static-method", + }); + }, +); + +it("does not upgrade absence when even one recorded request is excluded", () => { + const result = run( + [{ value: "/missing" }], + [{ path: "/known" }, { path: "/omitted", method: "" }], + ); + if (!result.ok) throw result.error; + expect(result.value.static_endpoints[0]).toMatchObject({ + status: "unknown", + reason: "capture-incomplete", + }); +}); + +it("charges comparison state against the inherited representation allowance", () => { + const result = run([{ value: "/large".repeat(500) }], [{ path: "/known" }], { + budget: 4096, + }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); +}); diff --git a/src/domain/javascript/staticObservedEndpoints.ts b/src/domain/javascript/staticObservedEndpoints.ts new file mode 100644 index 0000000000..ab8fa0810b --- /dev/null +++ b/src/domain/javascript/staticObservedEndpoints.ts @@ -0,0 +1,355 @@ +import { digestCanonicalValue } from "../canonicalDigest.js"; +import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js"; +import { parseRecordedTransportUrl } from "../recordedTransportUrl.js"; +import { err, ok, type Result } from "../result.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import type { JavaScriptApplicationGraph } from "./javascriptApplicationGraph.js"; +import { + WebNetworkApiSurfaceAccumulator, + type ApiSurfaceComparisonGroup, +} from "../analysisView/WebNetworkApiSurfaceAccumulator.js"; +import { + ENTRY_BYTES, + GROUP_BYTES, + scanRouteTemplate, + textBytes, +} from "../analysisView/webNetworkRouteTemplates.js"; +import type { StaticObservedEndpointComparison } from "./staticObservedEndpointSchemas.js"; + +type Endpoint = StaticObservedEndpointComparison["static_endpoints"][number]; +type Group = StaticObservedEndpointComparison["observed_groups"][number]; +type Match = StaticObservedEndpointComparison["matches"][number]; +const identity = (prefix: string, facts: unknown) => + `${prefix}_${digestCanonicalValue(facts)}`; +const routeKey = (origin: string, path: string, rules: string) => + JSON.stringify([origin, path, rules]); + +/** Interpret only method-bearing API spellings; fetch options and XHR methods stay unknown. */ +const methodFor = ( + mechanism: unknown, +): Pick => { + if (typeof mechanism !== "string") + return { method: null, method_basis: "unknown" }; + if (/(?:^|[.:])EventSource$/u.test(mechanism)) + return { method: "GET", method_basis: "api-definition" }; + if (/(?:^|[.:])sendBeacon$/u.test(mechanism)) + return { method: "POST", method_basis: "api-definition" }; + const method = /(?:^|[.:])(get|post|put|patch|delete|head|options)$/u.exec( + mechanism, + )?.[1]; + return method === undefined + ? { method: null, method_basis: "unknown" } + : { method: method.toUpperCase(), method_basis: "callee-spelling" }; +}; + +/** Keep recorded spelling; targets requiring URL normalization stay unknown instead of acquiring a different identity. */ +const targetFor = ( + value: unknown, + origin: string | undefined, +): Pick => { + const unknown = (reason: Endpoint["reason"]) => ({ + origin: null, + origin_basis: "unknown" as const, + path: null, + reason, + }); + if (typeof value !== "string") return unknown("unparseable-target"); + if (value.startsWith("/") && !value.startsWith("//")) { + const path = value.split(/[?#]/u, 1)[0]!; + if (origin === undefined) return { ...unknown("origin-required"), path }; + // Use the same recorded URL owner as HAR, rather than URL.pathname's normalization. + const parsed = parseRecordedTransportUrl(`${origin}${path}`); + if ( + parsed.path !== path || + parsed.origin !== origin || + parsed.credentials !== null + ) + return unknown("unparseable-target"); + if (URL.parse(`${origin}${path}`)?.pathname !== path) + return unknown("normalization-required"); + return { origin, origin_basis: "caller-origin", path, reason: null }; + } + if (!/^https?:\/\//iu.test(value)) + return unknown( + /^[A-Za-z][A-Za-z0-9+.-]*:/u.test(value) + ? "non-http-target" + : "unresolved-target", + ); + const parsed = parseRecordedTransportUrl(value); + if ( + parsed.path === null || + parsed.origin === null || + parsed.credentials !== null + ) + return unknown("unparseable-target"); + if (URL.parse(value)?.pathname !== parsed.path) + return unknown("normalization-required"); + return { + origin: parsed.origin, + origin_basis: "literal-url", + path: parsed.path, + reason: null, + }; +}; + +interface ComparisonInput { + readonly applicationEvidenceId: string; + readonly captureEvidenceId: string; + readonly graph: JavaScriptApplicationGraph; + readonly accumulator: WebNetworkApiSurfaceAccumulator; + readonly origin?: string; + readonly captureComplete: boolean; +} +interface IndexedGroup { + readonly source: ApiSurfaceComparisonGroup; + readonly row: Group; +} +interface ObservedIndex { + readonly rows: Group[]; + readonly routes: Map< + string, + { methods: Map; ambiguous: boolean } + >; + readonly origins: Set; + readonly complete: boolean; +} +type Node = JavaScriptApplicationGraph["nodes"][number]; + +/** Each route owns its methods and ambiguity; a group is never guessed from display text. */ +const indexObservedGroups = ( + input: ComparisonInput, +): Result => { + const { accumulator } = input; + const index: ObservedIndex = { + rows: [], + routes: new Map(), + origins: new Set(), + complete: + input.captureComplete && + accumulator.comparisonCoverage().excluded_records === 0, + }; + for (const source of accumulator.comparisonGroups()) { + const admitted = accumulator.admitDerivedBytes( + GROUP_BYTES + ENTRY_BYTES * source.record_ordinals.length, + ); + if (!admitted.ok) return admitted; + const row: Group = { + group_id: identity("observed_endpoint", [ + input.captureEvidenceId, + source.method, + source.origin, + source.path_template, + source.templated_segments, + ]), + method: source.method, + origin: source.origin, + path_template: source.path_template, + templated_segments: [...source.templated_segments], + record_ordinals: [...source.record_ordinals], + status: "observed-without-static-match", + reason: null, + }; + index.rows.push(row); + index.origins.add(source.origin); + const key = routeKey(source.origin, source.path_template, source.rules); + const bucket = index.routes.get(key); + if (bucket === undefined) + index.routes.set(key, { + methods: new Map([[source.method, { source, row }]]), + ambiguous: false, + }); + else bucket.methods.set(source.method, { source, row }); + } + return ok(index); +}; + +const readStaticEndpoint = ( + input: ComparisonInput, + node: Node, + observation: Node["observations"][number], +): Result => { + const properties = observation.properties; + if (properties.endpoint_kind !== "network") return ok(undefined); + const admitted = input.accumulator.admitDerivedBytes( + GROUP_BYTES + + (typeof properties.value === "string" + ? textBytes(properties.value.length) + : 0), + ); + if (!admitted.ok) return admitted; + const candidate = properties.interpretation === "candidate"; + const partial = properties.resolution === "partial"; + let reason: Endpoint["reason"] = null; + if (candidate) reason = "candidate-endpoint"; + else if (observation.evidence.state !== "observed") + reason = "unconfirmed-static-observation"; + else if (partial) reason = "partial-target"; + return ok({ + endpoint_id: identity("static_endpoint", [ + input.applicationEvidenceId, + node.node_id, + observation.observation_id, + ]), + node_id: node.node_id, + observation_id: observation.observation_id, + ...methodFor(reason === null ? properties.mechanism : undefined), + ...(reason !== null + ? { + origin: null, + origin_basis: "unknown" as const, + path: null, + reason, + } + : targetFor(properties.value, input.origin)), + status: "unknown", + }); +}; + +const matchEndpoint = ( + endpoint: Endpoint, + index: ObservedIndex, + budget: WebNetworkApiSurfaceAccumulator, +): Result => { + if ( + endpoint.reason !== null || + endpoint.path === null || + endpoint.origin === null + ) + return ok(undefined); + let failure: AnalysisError | undefined; + const route = scanRouteTemplate(endpoint.path, (bytes) => { + const admitted = budget.admitDerivedBytes(bytes); + if (!admitted.ok) failure = admitted.error; + return admitted.ok; + }); + if (route === null) { + if (failure === undefined) + throw new Error("Route reservation failed without its owned error"); + return err(failure); + } + const bucket = index.routes.get( + routeKey(endpoint.origin, route.template, route.rules), + ); + if ( + endpoint.method === null && + bucket !== undefined && + bucket.methods.size > 1 + ) { + endpoint.reason = "ambiguous-method"; + bucket.ambiguous = true; + return ok(undefined); + } + const selected = + endpoint.method === null + ? bucket?.methods.values().next().value + : bucket?.methods.get(endpoint.method); + if (selected !== undefined) { + const admitted = budget.admitDerivedBytes(GROUP_BYTES); + if (!admitted.ok) return admitted; + endpoint.status = "matched"; + selected.row.status = "matched"; + const exactOrdinal = selected.source.exactPathOrdinal(endpoint.path); + const basis = + exactOrdinal !== undefined + ? ("exact" as const) + : ("template-compatible" as const); + return ok({ + match_id: identity("endpoint_match", [ + endpoint.endpoint_id, + selected.row.group_id, + basis, + ]), + endpoint_id: endpoint.endpoint_id, + group_id: selected.row.group_id, + basis, + supporting_record_ordinal: + exactOrdinal ?? selected.source.record_ordinals[0]!, + }); + } + if (!index.complete) endpoint.reason = "capture-incomplete"; + else if (!index.origins.has(endpoint.origin)) + endpoint.reason = "origin-not-exercised"; + else { + endpoint.status = "not-observed-in-capture"; + endpoint.reason = "no-observed-match"; + } + return ok(undefined); +}; + +const finalizeObservedGroups = (index: ObservedIndex): void => { + for (const bucket of index.routes.values()) { + if (!bucket.ambiguous) continue; + for (const { row } of bucket.methods.values()) { + if (row.status === "matched") continue; + row.status = "unknown"; + row.reason = "ambiguous-static-method"; + } + } + index.rows.sort((a, b) => compare(a.group_id, b.group_id)); +}; + +const summarize = ( + endpoints: readonly Endpoint[], + groups: readonly Group[], + matchCount: number, +): StaticObservedEndpointComparison["summary"] => { + const summary = { + static_network_endpoint_observations: endpoints.length, + matched: 0, + not_observed_in_capture: 0, + unknown: 0, + observed_groups: groups.length, + observed_without_static_match: 0, + observed_unknown: 0, + matches: matchCount, + }; + for (const endpoint of endpoints) { + if (endpoint.status === "matched") summary.matched++; + else if (endpoint.status === "unknown") summary.unknown++; + else summary.not_observed_in_capture++; + } + for (const group of groups) { + if (group.status === "observed-without-static-match") + summary.observed_without_static_match++; + else if (group.status === "unknown") summary.observed_unknown++; + } + return summary; +}; + +/** Join admitted observations in indexed work; retain one concrete witness per pairing. */ +export const compareStaticObservedEndpoints = ( + input: ComparisonInput, +): Result< + Pick< + StaticObservedEndpointComparison, + "static_endpoints" | "observed_groups" | "matches" | "summary" + >, + AnalysisError +> => { + const indexed = indexObservedGroups(input); + if (!indexed.ok) return indexed; + const index = indexed.value; + const endpoints: Endpoint[] = []; + const matches: Match[] = []; + for (const node of input.graph.nodes) { + if (node.kind !== "endpoint") continue; + for (const observation of node.observations) { + const endpoint = readStaticEndpoint(input, node, observation); + if (!endpoint.ok) return endpoint; + if (endpoint.value === undefined) continue; + endpoints.push(endpoint.value); + const match = matchEndpoint(endpoint.value, index, input.accumulator); + if (!match.ok) return match; + if (match.value !== undefined) matches.push(match.value); + } + } + finalizeObservedGroups(index); + endpoints.sort((a, b) => compare(a.endpoint_id, b.endpoint_id)); + matches.sort((a, b) => compare(a.match_id, b.match_id)); + return ok({ + static_endpoints: endpoints, + observed_groups: index.rows, + matches, + summary: summarize(endpoints, index.rows, matches.length), + }); +}; diff --git a/src/server/registerApplicationTools.ts b/src/server/registerApplicationTools.ts index a68f218579..ae392dc1ca 100644 --- a/src/server/registerApplicationTools.ts +++ b/src/server/registerApplicationTools.ts @@ -3,6 +3,7 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import { registerTraceFeatureTool } from "./registerApplicationTools/traceFeature.js"; import { registerTraceJavaScriptSemanticsTool } from "./registerApplicationTools/traceSemantics.js"; import { registerCompareApplicationVersionsTool } from "./registerApplicationTools/compareVersions.js"; +import { registerCompareEndpointsTool } from "./registerApplicationTools/compareEndpoints.js"; import { registerCompareSourceToBundleTool } from "./registerApplicationTools/compareSourceToBundle.js"; import { registerCompareJavaScriptExportShapesTool } from "./registerApplicationTools/compareExportShapes.js"; import { registerCoverageTools } from "./registerApplicationTools/coverage.js"; @@ -23,6 +24,7 @@ export const registerApplicationTools = ( registerTraceFeatureTool(server, options); registerTraceJavaScriptSemanticsTool(server, options); registerCompareApplicationVersionsTool(server, options); + registerCompareEndpointsTool(server, options); registerCompareSourceToBundleTool(server, options); registerCompareJavaScriptExportShapesTool(server, options); registerReconstructionObligationLedgerTool(server, options); diff --git a/src/server/registerApplicationTools/compareEndpoints.ts b/src/server/registerApplicationTools/compareEndpoints.ts new file mode 100644 index 0000000000..69b4b156de --- /dev/null +++ b/src/server/registerApplicationTools/compareEndpoints.ts @@ -0,0 +1,60 @@ +import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; +import { runAdmittedToolOperation } from "../admittedToolOperation.js"; +import { requireAnalysisViewRetention } from "../../application/analysisView/AnalysisViewService.js"; +import { compareStaticAndObservedEndpointsEvidence } from "../../application/javascript/StaticObservedEndpointComparisonService.js"; +import { toolContract } from "../../contracts/toolContracts.js"; +import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; +import { logToolExecution } from "../toolLogging.js"; +import { deliverApplicationEvidence } from "./helpers.js"; +import type { ApplicationToolRegistration } from "./types.js"; + +const contract = toolContract("compare_static_and_observed_endpoints"); + +/** Both adapters execute the same authenticated, provider-free comparison. */ +export const registerCompareEndpointsTool = ( + server: EvidenceMcpServer, + options: ApplicationToolRegistration, +): void => { + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async (input, context) => + runAdmittedToolOperation( + server, + options.withAdmittedAnalysis, + contract.name, + context.mcpReq.signal, + async () => { + if (input.detail === "summary" || input.view !== undefined) { + const retention = requireAnalysisViewRetention( + contract.name, + options, + ); + if (!retention.ok) + return server.delivery.toCallToolResult(retention, contract); + } + const result = await logToolExecution( + options.logger, + contract.name, + () => + compareStaticAndObservedEndpointsEvidence( + input, + options.evidenceById, + { signal: context.mcpReq.signal }, + ), + ); + if (!result.ok) + return server.delivery.toCallToolResult(result, contract); + return deliverApplicationEvidence( + server, + options, + contract, + result.value, + input.view ?? + (input.detail === "summary" ? { kind: "summary" } : undefined), + context.mcpReq.signal, + ); + }, + ), + ); +}; diff --git a/tests/acceptance/applications/staticObservedEndpoints.test.ts b/tests/acceptance/applications/staticObservedEndpoints.test.ts new file mode 100644 index 0000000000..83b458b2e5 --- /dev/null +++ b/tests/acceptance/applications/staticObservedEndpoints.test.ts @@ -0,0 +1,409 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { expect } from "vitest"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../../src/domain/evidence.js"; +import { + createEvidenceBundle, + evidenceBundleSchema, +} from "../../../src/domain/evidenceBundle.js"; +import { + staticObservedEndpointComparisonSchema, + type StaticObservedEndpointComparison, +} from "../../../src/domain/javascript/staticObservedEndpointSchemas.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js"; + +const prepareInputs = async (cli: TestCli) => { + const root = await createTestTempDirectory("rea-static-observed-endpoints-"); + const appPath = join(root, "app"); + await mkdir(appPath); + const fixture = new URL( + "../../fixtures/javascript-endpoint-capture/", + import.meta.url, + ); + const sourcePath = join(appPath, "app.js"); + const source = await readFile(new URL("app.js.txt", fixture), "utf8"); + await writeFile(sourcePath, source); + await writeFile( + join(appPath, "index.html"), + await readFile(new URL("index.html.txt", fixture)), + ); + const harPath = join(root, "capture.har"); + const harBytes = await readFile( + new URL("recording.har.txt", fixture), + "utf8", + ); + expect(createHash("sha256").update(harBytes).digest("hex")).toBe( + "f0e7d2c869c811d4272a7b2916c6531c8c6cc7938336a35575e2a6b7b6346714", + ); + await writeFile(harPath, harBytes); + const run = cli.sharedCliRun?.bind(cli) ?? cli.run.bind(cli); + const analyzed = await run({ + arguments: [ + "analyze-javascript-application", + appPath, + "--artifact-format", + "directory", + "--json", + ], + }); + expect(analyzed.exitCode, analyzed.stderr).toBe(0); + const application = parseEvidence(analyzed.json); + const inspected = await run({ + arguments: ["inspect-web-network-capture", harPath, "har", "--json"], + }); + expect(inspected.exitCode, inspected.stderr).toBe(0); + const capture = parseEvidence(inspected.json); + const input = { application, capture, origin: "http://127.0.0.1:42845" }; + const inputPath = join(root, "comparison-input.json"); + await writeFile(inputPath, JSON.stringify(input)); + + return { + root, + sourcePath, + source, + harPath, + harBytes, + run, + application, + capture, + input, + inputPath, + }; +}; + +type McpCall = Awaited>["call"]; +const assertPagedComparison = async ( + call: McpCall, + report: StaticObservedEndpointComparison, + complete: Evidence, + input: Parameters[1], +) => { + const pageRequest = { + source: { kind: "retained-evidence", evidence_id: complete.evidence_id }, + view: { + kind: "endpoint-comparison", + collection: "static-endpoints", + fields: ["path", "status", "reason"], + offset: 0, + limit: 2, + }, + }; + const first = await call("inspect_analysis_view", pageRequest); + const firstView = analysisViewResultSchema.parse( + parseEvidence(first.structuredContent).normalized_result, + ); + expect(firstView).toMatchObject({ + source_coverage: report.source_coverage, + summary: report.summary, + coverage: { examined: 2, total: 7, exhausted: false, next_offset: 2 }, + }); + if (firstView.kind !== "endpoint-comparison") + throw new Error("Expected endpoint page"); + const inline = await call("compare_static_and_observed_endpoints", { + ...input, + detail: "complete", + view: pageRequest.view, + }); + expect(inline.isError, JSON.stringify(inline.content)).not.toBe(true); + expect(parseEvidence(inline.structuredContent).normalized_result).toEqual( + firstView, + ); + expect(firstView.items).toEqual( + report.static_endpoints + .slice(0, 2) + .map( + ({ endpoint_id, node_id, observation_id, path, status, reason }) => ({ + endpoint_id, + node_id, + observation_id, + path, + status, + reason, + }), + ), + ); + const empty = await call("inspect_analysis_view", { + ...pageRequest, + view: { ...pageRequest.view, statuses: [], fields: [] }, + }); + expect( + parseEvidence(empty.structuredContent).normalized_result, + ).toMatchObject({ + items: [], + coverage: { total: 0, exhausted: true }, + summary: report.summary, + }); + const matchPage = await call("inspect_analysis_view", { + source: pageRequest.source, + view: { kind: "endpoint-comparison", collection: "matches", fields: [] }, + }); + expect( + parseEvidence(matchPage.structuredContent).normalized_result, + ).toMatchObject({ + items: report.matches.map(({ match_id, endpoint_id, group_id }) => ({ + match_id, + endpoint_id, + group_id, + })), + }); + + return { firstView, pageRequest }; +}; + +const assertCliPublication = async ( + context: Awaited>, + comparison: { + readonly complete: Evidence; + readonly summary: Awaited>; + readonly views: Awaited>; + }, +) => { + const { root, run, inputPath, application, capture } = context; + const { + complete, + summary, + views: { firstView, pageRequest }, + } = comparison; + const bundlePath = join(root, "comparison.bundle.json"); + const cliSummary = await run({ + arguments: [ + "compare-static-and-observed-endpoints", + inputPath, + "--detail", + "summary", + "--evidence_output", + bundlePath, + "--json", + ], + }); + expect(cliSummary.exitCode, cliSummary.stderr).toBe(0); + expect(cliSummary.json).toMatchObject({ + summary: summary.structuredContent, + }); + const bundle = evidenceBundleSchema.parse( + JSON.parse(await readFile(bundlePath, "utf8")), + ); + expect(new Set(bundle.records.map(({ evidence_id }) => evidence_id))).toEqual( + new Set([ + application.evidence_id, + capture.evidence_id, + complete.evidence_id, + ]), + ); + const fileViewPath = join(root, "page-input.json"); + await writeFile( + fileViewPath, + JSON.stringify({ + ...pageRequest, + source: { + kind: "evidence-file", + path: bundlePath, + evidence_id: complete.evidence_id, + }, + }), + ); + const cliPage = await run({ + arguments: ["inspect-analysis-view", fileViewPath, "--json"], + }); + expect(cliPage.exitCode, cliPage.stderr).toBe(0); + expect(parseEvidence(cliPage.json).normalized_result).toEqual(firstView); + const inlineCli = await run({ + arguments: [ + "compare-static-and-observed-endpoints", + inputPath, + "--detail", + "summary", + "--view", + JSON.stringify(pageRequest.view), + "--evidence_output", + join(root, "selected.bundle.json"), + "--json", + ], + }); + expect(inlineCli.exitCode, inlineCli.stderr).toBe(0); + expect(inlineCli.json).toMatchObject({ + summary: { normalized_result: firstView }, + }); + const invalidView = await run({ + arguments: [ + "compare-static-and-observed-endpoints", + inputPath, + "--view", + JSON.stringify(pageRequest.view), + "--json", + ], + }); + expect(invalidView.exitCode).not.toBe(0); + expect(invalidView.json).toMatchObject({ code: "invalid_request" }); + const missingSink = await run({ + arguments: [ + "compare-static-and-observed-endpoints", + inputPath, + "--detail", + "summary", + "--json", + ], + }); + expect(missingSink.exitCode).not.toBe(0); + expect(missingSink.json).toMatchObject({ code: "invalid_request" }); +}; + +cliTest( + "compares real source and decoded HAR through CLI/MCP, retains complete facts and pages stable identities", + async ({ cli }) => { + const context = await prepareInputs(cli); + const { + root, + sourcePath, + source, + harPath, + harBytes, + run, + application, + capture, + input, + inputPath, + } = context; + const compared = await run({ + arguments: ["compare-static-and-observed-endpoints", inputPath, "--json"], + }); + expect(compared.exitCode, compared.stderr).toBe(0); + const complete = parseEvidence(compared.json); + const report = staticObservedEndpointComparisonSchema.parse( + complete.normalized_result, + ); + expect(report.summary).toMatchObject({ + static_network_endpoint_observations: 7, + matched: 3, + not_observed_in_capture: 1, + unknown: 3, + observed_without_static_match: 3, + }); + expect(new Set(report.matches.map(({ basis }) => basis))).toEqual( + new Set(["exact", "template-compatible"]), + ); + expect(complete.evidence_links).toEqual( + [application.evidence_id, capture.evidence_id].sort(), + ); + expect(JSON.stringify(report)).not.toMatch(/kept-source-only|fragment|KEY/); + const { client, call } = await connectLocalToolsMcp({ + REA_MCP_INPUT_SCHEMA_PROFILE: "full", + REA_MCP_TOOL_FAMILIES: "application,session", + }); + const schemas = await advertisedMcpToolSchemas( + client, + "compare_static_and_observed_endpoints", + ); + expect(schemas.ajv.validate(schemas.inputSchema, input)).toBe(true); + expect( + schemas.ajv.validate(schemas.inputSchema, { ...input, detail: "wrong" }), + ).toBe(false); + const retained = { + application: { + kind: "retained-evidence", + evidence_id: application.evidence_id, + }, + capture: { kind: "retained-evidence", evidence_id: capture.evidence_id }, + origin: input.origin, + }; + let bundleNumber = 0; + const importRecords = async ( + records: Parameters[0], + ) => { + const path = join(root, `sources-${bundleNumber++}.bundle.json`); + await writeFile(path, JSON.stringify(createEvidenceBundle(records))); + const response = await call("import_evidence_bundle", { path }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + }; + await importRecords([application, capture]); + const fullMcp = await call("compare_static_and_observed_endpoints", { + ...retained, + detail: "complete", + }); + expect(fullMcp.isError, JSON.stringify(fullMcp.content)).not.toBe(true); + expect(parseEvidence(fullMcp.structuredContent)).toEqual(complete); + const summary = await call( + "compare_static_and_observed_endpoints", + retained, + ); + expect(summary.isError).not.toBe(true); + const summaryView = analysisViewResultSchema.parse( + parseEvidence(summary.structuredContent).normalized_result, + ); + expect(summaryView).toMatchObject({ + kind: "summary", + parent_evidence_id: complete.evidence_id, + summary: { + summary: report.summary, + source_coverage: report.source_coverage, + }, + }); + expect(summaryView).not.toHaveProperty("static_endpoints"); + + const views = await assertPagedComparison(call, report, complete, retained); + await assertCliPublication(context, { complete, summary, views }); + const tampered = await call("compare_static_and_observed_endpoints", { + ...retained, + capture: { ...capture, normalized_result: {} }, + }); + expect(parseMcpToolError(tampered).error).toMatchObject({ + code: "evidence_integrity_mismatch", + }); + const sourceCapture = webNetworkCaptureSchema.parse( + capture.normalized_result, + ); + const replacement = (parameters: object, result: unknown) => + createEvidence(undefined, capture.provider, { + operation: capture.operation, + parameters, + result, + confidence: "observed", + }); + const declared = replacement( + { ...capture.parameters, sensitive_value_count: 1 }, + sourceCapture, + ); + await importRecords([declared]); + const refused = await call("compare_static_and_observed_endpoints", { + ...retained, + capture: { kind: "retained-evidence", evidence_id: declared.evidence_id }, + }); + expect(parseMcpToolError(refused).error).toMatchObject({ + code: "unsupported_target", + }); + expect(JSON.stringify(refused)).toContain("sensitive_values_unretained"); + const subset = replacement(capture.parameters, { + ...sourceCapture, + records: sourceCapture.records.slice(0, 1), + }); + await importRecords([subset]); + const limited = await call("compare_static_and_observed_endpoints", { + ...retained, + capture: { kind: "retained-evidence", evidence_id: subset.evidence_id }, + detail: "complete", + }); + expect(limited.isError).not.toBe(true); + expect( + staticObservedEndpointComparisonSchema + .parse(parseEvidence(limited.structuredContent).normalized_result) + .static_endpoints.find(({ path }) => path === "/api/admin/export"), + ).toMatchObject({ status: "unknown", reason: "capture-incomplete" }); + expect(await readFile(sourcePath, "utf8")).toBe(source); + expect( + createHash("sha256") + .update(await readFile(harPath)) + .digest("hex"), + ).toBe(createHash("sha256").update(harBytes).digest("hex")); + }, +); diff --git a/tests/fixtures/javascript-endpoint-capture/README.md b/tests/fixtures/javascript-endpoint-capture/README.md new file mode 100644 index 0000000000..e68ab14d85 --- /dev/null +++ b/tests/fixtures/javascript-endpoint-capture/README.md @@ -0,0 +1,21 @@ +# Static endpoint and capture fixture + +This source-owned loopback application was recorded on Linux with Playwright +1.64.1 and Chromium headless shell 153.0.8010.12. `recording.har.txt` is the original +HAR 1.2 from `browser.newContext({recordHar: {mode: "full", content: "embed"}})`; +its SHA-256 is `f0e7d2c869c811d4272a7b2916c6531c8c6cc7938336a35575e2a6b7b6346714`. +The `.txt` suffix preserves the recorder bytes without JSON formatting; the test +copies it to `capture.har` before production decoding. +The recorded origin is `http://127.0.0.1:42845`. Do not reserialize the HAR. + +The server returned the two adjacent text files as `/app.js` and `/`, and JSON +for the API paths. It streamed each response in two chunks 40 ms apart. The +browser awaited all three fetch response bodies before context close. The five +recorded requests are `/`, `/app.js`, `/api/users?ordinary=kept-source-only`, +`/users/123` and `/observed-only`. The script's unused function bodies were +never called. Transport authentication headers and cookies are absent. + +The public acceptance test copies these sources into a temporary application +and runs the production static analyzer, HAR decoder, comparison and retained +views through CLI/MCP. It replays this recording offline and needs no browser +or network. The recording proves only this small application's selected run. diff --git a/tests/fixtures/javascript-endpoint-capture/app.js.txt b/tests/fixtures/javascript-endpoint-capture/app.js.txt new file mode 100644 index 0000000000..69d455e030 --- /dev/null +++ b/tests/fixtures/javascript-endpoint-capture/app.js.txt @@ -0,0 +1,6 @@ + +function unusedAdmin() { return fetch('/api/admin/export'); } +function templateCandidate() { return fetch('/users/789'); } +function neverContacted() { return fetch('https://never-contact.invalid/route'); } +function translatedValue() { return translate.get('KEY'); } +Promise.all([fetch('/api/users?ordinary=kept-source-only#fragment'), fetch('/users/123'), fetch(window.dynamicTarget)].map(request => request.then(response => response.text()))).then(() => window.done = true); diff --git a/tests/fixtures/javascript-endpoint-capture/index.html.txt b/tests/fixtures/javascript-endpoint-capture/index.html.txt new file mode 100644 index 0000000000..d1f5160a22 --- /dev/null +++ b/tests/fixtures/javascript-endpoint-capture/index.html.txt @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fixtures/javascript-endpoint-capture/recording.har.txt b/tests/fixtures/javascript-endpoint-capture/recording.har.txt new file mode 100644 index 0000000000..9d64ce498d --- /dev/null +++ b/tests/fixtures/javascript-endpoint-capture/recording.har.txt @@ -0,0 +1 @@ +{"log":{"version":"1.2","creator":{"name":"Playwright","version":"1.64.1"},"browser":{"name":"chromium","version":"153.0.8010.12"},"pages":[{"startedDateTime":"2026-11-06T03:22:56.796Z","id":"page@07d055c0281abfb821900dbff3f19b1f","title":"","pageTimings":{"onContentLoad":98,"onLoad":98}}],"entries":[{"pageref":"page@07d055c0281abfb821900dbff3f19b1f","startedDateTime":"2026-11-06T03:22:56.799Z","time":48.571000000000005,"request":{"method":"GET","url":"http://127.0.0.1:42845/","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"},{"name":"Accept-Encoding","value":"gzip, deflate, br, zstd"},{"name":"Connection","value":"keep-alive"},{"name":"Host","value":"127.0.0.1:42845"},{"name":"Sec-Fetch-Dest","value":"document"},{"name":"Sec-Fetch-Mode","value":"navigate"},{"name":"Sec-Fetch-Site","value":"none"},{"name":"Sec-Fetch-User","value":"?1"},{"name":"Upgrade-Insecure-Requests","value":"1"},{"name":"User-Agent","value":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/153.0.8010.12 Safari/537.36"},{"name":"sec-ch-ua","value":"\"HeadlessChrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\""},{"name":"sec-ch-ua-mobile","value":"?0"},{"name":"sec-ch-ua-platform","value":"\"Linux\""}],"queryString":[],"headersSize":630,"bodySize":0},"response":{"status":200,"statusText":"OK","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Connection","value":"keep-alive"},{"name":"Content-Type","value":"text/html"},{"name":"Date","value":"Fri, 06 Nov 2026 03:22:56 GMT"},{"name":"Keep-Alive","value":"timeout=5"},{"name":"Transfer-Encoding","value":"chunked"}],"content":{"size":85,"mimeType":"text/html","compression":0,"text":""},"headersSize":156,"bodySize":102,"redirectURL":"","_transferSize":258},"cache":{},"timings":{"dns":0.063,"connect":0.148,"ssl":1.43,"send":0,"wait":6.175,"receive":40.755},"_frameref":"frame@2eea7869e2d6761b00a3030963155215","_monotonicTime":589.594,"_resourceType":"document","serverIPAddress":"127.0.0.1","_serverPort":42845,"_securityDetails":{}},{"pageref":"page@07d055c0281abfb821900dbff3f19b1f","startedDateTime":"2026-11-06T03:22:56.848Z","time":44.629999999999995,"request":{"method":"GET","url":"http://127.0.0.1:42845/app.js","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Accept","value":"*/*"},{"name":"Accept-Encoding","value":"gzip, deflate, br, zstd"},{"name":"Connection","value":"keep-alive"},{"name":"Host","value":"127.0.0.1:42845"},{"name":"Referer","value":"http://127.0.0.1:42845/"},{"name":"Sec-Fetch-Dest","value":"script"},{"name":"Sec-Fetch-Mode","value":"no-cors"},{"name":"Sec-Fetch-Site","value":"same-origin"},{"name":"User-Agent","value":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/153.0.8010.12 Safari/537.36"},{"name":"sec-ch-ua","value":"\"HeadlessChrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\""},{"name":"sec-ch-ua-mobile","value":"?0"},{"name":"sec-ch-ua-platform","value":"\"Linux\""}],"queryString":[],"headersSize":492,"bodySize":0},"response":{"status":200,"statusText":"OK","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Connection","value":"keep-alive"},{"name":"Content-Type","value":"text/javascript"},{"name":"Date","value":"Fri, 06 Nov 2026 03:22:56 GMT"},{"name":"Keep-Alive","value":"timeout=5"},{"name":"Transfer-Encoding","value":"chunked"}],"content":{"size":477,"mimeType":"text/javascript","compression":0,"text":"\nfunction unusedAdmin() { return fetch('/api/admin/export'); }\nfunction templateCandidate() { return fetch('/users/789'); }\nfunction neverContacted() { return fetch('https://never-contact.invalid/route'); }\nfunction translatedValue() { return translate.get('KEY'); }\nPromise.all([fetch('/api/users?ordinary=kept-source-only#fragment'), fetch('/users/123'), fetch(window.dynamicTarget)].map(request => request.then(response => response.text()))).then(() => window.done = true);\n"},"headersSize":162,"bodySize":494,"redirectURL":"","_transferSize":656},"cache":{},"timings":{"dns":-1,"connect":-1,"ssl":-1,"send":0,"wait":0.937,"receive":43.693},"_frameref":"frame@2eea7869e2d6761b00a3030963155215","_monotonicTime":638.947,"_resourceType":"script","serverIPAddress":"127.0.0.1","_serverPort":42845,"_securityDetails":{}},{"pageref":"page@07d055c0281abfb821900dbff3f19b1f","startedDateTime":"2026-11-06T03:22:56.894Z","time":44.669000000000004,"request":{"method":"GET","url":"http://127.0.0.1:42845/api/users?ordinary=kept-source-only","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Accept","value":"*/*"},{"name":"Accept-Encoding","value":"gzip, deflate, br, zstd"},{"name":"Connection","value":"keep-alive"},{"name":"Host","value":"127.0.0.1:42845"},{"name":"Referer","value":"http://127.0.0.1:42845/"},{"name":"Sec-Fetch-Dest","value":"empty"},{"name":"Sec-Fetch-Mode","value":"cors"},{"name":"Sec-Fetch-Site","value":"same-origin"},{"name":"User-Agent","value":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/153.0.8010.12 Safari/537.36"},{"name":"sec-ch-ua","value":"\"HeadlessChrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\""},{"name":"sec-ch-ua-mobile","value":"?0"},{"name":"sec-ch-ua-platform","value":"\"Linux\""}],"queryString":[{"name":"ordinary","value":"kept-source-only"}],"headersSize":491,"bodySize":0},"response":{"status":200,"statusText":"OK","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Connection","value":"keep-alive"},{"name":"Content-Type","value":"application/json"},{"name":"Date","value":"Fri, 06 Nov 2026 03:22:56 GMT"},{"name":"Keep-Alive","value":"timeout=5"},{"name":"Transfer-Encoding","value":"chunked"}],"content":{"size":11,"mimeType":"application/json","compression":0,"text":"{\"ok\":true}"},"headersSize":163,"bodySize":26,"redirectURL":"","_transferSize":189},"cache":{},"timings":{"dns":-1,"connect":-1,"ssl":-1,"send":0,"wait":3.066,"receive":41.603},"_frameref":"frame@2eea7869e2d6761b00a3030963155215","_monotonicTime":680.638,"_resourceType":"fetch","serverIPAddress":"127.0.0.1","_serverPort":42845,"_securityDetails":{}},{"pageref":"page@07d055c0281abfb821900dbff3f19b1f","startedDateTime":"2026-11-06T03:22:56.894Z","time":48.574,"request":{"method":"GET","url":"http://127.0.0.1:42845/users/123","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Accept","value":"*/*"},{"name":"Accept-Encoding","value":"gzip, deflate, br, zstd"},{"name":"Connection","value":"keep-alive"},{"name":"Host","value":"127.0.0.1:42845"},{"name":"Referer","value":"http://127.0.0.1:42845/"},{"name":"Sec-Fetch-Dest","value":"empty"},{"name":"Sec-Fetch-Mode","value":"cors"},{"name":"Sec-Fetch-Site","value":"same-origin"},{"name":"User-Agent","value":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/153.0.8010.12 Safari/537.36"},{"name":"sec-ch-ua","value":"\"HeadlessChrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\""},{"name":"sec-ch-ua-mobile","value":"?0"},{"name":"sec-ch-ua-platform","value":"\"Linux\""}],"queryString":[],"headersSize":491,"bodySize":0},"response":{"status":200,"statusText":"OK","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Connection","value":"keep-alive"},{"name":"Content-Type","value":"application/json"},{"name":"Date","value":"Fri, 06 Nov 2026 03:22:56 GMT"},{"name":"Keep-Alive","value":"timeout=5"},{"name":"Transfer-Encoding","value":"chunked"}],"content":{"size":11,"mimeType":"application/json","compression":0,"text":"{\"ok\":true}"},"headersSize":163,"bodySize":26,"redirectURL":"","_transferSize":189},"cache":{},"timings":{"dns":0.007,"connect":0.494,"ssl":1.603,"send":0,"wait":4.798,"receive":41.672},"_frameref":"frame@2eea7869e2d6761b00a3030963155215","_monotonicTime":682.181,"_resourceType":"fetch","serverIPAddress":"127.0.0.1","_serverPort":42845,"_securityDetails":{}},{"pageref":"page@07d055c0281abfb821900dbff3f19b1f","startedDateTime":"2026-11-06T03:22:56.894Z","time":51.590999999999994,"request":{"method":"GET","url":"http://127.0.0.1:42845/observed-only","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Accept","value":"*/*"},{"name":"Accept-Encoding","value":"gzip, deflate, br, zstd"},{"name":"Connection","value":"keep-alive"},{"name":"Host","value":"127.0.0.1:42845"},{"name":"Referer","value":"http://127.0.0.1:42845/"},{"name":"Sec-Fetch-Dest","value":"empty"},{"name":"Sec-Fetch-Mode","value":"cors"},{"name":"Sec-Fetch-Site","value":"same-origin"},{"name":"User-Agent","value":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/153.0.8010.12 Safari/537.36"},{"name":"sec-ch-ua","value":"\"HeadlessChrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\""},{"name":"sec-ch-ua-mobile","value":"?0"},{"name":"sec-ch-ua-platform","value":"\"Linux\""}],"queryString":[],"headersSize":495,"bodySize":0},"response":{"status":200,"statusText":"OK","httpVersion":"HTTP/1.1","cookies":[],"headers":[{"name":"Connection","value":"keep-alive"},{"name":"Content-Type","value":"application/json"},{"name":"Date","value":"Fri, 06 Nov 2026 03:22:56 GMT"},{"name":"Keep-Alive","value":"timeout=5"},{"name":"Transfer-Encoding","value":"chunked"}],"content":{"size":11,"mimeType":"application/json","compression":0,"text":"{\"ok\":true}"},"headersSize":163,"bodySize":26,"redirectURL":"","_transferSize":189},"cache":{},"timings":{"dns":0.004,"connect":0.202,"ssl":1.286,"send":0,"wait":7.949,"receive":42.15},"_frameref":"frame@2eea7869e2d6761b00a3030963155215","_monotonicTime":682.414,"_resourceType":"fetch","serverIPAddress":"127.0.0.1","_serverPort":42845,"_securityDetails":{}}]}} \ No newline at end of file