From c5f50e80d8032baba7c3f6af4b84f8ce2d49e65d Mon Sep 17 00:00:00 2001 From: Kaoru0822 Date: Fri, 30 Oct 2026 01:03:35 -0700 Subject: [PATCH] fix(captures): admit browser-serialized HAR request URLs (#8112) * fix(captures): admit browser-serialized HAR request URLs har-schema@2.0.0 checks request.url with ajv-formats' strict RFC 3986 `uri` format. Browsers record and send URLs as WHATWG serializes them, which keeps characters such as `|` and `{` unescaped in the query, so one Google Fonts or GraphQL GET entry made inspect-web-network-capture and environment-build reject the whole HAR. HAR 1.2 defines request.url only as the absolute request URL. Own that correction in the HAR validator: every URL the RFC 3986 check accepts stays accepted, and a URL that WHATWG URL serialization reproduces unchanged is accepted too. Spellings WHATWG would rewrite (a backslash authority, a tab, a space) and relative references still fail. Fixes #8110 Co-Authored-By: Claude Opus 5.5 * docs(captures): state that RFC 3986 request URLs stay admitted The HAR request URL rule is "passes RFC 3986, or WHATWG serialization reproduces it unchanged". A spelling WHATWG would rewrite therefore fails only when it also fails the RFC 3986 check; an uppercase host still passes. Say so in the docs and the validator comment, and pin the accepted uppercase-host case in the CLI test. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- docs/web-network-captures.md | 10 +- src/browser/history/HarSchema.ts | 29 +++++ .../cli/harBrowserSerializedUrlsCli.test.ts | 118 ++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 tests/boundary/cli/harBrowserSerializedUrlsCli.test.ts diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index cc8f7d9aa8..1310b2d578 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -72,7 +72,15 @@ are rejected before materialization. Prototype-named members such as `__proto__` are preserved as ordinary own JSON members without prototype mutation. Original capture files remain unchanged. REA owns specific schema corrections because pinned upstream schemas omit -object types and use the unrecognized `min` keyword. Following +object types and use the unrecognized `min` keyword. The pinned schema also +checks `request.url` as an RFC 3986 `uri`, while HAR 1.2 defines it only as the +absolute request URL. REA additionally admits a URL that WHATWG URL +serialization reproduces unchanged, which is how browsers record and send it, +for example with an unescaped `|` or `{` in the query. Every URL the RFC 3986 +check accepts stays admitted, even when WHATWG would rewrite it, such as an +uppercase host. Spellings that fail the RFC 3986 check and that WHATWG would +rewrite, such as a backslash authority, a tab or a space, still fail. +Following [HAR 1.2](https://github.com/ahmadnassri/har-spec/blob/81adb2fcf845e81ff3de728cf6829a0e6ad63e95/versions/1.2.md#timings), `timings` must be an object with nonnegative `send`, `wait` and `receive`; optional `blocked`, `dns`, `connect` and `ssl` may be absent, `-1`, or diff --git a/src/browser/history/HarSchema.ts b/src/browser/history/HarSchema.ts index 6b4de142fe..f50d91d392 100644 --- a/src/browser/history/HarSchema.ts +++ b/src/browser/history/HarSchema.ts @@ -44,10 +44,18 @@ export const createHarValidator = (): (( if (typeof plugin !== "function") throw new Error("Pinned AJV formats plugin is unavailable"); plugin(ajv); + ajv.addFormat(HAR_REQUEST_URL_FORMAT, harRequestUrlValidator(plugin)); const propertiesSchema = z.record(z.string(), z.unknown()); const correctedSchemas: Record> = { timings: timingsSchema, cache: { ...schemas.cache, type: "object" }, + request: { + ...schemas.request, + properties: { + ...propertiesSchema.parse(schemas.request?.properties), + url: { type: "string", format: HAR_REQUEST_URL_FORMAT }, + }, + }, entry: { ...schemas.entry, properties: { @@ -123,3 +131,24 @@ const allowObservedProducerRepresentations = ( } return adaptedPlaywrightEntries; }; + +const HAR_REQUEST_URL_FORMAT = "absolute-url"; + +// HAR 1.2 defines request.url only as the absolute request URL. har-schema's +// RFC 3986 `uri` format rejects URLs exactly as browsers serialize and send +// them, such as an unescaped `|` or `{` in the query. Keep every RFC 3986 URL +// admitted, even one WHATWG would rewrite. Otherwise admit only a spelling that +// WHATWG URL serialization reproduces unchanged, so a URL admitted by this +// branch reads the same under WHATWG as it was recorded. +const harRequestUrlValidator = ( + plugin: object, +): ((value: string) => boolean) => { + const getFormat: unknown = Reflect.get(plugin, "get"); + const rfc3986 = isCallable(getFormat) ? getFormat("uri") : undefined; + if (!isCallable(rfc3986)) + throw new Error("Pinned AJV uri format is unavailable"); + return (value) => rfc3986(value) === true || URL.parse(value)?.href === value; +}; + +const isCallable = (value: unknown): value is (argument: string) => unknown => + typeof value === "function"; diff --git a/tests/boundary/cli/harBrowserSerializedUrlsCli.test.ts b/tests/boundary/cli/harBrowserSerializedUrlsCli.test.ts new file mode 100644 index 0000000000..b50fb8e87b --- /dev/null +++ b/tests/boundary/cli/harBrowserSerializedUrlsCli.test.ts @@ -0,0 +1,118 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { historicalHar } from "../../fixtures/historicalHar.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +/** A HAR whose entries copy the fixture entry with each request URL. */ +const harWithRequestUrls = (urls: readonly string[]) => { + const har = historicalHar(); + const [entry] = har.log.entries; + if (entry === undefined) throw new Error("Expected a fixture HAR entry"); + return { + log: { + ...har.log, + entries: urls.map((url) => ({ + ...entry, + request: { ...entry.request, url }, + })), + }, + }; +}; + +const writeHar = async (urls: readonly string[]): Promise => { + const path = join( + await createTestTempDirectory("rea-har-request-url-"), + "capture.har", + ); + await writeFile(path, JSON.stringify(harWithRequestUrls(urls))); + return path; +}; + +// Each URL is exactly what a WHATWG URL serializer emits, as browsers record +// and send it; RFC 3986 alone rejects the unescaped characters. +const BROWSER_SERIALIZED = [ + "https://fonts.googleapis.com/css?family=Roboto:400,700|Open+Sans", + "https://api.example.test/graphql?variables={%22id%22:1}", + "https://api.example.test/search?q=`x`^[1]&rate=100%", + "https://api.example.test/a|b?q=c\\d", +]; + +cliTest( + "keeps admitting an RFC 3986 request URL that WHATWG would rewrite", + async ({ cli }) => { + const url = "https://EXAMPLE.test/a"; + expect(new URL(url).href).not.toBe(url); + const har = await writeHar([url]); + const inspected = await cli.run({ + arguments: ["inspect-web-network-capture", har, "har", "--json"], + }); + expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0); + expect(inspected.json).toMatchObject({ + normalized_result: { records: [{ reported: { request: { url } } }] }, + }); + }, +); + +cliTest( + "inspects and builds HARs whose request URLs are browser serializations", + async ({ cli }) => { + for (const url of BROWSER_SERIALIZED) + expect(new URL(url).href, "fixture must be a WHATWG serialization").toBe( + url, + ); + const har = await writeHar(BROWSER_SERIALIZED); + const inspected = await cli.run({ + arguments: ["inspect-web-network-capture", har, "har", "--json"], + }); + expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0); + expect(inspected.json).toMatchObject({ + normalized_result: { + total_records: BROWSER_SERIALIZED.length, + records: BROWSER_SERIALIZED.map((url) => ({ + reported: { request: { url } }, + })), + }, + }); + const built = await cli.run({ + arguments: ["environment-build", har, "--dry-run", "--json"], + timeoutMs: 60_000, + }); + expect(built.exitCode, built.stdout + built.stderr).toBe(0); + expect(built.json).toMatchObject({ + status: "planned", + capture: { records: BROWSER_SERIALIZED.length, input: "har" }, + }); + }, +); + +cliTest.for([ + // WHATWG rewrites these spellings, so the recorded text and its reading differ. + { name: "backslash authority", url: "http:\\\\user:pass@host.test/" }, + { name: "tab", url: "https://host.test/a\tb" }, + { name: "space", url: "https://host.test/a b" }, + { name: "relative reference", url: "/a|b" }, +])( + "still rejects a request URL that is neither RFC 3986 nor a WHATWG serialization: $name", + async ({ url }, { cli }) => { + const har = await writeHar([url]); + const result = await cli.run({ + arguments: ["inspect-web-network-capture", har, "har", "--json"], + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["capture_path", "/log/entries/0/request/url"], + reason: "invalid_format", + }, + ], + }, + }); + }, +);