diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 886d2d2c24..7a6e0f6f2e 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -63,7 +63,7 @@ redaction, and its live status clears only after a confirmed retry. ## Formats and upstream profiles -HAR uses unchanged `har-schema@2.0.0` draft-06 schemas, `ajv@8.20.0`, +HAR uses `har-schema@2.0.0` draft-06 schemas, `ajv@8.20.0`, `ajv-formats@3.0.1`, `jsonc-parser@3.3.1` JSON visitor, and `lossless-json@4.3.1` numeric representations, pinned in the npm lockfile. Only the HAR 1.2 profile is supported. Unknown extensions remain reported @@ -71,6 +71,14 @@ evidence. Duplicate object keys, including equal values and escaped spellings, are rejected before materialization. Prototype-named members such as `__proto__` are preserved as ordinary own JSON members without prototype mutation. Original capture files remain unchanged. +REA supplements the pinned schema's missing timing constraints: `timings` must +be an object with nonnegative `send`, `wait`, and `receive`; optional timing +phases and page timings may be nonnegative or exactly `-1` for unavailable. +Total request time must be nonnegative. Invalid timing fields refuse the entire +capture, even when the affected records would be redacted or unselected. +Body readers decode plain text and supported `base64` representations. Other +declared encodings remain preserved in reported Evidence, but decoded body +content is unavailable; their encoded characters are never treated as text. The observed mitmproxy 12.2.3 `SaveHar` profile can emit `postData.text: null` for a missing request body. REA preserves that reported null, omitting the optional field only from its validation copy for this exact producer profile; @@ -261,8 +269,8 @@ not register or wrap the Fiddler MCP server. - `numeric_literals` preserve HAR number lexemes and native integer/float representations, including unsafe integers and non-finite native floats. Values that cannot be safely represented as JSON numbers are `null` in the - reported view. Unsafe standard HAR numeric fields fail schema validation; - unsafe extension numbers remain available through their sidecars. + reported view, including otherwise valid standard HAR numeric fields; + their exact source values remain available through the numeric sidecars. - `binary_fields` retain exposed bytes, base64 and SHA-256 independently of declared sizes. Valid HAR `content.encoding: "base64"` is decoded exactly; producer-decoded Unicode text, missing text and unknown encodings do not diff --git a/src/browser/history/HarCapture.test.ts b/src/browser/history/HarCapture.test.ts index 1eb3067692..3b02138c5c 100644 --- a/src/browser/history/HarCapture.test.ts +++ b/src/browser/history/HarCapture.test.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import { expect, it } from "vitest"; +import { readRecordedResponseBody } from "../../domain/analysisView/recordedResponseBody.js"; import { historicalHar, harWithContainerLookalikes, @@ -7,6 +8,124 @@ import { import { decodeCaptureText } from "./CaptureJson.js"; import { decodeHarCapture } from "./HarCapture.js"; +it.each([ + ["null timing object", { timings: null }, "/timings"], + ["array timing object", { timings: [] }, "/timings"], + ["missing required phase", { timings: { send: 0, receive: 0 } }, "/timings"], + [ + "negative required phase", + { timings: { send: 0, wait: -2, receive: 0 } }, + "/timings/wait", + ], + [ + "unknown required phase", + { timings: { send: -1, wait: 0, receive: 0 } }, + "/timings/send", + ], + [ + "fractional negative optional phase", + { timings: { send: 0, wait: 0, receive: 0, dns: -0.5 } }, + "/timings/dns", + ], + ["negative total", { time: -1 }, "/time"], +] as const)( + "refuses HAR %s before publishing records", + (_name, edited, pointer) => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]!; + const text = JSON.stringify({ + log: { ...fixture.log, entries: [{ ...first, ...edited }] }, + }); + for (const sensitive of [[], ["entries"]]) + expect(() => decodeHarCapture(text, sensitive)).toThrowError( + expect.objectContaining({ + reason: "format", + pointer: `/log/entries/0${pointer}`, + }), + ); + }, +); + +it("retains supported HAR phase sentinels and fractional elapsed times", () => { + const fixture = historicalHar(); + const timings = { send: 0, wait: 1.25, receive: 0, dns: -1, ssl: -1 }; + const decoded = decodeHarCapture( + JSON.stringify({ + log: { + ...fixture.log, + entries: [{ ...fixture.log.entries[0]!, time: 1.25, timings }], + }, + }), + [], + ); + expect(decoded.records[0]?.reported).toMatchObject({ time: 1.25, timings }); +}); + +it.each([-2, -0.5, null])("refuses invalid HAR page timing %s", (onLoad) => { + const fixture = historicalHar(); + const text = JSON.stringify({ + log: { + ...fixture.log, + pages: [ + { + id: "page", + title: "Page", + startedDateTime: fixture.log.entries[0]!.startedDateTime, + pageTimings: { onLoad }, + }, + ], + }, + }); + expect(() => decodeHarCapture(text, [])).toThrowError( + expect.objectContaining({ + reason: "format", + pointer: "/log/pages/0/pageTimings/onLoad", + }), + ); +}); + +it.each([ + [undefined, "hello", { bodyText: "hello" }], + ["base64", "aGVsbG8=", { bodyText: "hello" }], + ["hex", "68656c6c6f", { bodyText: null, unsupportedEncoding: "hex" }], + [ + "base64url", + "aGVsbG8", + { bodyText: null, unsupportedEncoding: "base64url" }, + ], +] as const)( + "keeps HAR body encoding %s distinct from decoded text", + (encoding, text, expected) => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]!; + const content = { + size: 5, + mimeType: "text/plain", + text, + ...(encoding === undefined ? {} : { encoding }), + }; + const decoded = decodeHarCapture( + JSON.stringify({ + log: { + ...fixture.log, + entries: [{ ...first, response: { ...first.response, content } }], + }, + }), + [], + ); + const record = decoded.records[0]!; + expect(record.reported).toMatchObject({ + response: { + content: { + ...(encoding === undefined ? {} : { encoding }), + ...(encoding === "base64" ? {} : { text }), + }, + }, + }); + expect(readRecordedResponseBody(record, 262_144)).toEqual(expected); + }, +); + it("interprets HTTP fields only within HAR entries, preserving document lookalikes", () => { const fixture = harWithContainerLookalikes(); const decoded = decodeHarCapture(JSON.stringify(fixture), []); diff --git a/src/browser/history/HarSchema.ts b/src/browser/history/HarSchema.ts index cb4a55a572..709a4a14ef 100644 --- a/src/browser/history/HarSchema.ts +++ b/src/browser/history/HarSchema.ts @@ -5,7 +5,51 @@ import { z } from "zod"; import { CaptureFormatError } from "./CaptureFormatError.js"; import { captureObject } from "./CaptureRedaction.js"; -/** Validate using the unchanged, pinned upstream HAR 1.2 draft-06 schemas. */ +// har-schema 2.0.0 omits timings' object type and spells numeric bounds `min`. +// Add the HAR 1.2 constraints without modifying the imported schema or source. +const elapsedMilliseconds = { type: "number", minimum: 0 }; +const optionalMilliseconds = { anyOf: [{ const: -1 }, elapsedMilliseconds] }; +const timingRequirements = { + properties: { + log: { + properties: { + entries: { + items: { + properties: { + time: elapsedMilliseconds, + timings: { + type: "object", + properties: { + send: elapsedMilliseconds, + wait: elapsedMilliseconds, + receive: elapsedMilliseconds, + blocked: optionalMilliseconds, + dns: optionalMilliseconds, + connect: optionalMilliseconds, + ssl: optionalMilliseconds, + }, + }, + }, + }, + }, + pages: { + items: { + properties: { + pageTimings: { + properties: { + onContentLoad: optionalMilliseconds, + onLoad: optionalMilliseconds, + }, + }, + }, + }, + }, + }, + }, + }, +}; + +/** Validate the pinned HAR 1.2 schemas with explicit specification timing constraints. */ export const createHarValidator = (): (( value: unknown, ) => readonly number[]) => { @@ -20,9 +64,9 @@ export const createHarValidator = (): (( throw new Error("Pinned AJV formats plugin is unavailable"); plugin(ajv); for (const schema of Object.values(schemas)) ajv.addSchema(schema); - const validate = ajv.getSchema("har.json#"); - if (validate === undefined) - throw new Error("Pinned HAR root schema is unavailable"); + const validate = ajv.compile({ + allOf: [{ $ref: "har.json#" }, timingRequirements], + }); return (value) => { const adaptedPlaywrightEntries = allowObservedProducerRepresentations(value); diff --git a/src/domain/analysisView/recordedResponseBody.ts b/src/domain/analysisView/recordedResponseBody.ts index 1adf258774..0fba68d223 100644 --- a/src/domain/analysisView/recordedResponseBody.ts +++ b/src/domain/analysisView/recordedResponseBody.ts @@ -3,8 +3,10 @@ import type { WebNetworkCaptureRecord } from "../webNetworkCapture.js"; /** One record's retained response text, as replay and state extraction both read it. */ export interface RecordedResponseBody { - /** Producer-decoded response text, or null when the record retained none. */ + /** Producer-decoded response text, or null when absent or decoding is unavailable. */ readonly bodyText: string | null; + /** A retained representation whose declared encoding this reader cannot decode. */ + readonly unsupportedEncoding?: string; /** Set when the retained bytes are not UTF-8, so `bodyText` holds their base64 rather than a lossy decoding. */ readonly bodyEncoding?: "base64"; /** Set when only a prefix of a large base64 body was decoded. */ @@ -37,7 +39,14 @@ export const readRecordedResponseBody = ( if (!content.success) return { bodyText: null }; const { text, encoding } = content.data.response.content; if (typeof text !== "string") return { bodyText: null }; - if (encoding !== "base64") return { bodyText: text }; + if (encoding === undefined) return { bodyText: text }; + if (encoding !== "base64") + return { + bodyText: null, + ...(typeof encoding === "string" + ? { unsupportedEncoding: encoding } + : {}), + }; const binary = record.binary_fields.find( (field) => field.pointer === "/response/content/text" && field.state === "retained",