diff --git a/docs/guides/web/browser-scenario-contract.md b/docs/guides/web/browser-scenario-contract.md index 5e1eb39ec0..159d583fd5 100644 --- a/docs/guides/web/browser-scenario-contract.md +++ b/docs/guides/web/browser-scenario-contract.md @@ -696,12 +696,21 @@ request URL when CDP has no response body. Event and transaction references identify the associated Playwright response object, but do not verify that the retained bytes came from the original network transaction. Successful selected response bodies include this limitation in the capture. These bytes are not -the compressed wire representation. +the compressed wire representation. Playwright body media types come from +the duplicate-preserving header read and carry `media_type_source` set to +`content-type-header`. Identical repeated declarations retain one value. +Conflicting, unavailable or redacted declarations leave the type null. +Selecting a body allows that lookup even when full header values are omitted +from Evidence. Views use any retained type declarations to interpret captures +that predate `media_type_source`. Earlier REA versions can reject captures +that include the new field. Headers preserve producer order and duplicates. Authorization, Proxy-Authorization, Cookie, and Set-Cookie values are null with `redacted: true`; ordinary values remain intact. Declared UTF-8 secret values -and their standard URI/form or JSON-escaped representations are replaced without -decoding unrelated binary bytes. Other encodings are not interpreted. Digests describe the +and their standard URI/form or JSON-escaped representations are replaced. +Known binary bodies are redacted as bytes. For an unknown media type, a JSON +redaction pass can remove decoded secret spellings without assigning a type. +Other encodings are not interpreted. Digests describe the retained representation after redaction. The selected body-byte limit also bounds that retained representation after secret expansion, before base64 serialization. Credential header values are diff --git a/src/browser/BrowserScenarioResponseValues.ts b/src/browser/BrowserScenarioResponseValues.ts index 60b92e308a..8e4960332e 100644 --- a/src/browser/BrowserScenarioResponseValues.ts +++ b/src/browser/BrowserScenarioResponseValues.ts @@ -177,7 +177,11 @@ export const redactResponseSource = ( redact: (value: string) => string, maxBytes: number, maxAllocationBytes: number, -): Result => { + acceptNonJson = false, +): Result< + { readonly json: boolean; readonly bytes?: Buffer }, + ResponseValueIssue +> => { const markers = new Map( values.map(({ entry, value }) => [ JSON.stringify(jsonPointerTokens(entry.json_pointer)), @@ -213,6 +217,7 @@ export const redactResponseSource = ( "A marked source field has conflicting scalar/container representations; its source content was withheld.", ); }; + const malformedJson = new Error("The marked source is not valid JSON."); try { visit( text, @@ -249,15 +254,16 @@ export const redactResponseSource = ( if (redacted !== property) replace(offset, length, redacted); }, onError: () => { - throw new Error("The marked source is not valid JSON."); + throw malformedJson; }, }, { disallowComments: true, allowTrailingComma: false }, ); - if (parts.length === 0) return ok(undefined); + if (parts.length === 0) return ok({ json: true }); parts.push(text.slice(cursor)); - return ok(Buffer.from(parts.join(""))); + return ok({ json: true, bytes: Buffer.from(parts.join("")) }); } catch (cause: unknown) { + if (acceptNonJson && cause === malformedJson) return ok({ json: false }); return err( issue( cause instanceof Error diff --git a/src/browser/BrowserScenarioSecrets.ts b/src/browser/BrowserScenarioSecrets.ts index 4d0067c231..e93888764d 100644 --- a/src/browser/BrowserScenarioSecrets.ts +++ b/src/browser/BrowserScenarioSecrets.ts @@ -566,10 +566,10 @@ export class BrowserScenarioSecrets { retention.maxAllocationBytes, ); if (!redacted.ok) throw new Error(redacted.error.message); - return redacted.value ?? body; + return redacted.value.bytes ?? body; } - /** Decode JSON only when redaction is active; other bytes retain their existing binary policy. */ + /** Check JSON spellings only while redaction is active, including bodies of unknown type. */ redactJsonBody( body: Buffer, retention: { @@ -577,7 +577,8 @@ export class BrowserScenarioSecrets { readonly maxAllocationBytes: number; }, scope: ScenarioRedactionScope = "current", - ): Buffer { + mode: "json" | "unknown" = "json", + ): { readonly bytes: Buffer; readonly json: boolean } { if ( body.length === 0 || (scope === "response" @@ -585,17 +586,24 @@ export class BrowserScenarioSecrets { : this.textPatterns ).length === 0 ) - return body; - const text = decodeStrictUtf8(body, "preserve"); + return { bytes: body, json: mode === "json" }; + let text: string; + try { + text = decodeStrictUtf8(body, "preserve"); + } catch (cause: unknown) { + if (mode === "unknown") return { bytes: body, json: false }; + throw cause; + } const redacted = redactResponseSource( text, [], (value) => this.redact(value, scope), retention.maxBytes, retention.maxAllocationBytes, + mode === "unknown", ); if (!redacted.ok) throw new Error(redacted.error.message); - return redacted.value ?? body; + return { bytes: redacted.value.bytes ?? body, json: redacted.value.json }; } private retainSavedRedactions( diff --git a/src/browser/PlaywrightScenarioEvents.test.ts b/src/browser/PlaywrightScenarioEvents.test.ts index 217896c76a..b1f2423a7a 100644 --- a/src/browser/PlaywrightScenarioEvents.test.ts +++ b/src/browser/PlaywrightScenarioEvents.test.ts @@ -463,18 +463,21 @@ it("caps live descendant listeners and reuses capacity after closure", () => { expect(sockets[2048]?.on).toHaveBeenCalledTimes(3); }); +interface SourceTransactionCaptureOptions { + fail?: boolean; + followup?: boolean; + metadataOnly?: boolean; + learnDirect?: boolean; + latePageValue?: string; + value?: string; + padding?: number; + metadataLimit?: number; + requestContent?: { mediaType: string; body: string; typeSecret: string }; +} + /** Emit the request before attaching the source policy, matching Playwright listener order. */ const sourceTransactionCapture = async ( - options: { - fail?: boolean; - followup?: boolean; - metadataOnly?: boolean; - learnDirect?: boolean; - latePageValue?: string; - value?: string; - padding?: number; - metadataLimit?: number; - } = {}, + options: SourceTransactionCaptureOptions = {}, ) => { const listeners = new Map void>(); const context = { @@ -517,6 +520,14 @@ const sourceTransactionCapture = async ( }, }); const secrets = resolveScenarioSecrets(parsed, {}); + if (options.requestContent !== undefined) + secrets.save( + "earlier.media", + "media", + options.requestContent.typeSecret, + true, + ); + const mediaType = options.requestContent?.mediaType ?? "application/json"; const capture = new PlaywrightScenarioEvents({ page, context, @@ -531,15 +542,15 @@ const sourceTransactionCapture = async ( url: () => url, resourceType: () => "fetch", redirectedFrom: () => null, - postDataBuffer: () => Buffer.from(body), + postDataBuffer: () => Buffer.from(options.requestContent?.body ?? body), failure: () => null, sizes: async () => ({ responseBodySize: Buffer.byteLength(body) }), headers: () => ({ - "content-type": "application/json", + "content-type": mediaType, "x-source-token": value, }), headersArray: async () => [ - { name: "Content-Type", value: "application/json" }, + { name: "Content-Type", value: mediaType }, { name: "X-Source-Token", value }, ], }; @@ -610,6 +621,34 @@ const sourceTransactionCapture = async ( return { capture, sourceId, value, url }; }; +it("keeps byte redaction for a hidden binary declaration during the final privacy pass", async () => { + const typeSecret = "marked-media-parameter"; + const { capture, sourceId } = await sourceTransactionCapture({ + learnDirect: true, + value: "1000", + requestContent: { + mediaType: 'application/octet-stream; note="' + typeSecret + '"', + typeSecret, + body: '{"code":1e3,"literal":"1000"}', + }, + }); + const request = capture + .result() + .items.find( + (event) => + event.kind === "network-content" && + event.phase === "request" && + event.transaction_id === sourceId, + ); + if (request?.kind !== "network-content" || request.body.state !== "captured") + throw new Error("Expected captured request bytes"); + expect(request.body.media_type).toBeNull(); + expect(request.body.redacted).toBe(true); + const text = Buffer.from(request.body.content, "base64").toString("utf8"); + expect(text).toContain('"code":1e3'); + expect(text).not.toContain("1000"); +}); + it("redacts earlier source transaction metadata and request content before publishing", async () => { const { capture, sourceId, value } = await sourceTransactionCapture(); const source = capture diff --git a/src/browser/PlaywrightScenarioEvents.ts b/src/browser/PlaywrightScenarioEvents.ts index af7e6903c8..ff2118ad31 100644 --- a/src/browser/PlaywrightScenarioEvents.ts +++ b/src/browser/PlaywrightScenarioEvents.ts @@ -26,8 +26,13 @@ import type { } from "../domain/browserNetworkEvidence.js"; import type { BrowserScenarioCaptureLimits } from "../domain/browserScenarioCaptureLimits.js"; import { DEFAULT_BROWSER_SCENARIO_CAPTURE_LIMITS } from "../domain/browserScenarioCaptureLimits.js"; -import { captureNetworkBody } from "./network/PlaywrightNetworkContent.js"; +import { + captureNetworkBody, + type NetworkBodyRedaction, + type NetworkReadOptions, +} from "./network/PlaywrightNetworkContent.js"; import { PlaywrightScenarioNetwork } from "./network/PlaywrightScenarioNetwork.js"; +import { CONTENT_TYPE_HEADER } from "../domain/browserNetworkMediaType.js"; type EventName = | "console" @@ -101,6 +106,11 @@ type EventKind = BrowserScenarioEvent["kind"]; /** Arrival-ordered Playwright event capture with current-step attribution. */ export class PlaywrightScenarioEvents { private readonly items: BrowserScenarioEvent[] = []; + /** Keep only the observed redaction mode for retained bodies, outside Evidence. */ + private readonly networkBodyRedaction = new WeakMap< + BrowserScenarioEvent, + NetworkBodyRedaction + >(); private readonly retained: Budget; private dropped = 0; private readonly omittedKinds = new Map< @@ -156,7 +166,7 @@ export class PlaywrightScenarioEvents { selection: network, limits: this.limits, secrets: options.secrets, - emit: (event) => this.push(event), + emit: (event, bodyRedaction) => this.push(event, bodyRedaction), admitText: (kind, values) => this.admitProducerText(kind, values, true), admitContentBody: (input) => this.admitContentBody(input), omitNetwork: (id, kind, reason) => this.omitNetwork(id, kind, reason), @@ -515,31 +525,29 @@ export class PlaywrightScenarioEvents { }; if (body.state === "captured") { const prior = body; - const source = { - headers: (): Record => - prior.media_type === null - ? {} - : { "content-type": prior.media_type }, - headersArray: async () => [], + const readOptions: NetworkReadOptions = { + redactionScope: "response", + timeoutMs: 5000, + maxBodyBytes: this.limits.max_network_body_bytes, + maxBodyAllocationBytes: Math.min( + this.limits.max_network_body_allocation_bytes, + available, + ), + maxHeaderAllocationBytes: + this.limits.max_network_header_allocation_bytes, + maxMetadataAllocationBytes: + this.limits.max_network_metadata_allocation_bytes, }; const protectedBody = await captureNetworkBody( async () => Buffer.from(prior.content, "base64"), event.phase, - source, + () => ({ + value: prior.media_type, + redaction: this.networkBodyRedaction.get(event) ?? "unknown", + declarationNames: [CONTENT_TYPE_HEADER], + }), this.secrets, - { - redactionScope: "response", - timeoutMs: 5000, - maxBodyBytes: this.limits.max_network_body_bytes, - maxBodyAllocationBytes: Math.min( - this.limits.max_network_body_allocation_bytes, - available, - ), - maxHeaderAllocationBytes: - this.limits.max_network_header_allocation_bytes, - maxMetadataAllocationBytes: - this.limits.max_network_metadata_allocation_bytes, - }, + readOptions, ); body = protectedBody.state === "omitted" @@ -575,7 +583,10 @@ export class PlaywrightScenarioEvents { : { ...event, ...metadata }; } - private push(event: UnindexedEvent): BrowserScenarioEventAdmission { + private push( + event: UnindexedEvent, + bodyRedaction?: NetworkBodyRedaction, + ): BrowserScenarioEventAdmission { this.sequence += 1; const cost = allocationCost(event, this.limits.max_event_allocation_bytes) + 128; @@ -593,6 +604,8 @@ export class PlaywrightScenarioEvents { sequence: this.sequence, step_index: this.stepIndex, }); + if (bodyRedaction !== undefined) + this.networkBodyRedaction.set(parsed, bodyRedaction); this.items.push(parsed); return { status: "retained", sequence: this.sequence }; } @@ -667,6 +680,7 @@ export class PlaywrightScenarioEvents { bytes: input.observedBytes, sha256: "0".repeat(64), media_type: null, + media_type_source: "content-type-header", redacted: false, }, }, diff --git a/src/browser/network/PlaywrightNetworkContent.test.ts b/src/browser/network/PlaywrightNetworkContent.test.ts index aba688b179..ec93e53644 100644 --- a/src/browser/network/PlaywrightNetworkContent.test.ts +++ b/src/browser/network/PlaywrightNetworkContent.test.ts @@ -4,6 +4,7 @@ import { resolveScenarioSecrets } from "../../../tests/fixtures/browserScenarioS import { captureNetworkBody, captureNetworkHeaders, + networkBodyMediaType, } from "./PlaywrightNetworkContent.js"; const scenario = browserScenarioSchema.parse({ @@ -20,17 +21,15 @@ const options = { maxHeaderAllocationBytes: 16384, maxMetadataAllocationBytes: 4096, }; -const source = { - headers: () => ({ "content-type": "text/plain" }), - headersArray: async () => [], -}; +const textType = () => networkBodyMediaType("text/plain"); +const jsonType = () => networkBodyMediaType("application/json"); it("enforces the retained body byte limit after secret expansion", async () => { expect( await captureNetworkBody( async () => Buffer.from("xxxx"), "response", - source, + textType, secrets, options, ), @@ -43,7 +42,7 @@ it("enforces the retained body byte limit after secret expansion", async () => { await captureNetworkBody( async () => Buffer.from("safe"), "response", - source, + textType, secrets, options, ), @@ -56,17 +55,20 @@ it("enforces the retained body byte limit after secret expansion", async () => { it("charges discarded credential header values only for their retained projection", async () => { const read = async (name: string) => - captureNetworkHeaders( - { - headers: () => ({}), - headersArray: async () => [ - { name, value: "v".repeat(50_000) }, - { name: "X-Observed", value: "ordinary" }, - ], - }, - secrets, - options, - ); + ( + await captureNetworkHeaders( + { + headers: () => ({}), + headersArray: async () => [ + { name, value: "v".repeat(50_000) }, + { name: "X-Observed", value: "ordinary" }, + ], + }, + true, + secrets, + options, + ) + ).headers; expect(await read("Cookie")).toMatchObject({ state: "captured", items: [ @@ -85,12 +87,7 @@ it("redacts later JSON spellings of saved secrets without reserializing unmarked const captured = await captureNetworkBody( async () => Buffer.from(body), "response", - { - headers: () => ({ - "content-type": "application/problem+json; charset=utf-8", - }), - headersArray: async () => [], - }, + jsonType, held, { ...options, maxBodyBytes: 4096 }, ); @@ -109,10 +106,7 @@ it("keeps an empty JSON-labelled response as observed empty bytes", async () => const captured = await captureNetworkBody( async () => Buffer.alloc(0), "response", - { - headers: () => ({ "content-type": "application/json" }), - headersArray: async () => [], - }, + jsonType, secrets, options, ); @@ -125,10 +119,7 @@ it("checks the JSON decoding budget before invoking its decoder", async () => { const captured = await captureNetworkBody( async () => Buffer.from('{"value":"untouched"}'), "response", - { - headers: () => ({ "content-type": "application/json" }), - headersArray: async () => [], - }, + jsonType, held, { ...options, maxBodyBytes: 4096, maxBodyAllocationBytes: 1 }, ); @@ -147,10 +138,7 @@ it.each(['{"token":"private-token-729d"}', 'A","B'])( const captured = await captureNetworkBody( async () => Buffer.from(body), "request", - { - headers: () => ({ "content-type": "application/json" }), - headersArray: async () => [], - }, + jsonType, held, { ...options, maxBodyBytes: 4096 }, ); diff --git a/src/browser/network/PlaywrightNetworkContent.ts b/src/browser/network/PlaywrightNetworkContent.ts index 71f42111d6..055a9d7d5c 100644 --- a/src/browser/network/PlaywrightNetworkContent.ts +++ b/src/browser/network/PlaywrightNetworkContent.ts @@ -5,6 +5,10 @@ import { mediaTypeEssence, } from "../../domain/analysisView/recordedMediaType.js"; import { safeParseJson } from "../../domain/safeJson.js"; +import { + browserContentType, + CONTENT_TYPE_HEADER, +} from "../../domain/browserNetworkMediaType.js"; import { AnalysisCancelledError, @@ -71,12 +75,67 @@ const errorMessage = ( : "Network read failed; diagnostic exceeds the allocation budget."; }; +export type NetworkBodyRedaction = "json" | "unknown" | "bytes"; + +interface BodyMediaType { + readonly value: string | null; + readonly redaction: NetworkBodyRedaction; + readonly declarationNames: readonly string[]; +} + +interface NetworkHeaderCapture { + readonly headers: BrowserNetworkHeaders; + readonly mediaType: BodyMediaType; +} + +export const networkBodyMediaType = ( + value: string | null, + declarationNames: readonly string[] = [CONTENT_TYPE_HEADER], +): BodyMediaType => ({ + value, + declarationNames, + redaction: + value === null + ? "unknown" + : isJson(mediaTypeEssence(value)) + ? "json" + : "bytes", +}); + +/** Publish only the type that can still be read after privacy projection. */ +const retainNetworkMediaType = ( + { value, declarationNames }: BodyMediaType, + secrets: BrowserScenarioSecrets, + options: NetworkReadOptions, +): string | null => { + if (value === null) return null; + if ( + secrets.redactionAllocationCost(value.length, options.redactionScope) > + options.maxHeaderAllocationBytes || + declarationNames.some( + (name) => secrets.redact(name, options.redactionScope) !== name, + ) || + secrets + .redactBytes(Buffer.from(value), options.redactionScope) + .toString("utf8") !== value + ) + return null; + return value; +}; + /** Read actual producer headers without flattening duplicates or exposing credentials. */ export const captureNetworkHeaders = async ( source: NetworkHeadersSource, + retainValues: boolean, secrets: BrowserScenarioSecrets, options: NetworkReadOptions, -): Promise => { +): Promise => { + const unavailable = (message: string): NetworkHeaderCapture => ({ + headers: retainValues + ? { state: "unavailable", message } + : { state: "not_requested" }, + mediaType: networkBodyMediaType(null), + }); try { const headers = await withPlaywrightExecutionBoundary( () => source.headersArray(), @@ -88,7 +147,12 @@ export const captureNetworkHeaders = async ( BrowserNetworkHeaders, { state: "captured" } >["items"] = []; + const declarations: string[] = []; + const declarationNames: string[] = []; + let typeRedacted = false; for (const { name, value } of headers) { + const contentType = name.toLowerCase() === CONTENT_TYPE_HEADER; + if (!retainValues && !contentType) continue; const credential = isTransportCredentialHeader(name); cost += secrets.redactionAllocationCost( @@ -96,33 +160,44 @@ export const captureNetworkHeaders = async ( options.redactionScope, ) + 128; if (cost > options.maxHeaderAllocationBytes) - return { - state: "unavailable", - message: `Headers exceed the ${String(options.maxHeaderAllocationBytes)}-byte allocation budget.`, - }; + return unavailable( + `Headers exceed the ${String(options.maxHeaderAllocationBytes)}-byte allocation budget.`, + ); + if (contentType) { + declarationNames.push(name); + declarations.push(value); + } + if (!retainValues) continue; const safeName = secrets.redact(name, options.redactionScope); const safeValue = credential ? null : secrets .redactBytes(Buffer.from(value), options.redactionScope) .toString("utf8"); - items.push({ - name: safeName, - value: safeValue, - redacted: credential || safeName !== name || safeValue !== value, - }); + const redacted = credential || safeName !== name || safeValue !== value; + items.push({ name: safeName, value: safeValue, redacted }); + if (contentType) typeRedacted ||= redacted; } - return { state: "captured", items }; - } catch (cause: unknown) { + const observed = browserContentType(declarations); + const mediaType = networkBodyMediaType( + observed.state === "known" ? observed.value : null, + declarationNames, + ); return { - state: "unavailable", - message: errorMessage( + headers: retainValues + ? { state: "captured", items } + : { state: "not_requested" }, + mediaType: typeRedacted ? { ...mediaType, value: null } : mediaType, + }; + } catch (cause: unknown) { + return unavailable( + errorMessage( cause, secrets, options.maxMetadataAllocationBytes, options.redactionScope, ), - }; + ); } }; @@ -130,7 +205,7 @@ export const captureNetworkHeaders = async ( export const captureNetworkBody = async ( read: () => Promise, phase: "request" | "response", - source: NetworkHeadersSource, + getMediaType: () => BodyMediaType | Promise, secrets: BrowserScenarioSecrets, options: NetworkReadOptions, preflight?: NetworkBodyPreflight, @@ -168,7 +243,6 @@ export const captureNetworkBody = async ( ); if (!protection.retainBody) return { state: "not_requested" }; } - const contentType = source.headers()["content-type"]; const bodyAllocationCost = secrets.redactionAllocationCost( Math.max(observed.length, sourceBody.length), options.redactionScope, @@ -189,43 +263,28 @@ export const captureNetworkBody = async ( limit_allocation_bytes: maxBodyAllocationBytes, observed_allocation_bytes: observedBodyAllocationBytes, }; - if ( - contentType !== undefined && - secrets.redactionAllocationCost( - contentType.length, - options.redactionScope, - ) > options.maxHeaderAllocationBytes - ) - return { - state: "unavailable", - reason: "body-allocation-limit", - message: - "Content type exceeds the selected network header allocation budget.", - limit_allocation_bytes: options.maxHeaderAllocationBytes, - observed_allocation_bytes: secrets.redactionAllocationCost( - contentType.length, - options.redactionScope, - ), - }; - const jsonBody = - protection !== undefined || - (contentType !== undefined && isJson(mediaTypeEssence(contentType))); + const mediaType = await getMediaType(); const admission = await preflight?.({ observedBytes: observed.length, allocationBytes: observedBodyAllocationBytes, - mediaType: contentType, + mediaType: mediaType.value ?? undefined, }); if (admission?.status === "omitted") return { state: "omitted", reason: admission.reason }; - if (protection === undefined && jsonBody) - sourceBody = secrets.redactJsonBody( + let jsonBody = protection !== undefined; + if (protection === undefined && mediaType.redaction !== "bytes") { + const redacted = secrets.redactJsonBody( observed, { maxBytes: options.maxBodyBytes, maxAllocationBytes: options.maxBodyAllocationBytes, }, options.redactionScope, + mediaType.redaction, ); + sourceBody = redacted.bytes; + jsonBody = redacted.json; + } const retained = secrets.redactBytes(sourceBody, options.redactionScope); if ( jsonBody && @@ -257,12 +316,8 @@ export const captureNetworkBody = async ( content: retained.toString("base64"), bytes: retained.length, sha256: sha256Hex(retained), - media_type: - contentType === undefined - ? null - : secrets - .redactBytes(Buffer.from(contentType), options.redactionScope) - .toString("utf8"), + media_type: retainNetworkMediaType(mediaType, secrets, options), + media_type_source: "content-type-header", redacted: !retained.equals(observed), }; } catch (cause: unknown) { diff --git a/src/browser/network/PlaywrightScenarioNetwork.test.ts b/src/browser/network/PlaywrightScenarioNetwork.test.ts index fedc66d8bb..fb9859c438 100644 --- a/src/browser/network/PlaywrightScenarioNetwork.test.ts +++ b/src/browser/network/PlaywrightScenarioNetwork.test.ts @@ -567,7 +567,7 @@ it.each([ response_body: false, header_values: false, }, - reads: [1, 0, 0, 0], + reads: [1, 1, 0, 0], phases: ["request"], }, { @@ -576,7 +576,7 @@ it.each([ response_body: true, header_values: false, }, - reads: [0, 0, 1, 0], + reads: [0, 0, 1, 1], phases: ["response"], }, { @@ -589,7 +589,7 @@ it.each([ phases: ["request", "response"], }, ])( - "reads only the selected content $selection", + "reads selected bodies and the headers needed for their type $selection", async ({ selection: selected, reads, phases }) => { const { collector, events } = harness(selected); const request = new RequestFixture("input"); @@ -609,6 +609,12 @@ it.each([ .filter((event) => event.kind === "network-content") .map((event) => event.phase), ).toEqual(phases); + if (!selected.header_values) + expect( + events + .filter((event) => event.kind === "network-content") + .map(({ headers }) => headers), + ).toEqual([{ state: "not_requested" }]); }, ); @@ -985,19 +991,20 @@ it("preserves read errors and does not turn unexposed or empty bytes into the sa expect(collector.limitations().length).toBeGreaterThan(0); }); -it("redacts declared encodings in body media metadata", async () => { +it("keeps a redacted declaration out of body media metadata", async () => { const { collector, events } = harness({ request_body: false, response_body: true, - header_values: false, + header_values: true, }); const request = new RequestFixture("input"); const response = new ResponseFixture(request, "ordinary-bytes"); - const headers = response.headers(); - response.headers = () => ({ - ...headers, - "content-type": `text/plain; fixture=${encodeURIComponent("私密")}`, - }); + response.headersArray = async () => [ + { + name: "Content-Type", + value: `text/plain; fixture=${encodeURIComponent("私密")}`, + }, + ]; collector.request(request); collector.response(response); collector.finished(request); @@ -1006,14 +1013,94 @@ it("redacts declared encodings in body media metadata", async () => { expect.objectContaining({ kind: "network-content", phase: "response", + headers: { + state: "captured", + items: [ + { + name: "Content-Type", + value: "text/plain; fixture=[REDACTED:declared]", + redacted: true, + }, + ], + }, body: expect.objectContaining({ state: "captured", - media_type: "text/plain; fixture=[REDACTED:declared]", + media_type: null, }), }), ); }); +it.each([ + { + declarations: [ + "application/json; charset=utf-8", + "application/octet-stream", + ], + type: null, + }, + { + declarations: [ + "application/octet-stream", + "application/json; charset=utf-8", + ], + type: null, + }, + { + declarations: ["application/json; charset=utf-8, application/octet-stream"], + type: null, + }, + { + declarations: ["application/json", "application/json"], + type: "application/json", + }, + { + declarations: ['application/json; note="one,two"'], + type: 'application/json; note="one,two"', + }, + { declarations: null, type: null }, +])( + "derives body type once from retained declarations $declarations", + async ({ declarations, type }) => { + const { collector, events } = harness({ + request_body: false, + response_body: true, + header_values: false, + }); + const request = new RequestFixture("input"); + const response = new ResponseFixture(request, '{"value":"\\u79c1\\u5bc6"}'); + response.headers = () => ({ + "content-type": declarations?.join(", ") ?? "application/json", + }); + response.headersArray = async () => { + response.headerReads += 1; + if (declarations === null) throw new Error("Headers unavailable"); + return declarations.map((value) => ({ name: "Content-Type", value })); + }; + collector.request(request); + collector.response(response); + collector.finished(request); + await collector.finish(); + const content = events.find( + (event) => event.kind === "network-content" && event.phase === "response", + ); + expect(response.headerReads).toBe(1); + expect(content).toMatchObject({ + headers: { state: "not_requested" }, + body: { + state: "captured", + media_type: type, + media_type_source: "content-type-header", + }, + }); + if (content?.kind !== "network-content") + throw new Error("Missing response content"); + expect(JSON.parse(bodyText(content))).toEqual({ + value: "[REDACTED:declared]", + }); + }, +); + it("bounds admitted transactions before invoking content producers", async () => { const { collector, events } = harness(); const admitted: RequestFixture[] = []; diff --git a/src/browser/network/PlaywrightScenarioNetwork.ts b/src/browser/network/PlaywrightScenarioNetwork.ts index 4c39ac47dd..37655f3f05 100644 --- a/src/browser/network/PlaywrightScenarioNetwork.ts +++ b/src/browser/network/PlaywrightScenarioNetwork.ts @@ -17,6 +17,7 @@ import { } from "../BrowserScenarioResponseValues.js"; import { captureNetworkBody, + type NetworkBodyRedaction, captureNetworkHeaders, type NetworkBodyCapture, type NetworkHeadersSource, @@ -100,6 +101,7 @@ interface NetworkCaptureOptions { readonly secrets: BrowserScenarioSecrets; readonly emit: ( event: UnindexedNetworkEvent, + bodyRedaction?: NetworkBodyRedaction, ) => BrowserScenarioEventAdmission; /** Check aggregate event capacity before redacting producer-owned metadata. */ readonly admitText: ( @@ -647,11 +649,14 @@ export class PlaywrightScenarioNetwork { const readOptions = privacyOnly ? { ...this.readOptions, maxBodyBytes: MAX_SAVED_RESPONSE_BYTES } : this.readOptions; - let headers: Promise | undefined; - const getHeaders = (): Promise => - (headers ??= includeHeaders - ? captureNetworkHeaders(source, this.options.secrets, readOptions) - : Promise.resolve({ state: "not_requested" })); + let headers: ReturnType | undefined; + const getHeaders = () => + (headers ??= captureNetworkHeaders( + source, + includeHeaders, + this.options.secrets, + readOptions, + )); // Existing reads keep their order. Marked responses establish secret // spellings before either their headers or their body can be retained. if (!protectedResponse && includeHeaders) await getHeaders(); @@ -673,7 +678,7 @@ export class PlaywrightScenarioNetwork { return bounded.value; }, phase, - source, + async () => (await getHeaders()).mediaType, this.options.secrets, readOptions, async (body) => @@ -681,7 +686,7 @@ export class PlaywrightScenarioNetwork { transactionId: transaction.id, sourceEventSequence: sequence, phase, - headers: await getHeaders(), + headers: (await getHeaders()).headers, observedBytes: body.observedBytes, allocationBytes: body.allocationBytes, mediaType: body.mediaType, @@ -708,7 +713,9 @@ export class PlaywrightScenarioNetwork { message: "Response header values were withheld because marked source values could not be read safely.", } - : await getHeaders(); + : includeHeaders + ? (await getHeaders()).headers + : { state: "not_requested" }; const retainedBody = includeBody ? observedBody : { state: "not_requested" as const }; @@ -723,14 +730,21 @@ export class PlaywrightScenarioNetwork { this.gaps.add( "Some selected network content was unavailable; per-transaction content states retain the reason.", ); - const admission = this.options.emit({ - kind: "network-content", - transaction_id: transaction.id, - source_event_sequence: sequence, - phase, - headers: retainedHeaders, - body: retainedBody, - }); + const bodyRedaction = + retainedBody.state === "captured" + ? (await getHeaders()).mediaType.redaction + : undefined; + const admission = this.options.emit( + { + kind: "network-content", + transaction_id: transaction.id, + source_event_sequence: sequence, + phase, + headers: retainedHeaders, + body: retainedBody, + }, + bodyRedaction, + ); if (admission.status === "omitted") this.options.omitNetwork( transaction.id, diff --git a/src/domain/analysisView/scenarioCaptureRecords.test.ts b/src/domain/analysisView/scenarioCaptureRecords.test.ts index b0cf3f96f5..204335dd41 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.test.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.test.ts @@ -104,7 +104,23 @@ describe("scenario Content-Type evidence", () => { ], }, ]); - const projected = scenarioCaptureRecords(capture, "manifest"); + const projected = scenarioCaptureRecords( + { + ...capture, + events: { + ...capture.events, + items: capture.events.items.map((event) => + event.kind === "network-content" && event.body.state === "captured" + ? { + ...event, + body: { ...event.body, media_type_source: "independent" }, + } + : event, + ), + }, + }, + "manifest", + ); expect(projected.records).toHaveLength(2); for (const record of projected.records) expect(record.reported).toMatchObject({ @@ -113,6 +129,61 @@ describe("scenario Content-Type evidence", () => { }); }); +it("uses retained declarations to correct legacy body metadata and honors new explicit uncertainty", () => { + const capture = scenarioCaptureFixture([ + { + method: "GET", + url: "/legacy-conflict", + status: 200, + body: "{}", + mediaType: "application/json; charset=utf-8, application/octet-stream", + responseHeaders: [ + { name: "Content-Type", value: "application/json; charset=utf-8" }, + { name: "content-type", value: "application/octet-stream" }, + ], + }, + { + method: "GET", + url: "/legacy-agreement", + status: 200, + body: "{}", + mediaType: "application/json, application/json", + responseHeaders: [ + { name: "Content-Type", value: "application/json" }, + { name: "content-type", value: "application/json" }, + ], + }, + { + method: "GET", + url: "/legacy-coalesced-without-headers", + status: 200, + body: "{}", + mediaType: "application/json; charset=utf-8, application/octet-stream", + }, + ]); + const content = capture.events.items.filter( + (event) => event.kind === "network-content", + ); + expect(content.map(scenarioContentMediaType)).toEqual([ + undefined, + "application/json", + undefined, + ]); + const later = content[1]; + if (later?.body.state !== "captured") + throw new Error("Missing captured body"); + expect( + scenarioContentMediaType({ + ...later, + body: { + ...later.body, + media_type: null, + media_type_source: "content-type-header", + }, + }), + ).toBeUndefined(); +}); + describe("scenarioCaptureRecords", () => { it("projects each transaction, in request order, as a HAR-shaped record with its retained bytes", () => { const projected = scenarioCaptureRecords( diff --git a/src/domain/analysisView/scenarioCaptureRecords.ts b/src/domain/analysisView/scenarioCaptureRecords.ts index 8cf1388a04..7a8a17da6f 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.ts @@ -1,9 +1,7 @@ import { AnalysisUnsupportedTargetError } from "../analysisErrorCore.js"; import type { AnalysisError } from "../analysisErrorBase.js"; -import type { - BrowserNetworkBody, - BrowserNetworkHeaders, -} from "../browserNetworkEvidence.js"; +import type { BrowserNetworkBody } from "../browserNetworkEvidence.js"; +import { retainedNetworkMediaType } from "../browserNetworkMediaType.js"; import { retainedBrowserScenarioCaptureSchema, type RetainedBrowserScenarioCapture, @@ -47,23 +45,6 @@ const onlyContent = ( return found?.length === 1 ? found[0] : undefined; }; -const headerValue = ( - headers: BrowserNetworkHeaders | undefined, - name: string, -): string | undefined => { - if (headers?.state !== "captured") return undefined; - let value: string | undefined; - for (const item of headers.items) { - if (item.name.toLowerCase() !== name) continue; - // A singleton field has no certain value if declarations disagree or one - // was excluded. Never let ordering choose an interpretation of Evidence. - if (item.value === null || (value !== undefined && value !== item.value)) - return undefined; - value = item.value; - } - return value; -}; - const captured = ( content: ContentEvent | undefined, ): CapturedBody | undefined => @@ -88,12 +69,11 @@ const utf8Text = (body: CapturedBody): string | undefined => { } }; -/** A content event's media type: its captured body's, else its retained Content-Type value. */ +/** Use the retained type fact for the content event. */ export const scenarioContentMediaType = ( content: ContentEvent | undefined, ): string | undefined => - captured(content)?.media_type ?? - headerValue(content?.headers, "content-type"); + retainedNetworkMediaType(content?.body, content?.headers); /** Whether a request sent no body: Playwright exposed none and the request names no Content-Type, which the bodies browsers send carry. */ const unsentBody = ( diff --git a/src/domain/browserNetworkEvidence.ts b/src/domain/browserNetworkEvidence.ts index 9e330409c0..9c101896c5 100644 --- a/src/domain/browserNetworkEvidence.ts +++ b/src/domain/browserNetworkEvidence.ts @@ -74,6 +74,9 @@ export const browserNetworkBodySchema = z.union([ bytes: z.int().min(0), sha256: z.string().regex(/^[a-f0-9]{64}$/u), media_type: z.string().nullable(), + media_type_source: z + .enum(["content-type-header", "independent"]) + .optional(), redacted: z.boolean(), }) .superRefine((artifact, context) => { diff --git a/src/domain/browserNetworkMediaType.ts b/src/domain/browserNetworkMediaType.ts new file mode 100644 index 0000000000..eac7ebf21b --- /dev/null +++ b/src/domain/browserNetworkMediaType.ts @@ -0,0 +1,68 @@ +import { mediaTypeEssence } from "./analysisView/recordedMediaType.js"; +import type { + BrowserNetworkBody, + BrowserNetworkHeaders, +} from "./browserNetworkEvidence.js"; + +export const CONTENT_TYPE_HEADER = "content-type"; + +/** An HTTP singleton media type can contain commas only inside a quoted parameter. */ +const isSingleMediaType = (value: string): boolean => { + if (/[\r\n]/u.test(value) || mediaTypeEssence(value) === null) return false; + let quoted = false; + for (let index = 0; index < value.length; index += 1) { + const character = value[index]; + if (character === '"') quoted = !quoted; + else if (quoted && character === "\\") index += 1; + else if (!quoted && character === ",") return false; + } + return !quoted; +}; + +export type BrowserContentType = + | { readonly state: "absent" | "uncertain" } + | { readonly state: "known"; readonly value: string }; + +/** Interpret declarations without selecting an arbitrary occurrence or a masked value. */ +export const browserContentType = ( + declarations: Iterable, +): BrowserContentType => { + let value: string | undefined; + for (const declaration of declarations) { + if ( + declaration === null || + (value === undefined + ? !isSingleMediaType(declaration) + : value !== declaration) + ) + return { state: "uncertain" }; + value = declaration; + } + return value === undefined ? { state: "absent" } : { state: "known", value }; +}; + +function* retainedDeclarations( + items: Extract["items"], +): Generator { + for (const item of items) + if (item.name.toLowerCase() === CONTENT_TYPE_HEADER) + yield item.redacted ? null : item.value; +} + +/** Project a retained type without inferring certainty from a legacy flattened header. */ +export const retainedNetworkMediaType = ( + body: BrowserNetworkBody | undefined, + headers: BrowserNetworkHeaders | undefined, +): string | undefined => { + const captured = body?.state === "captured" ? body : undefined; + // Newly retained body metadata includes the provenance of an explicit unknown. + if (captured?.media_type_source !== undefined) + return captured.media_type ?? undefined; + if (headers?.state === "captured") { + const type = browserContentType(retainedDeclarations(headers.items)); + if (type.state === "known") return type.value; + if (type.state === "uncertain") return undefined; + } + const legacy = captured?.media_type; + return legacy != null && isSingleMediaType(legacy) ? legacy : undefined; +}; diff --git a/src/domain/scenarioScriptCapture.ts b/src/domain/scenarioScriptCapture.ts index 7ab02a78c0..26d054571d 100644 --- a/src/domain/scenarioScriptCapture.ts +++ b/src/domain/scenarioScriptCapture.ts @@ -1,6 +1,6 @@ import type { BrowserScenarioCapture } from "./browserScenarioCapture.js"; import type { BrowserScenarioEvent } from "./browserScenarioCaptureValues.js"; -import type { BrowserNetworkBody } from "./browserNetworkEvidence.js"; +import { retainedNetworkMediaType } from "./browserNetworkMediaType.js"; import type { CapturedWebScript } from "./webScriptExport.js"; import { indexBrowserScenarioTransactions, @@ -8,6 +8,7 @@ import { } from "./browserScenarioTransactions.js"; type RequestEvent = Extract; +type ContentEvent = Extract; /** Join script response bytes only through explicit transaction and event identity. */ export const scenarioScripts = ( @@ -34,7 +35,7 @@ const projectRequest = ( : (index.contents.get(response.sequence) ?? []).filter( ({ phase }) => phase === "response", ); - const body = bodies.length === 1 ? bodies[0]?.body : undefined; + const content = bodies.length === 1 ? bodies[0] : undefined; const ambiguous = index.requestCounts.get(id) !== 1 || matches.length > 1 || @@ -48,7 +49,7 @@ const projectRequest = ( status: ambiguous ? null : (response?.status ?? null), }, url: request.url.url, - content: projectBody(body, ambiguous), + content: projectBody(content, ambiguous), }; }; @@ -58,7 +59,7 @@ const unavailable = ( ): CapturedWebScript["content"] => ({ state: "unavailable", reason, message }); const projectBody = ( - body: BrowserNetworkBody | undefined, + content: ContentEvent | undefined, ambiguous: boolean, ): CapturedWebScript["content"] => { if (ambiguous) @@ -66,11 +67,12 @@ const projectBody = ( "ambiguous-transaction", "The capture does not identify one request, response, and content record for this transaction.", ); - if (body === undefined) + if (content === undefined) return unavailable( "response-content-not-retained", "No selected, completed script response bytes were retained for this request.", ); + const { body, headers } = content; if (body.state === "unavailable") return unavailable(body.reason, body.message); if (body.state !== "captured") @@ -90,7 +92,7 @@ const projectBody = ( state: "captured", bytes: Buffer.from(body.content, "base64"), sha256: body.sha256, - media_type: body.media_type, + media_type: retainedNetworkMediaType(body, headers) ?? null, redacted: body.redacted, representation: body.representation, };