fix(browser): derive body media types from observed headers (#12159)

* fix(browser): derive scenario media types from observed headers

Issue #12124

* fix(browser): retain body redaction mode through privacy replay
This commit is contained in:
morluto authored and GitHub committed 2026-11-19 03:31:13 +00:00
1 parent 5dc54183f2
commit 261c6d06d1
14 files changed
+533 -189

No files matched your search

+12 -3
View File
@@ -696,12 +696,21 @@ request URL when CDP has no response body. Event and transaction references
identify the associated Playwright response object, but do not verify that the
retained bytes came from the original network transaction. Successful selected
response bodies include this limitation in the capture. These bytes are not
the compressed wire representation.
the compressed wire representation. Playwright body media types come from
the duplicate-preserving header read and carry `media_type_source` set to
`content-type-header`. Identical repeated declarations retain one value.
Conflicting, unavailable or redacted declarations leave the type null.
Selecting a body allows that lookup even when full header values are omitted
from Evidence. Views use any retained type declarations to interpret captures
that predate `media_type_source`. Earlier REA versions can reject captures
that include the new field.
Headers preserve producer order and duplicates. Authorization,
Proxy-Authorization, Cookie, and Set-Cookie values are null with
`redacted: true`; ordinary values remain intact. Declared UTF-8 secret values
and their standard URI/form or JSON-escaped representations are replaced without
decoding unrelated binary bytes. Other encodings are not interpreted. Digests describe the
and their standard URI/form or JSON-escaped representations are replaced.
Known binary bodies are redacted as bytes. For an unknown media type, a JSON
redaction pass can remove decoded secret spellings without assigning a type.
Other encodings are not interpreted. Digests describe the
retained representation after redaction.
The selected body-byte limit also bounds that retained representation after
secret expansion, before base64 serialization. Credential header values are
+10 -4
View File
@@ -177,7 +177,11 @@ export const redactResponseSource = (
redact: (value: string) => string,
maxBytes: number,
maxAllocationBytes: number,
): Result<Buffer | undefined, ResponseValueIssue> => {
acceptNonJson = false,
): Result<
{ readonly json: boolean; readonly bytes?: Buffer },
ResponseValueIssue
> => {
const markers = new Map(
values.map(({ entry, value }) => [
JSON.stringify(jsonPointerTokens(entry.json_pointer)),
@@ -213,6 +217,7 @@ export const redactResponseSource = (
"A marked source field has conflicting scalar/container representations; its source content was withheld.",
);
};
const malformedJson = new Error("The marked source is not valid JSON.");
try {
visit(
text,
@@ -249,15 +254,16 @@ export const redactResponseSource = (
if (redacted !== property) replace(offset, length, redacted);
},
onError: () => {
throw new Error("The marked source is not valid JSON.");
throw malformedJson;
},
},
{ disallowComments: true, allowTrailingComma: false },
);
if (parts.length === 0) return ok(undefined);
if (parts.length === 0) return ok({ json: true });
parts.push(text.slice(cursor));
return ok(Buffer.from(parts.join("")));
return ok({ json: true, bytes: Buffer.from(parts.join("")) });
} catch (cause: unknown) {
if (acceptNonJson && cause === malformedJson) return ok({ json: false });
return err(
issue(
cause instanceof Error
+14 -6
View File
@@ -566,10 +566,10 @@ export class BrowserScenarioSecrets {
retention.maxAllocationBytes,
);
if (!redacted.ok) throw new Error(redacted.error.message);
return redacted.value ?? body;
return redacted.value.bytes ?? body;
}
/** Decode JSON only when redaction is active; other bytes retain their existing binary policy. */
/** Check JSON spellings only while redaction is active, including bodies of unknown type. */
redactJsonBody(
body: Buffer,
retention: {
@@ -577,7 +577,8 @@ export class BrowserScenarioSecrets {
readonly maxAllocationBytes: number;
},
scope: ScenarioRedactionScope = "current",
): Buffer {
mode: "json" | "unknown" = "json",
): { readonly bytes: Buffer; readonly json: boolean } {
if (
body.length === 0 ||
(scope === "response"
@@ -585,17 +586,24 @@ export class BrowserScenarioSecrets {
: this.textPatterns
).length === 0
)
return body;
const text = decodeStrictUtf8(body, "preserve");
return { bytes: body, json: mode === "json" };
let text: string;
try {
text = decodeStrictUtf8(body, "preserve");
} catch (cause: unknown) {
if (mode === "unknown") return { bytes: body, json: false };
throw cause;
}
const redacted = redactResponseSource(
text,
[],
(value) => this.redact(value, scope),
retention.maxBytes,
retention.maxAllocationBytes,
mode === "unknown",
);
if (!redacted.ok) throw new Error(redacted.error.message);
return redacted.value ?? body;
return { bytes: redacted.value.bytes ?? body, json: redacted.value.json };
}
private retainSavedRedactions(
+52 -13
View File
@@ -463,18 +463,21 @@ it("caps live descendant listeners and reuses capacity after closure", () => {
expect(sockets[2048]?.on).toHaveBeenCalledTimes(3);
});
interface SourceTransactionCaptureOptions {
fail?: boolean;
followup?: boolean;
metadataOnly?: boolean;
learnDirect?: boolean;
latePageValue?: string;
value?: string;
padding?: number;
metadataLimit?: number;
requestContent?: { mediaType: string; body: string; typeSecret: string };
}
/** Emit the request before attaching the source policy, matching Playwright listener order. */
const sourceTransactionCapture = async (
options: {
fail?: boolean;
followup?: boolean;
metadataOnly?: boolean;
learnDirect?: boolean;
latePageValue?: string;
value?: string;
padding?: number;
metadataLimit?: number;
} = {},
options: SourceTransactionCaptureOptions = {},
) => {
const listeners = new Map<string, (value: unknown) => void>();
const context = {
@@ -517,6 +520,14 @@ const sourceTransactionCapture = async (
},
});
const secrets = resolveScenarioSecrets(parsed, {});
if (options.requestContent !== undefined)
secrets.save(
"earlier.media",
"media",
options.requestContent.typeSecret,
true,
);
const mediaType = options.requestContent?.mediaType ?? "application/json";
const capture = new PlaywrightScenarioEvents({
page,
context,
@@ -531,15 +542,15 @@ const sourceTransactionCapture = async (
url: () => url,
resourceType: () => "fetch",
redirectedFrom: () => null,
postDataBuffer: () => Buffer.from(body),
postDataBuffer: () => Buffer.from(options.requestContent?.body ?? body),
failure: () => null,
sizes: async () => ({ responseBodySize: Buffer.byteLength(body) }),
headers: () => ({
"content-type": "application/json",
"content-type": mediaType,
"x-source-token": value,
}),
headersArray: async () => [
{ name: "Content-Type", value: "application/json" },
{ name: "Content-Type", value: mediaType },
{ name: "X-Source-Token", value },
],
};
@@ -610,6 +621,34 @@ const sourceTransactionCapture = async (
return { capture, sourceId, value, url };
};
it("keeps byte redaction for a hidden binary declaration during the final privacy pass", async () => {
const typeSecret = "marked-media-parameter";
const { capture, sourceId } = await sourceTransactionCapture({
learnDirect: true,
value: "1000",
requestContent: {
mediaType: 'application/octet-stream; note="' + typeSecret + '"',
typeSecret,
body: '{"code":1e3,"literal":"1000"}',
},
});
const request = capture
.result()
.items.find(
(event) =>
event.kind === "network-content" &&
event.phase === "request" &&
event.transaction_id === sourceId,
);
if (request?.kind !== "network-content" || request.body.state !== "captured")
throw new Error("Expected captured request bytes");
expect(request.body.media_type).toBeNull();
expect(request.body.redacted).toBe(true);
const text = Buffer.from(request.body.content, "base64").toString("utf8");
expect(text).toContain('"code":1e3');
expect(text).not.toContain("1000");
});
it("redacts earlier source transaction metadata and request content before publishing", async () => {
const { capture, sourceId, value } = await sourceTransactionCapture();
const source = capture
+37 -23
View File
@@ -26,8 +26,13 @@ import type {
} from "../domain/browserNetworkEvidence.js";
import type { BrowserScenarioCaptureLimits } from "../domain/browserScenarioCaptureLimits.js";
import { DEFAULT_BROWSER_SCENARIO_CAPTURE_LIMITS } from "../domain/browserScenarioCaptureLimits.js";
import { captureNetworkBody } from "./network/PlaywrightNetworkContent.js";
import {
captureNetworkBody,
type NetworkBodyRedaction,
type NetworkReadOptions,
} from "./network/PlaywrightNetworkContent.js";
import { PlaywrightScenarioNetwork } from "./network/PlaywrightScenarioNetwork.js";
import { CONTENT_TYPE_HEADER } from "../domain/browserNetworkMediaType.js";
type EventName =
| "console"
@@ -101,6 +106,11 @@ type EventKind = BrowserScenarioEvent["kind"];
/** Arrival-ordered Playwright event capture with current-step attribution. */
export class PlaywrightScenarioEvents {
private readonly items: BrowserScenarioEvent[] = [];
/** Keep only the observed redaction mode for retained bodies, outside Evidence. */
private readonly networkBodyRedaction = new WeakMap<
BrowserScenarioEvent,
NetworkBodyRedaction
>();
private readonly retained: Budget;
private dropped = 0;
private readonly omittedKinds = new Map<
@@ -156,7 +166,7 @@ export class PlaywrightScenarioEvents {
selection: network,
limits: this.limits,
secrets: options.secrets,
emit: (event) => this.push(event),
emit: (event, bodyRedaction) => this.push(event, bodyRedaction),
admitText: (kind, values) => this.admitProducerText(kind, values, true),
admitContentBody: (input) => this.admitContentBody(input),
omitNetwork: (id, kind, reason) => this.omitNetwork(id, kind, reason),
@@ -515,31 +525,29 @@ export class PlaywrightScenarioEvents {
};
if (body.state === "captured") {
const prior = body;
const source = {
headers: (): Record<string, string> =>
prior.media_type === null
? {}
: { "content-type": prior.media_type },
headersArray: async () => [],
const readOptions: NetworkReadOptions = {
redactionScope: "response",
timeoutMs: 5000,
maxBodyBytes: this.limits.max_network_body_bytes,
maxBodyAllocationBytes: Math.min(
this.limits.max_network_body_allocation_bytes,
available,
),
maxHeaderAllocationBytes:
this.limits.max_network_header_allocation_bytes,
maxMetadataAllocationBytes:
this.limits.max_network_metadata_allocation_bytes,
};
const protectedBody = await captureNetworkBody(
async () => Buffer.from(prior.content, "base64"),
event.phase,
source,
() => ({
value: prior.media_type,
redaction: this.networkBodyRedaction.get(event) ?? "unknown",
declarationNames: [CONTENT_TYPE_HEADER],
}),
this.secrets,
{
redactionScope: "response",
timeoutMs: 5000,
maxBodyBytes: this.limits.max_network_body_bytes,
maxBodyAllocationBytes: Math.min(
this.limits.max_network_body_allocation_bytes,
available,
),
maxHeaderAllocationBytes:
this.limits.max_network_header_allocation_bytes,
maxMetadataAllocationBytes:
this.limits.max_network_metadata_allocation_bytes,
},
readOptions,
);
body =
protectedBody.state === "omitted"
@@ -575,7 +583,10 @@ export class PlaywrightScenarioEvents {
: { ...event, ...metadata };
}
private push(event: UnindexedEvent): BrowserScenarioEventAdmission {
private push(
event: UnindexedEvent,
bodyRedaction?: NetworkBodyRedaction,
): BrowserScenarioEventAdmission {
this.sequence += 1;
const cost =
allocationCost(event, this.limits.max_event_allocation_bytes) + 128;
@@ -593,6 +604,8 @@ export class PlaywrightScenarioEvents {
sequence: this.sequence,
step_index: this.stepIndex,
});
if (bodyRedaction !== undefined)
this.networkBodyRedaction.set(parsed, bodyRedaction);
this.items.push(parsed);
return { status: "retained", sequence: this.sequence };
}
@@ -667,6 +680,7 @@ export class PlaywrightScenarioEvents {
bytes: input.observedBytes,
sha256: "0".repeat(64),
media_type: null,
media_type_source: "content-type-header",
redacted: false,
},
},
@@ -4,6 +4,7 @@ import { resolveScenarioSecrets } from "../../../tests/fixtures/browserScenarioS
import {
captureNetworkBody,
captureNetworkHeaders,
networkBodyMediaType,
} from "./PlaywrightNetworkContent.js";
const scenario = browserScenarioSchema.parse({
@@ -20,17 +21,15 @@ const options = {
maxHeaderAllocationBytes: 16384,
maxMetadataAllocationBytes: 4096,
};
const source = {
headers: () => ({ "content-type": "text/plain" }),
headersArray: async () => [],
};
const textType = () => networkBodyMediaType("text/plain");
const jsonType = () => networkBodyMediaType("application/json");
it("enforces the retained body byte limit after secret expansion", async () => {
expect(
await captureNetworkBody(
async () => Buffer.from("xxxx"),
"response",
source,
textType,
secrets,
options,
),
@@ -43,7 +42,7 @@ it("enforces the retained body byte limit after secret expansion", async () => {
await captureNetworkBody(
async () => Buffer.from("safe"),
"response",
source,
textType,
secrets,
options,
),
@@ -56,17 +55,20 @@ it("enforces the retained body byte limit after secret expansion", async () => {
it("charges discarded credential header values only for their retained projection", async () => {
const read = async (name: string) =>
captureNetworkHeaders(
{
headers: () => ({}),
headersArray: async () => [
{ name, value: "v".repeat(50_000) },
{ name: "X-Observed", value: "ordinary" },
],
},
secrets,
options,
);
(
await captureNetworkHeaders(
{
headers: () => ({}),
headersArray: async () => [
{ name, value: "v".repeat(50_000) },
{ name: "X-Observed", value: "ordinary" },
],
},
true,
secrets,
options,
)
).headers;
expect(await read("Cookie")).toMatchObject({
state: "captured",
items: [
@@ -85,12 +87,7 @@ it("redacts later JSON spellings of saved secrets without reserializing unmarked
const captured = await captureNetworkBody(
async () => Buffer.from(body),
"response",
{
headers: () => ({
"content-type": "application/problem+json; charset=utf-8",
}),
headersArray: async () => [],
},
jsonType,
held,
{ ...options, maxBodyBytes: 4096 },
);
@@ -109,10 +106,7 @@ it("keeps an empty JSON-labelled response as observed empty bytes", async () =>
const captured = await captureNetworkBody(
async () => Buffer.alloc(0),
"response",
{
headers: () => ({ "content-type": "application/json" }),
headersArray: async () => [],
},
jsonType,
secrets,
options,
);
@@ -125,10 +119,7 @@ it("checks the JSON decoding budget before invoking its decoder", async () => {
const captured = await captureNetworkBody(
async () => Buffer.from('{"value":"untouched"}'),
"response",
{
headers: () => ({ "content-type": "application/json" }),
headersArray: async () => [],
},
jsonType,
held,
{ ...options, maxBodyBytes: 4096, maxBodyAllocationBytes: 1 },
);
@@ -147,10 +138,7 @@ it.each(['{"token":"private-token-729d"}', 'A","B'])(
const captured = await captureNetworkBody(
async () => Buffer.from(body),
"request",
{
headers: () => ({ "content-type": "application/json" }),
headersArray: async () => [],
},
jsonType,
held,
{ ...options, maxBodyBytes: 4096 },
);
+102 -47
View File
@@ -5,6 +5,10 @@ import {
mediaTypeEssence,
} from "../../domain/analysisView/recordedMediaType.js";
import { safeParseJson } from "../../domain/safeJson.js";
import {
browserContentType,
CONTENT_TYPE_HEADER,
} from "../../domain/browserNetworkMediaType.js";
import {
AnalysisCancelledError,
@@ -71,12 +75,67 @@ const errorMessage = (
: "Network read failed; diagnostic exceeds the allocation budget.";
};
export type NetworkBodyRedaction = "json" | "unknown" | "bytes";
interface BodyMediaType {
readonly value: string | null;
readonly redaction: NetworkBodyRedaction;
readonly declarationNames: readonly string[];
}
interface NetworkHeaderCapture {
readonly headers: BrowserNetworkHeaders;
readonly mediaType: BodyMediaType;
}
export const networkBodyMediaType = (
value: string | null,
declarationNames: readonly string[] = [CONTENT_TYPE_HEADER],
): BodyMediaType => ({
value,
declarationNames,
redaction:
value === null
? "unknown"
: isJson(mediaTypeEssence(value))
? "json"
: "bytes",
});
/** Publish only the type that can still be read after privacy projection. */
const retainNetworkMediaType = (
{ value, declarationNames }: BodyMediaType,
secrets: BrowserScenarioSecrets,
options: NetworkReadOptions,
): string | null => {
if (value === null) return null;
if (
secrets.redactionAllocationCost(value.length, options.redactionScope) >
options.maxHeaderAllocationBytes ||
declarationNames.some(
(name) => secrets.redact(name, options.redactionScope) !== name,
) ||
secrets
.redactBytes(Buffer.from(value), options.redactionScope)
.toString("utf8") !== value
)
return null;
return value;
};
/** Read actual producer headers without flattening duplicates or exposing credentials. */
export const captureNetworkHeaders = async (
source: NetworkHeadersSource,
retainValues: boolean,
secrets: BrowserScenarioSecrets,
options: NetworkReadOptions,
): Promise<BrowserNetworkHeaders> => {
): Promise<NetworkHeaderCapture> => {
const unavailable = (message: string): NetworkHeaderCapture => ({
headers: retainValues
? { state: "unavailable", message }
: { state: "not_requested" },
mediaType: networkBodyMediaType(null),
});
try {
const headers = await withPlaywrightExecutionBoundary(
() => source.headersArray(),
@@ -88,7 +147,12 @@ export const captureNetworkHeaders = async (
BrowserNetworkHeaders,
{ state: "captured" }
>["items"] = [];
const declarations: string[] = [];
const declarationNames: string[] = [];
let typeRedacted = false;
for (const { name, value } of headers) {
const contentType = name.toLowerCase() === CONTENT_TYPE_HEADER;
if (!retainValues && !contentType) continue;
const credential = isTransportCredentialHeader(name);
cost +=
secrets.redactionAllocationCost(
@@ -96,33 +160,44 @@ export const captureNetworkHeaders = async (
options.redactionScope,
) + 128;
if (cost > options.maxHeaderAllocationBytes)
return {
state: "unavailable",
message: `Headers exceed the ${String(options.maxHeaderAllocationBytes)}-byte allocation budget.`,
};
return unavailable(
`Headers exceed the ${String(options.maxHeaderAllocationBytes)}-byte allocation budget.`,
);
if (contentType) {
declarationNames.push(name);
declarations.push(value);
}
if (!retainValues) continue;
const safeName = secrets.redact(name, options.redactionScope);
const safeValue = credential
? null
: secrets
.redactBytes(Buffer.from(value), options.redactionScope)
.toString("utf8");
items.push({
name: safeName,
value: safeValue,
redacted: credential || safeName !== name || safeValue !== value,
});
const redacted = credential || safeName !== name || safeValue !== value;
items.push({ name: safeName, value: safeValue, redacted });
if (contentType) typeRedacted ||= redacted;
}
return { state: "captured", items };
} catch (cause: unknown) {
const observed = browserContentType(declarations);
const mediaType = networkBodyMediaType(
observed.state === "known" ? observed.value : null,
declarationNames,
);
return {
state: "unavailable",
message: errorMessage(
headers: retainValues
? { state: "captured", items }
: { state: "not_requested" },
mediaType: typeRedacted ? { ...mediaType, value: null } : mediaType,
};
} catch (cause: unknown) {
return unavailable(
errorMessage(
cause,
secrets,
options.maxMetadataAllocationBytes,
options.redactionScope,
),
};
);
}
};
@@ -130,7 +205,7 @@ export const captureNetworkHeaders = async (
export const captureNetworkBody = async (
read: () => Promise<Buffer | null>,
phase: "request" | "response",
source: NetworkHeadersSource,
getMediaType: () => BodyMediaType | Promise<BodyMediaType>,
secrets: BrowserScenarioSecrets,
options: NetworkReadOptions,
preflight?: NetworkBodyPreflight,
@@ -168,7 +243,6 @@ export const captureNetworkBody = async (
);
if (!protection.retainBody) return { state: "not_requested" };
}
const contentType = source.headers()["content-type"];
const bodyAllocationCost = secrets.redactionAllocationCost(
Math.max(observed.length, sourceBody.length),
options.redactionScope,
@@ -189,43 +263,28 @@ export const captureNetworkBody = async (
limit_allocation_bytes: maxBodyAllocationBytes,
observed_allocation_bytes: observedBodyAllocationBytes,
};
if (
contentType !== undefined &&
secrets.redactionAllocationCost(
contentType.length,
options.redactionScope,
) > options.maxHeaderAllocationBytes
)
return {
state: "unavailable",
reason: "body-allocation-limit",
message:
"Content type exceeds the selected network header allocation budget.",
limit_allocation_bytes: options.maxHeaderAllocationBytes,
observed_allocation_bytes: secrets.redactionAllocationCost(
contentType.length,
options.redactionScope,
),
};
const jsonBody =
protection !== undefined ||
(contentType !== undefined && isJson(mediaTypeEssence(contentType)));
const mediaType = await getMediaType();
const admission = await preflight?.({
observedBytes: observed.length,
allocationBytes: observedBodyAllocationBytes,
mediaType: contentType,
mediaType: mediaType.value ?? undefined,
});
if (admission?.status === "omitted")
return { state: "omitted", reason: admission.reason };
if (protection === undefined && jsonBody)
sourceBody = secrets.redactJsonBody(
let jsonBody = protection !== undefined;
if (protection === undefined && mediaType.redaction !== "bytes") {
const redacted = secrets.redactJsonBody(
observed,
{
maxBytes: options.maxBodyBytes,
maxAllocationBytes: options.maxBodyAllocationBytes,
},
options.redactionScope,
mediaType.redaction,
);
sourceBody = redacted.bytes;
jsonBody = redacted.json;
}
const retained = secrets.redactBytes(sourceBody, options.redactionScope);
if (
jsonBody &&
@@ -257,12 +316,8 @@ export const captureNetworkBody = async (
content: retained.toString("base64"),
bytes: retained.length,
sha256: sha256Hex(retained),
media_type:
contentType === undefined
? null
: secrets
.redactBytes(Buffer.from(contentType), options.redactionScope)
.toString("utf8"),
media_type: retainNetworkMediaType(mediaType, secrets, options),
media_type_source: "content-type-header",
redacted: !retained.equals(observed),
};
} catch (cause: unknown) {
@@ -567,7 +567,7 @@ it.each([
response_body: false,
header_values: false,
},
reads: [1, 0, 0, 0],
reads: [1, 1, 0, 0],
phases: ["request"],
},
{
@@ -576,7 +576,7 @@ it.each([
response_body: true,
header_values: false,
},
reads: [0, 0, 1, 0],
reads: [0, 0, 1, 1],
phases: ["response"],
},
{
@@ -589,7 +589,7 @@ it.each([
phases: ["request", "response"],
},
])(
"reads only the selected content $selection",
"reads selected bodies and the headers needed for their type $selection",
async ({ selection: selected, reads, phases }) => {
const { collector, events } = harness(selected);
const request = new RequestFixture("input");
@@ -609,6 +609,12 @@ it.each([
.filter((event) => event.kind === "network-content")
.map((event) => event.phase),
).toEqual(phases);
if (!selected.header_values)
expect(
events
.filter((event) => event.kind === "network-content")
.map(({ headers }) => headers),
).toEqual([{ state: "not_requested" }]);
},
);
@@ -985,19 +991,20 @@ it("preserves read errors and does not turn unexposed or empty bytes into the sa
expect(collector.limitations().length).toBeGreaterThan(0);
});
it("redacts declared encodings in body media metadata", async () => {
it("keeps a redacted declaration out of body media metadata", async () => {
const { collector, events } = harness({
request_body: false,
response_body: true,
header_values: false,
header_values: true,
});
const request = new RequestFixture("input");
const response = new ResponseFixture(request, "ordinary-bytes");
const headers = response.headers();
response.headers = () => ({
...headers,
"content-type": `text/plain; fixture=${encodeURIComponent("私密")}`,
});
response.headersArray = async () => [
{
name: "Content-Type",
value: `text/plain; fixture=${encodeURIComponent("私密")}`,
},
];
collector.request(request);
collector.response(response);
collector.finished(request);
@@ -1006,14 +1013,94 @@ it("redacts declared encodings in body media metadata", async () => {
expect.objectContaining({
kind: "network-content",
phase: "response",
headers: {
state: "captured",
items: [
{
name: "Content-Type",
value: "text/plain; fixture=[REDACTED:declared]",
redacted: true,
},
],
},
body: expect.objectContaining({
state: "captured",
media_type: "text/plain; fixture=[REDACTED:declared]",
media_type: null,
}),
}),
);
});
it.each([
{
declarations: [
"application/json; charset=utf-8",
"application/octet-stream",
],
type: null,
},
{
declarations: [
"application/octet-stream",
"application/json; charset=utf-8",
],
type: null,
},
{
declarations: ["application/json; charset=utf-8, application/octet-stream"],
type: null,
},
{
declarations: ["application/json", "application/json"],
type: "application/json",
},
{
declarations: ['application/json; note="one,two"'],
type: 'application/json; note="one,two"',
},
{ declarations: null, type: null },
])(
"derives body type once from retained declarations $declarations",
async ({ declarations, type }) => {
const { collector, events } = harness({
request_body: false,
response_body: true,
header_values: false,
});
const request = new RequestFixture("input");
const response = new ResponseFixture(request, '{"value":"\\u79c1\\u5bc6"}');
response.headers = () => ({
"content-type": declarations?.join(", ") ?? "application/json",
});
response.headersArray = async () => {
response.headerReads += 1;
if (declarations === null) throw new Error("Headers unavailable");
return declarations.map((value) => ({ name: "Content-Type", value }));
};
collector.request(request);
collector.response(response);
collector.finished(request);
await collector.finish();
const content = events.find(
(event) => event.kind === "network-content" && event.phase === "response",
);
expect(response.headerReads).toBe(1);
expect(content).toMatchObject({
headers: { state: "not_requested" },
body: {
state: "captured",
media_type: type,
media_type_source: "content-type-header",
},
});
if (content?.kind !== "network-content")
throw new Error("Missing response content");
expect(JSON.parse(bodyText(content))).toEqual({
value: "[REDACTED:declared]",
});
},
);
it("bounds admitted transactions before invoking content producers", async () => {
const { collector, events } = harness();
const admitted: RequestFixture[] = [];
@@ -17,6 +17,7 @@ import {
} from "../BrowserScenarioResponseValues.js";
import {
captureNetworkBody,
type NetworkBodyRedaction,
captureNetworkHeaders,
type NetworkBodyCapture,
type NetworkHeadersSource,
@@ -100,6 +101,7 @@ interface NetworkCaptureOptions {
readonly secrets: BrowserScenarioSecrets;
readonly emit: (
event: UnindexedNetworkEvent,
bodyRedaction?: NetworkBodyRedaction,
) => BrowserScenarioEventAdmission;
/** Check aggregate event capacity before redacting producer-owned metadata. */
readonly admitText: (
@@ -647,11 +649,14 @@ export class PlaywrightScenarioNetwork {
const readOptions = privacyOnly
? { ...this.readOptions, maxBodyBytes: MAX_SAVED_RESPONSE_BYTES }
: this.readOptions;
let headers: Promise<BrowserNetworkHeaders> | undefined;
const getHeaders = (): Promise<BrowserNetworkHeaders> =>
(headers ??= includeHeaders
? captureNetworkHeaders(source, this.options.secrets, readOptions)
: Promise.resolve({ state: "not_requested" }));
let headers: ReturnType<typeof captureNetworkHeaders> | undefined;
const getHeaders = () =>
(headers ??= captureNetworkHeaders(
source,
includeHeaders,
this.options.secrets,
readOptions,
));
// Existing reads keep their order. Marked responses establish secret
// spellings before either their headers or their body can be retained.
if (!protectedResponse && includeHeaders) await getHeaders();
@@ -673,7 +678,7 @@ export class PlaywrightScenarioNetwork {
return bounded.value;
},
phase,
source,
async () => (await getHeaders()).mediaType,
this.options.secrets,
readOptions,
async (body) =>
@@ -681,7 +686,7 @@ export class PlaywrightScenarioNetwork {
transactionId: transaction.id,
sourceEventSequence: sequence,
phase,
headers: await getHeaders(),
headers: (await getHeaders()).headers,
observedBytes: body.observedBytes,
allocationBytes: body.allocationBytes,
mediaType: body.mediaType,
@@ -708,7 +713,9 @@ export class PlaywrightScenarioNetwork {
message:
"Response header values were withheld because marked source values could not be read safely.",
}
: await getHeaders();
: includeHeaders
? (await getHeaders()).headers
: { state: "not_requested" };
const retainedBody = includeBody
? observedBody
: { state: "not_requested" as const };
@@ -723,14 +730,21 @@ export class PlaywrightScenarioNetwork {
this.gaps.add(
"Some selected network content was unavailable; per-transaction content states retain the reason.",
);
const admission = this.options.emit({
kind: "network-content",
transaction_id: transaction.id,
source_event_sequence: sequence,
phase,
headers: retainedHeaders,
body: retainedBody,
});
const bodyRedaction =
retainedBody.state === "captured"
? (await getHeaders()).mediaType.redaction
: undefined;
const admission = this.options.emit(
{
kind: "network-content",
transaction_id: transaction.id,
source_event_sequence: sequence,
phase,
headers: retainedHeaders,
body: retainedBody,
},
bodyRedaction,
);
if (admission.status === "omitted")
this.options.omitNetwork(
transaction.id,
@@ -104,7 +104,23 @@ describe("scenario Content-Type evidence", () => {
],
},
]);
const projected = scenarioCaptureRecords(capture, "manifest");
const projected = scenarioCaptureRecords(
{
...capture,
events: {
...capture.events,
items: capture.events.items.map((event) =>
event.kind === "network-content" && event.body.state === "captured"
? {
...event,
body: { ...event.body, media_type_source: "independent" },
}
: event,
),
},
},
"manifest",
);
expect(projected.records).toHaveLength(2);
for (const record of projected.records)
expect(record.reported).toMatchObject({
@@ -113,6 +129,61 @@ describe("scenario Content-Type evidence", () => {
});
});
it("uses retained declarations to correct legacy body metadata and honors new explicit uncertainty", () => {
const capture = scenarioCaptureFixture([
{
method: "GET",
url: "/legacy-conflict",
status: 200,
body: "{}",
mediaType: "application/json; charset=utf-8, application/octet-stream",
responseHeaders: [
{ name: "Content-Type", value: "application/json; charset=utf-8" },
{ name: "content-type", value: "application/octet-stream" },
],
},
{
method: "GET",
url: "/legacy-agreement",
status: 200,
body: "{}",
mediaType: "application/json, application/json",
responseHeaders: [
{ name: "Content-Type", value: "application/json" },
{ name: "content-type", value: "application/json" },
],
},
{
method: "GET",
url: "/legacy-coalesced-without-headers",
status: 200,
body: "{}",
mediaType: "application/json; charset=utf-8, application/octet-stream",
},
]);
const content = capture.events.items.filter(
(event) => event.kind === "network-content",
);
expect(content.map(scenarioContentMediaType)).toEqual([
undefined,
"application/json",
undefined,
]);
const later = content[1];
if (later?.body.state !== "captured")
throw new Error("Missing captured body");
expect(
scenarioContentMediaType({
...later,
body: {
...later.body,
media_type: null,
media_type_source: "content-type-header",
},
}),
).toBeUndefined();
});
describe("scenarioCaptureRecords", () => {
it("projects each transaction, in request order, as a HAR-shaped record with its retained bytes", () => {
const projected = scenarioCaptureRecords(
@@ -1,9 +1,7 @@
import { AnalysisUnsupportedTargetError } from "../analysisErrorCore.js";
import type { AnalysisError } from "../analysisErrorBase.js";
import type {
BrowserNetworkBody,
BrowserNetworkHeaders,
} from "../browserNetworkEvidence.js";
import type { BrowserNetworkBody } from "../browserNetworkEvidence.js";
import { retainedNetworkMediaType } from "../browserNetworkMediaType.js";
import {
retainedBrowserScenarioCaptureSchema,
type RetainedBrowserScenarioCapture,
@@ -47,23 +45,6 @@ const onlyContent = (
return found?.length === 1 ? found[0] : undefined;
};
const headerValue = (
headers: BrowserNetworkHeaders | undefined,
name: string,
): string | undefined => {
if (headers?.state !== "captured") return undefined;
let value: string | undefined;
for (const item of headers.items) {
if (item.name.toLowerCase() !== name) continue;
// A singleton field has no certain value if declarations disagree or one
// was excluded. Never let ordering choose an interpretation of Evidence.
if (item.value === null || (value !== undefined && value !== item.value))
return undefined;
value = item.value;
}
return value;
};
const captured = (
content: ContentEvent | undefined,
): CapturedBody | undefined =>
@@ -88,12 +69,11 @@ const utf8Text = (body: CapturedBody): string | undefined => {
}
};
/** A content event's media type: its captured body's, else its retained Content-Type value. */
/** Use the retained type fact for the content event. */
export const scenarioContentMediaType = (
content: ContentEvent | undefined,
): string | undefined =>
captured(content)?.media_type ??
headerValue(content?.headers, "content-type");
retainedNetworkMediaType(content?.body, content?.headers);
/** Whether a request sent no body: Playwright exposed none and the request names no Content-Type, which the bodies browsers send carry. */
const unsentBody = (
+3
View File
@@ -74,6 +74,9 @@ export const browserNetworkBodySchema = z.union([
bytes: z.int().min(0),
sha256: z.string().regex(/^[a-f0-9]{64}$/u),
media_type: z.string().nullable(),
media_type_source: z
.enum(["content-type-header", "independent"])
.optional(),
redacted: z.boolean(),
})
.superRefine((artifact, context) => {
+68
View File
@@ -0,0 +1,68 @@
import { mediaTypeEssence } from "./analysisView/recordedMediaType.js";
import type {
BrowserNetworkBody,
BrowserNetworkHeaders,
} from "./browserNetworkEvidence.js";
export const CONTENT_TYPE_HEADER = "content-type";
/** An HTTP singleton media type can contain commas only inside a quoted parameter. */
const isSingleMediaType = (value: string): boolean => {
if (/[\r\n]/u.test(value) || mediaTypeEssence(value) === null) return false;
let quoted = false;
for (let index = 0; index < value.length; index += 1) {
const character = value[index];
if (character === '"') quoted = !quoted;
else if (quoted && character === "\\") index += 1;
else if (!quoted && character === ",") return false;
}
return !quoted;
};
export type BrowserContentType =
| { readonly state: "absent" | "uncertain" }
| { readonly state: "known"; readonly value: string };
/** Interpret declarations without selecting an arbitrary occurrence or a masked value. */
export const browserContentType = (
declarations: Iterable<string | null>,
): BrowserContentType => {
let value: string | undefined;
for (const declaration of declarations) {
if (
declaration === null ||
(value === undefined
? !isSingleMediaType(declaration)
: value !== declaration)
)
return { state: "uncertain" };
value = declaration;
}
return value === undefined ? { state: "absent" } : { state: "known", value };
};
function* retainedDeclarations(
items: Extract<BrowserNetworkHeaders, { state: "captured" }>["items"],
): Generator<string | null> {
for (const item of items)
if (item.name.toLowerCase() === CONTENT_TYPE_HEADER)
yield item.redacted ? null : item.value;
}
/** Project a retained type without inferring certainty from a legacy flattened header. */
export const retainedNetworkMediaType = (
body: BrowserNetworkBody | undefined,
headers: BrowserNetworkHeaders | undefined,
): string | undefined => {
const captured = body?.state === "captured" ? body : undefined;
// Newly retained body metadata includes the provenance of an explicit unknown.
if (captured?.media_type_source !== undefined)
return captured.media_type ?? undefined;
if (headers?.state === "captured") {
const type = browserContentType(retainedDeclarations(headers.items));
if (type.state === "known") return type.value;
if (type.state === "uncertain") return undefined;
}
const legacy = captured?.media_type;
return legacy != null && isSingleMediaType(legacy) ? legacy : undefined;
};
+8 -6
View File
@@ -1,6 +1,6 @@
import type { BrowserScenarioCapture } from "./browserScenarioCapture.js";
import type { BrowserScenarioEvent } from "./browserScenarioCaptureValues.js";
import type { BrowserNetworkBody } from "./browserNetworkEvidence.js";
import { retainedNetworkMediaType } from "./browserNetworkMediaType.js";
import type { CapturedWebScript } from "./webScriptExport.js";
import {
indexBrowserScenarioTransactions,
@@ -8,6 +8,7 @@ import {
} from "./browserScenarioTransactions.js";
type RequestEvent = Extract<BrowserScenarioEvent, { kind: "request" }>;
type ContentEvent = Extract<BrowserScenarioEvent, { kind: "network-content" }>;
/** Join script response bytes only through explicit transaction and event identity. */
export const scenarioScripts = (
@@ -34,7 +35,7 @@ const projectRequest = (
: (index.contents.get(response.sequence) ?? []).filter(
({ phase }) => phase === "response",
);
const body = bodies.length === 1 ? bodies[0]?.body : undefined;
const content = bodies.length === 1 ? bodies[0] : undefined;
const ambiguous =
index.requestCounts.get(id) !== 1 ||
matches.length > 1 ||
@@ -48,7 +49,7 @@ const projectRequest = (
status: ambiguous ? null : (response?.status ?? null),
},
url: request.url.url,
content: projectBody(body, ambiguous),
content: projectBody(content, ambiguous),
};
};
@@ -58,7 +59,7 @@ const unavailable = (
): CapturedWebScript["content"] => ({ state: "unavailable", reason, message });
const projectBody = (
body: BrowserNetworkBody | undefined,
content: ContentEvent | undefined,
ambiguous: boolean,
): CapturedWebScript["content"] => {
if (ambiguous)
@@ -66,11 +67,12 @@ const projectBody = (
"ambiguous-transaction",
"The capture does not identify one request, response, and content record for this transaction.",
);
if (body === undefined)
if (content === undefined)
return unavailable(
"response-content-not-retained",
"No selected, completed script response bytes were retained for this request.",
);
const { body, headers } = content;
if (body.state === "unavailable")
return unavailable(body.reason, body.message);
if (body.state !== "captured")
@@ -90,7 +92,7 @@ const projectBody = (
state: "captured",
bytes: Buffer.from(body.content, "base64"),
sha256: body.sha256,
media_type: body.media_type,
media_type: retainedNetworkMediaType(body, headers) ?? null,
redacted: body.redacted,
representation: body.representation,
};