fix(capture): distinguish corrupt SAZ metadata bytes (#13204)

This commit is contained in:
morluto authored and GitHub committed 2026-11-21 20:51:33 +00:00
1 parent a2faf4cc34
commit ff3919f3eb
4 files changed
+184 -244

No files matched your search

+67 -75
View File
@@ -5,89 +5,81 @@ import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { decodeSazCapture } from "./SazCapture.js";
it.each(["", "./", "././"])(
"preserves SAZ source member names while classifying logical paths: %j",
async (prefix) => {
const writer = new ZipWriter(new Uint8ArrayWriter(), {
useWebWorkers: false,
});
await writer.add(
`${prefix}raw/1_c.txt`,
new TextReader("GET /path HTTP/1.1\r\n\r\nbody"),
);
for (const [name, text] of [
["s.txt", "HTTP/1.1 200 OK\r\n\r\nresponse-body"],
["m.xml", "<Session><SessionFlags /></Session>"],
["w.txt", "stored-websocket-bytes"],
["g.txt", "stored-grpc-bytes"],
] as const)
await writer.add(`${prefix}raw/1_${name}`, new TextReader(text));
await writer.add(`${prefix}notes.txt`, new TextReader("ordinary evidence"));
const bytes = Buffer.from(await writer.close());
const root = await createTestTempDirectory("rea-saz-member-names-");
const path = join(root, "capture.saz");
await writeFile(path, bytes);
it("preserves mixed source spellings and session ordering within one SAZ", async () => {
const entries = [
["././raw/42_s.txt", "raw/42_s.txt", "HTTP/1.1 200 OK\r\n\r\nresponse"],
["raw/7_c.txt", "raw/7_c.txt", "GET /ordinary HTTP/1.1\r\n\r\n"],
["./raw/42_c.txt", "raw/42_c.txt", "GET /prefixed HTTP/1.1\r\n\r\nbody"],
["./raw/42_m.xml", "raw/42_m.xml", "<Session/>"],
["././raw/42_w.txt", "raw/42_w.txt", "stored websocket bytes"],
["raw/7_g.txt", "raw/7_g.txt", "stored grpc bytes"],
["./notes.txt", "notes.txt", "ordinary evidence"],
] as const;
const writer = new ZipWriter(new Uint8ArrayWriter(), {
useWebWorkers: false,
});
for (const [source, , content] of entries)
await writer.add(source, new TextReader(content));
const bytes = Buffer.from(await writer.close());
const root = await createTestTempDirectory("rea-saz-member-names-");
const path = join(root, "capture.saz");
await writeFile(path, bytes);
const capture = await decodeSazCapture(path, []);
expect(capture.records).toHaveLength(1);
expect(capture.records[0]).toMatchObject({
const capture = await decodeSazCapture(path, []);
expect(capture.records).toMatchObject([
{
ordinal: 0,
location: {
kind: "archive-members",
members: ["c.txt", "s.txt", "m.xml", "w.txt", "g.txt"].map(
(name) => `${prefix}raw/1_${name}`,
),
members: [
"././raw/42_s.txt",
"./raw/42_c.txt",
"./raw/42_m.xml",
"././raw/42_w.txt",
],
},
reported: {
request: { member: `${prefix}raw/1_c.txt`, url: "/path", body: "body" },
response: { member: `${prefix}raw/1_s.txt`, body: "response-body" },
metadata: { member: `${prefix}raw/1_m.xml` },
session_id: "42",
request: { member: "./raw/42_c.txt", url: "/prefixed", body: "body" },
response: { member: "././raw/42_s.txt", body: "response" },
metadata: { member: "./raw/42_m.xml" },
websocket: {
member: `${prefix}raw/1_w.txt`,
messages: "stored-websocket-bytes",
member: "././raw/42_w.txt",
messages: "stored websocket bytes",
},
grpc: { member: `${prefix}raw/1_g.txt`, messages: "stored-grpc-bytes" },
},
});
expect(capture.container.reported).toMatchObject({
members: [
...["c.txt", "s.txt", "m.xml", "w.txt", "g.txt"].map((name) => ({
path: `${prefix}raw/1_${name}`,
})),
{ path: `${prefix}notes.txt` },
],
});
if (prefix !== "") {
expect(capture.records[0]?.limitations).toContainEqual(
expect.stringContaining("leading current-directory prefix"),
);
expect(capture.container.reported).toMatchObject({
members: [
...["c.txt", "s.txt", "m.xml", "w.txt", "g.txt"].map((name) => ({
logical_path: `raw/1_${name}`,
limitations: [
expect.stringContaining("leading current-directory prefix"),
],
})),
{
logical_path: "notes.txt",
limitations: [
expect.stringContaining("leading current-directory prefix"),
],
},
],
});
} else {
expect(capture.records[0]?.limitations).toEqual([]);
expect(capture.container.reported).not.toHaveProperty(
"members.0.logical_path",
);
expect(capture.container.reported).not.toHaveProperty(
"members.0.limitations",
);
}
expect((await readFile(path)).equals(bytes)).toBe(true);
},
);
},
{
ordinal: 1,
location: {
kind: "archive-members",
members: ["raw/7_c.txt", "raw/7_g.txt"],
},
reported: {
session_id: "7",
request: { member: "raw/7_c.txt", url: "/ordinary" },
grpc: { member: "raw/7_g.txt", messages: "stored grpc bytes" },
},
limitations: [],
},
]);
const prefixNote = expect.stringContaining(
"leading current-directory prefix",
);
expect(capture.records[0]?.limitations).toContainEqual(prefixNote);
expect(capture.container.reported).toHaveProperty(
"members",
entries.map(([path, logical_path, content]) => ({
path,
kind: "file",
bytes: Buffer.byteLength(content),
...(path === logical_path
? {}
: { logical_path, limitations: [prefixNote] }),
})),
);
expect((await readFile(path)).equals(bytes)).toBe(true);
});
it("keeps rejecting distinct source names that collide on one logical session member", async () => {
const writer = new ZipWriter(new Uint8ArrayWriter(), {
+29 -99
View File
@@ -1,100 +1,41 @@
import { expect, it } from "vitest";
import { CaptureFormatError } from "./CaptureFormatError.js";
import { decodeSazMetadata } from "./SazMetadata.js";
it.each([
["UTF-8", Buffer.from('<?xml version="1.0" encoding="UTF-8"?><Session/>')],
[
"BOM-marked UTF-16LE",
Buffer.concat([
Buffer.from([0xff, 0xfe]),
Buffer.from(
'<?xml version="1.0" encoding="UTF-16"?><Session/>',
"utf16le",
),
]),
],
] as const)("accepts matching %s XML declarations", (_label, bytes) => {
expect(() => decodeSazMetadata(bytes)).not.toThrow();
});
it.each([
[
"UTF-16 declaration on UTF-8 bytes",
Buffer.from('<?xml version="1.0" encoding="UTF-16"?><Session/>'),
"format",
],
[
"UTF-8 declaration on UTF-16 bytes",
Buffer.concat([
Buffer.from([0xff, 0xfe]),
Buffer.from(
'<?xml version="1.0" encoding="UTF-8"?><Session/>',
"utf16le",
),
]),
"format",
],
[
"unsupported declared encoding",
Buffer.from('<?xml version="1.0" encoding="ISO-8859-1"?><Session/>'),
"unsupported",
],
] as const)("rejects %s", (_label, bytes, reason) => {
expect(() => decodeSazMetadata(bytes)).toThrow(
expect.objectContaining({ replyReason: reason }),
it("reports a declared legacy XML encoding as unsupported", () => {
expect(() =>
decodeSazMetadata(
Buffer.from('<?xml version="1.0" encoding="ISO-8859-1"?><Session/>'),
),
).toThrow(
expect.objectContaining({
failure: expect.objectContaining({
kind: "unsupported",
format: "web-network-capture",
}),
message: expect.stringContaining("ISO-8859-1"),
}),
);
});
it("retains U+FFFD in well-formed metadata as an ordinary XML character", () => {
it("preserves XML node order, attributes, entities, and literal replacement characters", () => {
const decoded = decodeSazMetadata(
Buffer.from('<Session><SessionFlag N="ui-comments" V="caf�"/>�</Session>'),
Buffer.from(
'<?xml version="1.0"?><Session><!-- note & literal --><Value title="caf�">A &amp; B�</Value><C><![CDATA[C & D]]></C></Session>',
),
);
expect(decoded).toMatchObject({
name: "#document",
children: [
{
name: "Session",
children: [
{
name: "SessionFlag",
attributes: [
{ name: "N", value: "ui-comments" },
{ name: "V", value: "caf�" },
],
},
{ name: "#text", value: "�" },
],
},
],
});
});
it.each([
"<Session>a & b</Session>",
"<Session>a\u0000b</Session>",
"<Session>&#xD800;</Session>",
"<Session a=1>�</Session>",
'<Session a="x"b="�"/>',
"<Session><Broken>�</Session>",
])("still rejects malformed metadata that also contains U+FFFD: %j", (xml) => {
expect(() => decodeSazMetadata(Buffer.from(xml))).toThrow(
"SAZ metadata is not well-formed XML.",
);
});
it("preserves valid comments, CDATA, declarations, and predefined entities", () => {
const xml =
'<?xml version="1.0"?><Session><!-- note & literal --><Value>A &amp; B</Value><C><![CDATA[C & D]]></C></Session>';
const decoded = decodeSazMetadata(Buffer.from(xml));
expect(decoded).toMatchObject({
children: [
{ name: "xml" },
{
name: "Session",
children: [
{ name: "#comment", value: " note & literal " },
{ name: "Value", children: [{ name: "#text", value: "A & B" }] },
{
name: "Value",
attributes: [{ name: "title", value: "caf�" }],
children: [{ name: "#text", value: "A & B�" }],
},
{ name: "C", children: [{ name: "#cdata-section", value: "C & D" }] },
],
},
@@ -102,24 +43,13 @@ it("preserves valid comments, CDATA, declarations, and predefined entities", ()
});
});
it("rejects DTDs without resolving entities", () => {
expect(() =>
decodeSazMetadata(
Buffer.from('<!DOCTYPE Session [<!ENTITY value "external">]><Session/>'),
),
).toThrow(
it("refuses malformed markup even when the XML contains a literal replacement character", () => {
expect(() => decodeSazMetadata(Buffer.from('<Session a="x"b="�"/>'))).toThrow(
expect.objectContaining({
replyReason: "unsupported",
} satisfies Partial<CaptureFormatError>),
failure: expect.objectContaining({
kind: "malformed",
format: "web-network-capture",
}),
}),
);
});
it("refuses duplicate expanded attribute names rather than dropping source values", () => {
expect(() =>
decodeSazMetadata(
Buffer.from(
'<Session xmlns:p="urn:a" xmlns:q="urn:a" p:x="first" q:x="last"/>',
),
),
).toThrow("SAZ metadata is not well-formed XML.");
});
+6 -16
View File
@@ -14,29 +14,19 @@ export const decodeSazMetadata = (bytes: Buffer): JsonValue => {
try {
xml = decodeXmlPlistText(bytes);
} catch (cause: unknown) {
if (
cause instanceof XmlPlistTextDecodeError &&
cause.reason === "unsupported-encoding"
)
if (!(cause instanceof XmlPlistTextDecodeError)) throw cause;
if (cause.reason === "unsupported-encoding")
throw new CaptureFormatError(
"unsupported",
`SAZ metadata declares unsupported XML encoding ${cause.declaredEncoding}.`,
"",
cause,
);
if (
cause instanceof XmlPlistTextDecodeError &&
cause.reason === "encoding-mismatch"
)
throw new CaptureFormatError(
"malformed",
"SAZ metadata encoding declaration disagrees with its bytes.",
"",
cause,
);
throw new CaptureFormatError(
"unsupported",
"SAZ metadata must use UTF-8 or BOM-marked UTF-16.",
"malformed",
cause.reason === "encoding-mismatch"
? "SAZ metadata encoding declaration disagrees with its bytes."
: "SAZ metadata contains invalid bytes for its detected XML encoding.",
"",
cause,
);
+82 -54
View File
@@ -701,9 +701,10 @@ it("reports zero-byte SAZ HTTP members as null instead of rejecting the archive"
]);
const path = await fixture(bytes);
const capture = webNetworkCaptureSchema.parse(
(await cli(path)).normalized_result,
(await cli(path, ["--record", "0", "--record", "1"])).normalized_result,
);
expect(capture.total_records).toBe(2);
expect(capture.records.map(({ ordinal }) => ordinal)).toEqual([0, 1]);
expect(capture.limitations).toContainEqual(
expect.stringMatching(
/\babsent\b[^.]*\bzero-byte\b[^.]*\bHTTP members\b[^.]*\bnull\b/iu,
@@ -728,58 +729,45 @@ it("reports zero-byte SAZ HTTP members as null instead of rejecting the archive"
expect(capture.records[1]?.limitations).toEqual([
"The stored request member is empty (zero bytes); the request is reported as null.",
]);
const selected = webNetworkCaptureSchema.parse(
(await cli(path, ["--record", "0"])).normalized_result,
);
expect(selected.records).toMatchObject([
{ ordinal: 0, reported: { response: null } },
]);
});
it("validates SAZ XML declarations against archive member bytes at the CLI boundary", async () => {
const withMetadata = (metadata: Buffer): [string, Buffer][] =>
members().map(([name, bytes]) =>
name === "raw/0042_m.xml" ? [name, metadata] : [name, bytes],
);
const utf8 = Buffer.from(
'<?xml version="1.0" encoding="UTF-8"?><Session SID="0042"/>',
);
const utf16le = Buffer.concat([
Buffer.from([0xff, 0xfe]),
Buffer.from(
'<?xml version="1.0" encoding="UTF-16"?><Session SID="0042"/>',
"utf16le",
it.each([
{
name: "a declaration that disagrees with the bytes",
metadata: Buffer.from(
'<?xml version="1.0" encoding="UTF-16"?><Session SID="1"/>',
),
]);
for (const metadata of [utf8, utf16le]) {
const result = await cli(
await fixture(await archive(withMetadata(metadata))),
);
expect(
webNetworkCaptureSchema.parse(result.normalized_result).records,
).toHaveLength(2);
}
const mismatch = Buffer.from(
'<?xml version="1.0" encoding="UTF-16"?><Session SID="0042"/>',
);
const rejected = await cliResponse(
await fixture(await archive(withMetadata(mismatch))),
["--record", "1"],
);
expect(rejected.ok).toBe(false);
expect(JSON.parse(rejected.stdout)).toMatchObject({
category: "invalid_input",
details: {
issues: [
expect.objectContaining({
message: expect.stringContaining(
"SAZ metadata encoding declaration disagrees with its bytes",
),
}),
],
},
});
});
issue: /encoding declaration disagrees/u,
},
{
name: "corrupt UTF-8 bytes",
metadata: Buffer.concat([
Buffer.from('<?xml version="1.0" encoding="UTF-8"?><Session>'),
Buffer.from([0xff]),
Buffer.from("</Session>"),
]),
issue: /invalid bytes/u,
},
])(
"rejects SAZ metadata with $name before selection",
async ({ metadata, issue }) => {
const bytes = await archive([
["raw/1_m.xml", metadata],
["raw/2_c.txt", Buffer.from("GET /valid HTTP/1.1\r\n\r\n")],
]);
const rejected = await cliResponse(await fixture(bytes), ["--record", "1"]);
expect(rejected.ok).toBe(false);
expect(JSON.parse(rejected.stdout)).toMatchObject({
code: "invalid_request",
category: "invalid_input",
details: {
issues: [
expect.objectContaining({ message: expect.stringMatching(issue) }),
],
},
});
},
);
it("excludes explicit literals from XML, HTTP byte views, member coordinates and binary sidecars", async () => {
const path = await fixture(await archive());
@@ -821,7 +809,7 @@ it("excludes explicit literals from XML, HTTP byte views, member coordinates and
expect(text).not.toContain(secret);
});
it("retains folded headers, stored chunk framing and BOM-marked UTF-16 metadata", async () => {
it("retains folded headers, stored chunk framing and both XML encodings", async () => {
const body = "3\r\none\r\n0\r\n\r\n";
const path = await fixture(
await archive([
@@ -834,7 +822,7 @@ it("retains folded headers, stored chunk framing and BOM-marked UTF-16 metadata"
[
"raw/1_m.xml",
Buffer.from(
'\ufeff<?xml version="1.0" encoding="utf-16"?><Session><Note>\u03b2</Note><Source><![CDATA[<!DOCTYPE html>]]></Source></Session>',
'\ufeff<?xml version="1.0" encoding="utf-16"?><Session SID="1"><Note>\u03b2</Note><Source><![CDATA[<!DOCTYPE html>]]></Source></Session>',
"utf16le",
),
],
@@ -844,6 +832,12 @@ it("retains folded headers, stored chunk framing and BOM-marked UTF-16 metadata"
"CONNECT example.test:443 HTTP/1.1\nHost: example.test:443\n\n",
),
],
[
"raw/2_m.xml",
Buffer.from(
'<?xml version="1.0" encoding="UTF-8"?><Session SID="2"><Note>雪</Note></Session>',
),
],
]),
);
const evidence = await cli(path);
@@ -864,7 +858,41 @@ it("retains folded headers, stored chunk framing and BOM-marked UTF-16 metadata"
content_base64: Buffer.from(body).toString("base64"),
}),
);
expect(JSON.stringify(capture.records[0]?.reported)).toContain("\u03b2");
expect(capture.records[0]?.reported).toHaveProperty(
"metadata.document.children.1",
expect.objectContaining({
name: "Session",
attributes: [{ name: "SID", value: "1" }],
children: [
expect.objectContaining({
name: "Note",
children: [expect.objectContaining({ name: "#text", value: "β" })],
}),
expect.objectContaining({
name: "Source",
children: [
expect.objectContaining({
name: "#cdata-section",
value: "<!DOCTYPE html>",
}),
],
}),
],
}),
);
expect(capture.records[1]?.reported).toHaveProperty(
"metadata.document.children.1",
expect.objectContaining({
name: "Session",
attributes: [{ name: "SID", value: "2" }],
children: [
expect.objectContaining({
name: "Note",
children: [expect.objectContaining({ name: "#text", value: "雪" })],
}),
],
}),
);
expect(capture.records[1]?.reported).toMatchObject({
request: { method: "CONNECT", url: "example.test:443", body: "" },
});
@@ -889,7 +917,7 @@ it.each([
entries.push([
"raw/8_m.xml",
Buffer.from(
'<!DOCTYPE Session [<!ENTITY external SYSTEM "file:///private-input">]><Session>&external;</Session>',
'<!DOCTYPE Session [<!ENTITY external SYSTEM "file:///private-input">]><Session/>',
),
]);
if (variant === "unknown-member")