diff --git a/src/browser/history/protobuf/ProtobufSchema.ts b/src/browser/history/protobuf/ProtobufSchema.ts index 3bbcf545b3..3b6aed3c1f 100644 --- a/src/browser/history/protobuf/ProtobufSchema.ts +++ b/src/browser/history/protobuf/ProtobufSchema.ts @@ -1,4 +1,5 @@ import { captureBudgets } from "../CaptureBudgets.js"; +import { strictUtf8Decoder } from "../../../domain/strictUtf8.js"; import { err, ok, type Result } from "../../../domain/result.js"; import { createFileRegistry, @@ -17,7 +18,11 @@ import { type DescriptorProto, type FileDescriptorSet, } from "@bufbuild/protobuf/wkt"; -import { BinaryReader } from "@bufbuild/protobuf/wire"; +import { + BinaryReader, + configureTextEncoding, + getTextEncoding, +} from "@bufbuild/protobuf/wire"; import type { ProtobufField } from "../../../domain/protobufWire.js"; import { CaptureFormatError } from "../CaptureFormatError.js"; import { @@ -61,6 +66,27 @@ const floatingValue = (value: number): string => // Descriptor custom options and future fields remain opaque identity facts. const descriptorEquality = { registry: createRegistry(), unknown: true }; +/** Descriptor strings are protobuf field data, including any initial U+FEFF. */ +const descriptorUtf8 = strictUtf8Decoder("preserve"); +const readDescriptorSet = (bytes: Uint8Array): FileDescriptorSet => { + const encoding = getTextEncoding(); + // The SDK has no per-fromBinary text-decoder option. Scope its encoding + // hook to this synchronous decode; never change ambient TextDecoder or + // leave different encoding semantics installed for another SDK consumer. + configureTextEncoding({ + ...encoding, + decodeUtf8: (value) => descriptorUtf8.decode(value), + }); + try { + return fromBinary(FileDescriptorSetSchema, bytes, { + readUnknownFields: true, + recursionLimit: captureBudgets().depth, + }); + } finally { + configureTextEncoding(encoding); + } +}; + const chargeQualifiedName = ( budget: ProtobufBudget, name: string, @@ -259,10 +285,7 @@ export class ProtobufSchema implements WireSchema { }); let descriptor: FileDescriptorSet; try { - descriptor = fromBinary(FileDescriptorSetSchema, bytes, { - readUnknownFields: true, - recursionLimit: captureBudgets().depth, - }); + descriptor = readDescriptorSet(bytes); } catch (cause) { return err({ kind: "malformed-descriptor", diff --git a/src/browser/history/protobuf/ProtobufWire.test.ts b/src/browser/history/protobuf/ProtobufWire.test.ts index 156a7a4fd8..4299645e07 100644 --- a/src/browser/history/protobuf/ProtobufWire.test.ts +++ b/src/browser/history/protobuf/ProtobufWire.test.ts @@ -1,10 +1,14 @@ import { expect, it } from "vitest"; import { create, toBinary } from "@bufbuild/protobuf"; import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt"; +import { getTextEncoding } from "@bufbuild/protobuf/wire"; import { decodeProtobufWire, ProtobufBudget } from "./ProtobufWire.js"; import { decodeProtobufBody } from "./ProtobufBody.js"; import { ProtobufSchema } from "./ProtobufSchema.js"; -import { protobufCaptureDescriptor } from "../../../../tests/fixtures/protobufCapture.js"; +import { + protobufBomDescriptor, + protobufCaptureDescriptor, +} from "../../../../tests/fixtures/protobufCapture.js"; const same = (value: string) => value; const wire = (hex: string) => { @@ -23,6 +27,89 @@ const readSchema = async (...args: Parameters) => { if (!admitted.ok) throw new Error(admitted.error.message); return admitted.value; }; + +it("keeps protoc's distinct BOM-prefixed import identities when annotating nested wire fields", async () => { + const encoding = getTextEncoding(); + const budget = new ProtobufBudget(); + const schema = await readSchema( + protobufBomDescriptor(), + "capture.Root", + budget, + same, + ); + const bytes = Buffer.from("0a02080712050a036f6e65", "hex"); + const decoded = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + budget, + same, + schema, + ); + expect(decoded.error).toBeUndefined(); + expect(decoded.fields).toMatchObject([ + { schema_name: "plain", schema_type: "plain.Item" }, + { schema_name: "number", schema_type: "uint32", schema_value: "7" }, + { schema_name: "marked", schema_type: "marked.Item" }, + { schema_name: "text", schema_type: "string", schema_value: "one" }, + ]); + expect(getTextEncoding()).toEqual(encoding); +}); + +it("retains BOM-bearing option differences in repeated descriptor files", async () => { + const encoding = getTextEncoding(); + const file = { + name: "same.proto", + syntax: "proto3", + messageType: [{ name: "M" }], + }; + const bytes = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { ...file, options: { javaPackage: "plain" } }, + { ...file, options: { javaPackage: "\uFEFFplain" } }, + ], + }), + ); + expect( + await ProtobufSchema.read(bytes, "M", new ProtobufBudget(), same), + ).toMatchObject({ + ok: false, + error: { + kind: "malformed-descriptor", + message: "Conflicting file identity in FileDescriptorSet", + }, + }); + expect(getTextEncoding()).toEqual(encoding); +}); + +it("restores SDK encoding after a descriptor string fails UTF-8 decoding", async () => { + const encoding = getTextEncoding(); + const bytes = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "bad.proto", + messageType: [{ name: "M" }], + options: { javaPackage: "unique-invalid-text" }, + }, + ], + }), + ); + // A string option has no registry-name constraints that could hide a lossy + // SDK decode. Corrupt its first byte without changing any wire extents. + bytes[Buffer.from(bytes).indexOf("unique-invalid-text")] = 0xff; + expect( + await ProtobufSchema.read(bytes, "M", new ProtobufBudget(), same), + ).toMatchObject({ + ok: false, + error: { kind: "malformed-descriptor" }, + }); + expect(getTextEncoding()).toEqual(encoding); +}); const source = { pointer: "/response/content/text", content_type: "application/grpc-web+proto", diff --git a/tests/acceptance/tools/protobufDescriptorFiles.test.ts b/tests/acceptance/tools/protobufDescriptorFiles.test.ts index b65b98ea3a..dac5691cd2 100644 --- a/tests/acceptance/tools/protobufDescriptorFiles.test.ts +++ b/tests/acceptance/tools/protobufDescriptorFiles.test.ts @@ -17,7 +17,10 @@ import { analysisViewResultSchema } from "../../../src/domain/analysisView/analy import { parseEvidence } from "../../../src/domain/evidence.js"; import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -import { protobufHar } from "../../fixtures/protobufCapture.js"; +import { + protobufBomDescriptor, + protobufHar, +} from "../../fixtures/protobufCapture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -85,6 +88,11 @@ const conflictingFiles = (change: string, reverse: boolean) => { } else if (change === "presence") { shared.options = create(FileOptionsSchema); duplicate.options = create(FileOptionsSchema, { javaMultipleFiles: false }); + } else if (change === "bom-option") { + shared.options = create(FileOptionsSchema, { javaPackage: "plain" }); + duplicate.options = create(FileOptionsSchema, { + javaPackage: "\uFEFFplain", + }); } else if (change === "opaque-file") { duplicate.$unknown = [ { no: 50000, wireType: WireType.Varint, data: Uint8Array.of(1) }, @@ -125,6 +133,7 @@ cliTest( "package", "imports", "presence", + "bom-option", "opaque-file", "opaque-option", ]) @@ -206,6 +215,75 @@ cliTest( }, ); +cliTest( + "preserves protoc BOM-prefixed imports and their capture evidence through CLI/MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-protobuf-bom-imports-"); + const descriptor = join(root, "schema.pb"); + const capture = join(root, "capture.har"); + const bytes = protobufBomDescriptor(); + const text = protobufHar( + Buffer.from("0a02080712050a036f6e65", "hex"), + "application/x-protobuf", + ); + await writeFile(descriptor, bytes); + await writeFile(capture, text); + const output = await cli.run({ + arguments: [ + "inspect-web-network-capture", + capture, + "har", + "--record", + "0", + "--protobuf-schema", + descriptor, + "--protobuf-message", + "capture.Root", + "--json", + ], + }); + expect(output.exitCode, output.stdout + output.stderr).toBe(0); + const evidence = parseEvidence(output.json); + const report = webNetworkCaptureSchema.parse(evidence.normalized_result); + expect(report.artifact).toMatchObject({ + path: capture, + sha256: sha256(Buffer.from(text)), + }); + expect(report.protobuf_coverage?.status).toBe("complete"); + expect(report.records[0]?.protobuf?.[0]).toMatchObject({ + schema: { + descriptor_set_path: descriptor, + descriptor_set_sha256: sha256(bytes), + descriptor_set_bytes: bytes.length, + }, + fields: [ + { schema_name: "plain", schema_type: "plain.Item" }, + { schema_name: "number", schema_value: "7" }, + { schema_name: "marked", schema_type: "marked.Item" }, + { schema_name: "text", schema_value: "one" }, + ], + }); + const call = await cli.sharedMcpCall!(); + const response = await call("inspect_web_network_capture", { + capture_path: capture, + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: "capture.Root", + }, + }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(parseEvidence(response.structuredContent).evidence_id).toBe( + evidence.evidence_id, + ); + expect((await readFile(descriptor)).equals(bytes)).toBe(true); + expect(await readFile(capture, "utf8")).toBe(text); + }, +); + /** Reorder known FileDescriptorProto fields without changing their facts. */ const reorderedSharedFile = () => { const shared = descriptorDefinition().file[0]!; diff --git a/tests/fixtures/protobufCapture.ts b/tests/fixtures/protobufCapture.ts index 6893cedd93..375c872e2d 100644 --- a/tests/fixtures/protobufCapture.ts +++ b/tests/fixtures/protobufCapture.ts @@ -1,6 +1,24 @@ import { create, toBinary } from "@bufbuild/protobuf"; import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt"; +/** + * protoc 31.1 --include_imports golden, with two distinct imported filenames: + * dep.proto: syntax="proto3"; package plain; message Item { uint32 number=1; } + * \uFEFFdep.proto: syntax="proto3"; package marked; message Item { string text=1; } + * root.proto imports both and defines capture.Root with plain.Item plain=1 + * and marked.Item marked=2. U+FEFF is filename data, not a document BOM. + */ +export const protobufBomDescriptor = (): Buffer => + Buffer.from( + "0a3a0a096465702e70726f746f1205706c61696e221e0a044974656d12160a066e756d6265721801200128" + + "0d52066e756d626572620670726f746f330a3a0a0cefbbbf6465702e70726f746f12066d61726b6564221a" + + "0a044974656d12120a0474657874180120012809520474657874620670726f746f330a87010a0a726f6f74" + + "2e70726f746f1207636170747572651a096465702e70726f746f1a0cefbbbf6465702e70726f746f224f0a" + + "04526f6f7412210a05706c61696e18012001280b320b2e706c61696e2e4974656d5205706c61696e12240a" + + "066d61726b656418022001280b320c2e6d61726b65642e4974656d52066d61726b6564620670726f746f33", + "hex", + ); + export const protobufFrame = (body: Buffer, flag = 0): Buffer => { const header = Buffer.alloc(5); header[0] = flag;