fix(protobuf): preserve descriptor string identity (#11834)

Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
ShuaixinHuangandShuaixinHuang authored and GitHub committed 2026-11-18 14:59:49 -05:00
1 parent e7ed93004a
commit 0db5f93df6
4 files changed
+213 -7

No files matched your search

+28 -5
View File
@@ -1,4 +1,5 @@
import { captureBudgets } from "../CaptureBudgets.js";
import { strictUtf8Decoder } from "../../../domain/strictUtf8.js";
import { err, ok, type Result } from "../../../domain/result.js";
import {
createFileRegistry,
@@ -17,7 +18,11 @@ import {
type DescriptorProto,
type FileDescriptorSet,
} from "@bufbuild/protobuf/wkt";
import { BinaryReader } from "@bufbuild/protobuf/wire";
import {
BinaryReader,
configureTextEncoding,
getTextEncoding,
} from "@bufbuild/protobuf/wire";
import type { ProtobufField } from "../../../domain/protobufWire.js";
import { CaptureFormatError } from "../CaptureFormatError.js";
import {
@@ -61,6 +66,27 @@ const floatingValue = (value: number): string =>
// Descriptor custom options and future fields remain opaque identity facts.
const descriptorEquality = { registry: createRegistry(), unknown: true };
/** Descriptor strings are protobuf field data, including any initial U+FEFF. */
const descriptorUtf8 = strictUtf8Decoder("preserve");
const readDescriptorSet = (bytes: Uint8Array): FileDescriptorSet => {
const encoding = getTextEncoding();
// The SDK has no per-fromBinary text-decoder option. Scope its encoding
// hook to this synchronous decode; never change ambient TextDecoder or
// leave different encoding semantics installed for another SDK consumer.
configureTextEncoding({
...encoding,
decodeUtf8: (value) => descriptorUtf8.decode(value),
});
try {
return fromBinary(FileDescriptorSetSchema, bytes, {
readUnknownFields: true,
recursionLimit: captureBudgets().depth,
});
} finally {
configureTextEncoding(encoding);
}
};
const chargeQualifiedName = (
budget: ProtobufBudget,
name: string,
@@ -259,10 +285,7 @@ export class ProtobufSchema implements WireSchema {
});
let descriptor: FileDescriptorSet;
try {
descriptor = fromBinary(FileDescriptorSetSchema, bytes, {
readUnknownFields: true,
recursionLimit: captureBudgets().depth,
});
descriptor = readDescriptorSet(bytes);
} catch (cause) {
return err({
kind: "malformed-descriptor",
@@ -1,10 +1,14 @@
import { expect, it } from "vitest";
import { create, toBinary } from "@bufbuild/protobuf";
import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt";
import { getTextEncoding } from "@bufbuild/protobuf/wire";
import { decodeProtobufWire, ProtobufBudget } from "./ProtobufWire.js";
import { decodeProtobufBody } from "./ProtobufBody.js";
import { ProtobufSchema } from "./ProtobufSchema.js";
import { protobufCaptureDescriptor } from "../../../../tests/fixtures/protobufCapture.js";
import {
protobufBomDescriptor,
protobufCaptureDescriptor,
} from "../../../../tests/fixtures/protobufCapture.js";
const same = (value: string) => value;
const wire = (hex: string) => {
@@ -23,6 +27,89 @@ const readSchema = async (...args: Parameters<typeof ProtobufSchema.read>) => {
if (!admitted.ok) throw new Error(admitted.error.message);
return admitted.value;
};
it("keeps protoc's distinct BOM-prefixed import identities when annotating nested wire fields", async () => {
const encoding = getTextEncoding();
const budget = new ProtobufBudget();
const schema = await readSchema(
protobufBomDescriptor(),
"capture.Root",
budget,
same,
);
const bytes = Buffer.from("0a02080712050a036f6e65", "hex");
const decoded = await decodeProtobufWire(
bytes,
0,
bytes.length,
0,
budget,
same,
schema,
);
expect(decoded.error).toBeUndefined();
expect(decoded.fields).toMatchObject([
{ schema_name: "plain", schema_type: "plain.Item" },
{ schema_name: "number", schema_type: "uint32", schema_value: "7" },
{ schema_name: "marked", schema_type: "marked.Item" },
{ schema_name: "text", schema_type: "string", schema_value: "one" },
]);
expect(getTextEncoding()).toEqual(encoding);
});
it("retains BOM-bearing option differences in repeated descriptor files", async () => {
const encoding = getTextEncoding();
const file = {
name: "same.proto",
syntax: "proto3",
messageType: [{ name: "M" }],
};
const bytes = toBinary(
FileDescriptorSetSchema,
create(FileDescriptorSetSchema, {
file: [
{ ...file, options: { javaPackage: "plain" } },
{ ...file, options: { javaPackage: "\uFEFFplain" } },
],
}),
);
expect(
await ProtobufSchema.read(bytes, "M", new ProtobufBudget(), same),
).toMatchObject({
ok: false,
error: {
kind: "malformed-descriptor",
message: "Conflicting file identity in FileDescriptorSet",
},
});
expect(getTextEncoding()).toEqual(encoding);
});
it("restores SDK encoding after a descriptor string fails UTF-8 decoding", async () => {
const encoding = getTextEncoding();
const bytes = toBinary(
FileDescriptorSetSchema,
create(FileDescriptorSetSchema, {
file: [
{
name: "bad.proto",
messageType: [{ name: "M" }],
options: { javaPackage: "unique-invalid-text" },
},
],
}),
);
// A string option has no registry-name constraints that could hide a lossy
// SDK decode. Corrupt its first byte without changing any wire extents.
bytes[Buffer.from(bytes).indexOf("unique-invalid-text")] = 0xff;
expect(
await ProtobufSchema.read(bytes, "M", new ProtobufBudget(), same),
).toMatchObject({
ok: false,
error: { kind: "malformed-descriptor" },
});
expect(getTextEncoding()).toEqual(encoding);
});
const source = {
pointer: "/response/content/text",
content_type: "application/grpc-web+proto",
@@ -17,7 +17,10 @@ import { analysisViewResultSchema } from "../../../src/domain/analysisView/analy
import { parseEvidence } from "../../../src/domain/evidence.js";
import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { protobufHar } from "../../fixtures/protobufCapture.js";
import {
protobufBomDescriptor,
protobufHar,
} from "../../fixtures/protobufCapture.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { cliTest } from "../../support/cli/cliFixture.js";
@@ -85,6 +88,11 @@ const conflictingFiles = (change: string, reverse: boolean) => {
} else if (change === "presence") {
shared.options = create(FileOptionsSchema);
duplicate.options = create(FileOptionsSchema, { javaMultipleFiles: false });
} else if (change === "bom-option") {
shared.options = create(FileOptionsSchema, { javaPackage: "plain" });
duplicate.options = create(FileOptionsSchema, {
javaPackage: "\uFEFFplain",
});
} else if (change === "opaque-file") {
duplicate.$unknown = [
{ no: 50000, wireType: WireType.Varint, data: Uint8Array.of(1) },
@@ -125,6 +133,7 @@ cliTest(
"package",
"imports",
"presence",
"bom-option",
"opaque-file",
"opaque-option",
])
@@ -206,6 +215,75 @@ cliTest(
},
);
cliTest(
"preserves protoc BOM-prefixed imports and their capture evidence through CLI/MCP",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-protobuf-bom-imports-");
const descriptor = join(root, "schema.pb");
const capture = join(root, "capture.har");
const bytes = protobufBomDescriptor();
const text = protobufHar(
Buffer.from("0a02080712050a036f6e65", "hex"),
"application/x-protobuf",
);
await writeFile(descriptor, bytes);
await writeFile(capture, text);
const output = await cli.run({
arguments: [
"inspect-web-network-capture",
capture,
"har",
"--record",
"0",
"--protobuf-schema",
descriptor,
"--protobuf-message",
"capture.Root",
"--json",
],
});
expect(output.exitCode, output.stdout + output.stderr).toBe(0);
const evidence = parseEvidence(output.json);
const report = webNetworkCaptureSchema.parse(evidence.normalized_result);
expect(report.artifact).toMatchObject({
path: capture,
sha256: sha256(Buffer.from(text)),
});
expect(report.protobuf_coverage?.status).toBe("complete");
expect(report.records[0]?.protobuf?.[0]).toMatchObject({
schema: {
descriptor_set_path: descriptor,
descriptor_set_sha256: sha256(bytes),
descriptor_set_bytes: bytes.length,
},
fields: [
{ schema_name: "plain", schema_type: "plain.Item" },
{ schema_name: "number", schema_value: "7" },
{ schema_name: "marked", schema_type: "marked.Item" },
{ schema_name: "text", schema_value: "one" },
],
});
const call = await cli.sharedMcpCall!();
const response = await call("inspect_web_network_capture", {
capture_path: capture,
format: "har",
record_ordinals: [0],
protobuf: {
schema: {
descriptor_set_path: descriptor,
message_type: "capture.Root",
},
},
});
expect(response.isError, JSON.stringify(response)).not.toBe(true);
expect(parseEvidence(response.structuredContent).evidence_id).toBe(
evidence.evidence_id,
);
expect((await readFile(descriptor)).equals(bytes)).toBe(true);
expect(await readFile(capture, "utf8")).toBe(text);
},
);
/** Reorder known FileDescriptorProto fields without changing their facts. */
const reorderedSharedFile = () => {
const shared = descriptorDefinition().file[0]!;
+18
View File
@@ -1,6 +1,24 @@
import { create, toBinary } from "@bufbuild/protobuf";
import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt";
/**
* protoc 31.1 --include_imports golden, with two distinct imported filenames:
* dep.proto: syntax="proto3"; package plain; message Item { uint32 number=1; }
* \uFEFFdep.proto: syntax="proto3"; package marked; message Item { string text=1; }
* root.proto imports both and defines capture.Root with plain.Item plain=1
* and marked.Item marked=2. U+FEFF is filename data, not a document BOM.
*/
export const protobufBomDescriptor = (): Buffer =>
Buffer.from(
"0a3a0a096465702e70726f746f1205706c61696e221e0a044974656d12160a066e756d6265721801200128" +
"0d52066e756d626572620670726f746f330a3a0a0cefbbbf6465702e70726f746f12066d61726b6564221a" +
"0a044974656d12120a0474657874180120012809520474657874620670726f746f330a87010a0a726f6f74" +
"2e70726f746f1207636170747572651a096465702e70726f746f1a0cefbbbf6465702e70726f746f224f0a" +
"04526f6f7412210a05706c61696e18012001280b320b2e706c61696e2e4974656d5205706c61696e12240a" +
"066d61726b656418022001280b320c2e6d61726b65642e4974656d52066d61726b6564620670726f746f33",
"hex",
);
export const protobufFrame = (body: Buffer, flag = 0): Buffer => {
const header = Buffer.alloc(5);
header[0] = flag;