mirror of
https://github.com/morluto/rea.git
synced 2026-11-18 20:53:00 +00:00
fix(protobuf): preserve descriptor string identity (#11834)
Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
1 parent
e7ed93004a
commit
0db5f93df6
4 files changed
+213
-7
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import { captureBudgets } from "../CaptureBudgets.js";
|
||||
import { strictUtf8Decoder } from "../../../domain/strictUtf8.js";
|
||||
import { err, ok, type Result } from "../../../domain/result.js";
|
||||
import {
|
||||
createFileRegistry,
|
||||
@@ -17,7 +18,11 @@ import {
|
||||
type DescriptorProto,
|
||||
type FileDescriptorSet,
|
||||
} from "@bufbuild/protobuf/wkt";
|
||||
import { BinaryReader } from "@bufbuild/protobuf/wire";
|
||||
import {
|
||||
BinaryReader,
|
||||
configureTextEncoding,
|
||||
getTextEncoding,
|
||||
} from "@bufbuild/protobuf/wire";
|
||||
import type { ProtobufField } from "../../../domain/protobufWire.js";
|
||||
import { CaptureFormatError } from "../CaptureFormatError.js";
|
||||
import {
|
||||
@@ -61,6 +66,27 @@ const floatingValue = (value: number): string =>
|
||||
// Descriptor custom options and future fields remain opaque identity facts.
|
||||
const descriptorEquality = { registry: createRegistry(), unknown: true };
|
||||
|
||||
/** Descriptor strings are protobuf field data, including any initial U+FEFF. */
|
||||
const descriptorUtf8 = strictUtf8Decoder("preserve");
|
||||
const readDescriptorSet = (bytes: Uint8Array): FileDescriptorSet => {
|
||||
const encoding = getTextEncoding();
|
||||
// The SDK has no per-fromBinary text-decoder option. Scope its encoding
|
||||
// hook to this synchronous decode; never change ambient TextDecoder or
|
||||
// leave different encoding semantics installed for another SDK consumer.
|
||||
configureTextEncoding({
|
||||
...encoding,
|
||||
decodeUtf8: (value) => descriptorUtf8.decode(value),
|
||||
});
|
||||
try {
|
||||
return fromBinary(FileDescriptorSetSchema, bytes, {
|
||||
readUnknownFields: true,
|
||||
recursionLimit: captureBudgets().depth,
|
||||
});
|
||||
} finally {
|
||||
configureTextEncoding(encoding);
|
||||
}
|
||||
};
|
||||
|
||||
const chargeQualifiedName = (
|
||||
budget: ProtobufBudget,
|
||||
name: string,
|
||||
@@ -259,10 +285,7 @@ export class ProtobufSchema implements WireSchema {
|
||||
});
|
||||
let descriptor: FileDescriptorSet;
|
||||
try {
|
||||
descriptor = fromBinary(FileDescriptorSetSchema, bytes, {
|
||||
readUnknownFields: true,
|
||||
recursionLimit: captureBudgets().depth,
|
||||
});
|
||||
descriptor = readDescriptorSet(bytes);
|
||||
} catch (cause) {
|
||||
return err({
|
||||
kind: "malformed-descriptor",
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { create, toBinary } from "@bufbuild/protobuf";
|
||||
import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt";
|
||||
import { getTextEncoding } from "@bufbuild/protobuf/wire";
|
||||
import { decodeProtobufWire, ProtobufBudget } from "./ProtobufWire.js";
|
||||
import { decodeProtobufBody } from "./ProtobufBody.js";
|
||||
import { ProtobufSchema } from "./ProtobufSchema.js";
|
||||
import { protobufCaptureDescriptor } from "../../../../tests/fixtures/protobufCapture.js";
|
||||
import {
|
||||
protobufBomDescriptor,
|
||||
protobufCaptureDescriptor,
|
||||
} from "../../../../tests/fixtures/protobufCapture.js";
|
||||
|
||||
const same = (value: string) => value;
|
||||
const wire = (hex: string) => {
|
||||
@@ -23,6 +27,89 @@ const readSchema = async (...args: Parameters<typeof ProtobufSchema.read>) => {
|
||||
if (!admitted.ok) throw new Error(admitted.error.message);
|
||||
return admitted.value;
|
||||
};
|
||||
|
||||
it("keeps protoc's distinct BOM-prefixed import identities when annotating nested wire fields", async () => {
|
||||
const encoding = getTextEncoding();
|
||||
const budget = new ProtobufBudget();
|
||||
const schema = await readSchema(
|
||||
protobufBomDescriptor(),
|
||||
"capture.Root",
|
||||
budget,
|
||||
same,
|
||||
);
|
||||
const bytes = Buffer.from("0a02080712050a036f6e65", "hex");
|
||||
const decoded = await decodeProtobufWire(
|
||||
bytes,
|
||||
0,
|
||||
bytes.length,
|
||||
0,
|
||||
budget,
|
||||
same,
|
||||
schema,
|
||||
);
|
||||
expect(decoded.error).toBeUndefined();
|
||||
expect(decoded.fields).toMatchObject([
|
||||
{ schema_name: "plain", schema_type: "plain.Item" },
|
||||
{ schema_name: "number", schema_type: "uint32", schema_value: "7" },
|
||||
{ schema_name: "marked", schema_type: "marked.Item" },
|
||||
{ schema_name: "text", schema_type: "string", schema_value: "one" },
|
||||
]);
|
||||
expect(getTextEncoding()).toEqual(encoding);
|
||||
});
|
||||
|
||||
it("retains BOM-bearing option differences in repeated descriptor files", async () => {
|
||||
const encoding = getTextEncoding();
|
||||
const file = {
|
||||
name: "same.proto",
|
||||
syntax: "proto3",
|
||||
messageType: [{ name: "M" }],
|
||||
};
|
||||
const bytes = toBinary(
|
||||
FileDescriptorSetSchema,
|
||||
create(FileDescriptorSetSchema, {
|
||||
file: [
|
||||
{ ...file, options: { javaPackage: "plain" } },
|
||||
{ ...file, options: { javaPackage: "\uFEFFplain" } },
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
await ProtobufSchema.read(bytes, "M", new ProtobufBudget(), same),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "malformed-descriptor",
|
||||
message: "Conflicting file identity in FileDescriptorSet",
|
||||
},
|
||||
});
|
||||
expect(getTextEncoding()).toEqual(encoding);
|
||||
});
|
||||
|
||||
it("restores SDK encoding after a descriptor string fails UTF-8 decoding", async () => {
|
||||
const encoding = getTextEncoding();
|
||||
const bytes = toBinary(
|
||||
FileDescriptorSetSchema,
|
||||
create(FileDescriptorSetSchema, {
|
||||
file: [
|
||||
{
|
||||
name: "bad.proto",
|
||||
messageType: [{ name: "M" }],
|
||||
options: { javaPackage: "unique-invalid-text" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
// A string option has no registry-name constraints that could hide a lossy
|
||||
// SDK decode. Corrupt its first byte without changing any wire extents.
|
||||
bytes[Buffer.from(bytes).indexOf("unique-invalid-text")] = 0xff;
|
||||
expect(
|
||||
await ProtobufSchema.read(bytes, "M", new ProtobufBudget(), same),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "malformed-descriptor" },
|
||||
});
|
||||
expect(getTextEncoding()).toEqual(encoding);
|
||||
});
|
||||
const source = {
|
||||
pointer: "/response/content/text",
|
||||
content_type: "application/grpc-web+proto",
|
||||
|
||||
@@ -17,7 +17,10 @@ import { analysisViewResultSchema } from "../../../src/domain/analysisView/analy
|
||||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
import { protobufHar } from "../../fixtures/protobufCapture.js";
|
||||
import {
|
||||
protobufBomDescriptor,
|
||||
protobufHar,
|
||||
} from "../../fixtures/protobufCapture.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
|
||||
@@ -85,6 +88,11 @@ const conflictingFiles = (change: string, reverse: boolean) => {
|
||||
} else if (change === "presence") {
|
||||
shared.options = create(FileOptionsSchema);
|
||||
duplicate.options = create(FileOptionsSchema, { javaMultipleFiles: false });
|
||||
} else if (change === "bom-option") {
|
||||
shared.options = create(FileOptionsSchema, { javaPackage: "plain" });
|
||||
duplicate.options = create(FileOptionsSchema, {
|
||||
javaPackage: "\uFEFFplain",
|
||||
});
|
||||
} else if (change === "opaque-file") {
|
||||
duplicate.$unknown = [
|
||||
{ no: 50000, wireType: WireType.Varint, data: Uint8Array.of(1) },
|
||||
@@ -125,6 +133,7 @@ cliTest(
|
||||
"package",
|
||||
"imports",
|
||||
"presence",
|
||||
"bom-option",
|
||||
"opaque-file",
|
||||
"opaque-option",
|
||||
])
|
||||
@@ -206,6 +215,75 @@ cliTest(
|
||||
},
|
||||
);
|
||||
|
||||
cliTest(
|
||||
"preserves protoc BOM-prefixed imports and their capture evidence through CLI/MCP",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-protobuf-bom-imports-");
|
||||
const descriptor = join(root, "schema.pb");
|
||||
const capture = join(root, "capture.har");
|
||||
const bytes = protobufBomDescriptor();
|
||||
const text = protobufHar(
|
||||
Buffer.from("0a02080712050a036f6e65", "hex"),
|
||||
"application/x-protobuf",
|
||||
);
|
||||
await writeFile(descriptor, bytes);
|
||||
await writeFile(capture, text);
|
||||
const output = await cli.run({
|
||||
arguments: [
|
||||
"inspect-web-network-capture",
|
||||
capture,
|
||||
"har",
|
||||
"--record",
|
||||
"0",
|
||||
"--protobuf-schema",
|
||||
descriptor,
|
||||
"--protobuf-message",
|
||||
"capture.Root",
|
||||
"--json",
|
||||
],
|
||||
});
|
||||
expect(output.exitCode, output.stdout + output.stderr).toBe(0);
|
||||
const evidence = parseEvidence(output.json);
|
||||
const report = webNetworkCaptureSchema.parse(evidence.normalized_result);
|
||||
expect(report.artifact).toMatchObject({
|
||||
path: capture,
|
||||
sha256: sha256(Buffer.from(text)),
|
||||
});
|
||||
expect(report.protobuf_coverage?.status).toBe("complete");
|
||||
expect(report.records[0]?.protobuf?.[0]).toMatchObject({
|
||||
schema: {
|
||||
descriptor_set_path: descriptor,
|
||||
descriptor_set_sha256: sha256(bytes),
|
||||
descriptor_set_bytes: bytes.length,
|
||||
},
|
||||
fields: [
|
||||
{ schema_name: "plain", schema_type: "plain.Item" },
|
||||
{ schema_name: "number", schema_value: "7" },
|
||||
{ schema_name: "marked", schema_type: "marked.Item" },
|
||||
{ schema_name: "text", schema_value: "one" },
|
||||
],
|
||||
});
|
||||
const call = await cli.sharedMcpCall!();
|
||||
const response = await call("inspect_web_network_capture", {
|
||||
capture_path: capture,
|
||||
format: "har",
|
||||
record_ordinals: [0],
|
||||
protobuf: {
|
||||
schema: {
|
||||
descriptor_set_path: descriptor,
|
||||
message_type: "capture.Root",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(response.isError, JSON.stringify(response)).not.toBe(true);
|
||||
expect(parseEvidence(response.structuredContent).evidence_id).toBe(
|
||||
evidence.evidence_id,
|
||||
);
|
||||
expect((await readFile(descriptor)).equals(bytes)).toBe(true);
|
||||
expect(await readFile(capture, "utf8")).toBe(text);
|
||||
},
|
||||
);
|
||||
|
||||
/** Reorder known FileDescriptorProto fields without changing their facts. */
|
||||
const reorderedSharedFile = () => {
|
||||
const shared = descriptorDefinition().file[0]!;
|
||||
|
||||
Vendored
+18
@@ -1,6 +1,24 @@
|
||||
import { create, toBinary } from "@bufbuild/protobuf";
|
||||
import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt";
|
||||
|
||||
/**
|
||||
* protoc 31.1 --include_imports golden, with two distinct imported filenames:
|
||||
* dep.proto: syntax="proto3"; package plain; message Item { uint32 number=1; }
|
||||
* \uFEFFdep.proto: syntax="proto3"; package marked; message Item { string text=1; }
|
||||
* root.proto imports both and defines capture.Root with plain.Item plain=1
|
||||
* and marked.Item marked=2. U+FEFF is filename data, not a document BOM.
|
||||
*/
|
||||
export const protobufBomDescriptor = (): Buffer =>
|
||||
Buffer.from(
|
||||
"0a3a0a096465702e70726f746f1205706c61696e221e0a044974656d12160a066e756d6265721801200128" +
|
||||
"0d52066e756d626572620670726f746f330a3a0a0cefbbbf6465702e70726f746f12066d61726b6564221a" +
|
||||
"0a044974656d12120a0474657874180120012809520474657874620670726f746f330a87010a0a726f6f74" +
|
||||
"2e70726f746f1207636170747572651a096465702e70726f746f1a0cefbbbf6465702e70726f746f224f0a" +
|
||||
"04526f6f7412210a05706c61696e18012001280b320b2e706c61696e2e4974656d5205706c61696e12240a" +
|
||||
"066d61726b656418022001280b320c2e6d61726b65642e4974656d52066d61726b6564620670726f746f33",
|
||||
"hex",
|
||||
);
|
||||
|
||||
export const protobufFrame = (body: Buffer, flag = 0): Buffer => {
|
||||
const header = Buffer.alloc(5);
|
||||
header[0] = flag;
|
||||
|
||||
Reference in new issue
Block a user