mirror of
https://github.com/morluto/rea.git
synced 2026-11-10 08:47:00 +00:00
feat(go): recover bounded Go string literals from (ptr, len) headers (#9206)
* feat(go): recover bounded Go string literals from (ptr, len) headers * fix(go): deduplicate header scan spans, bound rejected cache, and reject Cf * fix(go): allow U+200C ZWNJ and U+200D ZWJ in Go string literal recovery * fix(go): index Mach-O load regions, validate MCP schema, and add feature specs * test(go): cover Mach-O region-index cancellation and load-region boundaries
This commit is contained in:
Ryan S authored and GitHub committed
1 parent
5553acfe50
commit
94182f331e
26 files changed
+2709
-44
No files matched your search
@@ -264,3 +264,88 @@ it.each([
|
||||
expect(reason.cleanupIncomplete).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("validates string_literals input options and output invariants", async () => {
|
||||
let calls = 0;
|
||||
const service = new GoBinaryService({
|
||||
close: async () => {},
|
||||
identity,
|
||||
inspect: () => {
|
||||
calls += 1;
|
||||
return Promise.resolve(ok(report));
|
||||
},
|
||||
});
|
||||
for (const invalidInput of [
|
||||
{ path: report.artifact.path, max_string_literals: 10 },
|
||||
{
|
||||
path: report.artifact.path,
|
||||
include_string_literals: false,
|
||||
max_string_literals: 10,
|
||||
},
|
||||
{
|
||||
path: report.artifact.path,
|
||||
include_string_literals: true,
|
||||
max_string_literals: 0,
|
||||
},
|
||||
{
|
||||
path: report.artifact.path,
|
||||
include_string_literals: true,
|
||||
max_string_literals: 10_001,
|
||||
},
|
||||
]) {
|
||||
const refused = await service.inspect(invalidInput);
|
||||
expect(refused.ok).toBe(false);
|
||||
if (!refused.ok) expect(refused.error).toBeInstanceOf(AnalysisInputError);
|
||||
}
|
||||
expect(calls).toBe(0);
|
||||
|
||||
const baseLiterals: NonNullable<GoBinary["string_literals"]> = {
|
||||
evidence_kind: "inferred",
|
||||
discovered_count: 2,
|
||||
aggregate_bytes: 10,
|
||||
max_items: 100,
|
||||
truncated: false,
|
||||
items: [
|
||||
{
|
||||
value: "alpha",
|
||||
location: { offset: 64, bytes: 5 },
|
||||
first_header_offset: 256,
|
||||
reference_count: 1,
|
||||
},
|
||||
{
|
||||
value: "bravo",
|
||||
location: { offset: 69, bytes: 5 },
|
||||
first_header_offset: 272,
|
||||
reference_count: 2,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
for (const malformedStrings of [
|
||||
{ ...baseLiterals, aggregate_bytes: 9 },
|
||||
{ ...baseLiterals, discovered_count: 3, truncated: false },
|
||||
{
|
||||
...baseLiterals,
|
||||
items: [baseLiterals.items[1]!, baseLiterals.items[0]!],
|
||||
},
|
||||
{
|
||||
...baseLiterals,
|
||||
items: [
|
||||
{ ...baseLiterals.items[0]!, first_header_offset: 1020 },
|
||||
baseLiterals.items[1]!,
|
||||
],
|
||||
},
|
||||
]) {
|
||||
const out = await new GoBinaryService({
|
||||
close: async () => {},
|
||||
identity,
|
||||
inspect: () =>
|
||||
Promise.resolve(ok({ ...report, string_literals: malformedStrings })),
|
||||
}).inspect({
|
||||
path: report.artifact.path,
|
||||
include_string_literals: true,
|
||||
});
|
||||
expect(out.ok).toBe(false);
|
||||
if (!out.ok) expect(out.error).toBeInstanceOf(AnalysisOutputError);
|
||||
}
|
||||
});
|
||||
+27
-1
@@ -12,12 +12,18 @@ import {
|
||||
import { localPathStringSchema } from "../domain/localPath.js";
|
||||
import { absoluteCallerPath } from "../filesystem/callerPath.js";
|
||||
import { machoSliceArchitectureSchema } from "../domain/apple/machoTargetSelection.js";
|
||||
import {
|
||||
DEFAULT_MAX_GO_STRING_LITERALS,
|
||||
MAX_GO_STRING_LITERALS_LIMIT,
|
||||
goMaxStringLiteralsSchema,
|
||||
} from "../domain/go/goBinary.js";
|
||||
import { cliDecimalField } from "./options.js";
|
||||
|
||||
/** Resolve caller paths and inspect static build metadata through the shared workflow. */
|
||||
export const registerGoCommands = (cli: CliInstance, logger: Logger): void => {
|
||||
cli.command(CLI_COMMANDS.inspectGoBinary, {
|
||||
description:
|
||||
"Inspect embedded Go compiler, modules and build settings without target execution",
|
||||
"Inspect embedded Go compiler, modules, build settings and optional static (ptr, len) string literals without target execution",
|
||||
args: z.object({
|
||||
path: localPathStringSchema.describe(
|
||||
"Local ELF, PE, thin Mach-O or universal Mach-O image",
|
||||
@@ -30,6 +36,17 @@ export const registerGoCommands = (cli: CliInstance, logger: Logger): void => {
|
||||
.describe(
|
||||
"CPU family of the slice to read from a universal Mach-O; required for universal files",
|
||||
),
|
||||
includeStringLiterals: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.describe(
|
||||
"Also recover bounded inferred Go UTF-8 string literals from static (ptr, len) headers",
|
||||
),
|
||||
maxStringLiterals: cliDecimalField(
|
||||
goMaxStringLiteralsSchema.optional(),
|
||||
).describe(
|
||||
`Maximum distinct recovered string literals (1-${String(MAX_GO_STRING_LITERALS_LIMIT)}; default ${String(DEFAULT_MAX_GO_STRING_LITERALS)}; requires --include-string-literals)`,
|
||||
),
|
||||
}),
|
||||
run: ({ args, options }) =>
|
||||
withCommandCancellation(async (signal) => {
|
||||
@@ -47,6 +64,15 @@ export const registerGoCommands = (cli: CliInstance, logger: Logger): void => {
|
||||
...(options.architecture === undefined
|
||||
? {}
|
||||
: { architecture: options.architecture }),
|
||||
...(options.includeStringLiterals === undefined
|
||||
? {}
|
||||
: {
|
||||
include_string_literals:
|
||||
options.includeStringLiterals,
|
||||
}),
|
||||
...(options.maxStringLiterals === undefined
|
||||
? {}
|
||||
: { max_string_literals: options.maxStringLiterals }),
|
||||
},
|
||||
{ signal },
|
||||
),
|
||||
|
||||
@@ -15,7 +15,7 @@ export const GO_TOOL_CONTRACTS = [
|
||||
kind: "artifact-provider",
|
||||
requiresActiveTarget: true,
|
||||
description:
|
||||
"Inspect embedded Go compiler version, main module, dependency/replacement versions and build settings in an explicit local ELF, PE or thin Mach-O image, or the slice of a universal Mach-O named by architecture (read without lipo; artifact stays the universal file, selected_slice records the slice and source ranges stay file offsets). Default detail complete returns original artifact SHA-256, exact build-info source ranges, raw framed module bytes and unparsed module-text lines inline as observed Evidence. Explicit detail summary retains the same complete parent and returns compiler/main-module identity, counts, source ranges and parent_evidence_id. Query retained rows through inspect_analysis_view kind go, collection dependencies or settings; select exact dependency paths or setting keys, indices, fields and pagination. Rows always retain their original index scoped to the parent Evidence; empty filters select none. Runtime dependency completeness remains unknown. Supports inline and historical pointer encodings without installing Go, opening a disassembler, executing the target, writing temporary files or accessing the network. Missing metadata is reported as unknown, not proof that an image is not Go. Does not recover functions, source mappings, types or runtime behavior. Resource guards bound files to 256 MiB and charge each distinct build-info search span in full against a cumulative 256 MiB header-search budget, with partial overlaps charged separately; they also bound aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and complete inline metadata to 8 MiB; malformed, unsupported and oversized inputs receive typed failures rather than partial success.",
|
||||
"Inspect embedded Go compiler version, main module, dependency/replacement versions, build settings and optional inferred static (ptr, len) UTF-8 string literals in an explicit local ELF, PE or thin Mach-O image, or the slice of a universal Mach-O named by architecture (read without lipo; artifact stays the universal file, selected_slice records the slice and source ranges stay file offsets). Default detail complete returns original artifact SHA-256, exact build-info source ranges, raw framed module bytes and unparsed module-text lines inline as observed Evidence; include_string_literals: true adds bounded deduplicated string_literals (default max_string_literals 2,000, max 10,000, <=4,096 bytes each, <=512 KiB aggregate, 64 MiB header-scan budget) recovered from word-aligned (ptr, len) headers in non-executable mapped data pointing within single read-only spans. Explicit detail summary retains the same complete parent and returns compiler/main-module identity, counts, source ranges and parent_evidence_id. Query retained rows through inspect_analysis_view kind go, collection dependencies, settings or string_literals (exact values or substring contains when collected); select indices, fields and pagination. Rows always retain their original index scoped to the parent Evidence; empty filters select none. Runtime dependency completeness and string reachability remain unknown. Supports inline and historical pointer encodings without installing Go, opening a disassembler, executing the target, writing temporary files or accessing the network. Missing metadata is reported as unknown, not proof that an image is not Go. Does not recover functions, source mappings, types or runtime behavior. Resource guards bound files to 256 MiB and charge each distinct build-info search span in full against a cumulative 256 MiB header-search budget, with partial overlaps charged separately; they also bound aggregate embedded build-info strings to 1 MiB, structural table decoding to 16 MiB and complete inline metadata to 8 MiB; malformed, unsupported and oversized inputs receive typed failures rather than partial success.",
|
||||
inputSchema: inspectGoBinaryInputSchema,
|
||||
outputSchema: analysisDetailOutputSchemaOf(
|
||||
goBinarySchema,
|
||||
|
||||
@@ -149,3 +149,92 @@ it("revalidates the parent subject after immutable Go report facts have been cac
|
||||
).ok,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses string_literals view when unrequested and projects filtered/truncated string_literals when present", () => {
|
||||
const unrequested = projectAnalysisView(parent(), {
|
||||
kind: "go",
|
||||
collection: "string_literals",
|
||||
offset: 0,
|
||||
limit: 64,
|
||||
});
|
||||
expect(unrequested.ok).toBe(false);
|
||||
if (!unrequested.ok) {
|
||||
expect(unrequested.error).toBeInstanceOf(AnalysisInputError);
|
||||
if (unrequested.error instanceof AnalysisInputError)
|
||||
expect(unrequested.error.issues[0]?.path).toEqual(["view", "collection"]);
|
||||
}
|
||||
|
||||
const withStrings: GoBinary = {
|
||||
...report,
|
||||
string_literals: {
|
||||
evidence_kind: "inferred",
|
||||
discovered_count: 5,
|
||||
aggregate_bytes: 52,
|
||||
max_items: 3,
|
||||
truncated: true,
|
||||
items: [
|
||||
{
|
||||
value: "ListResourceProviders",
|
||||
location: { offset: 128, bytes: 21 },
|
||||
first_header_offset: 300,
|
||||
reference_count: 1,
|
||||
},
|
||||
{
|
||||
value: "genai",
|
||||
location: { offset: 149, bytes: 5 },
|
||||
first_header_offset: 316,
|
||||
reference_count: 2,
|
||||
},
|
||||
{
|
||||
value: "https://example.invalid/v1",
|
||||
location: { offset: 154, bytes: 26 },
|
||||
first_header_offset: 332,
|
||||
reference_count: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const stringParent = { ...parent(), normalizedResult: withStrings };
|
||||
const summary = projectAnalysisView(stringParent, { kind: "summary" });
|
||||
expect(summary.ok).toBe(true);
|
||||
if (
|
||||
!summary.ok ||
|
||||
summary.value.kind !== "summary" ||
|
||||
summary.value.parent_operation !== "inspect_go_binary"
|
||||
)
|
||||
throw new Error("Expected Go summary view");
|
||||
expect(summary.value.source_coverage.string_literals).toBe("truncated");
|
||||
expect(summary.value.summary).toMatchObject({
|
||||
string_literals: {
|
||||
evidence_kind: "inferred",
|
||||
available_count: 3,
|
||||
discovered_count: 5,
|
||||
aggregate_bytes: 52,
|
||||
max_items: 3,
|
||||
truncated: true,
|
||||
},
|
||||
});
|
||||
expect(summary.value.unknowns.join(" ")).toContain("truncated");
|
||||
|
||||
const filtered = projectAnalysisView(stringParent, {
|
||||
kind: "go",
|
||||
collection: "string_literals",
|
||||
contains: "example.invalid",
|
||||
fields: ["value", "reference_count"],
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
});
|
||||
expect(filtered.ok).toBe(true);
|
||||
if (!filtered.ok || filtered.value.kind !== "go")
|
||||
throw new Error("Expected Go string_literals page");
|
||||
expect(filtered.value.page).toEqual({
|
||||
collection: "string_literals",
|
||||
rows: [
|
||||
{
|
||||
index: 2,
|
||||
value: "https://example.invalid/v1",
|
||||
reference_count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -4,8 +4,10 @@ import { z } from "zod";
|
||||
import {
|
||||
goBinarySchema,
|
||||
goBuildInfoSchema,
|
||||
goMaxStringLiteralsSchema,
|
||||
goModuleMetadataSchema,
|
||||
goModuleSchema,
|
||||
goStringLiteralSchema,
|
||||
} from "../go/goBinary.js";
|
||||
import { err, ok } from "../result.js";
|
||||
import type { AnalysisViewParent } from "./analysisViewParent.js";
|
||||
@@ -47,6 +49,13 @@ export const goBinaryViewRequestSchema = z.discriminatedUnion("collection", [
|
||||
keys: z.array(z.string()).optional(),
|
||||
fields: z.array(settingSchema.keyof()).optional(),
|
||||
}),
|
||||
z.strictObject({
|
||||
...page,
|
||||
collection: z.literal("string_literals"),
|
||||
values: z.array(z.string()).optional(),
|
||||
contains: z.string().min(1).optional(),
|
||||
fields: z.array(goStringLiteralSchema.keyof()).optional(),
|
||||
}),
|
||||
]);
|
||||
|
||||
export const goBinaryViewPageSchema = z.discriminatedUnion("collection", [
|
||||
@@ -58,6 +67,12 @@ export const goBinaryViewPageSchema = z.discriminatedUnion("collection", [
|
||||
collection: z.literal("settings"),
|
||||
rows: z.array(settingSchema.partial().extend({ index: indexSchema })),
|
||||
}),
|
||||
z.strictObject({
|
||||
collection: z.literal("string_literals"),
|
||||
rows: z.array(
|
||||
goStringLiteralSchema.partial().extend({ index: indexSchema }),
|
||||
),
|
||||
}),
|
||||
]);
|
||||
|
||||
export const goBinaryViewContextSchema = z.strictObject({
|
||||
@@ -65,6 +80,7 @@ export const goBinaryViewContextSchema = z.strictObject({
|
||||
build_info_available: z.boolean(),
|
||||
module_parse_complete: z.boolean().nullable(),
|
||||
runtime_dependencies: z.literal("unknown"),
|
||||
string_literals: z.enum(["not-requested", "complete", "truncated"]),
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -100,6 +116,16 @@ export const goBinarySummarySchema = z.strictObject({
|
||||
}),
|
||||
})
|
||||
.nullable(),
|
||||
string_literals: z
|
||||
.strictObject({
|
||||
evidence_kind: z.literal("inferred"),
|
||||
available_count: count,
|
||||
discovered_count: count,
|
||||
aggregate_bytes: count,
|
||||
max_items: goMaxStringLiteralsSchema,
|
||||
truncated: z.boolean(),
|
||||
})
|
||||
.exactOptional(),
|
||||
});
|
||||
|
||||
const parseReport = createAnalysisViewResultReader(goBinarySchema);
|
||||
@@ -144,6 +170,7 @@ export const projectGoBinaryView = (
|
||||
]),
|
||||
);
|
||||
const info = report.build_info;
|
||||
const stringLiterals = report.string_literals;
|
||||
const runtimeUnknown =
|
||||
"Embedded build records do not establish complete runtime dependencies or target behavior.";
|
||||
const unknowns = [
|
||||
@@ -164,6 +191,16 @@ export const projectGoBinaryView = (
|
||||
]
|
||||
: []),
|
||||
]),
|
||||
...(stringLiterals === undefined
|
||||
? []
|
||||
: [
|
||||
"Recovered string_literals reflect static (ptr, len) pairs into read-only data; stack/immediate strings and runtime reachability remain unknown.",
|
||||
...(stringLiterals.truncated
|
||||
? [
|
||||
"Static (ptr, len) string_literals were truncated by item, aggregate-byte or scan budgets; additional candidates remain uninspected.",
|
||||
]
|
||||
: []),
|
||||
]),
|
||||
];
|
||||
const shared = {
|
||||
parent_evidence_id: parent.evidenceId,
|
||||
@@ -174,6 +211,12 @@ export const projectGoBinaryView = (
|
||||
build_info_available: info !== null,
|
||||
module_parse_complete: info?.module.complete ?? null,
|
||||
runtime_dependencies: "unknown" as const,
|
||||
string_literals:
|
||||
stringLiterals === undefined
|
||||
? ("not-requested" as const)
|
||||
: stringLiterals.truncated
|
||||
? ("truncated" as const)
|
||||
: ("complete" as const),
|
||||
},
|
||||
limitations: boundedViewMessages(
|
||||
[...parent.limitations, ...report.limitations],
|
||||
@@ -220,6 +263,18 @@ export const projectGoBinaryView = (
|
||||
},
|
||||
},
|
||||
},
|
||||
...(stringLiterals === undefined
|
||||
? {}
|
||||
: {
|
||||
string_literals: {
|
||||
evidence_kind: stringLiterals.evidence_kind,
|
||||
available_count: stringLiterals.items.length,
|
||||
discovered_count: stringLiterals.discovered_count,
|
||||
aggregate_bytes: stringLiterals.aggregate_bytes,
|
||||
max_items: stringLiterals.max_items,
|
||||
truncated: stringLiterals.truncated,
|
||||
},
|
||||
}),
|
||||
},
|
||||
coverage: completeWithinViewCoverage(1, 1),
|
||||
});
|
||||
@@ -227,6 +282,51 @@ export const projectGoBinaryView = (
|
||||
view.indices === undefined ? undefined : new Set(view.indices);
|
||||
const fields =
|
||||
view.fields === undefined ? undefined : new Set<string>(view.fields);
|
||||
if (view.collection === "string_literals") {
|
||||
if (stringLiterals === undefined)
|
||||
return err(
|
||||
analysisViewInputError([
|
||||
{
|
||||
path: ["view", "collection"],
|
||||
reason: "invalid_value",
|
||||
message:
|
||||
"Parent Go Evidence did not collect string_literals; re-run inspect_go_binary with include_string_literals: true.",
|
||||
},
|
||||
]),
|
||||
);
|
||||
const values = view.values === undefined ? undefined : new Set(view.values);
|
||||
const needle = view.contains;
|
||||
const selected = selectViewRows(
|
||||
stringLiterals.items,
|
||||
view,
|
||||
(row, index) =>
|
||||
(indices?.has(index) ?? true) &&
|
||||
(values?.has(row.value) ?? true) &&
|
||||
(needle === undefined || row.value.includes(needle)),
|
||||
(row, index) => ({
|
||||
index,
|
||||
...((fields?.has("value") ?? true) && { value: row.value }),
|
||||
...((fields?.has("location") ?? true) && { location: row.location }),
|
||||
...((fields?.has("first_header_offset") ?? true) && {
|
||||
first_header_offset: row.first_header_offset,
|
||||
}),
|
||||
...((fields?.has("reference_count") ?? true) && {
|
||||
reference_count: row.reference_count,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
return ok({
|
||||
kind: "go" as const,
|
||||
view,
|
||||
...shared,
|
||||
page: { collection: "string_literals" as const, rows: selected.rows },
|
||||
coverage: pageViewCoverage(
|
||||
view.offset,
|
||||
selected.rows.length,
|
||||
selected.total,
|
||||
),
|
||||
});
|
||||
}
|
||||
if (view.collection === "dependencies") {
|
||||
const paths = view.paths === undefined ? undefined : new Set(view.paths);
|
||||
const selected = selectViewRows(
|
||||
|
||||
+165
-12
@@ -17,6 +17,40 @@ const sourceRangeSchema = z.strictObject({
|
||||
offset: z.int().nonnegative(),
|
||||
bytes: z.int().nonnegative(),
|
||||
});
|
||||
|
||||
export const DEFAULT_MAX_GO_STRING_LITERALS = 2000;
|
||||
export const MAX_GO_STRING_LITERALS_LIMIT = 10_000;
|
||||
export const MAX_GO_STRING_LITERAL_BYTES = 4096;
|
||||
export const MAX_GO_STRING_AGGREGATE_BYTES = 512 * 1024;
|
||||
export const MAX_GO_STRING_HEADER_SCAN_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
export const goMaxStringLiteralsSchema = z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(MAX_GO_STRING_LITERALS_LIMIT);
|
||||
|
||||
/** One distinct printable UTF-8 literal inferred from static (ptr, len) headers. */
|
||||
export const goStringLiteralSchema = z.strictObject({
|
||||
value: z.string().min(1),
|
||||
location: z.strictObject({
|
||||
offset: z.int().nonnegative(),
|
||||
bytes: z.int().min(1).max(MAX_GO_STRING_LITERAL_BYTES),
|
||||
}),
|
||||
first_header_offset: z.int().nonnegative(),
|
||||
reference_count: z.int().positive(),
|
||||
});
|
||||
|
||||
/** Bounded static (ptr, len) string-header inventory, separate from observed build_info. */
|
||||
export const goStringLiteralsSchema = z.strictObject({
|
||||
evidence_kind: z.literal("inferred"),
|
||||
items: z.array(goStringLiteralSchema).max(MAX_GO_STRING_LITERALS_LIMIT),
|
||||
discovered_count: z.int().nonnegative(),
|
||||
aggregate_bytes: z.int().nonnegative().max(MAX_GO_STRING_AGGREGATE_BYTES),
|
||||
max_items: goMaxStringLiteralsSchema,
|
||||
truncated: z.boolean(),
|
||||
});
|
||||
|
||||
const moduleIdentitySchema = z.strictObject({
|
||||
path: z.string(),
|
||||
version: z.string(),
|
||||
@@ -40,18 +74,42 @@ export const goModuleMetadataSchema = z.strictObject({
|
||||
});
|
||||
|
||||
/** One exact local artifact, independent of any active disassembler target. */
|
||||
export const inspectGoBinaryInputSchema = z.strictObject({
|
||||
path: absoluteLocalPathStringSchema.describe(
|
||||
"Absolute filesystem path to the selected executable or library",
|
||||
),
|
||||
architecture: machoSliceArchitectureSchema.exactOptional(),
|
||||
detail: z
|
||||
.enum(["complete", "summary"])
|
||||
.default("complete")
|
||||
.describe(
|
||||
"complete returns all metadata inline; summary retains complete Evidence and returns build identity and counts for subsequent retained Go views.",
|
||||
export const inspectGoBinaryInputSchema = z
|
||||
.strictObject({
|
||||
path: absoluteLocalPathStringSchema.describe(
|
||||
"Absolute filesystem path to the selected executable or library",
|
||||
),
|
||||
});
|
||||
architecture: machoSliceArchitectureSchema.exactOptional(),
|
||||
include_string_literals: z
|
||||
.boolean()
|
||||
.exactOptional()
|
||||
.describe(
|
||||
"Also recover bounded inferred Go UTF-8 string literals referenced by static (ptr, len) headers in non-executable mapped data",
|
||||
),
|
||||
max_string_literals: goMaxStringLiteralsSchema
|
||||
.describe(
|
||||
`Maximum distinct recovered string literals (1-${String(MAX_GO_STRING_LITERALS_LIMIT)}; default ${String(DEFAULT_MAX_GO_STRING_LITERALS)}; requires include_string_literals: true)`,
|
||||
)
|
||||
.exactOptional(),
|
||||
detail: z
|
||||
.enum(["complete", "summary"])
|
||||
.default("complete")
|
||||
.describe(
|
||||
"complete returns all metadata inline; summary retains complete Evidence and returns build identity and counts for subsequent retained Go views.",
|
||||
),
|
||||
})
|
||||
.superRefine((input, context) => {
|
||||
if (
|
||||
input.max_string_literals !== undefined &&
|
||||
input.include_string_literals !== true
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["max_string_literals"],
|
||||
message:
|
||||
"max_string_literals requires include_string_literals to be true.",
|
||||
});
|
||||
});
|
||||
|
||||
/** Compiler-written build information and its original file-backed source ranges. */
|
||||
export const goBuildInfoSchema = z.strictObject({
|
||||
@@ -81,11 +139,14 @@ export const goBinarySchema = z
|
||||
/** The universal slice that was read; source ranges stay file offsets inside it. */
|
||||
selected_slice: machoTargetSelectionSchema.exactOptional(),
|
||||
build_info: goBuildInfoSchema.nullable(),
|
||||
string_literals: goStringLiteralsSchema.exactOptional(),
|
||||
limitations: z.array(z.string()),
|
||||
})
|
||||
.superRefine((report, context) => {
|
||||
const info = report.build_info;
|
||||
const slice = report.selected_slice;
|
||||
const stringLiterals = report.string_literals;
|
||||
const headerBytes = (report.bits / 8) * 2;
|
||||
if (
|
||||
slice !== undefined &&
|
||||
(!selectedSliceAgreesWithReport(
|
||||
@@ -112,14 +173,104 @@ export const goBinarySchema = z
|
||||
range.offset,
|
||||
range.bytes,
|
||||
),
|
||||
)) ||
|
||||
(stringLiterals !== undefined &&
|
||||
stringLiterals.items.some(
|
||||
(item) =>
|
||||
!machoSelectionContainsRange(
|
||||
slice,
|
||||
report.artifact.bytes,
|
||||
item.location.offset,
|
||||
item.location.bytes,
|
||||
) ||
|
||||
!machoSelectionContainsRange(
|
||||
slice,
|
||||
report.artifact.bytes,
|
||||
item.first_header_offset,
|
||||
headerBytes,
|
||||
),
|
||||
)))
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["selected_slice"],
|
||||
message:
|
||||
"Selected universal slice must be a FAT slice of the artifact whose CPU matches the report's Mach-O architecture and that contains the build-info source ranges.",
|
||||
"Selected universal slice must be a FAT slice of the artifact whose CPU matches the report's Mach-O architecture and that contains the reported source ranges.",
|
||||
});
|
||||
if (stringLiterals !== undefined) {
|
||||
if (stringLiterals.items.length > stringLiterals.max_items)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["string_literals", "items"],
|
||||
message: "Recovered Go string literals exceed max_items.",
|
||||
});
|
||||
if (stringLiterals.discovered_count < stringLiterals.items.length)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["string_literals", "discovered_count"],
|
||||
message:
|
||||
"discovered_count must be at least the number of retained string_literals items.",
|
||||
});
|
||||
if (
|
||||
stringLiterals.discovered_count > stringLiterals.items.length &&
|
||||
!stringLiterals.truncated
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["string_literals", "truncated"],
|
||||
message:
|
||||
"truncated must be true when discovered_count exceeds retained items.",
|
||||
});
|
||||
let sumBytes = 0;
|
||||
let previous: GoStringLiteral | undefined;
|
||||
for (const [index, item] of stringLiterals.items.entries()) {
|
||||
sumBytes += item.location.bytes;
|
||||
if (Buffer.byteLength(item.value, "utf8") !== item.location.bytes)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["string_literals", "items", index, "value"],
|
||||
message:
|
||||
"Decoded Go string literal UTF-8 length does not match its source byte length.",
|
||||
});
|
||||
if (
|
||||
item.location.offset > report.artifact.bytes ||
|
||||
item.location.bytes > report.artifact.bytes - item.location.offset
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["string_literals", "items", index, "location"],
|
||||
message:
|
||||
"Go string literal source range is outside the selected artifact.",
|
||||
});
|
||||
if (item.first_header_offset > report.artifact.bytes - headerBytes)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["string_literals", "items", index, "first_header_offset"],
|
||||
message:
|
||||
"Go string literal header offset is outside the selected artifact.",
|
||||
});
|
||||
if (
|
||||
previous !== undefined &&
|
||||
(item.location.offset < previous.location.offset ||
|
||||
(item.location.offset === previous.location.offset &&
|
||||
item.location.bytes <= previous.location.bytes))
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["string_literals", "items", index, "location"],
|
||||
message:
|
||||
"Go string literals must be strictly ordered by (location.offset, location.bytes).",
|
||||
});
|
||||
previous = item;
|
||||
}
|
||||
if (sumBytes !== stringLiterals.aggregate_bytes)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["string_literals", "aggregate_bytes"],
|
||||
message:
|
||||
"aggregate_bytes does not match the sum of retained Go string literal byte lengths.",
|
||||
});
|
||||
}
|
||||
if (info === null) return;
|
||||
if (info.header_offset > report.artifact.bytes - 32)
|
||||
context.addIssue({
|
||||
@@ -153,6 +304,8 @@ export const goBinarySchema = z
|
||||
}
|
||||
});
|
||||
|
||||
export type GoStringLiteral = z.infer<typeof goStringLiteralSchema>;
|
||||
export type GoStringLiterals = z.infer<typeof goStringLiteralsSchema>;
|
||||
export type GoModule = z.infer<typeof goModuleSchema>;
|
||||
export type GoModuleMetadata = z.infer<typeof goModuleMetadataSchema>;
|
||||
export type GoBinary = z.infer<typeof goBinarySchema>;
|
||||
|
||||
+37
-2
@@ -10,12 +10,16 @@ import {
|
||||
type ZeroFillMapping,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
import { readBinaryContainer } from "../native/metadata/readBinaryContainer.js";
|
||||
import type { GoModuleMetadata } from "../domain/go/goBinary.js";
|
||||
import type {
|
||||
GoModuleMetadata,
|
||||
GoStringLiterals,
|
||||
} from "../domain/go/goBinary.js";
|
||||
import { parseGoModuleBytes } from "./GoModuleText.js";
|
||||
import {
|
||||
goBuildInfoSearch,
|
||||
type GoBuildInfoSearchRange,
|
||||
} from "./GoBuildInfoSearch.js";
|
||||
import { recoverGoStringLiterals } from "./GoStringLiterals.js";
|
||||
|
||||
const MAGIC = Buffer.from("ff20476f206275696c64696e663a", "hex");
|
||||
const MODULE_START = Buffer.from("3077af0c9274080241e1c107e6d618e6", "hex");
|
||||
@@ -49,14 +53,43 @@ export interface GoBinaryImage {
|
||||
readonly bits: 32 | 64;
|
||||
readonly byte_order: "little" | "big";
|
||||
readonly build_info: GoBuildInfo | null;
|
||||
readonly string_literals?: GoStringLiterals;
|
||||
readonly macho_chained_fixups?: boolean;
|
||||
/** Native section-name coverage, retained for discovery limitations. */
|
||||
readonly section_names_limitation?: string;
|
||||
}
|
||||
|
||||
export interface ReadGoBinaryImageOptions {
|
||||
readonly includeStringLiterals?: boolean;
|
||||
readonly maxStringLiterals?: number | undefined;
|
||||
readonly checkActive?: (() => void) | undefined;
|
||||
}
|
||||
|
||||
/** Inspect bounded executable bytes using native mappings, without loading or executing code. */
|
||||
export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => {
|
||||
export const readGoBinaryImage = (
|
||||
bytes: Buffer,
|
||||
options?: ReadGoBinaryImageOptions,
|
||||
): GoBinaryImage => {
|
||||
const container = readBinaryContainer(bytes);
|
||||
const { format, architecture, bits, byte_order } = container;
|
||||
const stringLiterals =
|
||||
options?.includeStringLiterals === true
|
||||
? recoverGoStringLiterals(
|
||||
bytes,
|
||||
container,
|
||||
options.maxStringLiterals,
|
||||
options.checkActive,
|
||||
)
|
||||
: undefined;
|
||||
const stringMetadata =
|
||||
stringLiterals === undefined
|
||||
? {}
|
||||
: {
|
||||
string_literals: stringLiterals,
|
||||
...(container.macho_chained_fixups === true
|
||||
? { macho_chained_fixups: true }
|
||||
: {}),
|
||||
};
|
||||
let selected: { search: GoBuildInfoSearchRange; header: number } | null =
|
||||
null;
|
||||
const searched = new Set<string>();
|
||||
@@ -88,6 +121,7 @@ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => {
|
||||
bits,
|
||||
byte_order,
|
||||
build_info: null,
|
||||
...stringMetadata,
|
||||
...(sectionNamesLimitation === undefined
|
||||
? {}
|
||||
: { section_names_limitation: sectionNamesLimitation }),
|
||||
@@ -214,6 +248,7 @@ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => {
|
||||
module_location: module.location,
|
||||
module: parseGoModuleBytes(moduleBody),
|
||||
},
|
||||
...stringMetadata,
|
||||
...(sectionNamesLimitation === undefined
|
||||
? {}
|
||||
: { section_names_limitation: sectionNamesLimitation }),
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
import {
|
||||
goBinarySchema,
|
||||
type GoBinary,
|
||||
type GoStringLiterals,
|
||||
type InspectGoBinaryInput,
|
||||
} from "../domain/go/goBinary.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
@@ -167,10 +168,16 @@ const outputLimit = (path: string) =>
|
||||
const limitationsFor = (
|
||||
info: GoBinaryImage["build_info"],
|
||||
sectionNamesLimitation: string | undefined,
|
||||
stringLiterals: GoStringLiterals | undefined,
|
||||
machoChainedFixups: boolean | undefined,
|
||||
): string[] => [
|
||||
"Embedded build metadata is observed file content, not proof of compiler identity, dependency authenticity or runtime behavior.",
|
||||
"Only Go build information is inspected; function names, source mappings, type metadata and decompiled code are not recovered.",
|
||||
"Resource guards bound stable artifact reads and cumulative header searches to 256 MiB each, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits.",
|
||||
stringLiterals === undefined
|
||||
? "Only Go build information is inspected; function names, source mappings, type metadata and decompiled code are not recovered."
|
||||
: "Only Go build information and inferred static (ptr, len) string-header candidates are inspected; function names, source mappings, type metadata and decompiled code are not recovered.",
|
||||
stringLiterals === undefined
|
||||
? "Resource guards bound stable artifact reads and cumulative header searches to 256 MiB each, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits."
|
||||
: "Resource guards bound stable artifact reads and cumulative header searches to 256 MiB each, aggregate embedded build-info strings to 1 MiB, structural table decoding to 16 MiB, static string-header scanning to 64 MiB, recovered string literals to 512 KiB (up to 4,096 bytes each) and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits.",
|
||||
...(info !== null && (info.go_version === null || info.module_text === null)
|
||||
? [
|
||||
"Some embedded Go strings contain non-UTF8 bytes; undecodable text is null and exact source bytes are preserved as base64.",
|
||||
@@ -192,6 +199,21 @@ const limitationsFor = (
|
||||
"Some embedded module-text records could not be parsed completely; their original lines are preserved and additional metadata remains unknown.",
|
||||
]
|
||||
: []),
|
||||
...(stringLiterals === undefined
|
||||
? []
|
||||
: [
|
||||
"Recovered string_literals are inferred from pointer-aligned (ptr, len) pairs in non-executable mapped data pointing to printable UTF-8 slices within single read-only spans; compiler-inlined or stack-constructed strings without static headers, unapplied relocation addends and runtime reachability are not established.",
|
||||
...(machoChainedFixups === true
|
||||
? [
|
||||
"This Mach-O declares LC_DYLD_CHAINED_FIXUPS; rebase-encoded pointer slots in writable data may not hold raw virtual addresses statically, so (ptr, len) recovery can miss writable-section string headers.",
|
||||
]
|
||||
: []),
|
||||
...(stringLiterals.truncated
|
||||
? [
|
||||
"Recovered string_literals were truncated at the configured item, 512 KiB aggregate-byte or 64 MiB scan budget; additional static (ptr, len) candidates may exist.",
|
||||
]
|
||||
: []),
|
||||
]),
|
||||
];
|
||||
|
||||
/** Locate image-relative build-info ranges in the selected file. */
|
||||
@@ -214,6 +236,24 @@ const inFile = (
|
||||
},
|
||||
};
|
||||
|
||||
const inFileStringLiterals = (
|
||||
literals: GoStringLiterals | undefined,
|
||||
fileOffset: number,
|
||||
): GoStringLiterals | undefined =>
|
||||
literals === undefined || fileOffset === 0
|
||||
? literals
|
||||
: {
|
||||
...literals,
|
||||
items: literals.items.map((item) => ({
|
||||
...item,
|
||||
location: {
|
||||
...item.location,
|
||||
offset: item.location.offset + fileOffset,
|
||||
},
|
||||
first_header_offset: item.first_header_offset + fileOffset,
|
||||
})),
|
||||
};
|
||||
|
||||
/** Stable in-process metadata inspection without subprocesses, temporary files or Go installation. */
|
||||
export class GoBinaryProvider implements GoBinaryPort {
|
||||
readonly identity = GO_BINARY_PROVIDER_IDENTITY;
|
||||
@@ -252,12 +292,22 @@ export class GoBinaryProvider implements GoBinaryPort {
|
||||
snapshot.bytes,
|
||||
input.architecture,
|
||||
);
|
||||
const image = readGoBinaryImage(selected.bytes);
|
||||
const image = readGoBinaryImage(selected.bytes, {
|
||||
includeStringLiterals: input.include_string_literals === true,
|
||||
maxStringLiterals: input.max_string_literals,
|
||||
checkActive: () => assertArtifactReadActive(options?.signal),
|
||||
});
|
||||
const {
|
||||
section_names_limitation: sectionNamesLimitation,
|
||||
macho_chained_fixups: machoChainedFixups,
|
||||
string_literals: rawStringLiterals,
|
||||
...publicImage
|
||||
} = image;
|
||||
const info = inFile(image.build_info, selected.fileOffset);
|
||||
const stringLiterals = inFileStringLiterals(
|
||||
rawStringLiterals,
|
||||
selected.fileOffset,
|
||||
);
|
||||
const report = {
|
||||
...publicImage,
|
||||
artifact: {
|
||||
@@ -269,7 +319,15 @@ export class GoBinaryProvider implements GoBinaryPort {
|
||||
? {}
|
||||
: { selected_slice: selected.selection }),
|
||||
build_info: info,
|
||||
limitations: limitationsFor(info, sectionNamesLimitation),
|
||||
...(stringLiterals === undefined
|
||||
? {}
|
||||
: { string_literals: stringLiterals }),
|
||||
limitations: limitationsFor(
|
||||
info,
|
||||
sectionNamesLimitation,
|
||||
stringLiterals,
|
||||
machoChainedFixups,
|
||||
),
|
||||
};
|
||||
if (Buffer.byteLength(JSON.stringify(report), "utf8") > OUTPUT_BYTES)
|
||||
throw outputLimit(input.path);
|
||||
|
||||
@@ -0,0 +1,741 @@
|
||||
import { expect, it } from "vitest";
|
||||
import {
|
||||
MAX_GO_STRING_AGGREGATE_BYTES,
|
||||
MAX_GO_STRING_LITERAL_BYTES,
|
||||
} from "../domain/go/goBinary.js";
|
||||
import type {
|
||||
BinaryContainer,
|
||||
FileMapping,
|
||||
ImageSection,
|
||||
LoadRegion,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
import { createGoRodataStringFixture } from "../../tests/fixtures/go/rodataStringImage.js";
|
||||
import { readGoBinaryImage } from "./GoBinaryImage.js";
|
||||
import { recoverGoStringLiterals } from "./GoStringLiterals.js";
|
||||
|
||||
const LITERALS = [
|
||||
"ListResourceProviders",
|
||||
"genai",
|
||||
"https://example.invalid/v1",
|
||||
] as const;
|
||||
const PACKED_RODATA = Buffer.concat(
|
||||
LITERALS.map((value) => Buffer.from(value, "utf8")),
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ format: "elf", bits: 64, byteOrder: "little" },
|
||||
{ format: "elf", bits: 32, byteOrder: "little" },
|
||||
{ format: "elf", bits: 64, byteOrder: "big" },
|
||||
{ format: "pe", bits: 64, byteOrder: "little" },
|
||||
{ format: "pe", bits: 32, byteOrder: "little" },
|
||||
{ format: "macho", bits: 64, byteOrder: "little" },
|
||||
{ format: "macho", bits: 32, byteOrder: "little" },
|
||||
] as const)(
|
||||
"recovers exact non-NUL-terminated slices in $format ($bits-bit $byteOrder)",
|
||||
({ format, bits, byteOrder }) => {
|
||||
const width = bits / 8;
|
||||
const firstLen = Buffer.byteLength(LITERALS[0]);
|
||||
const secondLen = Buffer.byteLength(LITERALS[1]);
|
||||
const thirdLen = Buffer.byteLength(LITERALS[2]);
|
||||
const baseRo = 0x102000n;
|
||||
|
||||
const fixture = createGoRodataStringFixture({
|
||||
format,
|
||||
bits,
|
||||
byteOrder,
|
||||
rodata: PACKED_RODATA,
|
||||
headers: [
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: width,
|
||||
ptr: baseRo + BigInt(firstLen + secondLen),
|
||||
len: BigInt(thirdLen),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: width * 4,
|
||||
ptr: baseRo,
|
||||
len: BigInt(firstLen),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: width * 7,
|
||||
ptr: baseRo + BigInt(firstLen),
|
||||
len: BigInt(secondLen),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: width * 10,
|
||||
ptr: baseRo + BigInt(firstLen),
|
||||
len: BigInt(secondLen),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(readGoBinaryImage(fixture.bytes).string_literals).toBeUndefined();
|
||||
|
||||
const recovered = readGoBinaryImage(fixture.bytes, {
|
||||
includeStringLiterals: true,
|
||||
}).string_literals;
|
||||
expect(recovered).toEqual({
|
||||
evidence_kind: "inferred",
|
||||
discovered_count: 3,
|
||||
aggregate_bytes: PACKED_RODATA.length,
|
||||
max_items: 2000,
|
||||
truncated: false,
|
||||
items: [
|
||||
{
|
||||
value: "ListResourceProviders",
|
||||
location: { offset: fixture.rodataOffset, bytes: firstLen },
|
||||
first_header_offset: fixture.dataOffset + width * 4,
|
||||
reference_count: 1,
|
||||
},
|
||||
{
|
||||
value: "genai",
|
||||
location: {
|
||||
offset: fixture.rodataOffset + firstLen,
|
||||
bytes: secondLen,
|
||||
},
|
||||
first_header_offset: fixture.dataOffset + width * 7,
|
||||
reference_count: 2,
|
||||
},
|
||||
{
|
||||
value: "https://example.invalid/v1",
|
||||
location: {
|
||||
offset: fixture.rodataOffset + firstLen + secondLen,
|
||||
bytes: thirdLen,
|
||||
},
|
||||
first_header_offset: fixture.dataOffset + width,
|
||||
reference_count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ label: "sectionless ELF", sectionless: true, corruptElfShstrtab: false },
|
||||
{
|
||||
label: "ELF with unreadable .shstrtab",
|
||||
sectionless: false,
|
||||
corruptElfShstrtab: true,
|
||||
},
|
||||
])("recovers literals from $label via load regions", (mode) => {
|
||||
const baseRo = 0x102000n;
|
||||
const fixture = createGoRodataStringFixture({
|
||||
format: "elf",
|
||||
rodata: PACKED_RODATA,
|
||||
sectionless: mode.sectionless,
|
||||
corruptElfShstrtab: mode.corruptElfShstrtab,
|
||||
headers: [
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 0,
|
||||
ptr: baseRo,
|
||||
len: BigInt(LITERALS[0].length),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 16,
|
||||
ptr: baseRo + BigInt(LITERALS[0].length),
|
||||
len: BigInt(LITERALS[1].length),
|
||||
},
|
||||
],
|
||||
});
|
||||
const recovered = readGoBinaryImage(fixture.bytes, {
|
||||
includeStringLiterals: true,
|
||||
}).string_literals;
|
||||
expect(recovered?.items.map((item) => item.value)).toEqual([
|
||||
"ListResourceProviders",
|
||||
"genai",
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects unaligned, code-section, cross-section, out-of-bounds and non-printable UTF-8 candidates", () => {
|
||||
const validUnicode = Buffer.from("café\t日本語\n", "utf8");
|
||||
const invalidUtf8 = Buffer.from([0xff, 0xfe, 0x41]);
|
||||
const embeddedNul = Buffer.from("bad\0str", "utf8");
|
||||
const c1Control = Buffer.from([0xc2, 0x85]);
|
||||
const noncharacter = Buffer.from([0xef, 0xbf, 0xbf]);
|
||||
const maxAllowed = Buffer.alloc(MAX_GO_STRING_LITERAL_BYTES, 0x61);
|
||||
const overMax = Buffer.alloc(MAX_GO_STRING_LITERAL_BYTES + 1, 0x62);
|
||||
const rodata = Buffer.concat([
|
||||
validUnicode,
|
||||
invalidUtf8,
|
||||
embeddedNul,
|
||||
c1Control,
|
||||
noncharacter,
|
||||
maxAllowed,
|
||||
overMax,
|
||||
Buffer.from("tail", "utf8"),
|
||||
]);
|
||||
const secondRodata = Buffer.from("across-boundary", "utf8");
|
||||
const baseRo = 0x102000n;
|
||||
const tailOffset = rodata.length - 4;
|
||||
const badPrefixLen =
|
||||
validUnicode.length +
|
||||
invalidUtf8.length +
|
||||
embeddedNul.length +
|
||||
c1Control.length +
|
||||
noncharacter.length;
|
||||
|
||||
const fixture = createGoRodataStringFixture({
|
||||
format: "elf",
|
||||
rodata,
|
||||
secondRodata,
|
||||
headers: [
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 0,
|
||||
ptr: baseRo,
|
||||
len: BigInt(validUnicode.length),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 16,
|
||||
ptr: baseRo + BigInt(badPrefixLen),
|
||||
len: BigInt(MAX_GO_STRING_LITERAL_BYTES),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 32,
|
||||
ptr: baseRo + BigInt(badPrefixLen + maxAllowed.length),
|
||||
len: BigInt(MAX_GO_STRING_LITERAL_BYTES + 1),
|
||||
},
|
||||
{ section: "data", offsetInSection: 48, ptr: baseRo, len: 0n },
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 64,
|
||||
ptr: baseRo + BigInt(validUnicode.length),
|
||||
len: BigInt(invalidUtf8.length),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 80,
|
||||
ptr: baseRo + BigInt(validUnicode.length + invalidUtf8.length),
|
||||
len: BigInt(embeddedNul.length),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 96,
|
||||
ptr:
|
||||
baseRo +
|
||||
BigInt(validUnicode.length + invalidUtf8.length + embeddedNul.length),
|
||||
len: BigInt(c1Control.length),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 112,
|
||||
ptr: baseRo + BigInt(badPrefixLen - noncharacter.length),
|
||||
len: BigInt(noncharacter.length),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 128,
|
||||
ptr: baseRo + BigInt(tailOffset),
|
||||
len: 10n,
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 144,
|
||||
ptr: baseRo + BigInt(rodata.length),
|
||||
len: BigInt(secondRodata.length + 8),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 164,
|
||||
ptr: baseRo + BigInt(tailOffset),
|
||||
len: 4n,
|
||||
},
|
||||
{
|
||||
section: "text",
|
||||
offsetInSection: 0,
|
||||
ptr: baseRo + BigInt(tailOffset),
|
||||
len: 4n,
|
||||
},
|
||||
{ section: "data", offsetInSection: 192, ptr: 0x101000n, len: 4n },
|
||||
],
|
||||
});
|
||||
fixture.bytes.write("CODE", 0x1000, "utf8");
|
||||
|
||||
const recovered = readGoBinaryImage(fixture.bytes, {
|
||||
includeStringLiterals: true,
|
||||
}).string_literals;
|
||||
expect(recovered?.items.map((item) => item.value)).toEqual([
|
||||
"café\t日本語\n",
|
||||
"a".repeat(MAX_GO_STRING_LITERAL_BYTES),
|
||||
]);
|
||||
expect(recovered?.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("enforces maxStringLiterals and the 512 KiB aggregate byte cap deterministically", () => {
|
||||
const chunks = Array.from({ length: 130 }, (_, index) =>
|
||||
Buffer.from(
|
||||
`${String(index).padStart(4, "0")}:${"x".repeat(MAX_GO_STRING_LITERAL_BYTES - 5)}`,
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
const rodata = Buffer.concat(chunks);
|
||||
const baseRo = 0x102000n;
|
||||
const fixture = createGoRodataStringFixture({
|
||||
format: "macho",
|
||||
machoChainedFixups: true,
|
||||
rodata,
|
||||
headers: chunks.map((chunk, index) => ({
|
||||
section: "data" as const,
|
||||
offsetInSection: index * 16,
|
||||
ptr: baseRo + BigInt(index * chunk.length),
|
||||
len: BigInt(chunk.length),
|
||||
})),
|
||||
});
|
||||
|
||||
const cappedByCount = readGoBinaryImage(fixture.bytes, {
|
||||
includeStringLiterals: true,
|
||||
maxStringLiterals: 2,
|
||||
});
|
||||
expect(cappedByCount.macho_chained_fixups).toBe(true);
|
||||
expect(cappedByCount.string_literals).toMatchObject({
|
||||
discovered_count: 130,
|
||||
max_items: 2,
|
||||
truncated: true,
|
||||
aggregate_bytes: MAX_GO_STRING_LITERAL_BYTES * 2,
|
||||
});
|
||||
expect(cappedByCount.string_literals?.items).toHaveLength(2);
|
||||
expect(
|
||||
cappedByCount.string_literals?.items[0]?.value.startsWith("0000:"),
|
||||
).toBe(true);
|
||||
expect(
|
||||
cappedByCount.string_literals?.items[1]?.value.startsWith("0001:"),
|
||||
).toBe(true);
|
||||
|
||||
const cappedByBytes = readGoBinaryImage(fixture.bytes, {
|
||||
includeStringLiterals: true,
|
||||
maxStringLiterals: 500,
|
||||
});
|
||||
expect(cappedByBytes.string_literals).toMatchObject({
|
||||
discovered_count: 130,
|
||||
max_items: 500,
|
||||
aggregate_bytes: MAX_GO_STRING_AGGREGATE_BYTES,
|
||||
truncated: true,
|
||||
});
|
||||
expect(cappedByBytes.string_literals?.items).toHaveLength(
|
||||
MAX_GO_STRING_AGGREGATE_BYTES / MAX_GO_STRING_LITERAL_BYTES,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"counts physical headers once in file order across reversed sections and overlapping PT_LOAD (corruptShstrtab=%s)",
|
||||
(corruptElfShstrtab) => {
|
||||
const baseRo = 0x102000n;
|
||||
const firstLen = Buffer.byteLength(LITERALS[0]);
|
||||
const secondLen = Buffer.byteLength(LITERALS[1]);
|
||||
const fixture = createGoRodataStringFixture({
|
||||
format: "elf",
|
||||
rodata: PACKED_RODATA,
|
||||
corruptElfShstrtab,
|
||||
elfSplitReversedDataSections: true,
|
||||
headers: [
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 0x200,
|
||||
ptr: baseRo,
|
||||
len: BigInt(firstLen),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 0x40,
|
||||
ptr: baseRo,
|
||||
len: BigInt(firstLen),
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: 0x60,
|
||||
ptr: baseRo + BigInt(firstLen),
|
||||
len: BigInt(secondLen),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const recovered = readGoBinaryImage(fixture.bytes, {
|
||||
includeStringLiterals: true,
|
||||
}).string_literals;
|
||||
expect(recovered?.items).toEqual([
|
||||
{
|
||||
value: "ListResourceProviders",
|
||||
location: { offset: fixture.rodataOffset, bytes: firstLen },
|
||||
first_header_offset: fixture.dataOffset + 0x40,
|
||||
reference_count: 2,
|
||||
},
|
||||
{
|
||||
value: "genai",
|
||||
location: {
|
||||
offset: fixture.rodataOffset + firstLen,
|
||||
bytes: secondLen,
|
||||
},
|
||||
first_header_offset: fixture.dataOffset + 0x60,
|
||||
reference_count: 1,
|
||||
},
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects invisible/bidi Cf format controls while keeping U+200C ZWNJ, U+200D ZWJ and visible Unicode", () => {
|
||||
const zeroWidthSpace = Buffer.from("zero\u200Bwidth", "utf8");
|
||||
const rtlOverride = Buffer.from("rtl\u202Eoverride", "utf8");
|
||||
const byteOrderMark = Buffer.from("\uFEFFbom", "utf8");
|
||||
const softHyphen = Buffer.from("soft\u00ADhyphen", "utf8");
|
||||
const persianZwnj = Buffer.from("می\u200cخواهم", "utf8");
|
||||
const zwjEmoji = Buffer.from("👩\u200d💻", "utf8");
|
||||
const visibleUnicode = Buffer.from("visible-café", "utf8");
|
||||
const slices = [
|
||||
zeroWidthSpace,
|
||||
persianZwnj,
|
||||
rtlOverride,
|
||||
zwjEmoji,
|
||||
byteOrderMark,
|
||||
softHyphen,
|
||||
visibleUnicode,
|
||||
];
|
||||
let cursor = 0;
|
||||
const headers = slices.map((slice, index) => {
|
||||
const header = {
|
||||
section: "data" as const,
|
||||
offsetInSection: index * 16,
|
||||
ptr: 0x102000n + BigInt(cursor),
|
||||
len: BigInt(slice.length),
|
||||
};
|
||||
cursor += slice.length;
|
||||
return header;
|
||||
});
|
||||
const fixture = createGoRodataStringFixture({
|
||||
format: "elf",
|
||||
rodata: Buffer.concat(slices),
|
||||
headers,
|
||||
});
|
||||
|
||||
const recovered = readGoBinaryImage(fixture.bytes, {
|
||||
includeStringLiterals: true,
|
||||
}).string_literals;
|
||||
expect(recovered?.items.map((item) => item.value)).toEqual([
|
||||
"می\u200cخواهم",
|
||||
"👩\u200d💻",
|
||||
"visible-café",
|
||||
]);
|
||||
expect(recovered?.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("bounds rejected-slice caching across thousands of distinct NUL slices without dropping later literals", () => {
|
||||
const rejectedCount = 6000;
|
||||
const nulBlob = Buffer.alloc(rejectedCount, 0);
|
||||
const survivor = Buffer.from("recovered-after-nul-flood", "utf8");
|
||||
const rodata = Buffer.concat([nulBlob, survivor]);
|
||||
const baseRo = 0x102000n;
|
||||
const headers = Array.from({ length: rejectedCount }, (_, index) => ({
|
||||
section: "data" as const,
|
||||
offsetInSection: index * 8,
|
||||
ptr: baseRo + BigInt(index),
|
||||
len: 1n,
|
||||
}));
|
||||
headers.push(
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: rejectedCount * 8,
|
||||
ptr: baseRo,
|
||||
len: 1n,
|
||||
},
|
||||
{
|
||||
section: "data",
|
||||
offsetInSection: (rejectedCount + 1) * 8,
|
||||
ptr: baseRo + BigInt(rejectedCount),
|
||||
len: BigInt(survivor.length),
|
||||
},
|
||||
);
|
||||
const fixture = createGoRodataStringFixture({
|
||||
format: "elf",
|
||||
bits: 32,
|
||||
rodata,
|
||||
dataSize: (rejectedCount + 8) * 8,
|
||||
headers,
|
||||
});
|
||||
|
||||
const recovered = readGoBinaryImage(fixture.bytes, {
|
||||
includeStringLiterals: true,
|
||||
}).string_literals;
|
||||
expect(recovered).toEqual({
|
||||
evidence_kind: "inferred",
|
||||
discovered_count: 1,
|
||||
aggregate_bytes: survivor.length,
|
||||
max_items: 2000,
|
||||
truncated: false,
|
||||
items: [
|
||||
{
|
||||
value: "recovered-after-nul-flood",
|
||||
location: {
|
||||
offset: fixture.rodataOffset + rejectedCount,
|
||||
bytes: survivor.length,
|
||||
},
|
||||
first_header_offset: fixture.dataOffset + (rejectedCount + 1) * 8,
|
||||
reference_count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("indexes 25,000 Mach-O load regions and sections without quadratic stalls and honors region and section cancellation", () => {
|
||||
const count = 25_000;
|
||||
const slotBytes = 16;
|
||||
const bytes = Buffer.alloc((count + 1) * slotBytes);
|
||||
const literal = "macho-indexed";
|
||||
const literalBytes = Buffer.from(literal, "utf8");
|
||||
literalBytes.copy(bytes, 0);
|
||||
|
||||
const mappings: FileMapping[] = [];
|
||||
const loadRegions: LoadRegion[] = [];
|
||||
const sections: ImageSection[] = [];
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const offset = index * slotBytes;
|
||||
const address = 0x100000n + BigInt(index) * 0x2000n;
|
||||
const mapping: FileMapping = { address, offset, size: slotBytes };
|
||||
const readable = index !== count - 2;
|
||||
mappings.push(mapping);
|
||||
loadRegions.push({
|
||||
mapping,
|
||||
memory_size: BigInt(slotBytes),
|
||||
read: readable,
|
||||
write: index > 0,
|
||||
execute: false,
|
||||
});
|
||||
sections.push({
|
||||
format: "macho",
|
||||
name: index === 0 ? "__TEXT,__rodata" : "__DATA,__data",
|
||||
section_name: index === 0 ? "__rodata" : "__data",
|
||||
type: 0,
|
||||
flags: 0,
|
||||
address,
|
||||
declared_range: { offset: BigInt(offset), size: BigInt(slotBytes) },
|
||||
segment_mapping: mapping,
|
||||
segment_unmapped: false,
|
||||
offset,
|
||||
size: slotBytes,
|
||||
file_backed: true,
|
||||
read_only: index === 0,
|
||||
compressed: false,
|
||||
});
|
||||
}
|
||||
|
||||
const roAddr = 0x100000n;
|
||||
const unreadableHeaderOffset = (count - 2) * slotBytes;
|
||||
bytes.writeBigUInt64LE(roAddr, unreadableHeaderOffset);
|
||||
bytes.writeBigUInt64LE(
|
||||
BigInt(literalBytes.length),
|
||||
unreadableHeaderOffset + 8,
|
||||
);
|
||||
|
||||
const validHeaderOffset = (count - 1) * slotBytes;
|
||||
bytes.writeBigUInt64LE(roAddr, validHeaderOffset);
|
||||
bytes.writeBigUInt64LE(BigInt(literalBytes.length), validHeaderOffset + 8);
|
||||
|
||||
const container: BinaryContainer = {
|
||||
format: "macho",
|
||||
architecture: "arm64",
|
||||
bits: 64,
|
||||
byte_order: "little",
|
||||
mappings,
|
||||
load_regions: loadRegions,
|
||||
zero_fills: [],
|
||||
sections,
|
||||
};
|
||||
|
||||
const started = performance.now();
|
||||
const recovered = recoverGoStringLiterals(bytes, container);
|
||||
const elapsedMs = performance.now() - started;
|
||||
|
||||
expect(elapsedMs).toBeLessThan(500);
|
||||
expect(recovered.items).toEqual([
|
||||
{
|
||||
value: literal,
|
||||
location: { offset: 0, bytes: literalBytes.length },
|
||||
first_header_offset: validHeaderOffset,
|
||||
reference_count: 1,
|
||||
},
|
||||
]);
|
||||
|
||||
let inRegionLoop = false;
|
||||
let observedRegions = 0;
|
||||
let regionCancelChecks = 0;
|
||||
let inSectionLoop = false;
|
||||
let observedSections = 0;
|
||||
const trackedLoadRegions = new Proxy(loadRegions, {
|
||||
get(target, property, receiver) {
|
||||
if (typeof property === "string" && /^\d+$/u.test(property)) {
|
||||
inRegionLoop = true;
|
||||
observedRegions += 1;
|
||||
}
|
||||
return Reflect.get(target, property, receiver);
|
||||
},
|
||||
});
|
||||
const trackedSections = new Proxy(sections, {
|
||||
get(target, property, receiver) {
|
||||
if (typeof property === "string" && /^\d+$/u.test(property)) {
|
||||
inSectionLoop = true;
|
||||
observedSections += 1;
|
||||
}
|
||||
return Reflect.get(target, property, receiver);
|
||||
},
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
recoverGoStringLiterals(
|
||||
bytes,
|
||||
{
|
||||
...container,
|
||||
load_regions: trackedLoadRegions,
|
||||
sections: trackedSections,
|
||||
},
|
||||
2000,
|
||||
() => {
|
||||
if (inRegionLoop && !inSectionLoop) {
|
||||
regionCancelChecks += 1;
|
||||
if (regionCancelChecks >= 10)
|
||||
throw new Error("cancelled during region indexing");
|
||||
}
|
||||
},
|
||||
),
|
||||
).toThrow("cancelled during region indexing");
|
||||
expect(observedRegions).toBeGreaterThan(0);
|
||||
expect(observedRegions).toBeLessThan(count);
|
||||
expect(observedSections).toBe(0);
|
||||
|
||||
inSectionLoop = false;
|
||||
observedSections = 0;
|
||||
expect(() =>
|
||||
recoverGoStringLiterals(
|
||||
bytes,
|
||||
{ ...container, sections: trackedSections },
|
||||
2000,
|
||||
() => {
|
||||
if (inSectionLoop) throw new Error("cancelled during section indexing");
|
||||
},
|
||||
),
|
||||
).toThrow("cancelled during section indexing");
|
||||
expect(observedSections).toBeLessThan(count);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ target: "data", mode: "single", expected: ["macho-span-check"] },
|
||||
{ target: "data", mode: "touching", expected: ["macho-span-check"] },
|
||||
{ target: "data", mode: "overlapping", expected: ["macho-span-check"] },
|
||||
{ target: "data", mode: "gap", expected: [] },
|
||||
{ target: "data", mode: "unreadable-tail", expected: [] },
|
||||
{ target: "rodata", mode: "single", expected: ["macho-span-check"] },
|
||||
{ target: "rodata", mode: "touching", expected: ["macho-span-check"] },
|
||||
{ target: "rodata", mode: "overlapping", expected: ["macho-span-check"] },
|
||||
{ target: "rodata", mode: "gap", expected: [] },
|
||||
{ target: "rodata", mode: "unreadable-tail", expected: [] },
|
||||
] as const)(
|
||||
"requires contiguous readable Mach-O load_regions for $target ($mode)",
|
||||
({ target, mode, expected }) => {
|
||||
const literal = "macho-span-check";
|
||||
const bytes = Buffer.alloc(32);
|
||||
bytes.write(literal, 0, "utf8");
|
||||
const roAddr = 0x100000n;
|
||||
const dataAddr = 0x200000n;
|
||||
bytes.writeBigUInt64LE(roAddr, 16);
|
||||
bytes.writeBigUInt64LE(16n, 24);
|
||||
|
||||
const buildRegions = (
|
||||
address: bigint,
|
||||
offset: number,
|
||||
write: boolean,
|
||||
kind: "single" | "touching" | "overlapping" | "gap" | "unreadable-tail",
|
||||
): LoadRegion[] => {
|
||||
const region = (
|
||||
delta: number,
|
||||
size: number,
|
||||
read: boolean,
|
||||
): LoadRegion => ({
|
||||
mapping: {
|
||||
address: address + BigInt(delta),
|
||||
offset: offset + delta,
|
||||
size,
|
||||
},
|
||||
memory_size: BigInt(size),
|
||||
read,
|
||||
write,
|
||||
execute: false,
|
||||
});
|
||||
if (kind === "single") return [region(0, 16, true)];
|
||||
if (kind === "touching") return [region(0, 8, true), region(8, 8, true)];
|
||||
if (kind === "overlapping")
|
||||
return [region(0, 12, true), region(4, 12, true)];
|
||||
if (kind === "gap") return [region(0, 6, true), region(10, 6, true)];
|
||||
return [region(0, 8, true), region(8, 8, false)];
|
||||
};
|
||||
|
||||
const roMapping: FileMapping = { address: roAddr, offset: 0, size: 16 };
|
||||
const dataMapping: FileMapping = {
|
||||
address: dataAddr,
|
||||
offset: 16,
|
||||
size: 16,
|
||||
};
|
||||
const container: BinaryContainer = {
|
||||
format: "macho",
|
||||
architecture: "arm64",
|
||||
bits: 64,
|
||||
byte_order: "little",
|
||||
mappings: [roMapping, dataMapping],
|
||||
load_regions: [
|
||||
...buildRegions(
|
||||
roAddr,
|
||||
0,
|
||||
false,
|
||||
target === "rodata" ? mode : "single",
|
||||
),
|
||||
...buildRegions(
|
||||
dataAddr,
|
||||
16,
|
||||
true,
|
||||
target === "data" ? mode : "single",
|
||||
),
|
||||
],
|
||||
zero_fills: [],
|
||||
sections: [
|
||||
{
|
||||
format: "macho",
|
||||
name: "__TEXT,__rodata",
|
||||
section_name: "__rodata",
|
||||
type: 0,
|
||||
flags: 0,
|
||||
address: roAddr,
|
||||
declared_range: { offset: 0n, size: 16n },
|
||||
segment_mapping: roMapping,
|
||||
segment_unmapped: false,
|
||||
offset: 0,
|
||||
size: 16,
|
||||
file_backed: true,
|
||||
read_only: true,
|
||||
compressed: false,
|
||||
},
|
||||
{
|
||||
format: "macho",
|
||||
name: "__DATA,__data",
|
||||
section_name: "__data",
|
||||
type: 0,
|
||||
flags: 0,
|
||||
address: dataAddr,
|
||||
declared_range: { offset: 16n, size: 16n },
|
||||
segment_mapping: dataMapping,
|
||||
segment_unmapped: false,
|
||||
offset: 16,
|
||||
size: 16,
|
||||
file_backed: true,
|
||||
read_only: false,
|
||||
compressed: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const recovered = recoverGoStringLiterals(bytes, container);
|
||||
expect(recovered.items.map((item) => item.value)).toEqual(expected);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,589 @@
|
||||
import { isUtf8 } from "node:buffer";
|
||||
import {
|
||||
DEFAULT_MAX_GO_STRING_LITERALS,
|
||||
MAX_GO_STRING_AGGREGATE_BYTES,
|
||||
MAX_GO_STRING_HEADER_SCAN_BYTES,
|
||||
MAX_GO_STRING_LITERAL_BYTES,
|
||||
type GoStringLiteral,
|
||||
type GoStringLiterals,
|
||||
} from "../domain/go/goBinary.js";
|
||||
import {
|
||||
BinaryMappingFailure,
|
||||
createFileMappingIndex,
|
||||
imageFileMappings,
|
||||
type BinaryContainer,
|
||||
type FileMapping,
|
||||
type FileMappingIndex,
|
||||
type LoadRegion,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
|
||||
const MAX_TRACKED_CANDIDATES = 100_000;
|
||||
const MAX_REJECTED_SLICE_CACHE = 4096;
|
||||
const PE_MEM_READ = 0x40000000;
|
||||
const PE_MEM_EXECUTE = 0x20000000;
|
||||
const PE_MEM_DISCARDABLE = 0x02000000;
|
||||
const PE_CNT_CODE = 0x00000020;
|
||||
const PE_CNT_INITIALIZED_DATA = 0x00000040;
|
||||
const MACHO_ATTR_INSTRUCTIONS = 0x80000400;
|
||||
const MACHO_ATTR_DEBUG = 0x02000000;
|
||||
const DISALLOWED_GO_STRING_CODE_POINT =
|
||||
/(?![\t\n\r\u200c\u200d])[\p{Cc}\p{Cf}\p{Cs}\p{Co}\p{Cn}]/u;
|
||||
|
||||
interface CandidateLiteral {
|
||||
readonly offset: number;
|
||||
readonly bytes: number;
|
||||
firstHeaderOffset: number;
|
||||
referenceCount: number;
|
||||
}
|
||||
|
||||
interface HeaderScanSpan {
|
||||
readonly startOffset: number;
|
||||
readonly endOffset: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover bounded printable UTF-8 Go string literals referenced by word-aligned
|
||||
* static `(ptr, len)` pairs in non-executable mapped data.
|
||||
*/
|
||||
export const recoverGoStringLiterals = (
|
||||
bytes: Buffer,
|
||||
container: BinaryContainer,
|
||||
maxItems: number = DEFAULT_MAX_GO_STRING_LITERALS,
|
||||
checkActive?: () => void,
|
||||
): GoStringLiterals => {
|
||||
checkActive?.();
|
||||
const index = createFileMappingIndex(
|
||||
imageFileMappings(container),
|
||||
container.zero_fills,
|
||||
);
|
||||
checkActive?.();
|
||||
const readableRegions = createReadableLoadRegionIndex(
|
||||
container.load_regions,
|
||||
checkActive,
|
||||
);
|
||||
const targets = selectTargetSpans(
|
||||
bytes,
|
||||
container,
|
||||
index,
|
||||
readableRegions,
|
||||
checkActive,
|
||||
);
|
||||
const headerSpans = selectHeaderScanSpans(
|
||||
bytes,
|
||||
container,
|
||||
index,
|
||||
readableRegions,
|
||||
checkActive,
|
||||
);
|
||||
if (targets.length === 0 || headerSpans.length === 0)
|
||||
return {
|
||||
evidence_kind: "inferred",
|
||||
items: [],
|
||||
discovered_count: 0,
|
||||
aggregate_bytes: 0,
|
||||
max_items: maxItems,
|
||||
truncated: false,
|
||||
};
|
||||
|
||||
const { bits } = container;
|
||||
const little = container.byte_order === "little";
|
||||
const wordBytes = bits === 64 ? 8 : 4;
|
||||
const firstTarget = targets[0] as FileMapping;
|
||||
const lastTarget = targets[targets.length - 1] as FileMapping;
|
||||
const minTargetAddr = firstTarget.address;
|
||||
const maxTargetEnd = lastTarget.address + BigInt(lastTarget.size);
|
||||
|
||||
const accepted = new Map<number, CandidateLiteral>();
|
||||
const rejected = new Set<number>();
|
||||
const candidates: CandidateLiteral[] = [];
|
||||
let remainingBudget = MAX_GO_STRING_HEADER_SCAN_BYTES;
|
||||
let budgetTruncated = false;
|
||||
let steps = 0;
|
||||
|
||||
outer: for (const { startOffset, endOffset } of headerSpans) {
|
||||
if (remainingBudget < wordBytes * 2) {
|
||||
budgetTruncated = true;
|
||||
break;
|
||||
}
|
||||
remainingBudget -= wordBytes;
|
||||
let word0 = readWord(bytes, startOffset, bits, little);
|
||||
|
||||
for (
|
||||
let headerOffset = startOffset;
|
||||
headerOffset + wordBytes * 2 <= endOffset;
|
||||
headerOffset += wordBytes
|
||||
) {
|
||||
if ((++steps & 0xffff) === 0) checkActive?.();
|
||||
if (remainingBudget < wordBytes) {
|
||||
budgetTruncated = true;
|
||||
break outer;
|
||||
}
|
||||
remainingBudget -= wordBytes;
|
||||
const word1 = readWord(bytes, headerOffset + wordBytes, bits, little);
|
||||
const ptr = word0;
|
||||
const rawLen = word1;
|
||||
word0 = word1;
|
||||
|
||||
if (rawLen < 1n || rawLen > BigInt(MAX_GO_STRING_LITERAL_BYTES)) continue;
|
||||
if (ptr < minTargetAddr || ptr >= maxTargetEnd) continue;
|
||||
const targetIndex = lastSpanAtOrBelow(targets, ptr);
|
||||
if (targetIndex < 0) continue;
|
||||
const target = targets[targetIndex] as FileMapping;
|
||||
if (ptr + rawLen > target.address + BigInt(target.size)) continue;
|
||||
|
||||
const byteLen = Number(rawLen);
|
||||
const sliceOffset = target.offset + Number(ptr - target.address);
|
||||
if (
|
||||
headerOffset < sliceOffset + byteLen &&
|
||||
sliceOffset < headerOffset + wordBytes * 2
|
||||
)
|
||||
continue;
|
||||
|
||||
const key = sliceOffset * 8192 + byteLen;
|
||||
const existing = accepted.get(key);
|
||||
if (existing !== undefined) {
|
||||
existing.referenceCount += 1;
|
||||
if (headerOffset < existing.firstHeaderOffset)
|
||||
existing.firstHeaderOffset = headerOffset;
|
||||
continue;
|
||||
}
|
||||
if (rejected.has(key)) continue;
|
||||
|
||||
if (candidates.length >= MAX_TRACKED_CANDIDATES) {
|
||||
budgetTruncated = true;
|
||||
break outer;
|
||||
}
|
||||
if (remainingBudget < byteLen) {
|
||||
budgetTruncated = true;
|
||||
break outer;
|
||||
}
|
||||
remainingBudget -= byteLen;
|
||||
|
||||
const slice = bytes.subarray(sliceOffset, sliceOffset + byteLen);
|
||||
if (!isPrintableGoUtf8(slice)) {
|
||||
if (rejected.size >= MAX_REJECTED_SLICE_CACHE) rejected.clear();
|
||||
rejected.add(key);
|
||||
continue;
|
||||
}
|
||||
const entry: CandidateLiteral = {
|
||||
offset: sliceOffset,
|
||||
bytes: byteLen,
|
||||
firstHeaderOffset: headerOffset,
|
||||
referenceCount: 1,
|
||||
};
|
||||
accepted.set(key, entry);
|
||||
candidates.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
checkActive?.();
|
||||
candidates.sort(
|
||||
(left, right) => left.offset - right.offset || left.bytes - right.bytes,
|
||||
);
|
||||
|
||||
const items: GoStringLiteral[] = [];
|
||||
let aggregateBytes = 0;
|
||||
let outputTruncated = false;
|
||||
for (const candidate of candidates) {
|
||||
if (
|
||||
items.length >= maxItems ||
|
||||
aggregateBytes + candidate.bytes > MAX_GO_STRING_AGGREGATE_BYTES
|
||||
) {
|
||||
outputTruncated = true;
|
||||
break;
|
||||
}
|
||||
items.push({
|
||||
value: bytes
|
||||
.subarray(candidate.offset, candidate.offset + candidate.bytes)
|
||||
.toString("utf8"),
|
||||
location: { offset: candidate.offset, bytes: candidate.bytes },
|
||||
first_header_offset: candidate.firstHeaderOffset,
|
||||
reference_count: candidate.referenceCount,
|
||||
});
|
||||
aggregateBytes += candidate.bytes;
|
||||
}
|
||||
|
||||
return {
|
||||
evidence_kind: "inferred",
|
||||
items,
|
||||
discovered_count: candidates.length,
|
||||
aggregate_bytes: aggregateBytes,
|
||||
max_items: maxItems,
|
||||
truncated: budgetTruncated || outputTruncated,
|
||||
};
|
||||
};
|
||||
|
||||
const readWord = (
|
||||
bytes: Buffer,
|
||||
offset: number,
|
||||
bits: 32 | 64,
|
||||
little: boolean,
|
||||
): bigint =>
|
||||
bits === 32
|
||||
? BigInt(little ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset))
|
||||
: little
|
||||
? bytes.readBigUInt64LE(offset)
|
||||
: bytes.readBigUInt64BE(offset);
|
||||
|
||||
const isPrintableGoUtf8 = (slice: Buffer): boolean => {
|
||||
let hasNonAscii = false;
|
||||
for (const byte of slice) {
|
||||
if (byte >= 0x80) {
|
||||
hasNonAscii = true;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
(byte < 0x20 && byte !== 0x09 && byte !== 0x0a && byte !== 0x0d) ||
|
||||
byte === 0x7f
|
||||
)
|
||||
return false;
|
||||
}
|
||||
if (!hasNonAscii) return true;
|
||||
if (!isUtf8(slice)) return false;
|
||||
return !DISALLOWED_GO_STRING_CODE_POINT.test(slice.toString("utf8"));
|
||||
};
|
||||
|
||||
const admitMappedSpan = (
|
||||
bytes: Buffer,
|
||||
container: BinaryContainer,
|
||||
index: FileMappingIndex,
|
||||
address: bigint,
|
||||
offset: number,
|
||||
size: number,
|
||||
): FileMapping | null => {
|
||||
if (
|
||||
size <= 0 ||
|
||||
offset < 0 ||
|
||||
offset > bytes.length ||
|
||||
size > bytes.length - offset ||
|
||||
address < 0n ||
|
||||
address + BigInt(size) > 1n << BigInt(container.bits)
|
||||
)
|
||||
return null;
|
||||
try {
|
||||
return index.offset(address, size) === offset
|
||||
? { address, offset, size }
|
||||
: null;
|
||||
} catch (cause) {
|
||||
if (cause instanceof BinaryMappingFailure) return null;
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
interface ReadableLoadRegionIndex {
|
||||
readonly contains: (address: bigint, size: number) => boolean;
|
||||
}
|
||||
|
||||
const createReadableLoadRegionIndex = (
|
||||
loadRegions: readonly LoadRegion[] | undefined,
|
||||
checkActive?: () => void,
|
||||
): ReadableLoadRegionIndex => {
|
||||
if (loadRegions === undefined || loadRegions.length === 0)
|
||||
return { contains: () => false };
|
||||
const raw: { start: bigint; end: bigint }[] = [];
|
||||
let steps = 0;
|
||||
for (const region of loadRegions) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
if (!region.read || region.mapping.size <= 0) continue;
|
||||
raw.push({
|
||||
start: region.mapping.address,
|
||||
end: region.mapping.address + BigInt(region.mapping.size),
|
||||
});
|
||||
}
|
||||
if (raw.length === 0) return { contains: () => false };
|
||||
checkActive?.();
|
||||
raw.sort((left, right) =>
|
||||
left.start < right.start
|
||||
? -1
|
||||
: left.start > right.start
|
||||
? 1
|
||||
: left.end < right.end
|
||||
? -1
|
||||
: left.end > right.end
|
||||
? 1
|
||||
: 0,
|
||||
);
|
||||
const starts: bigint[] = [];
|
||||
const ends: bigint[] = [];
|
||||
let currentStart = (raw[0] as (typeof raw)[number]).start;
|
||||
let currentEnd = (raw[0] as (typeof raw)[number]).end;
|
||||
for (let index = 1; index < raw.length; index += 1) {
|
||||
if ((index & 0x3ff) === 0) checkActive?.();
|
||||
const span = raw[index] as (typeof raw)[number];
|
||||
if (span.start <= currentEnd) {
|
||||
if (span.end > currentEnd) currentEnd = span.end;
|
||||
continue;
|
||||
}
|
||||
starts.push(currentStart);
|
||||
ends.push(currentEnd);
|
||||
currentStart = span.start;
|
||||
currentEnd = span.end;
|
||||
}
|
||||
starts.push(currentStart);
|
||||
ends.push(currentEnd);
|
||||
return {
|
||||
contains: (address, size) => {
|
||||
if (size <= 0) return false;
|
||||
const index = lastBigIntAtOrBelow(starts, address);
|
||||
return index >= 0 && address + BigInt(size) <= (ends[index] as bigint);
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Retain read-only non-executable mapped spans without merging distinct sections,
|
||||
* so `[ptr, ptr + len)` cannot cross section boundaries.
|
||||
*/
|
||||
const selectTargetSpans = (
|
||||
bytes: Buffer,
|
||||
container: BinaryContainer,
|
||||
index: FileMappingIndex,
|
||||
readableRegions: ReadableLoadRegionIndex,
|
||||
checkActive?: () => void,
|
||||
): readonly FileMapping[] => {
|
||||
const raw: FileMapping[] = [];
|
||||
let steps = 0;
|
||||
if (container.sections.length > 0) {
|
||||
for (const section of container.sections) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
if (!section.read_only) continue;
|
||||
if (
|
||||
section.format === "pe" &&
|
||||
(section.characteristics & PE_MEM_DISCARDABLE) !== 0
|
||||
)
|
||||
continue;
|
||||
if (
|
||||
section.format === "macho" &&
|
||||
(section.segment_unmapped ||
|
||||
(section.flags & MACHO_ATTR_DEBUG) !== 0 ||
|
||||
!readableRegions.contains(section.address, section.size))
|
||||
)
|
||||
continue;
|
||||
const address =
|
||||
section.format === "pe"
|
||||
? (container.image_base ?? 0n) + BigInt(section.virtual_address)
|
||||
: section.address;
|
||||
const admitted = admitMappedSpan(
|
||||
bytes,
|
||||
container,
|
||||
index,
|
||||
address,
|
||||
section.offset,
|
||||
section.size,
|
||||
);
|
||||
if (admitted !== null) raw.push(admitted);
|
||||
}
|
||||
} else {
|
||||
for (const region of container.load_regions ?? []) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
if (!region.read || region.write || region.execute) continue;
|
||||
const admitted = admitMappedSpan(
|
||||
bytes,
|
||||
container,
|
||||
index,
|
||||
region.mapping.address,
|
||||
region.mapping.offset,
|
||||
region.mapping.size,
|
||||
);
|
||||
if (admitted !== null) raw.push(admitted);
|
||||
}
|
||||
}
|
||||
raw.sort((left, right) =>
|
||||
left.address < right.address
|
||||
? -1
|
||||
: left.address > right.address
|
||||
? 1
|
||||
: right.size - left.size,
|
||||
);
|
||||
const nonContained: FileMapping[] = [];
|
||||
let maxEnd = -1n;
|
||||
for (const span of raw) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
const end = span.address + BigInt(span.size);
|
||||
if (end <= maxEnd) continue;
|
||||
nonContained.push(span);
|
||||
maxEnd = end;
|
||||
}
|
||||
return nonContained;
|
||||
};
|
||||
|
||||
/** Select non-executable initialized data spans that may store static `(ptr, len)` pairs. */
|
||||
const selectHeaderScanSpans = (
|
||||
bytes: Buffer,
|
||||
container: BinaryContainer,
|
||||
index: FileMappingIndex,
|
||||
readableRegions: ReadableLoadRegionIndex,
|
||||
checkActive?: () => void,
|
||||
): readonly HeaderScanSpan[] => {
|
||||
const spans: FileMapping[] = [];
|
||||
let steps = 0;
|
||||
const push = (address: bigint, offset: number, size: number): void => {
|
||||
const admitted = admitMappedSpan(
|
||||
bytes,
|
||||
container,
|
||||
index,
|
||||
address,
|
||||
offset,
|
||||
size,
|
||||
);
|
||||
if (admitted !== null) spans.push(admitted);
|
||||
};
|
||||
|
||||
for (const section of container.sections) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
if (!section.file_backed || section.compressed) continue;
|
||||
if (section.format === "elf") {
|
||||
if (
|
||||
section.type !== 1 ||
|
||||
(section.flags & 2n) === 0n ||
|
||||
(section.flags & 4n) !== 0n
|
||||
)
|
||||
continue;
|
||||
push(section.address, section.offset, section.size);
|
||||
continue;
|
||||
}
|
||||
if (section.format === "macho") {
|
||||
if (
|
||||
section.segment_unmapped ||
|
||||
(section.flags & MACHO_ATTR_INSTRUCTIONS) !== 0 ||
|
||||
(section.flags & MACHO_ATTR_DEBUG) !== 0 ||
|
||||
!readableRegions.contains(section.address, section.size)
|
||||
)
|
||||
continue;
|
||||
push(section.address, section.offset, section.size);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
(section.characteristics & PE_MEM_READ) === 0 ||
|
||||
(section.characteristics & PE_MEM_EXECUTE) !== 0 ||
|
||||
(section.characteristics & PE_CNT_CODE) !== 0 ||
|
||||
(section.characteristics & PE_MEM_DISCARDABLE) !== 0 ||
|
||||
(section.characteristics & PE_CNT_INITIALIZED_DATA) === 0 ||
|
||||
container.image_base === undefined
|
||||
)
|
||||
continue;
|
||||
push(
|
||||
container.image_base + BigInt(section.virtual_address),
|
||||
section.offset,
|
||||
section.size,
|
||||
);
|
||||
}
|
||||
|
||||
if (container.sections.length === 0) {
|
||||
for (const region of container.load_regions ?? []) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
if (!region.read || region.execute) continue;
|
||||
push(region.mapping.address, region.mapping.offset, region.mapping.size);
|
||||
}
|
||||
} else if (
|
||||
container.format === "elf" &&
|
||||
container.section_names_limitation !== undefined
|
||||
) {
|
||||
for (const region of container.load_regions ?? []) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
if (!region.read || !region.write || region.execute) continue;
|
||||
push(region.mapping.address, region.mapping.offset, region.mapping.size);
|
||||
}
|
||||
}
|
||||
|
||||
return normalizeHeaderScanSpans(
|
||||
spans,
|
||||
container.bits === 64 ? 8 : 4,
|
||||
checkActive,
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Align candidate header spans to pointer-word boundaries, merge overlapping
|
||||
* same-phase intervals so physical headers are never counted twice, and sort
|
||||
* in ascending file-offset order so `first_header_offset` is minimal.
|
||||
*/
|
||||
const normalizeHeaderScanSpans = (
|
||||
spans: readonly FileMapping[],
|
||||
wordBytes: 4 | 8,
|
||||
checkActive?: () => void,
|
||||
): readonly HeaderScanSpan[] => {
|
||||
const wordWidth = BigInt(wordBytes);
|
||||
const aligned: {
|
||||
startOffset: number;
|
||||
endOffset: number;
|
||||
readonly phase: number;
|
||||
}[] = [];
|
||||
let steps = 0;
|
||||
for (const span of spans) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
const misalign = Number(
|
||||
(wordWidth - (span.address % wordWidth)) % wordWidth,
|
||||
);
|
||||
const startOffset = span.offset + misalign;
|
||||
const spanEnd = span.offset + span.size;
|
||||
if (spanEnd - startOffset < wordBytes * 2) continue;
|
||||
const endOffset =
|
||||
startOffset + Math.floor((spanEnd - startOffset) / wordBytes) * wordBytes;
|
||||
aligned.push({
|
||||
startOffset,
|
||||
endOffset,
|
||||
phase: startOffset % wordBytes,
|
||||
});
|
||||
}
|
||||
if (aligned.length === 0) return [];
|
||||
aligned.sort(
|
||||
(left, right) =>
|
||||
left.phase - right.phase ||
|
||||
left.startOffset - right.startOffset ||
|
||||
right.endOffset - left.endOffset,
|
||||
);
|
||||
const merged: HeaderScanSpan[] = [];
|
||||
let current = aligned[0] as (typeof aligned)[number];
|
||||
for (let index = 1; index < aligned.length; index += 1) {
|
||||
if ((++steps & 0x3ff) === 0) checkActive?.();
|
||||
const next = aligned[index] as (typeof aligned)[number];
|
||||
if (next.phase === current.phase && next.startOffset < current.endOffset) {
|
||||
if (next.endOffset > current.endOffset)
|
||||
current = { ...current, endOffset: next.endOffset };
|
||||
continue;
|
||||
}
|
||||
merged.push({
|
||||
startOffset: current.startOffset,
|
||||
endOffset: current.endOffset,
|
||||
});
|
||||
current = next;
|
||||
}
|
||||
merged.push({
|
||||
startOffset: current.startOffset,
|
||||
endOffset: current.endOffset,
|
||||
});
|
||||
merged.sort(
|
||||
(left, right) =>
|
||||
left.startOffset - right.startOffset || left.endOffset - right.endOffset,
|
||||
);
|
||||
return merged;
|
||||
};
|
||||
|
||||
const lastBigIntAtOrBelow = (
|
||||
sorted: readonly bigint[],
|
||||
value: bigint,
|
||||
): number => {
|
||||
let low = 0;
|
||||
let high = sorted.length;
|
||||
while (low < high) {
|
||||
const middle = (low + high) >>> 1;
|
||||
if ((sorted[middle] as bigint) <= value) low = middle + 1;
|
||||
else high = middle;
|
||||
}
|
||||
return low - 1;
|
||||
};
|
||||
|
||||
const lastSpanAtOrBelow = (
|
||||
sorted: readonly FileMapping[],
|
||||
address: bigint,
|
||||
): number => {
|
||||
let low = 0;
|
||||
let high = sorted.length;
|
||||
while (low < high) {
|
||||
const middle = (low + high) >>> 1;
|
||||
if ((sorted[middle] as FileMapping).address <= address) low = middle + 1;
|
||||
else high = middle;
|
||||
}
|
||||
return low - 1;
|
||||
};
|
||||
@@ -85,6 +85,7 @@ export type ImageSection = {
|
||||
/** Null when the ELF name table is unavailable or this name is unreadable. */
|
||||
readonly name: string | null;
|
||||
readonly type: number;
|
||||
readonly flags: bigint;
|
||||
readonly address: bigint;
|
||||
readonly declared_range: {
|
||||
readonly offset: bigint;
|
||||
@@ -105,6 +106,7 @@ export type ImageSection = {
|
||||
readonly name: string;
|
||||
readonly section_name: string;
|
||||
readonly type: number;
|
||||
readonly flags: number;
|
||||
readonly address: bigint;
|
||||
readonly declared_range: {
|
||||
readonly offset: bigint;
|
||||
|
||||
@@ -82,6 +82,7 @@ export const readElfImage = (
|
||||
format: "elf",
|
||||
name: section.name,
|
||||
type: section.type,
|
||||
flags: section.flags,
|
||||
address: section.address,
|
||||
declared_range: { offset: section.offset, size: section.size },
|
||||
...range,
|
||||
|
||||
@@ -230,6 +230,7 @@ const machoImageSection = ({
|
||||
name,
|
||||
section_name: sectionName,
|
||||
type: flags & 0xff,
|
||||
flags,
|
||||
address: sectionAddress,
|
||||
declared_range: { offset: sectionOffset, size: sectionSize },
|
||||
segment_mapping: segmentMapping,
|
||||
|
||||
Reference in new issue
Block a user