mirror of
https://github.com/morluto/rea.git
synced 2026-11-13 18:22:42 +00:00
feat(analysis): make resource budgets recoverable across CLI and MCP (#9706)
* refactor(analysis): centralize resource budgets and recovery facts * feat(analysis): expose ASAR and Ghidra runtime admission controls * feat(artifacts): expose reader admission and preserve limited recognition * feat(javascript): expose recoverable semantic precision budgets * feat(web): add recoverable source-map budgets and compact retained views * feat(web): make historical capture decoding budgets recoverable * fix(desktop): enforce selected view policies across capture and export * feat(runtime): make Frida and LLDB retention budgets selectable * feat(native): expose diagnostic budgets and preserve exhaustion causes * feat(managed): expose recoverable snapshot and representation budgets * feat(metadata): make Go and Rust inspection budgets recoverable * feat(wasm): expose representation and producer admissions * feat(evm): expose per-call worker resource budgets * feat(android): make resource and string-pool omissions recoverable * feat(producers): expose command and recorded-capture budgets * docs(analysis): record recovery contracts and integrated validation * fix(docs): link calibration oracle from published verification pages * fix(docs): restore desktop task anchors and receipt links * test(analysis): align fixtures with selectable budget contracts * fix(analysis): remove source-location projector dependency cycle * test(analysis): refresh budget recovery and verifier regressions * test(analysis): preserve format and publication recovery expectations * test(windows): align conformance with owned runtime admission * fix(desktop): preserve selected alias kinds in exported state * test(analysis): align provider fixtures with recoverable budgets * test(desktop): validate emitted alias metadata structurally * fix(ghidra): classify bounded Windows endpoint refusals * test(cli): assert selectable artifact admission recovery * fix(cli): preserve capture provenance in environment builds
This commit is contained in:
1 parent
55b6d2ca20
commit
cb4e5cfbff
515 files changed
+21066
-3637
No files matched your search
@@ -371,6 +371,7 @@ const partialMetadataResult = ({
|
||||
export const inspectManagedArtifactBytes = (
|
||||
bytes: Buffer,
|
||||
target: BinaryTarget,
|
||||
maxRepresentationBytes?: number,
|
||||
): ManagedArtifactInspection => {
|
||||
const layout = readManagedPeLayout(bytes);
|
||||
if (layout.cli === null) return unavailableResult(target, bytes, layout);
|
||||
@@ -386,6 +387,7 @@ export const inspectManagedArtifactBytes = (
|
||||
bytes,
|
||||
metadataOffset,
|
||||
cli.metadata.size,
|
||||
maxRepresentationBytes,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ManagedReaderFailure)) throw cause;
|
||||
|
||||
@@ -4,17 +4,15 @@ import { jsonStringByteLength } from "../domain/jsonStringByteLength.js";
|
||||
|
||||
import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js";
|
||||
import { ManagedReaderFailure } from "./ManagedReaderFailure.js";
|
||||
|
||||
/** Per-inspection capacity for eagerly retained metadata and its JSON projection. */
|
||||
const MAX_METADATA_REPRESENTATION_BYTES = 64 * 1024 * 1024;
|
||||
import { DEFAULT_MANAGED_REPRESENTATION_BYTES } from "../domain/managed/managedInspectionLimits.js";
|
||||
|
||||
/** The capacity owner's diagnostic, also used to admit a recovery envelope. */
|
||||
export class ManagedRepresentationFailure extends ManagedReaderFailure {
|
||||
constructor(offset: number | null) {
|
||||
constructor(offset: number | null, maximumBytes: number) {
|
||||
super({
|
||||
code: "resource-limit",
|
||||
scope: "metadata.representation",
|
||||
detail: `Managed metadata exceeds the ${String(MAX_METADATA_REPRESENTATION_BYTES)} byte representation budget`,
|
||||
detail: `Managed metadata exceeds max_metadata_representation_bytes=${String(maximumBytes)} bytes. Increase max_metadata_representation_bytes on the next inspection (CLI --max-metadata-representation-bytes), or select fewer type_names for member inspection. No REA restart is required; snapshot, transport and host memory limits are independent.`,
|
||||
offset,
|
||||
});
|
||||
}
|
||||
@@ -27,13 +25,21 @@ interface CachedString {
|
||||
|
||||
/** The layout owns decoding capacity and heap identity for all of its consumers. */
|
||||
export class ManagedDecodeBudget {
|
||||
#remaining = MAX_METADATA_REPRESENTATION_BYTES;
|
||||
#remaining: number;
|
||||
#transient = 0;
|
||||
readonly #rows = new Map<number, Set<number>>();
|
||||
readonly #strings = new Map<number, CachedString>();
|
||||
readonly #digests = new WeakMap<ArrayBufferLike, Map<string, string>>();
|
||||
readonly #prepaid = new WeakSet<object>();
|
||||
|
||||
constructor(readonly maximumBytes = DEFAULT_MANAGED_REPRESENTATION_BYTES) {
|
||||
if (!Number.isSafeInteger(maximumBytes) || maximumBytes < 1)
|
||||
throw new RangeError(
|
||||
"max_metadata_representation_bytes must be a positive safe integer",
|
||||
);
|
||||
this.#remaining = maximumBytes;
|
||||
}
|
||||
|
||||
reserve(bytes: number, offset: number | null): void {
|
||||
this.#admit(bytes, offset);
|
||||
this.#remaining -= bytes;
|
||||
@@ -68,7 +74,7 @@ export class ManagedDecodeBudget {
|
||||
bytes < 0 ||
|
||||
bytes > this.#remaining - this.#transient
|
||||
)
|
||||
throw new ManagedRepresentationFailure(offset);
|
||||
throw new ManagedRepresentationFailure(offset, this.maximumBytes);
|
||||
}
|
||||
|
||||
/** UTF-8 cannot decode to more UTF-16 units than bytes; JSON uses at most six bytes per unit. */
|
||||
|
||||
@@ -111,7 +111,11 @@ const unavailable = (
|
||||
}),
|
||||
);
|
||||
|
||||
const readMemberLayout = (bytes: Buffer, pe: ManagedPeLayout) => {
|
||||
const readMemberLayout = (
|
||||
bytes: Buffer,
|
||||
pe: ManagedPeLayout,
|
||||
maxRepresentationBytes?: number,
|
||||
) => {
|
||||
const cli = pe.cli;
|
||||
if (cli === null)
|
||||
throw new TypeError("Managed inventory requires CLI metadata");
|
||||
@@ -124,6 +128,7 @@ const readMemberLayout = (bytes: Buffer, pe: ManagedPeLayout) => {
|
||||
bytes,
|
||||
rootOffset,
|
||||
cli.metadata.size,
|
||||
maxRepresentationBytes,
|
||||
);
|
||||
const memberLists = readTypeMemberListFacts(bytes, layout);
|
||||
return { layout, memberLists };
|
||||
@@ -172,12 +177,14 @@ export const inspectManagedMembersBytes = (
|
||||
methodOffset = 0,
|
||||
methodLimit = Number.MAX_SAFE_INTEGER,
|
||||
userStrings: readUserStrings = true,
|
||||
maxRepresentationBytes,
|
||||
}: {
|
||||
readonly typeNames?: readonly string[];
|
||||
/** Zero-based offset into MethodDef rows. */
|
||||
readonly methodOffset?: number;
|
||||
readonly methodLimit?: number;
|
||||
readonly userStrings?: boolean;
|
||||
readonly maxRepresentationBytes?: number;
|
||||
} = {},
|
||||
): ManagedMemberInspection => {
|
||||
if (
|
||||
@@ -195,7 +202,11 @@ export const inspectManagedMembersBytes = (
|
||||
const issues: ManagedParseIssue[] = [];
|
||||
let recover: ReturnType<typeof prepareManagedMemberRecovery> | undefined;
|
||||
try {
|
||||
const { layout, memberLists } = readMemberLayout(bytes, pe);
|
||||
const { layout, memberLists } = readMemberLayout(
|
||||
bytes,
|
||||
pe,
|
||||
maxRepresentationBytes,
|
||||
);
|
||||
const methodTotal = layout.table(6)?.rowCount ?? 0;
|
||||
const methodRows = {
|
||||
offset: Math.min(methodOffset, methodTotal),
|
||||
|
||||
@@ -34,6 +34,7 @@ export const prepareManagedMemberRecovery = (
|
||||
): ((offset: number | null) => ManagedMemberInspection) => {
|
||||
const capacityIssue = new ManagedRepresentationFailure(
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
budget.maximumBytes,
|
||||
).issue;
|
||||
const report: ManagedMemberInspection = {
|
||||
...inventory,
|
||||
|
||||
@@ -449,6 +449,7 @@ export const readManagedMetadataLayout = (
|
||||
artifact: Buffer,
|
||||
rootOffset: number,
|
||||
size: number,
|
||||
maxRepresentationBytes?: number,
|
||||
): ManagedMetadataLayout => {
|
||||
if (size < 20 || rootOffset < 0 || rootOffset > artifact.length - size)
|
||||
throw managedFailure(
|
||||
@@ -458,7 +459,7 @@ export const readManagedMetadataLayout = (
|
||||
rootOffset,
|
||||
);
|
||||
const bytes = artifact.subarray(rootOffset, rootOffset + size);
|
||||
const budget = new ManagedDecodeBudget();
|
||||
const budget = new ManagedDecodeBudget(maxRepresentationBytes);
|
||||
const { streams, version, minimalDeltaOffset } = readMetadataStreams(
|
||||
bytes,
|
||||
rootOffset,
|
||||
|
||||
@@ -105,6 +105,7 @@ const readBoundaryInventory = (
|
||||
pe: ManagedPeLayout,
|
||||
cli: NonNullable<ManagedPeLayout["cli"]>,
|
||||
issues: ManagedParseIssue[],
|
||||
maxRepresentationBytes?: number,
|
||||
): {
|
||||
readonly layout: ManagedMetadataLayout;
|
||||
readonly inventory: Inventory;
|
||||
@@ -118,6 +119,7 @@ const readBoundaryInventory = (
|
||||
bytes,
|
||||
metadataOffset,
|
||||
cli.metadata.size,
|
||||
maxRepresentationBytes,
|
||||
);
|
||||
const memberLists = readTypeMemberListFacts(bytes, layout);
|
||||
const inventory = readManagedMetadataInventory(
|
||||
@@ -133,6 +135,7 @@ const readBoundaryInventory = (
|
||||
export const inspectManagedNativeBoundariesBytes = (
|
||||
bytes: Buffer,
|
||||
target: BinaryTarget,
|
||||
maxRepresentationBytes?: number,
|
||||
): ManagedNativeBoundaryInspection => {
|
||||
// An unmappable PE layout is invalid input for every managed operation.
|
||||
const pe = readManagedPeLayout(bytes);
|
||||
@@ -147,7 +150,13 @@ export const inspectManagedNativeBoundariesBytes = (
|
||||
let layout: ManagedMetadataLayout;
|
||||
let inventory: Inventory;
|
||||
try {
|
||||
({ layout, inventory } = readBoundaryInventory(bytes, pe, pe.cli, issues));
|
||||
({ layout, inventory } = readBoundaryInventory(
|
||||
bytes,
|
||||
pe,
|
||||
pe.cli,
|
||||
issues,
|
||||
maxRepresentationBytes,
|
||||
));
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ManagedReaderFailure)) throw cause;
|
||||
return emptyInspection(target, bytes, "malformed", {
|
||||
|
||||
@@ -22,14 +22,11 @@ import {
|
||||
AnalysisResourceConstraintError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js";
|
||||
import { DEFAULT_MANAGED_SNAPSHOT_BYTES } from "../domain/managed/managedInspectionLimits.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
import { RegularFileChangedError } from "../filesystem/RegularFile.js";
|
||||
import { readBoundedFileBytes } from "../process/BoundedFileBytes.js";
|
||||
|
||||
// Managed static inspection retains the complete PE snapshot in memory.
|
||||
export const MAX_MANAGED_SNAPSHOT_BYTES = 128 * 1024 * 1024;
|
||||
|
||||
interface ManagedSourceObservation {
|
||||
readonly byteLength: number;
|
||||
readonly sha256: string;
|
||||
@@ -42,6 +39,7 @@ interface ManagedSnapshot extends ManagedSourceObservation {
|
||||
|
||||
interface ManagedSnapshotReadOptions extends RegularFileReadOptions {
|
||||
readonly expectedSha256?: string;
|
||||
readonly maxBytes?: number;
|
||||
}
|
||||
|
||||
/** Admit one observed source digest before any parser or cache consumes its bytes. */
|
||||
@@ -69,13 +67,17 @@ export const verifyManagedSourceIdentity = (
|
||||
|
||||
const managedSnapshotCapacityError = (
|
||||
operation: string,
|
||||
maxBytes: number,
|
||||
): AnalysisResourceConstraintError =>
|
||||
new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"Managed static inspection requires an eagerly retained PE snapshot within its input byte budget.",
|
||||
{ max_snapshot_bytes: MAX_MANAGED_SNAPSHOT_BYTES },
|
||||
fixedAllowanceOptions("The managed PE snapshot allowance", "parser"),
|
||||
{ max_snapshot_bytes: maxBytes },
|
||||
{
|
||||
retryable: false,
|
||||
remediationAction: `Increase max_snapshot_bytes above ${String(maxBytes)} bytes on the next inspection (CLI --max-snapshot-bytes) to admit the complete selected file. No REA restart is required. Metadata representation, transport and host memory limits remain independent; repeating the unchanged request cannot raise this budget.`,
|
||||
},
|
||||
);
|
||||
|
||||
export const readManagedSnapshot = async (
|
||||
@@ -83,19 +85,17 @@ export const readManagedSnapshot = async (
|
||||
operation: string,
|
||||
options: ManagedSnapshotReadOptions = {},
|
||||
): Promise<ManagedSnapshot> => {
|
||||
const maxBytes = snapshotBudget(options);
|
||||
return readManagedFile(
|
||||
path,
|
||||
operation,
|
||||
async (handle, metadata) => {
|
||||
const signal = options.signal;
|
||||
if (metadata.size > MAX_MANAGED_SNAPSHOT_BYTES)
|
||||
throw managedSnapshotCapacityError(operation);
|
||||
const bytes = await readBoundedFileBytes(
|
||||
handle,
|
||||
MAX_MANAGED_SNAPSHOT_BYTES,
|
||||
{ signal },
|
||||
);
|
||||
if (bytes === undefined) throw managedSnapshotCapacityError(operation);
|
||||
if (metadata.size > maxBytes)
|
||||
throw managedSnapshotCapacityError(operation, maxBytes);
|
||||
const bytes = await readBoundedFileBytes(handle, maxBytes, { signal });
|
||||
if (bytes === undefined)
|
||||
throw managedSnapshotCapacityError(operation, maxBytes);
|
||||
signal?.throwIfAborted();
|
||||
return {
|
||||
byteLength: bytes.length,
|
||||
@@ -111,14 +111,15 @@ export const hashManagedSource = async (
|
||||
path: string,
|
||||
operation: string,
|
||||
options: ManagedSnapshotReadOptions = {},
|
||||
): Promise<ManagedSourceObservation> =>
|
||||
readManagedFile(
|
||||
): Promise<ManagedSourceObservation> => {
|
||||
const maxBytes = snapshotBudget(options);
|
||||
return readManagedFile(
|
||||
path,
|
||||
operation,
|
||||
async (handle, metadata) => {
|
||||
const signal = options.signal;
|
||||
if (metadata.size > MAX_MANAGED_SNAPSHOT_BYTES)
|
||||
throw managedSnapshotCapacityError(operation);
|
||||
if (metadata.size > maxBytes)
|
||||
throw managedSnapshotCapacityError(operation, maxBytes);
|
||||
const digest = createHash("sha256");
|
||||
let byteLength = 0;
|
||||
const chunk = Buffer.allocUnsafe(64 * 1024);
|
||||
@@ -127,13 +128,13 @@ export const hashManagedSource = async (
|
||||
const { bytesRead } = await handle.read(
|
||||
chunk,
|
||||
0,
|
||||
Math.min(chunk.length, MAX_MANAGED_SNAPSHOT_BYTES - byteLength + 1),
|
||||
Math.min(chunk.length, maxBytes - byteLength + 1),
|
||||
null,
|
||||
);
|
||||
signal?.throwIfAborted();
|
||||
if (bytesRead === 0) break;
|
||||
if (bytesRead > MAX_MANAGED_SNAPSHOT_BYTES - byteLength)
|
||||
throw managedSnapshotCapacityError(operation);
|
||||
if (bytesRead > maxBytes - byteLength)
|
||||
throw managedSnapshotCapacityError(operation, maxBytes);
|
||||
byteLength += bytesRead;
|
||||
digest.update(chunk.subarray(0, bytesRead));
|
||||
}
|
||||
@@ -141,6 +142,14 @@ export const hashManagedSource = async (
|
||||
},
|
||||
options,
|
||||
);
|
||||
};
|
||||
|
||||
const snapshotBudget = (options: ManagedSnapshotReadOptions): number => {
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MANAGED_SNAPSHOT_BYTES;
|
||||
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1)
|
||||
throw new RangeError("max_snapshot_bytes must be a positive safe integer");
|
||||
return maxBytes;
|
||||
};
|
||||
|
||||
const readManagedFile = <Value extends ManagedSourceObservation>(
|
||||
path: string,
|
||||
|
||||
@@ -39,6 +39,8 @@ import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js";
|
||||
import { inspectManagedNativeBoundariesBytes } from "./ManagedNativeBoundaryInspector.js";
|
||||
import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js";
|
||||
import { managedInspectionInputFailure } from "./ManagedReaderFailure.js";
|
||||
import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js";
|
||||
import type { z } from "zod";
|
||||
import {
|
||||
hashManagedSource,
|
||||
managedCleanupError,
|
||||
@@ -96,6 +98,26 @@ class ManagedStaticClient implements AnalysisClient {
|
||||
),
|
||||
);
|
||||
try {
|
||||
const input =
|
||||
operation === "inspect_managed_members"
|
||||
? managedMembersInputSchema.safeParse(parameters)
|
||||
: managedTargetInputSchema.safeParse(parameters);
|
||||
if (!input.success)
|
||||
return err(
|
||||
analysisInputErrorFromIssues(
|
||||
operation,
|
||||
input.error.issues,
|
||||
parameters,
|
||||
),
|
||||
);
|
||||
const snapshotOptions = {
|
||||
expectedSha256: this.target.sha256,
|
||||
signal: options?.signal,
|
||||
resources: this.#resources,
|
||||
...(input.data.max_snapshot_bytes === undefined
|
||||
? {}
|
||||
: { maxBytes: input.data.max_snapshot_bytes }),
|
||||
};
|
||||
if (options?.signal?.aborted === true)
|
||||
return err(
|
||||
new AnalysisCancelledError(operation, {
|
||||
@@ -105,16 +127,16 @@ class ManagedStaticClient implements AnalysisClient {
|
||||
const snapshot = this.#snapshotBytes;
|
||||
const observed =
|
||||
snapshot === undefined
|
||||
? await readManagedSnapshot(this.target.path, operation, {
|
||||
expectedSha256: this.target.sha256,
|
||||
signal: options?.signal,
|
||||
resources: this.#resources,
|
||||
})
|
||||
: await hashManagedSource(this.target.path, operation, {
|
||||
expectedSha256: this.target.sha256,
|
||||
signal: options?.signal,
|
||||
resources: this.#resources,
|
||||
});
|
||||
? await readManagedSnapshot(
|
||||
this.target.path,
|
||||
operation,
|
||||
snapshotOptions,
|
||||
)
|
||||
: await hashManagedSource(
|
||||
this.target.path,
|
||||
operation,
|
||||
snapshotOptions,
|
||||
);
|
||||
const admitted = verifyManagedSourceIdentity(
|
||||
this.target,
|
||||
observed,
|
||||
@@ -127,7 +149,7 @@ class ManagedStaticClient implements AnalysisClient {
|
||||
this.#snapshotBytes = bytes;
|
||||
const result = inspectManagedOperation(
|
||||
operation,
|
||||
parameters,
|
||||
input.data,
|
||||
bytes,
|
||||
this.target,
|
||||
);
|
||||
@@ -202,7 +224,7 @@ const isManagedOperation = (
|
||||
|
||||
const inspectManagedOperation = (
|
||||
operation: ManagedToolName,
|
||||
parameters: Readonly<Record<string, JsonValue>>,
|
||||
parameters: Omit<z.output<typeof managedMembersInputSchema>, "detail">,
|
||||
bytes: Buffer,
|
||||
target: BinaryTarget,
|
||||
):
|
||||
@@ -210,19 +232,29 @@ const inspectManagedOperation = (
|
||||
| ManagedMemberInspection
|
||||
| ManagedNativeBoundaryInspection => {
|
||||
if (operation === "inspect_managed_members") {
|
||||
const { type_names: typeNames } =
|
||||
managedMembersInputSchema.parse(parameters);
|
||||
return inspectManagedMembersBytes(
|
||||
const { type_names: typeNames } = parameters;
|
||||
return inspectManagedMembersBytes(bytes, target, {
|
||||
...(typeNames === undefined ? {} : { typeNames }),
|
||||
...(parameters.max_metadata_representation_bytes === undefined
|
||||
? {}
|
||||
: {
|
||||
maxRepresentationBytes:
|
||||
parameters.max_metadata_representation_bytes,
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (operation === "inspect_managed_artifact") {
|
||||
return inspectManagedArtifactBytes(
|
||||
bytes,
|
||||
target,
|
||||
typeNames === undefined ? {} : { typeNames },
|
||||
parameters.max_metadata_representation_bytes,
|
||||
);
|
||||
}
|
||||
managedTargetInputSchema.parse(parameters);
|
||||
if (operation === "inspect_managed_artifact") {
|
||||
return inspectManagedArtifactBytes(bytes, target);
|
||||
}
|
||||
return inspectManagedNativeBoundariesBytes(bytes, target);
|
||||
return inspectManagedNativeBoundariesBytes(
|
||||
bytes,
|
||||
target,
|
||||
parameters.max_metadata_representation_bytes,
|
||||
);
|
||||
};
|
||||
|
||||
const managedLocations = (
|
||||
|
||||
Reference in new issue
Block a user