test(limits): check documented limit defaults against code instead of per-file pins (#12720)

* test(limits): check documented limit defaults against code instead of per-file pins

The numeric-limit script now reads every default, floor, and maximum in
docs/design/rea-limit-inventory.md and compares it with ANALYSIS_SETTINGS
or the owner constant the parser enforces. A mismatch names the limit and
both values. Pins that only restated a covered value are removed.

Pwntools input bytes and the Kuna timeouts now use the shared exports, so
the registry value is the one the parser applies.

* fix(limits): reject a fractional documented limit cell

The cell parser treated a dot as the end of the integer, so 5000.5
compared equal to 5000. A documented number is now one complete integer,
with prose only after whitespace.

Co-authored-by: morluto <morluto@users.noreply.github.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-19 23:05:07 -05:00
1 parent d5818ad0da
commit dc2c31e76d
21 files changed
+730 -62

No files matched your search

+54
View File
@@ -63,3 +63,57 @@ export function requiredRuntimeLimitFailure(
readonly inRegistry: boolean;
},
): RequiredRuntimeLimitFailure | undefined;
/** One default, floor, and maximum from the numeric inventory tables. */
export type DocumentedNumericLimit = {
readonly name: string;
readonly section: string;
readonly defaultCell: string;
readonly floorCell: string;
readonly maximumCell: string;
};
/** Default, floor, and maximum exported by the analysis-setting registry or an owner. */
export type ExportedNumericLimit = {
readonly name: string;
readonly aliases?: readonly string[];
readonly defaultValue: number | null;
readonly minimum: number;
readonly maximum: number;
};
/** Read default, floor, and maximum from the three numeric inventory tables. */
export function parseDocumentedNumericLimits(
markdown: string,
): DocumentedNumericLimit[];
/**
* Read one documented cell.
* A fraction is unreadable. Prose is allowed only after a complete integer and whitespace.
*/
export function documentedLimitCell(
cell: string,
names: Readonly<Record<string, number>>,
):
| { readonly ok: true; readonly value: number | null }
| { readonly ok: false; readonly reason: string };
/**
* Compare documented limits with exported bounds.
* Each failure names the limit and both values.
*/
export function compareDocumentedNumericLimits(
rows: readonly DocumentedNumericLimit[],
bounds: readonly ExportedNumericLimit[],
names: Readonly<Record<string, number>>,
): string[];
/**
* Bounds ANALYSIS_SETTINGS and the restart-only owners export.
* `failures` names a registry entry that has no exported bound.
*/
export function loadExportedNumericLimitBounds(): Promise<{
readonly bounds: readonly ExportedNumericLimit[];
readonly names: Readonly<Record<string, number>>;
readonly failures: readonly string[];
}>;
+446
View File
@@ -8,6 +8,8 @@ import ts from "typescript6";
// reported separately and are not verified by this Node-only scan.
// Per-call limits belong to ANALYSIS_SETTINGS.
// A name in REQUIRED_RUNTIME_LIMITS that is absent from source fails closed.
// Each default, floor, and maximum in docs/design/rea-limit-inventory.md must
// equal ANALYSIS_SETTINGS or the owner export the parser enforces.
const root = fileURLToPath(new URL("../../", import.meta.url));
const definitionsPath = join(root, "src/domain/analysisSettingDefinitions.ts");
@@ -919,6 +921,415 @@ export const parseRuntimeSettingEnvironments = (source) => {
return names;
};
const DOCUMENTED_LIMIT_SECTIONS = [
"Runtime analysis settings",
"Per-call limits in the runtime system",
"Restart-only numeric limits",
];
const tableCells = (line) =>
line
.split("|")
.slice(1, -1)
.map((cell) => cell.trim());
const separatorRow = (cells) =>
cells.length > 0 && cells.every((cell) => /^:?-+:?$/u.test(cell));
/**
* Read default, floor, and maximum from the three numeric inventory tables.
* The limit name is the first cell. Later prose tables are not limits.
*
* @param {string} markdown
*/
export const parseDocumentedNumericLimits = (markdown) => {
const lines = markdown.split(/\r?\n/u);
const rows = [];
for (const title of DOCUMENTED_LIMIT_SECTIONS) {
const heading = lines.findIndex((line) => line.trim() === `## ${title}`);
if (heading < 0)
throw new Error(
`docs/design/rea-limit-inventory.md is missing "## ${title}"`,
);
let index = heading + 1;
while (index < lines.length && !lines[index].startsWith("|")) {
if (lines[index].startsWith("#"))
throw new Error(`## ${title} has no limit table`);
index += 1;
}
if (!lines[index]?.startsWith("|"))
throw new Error(`## ${title} has no limit table`);
const header = tableCells(lines[index]);
const column = (name) => {
const found = header.indexOf(name);
if (found < 0)
throw new Error(`## ${title} table is missing the ${name} column`);
return found;
};
const defaultColumn = column("Default");
const floorColumn = column("Floor");
const maximumColumn = column("Maximum");
if (header[0] !== "Setting" && header[0] !== "Environment")
throw new Error(
`## ${title} must name each limit in a Setting or Environment column`,
);
for (
index += 1;
index < lines.length && lines[index].startsWith("|");
index += 1
) {
const cells = tableCells(lines[index]);
if (separatorRow(cells)) continue;
const name = cells[0]?.replaceAll("`", "") ?? "";
if (name.length === 0)
throw new Error(`## ${title} has a row without a limit name`);
rows.push({
name,
section: title,
defaultCell: cells[defaultColumn] ?? "",
floorCell: cells[floorColumn] ?? "",
maximumCell: cells[maximumColumn] ?? "",
});
}
if (!rows.some((row) => row.section === title))
throw new Error(`## ${title} table has no limit rows`);
}
return rows;
};
/**
* A documented cell is one complete integer, optional prose after whitespace,
* `unset` (null), or one exported constant. A fraction is not that integer.
*
* @param {string} cell
* @param {Readonly<Record<string, number>>} names
* @returns {{ ok: true, value: number | null } | { ok: false, reason: string }}
*/
export const documentedLimitCell = (cell, names) => {
const text = cell.trim();
if (/^unset\b/iu.test(text)) return { ok: true, value: null };
const named = /^`([^`]+)`$/u.exec(text);
if (named) {
if (!Object.hasOwn(names, named[1]))
return { ok: false, reason: `unknown constant ${named[1]}` };
return { ok: true, value: names[named[1]] };
}
const integer = /^(\d+)(?=\s|$)/u.exec(text);
if (!integer)
return { ok: false, reason: `unreadable cell ${JSON.stringify(text)}` };
return { ok: true, value: Number(integer[1]) };
};
const showLimitValue = (value) => (value === null ? "unset" : String(value));
/**
* Compare one inventory row with the bound the code exports.
* Each failure names the limit and both values.
*
* @param {readonly {
* readonly name: string,
* readonly section: string,
* readonly defaultCell: string,
* readonly floorCell: string,
* readonly maximumCell: string,
* }[]} rows
* @param {readonly {
* readonly name: string,
* readonly aliases?: readonly string[],
* readonly defaultValue: number | null,
* readonly minimum: number,
* readonly maximum: number,
* }[]} bounds
* @param {Readonly<Record<string, number>>} names
* @returns {string[]}
*/
export const compareDocumentedNumericLimits = (rows, bounds, names) => {
const failures = [];
const byName = new Map();
for (const bound of bounds) {
for (const name of [bound.name, ...(bound.aliases ?? [])]) {
if (byName.has(name))
failures.push(`${name}: code exports more than one bound`);
byName.set(name, bound);
}
}
const seen = new Set();
const seenNames = new Set();
for (const row of rows) {
if (seenNames.has(row.name))
failures.push(`${row.name}: documented more than once`);
seenNames.add(row.name);
const bound = byName.get(row.name);
if (bound === undefined) {
failures.push(
`${row.name}: documented ${row.section} default ${JSON.stringify(row.defaultCell)}, floor ${JSON.stringify(row.floorCell)}, maximum ${JSON.stringify(row.maximumCell)}, but code exports no bound`,
);
continue;
}
seen.add(bound);
for (const [field, cell, codeValue] of [
["default", row.defaultCell, bound.defaultValue],
["floor", row.floorCell, bound.minimum],
["maximum", row.maximumCell, bound.maximum],
]) {
const documented = documentedLimitCell(cell, names);
if (!documented.ok) {
failures.push(
`${row.name} ${field}: documented ${JSON.stringify(cell)} (${documented.reason}), code ${showLimitValue(codeValue)}`,
);
continue;
}
if (documented.value !== codeValue)
failures.push(
`${row.name} ${field}: documented ${showLimitValue(documented.value)}, code ${showLimitValue(codeValue)}`,
);
}
}
for (const bound of bounds) {
if (seen.has(bound)) continue;
const label =
bound.aliases !== undefined && bound.aliases.length > 0
? `${bound.name} (${bound.aliases.join(", ")})`
: bound.name;
failures.push(
`${label}: code exports default ${showLimitValue(bound.defaultValue)}, floor ${showLimitValue(bound.minimum)}, maximum ${showLimitValue(bound.maximum)}, but docs/design/rea-limit-inventory.md has no row`,
);
}
return failures;
};
const exportedNumber = (module, name) => {
const value = module[name];
if (typeof value !== "number")
throw new Error(`${name} is not an exported number`);
return value;
};
const bound = (name, defaultValue, minimum, maximum, aliases = []) => ({
name,
...(aliases.length === 0 ? {} : { aliases }),
defaultValue,
minimum,
maximum,
});
/** Analysis-setting registry bounds, including each REA_ environment alias. */
const analysisSettingBounds = (settings) => {
const bounds = [];
for (const [id, definition] of Object.entries(settings)) {
if (
(typeof definition.defaultValue !== "number" &&
definition.defaultValue !== null) ||
typeof definition.minimum !== "number" ||
typeof definition.maximum !== "number"
)
throw new Error(
`${id} does not export a numeric default, floor, and maximum`,
);
bounds.push(
bound(
id,
definition.defaultValue,
definition.minimum,
definition.maximum,
typeof definition.environment === "string"
? [definition.environment]
: [],
),
);
}
return bounds;
};
/** One MCP startup phase, using the value resolveMcpStartupBudgets ships. */
const phaseBound = (budget) => {
if (budget.source !== "default")
throw new Error(`${budget.setting} unset resolution was not the default`);
return bound(
budget.setting,
budget.defaultMilliseconds,
budget.floorMilliseconds,
budget.maximumMilliseconds,
);
};
/** Restart-only bounds from the constants each owner parser enforces. */
const restartOnlyBounds = (owners) => {
const phases = owners.mcpBudgets.resolveMcpStartupBudgets({});
const reconnectSeconds = exportedNumber(
owners.hopperSession,
"HOPPER_SHUTDOWN_TIMEOUT_MS",
);
if (reconnectSeconds % 1000 !== 0)
throw new Error(
"HOPPER_SHUTDOWN_TIMEOUT_MS is not a whole number of seconds",
);
const seconds = reconnectSeconds / 1000;
const restart = [
bound(
"REA_DEBUGGER_MAX_PENDING_COMMANDS",
exportedNumber(owners.limits, "DEFAULT_DEBUGGER_MAX_PENDING_COMMANDS"),
exportedNumber(owners.limits, "MINIMUM_DEBUGGER_MAX_PENDING_COMMANDS"),
exportedNumber(owners.limits, "MAXIMUM_DEBUGGER_MAX_PENDING_COMMANDS"),
),
bound(
"REA_DEBUGGER_MAX_SESSIONS",
exportedNumber(owners.limits, "DEFAULT_DEBUGGER_MAX_SESSIONS"),
exportedNumber(owners.limits, "MINIMUM_DEBUGGER_MAX_SESSIONS"),
exportedNumber(owners.limits, "MAXIMUM_DEBUGGER_MAX_SESSIONS"),
),
bound(
"REA_HOPPER_DEMO_DIALOG_TIMEOUT_MS",
exportedNumber(
owners.hopperStartup,
"DEFAULT_HOPPER_DEMO_DIALOG_TIMEOUT_MS",
),
exportedNumber(
owners.hopperStartup,
"MINIMUM_HOPPER_DEMO_DIALOG_TIMEOUT_MS",
),
exportedNumber(owners.limits, "MAX_NODE_TIMER_DELAY_MS"),
),
bound(
"REA_JADX_ACTIVE_PROCESSOR_COUNT",
owners.jadx.JADX_JVM_DEFAULT,
exportedNumber(owners.jadx, "JADX_JVM_NUMERIC_MINIMUM"),
exportedNumber(owners.jadx, "JADX_ACTIVE_PROCESSOR_COUNT_MAXIMUM"),
),
bound(
"REA_JADX_HEAP_MIB",
owners.jadx.JADX_JVM_DEFAULT,
exportedNumber(owners.jadx, "JADX_JVM_NUMERIC_MINIMUM"),
exportedNumber(owners.jadx, "JADX_HEAP_MIB_MAXIMUM"),
),
bound(
"REA_KUNA_CACHE_MAX_BYTES",
exportedNumber(owners.kuna, "DEFAULT_KUNA_CACHE_MAX_BYTES"),
exportedNumber(owners.kuna, "MINIMUM_KUNA_CACHE_MAX_BYTES"),
exportedNumber(owners.kuna, "MAXIMUM_KUNA_CACHE_MAX_BYTES"),
),
bound(
"REA_KUNA_JOBS",
exportedNumber(owners.kuna, "DEFAULT_KUNA_JOBS"),
exportedNumber(owners.kuna, "MINIMUM_KUNA_JOBS"),
exportedNumber(owners.kuna, "MAXIMUM_KUNA_JOBS"),
),
phaseBound(phases.doctor),
phaseBound(phases.firstCatalog),
phaseBound(phases.initialize),
bound(
"REA_MCP_MAX_INPUT_ELEMENTS",
exportedNumber(owners.mcpInput, "DEFAULT_MCP_MAX_INPUT_ELEMENTS"),
exportedNumber(owners.mcpInput, "MINIMUM_MCP_MAX_INPUT_ELEMENTS"),
exportedNumber(owners.mcpInput, "MAXIMUM_MCP_MAX_INPUT_ELEMENTS"),
),
bound(
"REA_MCP_MAX_RESPONSE_BYTES",
exportedNumber(owners.sdk, "STDIO_DEFAULT_MAX_BUFFER_SIZE"),
exportedNumber(owners.mcpResponse, "MINIMUM_RESPONSE_BYTES"),
exportedNumber(owners.mcpResponse, "MAXIMUM_RESPONSE_BYTES"),
),
phaseBound(phases.toolsList),
bound("REA_RECONNECT_TIMEOUT_SECONDS", seconds, seconds, seconds),
bound(
"REA_SWIFTC_COMPILE_TIMEOUT_MS",
exportedNumber(
owners.startupTimeout,
"DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS",
),
exportedNumber(owners.limits, "TIMER_DURATION_MINIMUM_MS"),
exportedNumber(owners.limits, "MAX_NODE_TIMER_DELAY_MS"),
),
];
if (restart.some((entry) => entry.defaultValue === undefined))
throw new Error("A restart-only bound is missing its default");
return restart;
};
/** Registry entries must be the restart-only bounds, and the reverse. */
const restartRegistryFailures = (registry, restart) => {
const names = new Set(registry.map((entry) => entry.name));
const failures = [];
for (const name of names) {
if (!restart.some((entry) => entry.name === name))
failures.push(
`${name}: restart-only registry entry has no exported default, floor, and maximum`,
);
}
for (const entry of restart) {
if (!names.has(entry.name))
failures.push(
`${entry.name}: exported restart-only bound is not in NUMERIC_LIMIT_REGISTRY`,
);
}
return failures;
};
/**
* Bounds the running code exports.
* Analysis settings come from ANALYSIS_SETTINGS, which
* update_analysis_settings and inspect_analysis_settings share.
* Restart-only bounds are the constants each owner parser enforces.
* A registry name without those exports fails closed.
*/
export const loadExportedNumericLimitBounds = async () => {
const { importVerifierSource } =
await import("../lib/verifier-source-loader.mjs");
const source = (specifier) =>
importVerifierSource(specifier, import.meta.url);
const [
settings,
limits,
registry,
hopperStartup,
hopperSession,
jadx,
kuna,
mcpBudgets,
mcpInput,
mcpResponse,
startupTimeout,
sdk,
] = await Promise.all([
source("../../src/domain/analysisSettingDefinitions.ts"),
source("../../src/domain/analysisResourceLimits.ts"),
source("../../src/domain/numericLimitRegistry.ts"),
source("../../src/hopper/HopperStartupFailure.ts"),
source("../../src/hopper/HopperSessionValues.ts"),
source("../../src/android/JadxConfiguration.ts"),
source("../../src/kuna/KunaConfiguration.ts"),
source("../../src/config/mcpStartupBudgets.ts"),
source("../../src/config/mcpInputBudget.ts"),
source("../../src/config/mcpResponseBudget.ts"),
source("../../src/config/providerStartupTimeout.ts"),
import("@modelcontextprotocol/server"),
]);
const owners = {
limits,
hopperStartup,
hopperSession,
jadx,
kuna,
mcpBudgets,
mcpInput,
mcpResponse,
startupTimeout,
sdk,
};
const restart = restartOnlyBounds(owners);
return {
bounds: [...analysisSettingBounds(settings.ANALYSIS_SETTINGS), ...restart],
names: {
"Number.MAX_SAFE_INTEGER": Number.MAX_SAFE_INTEGER,
MAX_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: exportedNumber(
limits,
"MAX_GHIDRA_DECOMPILE_TIMEOUT_SECONDS",
),
},
failures: restartRegistryFailures(registry.NUMERIC_LIMIT_REGISTRY, restart),
};
};
const direct =
process.argv[1] !== undefined &&
import.meta.url === pathToFileURL(resolve(process.argv[1])).href;
@@ -1029,6 +1440,41 @@ if (direct) {
failures.push(`${relativePath} must report update_mode parameter`);
}
// The permission fixture copies this file into a tree with no package.json.
// A REA checkout always compares the inventory. A missing inventory here fails.
let checkout = false;
try {
checkout =
JSON.parse(await readFile(join(root, "package.json"), "utf8")).name ===
"rea-agents";
} catch (cause) {
if (cause?.code !== "ENOENT") throw cause;
}
if (checkout) {
try {
const inventory = await readFile(
join(root, "docs/design/rea-limit-inventory.md"),
"utf8",
);
const exported = await loadExportedNumericLimitBounds();
failures.push(
...exported.failures,
...compareDocumentedNumericLimits(
parseDocumentedNumericLimits(inventory),
exported.bounds,
exported.names,
),
);
} catch (cause) {
failures.push(
`documented limit values could not be checked: ${cause instanceof Error ? cause.message : String(cause)}`,
);
}
} else
console.log(
"Documented limit values skipped: this copy is not the REA checkout.",
);
if (failures.length > 0) {
console.error(`Numeric REA_ limit check failed:\n${failures.join("\n")}`);
process.exit(1);