mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 03:33:25 +00:00
test(cli): report CLI fields that admit values their contract refuses (#10575)
* test(cli): report CLI fields that admit values their contract refuses For every CLI command that reaches an MCP contract, compare each name-matched CLI field with its contract field on empty, NUL, caller-relative host path and numeric argv spellings. The 16 mismatches on main are recorded in a baseline with a counted ceiling that can only go down: a new mismatch fails, and so does a listed one that no longer occurs. * test: cover constrained CLI contract fields * fix(cli): admit relative web trace paths * test(cli): probe numeric contract fields by type, including bounds Numeric probes no longer require the contract to admit 0 or 1, so ranged fields such as inspect-byte-entropy --block-size (256..65536) are compared, with probes just past each declared bound. * fix(cli): describe caller-relative web trace paths and expect native Windows spellings The relative-path CLI arguments lost their contract descriptions; they now describe the caller-relative resolution. absoluteCallerPath renders native separators on Windows, so the web trace acceptance tests use native manifest paths for CLI and MCP parity. * test(cli): keep forward-slash Windows manifest coverage through MCP The CLI renders caller paths with native separators through absoluteCallerPath; MCP keeps the caller's spelling. The web trace parity tests now also call MCP with the forward-slash manifest spelling and expect the same trace with that path echoed, restoring the #1078 coverage.
This commit is contained in:
1 parent
d0fee4e58e
commit
48220827c4
10 files changed
+450
-45
No files matched your search
@@ -8,6 +8,8 @@ import type { Logger } from "pino";
|
||||
import type { CliInstance } from "./types.js";
|
||||
import { cliDecimalField } from "./options.js";
|
||||
import { withCommandCancellation } from "./commandCancellation.js";
|
||||
import { localPathStringSchema } from "../domain/localPath.js";
|
||||
import { absoluteCallerPath } from "../filesystem/callerPath.js";
|
||||
import { finishAnalysisCommand } from "./finishAnalysisCommand.js";
|
||||
|
||||
const importMapInputSchema =
|
||||
@@ -24,7 +26,9 @@ export const registerWebModuleCommands = (
|
||||
description:
|
||||
"Trace one captured script's native imports to URLs and captured source candidates",
|
||||
args: z.object({
|
||||
manifestPath: webModuleTraceInputSchema.shape.manifest_path,
|
||||
manifestPath: localPathStringSchema.describe(
|
||||
"export_web_scripts manifest path; a relative path resolves against the current directory",
|
||||
),
|
||||
scriptIndex: cliDecimalField(
|
||||
webModuleTraceInputSchema.shape.script_index.describe(
|
||||
"Zero-based manifest script index",
|
||||
@@ -49,7 +53,7 @@ export const registerWebModuleCommands = (
|
||||
() =>
|
||||
service.trace(
|
||||
{
|
||||
manifest_path: args.manifestPath,
|
||||
manifest_path: absoluteCallerPath(args.manifestPath),
|
||||
script_index: args.scriptIndex,
|
||||
...(options.importerUrl === undefined
|
||||
? {}
|
||||
|
||||
@@ -18,6 +18,8 @@ import type { Logger } from "pino";
|
||||
import type { CliInstance } from "./types.js";
|
||||
import { cliDecimalField } from "./options.js";
|
||||
import { withCommandCancellation } from "./commandCancellation.js";
|
||||
import { localPathStringSchema } from "../domain/localPath.js";
|
||||
import { absoluteCallerPath } from "../filesystem/callerPath.js";
|
||||
|
||||
const sourceMapSelection = webSourceLocationInputSchema.shape.source_map;
|
||||
const generatedPosition = webSourceLocationInputSchema.shape.generated_position;
|
||||
@@ -33,13 +35,17 @@ export const registerWebSourceLocationCommands = (
|
||||
description:
|
||||
"Trace one retained script position through an explicitly selected local source map",
|
||||
args: z.object({
|
||||
manifestPath: webSourceLocationInputSchema.shape.manifest_path,
|
||||
manifestPath: localPathStringSchema.describe(
|
||||
"export_web_scripts manifest path; a relative path resolves against the current directory",
|
||||
),
|
||||
scriptIndex: cliDecimalField(
|
||||
webSourceLocationInputSchema.shape.script_index.describe(
|
||||
"Zero-based manifest script index",
|
||||
),
|
||||
),
|
||||
sourceMapPath: sourceMapSelection.shape.path,
|
||||
sourceMapPath: localPathStringSchema.describe(
|
||||
"Local source-map path, not fetched; a relative path resolves against the current directory",
|
||||
),
|
||||
sourceMapUrl: sourceMapSelection.shape.url,
|
||||
line: cliDecimalField(
|
||||
generatedPosition.shape.line.describe("One-based generated line"),
|
||||
@@ -76,10 +82,10 @@ export const registerWebSourceLocationCommands = (
|
||||
run: () =>
|
||||
service.trace(
|
||||
{
|
||||
manifest_path: args.manifestPath,
|
||||
manifest_path: absoluteCallerPath(args.manifestPath),
|
||||
script_index: args.scriptIndex,
|
||||
source_map: {
|
||||
path: args.sourceMapPath,
|
||||
path: absoluteCallerPath(args.sourceMapPath),
|
||||
url: args.sourceMapUrl,
|
||||
},
|
||||
generated_position: { line: args.line, column: args.column },
|
||||
|
||||
@@ -24,7 +24,7 @@ cliTest(
|
||||
join(root, "files", "modules", "main.js"),
|
||||
artifacts.source,
|
||||
);
|
||||
let manifestPath = join(root, "manifest.json").replaceAll("\\", "/");
|
||||
let manifestPath = join(root, "manifest.json");
|
||||
await writeFile(manifestPath, JSON.stringify(manifest));
|
||||
if (process.platform !== "win32") {
|
||||
await mkdir(join(root, "lexical"));
|
||||
@@ -37,7 +37,15 @@ cliTest(
|
||||
REA_BROWSER_EXECUTABLE: "relative-missing-browser",
|
||||
};
|
||||
const cliResponse = await cli.run({
|
||||
arguments: ["trace-web-module-imports", manifestPath, "0", "--json"],
|
||||
arguments: [
|
||||
"trace-web-module-imports",
|
||||
process.platform === "win32"
|
||||
? "manifest.json"
|
||||
: "lexical/link/../manifest.json",
|
||||
"0",
|
||||
"--json",
|
||||
],
|
||||
cwd: root,
|
||||
environment: {
|
||||
...environment,
|
||||
// This single CLI invocation cannot reuse a newly written cache.
|
||||
@@ -86,6 +94,23 @@ cliTest(
|
||||
expect(response.isError).not.toBe(true);
|
||||
const expected = cliResponse.json;
|
||||
expect(response.structuredContent).toEqual(expected);
|
||||
// The CLI renders native separators; MCP keeps the caller's spelling and
|
||||
// still selects the same manifest root from a forward-slash Windows path.
|
||||
const forwardSlash = manifestPath.replaceAll("\\", "/");
|
||||
const forward = await client.callTool({
|
||||
name: "trace_web_module_imports",
|
||||
arguments: { manifest_path: forwardSlash, script_index: 0 },
|
||||
});
|
||||
expect(forward.isError, JSON.stringify(forward)).not.toBe(true);
|
||||
const { manifest: _manifest, ...cliRest } = (
|
||||
expected as { normalized_result: Record<string, unknown> }
|
||||
).normalized_result;
|
||||
expect(forward.structuredContent).toMatchObject({
|
||||
normalized_result: {
|
||||
...cliRest,
|
||||
manifest: { path: forwardSlash, reported_output_directory: root },
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -172,6 +172,29 @@ const expectColumnOutOfRange = (
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* The CLI renders native separators; MCP keeps the caller's spelling and still
|
||||
* selects the same manifest root from a forward-slash Windows path (#1078).
|
||||
*/
|
||||
const expectForwardSlashManifestTrace = async (
|
||||
client: Client,
|
||||
input: { readonly manifest_path: string } & Record<string, unknown>,
|
||||
cliJson: unknown,
|
||||
) => {
|
||||
const forwardSlash = input.manifest_path.replaceAll("\\", "/");
|
||||
const forward = await client.callTool({
|
||||
name: "trace_web_source_location",
|
||||
arguments: { ...input, manifest_path: forwardSlash },
|
||||
});
|
||||
expect(forward.isError, JSON.stringify(forward)).not.toBe(true);
|
||||
const { manifest: _manifest, ...cliRest } = (
|
||||
cliJson as { normalized_result: Record<string, unknown> }
|
||||
).normalized_result;
|
||||
expect(forward.structuredContent).toMatchObject({
|
||||
normalized_result: { ...cliRest, manifest: { path: forwardSlash } },
|
||||
});
|
||||
};
|
||||
|
||||
cliTest(
|
||||
"accepts numeric UTF-16 coordinates and runs the actual owned source-map codec without a browser",
|
||||
// Allow cold CLI and MCP ownership preparation across the full parity sequence.
|
||||
@@ -189,7 +212,7 @@ cliTest(
|
||||
);
|
||||
await mkdir(join(root, "files", "modules"), { recursive: true });
|
||||
await writeFile(join(root, "files", "modules", "main.js"), fixture.source);
|
||||
let manifest = join(root, "manifest.json").replaceAll("\\", "/");
|
||||
let manifest = join(root, "manifest.json");
|
||||
const map = join(root, "app.js.map");
|
||||
await writeFile(manifest, JSON.stringify(fixture.manifest));
|
||||
if (process.platform !== "win32") {
|
||||
@@ -202,14 +225,17 @@ cliTest(
|
||||
const response = await cli.run({
|
||||
arguments: [
|
||||
"trace-web-source-location",
|
||||
manifest,
|
||||
process.platform === "win32"
|
||||
? "manifest.json"
|
||||
: "lexical/link/../manifest.json",
|
||||
"0",
|
||||
map,
|
||||
"app.js.map",
|
||||
fixture.sourceMap.url,
|
||||
"1",
|
||||
"0",
|
||||
"--json",
|
||||
],
|
||||
cwd: root,
|
||||
environment: {
|
||||
REA_LOG_LEVEL: "silent",
|
||||
REA_BROWSER_EXECUTABLE: "relative-unconfigured-browser",
|
||||
@@ -273,23 +299,20 @@ cliTest(
|
||||
}
|
||||
});
|
||||
await client.connect(transport);
|
||||
const mcpResponse = await client.callTool({
|
||||
name: "trace_web_source_location",
|
||||
arguments: {
|
||||
manifest_path: manifest,
|
||||
script_index: 0,
|
||||
source_map: { path: map, url: fixture.sourceMap.url },
|
||||
generated_position: { line: 1, column: 0 },
|
||||
},
|
||||
});
|
||||
expect(mcpResponse.isError, JSON.stringify(mcpResponse)).not.toBe(true);
|
||||
expect(mcpResponse.structuredContent).toEqual(response.json);
|
||||
await expectEqualPositionSummary(client, {
|
||||
const input = {
|
||||
manifest_path: manifest,
|
||||
script_index: 0,
|
||||
source_map: { path: map, url: fixture.sourceMap.url },
|
||||
generated_position: { line: 1, column: 0 },
|
||||
};
|
||||
const mcpResponse = await client.callTool({
|
||||
name: "trace_web_source_location",
|
||||
arguments: input,
|
||||
});
|
||||
expect(mcpResponse.isError, JSON.stringify(mcpResponse)).not.toBe(true);
|
||||
expect(mcpResponse.structuredContent).toEqual(response.json);
|
||||
await expectForwardSlashManifestTrace(client, input, response.json);
|
||||
await expectEqualPositionSummary(client, input);
|
||||
const pastColumn = await client.callTool({
|
||||
name: "trace_web_source_location",
|
||||
arguments: {
|
||||
@@ -332,7 +355,7 @@ cliTest(
|
||||
const fixture = webSourceLocationFixture();
|
||||
await mkdir(join(root, "files", "modules"), { recursive: true });
|
||||
await writeFile(join(root, "files", "modules", "main.js"), fixture.source);
|
||||
const manifest = join(root, "manifest.json").replaceAll("\\", "/");
|
||||
const manifest = join(root, "manifest.json");
|
||||
await writeFile(manifest, JSON.stringify(fixture.manifest));
|
||||
const { call } = await connectLocalToolsMcp({
|
||||
REA_BROWSER_EXECUTABLE: "relative-unconfigured-browser",
|
||||
@@ -444,7 +467,7 @@ cliTest(
|
||||
const fixture = webSourceLocationFixture();
|
||||
await mkdir(join(root, "files", "modules"), { recursive: true });
|
||||
await writeFile(join(root, "files", "modules", "main.js"), fixture.source);
|
||||
const manifest = join(root, "manifest.json").replaceAll("\\", "/");
|
||||
const manifest = join(root, "manifest.json");
|
||||
await writeFile(manifest, JSON.stringify(fixture.manifest));
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const leaf: unknown = JSON.parse(fixture.sourceMap.text);
|
||||
@@ -552,7 +575,7 @@ cliTest(
|
||||
const fixture = webSourceLocationFixture(text);
|
||||
await mkdir(join(root, "files", "modules"), { recursive: true });
|
||||
await writeFile(join(root, "files", "modules", "main.js"), fixture.source);
|
||||
const manifest = join(root, "manifest.json").replaceAll("\\", "/");
|
||||
const manifest = join(root, "manifest.json");
|
||||
await writeFile(manifest, JSON.stringify(fixture.manifest));
|
||||
const map = join(root, "app.js.map");
|
||||
await writeFile(map, text);
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/** Canonical MCP operations reached by CLI commands with different names. */
|
||||
export const CLI_COMMAND_TOOL_ALIASES = Object.freeze({
|
||||
"environment-build": ["export_environment_bundle"],
|
||||
guidance: ["search_guidance", "read_guidance_section"],
|
||||
analyze: ["binary_overview"],
|
||||
inspect: ["binary_overview"],
|
||||
decompile: ["procedure_pseudo_code"],
|
||||
function: ["analyze_function"],
|
||||
instructions: ["read_function_instructions"],
|
||||
trace: ["trace_feature"],
|
||||
search: ["search_strings", "search_procedures"],
|
||||
"capture-process": ["capture_process_scenario"],
|
||||
compare: ["compare_bundles"],
|
||||
"list-objdump-architectures": ["inspect_with_objdump"],
|
||||
"debug-with-gdb": ["gdb_console"],
|
||||
"debug-with-rizin": ["rizin_debug_command"],
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
[
|
||||
"environment-build options.name empty",
|
||||
"environment-build options.name nul",
|
||||
"execute-rizin-command args.command nul",
|
||||
"export-web-scripts args.capturePath relative",
|
||||
"export-web-scripts args.outputDirectory relative",
|
||||
"extract-artifact options.outputRoot relative",
|
||||
"inspect-android-class options.mappingPath relative",
|
||||
"inspect-android-method options.mappingPath relative",
|
||||
"inspect-electron-fuses args.path empty",
|
||||
"inspect-native-api options.procedure empty",
|
||||
"inspect-native-data-type options.address empty",
|
||||
"inspect-native-data-type options.type empty",
|
||||
"recover-javascript-sources args.outputDirectory relative",
|
||||
"recover-javascript-sources args.path relative",
|
||||
"search-bytes options.cursor empty"
|
||||
]
|
||||
@@ -0,0 +1,303 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { Cli } from "incur";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import { createCli } from "../../../src/cli.js";
|
||||
import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js";
|
||||
import { CLI_COMMAND_TOOL_ALIASES } from "./cliContractAliases.js";
|
||||
|
||||
/**
|
||||
* Report-only parity between each CLI command's own field schemas and the MCP
|
||||
* contract it reaches. A CLI field that admits a value its contract refuses
|
||||
* fails late, under the wrong field name, or not at all. Mismatches already on
|
||||
* main are recorded in the baseline, which may only shrink: a new mismatch
|
||||
* fails, and so does a recorded one that no longer occurs.
|
||||
*
|
||||
* Fields are matched by mechanical name (camelCase or kebab-case to
|
||||
* snake_case). Renamed fields cannot be compared yet (W1 derives CLI fields
|
||||
* from contracts). Remove an entry from the baseline once its drift is fixed.
|
||||
*/
|
||||
|
||||
const BASELINE_CEILING = 16;
|
||||
const BASELINE_URL = new URL(
|
||||
"./cliContractSchemaParity.baseline.json",
|
||||
import.meta.url,
|
||||
);
|
||||
|
||||
type Probe = {
|
||||
readonly id: string;
|
||||
readonly value: unknown;
|
||||
/**
|
||||
* "refused": the contract refuses the value, so the CLI must refuse it too.
|
||||
* "resolved": the CLI resolves a caller-relative host path before the
|
||||
* contract sees it, so the CLI must admit the spelling the contract refuses.
|
||||
*/
|
||||
readonly expect: "refused" | "resolved";
|
||||
};
|
||||
|
||||
const admits = (schema: z.ZodType, value: unknown): boolean =>
|
||||
schema.safeParse(value).success;
|
||||
|
||||
const ABSOLUTE = process.platform === "win32" ? "C:\\rea\\x" : "/rea/x";
|
||||
const RELATIVE = "rea/x";
|
||||
|
||||
/** The host-path owner's refusal of a relative spelling (src/domain/localPath.ts). */
|
||||
const refusesAsRelativeHostPath = (contract: z.ZodType): boolean =>
|
||||
contract
|
||||
.safeParse(RELATIVE)
|
||||
.error?.issues.some(({ message }) =>
|
||||
message.includes("absolute local filesystem path"),
|
||||
) === true;
|
||||
|
||||
/** Numeric argv spellings: signs, exponents, radixes, separators, overflow. */
|
||||
const NUMERIC_SPELLINGS = [
|
||||
"01",
|
||||
"+1",
|
||||
"1e3",
|
||||
"0x10",
|
||||
"1.5",
|
||||
" 1",
|
||||
"-1",
|
||||
"1_000",
|
||||
"99999999999999999999",
|
||||
] as const;
|
||||
|
||||
/** The schema a field validates after optional, default, pipe and similar wrappers. */
|
||||
const innermost = (contract: z.ZodType): unknown => {
|
||||
let current: unknown = contract;
|
||||
for (
|
||||
let depth = 0;
|
||||
depth < 8 &&
|
||||
!(current instanceof z.ZodString) &&
|
||||
!(current instanceof z.ZodNumber);
|
||||
depth += 1
|
||||
) {
|
||||
const def = (current as { _zod?: { def?: Record<string, unknown> } })?._zod
|
||||
?.def;
|
||||
const inner = def?.["in"] ?? def?.["innerType"] ?? def?.["schema"];
|
||||
if (inner === undefined) break;
|
||||
current = inner;
|
||||
}
|
||||
return current;
|
||||
};
|
||||
|
||||
/** Probes that exercise refusals of one contract field. */
|
||||
const probesFor = (contract: z.ZodType): readonly Probe[] => {
|
||||
const probes: Probe[] = [];
|
||||
const unwrapped = innermost(contract);
|
||||
const text = admits(contract, "x")
|
||||
? "x"
|
||||
: admits(contract, ABSOLUTE)
|
||||
? ABSOLUTE
|
||||
: undefined;
|
||||
if (unwrapped instanceof z.ZodString && !admits(contract, ""))
|
||||
probes.push({ id: "empty", value: "", expect: "refused" });
|
||||
if (
|
||||
unwrapped instanceof z.ZodString &&
|
||||
!admits(contract, `${text ?? "x"}\u0000`)
|
||||
)
|
||||
probes.push({
|
||||
id: "nul",
|
||||
value: `${text ?? "x"}\u0000`,
|
||||
expect: "refused",
|
||||
});
|
||||
if (text !== undefined) {
|
||||
if (text === ABSOLUTE && refusesAsRelativeHostPath(contract))
|
||||
probes.push({ id: "relative", value: RELATIVE, expect: "resolved" });
|
||||
}
|
||||
// Numeric fields are identified by type, not by admitting 0 or 1, so a
|
||||
// range such as 256..65536 is probed too, including just past each bound.
|
||||
const numeric = unwrapped instanceof z.ZodNumber;
|
||||
if (numeric || admits(contract, 0) || admits(contract, 1)) {
|
||||
const minValue = numeric ? unwrapped.minValue : null;
|
||||
const maxValue = numeric ? unwrapped.maxValue : null;
|
||||
const bounds = [
|
||||
...(minValue === null || !Number.isFinite(minValue)
|
||||
? []
|
||||
: [String(minValue - 1)]),
|
||||
...(maxValue === null || !Number.isFinite(maxValue)
|
||||
? []
|
||||
: [String(maxValue + 1)]),
|
||||
].filter((value) => !admits(contract, Number(value)));
|
||||
for (const value of [...NUMERIC_SPELLINGS, ...bounds])
|
||||
probes.push({
|
||||
id: `numeric:${JSON.stringify(value)}`,
|
||||
value,
|
||||
expect: "refused",
|
||||
});
|
||||
}
|
||||
return probes;
|
||||
};
|
||||
|
||||
const snake = (name: string): string =>
|
||||
name.replaceAll("-", "_").replace(/[A-Z]/gu, (c) => `_${c.toLowerCase()}`);
|
||||
|
||||
/** The object shape a contract validates, through refinements and pipes. */
|
||||
const contractShape = (
|
||||
schema: unknown,
|
||||
): Readonly<Record<string, z.ZodType>> | undefined => {
|
||||
let current: unknown = schema;
|
||||
for (let depth = 0; depth < 8; depth += 1) {
|
||||
if (current instanceof z.ZodObject)
|
||||
return current.shape as Record<string, z.ZodType>;
|
||||
if (current instanceof z.ZodUnion) {
|
||||
const shapes = current.options.map((option) => contractShape(option));
|
||||
const first = shapes[0];
|
||||
if (first === undefined || shapes.some((shape) => shape === undefined))
|
||||
return undefined;
|
||||
const common = Object.fromEntries(
|
||||
Object.entries(first).filter(([key, value]) =>
|
||||
shapes.every((shape) => shape?.[key] === value),
|
||||
),
|
||||
);
|
||||
return Object.keys(common).length === 0 ? undefined : common;
|
||||
}
|
||||
const def = (current as { _zod?: { def?: Record<string, unknown> } })._zod
|
||||
?.def;
|
||||
const inner = def?.["in"] ?? def?.["innerType"] ?? def?.["schema"];
|
||||
if (inner === undefined) return undefined;
|
||||
current = inner;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/** One CLI token as incur passes it: a single argv string or the value. */
|
||||
const cliAdmittedValue = (
|
||||
field: z.ZodType,
|
||||
value: unknown,
|
||||
): { readonly admitted: boolean; readonly value?: unknown } => {
|
||||
const parsed = field.safeParse(value);
|
||||
return parsed.success
|
||||
? { admitted: true, value: parsed.data }
|
||||
: { admitted: false };
|
||||
};
|
||||
|
||||
const collectParity = () => {
|
||||
const cli = createCli();
|
||||
const commands = Cli.toCommands.get(
|
||||
cli as unknown as Parameters<typeof Cli.toCommands.get>[0],
|
||||
);
|
||||
if (commands === undefined) throw new Error("No CLI commands registered");
|
||||
const contracts = new Map<string, z.ZodType>(
|
||||
TOOL_CONTRACTS.map((contract) => [contract.name, contract.inputSchema]),
|
||||
);
|
||||
const mismatches: string[] = [];
|
||||
const uncomparable: string[] = [];
|
||||
let compared = 0;
|
||||
let commandsCompared = 0;
|
||||
let unmatchedFields = 0;
|
||||
for (const [name, entry] of commands) {
|
||||
if ("_alias" in entry) continue;
|
||||
const contractNames: readonly string[] = CLI_COMMAND_TOOL_ALIASES[
|
||||
name as keyof typeof CLI_COMMAND_TOOL_ALIASES
|
||||
] ?? [snake(name)];
|
||||
for (const contractName of contractNames) {
|
||||
const contract = contracts.get(contractName);
|
||||
if (contract === undefined) {
|
||||
uncomparable.push(`${name}: no contract ${contractName}`);
|
||||
continue;
|
||||
}
|
||||
const shape = contractShape(contract);
|
||||
if (shape === undefined) {
|
||||
uncomparable.push(`${name}: no object shape ${contractName}`);
|
||||
continue;
|
||||
}
|
||||
commandsCompared += 1;
|
||||
for (const group of ["args", "options"] as const) {
|
||||
const declared = (entry as unknown as Record<string, unknown>)[group];
|
||||
const fields: Readonly<Record<string, z.ZodType>> =
|
||||
declared instanceof z.ZodObject
|
||||
? (declared.shape as Record<string, z.ZodType>)
|
||||
: {};
|
||||
for (const [fieldName, cliField] of Object.entries(fields)) {
|
||||
const contractField = shape[snake(fieldName)];
|
||||
if (contractField === undefined) {
|
||||
unmatchedFields += 1;
|
||||
continue;
|
||||
}
|
||||
for (const probe of probesFor(contractField)) {
|
||||
compared += 1;
|
||||
const cliValue = cliAdmittedValue(cliField, probe.value);
|
||||
const mismatch =
|
||||
probe.expect === "resolved"
|
||||
? !cliValue.admitted
|
||||
: cliValue.admitted && !admits(contractField, cliValue.value);
|
||||
if (mismatch)
|
||||
mismatches.push(`${name} ${group}.${fieldName} ${probe.id}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
mismatches: [...new Set(mismatches)].sort(),
|
||||
compared,
|
||||
commandsCompared,
|
||||
unmatchedFields,
|
||||
uncomparable,
|
||||
};
|
||||
};
|
||||
|
||||
describe("CLI and contract field admission parity", () => {
|
||||
it("probes constrained strings without needing a valid sample", () => {
|
||||
const probes = probesFor(z.string().regex(/^[a-f0-9]{64}$/u));
|
||||
expect(probes.map(({ id }) => id)).toEqual(["empty", "nul"]);
|
||||
});
|
||||
it("probes numeric fields whose range admits neither 0 nor 1", () => {
|
||||
const ids = probesFor(z.number().int().min(256).max(65536).optional()).map(
|
||||
({ id }) => id,
|
||||
);
|
||||
expect(ids).toEqual(
|
||||
expect.arrayContaining([
|
||||
'numeric:"-1"',
|
||||
'numeric:"99999999999999999999"',
|
||||
'numeric:"255"',
|
||||
'numeric:"65537"',
|
||||
]),
|
||||
);
|
||||
});
|
||||
it("compares union fields shared by every operation", () => {
|
||||
const limit = z.number().int().positive();
|
||||
const schema = z.discriminatedUnion("operation", [
|
||||
z.object({ operation: z.literal("one"), limit }),
|
||||
z.object({ operation: z.literal("two"), limit }),
|
||||
]);
|
||||
expect(contractShape(schema)).toEqual({ limit });
|
||||
});
|
||||
it("admits no new value the contract refuses, and the baseline only shrinks", () => {
|
||||
const { mismatches, compared, commandsCompared, uncomparable } =
|
||||
collectParity();
|
||||
// CLI-only commands and non-object contract unions require a different comparison.
|
||||
expect(uncomparable).toEqual(
|
||||
JSON.parse(
|
||||
readFileSync(
|
||||
new URL(
|
||||
"./cliContractSchemaParity.uncomparable.json",
|
||||
import.meta.url,
|
||||
),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
// Guard against a vacuous run, such as a CLI that registers nothing.
|
||||
expect(commandsCompared).toBeGreaterThan(100);
|
||||
expect(compared).toBeGreaterThan(500);
|
||||
const baseline = JSON.parse(
|
||||
readFileSync(BASELINE_URL, "utf8"),
|
||||
) as readonly string[];
|
||||
// The recorded count can only go down; raise it only with a reviewed reason.
|
||||
expect(baseline.length).toBeLessThanOrEqual(BASELINE_CEILING);
|
||||
expect(new Set(baseline).size).toBe(baseline.length);
|
||||
const recorded = new Set(baseline);
|
||||
const observed = new Set(mismatches);
|
||||
expect(
|
||||
mismatches.filter((entry) => !recorded.has(entry)),
|
||||
"new CLI fields that admit values their contract refuses",
|
||||
).toEqual([]);
|
||||
expect(
|
||||
baseline.filter((entry) => !observed.has(entry)),
|
||||
"fixed mismatches still listed in the baseline; remove them",
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
[
|
||||
"setup: no contract setup",
|
||||
"doctor: no contract doctor",
|
||||
"uninstall: no contract uninstall",
|
||||
"update: no contract update",
|
||||
"capabilities: no contract capabilities",
|
||||
"providers: no contract providers",
|
||||
"import-reference-source: no contract import_reference_source",
|
||||
"project-managed-application-graph: no object shape project_managed_application_graph",
|
||||
"compare-web-captures: no object shape compare_web_captures",
|
||||
"validate-browser-scenario: no contract validate_browser_scenario",
|
||||
"guided-workflows: no object shape guided_workflows",
|
||||
"inspect-evidence-bundle: no contract inspect_evidence_bundle",
|
||||
"evidence-import: no contract evidence_import",
|
||||
"evidence-export: no contract evidence_export",
|
||||
"environment-export: no contract environment_export",
|
||||
"environment-compare: no contract environment_compare",
|
||||
"environment-serve: no contract environment_serve",
|
||||
"inspect-desktop-application-state: no contract inspect_desktop_application_state",
|
||||
"export-desktop-application: no contract export_desktop_application",
|
||||
"describe-desktop-application-task: no contract describe_desktop_application_task",
|
||||
"capture-desktop-application: no contract capture_desktop_application"
|
||||
]
|
||||
@@ -321,6 +321,9 @@ describe("web module CLI advertised input schema", () => {
|
||||
args({ manifestPath: resolve("manifest.json"), scriptIndex: 0 }),
|
||||
).toBe(true);
|
||||
expect(args({ manifestPath: "", scriptIndex: 0 })).toBe(false);
|
||||
expect(args({ manifestPath: "./manifest.json", scriptIndex: 0 })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(args({ manifestPath: "manifest\u0000.json", scriptIndex: 0 })).toBe(
|
||||
false,
|
||||
);
|
||||
@@ -352,12 +355,12 @@ describe("web source location CLI advertised input schema", () => {
|
||||
|
||||
expect(args(valid)).toBe(true);
|
||||
expect(args({ ...valid, manifestPath: "" })).toBe(false);
|
||||
expect(args({ ...valid, manifestPath: "./manifest.json" })).toBe(false);
|
||||
expect(args({ ...valid, manifestPath: "./manifest.json" })).toBe(true);
|
||||
expect(
|
||||
args({ ...valid, manifestPath: `${resolve("manifest.json")}\u0000` }),
|
||||
).toBe(false);
|
||||
expect(args({ ...valid, sourceMapPath: "" })).toBe(false);
|
||||
expect(args({ ...valid, sourceMapPath: "app.js.map" })).toBe(false);
|
||||
expect(args({ ...valid, sourceMapPath: "app.js.map" })).toBe(true);
|
||||
expect(
|
||||
args({ ...valid, sourceMapPath: `${resolve("app.js.map")}\u0000` }),
|
||||
).toBe(false);
|
||||
|
||||
@@ -5,23 +5,7 @@ import { createCliInventory } from "../../../scripts/lib/catalog-cli.mjs";
|
||||
import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js";
|
||||
import { CLI_COMMAND_NAMES } from "../../../src/cliCommandNames.js";
|
||||
|
||||
/** Canonical MCP operations reached by CLI commands with different names. */
|
||||
const CLI_COMMAND_TOOL_ALIASES = Object.freeze({
|
||||
"environment-build": ["export_environment_bundle"],
|
||||
guidance: ["search_guidance", "read_guidance_section"],
|
||||
analyze: ["binary_overview"],
|
||||
inspect: ["binary_overview"],
|
||||
decompile: ["procedure_pseudo_code"],
|
||||
function: ["analyze_function"],
|
||||
instructions: ["read_function_instructions"],
|
||||
trace: ["trace_feature"],
|
||||
search: ["search_strings", "search_procedures"],
|
||||
"capture-process": ["capture_process_scenario"],
|
||||
compare: ["compare_bundles"],
|
||||
"list-objdump-architectures": ["inspect_with_objdump"],
|
||||
"debug-with-gdb": ["gdb_console"],
|
||||
"debug-with-rizin": ["rizin_debug_command"],
|
||||
});
|
||||
import { CLI_COMMAND_TOOL_ALIASES } from "./cliContractAliases.js";
|
||||
|
||||
/** MCP operations without a dedicated CLI command or direct command alias. */
|
||||
const MCP_TOOLS_WITHOUT_DEDICATED_CLI = Object.freeze([
|
||||
|
||||
Reference in new issue
Block a user