Files
reverse-engineering-anything/tests/boundary/cli/cliContractSchemaParity.test.ts
T
morluto 48220827c4 test(cli): report CLI fields that admit values their contract refuses (#10575)
* test(cli): report CLI fields that admit values their contract refuses

For every CLI command that reaches an MCP contract, compare each
name-matched CLI field with its contract field on empty, NUL,
caller-relative host path and numeric argv spellings. The 16 mismatches on
main are recorded in a baseline with a counted ceiling that can only go
down: a new mismatch fails, and so does a listed one that no longer occurs.

* test: cover constrained CLI contract fields

* fix(cli): admit relative web trace paths

* test(cli): probe numeric contract fields by type, including bounds

Numeric probes no longer require the contract to admit 0 or 1, so ranged fields such as inspect-byte-entropy --block-size (256..65536) are compared, with probes just past each declared bound.

* fix(cli): describe caller-relative web trace paths and expect native Windows spellings

The relative-path CLI arguments lost their contract descriptions; they now describe the caller-relative resolution. absoluteCallerPath renders native separators on Windows, so the web trace acceptance tests use native manifest paths for CLI and MCP parity.

* test(cli): keep forward-slash Windows manifest coverage through MCP

The CLI renders caller paths with native separators through absoluteCallerPath; MCP keeps the caller's spelling. The web trace parity tests now also call MCP with the forward-slash manifest spelling and expect the same trace with that path echoed, restoring the #1078 coverage.
2026-11-15 17:07:09 +08:00

304 lines
10 KiB
TypeScript

import { readFileSync } from "node:fs";
import { Cli } from "incur";
import { describe, expect, it } from "vitest";
import { z } from "zod";
import { createCli } from "../../../src/cli.js";
import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js";
import { CLI_COMMAND_TOOL_ALIASES } from "./cliContractAliases.js";
/**
* Report-only parity between each CLI command's own field schemas and the MCP
* contract it reaches. A CLI field that admits a value its contract refuses
* fails late, under the wrong field name, or not at all. Mismatches already on
* main are recorded in the baseline, which may only shrink: a new mismatch
* fails, and so does a recorded one that no longer occurs.
*
* Fields are matched by mechanical name (camelCase or kebab-case to
* snake_case). Renamed fields cannot be compared yet (W1 derives CLI fields
* from contracts). Remove an entry from the baseline once its drift is fixed.
*/
const BASELINE_CEILING = 16;
const BASELINE_URL = new URL(
"./cliContractSchemaParity.baseline.json",
import.meta.url,
);
type Probe = {
readonly id: string;
readonly value: unknown;
/**
* "refused": the contract refuses the value, so the CLI must refuse it too.
* "resolved": the CLI resolves a caller-relative host path before the
* contract sees it, so the CLI must admit the spelling the contract refuses.
*/
readonly expect: "refused" | "resolved";
};
const admits = (schema: z.ZodType, value: unknown): boolean =>
schema.safeParse(value).success;
const ABSOLUTE = process.platform === "win32" ? "C:\\rea\\x" : "/rea/x";
const RELATIVE = "rea/x";
/** The host-path owner's refusal of a relative spelling (src/domain/localPath.ts). */
const refusesAsRelativeHostPath = (contract: z.ZodType): boolean =>
contract
.safeParse(RELATIVE)
.error?.issues.some(({ message }) =>
message.includes("absolute local filesystem path"),
) === true;
/** Numeric argv spellings: signs, exponents, radixes, separators, overflow. */
const NUMERIC_SPELLINGS = [
"01",
"+1",
"1e3",
"0x10",
"1.5",
" 1",
"-1",
"1_000",
"99999999999999999999",
] as const;
/** The schema a field validates after optional, default, pipe and similar wrappers. */
const innermost = (contract: z.ZodType): unknown => {
let current: unknown = contract;
for (
let depth = 0;
depth < 8 &&
!(current instanceof z.ZodString) &&
!(current instanceof z.ZodNumber);
depth += 1
) {
const def = (current as { _zod?: { def?: Record<string, unknown> } })?._zod
?.def;
const inner = def?.["in"] ?? def?.["innerType"] ?? def?.["schema"];
if (inner === undefined) break;
current = inner;
}
return current;
};
/** Probes that exercise refusals of one contract field. */
const probesFor = (contract: z.ZodType): readonly Probe[] => {
const probes: Probe[] = [];
const unwrapped = innermost(contract);
const text = admits(contract, "x")
? "x"
: admits(contract, ABSOLUTE)
? ABSOLUTE
: undefined;
if (unwrapped instanceof z.ZodString && !admits(contract, ""))
probes.push({ id: "empty", value: "", expect: "refused" });
if (
unwrapped instanceof z.ZodString &&
!admits(contract, `${text ?? "x"}\u0000`)
)
probes.push({
id: "nul",
value: `${text ?? "x"}\u0000`,
expect: "refused",
});
if (text !== undefined) {
if (text === ABSOLUTE && refusesAsRelativeHostPath(contract))
probes.push({ id: "relative", value: RELATIVE, expect: "resolved" });
}
// Numeric fields are identified by type, not by admitting 0 or 1, so a
// range such as 256..65536 is probed too, including just past each bound.
const numeric = unwrapped instanceof z.ZodNumber;
if (numeric || admits(contract, 0) || admits(contract, 1)) {
const minValue = numeric ? unwrapped.minValue : null;
const maxValue = numeric ? unwrapped.maxValue : null;
const bounds = [
...(minValue === null || !Number.isFinite(minValue)
? []
: [String(minValue - 1)]),
...(maxValue === null || !Number.isFinite(maxValue)
? []
: [String(maxValue + 1)]),
].filter((value) => !admits(contract, Number(value)));
for (const value of [...NUMERIC_SPELLINGS, ...bounds])
probes.push({
id: `numeric:${JSON.stringify(value)}`,
value,
expect: "refused",
});
}
return probes;
};
const snake = (name: string): string =>
name.replaceAll("-", "_").replace(/[A-Z]/gu, (c) => `_${c.toLowerCase()}`);
/** The object shape a contract validates, through refinements and pipes. */
const contractShape = (
schema: unknown,
): Readonly<Record<string, z.ZodType>> | undefined => {
let current: unknown = schema;
for (let depth = 0; depth < 8; depth += 1) {
if (current instanceof z.ZodObject)
return current.shape as Record<string, z.ZodType>;
if (current instanceof z.ZodUnion) {
const shapes = current.options.map((option) => contractShape(option));
const first = shapes[0];
if (first === undefined || shapes.some((shape) => shape === undefined))
return undefined;
const common = Object.fromEntries(
Object.entries(first).filter(([key, value]) =>
shapes.every((shape) => shape?.[key] === value),
),
);
return Object.keys(common).length === 0 ? undefined : common;
}
const def = (current as { _zod?: { def?: Record<string, unknown> } })._zod
?.def;
const inner = def?.["in"] ?? def?.["innerType"] ?? def?.["schema"];
if (inner === undefined) return undefined;
current = inner;
}
return undefined;
};
/** One CLI token as incur passes it: a single argv string or the value. */
const cliAdmittedValue = (
field: z.ZodType,
value: unknown,
): { readonly admitted: boolean; readonly value?: unknown } => {
const parsed = field.safeParse(value);
return parsed.success
? { admitted: true, value: parsed.data }
: { admitted: false };
};
const collectParity = () => {
const cli = createCli();
const commands = Cli.toCommands.get(
cli as unknown as Parameters<typeof Cli.toCommands.get>[0],
);
if (commands === undefined) throw new Error("No CLI commands registered");
const contracts = new Map<string, z.ZodType>(
TOOL_CONTRACTS.map((contract) => [contract.name, contract.inputSchema]),
);
const mismatches: string[] = [];
const uncomparable: string[] = [];
let compared = 0;
let commandsCompared = 0;
let unmatchedFields = 0;
for (const [name, entry] of commands) {
if ("_alias" in entry) continue;
const contractNames: readonly string[] = CLI_COMMAND_TOOL_ALIASES[
name as keyof typeof CLI_COMMAND_TOOL_ALIASES
] ?? [snake(name)];
for (const contractName of contractNames) {
const contract = contracts.get(contractName);
if (contract === undefined) {
uncomparable.push(`${name}: no contract ${contractName}`);
continue;
}
const shape = contractShape(contract);
if (shape === undefined) {
uncomparable.push(`${name}: no object shape ${contractName}`);
continue;
}
commandsCompared += 1;
for (const group of ["args", "options"] as const) {
const declared = (entry as unknown as Record<string, unknown>)[group];
const fields: Readonly<Record<string, z.ZodType>> =
declared instanceof z.ZodObject
? (declared.shape as Record<string, z.ZodType>)
: {};
for (const [fieldName, cliField] of Object.entries(fields)) {
const contractField = shape[snake(fieldName)];
if (contractField === undefined) {
unmatchedFields += 1;
continue;
}
for (const probe of probesFor(contractField)) {
compared += 1;
const cliValue = cliAdmittedValue(cliField, probe.value);
const mismatch =
probe.expect === "resolved"
? !cliValue.admitted
: cliValue.admitted && !admits(contractField, cliValue.value);
if (mismatch)
mismatches.push(`${name} ${group}.${fieldName} ${probe.id}`);
}
}
}
}
}
return {
mismatches: [...new Set(mismatches)].sort(),
compared,
commandsCompared,
unmatchedFields,
uncomparable,
};
};
describe("CLI and contract field admission parity", () => {
it("probes constrained strings without needing a valid sample", () => {
const probes = probesFor(z.string().regex(/^[a-f0-9]{64}$/u));
expect(probes.map(({ id }) => id)).toEqual(["empty", "nul"]);
});
it("probes numeric fields whose range admits neither 0 nor 1", () => {
const ids = probesFor(z.number().int().min(256).max(65536).optional()).map(
({ id }) => id,
);
expect(ids).toEqual(
expect.arrayContaining([
'numeric:"-1"',
'numeric:"99999999999999999999"',
'numeric:"255"',
'numeric:"65537"',
]),
);
});
it("compares union fields shared by every operation", () => {
const limit = z.number().int().positive();
const schema = z.discriminatedUnion("operation", [
z.object({ operation: z.literal("one"), limit }),
z.object({ operation: z.literal("two"), limit }),
]);
expect(contractShape(schema)).toEqual({ limit });
});
it("admits no new value the contract refuses, and the baseline only shrinks", () => {
const { mismatches, compared, commandsCompared, uncomparable } =
collectParity();
// CLI-only commands and non-object contract unions require a different comparison.
expect(uncomparable).toEqual(
JSON.parse(
readFileSync(
new URL(
"./cliContractSchemaParity.uncomparable.json",
import.meta.url,
),
"utf8",
),
),
);
// Guard against a vacuous run, such as a CLI that registers nothing.
expect(commandsCompared).toBeGreaterThan(100);
expect(compared).toBeGreaterThan(500);
const baseline = JSON.parse(
readFileSync(BASELINE_URL, "utf8"),
) as readonly string[];
// The recorded count can only go down; raise it only with a reviewed reason.
expect(baseline.length).toBeLessThanOrEqual(BASELINE_CEILING);
expect(new Set(baseline).size).toBe(baseline.length);
const recorded = new Set(baseline);
const observed = new Set(mismatches);
expect(
mismatches.filter((entry) => !recorded.has(entry)),
"new CLI fields that admit values their contract refuses",
).toEqual([]);
expect(
baseline.filter((entry) => !observed.has(entry)),
"fixed mismatches still listed in the baseline; remove them",
).toEqual([]);
});
});