test(cli): derive CLI tool coverage and schema parity from command registry (#11776)

* test(cli): derive CLI tool coverage and schema parity from command registry

* test(cli): check recorded routed operations and host operation bindings
This commit is contained in:
morluto authored and GitHub committed 2026-11-16 16:30:36 -05:00
1 parent e741366c94
commit e44f0077c9
11 files changed
+605 -389

No files matched your search

+28
View File
@@ -165,3 +165,31 @@ export const distinctOperations = (
const seen = new Set<string>();
return operations.filter(({ name }) => !seen.has(name) && seen.add(name));
};
/**
* Contract operations the command itself answers, excluding the session-target
* opener and any Evidence-target source operation it runs first.
*/
export const commandOwnOperations = (
entry: CliCommandEntry,
): readonly CliCommandOperation[] => {
const { source, operations } = entry;
if (source.kind === "contract") {
const opening = source.description.sessionTarget?.opening;
const targetSource = source.description.evidenceTargets?.source;
return operations.filter(
({ name }) => name !== opening && name !== targetSource,
);
}
if (source.kind === "routed") {
const own = new Set<string>(
source.descriptions.map((description) => description.contract.name),
);
return operations.filter(({ name }) => own.has(name));
}
if (source.kind === "document") {
const opening = source.description.sessionTarget?.opening;
return operations.filter(({ name }) => name !== opening);
}
return operations;
};
+84 -61
View File
@@ -483,71 +483,94 @@ const registerBuildCommand = (
logger: Logger,
environment: Readonly<NodeJS.ProcessEnv>,
): void => {
registerCliCommand(cli, CLI_COMMANDS.environmentBuild, {
description:
"Build an agent environment bundle from a recorded capture in one offline step: inspect a HAR (or read capture Evidence), project the environment manifest, derive the state, report which tools the environment answers from state, which replay and why each write is withheld, then write the bundle environment-export writes",
args: z.object({
capture: localPathStringSchema.describe(
"A .har file, or capture Evidence JSON from inspect-web-network-capture (HAR) or capture-browser-scenario",
),
directory: localPathStringSchema
.optional()
.describe(
"Bundle directory to create; must be absent or empty, nothing is overwritten; omit only with --dry-run",
registerCliCommand(
cli,
CLI_COMMANDS.environmentBuild,
{
description:
"Build an agent environment bundle from a recorded capture in one offline step: inspect a HAR (or read capture Evidence), project the environment manifest, derive the state, report which tools the environment answers from state, which replay and why each write is withheld, then write the bundle environment-export writes",
args: z.object({
capture: localPathStringSchema.describe(
"A .har file, or capture Evidence JSON from inspect-web-network-capture (HAR) or capture-browser-scenario",
),
}),
options: z.object({
name: z
.string()
.optional()
.describe(
"Env folder and env id (kebab-case); default the manifest's name, or the capture file's name when the origin is a loopback or IP address",
directory: localPathStringSchema
.optional()
.describe(
"Bundle directory to create; must be absent or empty, nothing is overwritten; omit only with --dry-run",
),
}),
options: z.object({
name: z
.string()
.optional()
.describe(
"Env folder and env id (kebab-case); default the manifest's name, or the capture file's name when the origin is a loopback or IP address",
),
templateRule: z
.array(z.enum(TEMPLATE_RULES))
.optional()
.describe(
"Opt-in template rule; repeat to select several. Replaces the default after-uid-literal, opaque-token, sibling-variance and prefix-variance",
),
formatRulesOnly: z
.boolean()
.default(false)
.describe("Apply only the four format rules, the views' own default"),
packageTarball: localPathStringSchema
.optional()
.describe(
"Packed rea-agents tarball (npm pack) copied into the env folder and installed by the Dockerfile instead of the registry package pinned to this version",
),
dryRun: z
.boolean()
.default(false)
.describe("Run every stage and report, but write nothing"),
// Decimal tokens only: numeric coercion would read "0x1" as 1 ms.
captureTimeoutMs: cliDecimalField(
inspectWebNetworkCaptureInputSchema.shape.capture_timeout_ms,
),
templateRule: z
.array(z.enum(TEMPLATE_RULES))
.optional()
.describe(
"Opt-in template rule; repeat to select several. Replaces the default after-uid-literal, opaque-token, sibling-variance and prefix-variance",
),
formatRulesOnly: z
.boolean()
.default(false)
.describe("Apply only the four format rules, the views' own default"),
packageTarball: localPathStringSchema
.optional()
.describe(
"Packed rea-agents tarball (npm pack) copied into the env folder and installed by the Dockerfile instead of the registry package pinned to this version",
),
dryRun: z
.boolean()
.default(false)
.describe("Run every stage and report, but write nothing"),
// Decimal tokens only: numeric coercion would read "0x1" as 1 ms.
captureTimeoutMs: cliDecimalField(
inspectWebNetworkCaptureInputSchema.shape.capture_timeout_ms,
),
}),
run: ({ args, options }) =>
withCommandCancellation((signal) =>
logCliCommand(logger, CLI_COMMANDS.environmentBuild, () =>
buildEnvironmentCommand(
{
capture: args.capture,
directory: args.directory,
name: options.name,
templateRule: options.templateRule,
formatRulesOnly: options.formatRulesOnly,
packageTarball: options.packageTarball,
dryRun: options.dryRun,
captureTimeoutMs: options.captureTimeoutMs,
},
environment,
logger,
signal,
}),
run: ({ args, options }) =>
withCommandCancellation((signal) =>
logCliCommand(logger, CLI_COMMANDS.environmentBuild, () =>
buildEnvironmentCommand(
{
capture: args.capture,
directory: args.directory,
name: options.name,
templateRule: options.templateRule,
formatRulesOnly: options.formatRulesOnly,
packageTarball: options.packageTarball,
dryRun: options.dryRun,
captureTimeoutMs: options.captureTimeoutMs,
},
environment,
logger,
signal,
),
),
),
),
});
},
{
operations: [
{
name: "export_environment_bundle",
flags: {
name: "--name",
template_rules: "--template-rule",
package_tarball: "--package-tarball",
dry_run: "--dry-run",
},
},
{
name: "inspect_web_network_capture",
flags: {
capture_timeout_ms: "--capture-timeout-ms",
},
},
],
},
);
};
const manifestArgument = z
+102 -78
View File
@@ -52,86 +52,110 @@ export const registerGuidanceCommands = (
logger: Logger,
resources: CliGuidanceResources = processGuidanceResources,
): void => {
registerCliCommand(cli, CLI_COMMANDS.guidance, {
description: "Search or read the installed REA workflow guidance",
args: z.object({}),
options: z
.object({
action: z
.enum(["search", "read"])
.default("search")
.describe("Guidance action to perform: search or read"),
query: searchGuidanceInputSchema.shape.query
.optional()
.describe(
"Required with search; all non-generic query words must match",
registerCliCommand(
cli,
CLI_COMMANDS.guidance,
{
description: "Search or read the installed REA workflow guidance",
args: z.object({}),
options: z
.object({
action: z
.enum(["search", "read"])
.default("search")
.describe("Guidance action to perform: search or read"),
query: searchGuidanceInputSchema.shape.query
.optional()
.describe(
"Required with search; all non-generic query words must match",
),
locator: readGuidanceSectionInputSchema.shape.locator
.optional()
.describe(
"Required with --action read; locator returned by search",
),
// Decimal tokens only: numeric coercion would read "0x2" as 2.
limit: cliDecimalField(searchGuidanceLimitSchema.optional()).describe(
"Maximum search matches; search only, defaults to 8",
),
locator: readGuidanceSectionInputSchema.shape.locator
.optional()
.describe("Required with --action read; locator returned by search"),
// Decimal tokens only: numeric coercion would read "0x2" as 2.
limit: cliDecimalField(searchGuidanceLimitSchema.optional()).describe(
"Maximum search matches; search only, defaults to 8",
),
cursor: searchGuidanceInputSchema.shape.cursor
.optional()
.describe("Continue a search page; search only"),
})
.superRefine((options, context) => {
if (options.action === "search") {
if (options.query === undefined)
context.addIssue({
code: "custom",
path: ["query"],
message: "--query is required with --action search",
});
if (options.locator !== undefined)
context.addIssue({
code: "custom",
path: ["locator"],
message: "--locator can only be used with --action read",
});
} else {
if (options.locator === undefined)
context.addIssue({
code: "custom",
path: ["locator"],
message: "--locator is required with --action read",
});
for (const option of ["query", "limit", "cursor"] as const)
if (options[option] !== undefined)
cursor: searchGuidanceInputSchema.shape.cursor
.optional()
.describe("Continue a search page; search only"),
})
.superRefine((options, context) => {
if (options.action === "search") {
if (options.query === undefined)
context.addIssue({
code: "custom",
path: [option],
message: `--${option} can only be used with --action search`,
path: ["query"],
message: "--query is required with --action search",
});
}
}),
run: ({ options }) =>
logCliCommand(logger, CLI_COMMANDS.guidance, async () => {
const result =
options.action === "search"
? await searchGuidance(
{
...(options.query === undefined
? {}
: { query: options.query }),
...(options.limit === undefined
? {}
: { limit: options.limit }),
...(options.cursor === undefined
? {}
: { cursor: options.cursor }),
},
resources.resources(),
)
: await readGuidanceSection(
{
locator: options.locator,
},
resources.resources(),
);
return result.ok ? result.value : projectAnalysisError(result.error);
}),
});
if (options.locator !== undefined)
context.addIssue({
code: "custom",
path: ["locator"],
message: "--locator can only be used with --action read",
});
} else {
if (options.locator === undefined)
context.addIssue({
code: "custom",
path: ["locator"],
message: "--locator is required with --action read",
});
for (const option of ["query", "limit", "cursor"] as const)
if (options[option] !== undefined)
context.addIssue({
code: "custom",
path: [option],
message: `--${option} can only be used with --action search`,
});
}
}),
run: ({ options }) =>
logCliCommand(logger, CLI_COMMANDS.guidance, async () => {
const result =
options.action === "search"
? await searchGuidance(
{
...(options.query === undefined
? {}
: { query: options.query }),
...(options.limit === undefined
? {}
: { limit: options.limit }),
...(options.cursor === undefined
? {}
: { cursor: options.cursor }),
},
resources.resources(),
)
: await readGuidanceSection(
{
locator: options.locator,
},
resources.resources(),
);
return result.ok ? result.value : projectAnalysisError(result.error);
}),
},
{
operations: [
{
name: "search_guidance",
flags: {
query: "--query",
limit: "--limit",
cursor: "--cursor",
},
},
{
name: "read_guidance_section",
flags: {
locator: "--locator",
},
},
],
},
);
};
+21 -12
View File
@@ -27,20 +27,29 @@ export const registerProcessCommands = (
logger: Logger,
environment: Readonly<Record<string, string | undefined>>,
): void => {
registerCliCommand(cli, CLI_COMMANDS.captureProcess, {
description: "Capture one caller-selected process scenario",
args: z.object({
scenario: localPathStringSchema.describe("Scenario JSON path"),
}),
run: ({ args }) =>
withCommandCancellation((signal) =>
logCliCommand(logger, "capture-process", () =>
captureProcessScenarioFile(args.scenario, environment, signal).catch(
projectProcessCliInputCleanupError,
registerCliCommand(
cli,
CLI_COMMANDS.captureProcess,
{
description: "Capture one caller-selected process scenario",
args: z.object({
scenario: localPathStringSchema.describe("Scenario JSON path"),
}),
run: ({ args }) =>
withCommandCancellation((signal) =>
logCliCommand(logger, "capture-process", () =>
captureProcessScenarioFile(
args.scenario,
environment,
signal,
).catch(projectProcessCliInputCleanupError),
),
),
),
});
},
{
operations: [{ name: "capture_process_scenario", flags: {} }],
},
);
registerContractCommand(
cli as CliInstance,
logger,
+171 -156
View File
@@ -233,163 +233,178 @@ export const registerReverseEngineeringCommands = (
),
),
);
registerCliCommand(cli, CLI_COMMANDS.debugWithGdb, {
description:
"Run one unrestricted GDB console command in an ephemeral MI session; provide an inferior TTY so a live target can be safely detached",
args: z.object({
command: z
.string()
.min(1)
.describe("One unrestricted GDB console command."),
}),
options: z.object({
executable: z
.string()
.min(1)
.optional()
.describe(
"Initial local file to hash and load without running it; command Evidence retains this initial selection.",
),
inferiorTty: z
.string()
.min(1)
.describe(
"Caller-selected durable TTY for output from a live GDB inferior.",
),
...analysisEvidenceDeliveryOptions,
}),
run: ({ args, options }) =>
withCommandCancellation((signal) =>
logCliCommand(logger, CLI_COMMANDS.debugWithGdb, async () => {
const delivery = {
providerId: "gdb",
operation: "gdb_console",
signal,
detail: options.detail,
evidenceOutput: options.evidence_output,
};
const inputError = cliEvidenceDeliveryInputError(delivery);
if (inputError !== undefined) return projectAnalysisError(inputError);
if (gdbOneShotCommandMayInvalidateDetachProof(args.command))
return projectAnalysisError(
new AnalysisInputError("gdb_console", undefined, [
{
path: ["command"],
reason: "invalid_value",
message:
"This command can invalidate the verified inferior output route; use the persistent GDB session workflow.",
},
]),
);
const manager = createGdbManager(environment);
let sessionId: string | undefined;
return deliverCliAnalysisEvidence({
...delivery,
run: async () => {
const started = await manager.start(signal, {
inferiorTty: options.inferiorTty,
...(options.executable === undefined
? {}
: { executable: options.executable }),
});
if (!started.ok) return started;
sessionId = started.value.session_id;
const executed = await manager.execute(
sessionId,
args.command,
signal,
registerCliCommand(
cli,
CLI_COMMANDS.debugWithGdb,
{
description:
"Run one unrestricted GDB console command in an ephemeral MI session; provide an inferior TTY so a live target can be safely detached",
args: z.object({
command: z
.string()
.min(1)
.describe("One unrestricted GDB console command."),
}),
options: z.object({
executable: z
.string()
.min(1)
.optional()
.describe(
"Initial local file to hash and load without running it; command Evidence retains this initial selection.",
),
inferiorTty: z
.string()
.min(1)
.describe(
"Caller-selected durable TTY for output from a live GDB inferior.",
),
...analysisEvidenceDeliveryOptions,
}),
run: ({ args, options }) =>
withCommandCancellation((signal) =>
logCliCommand(logger, CLI_COMMANDS.debugWithGdb, async () => {
const delivery = {
providerId: "gdb",
operation: "gdb_console",
signal,
detail: options.detail,
evidenceOutput: options.evidence_output,
};
const inputError = cliEvidenceDeliveryInputError(delivery);
if (inputError !== undefined)
return projectAnalysisError(inputError);
if (gdbOneShotCommandMayInvalidateDetachProof(args.command))
return projectAnalysisError(
new AnalysisInputError("gdb_console", undefined, [
{
path: ["command"],
reason: "invalid_value",
message:
"This command can invalidate the verified inferior output route; use the persistent GDB session workflow.",
},
]),
);
return executed.ok ? ok(executed.value.evidence) : executed;
},
service: {
close: () =>
closeDebuggerCommandSession(
manager,
sessionId,
"gdb",
"gdb_console",
(cleanup) =>
logger.error(
{ command: CLI_COMMANDS.debugWithGdb, cleanup },
"GDB cleanup failed after collecting command output",
),
),
},
});
}),
),
});
registerCliCommand(cli, CLI_COMMANDS.debugWithRizin, {
description:
"Run one unrestricted Rizin debugger command in an ephemeral session",
args: z.object({
path: localPathStringSchema.describe(
"Path to the local artifact to debug.",
),
command: z
.string()
.min(1)
.describe("One unrestricted Rizin debugger command.")
.refine((value) => !/[\r\n]/u.test(value)),
}),
options: z.object({
inferiorTty: z
.string()
.min(1)
.describe(
"Caller-selected Linux TTY for inferior stdin, stdout and stderr.",
),
backend: z
.string()
.min(1)
.optional()
.describe("Optional Rizin IO debugger backend name."),
}),
run: ({ args, options }) =>
withCommandCancellation((signal) =>
logCliCommand(logger, CLI_COMMANDS.debugWithRizin, async () => {
const manager = createRizinDebugSessionManager(environment);
let sessionId: string | undefined;
return finishAnalysisCommand(
async () => {
const started = await manager.start(
{
...args,
inferior_tty: options.inferiorTty,
...(options.backend === undefined
const manager = createGdbManager(environment);
let sessionId: string | undefined;
return deliverCliAnalysisEvidence({
...delivery,
run: async () => {
const started = await manager.start(signal, {
inferiorTty: options.inferiorTty,
...(options.executable === undefined
? {}
: { backend: options.backend }),
},
signal,
);
if (!started.ok) return started;
sessionId = started.value.session_id;
const executed = await manager.execute(
sessionId,
args.command,
signal,
);
return executed.ok ? ok(executed.value.evidence) : executed;
},
{
close: () =>
closeDebuggerCommandSession(
manager,
: { executable: options.executable }),
});
if (!started.ok) return started;
sessionId = started.value.session_id;
const executed = await manager.execute(
sessionId,
"rizin",
"rizin_debug_command",
(cleanup) =>
logger.error(
{ command: CLI_COMMANDS.debugWithRizin, cleanup },
"Rizin cleanup failed after collecting command output",
),
),
},
"rizin",
"rizin_debug_command",
signal,
).then(projectCliAnalysisResult);
}),
),
});
args.command,
signal,
);
return executed.ok ? ok(executed.value.evidence) : executed;
},
service: {
close: () =>
closeDebuggerCommandSession(
manager,
sessionId,
"gdb",
"gdb_console",
(cleanup) =>
logger.error(
{ command: CLI_COMMANDS.debugWithGdb, cleanup },
"GDB cleanup failed after collecting command output",
),
),
},
});
}),
),
},
{
operations: [{ name: "gdb_console", flags: {} }],
},
);
registerCliCommand(
cli,
CLI_COMMANDS.debugWithRizin,
{
description:
"Run one unrestricted Rizin debugger command in an ephemeral session",
args: z.object({
path: localPathStringSchema.describe(
"Path to the local artifact to debug.",
),
command: z
.string()
.min(1)
.describe("One unrestricted Rizin debugger command.")
.refine((value) => !/[\r\n]/u.test(value)),
}),
options: z.object({
inferiorTty: z
.string()
.min(1)
.describe(
"Caller-selected Linux TTY for inferior stdin, stdout and stderr.",
),
backend: z
.string()
.min(1)
.optional()
.describe("Optional Rizin IO debugger backend name."),
}),
run: ({ args, options }) =>
withCommandCancellation((signal) =>
logCliCommand(logger, CLI_COMMANDS.debugWithRizin, async () => {
const manager = createRizinDebugSessionManager(environment);
let sessionId: string | undefined;
return finishAnalysisCommand(
async () => {
const started = await manager.start(
{
...args,
inferior_tty: options.inferiorTty,
...(options.backend === undefined
? {}
: { backend: options.backend }),
},
signal,
);
if (!started.ok) return started;
sessionId = started.value.session_id;
const executed = await manager.execute(
sessionId,
args.command,
signal,
);
return executed.ok ? ok(executed.value.evidence) : executed;
},
{
close: () =>
closeDebuggerCommandSession(
manager,
sessionId,
"rizin",
"rizin_debug_command",
(cleanup) =>
logger.error(
{ command: CLI_COMMANDS.debugWithRizin, cleanup },
"Rizin cleanup failed after collecting command output",
),
),
},
"rizin",
"rizin_debug_command",
signal,
).then(projectCliAnalysisResult);
}),
),
},
{
operations: [{ name: "rizin_debug_command", flags: {} }],
},
);
};