test(cli): consolidate recovery and result control checks (#11891)

This commit is contained in:
morluto authored and GitHub committed 2026-11-18 07:17:37 +08:00
1 parent ebb606a139
commit c3209239c3
3 files changed
+95 -367

No files matched your search

-173
View File
@@ -1,173 +0,0 @@
import { cliCommand } from "../../../src/cli/commandRegistry.js";
import { createCommandCli } from "../../support/cli/commandCli.js";
import { expect, it } from "vitest";
import { z } from "zod";
import { registerContractCommand } from "../../../src/cli/contractCommand.js";
import { describeContractCommand } from "../../../src/cli/contractCliDescription.js";
import { projectCliInputFields } from "../../../src/cli/projectCliAnalysisResult.js";
import type { CliInstance } from "../../../src/cli/types.js";
import { AnalysisTimeoutError } from "../../../src/domain/analysisErrorCore.js";
import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js";
import { nativeDeadlineRecovery } from "../../../src/domain/native/nativeDeadlines.js";
import { silentLogger } from "../../../src/logger.js";
const contract = {
name: "inspect_control_fixture",
inputSchema: z.strictObject({
path: z.string(),
worker_timeout_ms: z.number().int().optional(),
}),
};
/** The failure a service reports: it names the contract field only. */
const deadlineFailure = () =>
projectAnalysisError(
new AnalysisTimeoutError(contract.name, 5, {
...nativeDeadlineRecovery({
timeoutMs: 5,
parameter: "worker_timeout_ms",
defaultMs: 1_000,
phase: "fixture deadline",
unchanged: "",
}),
}),
);
it("names the service's per-call control by field over MCP and by flag on the CLI", async () => {
const mcp = deadlineFailure();
expect(mcp.details?.limit).toMatchObject({
control_kind: "parameter",
parameter: "worker_timeout_ms",
});
expect(mcp.details?.limit).not.toHaveProperty("cli_flag");
expect(JSON.stringify(mcp)).not.toContain("--");
const cli = createCommandCli();
registerContractCommand(
cli as unknown as CliInstance,
silentLogger,
contract,
{
command: "inspect-control-fixture",
description: "Control fixture",
positionals: ["path"],
},
async () => deadlineFailure(),
);
const command = cliCommand(cli, "inspect-control-fixture")?.definition;
if (command === undefined) throw new Error("command is not registered");
const parse = (key: "args" | "options", value: unknown) =>
(Reflect.get(command, key) as { parse(v: unknown): unknown }).parse(value);
// A failed command sets the shared worker's exit status.
const previousExitCode = process.exitCode;
let output: ReturnType<typeof deadlineFailure>;
try {
output = (await (
Reflect.get(command, "run") as (value: unknown) => Promise<unknown>
)({
args: parse("args", { path: "fixture.bin" }),
options: parse("options", { workerTimeoutMs: 5 }),
})) as ReturnType<typeof deadlineFailure>;
} finally {
process.exitCode = previousExitCode;
}
expect(output.details?.limit).toEqual({
...(mcp.details?.limit as object),
cli_flag: "--worker-timeout-ms",
});
expect(output.remediation).toEqual(mcp.remediation);
});
it("names a member control by its member flag, else by its field's flag", () => {
const failure = (parameter: string) => ({
code: "provider_timeout",
details: { limit: { control_kind: "parameter", parameter } },
});
const binding = {
operation: "fixture",
fields: {},
flags: {
producer_limits: "--producer-limits",
"pool_policy.uri_limit": "--uri-limit",
},
};
expect(
projectCliInputFields(failure("producer_limits.output_bytes"), binding),
).toMatchObject({ details: { limit: { cli_flag: "--producer-limits" } } });
expect(
projectCliInputFields(failure("pool_policy.uri_limit"), binding),
).toMatchObject({ details: { limit: { cli_flag: "--uri-limit" } } });
// A control the command does not take as a flag keeps the field only.
const unbound = failure("analysis_timeout_ms");
expect(projectCliInputFields(unbound, binding)).toBe(unbound);
const success = { details: { limit: { parameter: "producer_limits" } } };
expect(projectCliInputFields(success, binding)).toBe(success);
// A registered setting that a request argument also selects names it.
expect(
projectCliInputFields(
{
code: "provider_timeout",
details: {
limit: {
setting: "fixture.timeout_ms",
parameter: "producer_limits.timeout_ms",
},
},
},
binding,
),
).toMatchObject({ details: { limit: { cli_flag: "--producer-limits" } } });
});
it("binds a nested flattened member to its own flag", () => {
const description = describeContractCommand(
{
name: "inspect_nested_fixture",
inputSchema: z.strictObject({
path: z.string(),
protobuf: z
.strictObject({
schema: z
.strictObject({ message_type: z.string().optional() })
.optional(),
})
.optional(),
}),
},
{
command: "inspect-nested-fixture",
description: "Nested fixture",
positionals: ["path"],
flatten: {
protobuf: {
schema: {
flatten: {
message_type: {
name: "protobufMessage",
description: "Message type",
},
},
},
},
},
},
);
expect(description.inputBinding.flags).toEqual({
"protobuf.schema.message_type": "--protobuf-message",
});
expect(
projectCliInputFields(
{
code: "resource_limit_exceeded",
details: {
limit: {
control_kind: "parameter",
parameter: "protobuf.schema.message_type",
},
},
},
description.inputBinding,
),
).toMatchObject({ details: { limit: { cli_flag: "--protobuf-message" } } });
});
+31 -69
View File
@@ -2,10 +2,11 @@ import { cliCommand } from "../../../src/cli/commandRegistry.js";
import { createCommandCli } from "../../support/cli/commandCli.js";
import { Writable } from "node:stream";
import { expect, it, vi } from "vitest";
import { expect, it } from "vitest";
import { z } from "zod";
import { formatCliOutput } from "../../../src/cli/cliOutputFormatter.js";
import { decode as decodeToon } from "@toon-format/toon";
import { parse as parseYaml } from "yaml";
import { parseCliOutputArguments } from "../../../src/cliOutput.js";
import { registerContractCommand } from "../../../src/cli/contractCommand.js";
import { createCliResultOutput } from "../../../src/cli/cliResultOutput.js";
@@ -135,56 +136,25 @@ it("reads a union value under the option the producer used", () => {
).toEqual({});
});
it("parses a union branch only when the value's keys cannot select it", () => {
it("reads declared controls without inspecting unrelated captured values", () => {
const limit = deadlineParameterLimit("worker_timeout_ms", 5, 1_000);
const spy = <Schema extends z.ZodType>(schema: Schema) => ({
schema,
// Spy through the base type: a generic `Schema` hides its method keys.
parse: vi.spyOn<z.ZodType, "safeParse">(schema, "safeParse"),
});
// A uniquely selected branch and a branch without controls are not parsed.
const view = spy(
z.strictObject({
kind: z.literal("view"),
limit: analysisParameterLimitSchema,
}),
);
const complete = spy(
z.strictObject({ limit: analysisParameterLimitSchema, total: z.number() }),
);
const plain = spy(z.strictObject({ count: z.number() }));
const other = spy(z.strictObject({ name: z.string() }));
const extra = {
get count(): number {
throw new Error("Control collection visited an unrelated captured value");
},
};
const schema = z.strictObject({
result: z.union([view.schema, complete.schema]),
extra: z.union([plain.schema, other.schema]).optional(),
result: z.strictObject({ limit: analysisParameterLimitSchema }),
extra: z.union([
z.strictObject({ count: z.literal(1) }),
z.strictObject({ timestamp: z.string() }),
]),
});
expect(
cliResultControlsIn(
{ result: { limit, total: 1 }, extra: { count: 1 } },
schema,
binding,
),
).toEqual({ worker_timeout_ms: "--worker-timeout-ms" });
for (const branch of [view, complete, plain, other])
expect(branch.parse).not.toHaveBeenCalled();
// Branches the keys cannot tell apart keep first-match order.
const captured = spy(
z.strictObject({ note: z.strictObject({ limit: z.unknown() }) }),
);
const declared = spy(
z.strictObject({
note: z.strictObject({ limit: analysisParameterLimitSchema }),
}),
);
expect(
cliResultControlsIn(
{ note: { limit } },
z.union([captured.schema, declared.schema]),
binding,
),
).toEqual({});
expect(captured.parse).toHaveBeenCalledTimes(1);
expect(declared.parse).not.toHaveBeenCalled();
cliResultControlsIn({ result: { limit }, extra }, schema, binding),
).toEqual({
worker_timeout_ms: "--worker-timeout-ms",
});
});
it("finds a control a recursive schema reaches only past a back-reference", () => {
@@ -247,13 +217,6 @@ const runFixtureCommand = async (produce: () => unknown) => {
});
};
it("records the controls of a contract command's result and returns the result unchanged", async () => {
const { output, controls } = await runFixtureCommand(partialResult);
expect(output).toEqual(partialResult());
// Only controls this command takes as options are rendered.
expect(controls).toEqual({ worker_timeout_ms: "--worker-timeout-ms" });
});
it("reads a failure's partial Evidence of the same operation at the declared positions only", async () => {
const failure = (operation: string) => ({
code: "timeout",
@@ -342,19 +305,18 @@ it.each(["yaml", "toon"] as const)(
["--format", format, "--full-output"],
cliControls,
);
expect(text).toBe(
`${formatCliOutput(
{
ok: true,
data: partialResult(),
meta: {
command: "fixture",
duration: "0ms",
cli_controls: cliControls,
},
},
format,
)}\n`,
);
const decoded = format === "yaml" ? parseYaml(text) : decodeToon(text);
expect(decoded).toEqual({
ok: true,
data: {
...partialResult(),
...(format === "toon" ? { trailing: null } : {}),
},
meta: {
command: "fixture",
duration: "0ms",
cli_controls: cliControls,
},
});
},
);
@@ -1,22 +1,14 @@
import { expect, it } from "vitest";
import {
AnalysisInputError,
AnalysisResourceConstraintError,
reattributedResourceConstraint,
} from "../../../src/domain/analysisErrorCore.js";
import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js";
import { analysisErrorWithCleanupFailure } from "../../../src/domain/analysisErrorCleanup.js";
import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js";
import { freezeJsonSnapshot } from "../../../src/domain/immutableJson.js";
import { ownedJsonObjectSchema } from "../../../src/domain/jsonValue.js";
import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js";
import { err, ok } from "../../../src/domain/result.js";
import { createCli } from "../../../src/cli.js";
import { cliCommand, cliCommands } from "../../../src/cli/commandRegistry.js";
import { BinaryTargetError } from "../../../src/domain/configurationErrors.js";
import { ArtifactOperationError } from "../../../src/domain/artifactOperationError.js";
import {
projectCliAnalysisResult,
projectCliCommandOutput,
projectCliInputFields,
} from "../../../src/cli/projectCliAnalysisResult.js";
@@ -27,7 +19,7 @@ const binding = {
flags: {},
};
it("projects bound input fields without changing canonical facts or nested identities", () => {
it("projects bound input fields without changing canonical facts or nested paths", () => {
const original = freezeJsonSnapshot(
projectAnalysisError(
new AnalysisInputError("inspect_keyed_archive", undefined, [
@@ -59,17 +51,6 @@ it("projects bound input fields without changing canonical facts or nested ident
{ path: ["path", "path", 0] },
{ path: ["root"] },
]);
const details = ownedJsonObjectSchema.parse(
ownedJsonObjectSchema.parse(mapped).details,
);
const mappedIssues = details.issues;
const originalIssues = original.details?.issues;
if (!Array.isArray(mappedIssues) || !Array.isArray(originalIssues))
throw new Error("Expected projected input issues");
expect(mappedIssues[1]).toBe(originalIssues[1]);
expect(ownedJsonObjectSchema.parse(mappedIssues[0]).observed).toBe(
ownedJsonObjectSchema.parse(originalIssues[0]).observed,
);
});
it("keeps cleanup history and retained output while binding the actionable primary issue", () => {
@@ -115,30 +96,23 @@ it("keeps cleanup history and retained output while binding the actionable prima
issues: [{ path: ["archive"], reason: "invalid_value" }],
},
});
const details = ownedJsonObjectSchema.parse(
ownedJsonObjectSchema.parse(mapped).details,
);
expect(details.diagnostics).toBe(original.details?.diagnostics);
expect(details.captured_output).toBe(original.details?.captured_output);
});
it("passes through successful data, unbound operations and unknown error values", () => {
const value = { path: "caller-selected data" };
expect(projectCliInputFields(value, binding)).toBe(value);
expect(projectCliInputFields(value, binding)).toEqual(value);
const targetError = projectAnalysisError(
new AnalysisInputError("open_binary", undefined, [
{ path: ["path"], reason: "invalid_format" },
]),
);
expect(projectCliInputFields(targetError, binding)).toBe(targetError);
expect(projectCliInputFields(targetError, binding)).toEqual(targetError);
const unbound = projectAnalysisError(
new AnalysisInputError("inspect_keyed_archive", undefined, [
{ path: ["root"], reason: "invalid_value" },
]),
);
expect(projectCliInputFields(unbound, binding)).toBe(unbound);
expect(projectCliAnalysisResult(err(targetError))).toBe(targetError);
expect(projectCliAnalysisResult(ok(value))).toBe(value);
expect(projectCliInputFields(unbound, binding)).toEqual(unbound);
const unknown = {
code: "future_failure",
details: {
@@ -146,47 +120,7 @@ it("passes through successful data, unbound operations and unknown error values"
issues: [{ path: ["path"] }],
},
};
expect(projectCliInputFields(unknown, binding)).toBe(unknown);
});
it("names related fields and operations by contract identity and renders only this command's flags", () => {
const constraint = new AnalysisResourceConstraintError(
"inspect_sqlite_database",
"memory",
"fixture limit",
null,
{
related: {
parameters: ["columns", "table", "row_limit"],
operations: ["inspect_artifact"],
},
},
);
// Re-attribution keeps the inputs the recovery names, as it keeps `limit`.
const projected = projectAnalysisError(
reattributedResourceConstraint("inspect_sqlite_database", constraint),
);
expect(projected.details).toMatchObject({
related_parameters: ["columns", "table", "row_limit"],
related_operations: ["inspect_artifact"],
});
const sqlite = {
operation: "inspect_sqlite_database",
fields: {},
flags: { columns: "--column", table: "--table" },
};
// A field the command takes no flag for is left to the contract name.
expect(projectCliInputFields(projected, sqlite)).toMatchObject({
details: { related_cli_flags: { columns: "--column", table: "--table" } },
});
// Another command's failure keeps its fields for that command to render.
expect(
projectCliInputFields(projected, { ...sqlite, operation: "inspect_plist" }),
).toBe(projected);
// An error that names nothing projects no related fields.
expect(
projectAnalysisError(new AnalysisInputError("inspect_plist")).details ?? {},
).not.toHaveProperty("related_parameters");
expect(projectCliInputFields(unknown, binding)).toEqual(unknown);
});
it("renders related_cli_flags and related_cli_commands from the CLI command registry across command shapes", () => {
@@ -202,35 +136,6 @@ it("renders related_cli_flags and related_cli_commands from the CLI command regi
)
throw new Error("Expected registered CLI commands");
// Open-path BinaryTargetError has no details.operation; flags come from the
// running command's open_binary bindings, while MCP projection stays flag-free.
const loadRejected = projectAnalysisError(
new BinaryTargetError("/tmp/image.bin", "unaligned entry", {
constraint: "load_selection_rejected",
}),
);
expect(loadRejected.details).toMatchObject({
related_parameters: ["format", "architecture", "entry_point"],
});
expect(loadRejected.details).not.toHaveProperty("related_cli_flags");
expect(JSON.stringify(loadRejected)).not.toContain("--target-format");
const cliLoadRejected = projectCliCommandOutput(
{ error: "Analysis failed", ...loadRejected },
{ operations: inspectEntry.operations, commands: entries },
);
expect(cliLoadRejected).toMatchObject({
error: "Analysis failed",
details: {
related_parameters: ["format", "architecture", "entry_point"],
related_cli_flags: {
format: "--target-format",
architecture: "--architecture",
entry_point: "--entry-point",
},
},
});
// Artifact integrity errors omit details.operation when artifactDetails are present.
const integrityFailure = projectAnalysisError(
new ArtifactOperationError("inspect_artifact", "integrity", {
@@ -293,30 +198,6 @@ it("renders related_cli_flags and related_cli_commands from the CLI command regi
},
});
// In one-shot mode where only the running registrar was loaded, recorded
// canonical commands from other registrars still resolve.
expect(
projectCliCommandOutput(
projectAnalysisError(
new BinaryTargetError("/tmp/module.wasm", "wasm module", {
constraint: "wasm_requires_static_inspection",
}),
),
{
operations: inspectEntry.operations,
commands: [inspectEntry],
recordedCommands: { "inspect-wasm-artifact": "wasm" },
},
),
).toMatchObject({
details: {
related_operations: ["inspect_wasm_artifact"],
related_cli_commands: {
inspect_wasm_artifact: "rea inspect-wasm-artifact",
},
},
});
// Non-contract host commands that bind contract operations render flags too.
const compareFailure = projectAnalysisError(
new AnalysisInputError("compare_bundles", {
@@ -334,3 +215,61 @@ it("renders related_cli_flags and related_cli_commands from the CLI command regi
},
});
});
it("names a flattened member flag or the JSON flag for its containing group", () => {
const cli = createCli();
const commands = cliCommands(cli);
const project = (command: string, limit: Record<string, string>) => {
const entry = cliCommand(cli, command);
if (entry === undefined) throw new Error(`Missing command ${command}`);
return projectCliCommandOutput(
{ code: "resource_constraint", details: { limit } },
{ operations: entry.operations, commands },
).details.limit;
};
expect(
project("inspect-web-network-capture", {
control_kind: "parameter",
parameter: "protobuf.schema.message_type",
}),
).toEqual({
control_kind: "parameter",
parameter: "protobuf.schema.message_type",
cli_flag: "--protobuf-message",
});
expect(
project("inspect-binary-layout", {
setting: "pwntools.max_input_bytes",
parameter: "producer_limits.input_bytes",
}),
).toEqual({
setting: "pwntools.max_input_bytes",
parameter: "producer_limits.input_bytes",
cli_flag: "--producer-limits",
});
const unavailable = {
control_kind: "parameter",
parameter: "source_map_limits.fetch_bytes",
};
expect(project("trace-web-source-location", unavailable)).toEqual(
unavailable,
);
const sourceLocation = cliCommand(cli, "trace-web-source-location");
if (sourceLocation === undefined)
throw new Error("Missing source location command");
const data = {
details: {
limit: {
control_kind: "parameter",
parameter: "source_map_limits.decoded_records",
},
},
};
expect(
projectCliCommandOutput(data, {
operations: sourceLocation.operations,
commands,
}),
).toEqual(data);
});