test(ida): consolidate advertised observation and effect checks (#13008)

* test(ida): consolidate advertised observation and effect checks

* test(ida): preserve empty-name and instruction-body contracts
This commit is contained in:
morluto authored and GitHub committed 2026-11-21 00:59:22 -05:00
1 parent d93a199629
commit 9bb06b29ca
4 files changed
+157 -186

No files matched your search

+29 -63
View File
@@ -1,7 +1,6 @@
import { expect, it } from "vitest";
import { RecordingIdaMcp } from "../../tests/fixtures/idaMcp.js";
import { IdaOperationRunner } from "./IdaOperationRunner.js";
import type { JsonValue } from "../domain/jsonValue.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
const target: BinaryTarget = {
@@ -60,71 +59,38 @@ it("rejects file coordinates before any symbol lookup", async () => {
});
});
it("retains usable function identities and raw spans without inventing body ranges", async () => {
it("projects the native xref source function and retains unused span evidence", async () => {
const producer = new RecordingIdaMcp(target, "attached");
const runner = new IdaOperationRunner(producer, "native");
const identity = { address: "0x1000", name: "main" };
const spans: Record<string, JsonValue>[] = [
{},
{ size: null },
{ size: { bytes: 32 } },
{ size: "0x20" },
];
for (const span of spans) {
const fn = { addr: "0x1000", name: "main", ...span };
const inventory = [{ data: [fn], next_offset: null }];
producer.overrideCall = (tool) => {
switch (tool) {
case "list_funcs":
return inventory;
case "lookup_funcs":
return [{ query: "main", fn, error: null }];
case "xrefs_to":
return [
{
xrefs: [{ addr: "0x1004", type: "code", fn }],
more: false,
error: null,
},
];
default:
return undefined;
}
};
expect(await runner.run("list_procedures", {})).toEqual([
{ address: "0x1000", value: "main" },
]);
expect(await runner.resolve("main")).toEqual(identity);
expect(await runner.xrefs("0x1000")).toEqual([
{ address: "0x1004", type: "code", function: identity },
]);
expect(
await runner.run("read_function_instructions", { procedure: "main" }),
).toMatchObject({
procedure: {
...identity,
body: {
available: false,
reason: expect.stringContaining("complete function body ranges"),
const response = [
{
xrefs: [
{
addr: "0x1004",
type: "code",
fn: { addr: "0x1000", name: "main", size: { bytes: 32 } },
},
},
});
expect(runner.raw).toContainEqual(
expect.objectContaining({ tool: "list_funcs", result: inventory }),
);
}
for (const fn of [
{ address: "invalid", name: "main" },
{ address: "0x1000", name: "" },
]) {
producer.overrideCall = (tool) =>
tool === "lookup_funcs"
? [{ query: "main", fn, error: null }]
: undefined;
await expect(runner.resolve("main")).rejects.toThrow();
}
],
more: false,
error: null,
},
];
producer.overrideCall = (tool) =>
tool === "xrefs_to" ? response : undefined;
expect(await runner.xrefs("0x1000")).toEqual([
{
address: "0x1004",
type: "code",
function: { address: "0x1000", name: "main" },
},
]);
expect(runner.raw).toEqual([
{
tool: "xrefs_to",
arguments: { addrs: ["0x1000"], limit: 1000 },
result: response,
},
]);
});
it.each([
@@ -110,7 +110,8 @@ describe.each(["attached", "headless"] as const)(
it.each<IdentityCase>([
{ label: "address", fields: { address: "invalid", addr: "invalid" } },
{ label: "name", fields: { name: null } },
{ label: "null name", fields: { name: null } },
{ label: "empty name", fields: { name: "" } },
])("rejects malformed $label despite unused size", async ({ fields }) => {
const fixture = await session(mode, {
...fields,
@@ -147,3 +148,30 @@ describe.each(["attached", "headless"] as const)(
});
},
);
it("preserves unknown native instruction body bounds despite an upstream size", async () => {
const { client, lookupObservation } = await session("attached", {
size: { bytes: 32 },
});
try {
const result = await client.execute("read_function_instructions", {
procedure: "main",
});
if (!result.ok) throw result.error;
expect(result.value.result).toMatchObject({
procedure: {
address: "0x1000",
name: "main",
body: {
available: false,
reason: expect.stringContaining("complete function body ranges"),
},
},
});
expect(observation(result.value.rawResult, "lookup_funcs")).toEqual(
lookupObservation,
);
} finally {
expect((await client.close()).ok).toBe(true);
}
});
@@ -99,72 +99,88 @@ it("keeps retryable IDA effects in composed discovery while preserving unavailab
});
it.each([
["native", false],
["native", true],
["modern", true],
] as const)(
"covers IDA provider effects in the global audit (%s, stdio=%s)",
(profileName, stdio) => {
for (const descriptor of idaCapabilities(
IDA_COMPATIBILITY_PROFILES[profileName],
stdio,
)) {
const audited = contractNamed(descriptor.operation)?.effects;
expect(audited, descriptor.operation).toBeDefined();
if (descriptor.effects.mayAccessNetwork)
expect(audited?.accessesNetwork, descriptor.operation).toBe(true);
if (descriptor.effects.mayShowUi)
expect(audited?.changesUiState, descriptor.operation).toBe(true);
if (descriptor.effects.launchesProcess)
expect(audited?.launchesProcess, descriptor.operation).toBe(true);
if (descriptor.effects.mayWriteFilesystem)
expect(audited?.writesFilesystem, descriptor.operation).toBe(true);
}
{
profile: "native",
stdio: false,
showUi: true,
launchesAndWrites: false,
},
);
{
profile: "native",
stdio: true,
showUi: true,
launchesAndWrites: true,
},
{
profile: "modern",
stdio: false,
showUi: false,
launchesAndWrites: true,
},
] as const)(
"audits and projects IDA effects for $profile (stdio=$stdio)",
({ profile, stdio, showUi, launchesAndWrites }) => {
const descriptors = idaCapabilities(
IDA_COMPATIBILITY_PROFILES[profile],
stdio,
);
for (const { operation, effects } of descriptors) {
expect(contractNamed(operation)?.effects, operation).toMatchObject({
accessesNetwork: true,
...(effects.mayShowUi ? { changesUiState: true } : {}),
...(effects.launchesProcess ? { launchesProcess: true } : {}),
...(effects.mayWriteFilesystem ? { writesFilesystem: true } : {}),
});
}
it.each([false, true])(
"retains selected provider UI/process/network effects in direct and composed discovery (UI=%s)",
(showUi) => {
const input = {
open: true,
kind: "executable",
capabilities: providerCapabilitySets(
idaCapabilities(
IDA_COMPATIBILITY_PROFILES[showUi ? "native" : "modern"],
false,
),
),
};
const entries = buildCapabilityInventory(input, enabledPolicy);
for (const name of [
"procedure_pseudo_code",
"analyze_function",
"batch_decompile",
"decompile_all",
"trace_native_values",
"inspect_native_api",
]) {
expect(entries.find((entry) => entry.name === name)).toMatchObject({
effects: {
changesUiState: showUi,
accessesNetwork: true,
},
annotations: { open_world: true },
});
}
for (const name of [
"procedure_assembly",
"read_function_instructions",
"list_procedures",
"search_strings",
"trace_feature",
]) {
expect(entries.find((entry) => entry.name === name)).toMatchObject({
effects: { changesUiState: false, accessesNetwork: true },
annotations: { open_world: true },
});
const entries = buildCapabilityInventory(
{
open: true,
kind: "executable",
capabilities: providerCapabilitySets(descriptors),
},
enabledPolicy,
);
for (const [names, changesUiState] of [
[
[
"procedure_pseudo_code",
"analyze_function",
"batch_decompile",
"decompile_all",
"trace_native_values",
"inspect_native_api",
],
showUi,
],
[
[
"procedure_assembly",
"read_function_instructions",
"list_procedures",
"search_strings",
"trace_feature",
],
false,
],
] as const) {
for (const name of names)
expect(
entries.find((entry) => entry.name === name),
name,
).toMatchObject({
effects: { changesUiState, accessesNetwork: true },
annotations: { open_world: true },
});
}
expect(
entries.find(({ name }) => name === "procedure_assembly"),
).toMatchObject({
effects: {
launchesProcess: launchesAndWrites,
writesFilesystem: launchesAndWrites,
},
});
},
);
@@ -310,28 +326,3 @@ it("includes reachable IDA document and procedure effects in address-context dis
annotations: { open_world: false },
});
});
it("keeps native attached UI effects limited to the operations that show them", () => {
const input = {
open: true,
kind: "executable",
capabilities: providerCapabilitySets(
idaCapabilities(IDA_COMPATIBILITY_PROFILES.native, true),
),
};
const entries = buildCapabilityInventory(input, enabledPolicy);
for (const [name, showUi] of [
["procedure_pseudo_code", true],
["analyze_function", true],
["procedure_assembly", false],
["read_function_instructions", false],
] as const) {
expect(
entries.find((entry) => entry.name === name),
name,
).toMatchObject({
effects: { changesUiState: showUi, accessesNetwork: true },
annotations: { open_world: true },
});
}
});
+20 -34
View File
@@ -5,8 +5,7 @@ import { afterEach, describe, expect, it } from "vitest";
import { createIdaTarget, RecordingIdaMcp } from "../../fixtures/idaMcp.js";
import { IdaSessionClient } from "../../../src/ida/IdaSessionClient.js";
import { idaCapabilities } from "../../../src/ida/IdaProviderCapabilities.js";
import { IDA_OPERATIONS } from "../../../src/ida/IdaProviderMetadata.js";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js";
import { createEvidence } from "../../../src/domain/evidence.js";
import type { JsonValue } from "../../../src/domain/jsonValue.js";
import { createToolResultDelivery } from "../../../src/server/toolResult.js";
@@ -32,26 +31,14 @@ const parametersFor = (
return { procedure: "main" };
};
/** Operations only the headless profile declares. */
const HEADLESS_ONLY: ReadonlySet<string> = new Set([
"resolve_containing_procedure",
"list_documents",
]);
describe.each(["attached", "headless"] as const)(
"IDA %s caller-visible contracts",
(mode) => {
const operations = IDA_OPERATIONS.filter(
(operation) => mode === "headless" || !HEADLESS_ONLY.has(operation),
);
it("includes the baseline operations needed by this contract matrix", () => {
expect(operations).toContain("list_procedures");
if (mode === "headless")
expect(operations).toContain("resolve_containing_procedure");
});
it.each(operations)(
"serializes %s against its named advertised output contract",
async (operation) => {
const profile =
IDA_COMPATIBILITY_PROFILES[mode === "attached" ? "native" : "modern"];
it.each(idaCapabilities(profile, true))(
"checks the advertised $operation output contract and UI effect",
async ({ operation, available, effects }) => {
const { root, target } = await createIdaTarget();
roots.push(root);
const producer = new RecordingIdaMcp(target, mode);
@@ -61,7 +48,7 @@ describe.each(["attached", "headless"] as const)(
args: [],
env: {},
mode,
protocol: mode === "attached" ? "native" : "modern",
protocol: profile.protocol,
workspaceRoot: root,
timeoutMs: 1000,
},
@@ -71,24 +58,19 @@ describe.each(["attached", "headless"] as const)(
try {
const parameters = parametersFor(operation);
const execution = await client.execute(operation, parameters);
const descriptor = idaCapabilities(
IDA_COMPATIBILITY_PROFILES[
mode === "attached" ? "native" : "modern"
],
true,
).find((capability) => capability.operation === operation);
expect(descriptor?.effects.mayShowUi).toBe(
expect(effects.mayShowUi).toBe(
producer.calls.some(({ name }) => guiNavigatingTool(mode, name)),
);
if (operation === "procedure_callers") {
expect(execution.ok).toBe(false);
if (!execution.ok)
expect(execution.error._tag).toBe(
"AnalysisCapabilityUnavailableError",
);
if (!available) {
expect(execution).toMatchObject({
ok: false,
error: { _tag: "AnalysisCapabilityUnavailableError", operation },
});
return;
}
if (!execution.ok) throw execution.error;
if (operation === "list_documents")
expect(execution.value.result).toEqual([target.path]);
const evidence = createEvidence(target, execution.value.provider, {
operation,
parameters,
@@ -97,7 +79,11 @@ describe.each(["attached", "headless"] as const)(
locations: execution.value.locations,
limitations: execution.value.limitations,
});
const contract = toolContract(operation);
const contract = TOOL_CONTRACTS.find(
({ name }) => name === operation,
);
if (contract === undefined)
throw new Error(`Missing tool contract for ${operation}`);
const serialized = delivery.toEvidenceToolResult(
evidence,
contract,