From 9bb06b29caa081d3aed924135df8a5910c33b306 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:59:22 -0500 Subject: [PATCH] test(ida): consolidate advertised observation and effect checks (#13008) * test(ida): consolidate advertised observation and effect checks * test(ida): preserve empty-name and instruction-body contracts --- src/ida/IdaOperationRunner.test.ts | 92 +++------- .../providers/ida/functionIdentity.test.ts | 30 +++- .../ida/idaCapabilityInventory.test.ts | 167 +++++++++--------- .../conformance/ida/providerContracts.test.ts | 54 +++--- 4 files changed, 157 insertions(+), 186 deletions(-) diff --git a/src/ida/IdaOperationRunner.test.ts b/src/ida/IdaOperationRunner.test.ts index d5ed39b8bd..535069cc86 100644 --- a/src/ida/IdaOperationRunner.test.ts +++ b/src/ida/IdaOperationRunner.test.ts @@ -1,7 +1,6 @@ import { expect, it } from "vitest"; import { RecordingIdaMcp } from "../../tests/fixtures/idaMcp.js"; import { IdaOperationRunner } from "./IdaOperationRunner.js"; -import type { JsonValue } from "../domain/jsonValue.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; const target: BinaryTarget = { @@ -60,71 +59,38 @@ it("rejects file coordinates before any symbol lookup", async () => { }); }); -it("retains usable function identities and raw spans without inventing body ranges", async () => { +it("projects the native xref source function and retains unused span evidence", async () => { const producer = new RecordingIdaMcp(target, "attached"); const runner = new IdaOperationRunner(producer, "native"); - const identity = { address: "0x1000", name: "main" }; - const spans: Record[] = [ - {}, - { size: null }, - { size: { bytes: 32 } }, - { size: "0x20" }, - ]; - for (const span of spans) { - const fn = { addr: "0x1000", name: "main", ...span }; - const inventory = [{ data: [fn], next_offset: null }]; - producer.overrideCall = (tool) => { - switch (tool) { - case "list_funcs": - return inventory; - case "lookup_funcs": - return [{ query: "main", fn, error: null }]; - case "xrefs_to": - return [ - { - xrefs: [{ addr: "0x1004", type: "code", fn }], - more: false, - error: null, - }, - ]; - default: - return undefined; - } - }; - - expect(await runner.run("list_procedures", {})).toEqual([ - { address: "0x1000", value: "main" }, - ]); - expect(await runner.resolve("main")).toEqual(identity); - expect(await runner.xrefs("0x1000")).toEqual([ - { address: "0x1004", type: "code", function: identity }, - ]); - expect( - await runner.run("read_function_instructions", { procedure: "main" }), - ).toMatchObject({ - procedure: { - ...identity, - body: { - available: false, - reason: expect.stringContaining("complete function body ranges"), + const response = [ + { + xrefs: [ + { + addr: "0x1004", + type: "code", + fn: { addr: "0x1000", name: "main", size: { bytes: 32 } }, }, - }, - }); - expect(runner.raw).toContainEqual( - expect.objectContaining({ tool: "list_funcs", result: inventory }), - ); - } - - for (const fn of [ - { address: "invalid", name: "main" }, - { address: "0x1000", name: "" }, - ]) { - producer.overrideCall = (tool) => - tool === "lookup_funcs" - ? [{ query: "main", fn, error: null }] - : undefined; - await expect(runner.resolve("main")).rejects.toThrow(); - } + ], + more: false, + error: null, + }, + ]; + producer.overrideCall = (tool) => + tool === "xrefs_to" ? response : undefined; + expect(await runner.xrefs("0x1000")).toEqual([ + { + address: "0x1004", + type: "code", + function: { address: "0x1000", name: "main" }, + }, + ]); + expect(runner.raw).toEqual([ + { + tool: "xrefs_to", + arguments: { addrs: ["0x1000"], limit: 1000 }, + result: response, + }, + ]); }); it.each([ diff --git a/tests/boundary/providers/ida/functionIdentity.test.ts b/tests/boundary/providers/ida/functionIdentity.test.ts index 8c56ced29a..7645286abf 100644 --- a/tests/boundary/providers/ida/functionIdentity.test.ts +++ b/tests/boundary/providers/ida/functionIdentity.test.ts @@ -110,7 +110,8 @@ describe.each(["attached", "headless"] as const)( it.each([ { label: "address", fields: { address: "invalid", addr: "invalid" } }, - { label: "name", fields: { name: null } }, + { label: "null name", fields: { name: null } }, + { label: "empty name", fields: { name: "" } }, ])("rejects malformed $label despite unused size", async ({ fields }) => { const fixture = await session(mode, { ...fields, @@ -147,3 +148,30 @@ describe.each(["attached", "headless"] as const)( }); }, ); + +it("preserves unknown native instruction body bounds despite an upstream size", async () => { + const { client, lookupObservation } = await session("attached", { + size: { bytes: 32 }, + }); + try { + const result = await client.execute("read_function_instructions", { + procedure: "main", + }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + procedure: { + address: "0x1000", + name: "main", + body: { + available: false, + reason: expect.stringContaining("complete function body ranges"), + }, + }, + }); + expect(observation(result.value.rawResult, "lookup_funcs")).toEqual( + lookupObservation, + ); + } finally { + expect((await client.close()).ok).toBe(true); + } +}); diff --git a/tests/boundary/providers/ida/idaCapabilityInventory.test.ts b/tests/boundary/providers/ida/idaCapabilityInventory.test.ts index 945fd99ba3..c4a97e7165 100644 --- a/tests/boundary/providers/ida/idaCapabilityInventory.test.ts +++ b/tests/boundary/providers/ida/idaCapabilityInventory.test.ts @@ -99,72 +99,88 @@ it("keeps retryable IDA effects in composed discovery while preserving unavailab }); it.each([ - ["native", false], - ["native", true], - ["modern", true], -] as const)( - "covers IDA provider effects in the global audit (%s, stdio=%s)", - (profileName, stdio) => { - for (const descriptor of idaCapabilities( - IDA_COMPATIBILITY_PROFILES[profileName], - stdio, - )) { - const audited = contractNamed(descriptor.operation)?.effects; - expect(audited, descriptor.operation).toBeDefined(); - if (descriptor.effects.mayAccessNetwork) - expect(audited?.accessesNetwork, descriptor.operation).toBe(true); - if (descriptor.effects.mayShowUi) - expect(audited?.changesUiState, descriptor.operation).toBe(true); - if (descriptor.effects.launchesProcess) - expect(audited?.launchesProcess, descriptor.operation).toBe(true); - if (descriptor.effects.mayWriteFilesystem) - expect(audited?.writesFilesystem, descriptor.operation).toBe(true); - } + { + profile: "native", + stdio: false, + showUi: true, + launchesAndWrites: false, }, -); + { + profile: "native", + stdio: true, + showUi: true, + launchesAndWrites: true, + }, + { + profile: "modern", + stdio: false, + showUi: false, + launchesAndWrites: true, + }, +] as const)( + "audits and projects IDA effects for $profile (stdio=$stdio)", + ({ profile, stdio, showUi, launchesAndWrites }) => { + const descriptors = idaCapabilities( + IDA_COMPATIBILITY_PROFILES[profile], + stdio, + ); + for (const { operation, effects } of descriptors) { + expect(contractNamed(operation)?.effects, operation).toMatchObject({ + accessesNetwork: true, + ...(effects.mayShowUi ? { changesUiState: true } : {}), + ...(effects.launchesProcess ? { launchesProcess: true } : {}), + ...(effects.mayWriteFilesystem ? { writesFilesystem: true } : {}), + }); + } -it.each([false, true])( - "retains selected provider UI/process/network effects in direct and composed discovery (UI=%s)", - (showUi) => { - const input = { - open: true, - kind: "executable", - capabilities: providerCapabilitySets( - idaCapabilities( - IDA_COMPATIBILITY_PROFILES[showUi ? "native" : "modern"], - false, - ), - ), - }; - const entries = buildCapabilityInventory(input, enabledPolicy); - for (const name of [ - "procedure_pseudo_code", - "analyze_function", - "batch_decompile", - "decompile_all", - "trace_native_values", - "inspect_native_api", - ]) { - expect(entries.find((entry) => entry.name === name)).toMatchObject({ - effects: { - changesUiState: showUi, - accessesNetwork: true, - }, - annotations: { open_world: true }, - }); - } - for (const name of [ - "procedure_assembly", - "read_function_instructions", - "list_procedures", - "search_strings", - "trace_feature", - ]) { - expect(entries.find((entry) => entry.name === name)).toMatchObject({ - effects: { changesUiState: false, accessesNetwork: true }, - annotations: { open_world: true }, - }); + const entries = buildCapabilityInventory( + { + open: true, + kind: "executable", + capabilities: providerCapabilitySets(descriptors), + }, + enabledPolicy, + ); + for (const [names, changesUiState] of [ + [ + [ + "procedure_pseudo_code", + "analyze_function", + "batch_decompile", + "decompile_all", + "trace_native_values", + "inspect_native_api", + ], + showUi, + ], + [ + [ + "procedure_assembly", + "read_function_instructions", + "list_procedures", + "search_strings", + "trace_feature", + ], + false, + ], + ] as const) { + for (const name of names) + expect( + entries.find((entry) => entry.name === name), + name, + ).toMatchObject({ + effects: { changesUiState, accessesNetwork: true }, + annotations: { open_world: true }, + }); } + expect( + entries.find(({ name }) => name === "procedure_assembly"), + ).toMatchObject({ + effects: { + launchesProcess: launchesAndWrites, + writesFilesystem: launchesAndWrites, + }, + }); }, ); @@ -310,28 +326,3 @@ it("includes reachable IDA document and procedure effects in address-context dis annotations: { open_world: false }, }); }); - -it("keeps native attached UI effects limited to the operations that show them", () => { - const input = { - open: true, - kind: "executable", - capabilities: providerCapabilitySets( - idaCapabilities(IDA_COMPATIBILITY_PROFILES.native, true), - ), - }; - const entries = buildCapabilityInventory(input, enabledPolicy); - for (const [name, showUi] of [ - ["procedure_pseudo_code", true], - ["analyze_function", true], - ["procedure_assembly", false], - ["read_function_instructions", false], - ] as const) { - expect( - entries.find((entry) => entry.name === name), - name, - ).toMatchObject({ - effects: { changesUiState: showUi, accessesNetwork: true }, - annotations: { open_world: true }, - }); - } -}); diff --git a/tests/conformance/ida/providerContracts.test.ts b/tests/conformance/ida/providerContracts.test.ts index de9ff7e6b7..a798602f2c 100644 --- a/tests/conformance/ida/providerContracts.test.ts +++ b/tests/conformance/ida/providerContracts.test.ts @@ -5,8 +5,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { createIdaTarget, RecordingIdaMcp } from "../../fixtures/idaMcp.js"; import { IdaSessionClient } from "../../../src/ida/IdaSessionClient.js"; import { idaCapabilities } from "../../../src/ida/IdaProviderCapabilities.js"; -import { IDA_OPERATIONS } from "../../../src/ida/IdaProviderMetadata.js"; -import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; import { createEvidence } from "../../../src/domain/evidence.js"; import type { JsonValue } from "../../../src/domain/jsonValue.js"; import { createToolResultDelivery } from "../../../src/server/toolResult.js"; @@ -32,26 +31,14 @@ const parametersFor = ( return { procedure: "main" }; }; -/** Operations only the headless profile declares. */ -const HEADLESS_ONLY: ReadonlySet = new Set([ - "resolve_containing_procedure", - "list_documents", -]); - describe.each(["attached", "headless"] as const)( "IDA %s caller-visible contracts", (mode) => { - const operations = IDA_OPERATIONS.filter( - (operation) => mode === "headless" || !HEADLESS_ONLY.has(operation), - ); - it("includes the baseline operations needed by this contract matrix", () => { - expect(operations).toContain("list_procedures"); - if (mode === "headless") - expect(operations).toContain("resolve_containing_procedure"); - }); - it.each(operations)( - "serializes %s against its named advertised output contract", - async (operation) => { + const profile = + IDA_COMPATIBILITY_PROFILES[mode === "attached" ? "native" : "modern"]; + it.each(idaCapabilities(profile, true))( + "checks the advertised $operation output contract and UI effect", + async ({ operation, available, effects }) => { const { root, target } = await createIdaTarget(); roots.push(root); const producer = new RecordingIdaMcp(target, mode); @@ -61,7 +48,7 @@ describe.each(["attached", "headless"] as const)( args: [], env: {}, mode, - protocol: mode === "attached" ? "native" : "modern", + protocol: profile.protocol, workspaceRoot: root, timeoutMs: 1000, }, @@ -71,24 +58,19 @@ describe.each(["attached", "headless"] as const)( try { const parameters = parametersFor(operation); const execution = await client.execute(operation, parameters); - const descriptor = idaCapabilities( - IDA_COMPATIBILITY_PROFILES[ - mode === "attached" ? "native" : "modern" - ], - true, - ).find((capability) => capability.operation === operation); - expect(descriptor?.effects.mayShowUi).toBe( + expect(effects.mayShowUi).toBe( producer.calls.some(({ name }) => guiNavigatingTool(mode, name)), ); - if (operation === "procedure_callers") { - expect(execution.ok).toBe(false); - if (!execution.ok) - expect(execution.error._tag).toBe( - "AnalysisCapabilityUnavailableError", - ); + if (!available) { + expect(execution).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCapabilityUnavailableError", operation }, + }); return; } if (!execution.ok) throw execution.error; + if (operation === "list_documents") + expect(execution.value.result).toEqual([target.path]); const evidence = createEvidence(target, execution.value.provider, { operation, parameters, @@ -97,7 +79,11 @@ describe.each(["attached", "headless"] as const)( locations: execution.value.locations, limitations: execution.value.limitations, }); - const contract = toolContract(operation); + const contract = TOOL_CONTRACTS.find( + ({ name }) => name === operation, + ); + if (contract === undefined) + throw new Error(`Missing tool contract for ${operation}`); const serialized = delivery.toEvidenceToolResult( evidence, contract,