mirror of
https://github.com/morluto/rea.git
synced 2026-11-21 22:15:50 +00:00
test(ida): consolidate advertised observation and effect checks (#13008)
* test(ida): consolidate advertised observation and effect checks * test(ida): preserve empty-name and instruction-body contracts
This commit is contained in:
1 parent
d93a199629
commit
9bb06b29ca
4 files changed
+157
-186
No files matched your search
@@ -1,7 +1,6 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { RecordingIdaMcp } from "../../tests/fixtures/idaMcp.js";
|
||||
import { IdaOperationRunner } from "./IdaOperationRunner.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
|
||||
|
||||
const target: BinaryTarget = {
|
||||
@@ -60,71 +59,38 @@ it("rejects file coordinates before any symbol lookup", async () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("retains usable function identities and raw spans without inventing body ranges", async () => {
|
||||
it("projects the native xref source function and retains unused span evidence", async () => {
|
||||
const producer = new RecordingIdaMcp(target, "attached");
|
||||
const runner = new IdaOperationRunner(producer, "native");
|
||||
const identity = { address: "0x1000", name: "main" };
|
||||
const spans: Record<string, JsonValue>[] = [
|
||||
{},
|
||||
{ size: null },
|
||||
{ size: { bytes: 32 } },
|
||||
{ size: "0x20" },
|
||||
];
|
||||
for (const span of spans) {
|
||||
const fn = { addr: "0x1000", name: "main", ...span };
|
||||
const inventory = [{ data: [fn], next_offset: null }];
|
||||
producer.overrideCall = (tool) => {
|
||||
switch (tool) {
|
||||
case "list_funcs":
|
||||
return inventory;
|
||||
case "lookup_funcs":
|
||||
return [{ query: "main", fn, error: null }];
|
||||
case "xrefs_to":
|
||||
return [
|
||||
{
|
||||
xrefs: [{ addr: "0x1004", type: "code", fn }],
|
||||
more: false,
|
||||
error: null,
|
||||
},
|
||||
];
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
expect(await runner.run("list_procedures", {})).toEqual([
|
||||
{ address: "0x1000", value: "main" },
|
||||
]);
|
||||
expect(await runner.resolve("main")).toEqual(identity);
|
||||
expect(await runner.xrefs("0x1000")).toEqual([
|
||||
{ address: "0x1004", type: "code", function: identity },
|
||||
]);
|
||||
expect(
|
||||
await runner.run("read_function_instructions", { procedure: "main" }),
|
||||
).toMatchObject({
|
||||
procedure: {
|
||||
...identity,
|
||||
body: {
|
||||
available: false,
|
||||
reason: expect.stringContaining("complete function body ranges"),
|
||||
const response = [
|
||||
{
|
||||
xrefs: [
|
||||
{
|
||||
addr: "0x1004",
|
||||
type: "code",
|
||||
fn: { addr: "0x1000", name: "main", size: { bytes: 32 } },
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(runner.raw).toContainEqual(
|
||||
expect.objectContaining({ tool: "list_funcs", result: inventory }),
|
||||
);
|
||||
}
|
||||
|
||||
for (const fn of [
|
||||
{ address: "invalid", name: "main" },
|
||||
{ address: "0x1000", name: "" },
|
||||
]) {
|
||||
producer.overrideCall = (tool) =>
|
||||
tool === "lookup_funcs"
|
||||
? [{ query: "main", fn, error: null }]
|
||||
: undefined;
|
||||
await expect(runner.resolve("main")).rejects.toThrow();
|
||||
}
|
||||
],
|
||||
more: false,
|
||||
error: null,
|
||||
},
|
||||
];
|
||||
producer.overrideCall = (tool) =>
|
||||
tool === "xrefs_to" ? response : undefined;
|
||||
expect(await runner.xrefs("0x1000")).toEqual([
|
||||
{
|
||||
address: "0x1004",
|
||||
type: "code",
|
||||
function: { address: "0x1000", name: "main" },
|
||||
},
|
||||
]);
|
||||
expect(runner.raw).toEqual([
|
||||
{
|
||||
tool: "xrefs_to",
|
||||
arguments: { addrs: ["0x1000"], limit: 1000 },
|
||||
result: response,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -110,7 +110,8 @@ describe.each(["attached", "headless"] as const)(
|
||||
|
||||
it.each<IdentityCase>([
|
||||
{ label: "address", fields: { address: "invalid", addr: "invalid" } },
|
||||
{ label: "name", fields: { name: null } },
|
||||
{ label: "null name", fields: { name: null } },
|
||||
{ label: "empty name", fields: { name: "" } },
|
||||
])("rejects malformed $label despite unused size", async ({ fields }) => {
|
||||
const fixture = await session(mode, {
|
||||
...fields,
|
||||
@@ -147,3 +148,30 @@ describe.each(["attached", "headless"] as const)(
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves unknown native instruction body bounds despite an upstream size", async () => {
|
||||
const { client, lookupObservation } = await session("attached", {
|
||||
size: { bytes: 32 },
|
||||
});
|
||||
try {
|
||||
const result = await client.execute("read_function_instructions", {
|
||||
procedure: "main",
|
||||
});
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.result).toMatchObject({
|
||||
procedure: {
|
||||
address: "0x1000",
|
||||
name: "main",
|
||||
body: {
|
||||
available: false,
|
||||
reason: expect.stringContaining("complete function body ranges"),
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(observation(result.value.rawResult, "lookup_funcs")).toEqual(
|
||||
lookupObservation,
|
||||
);
|
||||
} finally {
|
||||
expect((await client.close()).ok).toBe(true);
|
||||
}
|
||||
});
|
||||
@@ -99,72 +99,88 @@ it("keeps retryable IDA effects in composed discovery while preserving unavailab
|
||||
});
|
||||
|
||||
it.each([
|
||||
["native", false],
|
||||
["native", true],
|
||||
["modern", true],
|
||||
] as const)(
|
||||
"covers IDA provider effects in the global audit (%s, stdio=%s)",
|
||||
(profileName, stdio) => {
|
||||
for (const descriptor of idaCapabilities(
|
||||
IDA_COMPATIBILITY_PROFILES[profileName],
|
||||
stdio,
|
||||
)) {
|
||||
const audited = contractNamed(descriptor.operation)?.effects;
|
||||
expect(audited, descriptor.operation).toBeDefined();
|
||||
if (descriptor.effects.mayAccessNetwork)
|
||||
expect(audited?.accessesNetwork, descriptor.operation).toBe(true);
|
||||
if (descriptor.effects.mayShowUi)
|
||||
expect(audited?.changesUiState, descriptor.operation).toBe(true);
|
||||
if (descriptor.effects.launchesProcess)
|
||||
expect(audited?.launchesProcess, descriptor.operation).toBe(true);
|
||||
if (descriptor.effects.mayWriteFilesystem)
|
||||
expect(audited?.writesFilesystem, descriptor.operation).toBe(true);
|
||||
}
|
||||
{
|
||||
profile: "native",
|
||||
stdio: false,
|
||||
showUi: true,
|
||||
launchesAndWrites: false,
|
||||
},
|
||||
);
|
||||
{
|
||||
profile: "native",
|
||||
stdio: true,
|
||||
showUi: true,
|
||||
launchesAndWrites: true,
|
||||
},
|
||||
{
|
||||
profile: "modern",
|
||||
stdio: false,
|
||||
showUi: false,
|
||||
launchesAndWrites: true,
|
||||
},
|
||||
] as const)(
|
||||
"audits and projects IDA effects for $profile (stdio=$stdio)",
|
||||
({ profile, stdio, showUi, launchesAndWrites }) => {
|
||||
const descriptors = idaCapabilities(
|
||||
IDA_COMPATIBILITY_PROFILES[profile],
|
||||
stdio,
|
||||
);
|
||||
for (const { operation, effects } of descriptors) {
|
||||
expect(contractNamed(operation)?.effects, operation).toMatchObject({
|
||||
accessesNetwork: true,
|
||||
...(effects.mayShowUi ? { changesUiState: true } : {}),
|
||||
...(effects.launchesProcess ? { launchesProcess: true } : {}),
|
||||
...(effects.mayWriteFilesystem ? { writesFilesystem: true } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
it.each([false, true])(
|
||||
"retains selected provider UI/process/network effects in direct and composed discovery (UI=%s)",
|
||||
(showUi) => {
|
||||
const input = {
|
||||
open: true,
|
||||
kind: "executable",
|
||||
capabilities: providerCapabilitySets(
|
||||
idaCapabilities(
|
||||
IDA_COMPATIBILITY_PROFILES[showUi ? "native" : "modern"],
|
||||
false,
|
||||
),
|
||||
),
|
||||
};
|
||||
const entries = buildCapabilityInventory(input, enabledPolicy);
|
||||
for (const name of [
|
||||
"procedure_pseudo_code",
|
||||
"analyze_function",
|
||||
"batch_decompile",
|
||||
"decompile_all",
|
||||
"trace_native_values",
|
||||
"inspect_native_api",
|
||||
]) {
|
||||
expect(entries.find((entry) => entry.name === name)).toMatchObject({
|
||||
effects: {
|
||||
changesUiState: showUi,
|
||||
accessesNetwork: true,
|
||||
},
|
||||
annotations: { open_world: true },
|
||||
});
|
||||
}
|
||||
for (const name of [
|
||||
"procedure_assembly",
|
||||
"read_function_instructions",
|
||||
"list_procedures",
|
||||
"search_strings",
|
||||
"trace_feature",
|
||||
]) {
|
||||
expect(entries.find((entry) => entry.name === name)).toMatchObject({
|
||||
effects: { changesUiState: false, accessesNetwork: true },
|
||||
annotations: { open_world: true },
|
||||
});
|
||||
const entries = buildCapabilityInventory(
|
||||
{
|
||||
open: true,
|
||||
kind: "executable",
|
||||
capabilities: providerCapabilitySets(descriptors),
|
||||
},
|
||||
enabledPolicy,
|
||||
);
|
||||
for (const [names, changesUiState] of [
|
||||
[
|
||||
[
|
||||
"procedure_pseudo_code",
|
||||
"analyze_function",
|
||||
"batch_decompile",
|
||||
"decompile_all",
|
||||
"trace_native_values",
|
||||
"inspect_native_api",
|
||||
],
|
||||
showUi,
|
||||
],
|
||||
[
|
||||
[
|
||||
"procedure_assembly",
|
||||
"read_function_instructions",
|
||||
"list_procedures",
|
||||
"search_strings",
|
||||
"trace_feature",
|
||||
],
|
||||
false,
|
||||
],
|
||||
] as const) {
|
||||
for (const name of names)
|
||||
expect(
|
||||
entries.find((entry) => entry.name === name),
|
||||
name,
|
||||
).toMatchObject({
|
||||
effects: { changesUiState, accessesNetwork: true },
|
||||
annotations: { open_world: true },
|
||||
});
|
||||
}
|
||||
expect(
|
||||
entries.find(({ name }) => name === "procedure_assembly"),
|
||||
).toMatchObject({
|
||||
effects: {
|
||||
launchesProcess: launchesAndWrites,
|
||||
writesFilesystem: launchesAndWrites,
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -310,28 +326,3 @@ it("includes reachable IDA document and procedure effects in address-context dis
|
||||
annotations: { open_world: false },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps native attached UI effects limited to the operations that show them", () => {
|
||||
const input = {
|
||||
open: true,
|
||||
kind: "executable",
|
||||
capabilities: providerCapabilitySets(
|
||||
idaCapabilities(IDA_COMPATIBILITY_PROFILES.native, true),
|
||||
),
|
||||
};
|
||||
const entries = buildCapabilityInventory(input, enabledPolicy);
|
||||
for (const [name, showUi] of [
|
||||
["procedure_pseudo_code", true],
|
||||
["analyze_function", true],
|
||||
["procedure_assembly", false],
|
||||
["read_function_instructions", false],
|
||||
] as const) {
|
||||
expect(
|
||||
entries.find((entry) => entry.name === name),
|
||||
name,
|
||||
).toMatchObject({
|
||||
effects: { changesUiState: showUi, accessesNetwork: true },
|
||||
annotations: { open_world: true },
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -5,8 +5,7 @@ import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createIdaTarget, RecordingIdaMcp } from "../../fixtures/idaMcp.js";
|
||||
import { IdaSessionClient } from "../../../src/ida/IdaSessionClient.js";
|
||||
import { idaCapabilities } from "../../../src/ida/IdaProviderCapabilities.js";
|
||||
import { IDA_OPERATIONS } from "../../../src/ida/IdaProviderMetadata.js";
|
||||
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
||||
import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js";
|
||||
import { createEvidence } from "../../../src/domain/evidence.js";
|
||||
import type { JsonValue } from "../../../src/domain/jsonValue.js";
|
||||
import { createToolResultDelivery } from "../../../src/server/toolResult.js";
|
||||
@@ -32,26 +31,14 @@ const parametersFor = (
|
||||
return { procedure: "main" };
|
||||
};
|
||||
|
||||
/** Operations only the headless profile declares. */
|
||||
const HEADLESS_ONLY: ReadonlySet<string> = new Set([
|
||||
"resolve_containing_procedure",
|
||||
"list_documents",
|
||||
]);
|
||||
|
||||
describe.each(["attached", "headless"] as const)(
|
||||
"IDA %s caller-visible contracts",
|
||||
(mode) => {
|
||||
const operations = IDA_OPERATIONS.filter(
|
||||
(operation) => mode === "headless" || !HEADLESS_ONLY.has(operation),
|
||||
);
|
||||
it("includes the baseline operations needed by this contract matrix", () => {
|
||||
expect(operations).toContain("list_procedures");
|
||||
if (mode === "headless")
|
||||
expect(operations).toContain("resolve_containing_procedure");
|
||||
});
|
||||
it.each(operations)(
|
||||
"serializes %s against its named advertised output contract",
|
||||
async (operation) => {
|
||||
const profile =
|
||||
IDA_COMPATIBILITY_PROFILES[mode === "attached" ? "native" : "modern"];
|
||||
it.each(idaCapabilities(profile, true))(
|
||||
"checks the advertised $operation output contract and UI effect",
|
||||
async ({ operation, available, effects }) => {
|
||||
const { root, target } = await createIdaTarget();
|
||||
roots.push(root);
|
||||
const producer = new RecordingIdaMcp(target, mode);
|
||||
@@ -61,7 +48,7 @@ describe.each(["attached", "headless"] as const)(
|
||||
args: [],
|
||||
env: {},
|
||||
mode,
|
||||
protocol: mode === "attached" ? "native" : "modern",
|
||||
protocol: profile.protocol,
|
||||
workspaceRoot: root,
|
||||
timeoutMs: 1000,
|
||||
},
|
||||
@@ -71,24 +58,19 @@ describe.each(["attached", "headless"] as const)(
|
||||
try {
|
||||
const parameters = parametersFor(operation);
|
||||
const execution = await client.execute(operation, parameters);
|
||||
const descriptor = idaCapabilities(
|
||||
IDA_COMPATIBILITY_PROFILES[
|
||||
mode === "attached" ? "native" : "modern"
|
||||
],
|
||||
true,
|
||||
).find((capability) => capability.operation === operation);
|
||||
expect(descriptor?.effects.mayShowUi).toBe(
|
||||
expect(effects.mayShowUi).toBe(
|
||||
producer.calls.some(({ name }) => guiNavigatingTool(mode, name)),
|
||||
);
|
||||
if (operation === "procedure_callers") {
|
||||
expect(execution.ok).toBe(false);
|
||||
if (!execution.ok)
|
||||
expect(execution.error._tag).toBe(
|
||||
"AnalysisCapabilityUnavailableError",
|
||||
);
|
||||
if (!available) {
|
||||
expect(execution).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "AnalysisCapabilityUnavailableError", operation },
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!execution.ok) throw execution.error;
|
||||
if (operation === "list_documents")
|
||||
expect(execution.value.result).toEqual([target.path]);
|
||||
const evidence = createEvidence(target, execution.value.provider, {
|
||||
operation,
|
||||
parameters,
|
||||
@@ -97,7 +79,11 @@ describe.each(["attached", "headless"] as const)(
|
||||
locations: execution.value.locations,
|
||||
limitations: execution.value.limitations,
|
||||
});
|
||||
const contract = toolContract(operation);
|
||||
const contract = TOOL_CONTRACTS.find(
|
||||
({ name }) => name === operation,
|
||||
);
|
||||
if (contract === undefined)
|
||||
throw new Error(`Missing tool contract for ${operation}`);
|
||||
const serialized = delivery.toEvidenceToolResult(
|
||||
evidence,
|
||||
contract,
|
||||
|
||||
Reference in new issue
Block a user