From bb7bdd2fba4cbdd72e3c82df93fa64b7f0575799 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 20 Nov 2026 08:44:13 -0500 Subject: [PATCH] test(mcp): consolidate admission and lifecycle checks (#12833) * test(mcp): consolidate canonical batch capacity checks * test(mcp): keep separate cold-start prompt checks * test(mcp): consolidate byte entropy request checks * test(mcp): consolidate progress checks and remove waiting sleeps --- src/contracts/enhancedInputs.test.ts | 37 ---------- .../binaryDiagnosticsToolContracts.test.ts | 33 --------- src/server/DeferredEvidenceMcpServer.test.ts | 16 ++--- .../boundary/mcp/binaryDiagnosticsMcp.test.ts | 17 ++++- ....01-lists-the-complete-target-open.test.ts | 24 ++++++- .../enhancedTools.callPathsAndBatches.test.ts | 32 ++++++--- tests/boundary/mcp/progressMcp.test.ts | 69 ++++--------------- 7 files changed, 82 insertions(+), 146 deletions(-) delete mode 100644 src/contracts/enhancedInputs.test.ts delete mode 100644 src/contracts/native/binaryDiagnosticsToolContracts.test.ts diff --git a/src/contracts/enhancedInputs.test.ts b/src/contracts/enhancedInputs.test.ts deleted file mode 100644 index 6490fd94a3..0000000000 --- a/src/contracts/enhancedInputs.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Ajv2020 } from "ajv/dist/2020.js"; -import { describe, expect, it } from "vitest"; - -import { - enhancedInputSchemas, - MAX_ENHANCED_RECORDS, -} from "./enhancedInputs.js"; -import { advertisedInputJsonSchema } from "./toolSchemaMetadata.js"; -import { toolContract } from "./toolContracts.js"; - -describe("batch_decompile input contract", () => { - it("bounds addresses in the canonical and advertised schemas", () => { - const addresses = Array.from({ length: MAX_ENHANCED_RECORDS }, () => "0x0"); - const schema = enhancedInputSchemas.batch_decompile; - expect(schema.safeParse({ addresses }).success).toBe(true); - expect(schema.safeParse({ addresses: [...addresses, "0x1"] }).success).toBe( - false, - ); - - const advertised = advertisedInputJsonSchema( - toolContract("batch_decompile"), - ); - const advertisedAddresses = ( - advertised.properties as Record - ).addresses; - expect(advertisedAddresses).toMatchObject({ - type: "array", - maxItems: MAX_ENHANCED_RECORDS, - }); - - const ajv = new Ajv2020({ strict: false, validateFormats: false }); - expect(ajv.validateSchema(advertised)).toBe(true); - const validate = ajv.compile(advertised); - expect(validate({ addresses })).toBe(true); - expect(validate({ addresses: [...addresses, "0x1"] })).toBe(false); - }); -}); diff --git a/src/contracts/native/binaryDiagnosticsToolContracts.test.ts b/src/contracts/native/binaryDiagnosticsToolContracts.test.ts deleted file mode 100644 index 18a2ae9b06..0000000000 --- a/src/contracts/native/binaryDiagnosticsToolContracts.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { expect, it } from "vitest"; - -import { toolContract } from "../toolContracts.js"; - -const entropy = toolContract("inspect_byte_entropy"); - -it("warns that complete byte-entropy output can become very large", () => { - expect(entropy.description).toMatch( - /output.*large.*one row per selected block.*summary.*full list/iu, - ); - expect(entropy.description).toContain( - "Use a shorter length or a larger block_size", - ); -}); - -it("keeps the public entropy input closed after detail is added", () => { - expect(entropy.inputSchema.parse({ path: "/file" })).toMatchObject({ - path: "/file", - block_size: 4096, - offset: 0, - detail: "complete", - }); - expect( - entropy.inputSchema.safeParse({ - path: "/file", - detail: "summary", - extra: true, - }).success, - ).toBe(false); - expect( - entropy.inputSchema.safeParse({ path: "/file", block_size: 257 }).success, - ).toBe(false); -}); diff --git a/src/server/DeferredEvidenceMcpServer.test.ts b/src/server/DeferredEvidenceMcpServer.test.ts index f2f5aedffe..d619f5e3c5 100644 --- a/src/server/DeferredEvidenceMcpServer.test.ts +++ b/src/server/DeferredEvidenceMcpServer.test.ts @@ -354,8 +354,9 @@ it.each(["legacy", "auto"] as const)( }, ); -it("initializes through prompt retrieval or completion without an internal client", async () => { +it("completes a prompt argument as the first operation", async () => { const f = await fixture(); + expect(f.loads()).toBe(0); expect( ( await f.client.complete({ @@ -364,24 +365,21 @@ it("initializes through prompt retrieval or completion without an internal clien }) ).completion.values, ).toEqual(["Example"]); - expect( - ( - await f.client.getPrompt({ - name: "explain", - arguments: { document: "Example" }, - }) - ).messages[0]?.content, - ).toEqual({ type: "text", text: "Example" }); expect(f.loads()).toBe(1); }); it("retrieves a prompt as the first operation", async () => { const f = await fixture(); + expect(f.loads()).toBe(0); const result = await f.client.getPrompt({ name: "explain", arguments: { document: "Example" }, }); expect(result.messages).toHaveLength(1); + expect(result.messages[0]?.content).toEqual({ + type: "text", + text: "Example", + }); expect(f.loads()).toBe(1); }); diff --git a/tests/boundary/mcp/binaryDiagnosticsMcp.test.ts b/tests/boundary/mcp/binaryDiagnosticsMcp.test.ts index 9f4316b635..4f81f4d624 100644 --- a/tests/boundary/mcp/binaryDiagnosticsMcp.test.ts +++ b/tests/boundary/mcp/binaryDiagnosticsMcp.test.ts @@ -1,4 +1,4 @@ -import { createHash, randomBytes } from "node:crypto"; +import { createHash } from "node:crypto"; import { open, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -381,7 +381,7 @@ it("profiles real bytes with exact offsets, valid advertised schemas and read-on path, Buffer.concat([ Buffer.alloc(4096), - randomBytes(4096), + Buffer.from(Array.from({ length: 4096 }, (_, index) => index % 256)), Buffer.from("ordinary text ".repeat(400)), ]), ); @@ -402,6 +402,19 @@ it("profiles real bytes with exact offsets, valid advertised schemas and read-on InMemoryTransport.createLinkedPair(); await server.connect(serverTransport); await client.connect(clientTransport); + for (const [options, field, reason] of [ + [{ detail: "summary", surprise: true }, "surprise", "unknown_argument"], + [{ block_size: 257 }, "block_size", "invalid_value"], + ] as const) { + const rejected = await client.callTool({ + name: "inspect_byte_entropy", + arguments: { path, ...options }, + }); + expect(parseMcpToolError(rejected).error).toMatchObject({ + code: "invalid_request", + details: { issues: [{ path: [field], reason }] }, + }); + } const { ajv, inputSchema, toolDefinition } = await advertisedMcpToolSchemas( client, "inspect_byte_entropy", diff --git a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts index 828a28edff..a1c0f8bab4 100644 --- a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts +++ b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts @@ -1,5 +1,3 @@ -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { afterEach, describe, expect, it } from "vitest"; @@ -282,11 +280,31 @@ describe("enhanced MCP tools", () => { ["trace_native_values", { procedure: "0x1" }], ["decompile_all", { output_directory: join(outputParent, "out") }], ] as const; + const functionProgress: { + progress: number; + message?: string | undefined; + }[] = []; const results = await Promise.all( calls.map(async ([name, arguments_]) => - jsonResult(await client.callTool({ name, arguments: arguments_ })), + jsonResult( + await client.callTool( + { name, arguments: arguments_ }, + name === "analyze_function" + ? { onprogress: (update) => functionProgress.push(update) } + : undefined, + ), + ), ), ); + expect(functionProgress[0]?.message).toBe("analyze_function: started"); + expect(functionProgress.at(-1)?.message).toBe( + "analyze_function: completed", + ); + functionProgress.forEach(({ progress }, index) => { + expect(progress).toBeGreaterThan( + functionProgress[index - 1]?.progress ?? 0, + ); + }); expect(results[0]).toMatchObject({ count: 2, classes: [ diff --git a/tests/boundary/mcp/enhancedTools.callPathsAndBatches.test.ts b/tests/boundary/mcp/enhancedTools.callPathsAndBatches.test.ts index 35ed607e88..48d0fae259 100644 --- a/tests/boundary/mcp/enhancedTools.callPathsAndBatches.test.ts +++ b/tests/boundary/mcp/enhancedTools.callPathsAndBatches.test.ts @@ -1,3 +1,4 @@ +import { Ajv2020 } from "ajv/dist/2020.js"; import { canonicalInputSchema } from "../../fixtures/canonicalOutputSchema.js"; import { afterEach, describe, expect, it } from "vitest"; @@ -13,7 +14,10 @@ import { batchDecompileResultSchema } from "../../../src/domain/native/batchDeco import { ENHANCED_TOOL_CONTRACTS } from "../../../src/contracts/enhancedToolContracts.js"; import type { RelationshipCoverage } from "../../../src/domain/relationshipCoverage.js"; import { err, ok as resultOk } from "../../../src/domain/result.js"; -import { MAX_ENHANCED_RECORDS } from "../../../src/contracts/enhancedInputs.js"; +import { + enhancedInputSchemas, + MAX_ENHANCED_RECORDS, +} from "../../../src/contracts/enhancedInputs.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { @@ -473,17 +477,29 @@ it("advertises and enforces the batch input budget before provider calls", async ); const { tools } = await client.listTools(); expect(tools.some(({ name }) => name === "batch_decompile")).toBe(true); - // The budget is a canonical input constraint; the server enforces it - // whatever the compact advertisement retains. - expect(canonicalInputSchema("batch_decompile")).toMatchObject({ - properties: { addresses: { maxItems: MAX_ENHANCED_RECORDS } }, + // The compact wire presentation may widen the canonical array bound. + const canonical = canonicalInputSchema("batch_decompile"); + expect(canonical).toMatchObject({ + properties: { + addresses: { type: "array", maxItems: MAX_ENHANCED_RECORDS }, + }, }); + const addresses = Array.from({ length: MAX_ENHANCED_RECORDS }, () => "0x0"); + const overLimit = { addresses: [...addresses, "0x1"] }; + expect( + enhancedInputSchemas.batch_decompile.safeParse({ addresses }).success, + ).toBe(true); + const validate = new Ajv2020({ + strict: false, + validateFormats: false, + }).compile(canonical); + expect(validate({ addresses })).toBe(true); + expect(validate(overLimit)).toBe(false); + const result = await client.callTool({ name: "batch_decompile", - arguments: { - addresses: Array.from({ length: MAX_ENHANCED_RECORDS + 1 }, () => "0x0"), - }, + arguments: overLimit, }); expect(result.isError).toBe(true); diff --git a/tests/boundary/mcp/progressMcp.test.ts b/tests/boundary/mcp/progressMcp.test.ts index 4d8d981f84..4bf32c01bf 100644 --- a/tests/boundary/mcp/progressMcp.test.ts +++ b/tests/boundary/mcp/progressMcp.test.ts @@ -1,16 +1,12 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { setTimeout as delay } from "node:timers/promises"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, onTestFinished, vi } from "vitest"; import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; -import { - createAnalysisExecution, - type AnalysisOperationPort, -} from "../../../src/ports/AnalysisProvider.js"; +import type { AnalysisOperationPort } from "../../../src/ports/AnalysisProvider.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; -import { err, ok } from "../../../src/domain/result.js"; +import { err } from "../../../src/domain/result.js"; import { createServer } from "../../../src/server/createServer.js"; import { ToolResultDelivery } from "../../../src/server/ToolResultDelivery.js"; import { observed } from "../../fixtures/analysisExecution.js"; @@ -72,10 +68,15 @@ it.each(["current_address", "analyze_function"] as const)( describe("ordinary MCP progress and cancellation", () => { it("advances MCP progress when provider waiting does not advance completed work", async () => { + let now = 0; + const clock = vi.spyOn(performance, "now").mockImplementation(() => now); + onTestFinished(() => { + clock.mockRestore(); + }); const analysis: AnalysisOperationPort = { execute: async (_operation, _parameters, options) => { for (const message of ["import pending", "analysis pending"]) { - await delay(120); + now += 120; await options?.progress?.report({ phase: "ghidra_startup", completed: 0, @@ -104,14 +105,12 @@ describe("ordinary MCP progress and cancellation", () => { ); expect(result.isError).not.toBe(true); - expect(progress[0]?.progress).toBe(1); - expect(progress.map((update) => update.message)).toEqual( - expect.arrayContaining([ - "ghidra_startup: import pending", - "ghidra_startup: analysis pending", - ]), - ); - expect(progress.at(-1)?.message).toContain("completed"); + expect(progress.map(({ message }) => message)).toEqual([ + "current_address: started", + "ghidra_startup: import pending", + "ghidra_startup: analysis pending", + "current_address: completed", + ]); progress.forEach((update, index) => { expect(update.progress).toBeGreaterThan( progress[index - 1]?.progress ?? 0, @@ -121,44 +120,6 @@ describe("ordinary MCP progress and cancellation", () => { }, [() => client.close(), () => server.close()]); }); - it("keeps analyze_function progress strictly increasing from start to completion", async () => { - const analysis: AnalysisOperationPort = { - execute: (operation) => - Promise.resolve( - ok( - createAnalysisExecution( - operation === "analyze_function" - ? FUNCTION_COMPARISON_EXAMPLE.left.normalized_result - : "0x1000", - { id: "fixture", name: "Fixture", version: "1" }, - ), - ), - ), - }; - const server = createServer({ kind: "fixed", analysis }); - const client = testClient("function-progress-test"); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - const progress: number[] = []; - await withVerifierCleanup(async () => { - await server.connect(serverTransport); - await client.connect(clientTransport); - const result = await client.callTool( - { - name: "analyze_function", - arguments: { procedure: "main", detail: "complete" }, - }, - { onprogress: (update) => progress.push(update.progress) }, - ); - - expect(result.isError).not.toBe(true); - expect(progress.length).toBeGreaterThanOrEqual(2); - progress.forEach((value, index) => { - expect(value).toBeGreaterThan(progress[index - 1] ?? 0); - }); - }, [() => client.close(), () => server.close()]); - }); - it("propagates client cancellation to the active provider request", async () => { let providerObservedCancellation = false; let providerWasStarted = false;