reverse-engineering-anything/tests/boundary/mcp/functionComparisonMcp.test.ts

import { functionSnapshotFixture } from "../../../src/domain/functionDossierEvidence.fixture.js";
import { ok as resultOk } from "../../../src/domain/result.js";
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { describe, expect, it, onTestFinished } from "vitest";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js";
import { compareFunctions } from "../../../src/domain/functionComparison.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";

import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
import { createServer } from "../../../src/server/createServer.js";
import { observed } from "../../fixtures/analysisExecution.js";
import { reconstructionVerificationResultSchema } from "../../../src/domain/reconstructionVerificationSchemas.js";

it.each([
  { comparisonStatus: "unchanged", verificationStatus: "pass" },
  { comparisonStatus: "changed", verificationStatus: "fail" },
] as const)(
  "verifies public $comparisonStatus function comparison as $verificationStatus",
  async ({ comparisonStatus, verificationStatus }) => {
    let providerLaunches = 0;
    const session = createTestBinarySession(() => {
      providerLaunches += 1;
      throw new Error("Function verification must not launch a provider");
    });
    const left = FUNCTION_COMPARISON_EXAMPLE.left;
    const right =
      comparisonStatus === "changed"
        ? FUNCTION_COMPARISON_EXAMPLE.right
        : createEvidence(
            {
              path: "/tmp/function-unchanged",
              sha256: "2".repeat(64),
              format: "mach-o",
            },
            left.provider,
            {
              operation: left.operation,
              parameters: left.parameters,
              result: left.normalized_result,
              confidence: left.confidence,
              authority: left.authority,
              ...(left.relationship_coverage === null
                ? {}
                : { relationshipCoverage: left.relationship_coverage }),
            },
          );
    const server = createServer({ kind: "session", session });
    const client = new Client({ name: "function-claims", version: "1" });
    const [clientTransport, serverTransport] =
      InMemoryTransport.createLinkedPair();
    try {
      await server.connect(serverTransport);
      await client.connect(clientTransport);
      const compared = await client.callTool({
        name: "compare_functions",
        arguments: { left, right },
      });
      expect(compared.isError).not.toBe(true);
      const comparison = parseEvidence(compared.structuredContent);
      expect(comparison.limitations).not.toContainEqual(
        expect.stringContaining(
          "without address or generated-name normalization",
        ),
      );
      expect(comparison.normalized_result).toMatchObject({
        status: comparisonStatus,
        dimensions: expect.arrayContaining(
          ["pseudocode", "assembly"].map((dimension) =>
            expect.objectContaining({
              dimension,
              limitations: expect.arrayContaining([
                expect.stringContaining(
                  "without address or generated-name normalization",
                ),
              ]),
            }),
          ),
        ),
      });
      expect(comparison.parameters).toEqual({
        left_evidence_id: left.evidence_id,
        right_evidence_id: right.evidence_id,
      });
      expect(comparison.evidence_links).toEqual(
        expect.arrayContaining([left.evidence_id, right.evidence_id]),
      );

      const verified = await client.callTool({
        name: "verify_reconstruction",
        arguments: {
          specification: {
            name: "Function pseudocode comparison",
            claims: ["pseudocode", "assembly", "cfg", "overall"].map(
              (dimension) => ({
                kind: "structural-function",
                claim_id: dimension,
                title: `${dimension} agrees`,
                comparison_evidence_id: comparison.evidence_id,
                dimension,
              }),
            ),
          },
        },
      });
      expect(
        verified.isError,
        JSON.stringify(verified.structuredContent),
      ).not.toBe(true);
      const verification = parseEvidence(verified.structuredContent);
      expect(verification.normalized_result).toMatchObject({
        status: verificationStatus,
        summary: {
          total: 4,
          passed: verificationStatus === "pass" ? 4 : 2,
          failed: verificationStatus === "fail" ? 2 : 0,
          unknown: 0,
        },
        claims: {
          items: expect.arrayContaining([
            expect.objectContaining({
              dimension: "pseudocode",
              status: verificationStatus,
              observed_status: comparisonStatus,
              comparison_evidence_id: comparison.evidence_id,
              left_evidence_ids: [left.evidence_id],
              right_evidence_ids: [right.evidence_id],
              limitations: expect.arrayContaining([
                expect.stringContaining(
                  "without address or generated-name normalization",
                ),
              ]),
            }),
          ]),
        },
      });
      const claims = reconstructionVerificationResultSchema.parse(
        verification.normalized_result,
      ).claims.items;
      for (const claim of claims) {
        const caveats = claim.limitations.join(" ");
        if (claim.dimension === "cfg")
          expect(caveats).not.toMatch(/generated-name normalization/u);
        else expect(caveats).toMatch(/generated-name normalization/u);
        if (claim.dimension === "assembly" || claim.dimension === "overall")
          expect(caveats).toMatch(/Assembly is opaque/u);
        else expect(caveats).not.toMatch(/Assembly is opaque/u);
      }
      expect(
        session
          .exportEvidenceBundle()
          .records.find(({ evidence_id: id }) => id === comparison.evidence_id),
      ).toEqual(comparison);
    } finally {
      await Promise.allSettled([
        client.close(),
        server.close(),
        session.close(),
      ]);
      expect(providerLaunches).toBe(0);
    }
  },
);

describe("function comparison MCP integration", () => {
  it("records linked comparison Evidence and its residual unknown", async () => {
    const session = createTestBinarySession(() => ({
      health: () => Promise.resolve(),
      execute: () => Promise.resolve(observed(null)),
      close: () => Promise.resolve(resultOk(null)),
    }));
    expect(session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.left).ok).toBe(
      true,
    );
    expect(session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.right).ok).toBe(
      true,
    );
    const server = createServer({ kind: "session", session });
    const client = new Client({
      name: "function-comparison-test",
      version: "1",
    });
    const [clientTransport, serverTransport] =
      InMemoryTransport.createLinkedPair();
    try {
      await server.connect(serverTransport);
      await client.connect(clientTransport);
      const result = await client.callTool({
        name: "compare_functions",
        arguments: {
          left: FUNCTION_COMPARISON_EXAMPLE.left,
          right: FUNCTION_COMPARISON_EXAMPLE.right,
        },
      });
      expect(result.isError).not.toBe(true);
      expect(result.structuredContent).toMatchObject({
        normalized_result: { status: expect.any(String) },
        evidence_id: expect.stringMatching(/^ev_[a-f0-9]{64}$/u),
      });
      const unknowns = await client.callTool({
        name: "list_unknowns",
        arguments: { domain: "function-comparison" },
      });
      expect(unknowns.structuredContent).toMatchObject({
        result: {
          items: [
            expect.objectContaining({
              domain: "function-comparison",
              recommended_probes: [
                expect.objectContaining({
                  rationale:
                    "Capture complete dossiers for both functions under the same target context.",
                }),
              ],
            }),
          ],
        },
      });
    } finally {
      await Promise.allSettled([
        client.close(),
        server.close(),
        session.close(),
      ]);
    }
  });

  it("accepts inline Evidence without prior session registration", async () => {
    const session = createTestBinarySession(() => ({
      health: () => Promise.resolve(),
      execute: () => Promise.resolve(observed(null)),
      close: () => Promise.resolve(resultOk(null)),
    }));
    const server = createServer({ kind: "session", session });
    const client = new Client({ name: "function-link-test", version: "1" });
    const [clientTransport, serverTransport] =
      InMemoryTransport.createLinkedPair();
    try {
      await server.connect(serverTransport);
      await client.connect(clientTransport);
      const result = await client.callTool({
        name: "compare_functions",
        arguments: {
          left: FUNCTION_COMPARISON_EXAMPLE.left,
          right: FUNCTION_COMPARISON_EXAMPLE.right,
        },
      });
      expect(result.isError).not.toBe(true);
      expect(result.structuredContent).toMatchObject({
        normalized_result: { status: expect.any(String) },
      });
    } finally {
      await Promise.allSettled([
        client.close(),
        server.close(),
        session.close(),
      ]);
    }
  });

  it("rejects inline Evidence with the wrong operation", async () => {
    const session = createTestBinarySession(() => ({
      health: () => Promise.resolve(),
      execute: () => Promise.resolve(observed(null)),
      close: () => Promise.resolve(resultOk(null)),
    }));
    const wrong = createEvidence(
      undefined,
      { id: "fixture", name: "Fixture", version: "1" },
      {
        operation: "inventory_artifact",
        parameters: {},
        result: {},
      },
    );
    session.recordEvidence(wrong);
    session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.right);
    const server = createServer({ kind: "session", session });
    const client = new Client({
      name: "function-authority-test",
      version: "1",
    });
    const [clientTransport, serverTransport] =
      InMemoryTransport.createLinkedPair();
    try {
      await server.connect(serverTransport);
      await client.connect(clientTransport);
      const result = await client.callTool({
        name: "compare_functions",
        arguments: {
          left: wrong,
          right: FUNCTION_COMPARISON_EXAMPLE.right,
        },
      });
      expect(result.isError).toBe(true);
      expect(session.exportEvidenceBundle().records).toHaveLength(2);
    } finally {
      await Promise.allSettled([
        client.close(),
        server.close(),
        session.close(),
      ]);
    }
  });
});

it.each([
  { status: "changed", expected: "fail" },
  { status: "unknown", expected: "unknown" },
  { status: "unchanged", expected: "pass" },
] as const)(
  "checks the aggregate status of imported $status function comparisons",
  async ({ status, expected }) => {
    const left = FUNCTION_COMPARISON_EXAMPLE.left;
    const right =
      status === "changed"
        ? FUNCTION_COMPARISON_EXAMPLE.right
        : createEvidence(
            {
              path: "/fixture/function-right",
              sha256: "3".repeat(64),
              format: "mach-o",
            },
            status === "unknown"
              ? { ...left.provider, id: "other-provider" }
              : left.provider,
            {
              operation: left.operation,
              parameters: left.parameters,
              result: left.normalized_result,
              confidence: left.confidence,
              authority: left.authority,
              relationshipCoverage: { status: "complete" },
            },
          );
    const result = compareFunctions(
      functionSnapshotFixture(left),
      functionSnapshotFixture(right),
    );
    expect(result.status).toBe(status);
    const provider = {
      id: "rea-function-comparison",
      name: "REA function comparison",
      version: "1",
    };
    const observation = {
      operation: "compare_functions",
      predicateType: "rea.function-comparison",
      parameters: {
        left_evidence_id: left.evidence_id,
        right_evidence_id: right.evidence_id,
      },
      confidence: "derived" as const,
      authority: "analyst-inference" as const,
      evidenceLinks: [left.evidence_id, right.evidence_id],
    };
    const consistent = createEvidence(undefined, provider, {
      ...observation,
      result: jsonValueSchema.parse(result),
    });
    const inconsistent =
      status === "unchanged"
        ? null
        : createEvidence(undefined, provider, {
            ...observation,
            result: jsonValueSchema.parse({ ...result, status: "unchanged" }),
          });
    const directory = await createTestTempDirectory("rea-function-status-");
    const path = join(directory, "bundle.json");
    await writeFile(
      path,
      JSON.stringify(
        createEvidenceBundle([
          left,
          right,
          consistent,
          ...(inconsistent === null ? [] : [inconsistent]),
        ]),
      ),
    );
    const session = createTestBinarySession(() => {
      throw new Error("Imported comparison must not launch a provider");
    });
    const harness = await createApplicationMcpHarness({ session });
    onTestFinished(harness.close);
    const imported = await harness.client.callTool({
      name: "import_evidence_bundle",
      arguments: { path },
    });
    expect(imported.isError).not.toBe(true);
    const verification = (evidence_id: string) =>
      harness.client.callTool({
        name: "verify_reconstruction",
        arguments: {
          specification: {
            name: "Function aggregate",
            claims: [
              {
                kind: "structural-function",
                claim_id: "overall",
                title: "Function agrees",
                comparison_evidence_id: evidence_id,
                dimension: "overall",
              },
            ],
          },
        },
      });
    if (inconsistent !== null) {
      const rejected = await verification(inconsistent.evidence_id);
      expect(parseMcpToolError(rejected).error).toMatchObject({
        code: "evidence_integrity_mismatch",
        details: {
          reason: expect.stringContaining(
            "Function comparison status contradicts its dimensions or function match",
          ),
        },
      });
      expect(harness.session.evidenceById(inconsistent.evidence_id)).toEqual(
        inconsistent,
      );
    }
    const verified = await verification(consistent.evidence_id);
    expect(verified.isError).not.toBe(true);
    expect(
      parseEvidence(verified.structuredContent).normalized_result,
    ).toMatchObject({ status: expected });
    await harness.client.ping();
  },
);