fix(comparison): validate function result projections

Validate canonical dimension ownership, summary counts, and exact changed or unknown projections before investigation consumers publish derived Evidence. Reuse the shared dimension contract in selectors and keep historical source fixtures internally consistent.

Verify authentic malformed comparisons across all three MCP consumers while preserving valid reordered and unknown results and retained source Evidence.
This commit is contained in:
morluto committed 2026-10-23 15:54:55 +08:00
1 parent 02c8532901
commit 5f4e39fea2
5 files changed
+338 -37

No files matched your search

+52 -2
View File
@@ -1,5 +1,6 @@
import { z } from "zod";
import { overallStatus } from "./functionComparisonResults.js";
import { overallStatus, summarize } from "./functionComparisonResults.js";
import { canonicalJson } from "./comparisonSemantics.js";
import { evidenceInputSchema } from "./evidenceReference.js";
import { digestSchema } from "./../domain/digests.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
@@ -91,11 +92,60 @@ export const functionComparisonResultSchema = z
changed: z.int().min(0),
unknown: z.int().min(0),
}),
dimensions: z.array(functionDimensionSchema).length(8),
dimensions: z
.array(functionDimensionSchema)
.length(functionComparisonDimensionNameSchema.options.length),
changes: z.array(functionDimensionSchema),
limitations: z.array(z.string()),
})
.superRefine((result, context) => {
const dimensions = new Map<DimensionName, FunctionDimension>();
for (const [index, dimension] of result.dimensions.entries()) {
if (dimensions.has(dimension.dimension))
context.addIssue({
code: "custom",
path: ["dimensions", index, "dimension"],
message: "Function comparison dimensions must occur exactly once.",
});
dimensions.set(dimension.dimension, dimension);
}
const expectedSummary = summarize(result.dimensions);
for (const status of comparisonStatusSchema.options)
if (result.summary[status] !== expectedSummary[status])
context.addIssue({
code: "custom",
path: ["summary", status],
message: "Function comparison summary must count its dimensions.",
});
const changes = new Set<DimensionName>();
for (const [index, change] of result.changes.entries()) {
const dimension = dimensions.get(change.dimension);
if (
changes.has(change.dimension) ||
dimension === undefined ||
change.status === "unchanged" ||
canonicalJson(change) !== canonicalJson(dimension)
)
context.addIssue({
code: "custom",
path: ["changes", index],
message:
"Function comparison changes must cite each changed or unknown dimension exactly as recorded.",
});
changes.add(change.dimension);
}
if (
result.dimensions.some(
({ dimension, status }) =>
status !== "unchanged" && !changes.has(dimension),
)
)
context.addIssue({
code: "custom",
path: ["changes"],
message:
"Function comparison changes must include every changed or unknown dimension.",
});
if (
result.status !==
overallStatus(result.dimensions, result.function_match.status)
@@ -120,23 +120,27 @@ const comparison = (
target?: EvidenceSubjectTarget;
} = {},
resultInput = baselineResult,
): Evidence =>
createEvidence(identity.target, identity.provider ?? provider, {
): Evidence => {
const links = sources.map(({ evidence_id: id }) => id);
const dimensions = resultInput.dimensions.map((dimension) => ({
...dimension,
evidence_links: links,
}));
return createEvidence(identity.target, identity.provider ?? provider, {
predicateType: "rea.function-comparison",
operation: "compare_functions",
parameters,
result: jsonValueSchema.parse({
...resultInput,
dimensions: resultInput.dimensions.map((dimension) => ({
...dimension,
evidence_links: sources.map(({ evidence_id: id }) => id),
})),
dimensions,
changes: dimensions.filter(({ status }) => status !== "unchanged"),
}),
confidence: "derived",
authority: "analyst-inference",
evidenceLinks: sources.map(({ evidence_id: id }) => id),
evidenceLinks: links,
...observation,
});
};
const specification = (evidence: Evidence, dimension = "pseudocode") => ({
name: "Function reconstruction",
@@ -4,6 +4,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js";
import { digestSchema } from "./../domain/digests.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js";
import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const unknownIdSchema = prefixedDigestSchema("unk");
@@ -26,14 +27,7 @@ const functionClaimSchema = z.object({
kind: z.literal("structural-function"),
dimension: z.enum([
"overall",
"identity",
"pseudocode",
"assembly",
"comments",
"calls",
"references",
"strings_names",
"cfg",
...functionComparisonDimensionNameSchema.options,
]),
});
const artifactClaimSchema = z.object({
+11 -20
View File
@@ -14,7 +14,10 @@ import {
} from "./comparisonSourceEvidence.js";
import { evidenceSchema, type Evidence } from "./evidence.js";
import { hasComparisonEvidenceIdentity } from "./comparisonEvidenceIdentity.js";
import { functionComparisonResultSchema } from "./functionComparisonSchemas.js";
import {
functionComparisonDimensionNameSchema,
functionComparisonResultSchema,
} from "./functionComparisonSchemas.js";
import type { JsonValue } from "./jsonValue.js";
import {
comparisonStatusSchema,
@@ -24,16 +27,6 @@ import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const functionDimensionSchema = z.enum([
"identity",
"pseudocode",
"assembly",
"comments",
"calls",
"references",
"strings_names",
"cfg",
]);
const runtimeDimensionSchema = z.enum(PROCESS_COMPARISON_DIMENSIONS);
const expectedPatternSchema = z.enum([
"cochanged",
@@ -54,7 +47,10 @@ const artifactSelectorSchema = z
})
.strict();
const functionSelectorSchema = z
.object({ kind: z.literal("function"), dimension: functionDimensionSchema })
.object({
kind: z.literal("function"),
dimension: functionComparisonDimensionNameSchema,
})
.strict();
const staticSelectorSchema = z.discriminatedUnion("kind", [
artifactSelectorSchema,
@@ -333,16 +329,11 @@ const selectStatic = (
"Function selector requires function comparison Evidence",
);
const result = comparison.result;
const matches = result.dimensions.filter(
const match = result.dimensions.find(
({ dimension }) => dimension === selector.dimension,
);
const match = matches.at(0);
if (matches.length !== 1 || match === undefined)
throw callerInputError(
input,
path,
"Function selector must resolve exactly once",
);
if (match === undefined)
throw evidenceIntegrityError("Function comparison dimension missing");
return {
status: match.status,
conclusion_kind: match.conclusion_kind,
@@ -0,0 +1,262 @@
import { expect, it, onTestFinished } from "vitest";
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
import {
FUNCTION_COMPARISON_EVIDENCE,
INVESTIGATION_EXAMPLES,
PROCESS_CAPTURE_REFERENCE,
PROCESS_CAPTURE_RECONSTRUCTION,
PROCESS_COMPARISON_EVIDENCE,
} from "../../../src/contracts/investigationExamples.js";
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
import {
functionComparisonResultSchema,
type FunctionComparisonResult,
} from "../../../src/domain/functionComparisonSchemas.js";
import { summarize } from "../../../src/domain/functionComparisonResults.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
const conflictingDimensions = (result: FunctionComparisonResult) => {
const changed = result.dimensions.find(
(dimension) => dimension.dimension === "pseudocode",
);
if (changed?.status !== "changed")
throw new Error("Fixture requires a known pseudocode change");
const dimensions = result.dimensions.map((dimension) =>
dimension.dimension === "pseudocode"
? {
...changed,
status: "unchanged" as const,
conclusion_kind: "derived_relationship" as const,
}
: dimension.dimension === "cfg"
? changed
: dimension,
);
return {
...result,
dimensions,
summary: summarize(dimensions),
changes: dimensions.filter(({ status }) => status !== "unchanged"),
};
};
const unknownDimension = (result: FunctionComparisonResult) => {
const dimensions = result.dimensions.map((dimension) =>
dimension.dimension === "cfg"
? {
...dimension,
status: "unknown" as const,
left_digest: null,
right_digest: null,
text_delta: null,
conclusion_kind: "unresolved_branch" as const,
}
: dimension,
);
return {
...result,
dimensions,
summary: summarize(dimensions),
changes: dimensions.filter(({ status }) => status !== "unchanged"),
};
};
it.each([
{
name: "canonical",
valid: true,
result: (value: FunctionComparisonResult) => value,
},
{
name: "reordered canonical",
valid: true,
result: (value: FunctionComparisonResult) => ({
...value,
dimensions: [...value.dimensions].reverse(),
changes: [...value.changes].reverse(),
}),
},
{
name: "unknown dimension with complete changes",
valid: true,
result: unknownDimension,
},
{
name: "omitted unknown dimension with known changes retained",
valid: false,
result: (value: FunctionComparisonResult) => {
const unknown = unknownDimension(value);
return {
...unknown,
changes: unknown.changes.filter(({ dimension }) => dimension !== "cfg"),
};
},
},
{
name: "conflicting dimensions",
valid: false,
result: conflictingDimensions,
},
{
name: "reversed conflicting dimensions",
valid: false,
result: (value: FunctionComparisonResult) => {
const malformed = conflictingDimensions(value);
return {
...malformed,
dimensions: [...malformed.dimensions].reverse(),
changes: [...malformed.changes].reverse(),
};
},
},
{
name: "incorrect summary counts with the correct total",
valid: false,
result: (value: FunctionComparisonResult) => ({
...value,
summary: {
...value.summary,
changed: value.summary.changed - 1,
unchanged: value.summary.unchanged + 1,
},
}),
},
{
name: "missing changes",
valid: false,
result: (value: FunctionComparisonResult) => ({ ...value, changes: [] }),
},
{
name: "altered change provenance",
valid: false,
result: (value: FunctionComparisonResult) => ({
...value,
changes: value.changes.map((dimension, index) =>
index === 0
? {
...dimension,
limitations: [
...dimension.limitations,
"Unattributed change description.",
],
}
: dimension,
),
}),
},
{
name: "duplicate changes",
valid: false,
result: (value: FunctionComparisonResult) => ({
...value,
changes: [...value.changes, ...value.changes],
}),
},
])(
"validates $name function comparisons before investigation publication",
async (scenario) => {
const base = FUNCTION_COMPARISON_EVIDENCE;
const result = functionComparisonResultSchema.parse(base.normalized_result);
const comparison = createEvidence(undefined, base.provider, {
predicateType: base.predicate_type,
operation: base.operation,
parameters: base.parameters,
result: jsonValueSchema.parse(scenario.result(result)),
confidence: base.confidence,
authority: base.authority,
limitations: base.limitations,
evidenceLinks: base.evidence_links,
});
let providerLaunches = 0;
const session = createTestBinarySession(() => {
providerLaunches += 1;
throw new Error(
"Retained comparison inspection must not launch a provider",
);
});
const harness = await createApplicationMcpHarness({ session });
onTestFinished(harness.close);
for (const record of [
FUNCTION_COMPARISON_EXAMPLE.left,
FUNCTION_COMPARISON_EXAMPLE.right,
PROCESS_CAPTURE_REFERENCE,
PROCESS_CAPTURE_RECONSTRUCTION,
PROCESS_COMPARISON_EVIDENCE,
comparison,
])
expect(session.recordEvidence(record).ok).toBe(true);
const recordCount = session.exportEvidenceBundle().records.length;
const correlation = INVESTIGATION_EXAMPLES.correlate_static_and_runtime;
const calls = [
{
name: "find_changed_behavior",
arguments: { comparisons: [comparison] },
},
{
name: "correlate_static_and_runtime",
arguments: {
...correlation,
static_comparisons: [comparison],
mappings: correlation.mappings.map((mapping) => ({
...mapping,
static: {
...mapping.static,
comparison_evidence_id: comparison.evidence_id,
},
})),
},
},
{
name: "verify_reconstruction",
arguments: {
specification: {
name: "Function comparison consistency",
claims: [
{
kind: "structural-function",
claim_id: "pseudocode",
title: "Selected pseudocode remains equivalent",
comparison_evidence_id: comparison.evidence_id,
dimension: "pseudocode",
},
],
},
},
},
];
const responses = [];
for (const call of calls)
responses.push({ call, response: await harness.client.callTool(call) });
expect(
Object.fromEntries(
responses.map(({ call, response }) => [
call.name,
response.isError === true,
]),
),
).toEqual(
Object.fromEntries(calls.map(({ name }) => [name, !scenario.valid])),
);
for (const { call, response } of responses) {
if (scenario.valid)
expect(parseEvidence(response.structuredContent).operation).toBe(
call.name,
);
else
expect(parseMcpToolError(response)).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
category: "integrity_mismatch",
},
});
}
if (!scenario.valid)
expect(session.exportEvidenceBundle().records).toHaveLength(recordCount);
expect(session.evidenceById(comparison.evidence_id)).toEqual(comparison);
expect(providerLaunches).toBe(0);
},
);