fix(evidence): preserve call coverage in function comparisons

Treat missing, partial, unknown, and unresolved call coverage as insufficient for call-set equality. Combine IDA caller/callee coverage without losing either explanation and recheck source coverage when verifying historical comparisons.

Scope generated function-comparison unknowns with optional typed comparison/dimension facts, preserving legacy IDs and revision digests when absent. Keep unrelated dimensions conclusive, explicit dependencies applicable, and ambiguous function identity unknown. A confirmed change for matched functions remains a failure even if calls are incomplete.

Validation: function/reconstruction/residual-unknown and bundle tests, recorded IDA protocol regression, public MCP comparison/verification, and advertised SDK JSON Schema checks. Real IDA provider execution was not verified.

Fixes #2222
This commit is contained in:
morluto committed 2026-10-13 04:12:34 +08:00
1 parent c89972c877
commit 049132ab43
17 files changed
+871 -63

No files matched your search

@@ -50,6 +50,7 @@ const observe = (digit: string, text: string) =>
result: dossier(text),
confidence: "derived",
authority: "shipped-artifact",
relationshipCoverage: { status: "complete" },
},
);
+78
View File
@@ -0,0 +1,78 @@
import type { RelationshipCoverage } from "./relationshipCoverage.js";
import type { FunctionDossier } from "./hopperValues.js";
import { uniqueSorted } from "./canonicalOrdering.js";
/** Combine caller and callee read outcomes without losing either explanation. */
export const combineFunctionRelationshipCoverage = (
callers: RelationshipCoverage | undefined,
callees: RelationshipCoverage | undefined,
): RelationshipCoverage => {
const reads = [
{ direction: "callers" as const, coverage: callers },
{ direction: "callees" as const, coverage: callees },
];
const diagnostics = uniqueSorted(
reads.flatMap(({ coverage }) =>
coverage !== undefined && "diagnostics" in coverage
? coverage.diagnostics
: [],
),
);
const details = diagnostics.length > 0 ? { diagnostics } : {};
const reasons = reads.flatMap(({ direction, coverage }) => {
const label = direction === "callers" ? "Callers" : "Callees";
const limitations =
coverage === undefined
? ["Provider did not report relationship coverage."]
: coverage.status === "unknown"
? [coverage.reason]
: coverage.status === "partial"
? coverage.limitations
: [];
return limitations.map((reason) => `${label}: ${reason}`);
});
if (
reads.some(
({ coverage }) => coverage === undefined || coverage.status === "unknown",
)
)
return {
status: "unknown",
reason: uniqueSorted(reasons).join(" "),
...details,
};
if (reasons.length > 0)
return {
status: "partial",
limitations: uniqueSorted(reasons),
...details,
};
return { status: "complete", ...details };
};
/** Explain why two resolved call sets cannot establish call-set equality. */
export const functionCallCoverageLimitations = (
coverage: RelationshipCoverage | null,
dossier: Pick<FunctionDossier, "unresolved_calls">,
): string[] => {
const limitations =
coverage === null
? ["Provider did not report relationship coverage."]
: coverage.status === "unknown"
? [
coverage.reason,
...("diagnostics" in coverage ? coverage.diagnostics : []),
]
: coverage.status === "partial"
? [
...coverage.limitations,
...("diagnostics" in coverage ? coverage.diagnostics : []),
]
: [];
limitations.push(
...(dossier.unresolved_calls ?? []).map(
({ address, reason }) => `Unresolved call at ${address}: ${reason}`,
),
);
return uniqueSorted(limitations);
};
+116
View File
@@ -11,6 +11,7 @@ import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { jsonValueSchema } from "./jsonValue.js";
import { compareFunctions } from "./functionComparison.js";
import type { RelationshipCoverage } from "./relationshipCoverage.js";
const procedure = (address: string, name: string) => ({
address,
name,
@@ -96,6 +97,7 @@ const observe = (
digit: string,
value: ReturnType<typeof dossier>,
providerId = "rea-workflow",
relationshipCoverage: RelationshipCoverage | null = { status: "complete" },
): Evidence =>
createEvidence(
{
@@ -112,6 +114,7 @@ const observe = (
result: jsonValueSchema.parse(value),
confidence: "derived",
authority: "shipped-artifact",
...(relationshipCoverage === null ? {} : { relationshipCoverage }),
},
);
@@ -187,6 +190,100 @@ describe("function comparison normalized identity", () => {
});
});
describe("function call relationship coverage", () => {
const coverageCases: {
readonly label: string;
readonly coverage: RelationshipCoverage | null;
readonly limitation: string;
}[] = [
{
label: "missing",
coverage: null,
limitation: "Provider did not report relationship coverage.",
},
{
label: "partial",
coverage: {
status: "partial",
limitations: ["Call graph stopped at the read limit."],
},
limitation: "Call graph stopped at the read limit.",
},
];
it.each(coverageCases)(
"keeps equal empty calls unknown for $label coverage",
({ coverage, limitation }) => {
const left = observe("a", dossier("return 0;", "0x1000"));
const right = observe(
"b",
dossier("return 0;", "0x2000"),
"rea-workflow",
coverage,
);
const result = compareFunctions(left, right);
const calls = result.dimensions.find(
({ dimension }) => dimension === "calls",
);
expect(calls).toMatchObject({
status: "unknown",
conclusion_kind: "unresolved_branch",
left_count: 0,
right_count: 0,
limitations: [`Right: ${limitation}`],
});
expect(result.status).toBe("unknown");
},
);
it("retains complete observations with execution diagnostics as comparable", () => {
const coverage = {
status: "complete" as const,
diagnostics: ["Provider reported a non-call xref warning."],
};
const left = observe(
"c",
dossier("return 0;", "0x1000"),
"rea-workflow",
coverage,
);
const right = observe(
"d",
dossier("return 0;", "0x2000"),
"rea-workflow",
coverage,
);
expect(
compareFunctions(left, right).dimensions.find(
({ dimension }) => dimension === "calls",
),
).toMatchObject({ status: "unchanged", left_count: 0, right_count: 0 });
});
it("keeps unresolved call sites unknown despite complete resolved-set coverage", () => {
const left = observe("e", dossier("return 0;", "0x1000"));
const rightDossier = functionDossierSchema.parse({
...dossier("return 0;", "0x2000"),
unresolved_calls: [
{
address: "0x2010",
reason: "Indirect dispatch target could not be resolved.",
},
],
});
const right = observe("f", rightDossier);
expect(
compareFunctions(left, right).dimensions.find(
({ dimension }) => dimension === "calls",
),
).toMatchObject({
status: "unknown",
limitations: [
"Right: Unresolved call at 0x2010: Indirect dispatch target could not be resolved.",
],
});
});
});
describe("function collection collation ties", () => {
const composed = procedure("0x2000", "caf\u00e9");
const decomposed = procedure("0x3000", "cafe\u0301");
@@ -342,6 +439,7 @@ describe("function comparison ordering controls", () => {
result: jsonValueSchema.parse(result),
confidence: "derived",
authority: "shipped-artifact",
relationshipCoverage: { status: "complete" },
},
);
const left = controlled("0x1000", false);
@@ -525,7 +623,9 @@ describe("function comparison", () => {
limitations: [expect.stringContaining("did not expose reference kinds")],
});
});
});
describe("function comparison identity and malformed CFG", () => {
it("does not promote address-derived names through equal signatures", () => {
const autoDossier = (base: "0x1000" | "0x2000") =>
functionDossierSchema.parse({
@@ -546,6 +646,22 @@ describe("function comparison", () => {
comparison.dimensions.find(({ dimension }) => dimension === "identity"),
).toMatchObject({ status: "unknown" });
expect(comparison.status).toBe("unknown");
const changedText = compareFunctions(
observe("b", autoDossier("0x1000")),
observe(
"c",
functionDossierSchema.parse({
...autoDossier("0x2000"),
pseudocode: "return 1;",
}),
),
);
expect(changedText.dimensions).toEqual(
expect.arrayContaining([
expect.objectContaining({ dimension: "pseudocode", status: "changed" }),
]),
);
expect(changedText.status).toBe("unknown");
});
it("rejects invalid CFG normalization and preserves newline differences", () => {
@@ -233,6 +233,18 @@ const compareCalls = (
): FunctionDimension => {
const leftValues = callProjection(left);
const rightValues = callProjection(right);
const coverageLimitations = [
...left.callCoverageLimitations.map((reason) => `Left: ${reason}`),
...right.callCoverageLimitations.map((reason) => `Right: ${reason}`),
];
if (coverageLimitations.length > 0)
return unresolvedDimension({
dimension: "calls",
links: context.links,
leftCount: leftValues.length,
rightCount: rightValues.length,
limitations: coverageLimitations,
});
if ([...leftValues, ...rightValues].some(({ name }) => isAutoName(name)))
return unresolvedDimension({
dimension: "calls",
+4 -8
View File
@@ -69,15 +69,11 @@ export const overallStatus = (
dimensions: readonly FunctionDimension[],
match: FunctionComparisonResult["function_match"]["status"],
): FunctionComparisonResult["status"] => {
if (
match === "ambiguous" ||
dimensions.some(({ status }) => status === "unknown")
)
return "unknown";
if (match === "mismatched") return "changed";
return dimensions.some(({ status }) => status === "changed")
? "changed"
: "unchanged";
if (match === "ambiguous") return "unknown";
if (dimensions.some(({ status }) => status === "changed")) return "changed";
if (dimensions.some(({ status }) => status === "unknown")) return "unknown";
return "unchanged";
};
export const summarize = (dimensions: readonly FunctionDimension[]) => ({
+5 -3
View File
@@ -4,7 +4,7 @@ import { digestSchema } from "./../domain/digests.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
const comparisonStatusSchema = z.enum(["unchanged", "changed", "unknown"]);
const dimensionNameSchema = z.enum([
export const functionComparisonDimensionNameSchema = z.enum([
"identity",
"pseudocode",
"assembly",
@@ -29,7 +29,7 @@ const textDeltaSchema = z.object({
});
const functionDimensionContextShape = {
dimension: dimensionNameSchema,
dimension: functionComparisonDimensionNameSchema,
left_count: z.number().int().min(0).nullable(),
right_count: z.number().int().min(0).nullable(),
evidence_links: z.array(evidenceIdSchema).min(2),
@@ -98,4 +98,6 @@ export type FunctionComparisonResult = z.infer<
typeof functionComparisonResultSchema
>;
export type FunctionDimension = z.infer<typeof functionDimensionSchema>;
export type DimensionName = z.infer<typeof dimensionNameSchema>;
export type DimensionName = z.infer<
typeof functionComparisonDimensionNameSchema
>;
+7
View File
@@ -1,6 +1,7 @@
import { parseEvidence, type Evidence } from "./evidence.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { functionDossierSchema, type FunctionDossier } from "./hopperValues.js";
import { functionCallCoverageLimitations } from "./functionCallCoverage.js";
/** Complete function observation parsed from one analyze_function Evidence record. */
export interface FunctionSnapshot {
@@ -8,6 +9,7 @@ export interface FunctionSnapshot {
readonly subject: NonNullable<Evidence["subject"]>;
readonly dossier: FunctionDossier;
readonly limitations: readonly string[];
readonly callCoverageLimitations: readonly string[];
}
/** Parse one complete analyze_function Evidence record for comparison workflows. */
@@ -20,6 +22,10 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => {
if (evidence.subject === null)
throw new TypeError("Function comparison requires artifact-bound Evidence");
const dossier = functionDossierSchema.parse(evidence.normalized_result);
const callCoverageLimitations = functionCallCoverageLimitations(
evidence.relationship_coverage,
dossier,
);
return {
evidence,
subject: evidence.subject,
@@ -27,5 +33,6 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => {
limitations: [
...new Set([...evidence.limitations, ...dossier.limitations]),
].sort((left, right) => compareUnicodeCodePoints(left, right)),
callCoverageLimitations,
};
};
+73
View File
@@ -0,0 +1,73 @@
import { expect, it } from "vitest";
import { createResidualUnknown } from "./residualUnknown.js";
import { reconstructionClaimUnknowns } from "./reconstructionUnknowns.js";
const ev = (digit: string) => `ev_${digit.repeat(64)}`;
const base = {
severity: "high" as const,
domain: "function-comparison",
supporting_evidence_ids: [ev("a"), ev("b")],
contradicting_evidence_ids: [],
required_authority: null,
required_confidence: "derived" as const,
required_environment: null,
recommended_probes: [],
};
it("keeps explicit dependencies while excluding unrelated scoped unknowns", () => {
const comparisonId = ev("a");
const other = createResidualUnknown(
{
...base,
question: "Other dimension",
comparison_conclusion_scope: {
comparison_evidence_id: comparisonId,
dimensions: ["pseudocode"],
},
relationships: [],
},
ev("c"),
null,
);
const calls = createResidualUnknown(
{
...base,
question: "Call dimension",
comparison_conclusion_scope: {
comparison_evidence_id: comparisonId,
dimensions: ["calls"],
},
relationships: [{ type: "depends-on", unknown_id: other.unknown_id }],
},
ev("d"),
null,
);
const unrelated = createResidualUnknown(
{
...base,
question: "Unrelated dimension",
comparison_conclusion_scope: {
comparison_evidence_id: comparisonId,
dimensions: ["pseudocode"],
},
relationships: [],
},
ev("e"),
null,
);
const selected = reconstructionClaimUnknowns(
new Map([
[calls.unknown_id, calls],
[other.unknown_id, other],
[unrelated.unknown_id, unrelated],
]),
comparisonId,
[ev("b")],
"calls",
);
expect(selected.map(({ unknown_id }) => unknown_id)).toEqual([
calls.unknown_id,
other.unknown_id,
]);
});
+13
View File
@@ -28,13 +28,26 @@ export const reconstructionClaimUnknowns = (
heads: ReadonlyMap<string, ResidualUnknown>,
comparisonId: string,
sourceIds: readonly string[],
dimension?: string,
): ResidualUnknown[] => {
const appliesToClaim = (item: ResidualUnknown): boolean => {
const scope = item.comparison_conclusion_scope;
return (
scope === undefined ||
(scope.comparison_evidence_id === comparisonId &&
(dimension === undefined ||
scope.dimensions.some(
(scopedDimension) => scopedDimension === dimension,
)))
);
};
const evidenceIds = new Set([comparisonId, ...sourceIds]);
const relevantIds = new Set(
[...heads.values()]
.filter(
(item) =>
item.status !== "resolved" &&
appliesToClaim(item) &&
[
...item.supporting_evidence_ids,
...item.contradicting_evidence_ids,
@@ -69,6 +69,7 @@ const source = (
result: dossier,
confidence: "derived",
authority: "shipped-artifact",
relationshipCoverage: { status: "complete" },
...observation,
},
);
@@ -99,14 +100,15 @@ const comparison = (
provider?: EvidenceProvider;
target?: EvidenceSubjectTarget;
} = {},
resultInput = baselineResult,
): Evidence =>
createEvidence(identity.target, identity.provider ?? provider, {
predicateType: "rea.function-comparison",
operation: "compare_functions",
parameters,
result: jsonValueSchema.parse({
...baselineResult,
dimensions: baselineResult.dimensions.map((dimension) => ({
...resultInput,
dimensions: resultInput.dimensions.map((dimension) => ({
...dimension,
evidence_links: sources.map(({ evidence_id: id }) => id),
})),
@@ -195,6 +197,100 @@ describe("function reconstruction source parameters", () => {
});
});
describe("function reconstruction call coverage", () => {
it("rechecks old unchanged comparisons without affecting unrelated finite dimensions", () => {
const incomplete = source("7", {
relationshipCoverage: {
status: "unknown",
reason: "Call graph unavailable.",
},
});
const incompleteSides = {
left_evidence_id: left.evidence_id,
right_evidence_id: incomplete.evidence_id,
};
const oldUnchanged = comparison(incompleteSides, {}, [left, incomplete]);
for (const dimension of ["calls", "overall"] as const) {
const result = admittedVerification(
oldUnchanged,
[left, incomplete],
dimension,
)();
expect(result).toMatchObject({
status: "unknown",
summary: { total: 1, passed: 0, failed: 0, unknown: 1 },
claims: {
items: [
expect.objectContaining({
dimension,
status: "unknown",
observed_status: "unchanged",
limitations: expect.arrayContaining([
`Source ${incomplete.evidence_id}: Call graph unavailable.`,
]),
}),
],
},
});
}
expect(
admittedVerification(oldUnchanged, [left, incomplete], "pseudocode")(),
).toMatchObject({
status: "pass",
claims: {
items: [
expect.objectContaining({
dimension: "pseudocode",
status: "pass",
observed_status: "unchanged",
}),
],
},
});
});
it("keeps a confirmed historical overall change as a failure despite unknown calls", () => {
const changedDossier = jsonValueSchema.parse({
...functionDossierSchema.parse(dossier),
pseudocode: "return 1;",
});
const changedComplete = source("8", { result: changedDossier });
const incomplete = source("9", {
result: changedDossier,
relationshipCoverage: {
status: "unknown",
reason: "Call graph unavailable.",
},
});
const changedResult = compareFunctions(left, changedComplete);
const oldChanged = comparison(
{
left_evidence_id: left.evidence_id,
right_evidence_id: incomplete.evidence_id,
},
{},
[left, incomplete],
{},
changedResult,
);
expect(oldChanged.normalized_result).toMatchObject({ status: "changed" });
expect(
admittedVerification(oldChanged, [left, incomplete], "overall")(),
).toMatchObject({
status: "fail",
summary: { total: 1, passed: 0, failed: 1, unknown: 0 },
claims: {
items: [
expect.objectContaining({
status: "fail",
observed_status: "changed",
}),
],
},
});
});
});
describe("function reconstruction source aliases", () => {
it.each([
["no source declarations", {}],
+44 -8
View File
@@ -16,6 +16,7 @@ import {
isProcessEvidenceProvider,
} from "./process/processEvidenceProvider.js";
import { functionDossierSchema } from "./hopperValues.js";
import { functionCallCoverageLimitations } from "./functionCallCoverage.js";
import {
deriveProcessComparisonStatus,
PROCESS_COMPARISON_DIMENSIONS,
@@ -58,6 +59,11 @@ const providers = {
},
} as const;
interface FunctionSourceFacts {
readonly evidence_id: string;
readonly callCoverageLimitations: readonly string[];
}
const acceptsPredicate = (
expected: readonly string[],
actual: string,
@@ -137,14 +143,19 @@ const evaluateClaim = (
throw new TypeError(`Missing source Evidence ${id}`);
return source;
});
validateSourceKinds(claim, sources);
const functionSourceFacts = readValidatedSourceFacts(claim, sources);
const observed = observedStatus(claim, comparison);
const limitations = authorityLimitations(claim, sources, sides);
const relevantUnknowns = reconstructionClaimUnknowns(
const limitations = [
...authorityLimitations(claim, sources, sides),
...sourceCallCoverageLimitations(claim, functionSourceFacts, observed),
];
const relatedUnknowns = reconstructionClaimUnknowns(
heads,
comparison.evidence_id,
allSourceIds,
claim.dimension,
);
const relevantUnknowns = relatedUnknowns;
const status = classify(observed, limitations, relevantUnknowns);
const displayedUnknowns = relevantUnknowns;
return reconstructionClaimResultSchema.parse({
@@ -168,10 +179,10 @@ const evaluateClaim = (
});
};
const validateSourceKinds = (
const readValidatedSourceFacts = (
claim: Claim,
sources: readonly Evidence[],
): void => {
): readonly FunctionSourceFacts[] => {
const expected =
claim.kind === "behavioral"
? ["capture_process_scenario", "rea.process-capture"]
@@ -188,13 +199,22 @@ const validateSourceKinds = (
throw new TypeError(
"Comparison source Evidence has an unexpected observation type",
);
const functionSources: FunctionSourceFacts[] = [];
for (const source of sources) {
if (claim.kind === "behavioral")
parseProcessCapture(source.normalized_result);
else if (claim.kind === "structural-function")
functionDossierSchema.parse(source.normalized_result);
else artifactInventoryResultSchema.parse(source.normalized_result);
else if (claim.kind === "structural-function") {
const dossier = functionDossierSchema.parse(source.normalized_result);
functionSources.push({
evidence_id: source.evidence_id,
callCoverageLimitations: functionCallCoverageLimitations(
source.relationship_coverage,
dossier,
),
});
} else artifactInventoryResultSchema.parse(source.normalized_result);
}
return functionSources;
};
const validateComparisonIdentity = (claim: Claim, evidence: Evidence): void => {
@@ -317,6 +337,22 @@ const authorityLimitations = (
return limitations;
};
const sourceCallCoverageLimitations = (
claim: Claim,
functionSources: readonly FunctionSourceFacts[],
observed: ObservedStatus,
): string[] => {
if (
claim.kind !== "structural-function" ||
(claim.dimension !== "calls" &&
!(claim.dimension === "overall" && observed !== "changed"))
)
return [];
return functionSources.flatMap(({ evidence_id, callCoverageLimitations }) =>
callCoverageLimitations.map((reason) => `Source ${evidence_id}: ${reason}`),
);
};
const classify = (
observed: ObservedStatus,
limitations: readonly string[],
+43
View File
@@ -97,6 +97,49 @@ it("normalizes producer text before identity and revision authentication", () =>
expect(residualUnknownSchema.parse(created)).toEqual(created);
});
it("authenticates typed comparison scope and preserves it across revisions", () => {
const comparisonId = `ev_${"c".repeat(64)}`;
const scopedInput = recordUnknownInputSchema.parse({
...input,
domain: "function-comparison",
supporting_evidence_ids: [comparisonId],
comparison_conclusion_scope: {
comparison_evidence_id: comparisonId,
dimensions: ["calls"],
},
});
const created = createResidualUnknown(scopedInput, mutationId, scopeDigest);
const updated = updateResidualUnknown(
created,
{
...input,
unknown_id: created.unknown_id,
expected_revision: 1,
status: "investigating",
severity: "high",
supporting_evidence_ids: [comparisonId],
required_confidence: "observed",
recommended_probes: [],
resolution: null,
},
mutationId,
);
expect(updated.comparison_conclusion_scope).toEqual({
comparison_evidence_id: comparisonId,
dimensions: ["calls"],
});
expect(residualUnknownSchema.parse(updated)).toEqual(updated);
expect(
residualUnknownSchema.safeParse({
...created,
comparison_conclusion_scope: {
comparison_evidence_id: `ev_${"d".repeat(64)}`,
dimensions: ["calls"],
},
}).success,
).toBe(false);
});
it("keeps generated revisions subject to state and relationship validation", () => {
const created = createResidualUnknown(input, mutationId, scopeDigest);
const command = {
+84 -5
View File
@@ -7,6 +7,7 @@ import {
compositeKey,
} from "./unicodeCodePointOrder.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const unknownIdSchema = prefixedDigestSchema("unk");
@@ -23,6 +24,7 @@ const stableUnknownId = (
question: string,
domain: string,
scopeDigest: string | null,
comparisonConclusionScope?: z.infer<typeof comparisonConclusionScopeSchema>,
): string =>
`unk_${createHash("sha256")
.update(
@@ -30,6 +32,9 @@ const stableUnknownId = (
domain: domain.trim(),
question: question.replace(/\s+/gu, " ").trim(),
scope_digest: scopeDigest,
...(comparisonConclusionScope === undefined
? {}
: { comparison_conclusion_scope: comparisonConclusionScope }),
}),
)
.digest("hex")}`;
@@ -38,6 +43,23 @@ const probeSchema = z.object({
operation: z.string().min(1),
rationale: text,
});
const comparisonConclusionScopeSchema = z
.object({
comparison_evidence_id: evidenceIdSchema,
dimensions: z
.array(
z.union([z.literal("overall"), functionComparisonDimensionNameSchema]),
)
.min(1),
})
.strict()
.superRefine(({ dimensions }, context) => {
if (!sameOrderedValues(dimensions, sortedUnique(dimensions)))
context.addIssue({
code: "custom",
message: "Comparison dimensions must be sorted and unique",
});
});
const relationshipSchema = z.object({
type: z.enum(["depends-on", "related-to", "contradicts"]),
unknown_id: unknownIdSchema,
@@ -78,6 +100,7 @@ const residualUnknownObjectSchema = z.object({
]),
severity: z.enum(["low", "medium", "high", "critical"]),
domain: text,
comparison_conclusion_scope: comparisonConclusionScopeSchema.exactOptional(),
supporting_evidence_ids: z.array(evidenceIdSchema),
contradicting_evidence_ids: z.array(evidenceIdSchema),
required_authority: z
@@ -97,6 +120,8 @@ const residualUnknownObjectSchema = z.object({
mutation_evidence_ids: z.array(evidenceIdSchema).min(1),
});
export type ResidualUnknown = z.infer<typeof residualUnknownSchema>;
type UnknownRevision = Omit<
z.infer<typeof residualUnknownObjectSchema>,
"revision_digest"
@@ -105,7 +130,12 @@ type UnknownRevision = Omit<
const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
if (
value.unknown_id !==
stableUnknownId(value.question, value.domain, value.scope_digest)
stableUnknownId(
value.question,
value.domain,
value.scope_digest,
value.comparison_conclusion_scope,
)
)
context.addIssue({
code: "custom",
@@ -153,6 +183,29 @@ const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
code: "custom",
message: "Evidence IDs must be unique",
});
if (
value.comparison_conclusion_scope !== undefined &&
![
...value.supporting_evidence_ids,
...value.contradicting_evidence_ids,
].includes(value.comparison_conclusion_scope.comparison_evidence_id)
)
context.addIssue({
code: "custom",
message:
"Scoped comparison Evidence must support or contradict the unknown",
path: ["comparison_conclusion_scope"],
});
if (
value.comparison_conclusion_scope !== undefined &&
value.domain !== "function-comparison"
)
context.addIssue({
code: "custom",
message:
"Comparison conclusion scope requires function-comparison domain",
path: ["comparison_conclusion_scope"],
});
const contradictingEvidence = new Set(value.contradicting_evidence_ids);
if (value.supporting_evidence_ids.some((id) => contradictingEvidence.has(id)))
context.addIssue({
@@ -170,7 +223,7 @@ const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
});
};
export const residualUnknownSchema = residualUnknownObjectSchema.superRefine(
const residualUnknownValidatedSchema = residualUnknownObjectSchema.superRefine(
(value, context) => {
if (computeRevisionDigest(value) !== value.revision_digest)
context.addIssue({
@@ -182,11 +235,12 @@ export const residualUnknownSchema = residualUnknownObjectSchema.superRefine(
},
);
export const residualUnknownSchema = residualUnknownValidatedSchema;
const generatedRevisionSchema = residualUnknownObjectSchema
.omit({ revision_digest: true })
.superRefine(refineRevision);
export type ResidualUnknown = z.infer<typeof residualUnknownSchema>;
export type UnknownStatus = ResidualUnknown["status"];
/** Input for deterministic creation of one residual unknown. */
@@ -237,8 +291,21 @@ export const createResidualUnknown = (
): ResidualUnknown => {
const question = input.question.replace(/\s+/gu, " ").trim();
const domain = input.domain.trim();
const comparisonConclusionScope =
input.comparison_conclusion_scope === undefined
? undefined
: {
comparison_evidence_id:
input.comparison_conclusion_scope.comparison_evidence_id,
dimensions: input.comparison_conclusion_scope.dimensions,
};
const revision = {
unknown_id: stableUnknownId(question, domain, scopeDigest),
unknown_id: stableUnknownId(
question,
domain,
scopeDigest,
comparisonConclusionScope,
),
revision: 1,
previous_revision_digest: null,
scope_digest: scopeDigest,
@@ -246,6 +313,9 @@ export const createResidualUnknown = (
status:
input.contradicting_evidence_ids.length > 0 ? "contradicted" : "open",
domain,
...(comparisonConclusionScope === undefined
? {}
: { comparison_conclusion_scope: comparisonConclusionScope }),
...projectRevisionDetails(input),
resolution: null,
mutation_evidence_ids: [mutationEvidenceId],
@@ -267,6 +337,9 @@ export const updateResidualUnknown = (
question: current.question,
status: input.status,
domain: current.domain,
...(current.comparison_conclusion_scope === undefined
? {}
: { comparison_conclusion_scope: current.comparison_conclusion_scope }),
...projectRevisionDetails(input),
resolution: input.resolution,
mutation_evidence_ids: sortedUnique([
@@ -307,7 +380,7 @@ const projectRevisionDetails = (input: RevisionDetailsInput) => ({
});
const computeRevisionDigest = (
value: Omit<ResidualUnknown, "revision_digest"> | ResidualUnknown,
value: UnknownRevision | ResidualUnknown,
): string => {
const { revision_digest: _revisionDigest, ...semantic } =
"revision_digest" in value ? value : { ...value, revision_digest: "" };
@@ -323,6 +396,12 @@ const canonicalJson = (value: unknown): string => {
const sortedUnique = (values: readonly string[]): string[] =>
[...new Set(values)].sort(compareUnicodeCodePoints);
const sameOrderedValues = (
left: readonly string[],
right: readonly string[],
): boolean =>
left.length === right.length &&
left.every((value, index) => value === right[index]);
const sortedRelationships = (
values: readonly z.infer<typeof relationshipSchema>[],
+14 -3
View File
@@ -223,11 +223,22 @@ describe("native IDA provider observations", () => {
procedure: "main",
});
if (!dossier.ok) throw dossier.error;
expect(dossier.value.relationshipCoverage).toEqual({
expect(dossier.value.relationshipCoverage).toMatchObject({
status: "unknown",
reason:
"Upstream xrefs do not establish direct caller relationships.",
reason: more
? expect.stringContaining(
"Callees: Upstream callees capped results at 500; additional callees remain unknown.",
)
: expect.stringContaining(
"Callers: Upstream xrefs do not establish direct caller relationships.",
),
});
if (more)
expect(dossier.value.relationshipCoverage).toMatchObject({
reason: expect.stringContaining(
"Callers: Upstream xrefs do not establish direct caller relationships.",
),
});
expect(dossier.value.result).toMatchObject({
callers: [],
callees: [{ address: "0x2000", name: "puts" }],
+2 -28
View File
@@ -19,6 +19,7 @@ import {
} from "../domain/hopperValues.js";
import type { JsonValue } from "../domain/jsonValue.js";
import type { RelationshipCoverage } from "../domain/relationshipCoverage.js";
import { combineFunctionRelationshipCoverage } from "../domain/functionCallCoverage.js";
import { readCallGraphCallees, unavailableCallGraph } from "./IdaCallGraph.js";
import type { IdaMcpConnection } from "./IdaMcpConnection.js";
import type { IdaOperation } from "./IdaProviderCapabilities.js";
@@ -70,33 +71,6 @@ interface IdaRelationships {
readonly coverage: RelationshipCoverage;
}
const combinedRelationshipCoverage = (
callers: RelationshipCoverage,
callees: RelationshipCoverage,
): RelationshipCoverage => {
const coverage = [callers, callees];
const diagnostics = [
...new Set(
coverage.flatMap((item) =>
"diagnostics" in item ? item.diagnostics : [],
),
),
];
const details = diagnostics.length > 0 ? { diagnostics } : {};
const unknown = coverage.filter((item) => item.status === "unknown");
if (unknown.length > 0)
return {
status: "unknown",
reason: [...new Set(unknown.map((item) => item.reason))].join(" "),
...details,
};
const limitations = coverage.flatMap((item) =>
item.status === "partial" ? item.limitations : [],
);
return limitations.length > 0
? { status: "partial", limitations, ...details }
: { status: "complete", ...details };
};
/** The only lookup answers upstream reports for a name or address that is not a function. */
const MISSING_PROCEDURE_ANSWERS: ReadonlySet<string> = new Set([
"Not found",
@@ -632,7 +606,7 @@ export class IdaOperationRunner {
const pseudocode = await this.pseudocode(fn);
const callers = await this.relationships(fn, "callers");
const callees = await this.relationships(fn, "callees");
this.relationshipCoverage = combinedRelationshipCoverage(
this.relationshipCoverage = combineFunctionRelationshipCoverage(
callers.coverage,
callees.coverage,
);
+44 -6
View File
@@ -20,6 +20,7 @@ import { withAdmittedAnalysis } from "./analysisAdmission.js";
import { runAdmittedToolOperation } from "./admittedToolOperation.js";
import { compareFunctions } from "../domain/functionComparison.js";
import { uniqueSorted } from "../domain/canonicalOrdering.js";
/** Register explicit Evidence-backed function comparison. */
export const registerFunctionComparisonTool = (
server: EvidenceMcpServer,
@@ -102,12 +103,12 @@ export const registerFunctionComparisonTool = (
const recorded = recordDerivedEvidence(
session,
evidence,
functionUnknownInput({
status: comparison.status,
comparisonUnknownInput(
comparison,
leftIds,
rightIds,
comparisonEvidenceId: evidence.evidence_id,
}),
evidence.evidence_id,
),
);
return server.delivery.toEvidenceToolResult(
evidence,
@@ -119,26 +120,63 @@ export const registerFunctionComparisonTool = (
);
};
const comparisonUnknownInput = (
comparison: ReturnType<typeof compareFunctions>,
leftIds: readonly string[],
rightIds: readonly string[],
comparisonEvidenceId: string,
): RecordUnknownInput | undefined =>
functionUnknownInput({
status: comparison.status,
dimensions: comparison.dimensions
.filter(({ status }) =>
comparison.status === "unknown"
? status === "unknown"
: status === "changed",
)
.map(({ dimension }) => dimension),
leftIds,
rightIds,
comparisonEvidenceId,
});
const functionUnknownInput = ({
status,
dimensions,
leftIds,
rightIds,
comparisonEvidenceId,
}: {
status: ReturnType<typeof compareFunctions>["status"];
dimensions: readonly ReturnType<
typeof compareFunctions
>["dimensions"][number]["dimension"][];
leftIds: readonly string[];
rightIds: readonly string[];
comparisonEvidenceId: string;
}): RecordUnknownInput | undefined => {
if (status === "unchanged") return undefined;
const supportingIds = [...new Set([...leftIds, ...rightIds])];
const supportingIds = [
...new Set([...leftIds, ...rightIds, comparisonEvidenceId]),
];
const contradictingIds = rightIds.filter((id) => !leftIds.includes(id));
const contradictory = status === "changed";
return {
question: `Function comparison is ${status} (comparison ${comparisonEvidenceId})`,
severity: status === "changed" ? "medium" : "high",
domain: "function-comparison",
supporting_evidence_ids: contradictory ? [...leftIds] : supportingIds,
comparison_conclusion_scope: {
comparison_evidence_id: comparisonEvidenceId,
dimensions: uniqueSorted([
...(status === "unknown" || dimensions.length === 0
? ["overall" as const]
: []),
...dimensions,
]),
},
supporting_evidence_ids: contradictory
? [...leftIds, comparisonEvidenceId]
: supportingIds,
contradicting_evidence_ids: contradictory ? contradictingIds : [],
required_authority: "shipped-artifact",
required_confidence: "observed",
@@ -0,0 +1,233 @@
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { expect, it } from "vitest";
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
import { functionDossierSchema } from "../../../src/domain/hopperValues.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
import { createServer } from "../../../src/server/createServer.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
const sourceWithCoverage = (
template: typeof FUNCTION_COMPARISON_EXAMPLE.left,
observation: string,
result: typeof template.normalized_result,
relationshipCoverage:
| { readonly status: "complete" }
| { readonly status: "unknown"; readonly reason: string },
) => {
if (template.subject === null)
throw new Error("Function comparison fixtures must be artifact-bound");
return createEvidence(
{
path: `${template.subject.local_path}/${observation}`,
sha256: template.subject.digest.sha256,
format: template.subject.format,
},
template.provider,
{
operation: "analyze_function",
parameters: { ...template.parameters, observation },
result,
confidence: template.confidence,
authority: template.authority,
relationshipCoverage,
},
);
};
const verify = async (
call: (
name: string,
arguments_: Record<string, unknown>,
) => Promise<{
readonly isError?: boolean | undefined;
readonly structuredContent?: unknown | undefined;
}>,
comparison: ReturnType<typeof parseEvidence>,
dimensions: readonly ("calls" | "overall" | "pseudocode")[],
) => {
const response = await call("verify_reconstruction", {
specification: {
name: "Function call coverage verification",
claims: dimensions.map((dimension) => ({
kind: "structural-function",
claim_id: dimension,
title: `Check ${dimension}`,
comparison_evidence_id: comparison.evidence_id,
dimension,
})),
},
});
expect(response.isError, JSON.stringify(response.structuredContent)).not.toBe(
true,
);
return parseEvidence(response.structuredContent);
};
const assertChangedDespiteUnknownCalls = async (
call: Parameters<typeof verify>[0],
left: ReturnType<typeof parseEvidence>,
): Promise<void> => {
const changedRight = sourceWithCoverage(
FUNCTION_COMPARISON_EXAMPLE.right,
"coverage-changed-right",
jsonValueSchema.parse({
...functionDossierSchema.parse(
FUNCTION_COMPARISON_EXAMPLE.right.normalized_result,
),
pseudocode: "return 1;",
}),
{ status: "unknown", reason: "Call graph unavailable." },
);
const changed = await call("compare_functions", {
left,
right: changedRight,
});
expect(changed.isError, JSON.stringify(changed.structuredContent)).not.toBe(
true,
);
const changedComparison = parseEvidence(changed.structuredContent);
expect(changedComparison.normalized_result).toMatchObject({
status: "changed",
dimensions: expect.arrayContaining([
expect.objectContaining({ dimension: "calls", status: "unknown" }),
expect.objectContaining({ dimension: "pseudocode", status: "changed" }),
]),
});
const verification = await verify(call, changedComparison, ["overall"]);
expect(verification.normalized_result).toMatchObject({
status: "fail",
summary: { total: 1, passed: 0, failed: 1, unknown: 0 },
claims: {
items: [
expect.objectContaining({
claim_id: "overall",
status: "fail",
observed_status: "changed",
}),
],
},
});
};
it("keeps incomplete calls unknown through public comparison and verification while preserving confirmed changes", async () => {
const session = createTestBinarySession(() => {
throw new Error("Function comparison must not launch a provider");
});
const server = createServer({ kind: "session", session });
const client = new Client({ name: "function-coverage-test", version: "1" });
const [clientTransport, serverTransport] =
InMemoryTransport.createLinkedPair();
try {
await server.connect(serverTransport);
await client.connect(clientTransport);
const left = sourceWithCoverage(
FUNCTION_COMPARISON_EXAMPLE.left,
"coverage-left",
FUNCTION_COMPARISON_EXAMPLE.left.normalized_result,
{ status: "complete" },
);
const right = sourceWithCoverage(
FUNCTION_COMPARISON_EXAMPLE.right,
"coverage-right",
FUNCTION_COMPARISON_EXAMPLE.left.normalized_result,
{ status: "unknown", reason: "Call graph unavailable." },
);
expect(left.subject).not.toBeNull();
expect(right.subject).not.toBeNull();
expect(left.limitations).not.toContain(
"Artifact identity is unavailable for this observation.",
);
expect(right.limitations).not.toContain(
"Artifact identity is unavailable for this observation.",
);
const compared = await client.callTool({
name: "compare_functions",
arguments: { left, right },
});
expect(
compared.isError,
JSON.stringify(compared.structuredContent),
).not.toBe(true);
const comparison = parseEvidence(compared.structuredContent);
expect(session.evidenceById(left.evidence_id)?.subject).not.toBeNull();
expect(session.evidenceById(right.evidence_id)?.subject).not.toBeNull();
expect(comparison.normalized_result).toMatchObject({
status: "unknown",
summary: { unchanged: 7, changed: 0, unknown: 1 },
dimensions: expect.arrayContaining([
expect.objectContaining({
dimension: "calls",
status: "unknown",
conclusion_kind: "unresolved_branch",
left_count: 0,
right_count: 0,
limitations: ["Right: Call graph unavailable."],
evidence_links: [left.evidence_id, right.evidence_id],
}),
]),
});
expect(comparison.evidence_links).toEqual(
expect.arrayContaining([left.evidence_id, right.evidence_id]),
);
const unknowns = await client.callTool({
name: "list_unknowns",
arguments: { domain: "function-comparison" },
});
expect(unknowns.isError).not.toBe(true);
expect(unknowns.structuredContent).toMatchObject({
result: {
items: [
expect.objectContaining({
domain: "function-comparison",
status: "open",
supporting_evidence_ids: expect.arrayContaining([
left.evidence_id,
right.evidence_id,
]),
}),
],
},
});
const verification = await verify(
async (name, arguments_) =>
client.callTool({ name, arguments: arguments_ }),
comparison,
["calls", "overall", "pseudocode"],
);
expect(verification.normalized_result).toMatchObject({
status: "unknown",
summary: { total: 3, passed: 1, failed: 0, unknown: 2 },
claims: {
items: expect.arrayContaining([
expect.objectContaining({
claim_id: "calls",
status: "unknown",
observed_status: "unknown",
}),
expect.objectContaining({
claim_id: "overall",
status: "unknown",
observed_status: "unknown",
}),
expect.objectContaining({
claim_id: "pseudocode",
status: "pass",
observed_status: "unchanged",
}),
]),
},
});
await assertChangedDespiteUnknownCalls(
async (name, arguments_) =>
client.callTool({ name, arguments: arguments_ }),
left,
);
} finally {
await Promise.allSettled([client.close(), server.close(), session.close()]);
}
});