diff --git a/src/contracts/functionComparisonExample.ts b/src/contracts/functionComparisonExample.ts index 2ba8581d91..39dfe6f92e 100644 --- a/src/contracts/functionComparisonExample.ts +++ b/src/contracts/functionComparisonExample.ts @@ -50,6 +50,7 @@ const observe = (digit: string, text: string) => result: dossier(text), confidence: "derived", authority: "shipped-artifact", + relationshipCoverage: { status: "complete" }, }, ); diff --git a/src/domain/functionCallCoverage.ts b/src/domain/functionCallCoverage.ts new file mode 100644 index 0000000000..eb2c05a132 --- /dev/null +++ b/src/domain/functionCallCoverage.ts @@ -0,0 +1,78 @@ +import type { RelationshipCoverage } from "./relationshipCoverage.js"; +import type { FunctionDossier } from "./hopperValues.js"; +import { uniqueSorted } from "./canonicalOrdering.js"; + +/** Combine caller and callee read outcomes without losing either explanation. */ +export const combineFunctionRelationshipCoverage = ( + callers: RelationshipCoverage | undefined, + callees: RelationshipCoverage | undefined, +): RelationshipCoverage => { + const reads = [ + { direction: "callers" as const, coverage: callers }, + { direction: "callees" as const, coverage: callees }, + ]; + const diagnostics = uniqueSorted( + reads.flatMap(({ coverage }) => + coverage !== undefined && "diagnostics" in coverage + ? coverage.diagnostics + : [], + ), + ); + const details = diagnostics.length > 0 ? { diagnostics } : {}; + const reasons = reads.flatMap(({ direction, coverage }) => { + const label = direction === "callers" ? "Callers" : "Callees"; + const limitations = + coverage === undefined + ? ["Provider did not report relationship coverage."] + : coverage.status === "unknown" + ? [coverage.reason] + : coverage.status === "partial" + ? coverage.limitations + : []; + return limitations.map((reason) => `${label}: ${reason}`); + }); + if ( + reads.some( + ({ coverage }) => coverage === undefined || coverage.status === "unknown", + ) + ) + return { + status: "unknown", + reason: uniqueSorted(reasons).join(" "), + ...details, + }; + if (reasons.length > 0) + return { + status: "partial", + limitations: uniqueSorted(reasons), + ...details, + }; + return { status: "complete", ...details }; +}; + +/** Explain why two resolved call sets cannot establish call-set equality. */ +export const functionCallCoverageLimitations = ( + coverage: RelationshipCoverage | null, + dossier: Pick, +): string[] => { + const limitations = + coverage === null + ? ["Provider did not report relationship coverage."] + : coverage.status === "unknown" + ? [ + coverage.reason, + ...("diagnostics" in coverage ? coverage.diagnostics : []), + ] + : coverage.status === "partial" + ? [ + ...coverage.limitations, + ...("diagnostics" in coverage ? coverage.diagnostics : []), + ] + : []; + limitations.push( + ...(dossier.unresolved_calls ?? []).map( + ({ address, reason }) => `Unresolved call at ${address}: ${reason}`, + ), + ); + return uniqueSorted(limitations); +}; diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index f846217c54..acb505dac9 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -11,6 +11,7 @@ import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; import { jsonValueSchema } from "./jsonValue.js"; import { compareFunctions } from "./functionComparison.js"; +import type { RelationshipCoverage } from "./relationshipCoverage.js"; const procedure = (address: string, name: string) => ({ address, name, @@ -96,6 +97,7 @@ const observe = ( digit: string, value: ReturnType, providerId = "rea-workflow", + relationshipCoverage: RelationshipCoverage | null = { status: "complete" }, ): Evidence => createEvidence( { @@ -112,6 +114,7 @@ const observe = ( result: jsonValueSchema.parse(value), confidence: "derived", authority: "shipped-artifact", + ...(relationshipCoverage === null ? {} : { relationshipCoverage }), }, ); @@ -187,6 +190,100 @@ describe("function comparison normalized identity", () => { }); }); +describe("function call relationship coverage", () => { + const coverageCases: { + readonly label: string; + readonly coverage: RelationshipCoverage | null; + readonly limitation: string; + }[] = [ + { + label: "missing", + coverage: null, + limitation: "Provider did not report relationship coverage.", + }, + { + label: "partial", + coverage: { + status: "partial", + limitations: ["Call graph stopped at the read limit."], + }, + limitation: "Call graph stopped at the read limit.", + }, + ]; + it.each(coverageCases)( + "keeps equal empty calls unknown for $label coverage", + ({ coverage, limitation }) => { + const left = observe("a", dossier("return 0;", "0x1000")); + const right = observe( + "b", + dossier("return 0;", "0x2000"), + "rea-workflow", + coverage, + ); + const result = compareFunctions(left, right); + const calls = result.dimensions.find( + ({ dimension }) => dimension === "calls", + ); + expect(calls).toMatchObject({ + status: "unknown", + conclusion_kind: "unresolved_branch", + left_count: 0, + right_count: 0, + limitations: [`Right: ${limitation}`], + }); + expect(result.status).toBe("unknown"); + }, + ); + + it("retains complete observations with execution diagnostics as comparable", () => { + const coverage = { + status: "complete" as const, + diagnostics: ["Provider reported a non-call xref warning."], + }; + const left = observe( + "c", + dossier("return 0;", "0x1000"), + "rea-workflow", + coverage, + ); + const right = observe( + "d", + dossier("return 0;", "0x2000"), + "rea-workflow", + coverage, + ); + expect( + compareFunctions(left, right).dimensions.find( + ({ dimension }) => dimension === "calls", + ), + ).toMatchObject({ status: "unchanged", left_count: 0, right_count: 0 }); + }); + + it("keeps unresolved call sites unknown despite complete resolved-set coverage", () => { + const left = observe("e", dossier("return 0;", "0x1000")); + const rightDossier = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + unresolved_calls: [ + { + address: "0x2010", + reason: "Indirect dispatch target could not be resolved.", + }, + ], + }); + const right = observe("f", rightDossier); + expect( + compareFunctions(left, right).dimensions.find( + ({ dimension }) => dimension === "calls", + ), + ).toMatchObject({ + status: "unknown", + limitations: [ + "Right: Unresolved call at 0x2010: Indirect dispatch target could not be resolved.", + ], + }); + }); +}); + describe("function collection collation ties", () => { const composed = procedure("0x2000", "caf\u00e9"); const decomposed = procedure("0x3000", "cafe\u0301"); @@ -342,6 +439,7 @@ describe("function comparison ordering controls", () => { result: jsonValueSchema.parse(result), confidence: "derived", authority: "shipped-artifact", + relationshipCoverage: { status: "complete" }, }, ); const left = controlled("0x1000", false); @@ -525,7 +623,9 @@ describe("function comparison", () => { limitations: [expect.stringContaining("did not expose reference kinds")], }); }); +}); +describe("function comparison identity and malformed CFG", () => { it("does not promote address-derived names through equal signatures", () => { const autoDossier = (base: "0x1000" | "0x2000") => functionDossierSchema.parse({ @@ -546,6 +646,22 @@ describe("function comparison", () => { comparison.dimensions.find(({ dimension }) => dimension === "identity"), ).toMatchObject({ status: "unknown" }); expect(comparison.status).toBe("unknown"); + const changedText = compareFunctions( + observe("b", autoDossier("0x1000")), + observe( + "c", + functionDossierSchema.parse({ + ...autoDossier("0x2000"), + pseudocode: "return 1;", + }), + ), + ); + expect(changedText.dimensions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ dimension: "pseudocode", status: "changed" }), + ]), + ); + expect(changedText.status).toBe("unknown"); }); it("rejects invalid CFG normalization and preserves newline differences", () => { diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index f4d7d86a1d..27bb7b1762 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -233,6 +233,18 @@ const compareCalls = ( ): FunctionDimension => { const leftValues = callProjection(left); const rightValues = callProjection(right); + const coverageLimitations = [ + ...left.callCoverageLimitations.map((reason) => `Left: ${reason}`), + ...right.callCoverageLimitations.map((reason) => `Right: ${reason}`), + ]; + if (coverageLimitations.length > 0) + return unresolvedDimension({ + dimension: "calls", + links: context.links, + leftCount: leftValues.length, + rightCount: rightValues.length, + limitations: coverageLimitations, + }); if ([...leftValues, ...rightValues].some(({ name }) => isAutoName(name))) return unresolvedDimension({ dimension: "calls", diff --git a/src/domain/functionComparisonResults.ts b/src/domain/functionComparisonResults.ts index 95465c3e7f..93eb9d7522 100644 --- a/src/domain/functionComparisonResults.ts +++ b/src/domain/functionComparisonResults.ts @@ -69,15 +69,11 @@ export const overallStatus = ( dimensions: readonly FunctionDimension[], match: FunctionComparisonResult["function_match"]["status"], ): FunctionComparisonResult["status"] => { - if ( - match === "ambiguous" || - dimensions.some(({ status }) => status === "unknown") - ) - return "unknown"; if (match === "mismatched") return "changed"; - return dimensions.some(({ status }) => status === "changed") - ? "changed" - : "unchanged"; + if (match === "ambiguous") return "unknown"; + if (dimensions.some(({ status }) => status === "changed")) return "changed"; + if (dimensions.some(({ status }) => status === "unknown")) return "unknown"; + return "unchanged"; }; export const summarize = (dimensions: readonly FunctionDimension[]) => ({ diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts index 26c10b1fb8..bafc3def14 100644 --- a/src/domain/functionComparisonSchemas.ts +++ b/src/domain/functionComparisonSchemas.ts @@ -4,7 +4,7 @@ import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; const comparisonStatusSchema = z.enum(["unchanged", "changed", "unknown"]); -const dimensionNameSchema = z.enum([ +export const functionComparisonDimensionNameSchema = z.enum([ "identity", "pseudocode", "assembly", @@ -29,7 +29,7 @@ const textDeltaSchema = z.object({ }); const functionDimensionContextShape = { - dimension: dimensionNameSchema, + dimension: functionComparisonDimensionNameSchema, left_count: z.number().int().min(0).nullable(), right_count: z.number().int().min(0).nullable(), evidence_links: z.array(evidenceIdSchema).min(2), @@ -98,4 +98,6 @@ export type FunctionComparisonResult = z.infer< typeof functionComparisonResultSchema >; export type FunctionDimension = z.infer; -export type DimensionName = z.infer; +export type DimensionName = z.infer< + typeof functionComparisonDimensionNameSchema +>; diff --git a/src/domain/functionDossierEvidence.ts b/src/domain/functionDossierEvidence.ts index 5a1443c1c3..35f5c4b8b0 100644 --- a/src/domain/functionDossierEvidence.ts +++ b/src/domain/functionDossierEvidence.ts @@ -1,6 +1,7 @@ import { parseEvidence, type Evidence } from "./evidence.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; import { functionDossierSchema, type FunctionDossier } from "./hopperValues.js"; +import { functionCallCoverageLimitations } from "./functionCallCoverage.js"; /** Complete function observation parsed from one analyze_function Evidence record. */ export interface FunctionSnapshot { @@ -8,6 +9,7 @@ export interface FunctionSnapshot { readonly subject: NonNullable; readonly dossier: FunctionDossier; readonly limitations: readonly string[]; + readonly callCoverageLimitations: readonly string[]; } /** Parse one complete analyze_function Evidence record for comparison workflows. */ @@ -20,6 +22,10 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => { if (evidence.subject === null) throw new TypeError("Function comparison requires artifact-bound Evidence"); const dossier = functionDossierSchema.parse(evidence.normalized_result); + const callCoverageLimitations = functionCallCoverageLimitations( + evidence.relationship_coverage, + dossier, + ); return { evidence, subject: evidence.subject, @@ -27,5 +33,6 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => { limitations: [ ...new Set([...evidence.limitations, ...dossier.limitations]), ].sort((left, right) => compareUnicodeCodePoints(left, right)), + callCoverageLimitations, }; }; diff --git a/src/domain/reconstructionUnknowns.test.ts b/src/domain/reconstructionUnknowns.test.ts new file mode 100644 index 0000000000..9fa406b199 --- /dev/null +++ b/src/domain/reconstructionUnknowns.test.ts @@ -0,0 +1,73 @@ +import { expect, it } from "vitest"; + +import { createResidualUnknown } from "./residualUnknown.js"; +import { reconstructionClaimUnknowns } from "./reconstructionUnknowns.js"; + +const ev = (digit: string) => `ev_${digit.repeat(64)}`; +const base = { + severity: "high" as const, + domain: "function-comparison", + supporting_evidence_ids: [ev("a"), ev("b")], + contradicting_evidence_ids: [], + required_authority: null, + required_confidence: "derived" as const, + required_environment: null, + recommended_probes: [], +}; + +it("keeps explicit dependencies while excluding unrelated scoped unknowns", () => { + const comparisonId = ev("a"); + const other = createResidualUnknown( + { + ...base, + question: "Other dimension", + comparison_conclusion_scope: { + comparison_evidence_id: comparisonId, + dimensions: ["pseudocode"], + }, + relationships: [], + }, + ev("c"), + null, + ); + const calls = createResidualUnknown( + { + ...base, + question: "Call dimension", + comparison_conclusion_scope: { + comparison_evidence_id: comparisonId, + dimensions: ["calls"], + }, + relationships: [{ type: "depends-on", unknown_id: other.unknown_id }], + }, + ev("d"), + null, + ); + const unrelated = createResidualUnknown( + { + ...base, + question: "Unrelated dimension", + comparison_conclusion_scope: { + comparison_evidence_id: comparisonId, + dimensions: ["pseudocode"], + }, + relationships: [], + }, + ev("e"), + null, + ); + const selected = reconstructionClaimUnknowns( + new Map([ + [calls.unknown_id, calls], + [other.unknown_id, other], + [unrelated.unknown_id, unrelated], + ]), + comparisonId, + [ev("b")], + "calls", + ); + expect(selected.map(({ unknown_id }) => unknown_id)).toEqual([ + calls.unknown_id, + other.unknown_id, + ]); +}); diff --git a/src/domain/reconstructionUnknowns.ts b/src/domain/reconstructionUnknowns.ts index d05f96fe92..c09e814eb0 100644 --- a/src/domain/reconstructionUnknowns.ts +++ b/src/domain/reconstructionUnknowns.ts @@ -28,13 +28,26 @@ export const reconstructionClaimUnknowns = ( heads: ReadonlyMap, comparisonId: string, sourceIds: readonly string[], + dimension?: string, ): ResidualUnknown[] => { + const appliesToClaim = (item: ResidualUnknown): boolean => { + const scope = item.comparison_conclusion_scope; + return ( + scope === undefined || + (scope.comparison_evidence_id === comparisonId && + (dimension === undefined || + scope.dimensions.some( + (scopedDimension) => scopedDimension === dimension, + ))) + ); + }; const evidenceIds = new Set([comparisonId, ...sourceIds]); const relevantIds = new Set( [...heads.values()] .filter( (item) => item.status !== "resolved" && + appliesToClaim(item) && [ ...item.supporting_evidence_ids, ...item.contradicting_evidence_ids, diff --git a/src/domain/reconstructionVerification.functionSources.test.ts b/src/domain/reconstructionVerification.functionSources.test.ts index 4c4c04f043..5dc8db5897 100644 --- a/src/domain/reconstructionVerification.functionSources.test.ts +++ b/src/domain/reconstructionVerification.functionSources.test.ts @@ -69,6 +69,7 @@ const source = ( result: dossier, confidence: "derived", authority: "shipped-artifact", + relationshipCoverage: { status: "complete" }, ...observation, }, ); @@ -99,14 +100,15 @@ const comparison = ( provider?: EvidenceProvider; target?: EvidenceSubjectTarget; } = {}, + resultInput = baselineResult, ): Evidence => createEvidence(identity.target, identity.provider ?? provider, { predicateType: "rea.function-comparison", operation: "compare_functions", parameters, result: jsonValueSchema.parse({ - ...baselineResult, - dimensions: baselineResult.dimensions.map((dimension) => ({ + ...resultInput, + dimensions: resultInput.dimensions.map((dimension) => ({ ...dimension, evidence_links: sources.map(({ evidence_id: id }) => id), })), @@ -195,6 +197,100 @@ describe("function reconstruction source parameters", () => { }); }); +describe("function reconstruction call coverage", () => { + it("rechecks old unchanged comparisons without affecting unrelated finite dimensions", () => { + const incomplete = source("7", { + relationshipCoverage: { + status: "unknown", + reason: "Call graph unavailable.", + }, + }); + const incompleteSides = { + left_evidence_id: left.evidence_id, + right_evidence_id: incomplete.evidence_id, + }; + const oldUnchanged = comparison(incompleteSides, {}, [left, incomplete]); + for (const dimension of ["calls", "overall"] as const) { + const result = admittedVerification( + oldUnchanged, + [left, incomplete], + dimension, + )(); + expect(result).toMatchObject({ + status: "unknown", + summary: { total: 1, passed: 0, failed: 0, unknown: 1 }, + claims: { + items: [ + expect.objectContaining({ + dimension, + status: "unknown", + observed_status: "unchanged", + limitations: expect.arrayContaining([ + `Source ${incomplete.evidence_id}: Call graph unavailable.`, + ]), + }), + ], + }, + }); + } + expect( + admittedVerification(oldUnchanged, [left, incomplete], "pseudocode")(), + ).toMatchObject({ + status: "pass", + claims: { + items: [ + expect.objectContaining({ + dimension: "pseudocode", + status: "pass", + observed_status: "unchanged", + }), + ], + }, + }); + }); + + it("keeps a confirmed historical overall change as a failure despite unknown calls", () => { + const changedDossier = jsonValueSchema.parse({ + ...functionDossierSchema.parse(dossier), + pseudocode: "return 1;", + }); + const changedComplete = source("8", { result: changedDossier }); + const incomplete = source("9", { + result: changedDossier, + relationshipCoverage: { + status: "unknown", + reason: "Call graph unavailable.", + }, + }); + const changedResult = compareFunctions(left, changedComplete); + const oldChanged = comparison( + { + left_evidence_id: left.evidence_id, + right_evidence_id: incomplete.evidence_id, + }, + {}, + [left, incomplete], + {}, + changedResult, + ); + expect(oldChanged.normalized_result).toMatchObject({ status: "changed" }); + expect( + admittedVerification(oldChanged, [left, incomplete], "overall")(), + ).toMatchObject({ + status: "fail", + summary: { total: 1, passed: 0, failed: 1, unknown: 0 }, + claims: { + items: [ + expect.objectContaining({ + status: "fail", + observed_status: "changed", + }), + ], + }, + }); + }); +}); + describe("function reconstruction source aliases", () => { it.each([ ["no source declarations", {}], diff --git a/src/domain/reconstructionVerification.ts b/src/domain/reconstructionVerification.ts index 3dd9e3baf5..dd300a0a27 100644 --- a/src/domain/reconstructionVerification.ts +++ b/src/domain/reconstructionVerification.ts @@ -16,6 +16,7 @@ import { isProcessEvidenceProvider, } from "./process/processEvidenceProvider.js"; import { functionDossierSchema } from "./hopperValues.js"; +import { functionCallCoverageLimitations } from "./functionCallCoverage.js"; import { deriveProcessComparisonStatus, PROCESS_COMPARISON_DIMENSIONS, @@ -58,6 +59,11 @@ const providers = { }, } as const; +interface FunctionSourceFacts { + readonly evidence_id: string; + readonly callCoverageLimitations: readonly string[]; +} + const acceptsPredicate = ( expected: readonly string[], actual: string, @@ -137,14 +143,19 @@ const evaluateClaim = ( throw new TypeError(`Missing source Evidence ${id}`); return source; }); - validateSourceKinds(claim, sources); + const functionSourceFacts = readValidatedSourceFacts(claim, sources); const observed = observedStatus(claim, comparison); - const limitations = authorityLimitations(claim, sources, sides); - const relevantUnknowns = reconstructionClaimUnknowns( + const limitations = [ + ...authorityLimitations(claim, sources, sides), + ...sourceCallCoverageLimitations(claim, functionSourceFacts, observed), + ]; + const relatedUnknowns = reconstructionClaimUnknowns( heads, comparison.evidence_id, allSourceIds, + claim.dimension, ); + const relevantUnknowns = relatedUnknowns; const status = classify(observed, limitations, relevantUnknowns); const displayedUnknowns = relevantUnknowns; return reconstructionClaimResultSchema.parse({ @@ -168,10 +179,10 @@ const evaluateClaim = ( }); }; -const validateSourceKinds = ( +const readValidatedSourceFacts = ( claim: Claim, sources: readonly Evidence[], -): void => { +): readonly FunctionSourceFacts[] => { const expected = claim.kind === "behavioral" ? ["capture_process_scenario", "rea.process-capture"] @@ -188,13 +199,22 @@ const validateSourceKinds = ( throw new TypeError( "Comparison source Evidence has an unexpected observation type", ); + const functionSources: FunctionSourceFacts[] = []; for (const source of sources) { if (claim.kind === "behavioral") parseProcessCapture(source.normalized_result); - else if (claim.kind === "structural-function") - functionDossierSchema.parse(source.normalized_result); - else artifactInventoryResultSchema.parse(source.normalized_result); + else if (claim.kind === "structural-function") { + const dossier = functionDossierSchema.parse(source.normalized_result); + functionSources.push({ + evidence_id: source.evidence_id, + callCoverageLimitations: functionCallCoverageLimitations( + source.relationship_coverage, + dossier, + ), + }); + } else artifactInventoryResultSchema.parse(source.normalized_result); } + return functionSources; }; const validateComparisonIdentity = (claim: Claim, evidence: Evidence): void => { @@ -317,6 +337,22 @@ const authorityLimitations = ( return limitations; }; +const sourceCallCoverageLimitations = ( + claim: Claim, + functionSources: readonly FunctionSourceFacts[], + observed: ObservedStatus, +): string[] => { + if ( + claim.kind !== "structural-function" || + (claim.dimension !== "calls" && + !(claim.dimension === "overall" && observed !== "changed")) + ) + return []; + return functionSources.flatMap(({ evidence_id, callCoverageLimitations }) => + callCoverageLimitations.map((reason) => `Source ${evidence_id}: ${reason}`), + ); +}; + const classify = ( observed: ObservedStatus, limitations: readonly string[], diff --git a/src/domain/residualUnknown.test.ts b/src/domain/residualUnknown.test.ts index f48e80376f..5f07078f29 100644 --- a/src/domain/residualUnknown.test.ts +++ b/src/domain/residualUnknown.test.ts @@ -97,6 +97,49 @@ it("normalizes producer text before identity and revision authentication", () => expect(residualUnknownSchema.parse(created)).toEqual(created); }); +it("authenticates typed comparison scope and preserves it across revisions", () => { + const comparisonId = `ev_${"c".repeat(64)}`; + const scopedInput = recordUnknownInputSchema.parse({ + ...input, + domain: "function-comparison", + supporting_evidence_ids: [comparisonId], + comparison_conclusion_scope: { + comparison_evidence_id: comparisonId, + dimensions: ["calls"], + }, + }); + const created = createResidualUnknown(scopedInput, mutationId, scopeDigest); + const updated = updateResidualUnknown( + created, + { + ...input, + unknown_id: created.unknown_id, + expected_revision: 1, + status: "investigating", + severity: "high", + supporting_evidence_ids: [comparisonId], + required_confidence: "observed", + recommended_probes: [], + resolution: null, + }, + mutationId, + ); + expect(updated.comparison_conclusion_scope).toEqual({ + comparison_evidence_id: comparisonId, + dimensions: ["calls"], + }); + expect(residualUnknownSchema.parse(updated)).toEqual(updated); + expect( + residualUnknownSchema.safeParse({ + ...created, + comparison_conclusion_scope: { + comparison_evidence_id: `ev_${"d".repeat(64)}`, + dimensions: ["calls"], + }, + }).success, + ).toBe(false); +}); + it("keeps generated revisions subject to state and relationship validation", () => { const created = createResidualUnknown(input, mutationId, scopeDigest); const command = { diff --git a/src/domain/residualUnknown.ts b/src/domain/residualUnknown.ts index 88e0da49e2..0819a3964f 100644 --- a/src/domain/residualUnknown.ts +++ b/src/domain/residualUnknown.ts @@ -7,6 +7,7 @@ import { compositeKey, } from "./unicodeCodePointOrder.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; +import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const unknownIdSchema = prefixedDigestSchema("unk"); @@ -23,6 +24,7 @@ const stableUnknownId = ( question: string, domain: string, scopeDigest: string | null, + comparisonConclusionScope?: z.infer, ): string => `unk_${createHash("sha256") .update( @@ -30,6 +32,9 @@ const stableUnknownId = ( domain: domain.trim(), question: question.replace(/\s+/gu, " ").trim(), scope_digest: scopeDigest, + ...(comparisonConclusionScope === undefined + ? {} + : { comparison_conclusion_scope: comparisonConclusionScope }), }), ) .digest("hex")}`; @@ -38,6 +43,23 @@ const probeSchema = z.object({ operation: z.string().min(1), rationale: text, }); +const comparisonConclusionScopeSchema = z + .object({ + comparison_evidence_id: evidenceIdSchema, + dimensions: z + .array( + z.union([z.literal("overall"), functionComparisonDimensionNameSchema]), + ) + .min(1), + }) + .strict() + .superRefine(({ dimensions }, context) => { + if (!sameOrderedValues(dimensions, sortedUnique(dimensions))) + context.addIssue({ + code: "custom", + message: "Comparison dimensions must be sorted and unique", + }); + }); const relationshipSchema = z.object({ type: z.enum(["depends-on", "related-to", "contradicts"]), unknown_id: unknownIdSchema, @@ -78,6 +100,7 @@ const residualUnknownObjectSchema = z.object({ ]), severity: z.enum(["low", "medium", "high", "critical"]), domain: text, + comparison_conclusion_scope: comparisonConclusionScopeSchema.exactOptional(), supporting_evidence_ids: z.array(evidenceIdSchema), contradicting_evidence_ids: z.array(evidenceIdSchema), required_authority: z @@ -97,6 +120,8 @@ const residualUnknownObjectSchema = z.object({ mutation_evidence_ids: z.array(evidenceIdSchema).min(1), }); +export type ResidualUnknown = z.infer; + type UnknownRevision = Omit< z.infer, "revision_digest" @@ -105,7 +130,12 @@ type UnknownRevision = Omit< const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => { if ( value.unknown_id !== - stableUnknownId(value.question, value.domain, value.scope_digest) + stableUnknownId( + value.question, + value.domain, + value.scope_digest, + value.comparison_conclusion_scope, + ) ) context.addIssue({ code: "custom", @@ -153,6 +183,29 @@ const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => { code: "custom", message: "Evidence IDs must be unique", }); + if ( + value.comparison_conclusion_scope !== undefined && + ![ + ...value.supporting_evidence_ids, + ...value.contradicting_evidence_ids, + ].includes(value.comparison_conclusion_scope.comparison_evidence_id) + ) + context.addIssue({ + code: "custom", + message: + "Scoped comparison Evidence must support or contradict the unknown", + path: ["comparison_conclusion_scope"], + }); + if ( + value.comparison_conclusion_scope !== undefined && + value.domain !== "function-comparison" + ) + context.addIssue({ + code: "custom", + message: + "Comparison conclusion scope requires function-comparison domain", + path: ["comparison_conclusion_scope"], + }); const contradictingEvidence = new Set(value.contradicting_evidence_ids); if (value.supporting_evidence_ids.some((id) => contradictingEvidence.has(id))) context.addIssue({ @@ -170,7 +223,7 @@ const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => { }); }; -export const residualUnknownSchema = residualUnknownObjectSchema.superRefine( +const residualUnknownValidatedSchema = residualUnknownObjectSchema.superRefine( (value, context) => { if (computeRevisionDigest(value) !== value.revision_digest) context.addIssue({ @@ -182,11 +235,12 @@ export const residualUnknownSchema = residualUnknownObjectSchema.superRefine( }, ); +export const residualUnknownSchema = residualUnknownValidatedSchema; + const generatedRevisionSchema = residualUnknownObjectSchema .omit({ revision_digest: true }) .superRefine(refineRevision); -export type ResidualUnknown = z.infer; export type UnknownStatus = ResidualUnknown["status"]; /** Input for deterministic creation of one residual unknown. */ @@ -237,8 +291,21 @@ export const createResidualUnknown = ( ): ResidualUnknown => { const question = input.question.replace(/\s+/gu, " ").trim(); const domain = input.domain.trim(); + const comparisonConclusionScope = + input.comparison_conclusion_scope === undefined + ? undefined + : { + comparison_evidence_id: + input.comparison_conclusion_scope.comparison_evidence_id, + dimensions: input.comparison_conclusion_scope.dimensions, + }; const revision = { - unknown_id: stableUnknownId(question, domain, scopeDigest), + unknown_id: stableUnknownId( + question, + domain, + scopeDigest, + comparisonConclusionScope, + ), revision: 1, previous_revision_digest: null, scope_digest: scopeDigest, @@ -246,6 +313,9 @@ export const createResidualUnknown = ( status: input.contradicting_evidence_ids.length > 0 ? "contradicted" : "open", domain, + ...(comparisonConclusionScope === undefined + ? {} + : { comparison_conclusion_scope: comparisonConclusionScope }), ...projectRevisionDetails(input), resolution: null, mutation_evidence_ids: [mutationEvidenceId], @@ -267,6 +337,9 @@ export const updateResidualUnknown = ( question: current.question, status: input.status, domain: current.domain, + ...(current.comparison_conclusion_scope === undefined + ? {} + : { comparison_conclusion_scope: current.comparison_conclusion_scope }), ...projectRevisionDetails(input), resolution: input.resolution, mutation_evidence_ids: sortedUnique([ @@ -307,7 +380,7 @@ const projectRevisionDetails = (input: RevisionDetailsInput) => ({ }); const computeRevisionDigest = ( - value: Omit | ResidualUnknown, + value: UnknownRevision | ResidualUnknown, ): string => { const { revision_digest: _revisionDigest, ...semantic } = "revision_digest" in value ? value : { ...value, revision_digest: "" }; @@ -323,6 +396,12 @@ const canonicalJson = (value: unknown): string => { const sortedUnique = (values: readonly string[]): string[] => [...new Set(values)].sort(compareUnicodeCodePoints); +const sameOrderedValues = ( + left: readonly string[], + right: readonly string[], +): boolean => + left.length === right.length && + left.every((value, index) => value === right[index]); const sortedRelationships = ( values: readonly z.infer[], diff --git a/src/ida/IdaNativeProtocol.test.ts b/src/ida/IdaNativeProtocol.test.ts index 0e17167b39..f906730319 100644 --- a/src/ida/IdaNativeProtocol.test.ts +++ b/src/ida/IdaNativeProtocol.test.ts @@ -223,11 +223,22 @@ describe("native IDA provider observations", () => { procedure: "main", }); if (!dossier.ok) throw dossier.error; - expect(dossier.value.relationshipCoverage).toEqual({ + expect(dossier.value.relationshipCoverage).toMatchObject({ status: "unknown", - reason: - "Upstream xrefs do not establish direct caller relationships.", + reason: more + ? expect.stringContaining( + "Callees: Upstream callees capped results at 500; additional callees remain unknown.", + ) + : expect.stringContaining( + "Callers: Upstream xrefs do not establish direct caller relationships.", + ), }); + if (more) + expect(dossier.value.relationshipCoverage).toMatchObject({ + reason: expect.stringContaining( + "Callers: Upstream xrefs do not establish direct caller relationships.", + ), + }); expect(dossier.value.result).toMatchObject({ callers: [], callees: [{ address: "0x2000", name: "puts" }], diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index 530376458b..c30ec81bcb 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -19,6 +19,7 @@ import { } from "../domain/hopperValues.js"; import type { JsonValue } from "../domain/jsonValue.js"; import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; +import { combineFunctionRelationshipCoverage } from "../domain/functionCallCoverage.js"; import { readCallGraphCallees, unavailableCallGraph } from "./IdaCallGraph.js"; import type { IdaMcpConnection } from "./IdaMcpConnection.js"; import type { IdaOperation } from "./IdaProviderCapabilities.js"; @@ -70,33 +71,6 @@ interface IdaRelationships { readonly coverage: RelationshipCoverage; } -const combinedRelationshipCoverage = ( - callers: RelationshipCoverage, - callees: RelationshipCoverage, -): RelationshipCoverage => { - const coverage = [callers, callees]; - const diagnostics = [ - ...new Set( - coverage.flatMap((item) => - "diagnostics" in item ? item.diagnostics : [], - ), - ), - ]; - const details = diagnostics.length > 0 ? { diagnostics } : {}; - const unknown = coverage.filter((item) => item.status === "unknown"); - if (unknown.length > 0) - return { - status: "unknown", - reason: [...new Set(unknown.map((item) => item.reason))].join(" "), - ...details, - }; - const limitations = coverage.flatMap((item) => - item.status === "partial" ? item.limitations : [], - ); - return limitations.length > 0 - ? { status: "partial", limitations, ...details } - : { status: "complete", ...details }; -}; /** The only lookup answers upstream reports for a name or address that is not a function. */ const MISSING_PROCEDURE_ANSWERS: ReadonlySet = new Set([ "Not found", @@ -632,7 +606,7 @@ export class IdaOperationRunner { const pseudocode = await this.pseudocode(fn); const callers = await this.relationships(fn, "callers"); const callees = await this.relationships(fn, "callees"); - this.relationshipCoverage = combinedRelationshipCoverage( + this.relationshipCoverage = combineFunctionRelationshipCoverage( callers.coverage, callees.coverage, ); diff --git a/src/server/registerFunctionComparisonTool.ts b/src/server/registerFunctionComparisonTool.ts index b968972ede..405efd713d 100644 --- a/src/server/registerFunctionComparisonTool.ts +++ b/src/server/registerFunctionComparisonTool.ts @@ -20,6 +20,7 @@ import { withAdmittedAnalysis } from "./analysisAdmission.js"; import { runAdmittedToolOperation } from "./admittedToolOperation.js"; import { compareFunctions } from "../domain/functionComparison.js"; +import { uniqueSorted } from "../domain/canonicalOrdering.js"; /** Register explicit Evidence-backed function comparison. */ export const registerFunctionComparisonTool = ( server: EvidenceMcpServer, @@ -102,12 +103,12 @@ export const registerFunctionComparisonTool = ( const recorded = recordDerivedEvidence( session, evidence, - functionUnknownInput({ - status: comparison.status, + comparisonUnknownInput( + comparison, leftIds, rightIds, - comparisonEvidenceId: evidence.evidence_id, - }), + evidence.evidence_id, + ), ); return server.delivery.toEvidenceToolResult( evidence, @@ -119,26 +120,63 @@ export const registerFunctionComparisonTool = ( ); }; +const comparisonUnknownInput = ( + comparison: ReturnType, + leftIds: readonly string[], + rightIds: readonly string[], + comparisonEvidenceId: string, +): RecordUnknownInput | undefined => + functionUnknownInput({ + status: comparison.status, + dimensions: comparison.dimensions + .filter(({ status }) => + comparison.status === "unknown" + ? status === "unknown" + : status === "changed", + ) + .map(({ dimension }) => dimension), + leftIds, + rightIds, + comparisonEvidenceId, + }); + const functionUnknownInput = ({ status, + dimensions, leftIds, rightIds, comparisonEvidenceId, }: { status: ReturnType["status"]; + dimensions: readonly ReturnType< + typeof compareFunctions + >["dimensions"][number]["dimension"][]; leftIds: readonly string[]; rightIds: readonly string[]; comparisonEvidenceId: string; }): RecordUnknownInput | undefined => { if (status === "unchanged") return undefined; - const supportingIds = [...new Set([...leftIds, ...rightIds])]; + const supportingIds = [ + ...new Set([...leftIds, ...rightIds, comparisonEvidenceId]), + ]; const contradictingIds = rightIds.filter((id) => !leftIds.includes(id)); const contradictory = status === "changed"; return { question: `Function comparison is ${status} (comparison ${comparisonEvidenceId})`, severity: status === "changed" ? "medium" : "high", domain: "function-comparison", - supporting_evidence_ids: contradictory ? [...leftIds] : supportingIds, + comparison_conclusion_scope: { + comparison_evidence_id: comparisonEvidenceId, + dimensions: uniqueSorted([ + ...(status === "unknown" || dimensions.length === 0 + ? ["overall" as const] + : []), + ...dimensions, + ]), + }, + supporting_evidence_ids: contradictory + ? [...leftIds, comparisonEvidenceId] + : supportingIds, contradicting_evidence_ids: contradictory ? contradictingIds : [], required_authority: "shipped-artifact", required_confidence: "observed", diff --git a/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts b/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts new file mode 100644 index 0000000000..3d3295fc45 --- /dev/null +++ b/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts @@ -0,0 +1,233 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, it } from "vitest"; + +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; + +const sourceWithCoverage = ( + template: typeof FUNCTION_COMPARISON_EXAMPLE.left, + observation: string, + result: typeof template.normalized_result, + relationshipCoverage: + | { readonly status: "complete" } + | { readonly status: "unknown"; readonly reason: string }, +) => { + if (template.subject === null) + throw new Error("Function comparison fixtures must be artifact-bound"); + return createEvidence( + { + path: `${template.subject.local_path}/${observation}`, + sha256: template.subject.digest.sha256, + format: template.subject.format, + }, + template.provider, + { + operation: "analyze_function", + parameters: { ...template.parameters, observation }, + result, + confidence: template.confidence, + authority: template.authority, + relationshipCoverage, + }, + ); +}; + +const verify = async ( + call: ( + name: string, + arguments_: Record, + ) => Promise<{ + readonly isError?: boolean | undefined; + readonly structuredContent?: unknown | undefined; + }>, + comparison: ReturnType, + dimensions: readonly ("calls" | "overall" | "pseudocode")[], +) => { + const response = await call("verify_reconstruction", { + specification: { + name: "Function call coverage verification", + claims: dimensions.map((dimension) => ({ + kind: "structural-function", + claim_id: dimension, + title: `Check ${dimension}`, + comparison_evidence_id: comparison.evidence_id, + dimension, + })), + }, + }); + expect(response.isError, JSON.stringify(response.structuredContent)).not.toBe( + true, + ); + return parseEvidence(response.structuredContent); +}; + +const assertChangedDespiteUnknownCalls = async ( + call: Parameters[0], + left: ReturnType, +): Promise => { + const changedRight = sourceWithCoverage( + FUNCTION_COMPARISON_EXAMPLE.right, + "coverage-changed-right", + jsonValueSchema.parse({ + ...functionDossierSchema.parse( + FUNCTION_COMPARISON_EXAMPLE.right.normalized_result, + ), + pseudocode: "return 1;", + }), + { status: "unknown", reason: "Call graph unavailable." }, + ); + const changed = await call("compare_functions", { + left, + right: changedRight, + }); + expect(changed.isError, JSON.stringify(changed.structuredContent)).not.toBe( + true, + ); + const changedComparison = parseEvidence(changed.structuredContent); + expect(changedComparison.normalized_result).toMatchObject({ + status: "changed", + dimensions: expect.arrayContaining([ + expect.objectContaining({ dimension: "calls", status: "unknown" }), + expect.objectContaining({ dimension: "pseudocode", status: "changed" }), + ]), + }); + const verification = await verify(call, changedComparison, ["overall"]); + expect(verification.normalized_result).toMatchObject({ + status: "fail", + summary: { total: 1, passed: 0, failed: 1, unknown: 0 }, + claims: { + items: [ + expect.objectContaining({ + claim_id: "overall", + status: "fail", + observed_status: "changed", + }), + ], + }, + }); +}; + +it("keeps incomplete calls unknown through public comparison and verification while preserving confirmed changes", async () => { + const session = createTestBinarySession(() => { + throw new Error("Function comparison must not launch a provider"); + }); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "function-coverage-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const left = sourceWithCoverage( + FUNCTION_COMPARISON_EXAMPLE.left, + "coverage-left", + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, + { status: "complete" }, + ); + const right = sourceWithCoverage( + FUNCTION_COMPARISON_EXAMPLE.right, + "coverage-right", + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, + { status: "unknown", reason: "Call graph unavailable." }, + ); + expect(left.subject).not.toBeNull(); + expect(right.subject).not.toBeNull(); + expect(left.limitations).not.toContain( + "Artifact identity is unavailable for this observation.", + ); + expect(right.limitations).not.toContain( + "Artifact identity is unavailable for this observation.", + ); + const compared = await client.callTool({ + name: "compare_functions", + arguments: { left, right }, + }); + expect( + compared.isError, + JSON.stringify(compared.structuredContent), + ).not.toBe(true); + const comparison = parseEvidence(compared.structuredContent); + expect(session.evidenceById(left.evidence_id)?.subject).not.toBeNull(); + expect(session.evidenceById(right.evidence_id)?.subject).not.toBeNull(); + expect(comparison.normalized_result).toMatchObject({ + status: "unknown", + summary: { unchanged: 7, changed: 0, unknown: 1 }, + dimensions: expect.arrayContaining([ + expect.objectContaining({ + dimension: "calls", + status: "unknown", + conclusion_kind: "unresolved_branch", + left_count: 0, + right_count: 0, + limitations: ["Right: Call graph unavailable."], + evidence_links: [left.evidence_id, right.evidence_id], + }), + ]), + }); + expect(comparison.evidence_links).toEqual( + expect.arrayContaining([left.evidence_id, right.evidence_id]), + ); + + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "function-comparison" }, + }); + expect(unknowns.isError).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { + items: [ + expect.objectContaining({ + domain: "function-comparison", + status: "open", + supporting_evidence_ids: expect.arrayContaining([ + left.evidence_id, + right.evidence_id, + ]), + }), + ], + }, + }); + + const verification = await verify( + async (name, arguments_) => + client.callTool({ name, arguments: arguments_ }), + comparison, + ["calls", "overall", "pseudocode"], + ); + expect(verification.normalized_result).toMatchObject({ + status: "unknown", + summary: { total: 3, passed: 1, failed: 0, unknown: 2 }, + claims: { + items: expect.arrayContaining([ + expect.objectContaining({ + claim_id: "calls", + status: "unknown", + observed_status: "unknown", + }), + expect.objectContaining({ + claim_id: "overall", + status: "unknown", + observed_status: "unknown", + }), + expect.objectContaining({ + claim_id: "pseudocode", + status: "pass", + observed_status: "unchanged", + }), + ]), + }, + }); + + await assertChangedDespiteUnknownCalls( + async (name, arguments_) => + client.callTool({ name, arguments: arguments_ }), + left, + ); + } finally { + await Promise.allSettled([client.close(), server.close(), session.close()]); + } +});