mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
fix(evidence): preserve call coverage in function comparisons
Treat missing, partial, unknown, and unresolved call coverage as insufficient for call-set equality. Combine IDA caller/callee coverage without losing either explanation and recheck source coverage when verifying historical comparisons. Scope generated function-comparison unknowns with optional typed comparison/dimension facts, preserving legacy IDs and revision digests when absent. Keep unrelated dimensions conclusive, explicit dependencies applicable, and ambiguous function identity unknown. A confirmed change for matched functions remains a failure even if calls are incomplete. Validation: function/reconstruction/residual-unknown and bundle tests, recorded IDA protocol regression, public MCP comparison/verification, and advertised SDK JSON Schema checks. Real IDA provider execution was not verified. Fixes #2222
This commit is contained in:
1 parent
c89972c877
commit
049132ab43
17 files changed
+871
-63
No files matched your search
@@ -50,6 +50,7 @@ const observe = (digit: string, text: string) =>
|
||||
result: dossier(text),
|
||||
confidence: "derived",
|
||||
authority: "shipped-artifact",
|
||||
relationshipCoverage: { status: "complete" },
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import type { RelationshipCoverage } from "./relationshipCoverage.js";
|
||||
import type { FunctionDossier } from "./hopperValues.js";
|
||||
import { uniqueSorted } from "./canonicalOrdering.js";
|
||||
|
||||
/** Combine caller and callee read outcomes without losing either explanation. */
|
||||
export const combineFunctionRelationshipCoverage = (
|
||||
callers: RelationshipCoverage | undefined,
|
||||
callees: RelationshipCoverage | undefined,
|
||||
): RelationshipCoverage => {
|
||||
const reads = [
|
||||
{ direction: "callers" as const, coverage: callers },
|
||||
{ direction: "callees" as const, coverage: callees },
|
||||
];
|
||||
const diagnostics = uniqueSorted(
|
||||
reads.flatMap(({ coverage }) =>
|
||||
coverage !== undefined && "diagnostics" in coverage
|
||||
? coverage.diagnostics
|
||||
: [],
|
||||
),
|
||||
);
|
||||
const details = diagnostics.length > 0 ? { diagnostics } : {};
|
||||
const reasons = reads.flatMap(({ direction, coverage }) => {
|
||||
const label = direction === "callers" ? "Callers" : "Callees";
|
||||
const limitations =
|
||||
coverage === undefined
|
||||
? ["Provider did not report relationship coverage."]
|
||||
: coverage.status === "unknown"
|
||||
? [coverage.reason]
|
||||
: coverage.status === "partial"
|
||||
? coverage.limitations
|
||||
: [];
|
||||
return limitations.map((reason) => `${label}: ${reason}`);
|
||||
});
|
||||
if (
|
||||
reads.some(
|
||||
({ coverage }) => coverage === undefined || coverage.status === "unknown",
|
||||
)
|
||||
)
|
||||
return {
|
||||
status: "unknown",
|
||||
reason: uniqueSorted(reasons).join(" "),
|
||||
...details,
|
||||
};
|
||||
if (reasons.length > 0)
|
||||
return {
|
||||
status: "partial",
|
||||
limitations: uniqueSorted(reasons),
|
||||
...details,
|
||||
};
|
||||
return { status: "complete", ...details };
|
||||
};
|
||||
|
||||
/** Explain why two resolved call sets cannot establish call-set equality. */
|
||||
export const functionCallCoverageLimitations = (
|
||||
coverage: RelationshipCoverage | null,
|
||||
dossier: Pick<FunctionDossier, "unresolved_calls">,
|
||||
): string[] => {
|
||||
const limitations =
|
||||
coverage === null
|
||||
? ["Provider did not report relationship coverage."]
|
||||
: coverage.status === "unknown"
|
||||
? [
|
||||
coverage.reason,
|
||||
...("diagnostics" in coverage ? coverage.diagnostics : []),
|
||||
]
|
||||
: coverage.status === "partial"
|
||||
? [
|
||||
...coverage.limitations,
|
||||
...("diagnostics" in coverage ? coverage.diagnostics : []),
|
||||
]
|
||||
: [];
|
||||
limitations.push(
|
||||
...(dossier.unresolved_calls ?? []).map(
|
||||
({ address, reason }) => `Unresolved call at ${address}: ${reason}`,
|
||||
),
|
||||
);
|
||||
return uniqueSorted(limitations);
|
||||
};
|
||||
@@ -11,6 +11,7 @@ import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
|
||||
import { jsonValueSchema } from "./jsonValue.js";
|
||||
|
||||
import { compareFunctions } from "./functionComparison.js";
|
||||
import type { RelationshipCoverage } from "./relationshipCoverage.js";
|
||||
const procedure = (address: string, name: string) => ({
|
||||
address,
|
||||
name,
|
||||
@@ -96,6 +97,7 @@ const observe = (
|
||||
digit: string,
|
||||
value: ReturnType<typeof dossier>,
|
||||
providerId = "rea-workflow",
|
||||
relationshipCoverage: RelationshipCoverage | null = { status: "complete" },
|
||||
): Evidence =>
|
||||
createEvidence(
|
||||
{
|
||||
@@ -112,6 +114,7 @@ const observe = (
|
||||
result: jsonValueSchema.parse(value),
|
||||
confidence: "derived",
|
||||
authority: "shipped-artifact",
|
||||
...(relationshipCoverage === null ? {} : { relationshipCoverage }),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -187,6 +190,100 @@ describe("function comparison normalized identity", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("function call relationship coverage", () => {
|
||||
const coverageCases: {
|
||||
readonly label: string;
|
||||
readonly coverage: RelationshipCoverage | null;
|
||||
readonly limitation: string;
|
||||
}[] = [
|
||||
{
|
||||
label: "missing",
|
||||
coverage: null,
|
||||
limitation: "Provider did not report relationship coverage.",
|
||||
},
|
||||
{
|
||||
label: "partial",
|
||||
coverage: {
|
||||
status: "partial",
|
||||
limitations: ["Call graph stopped at the read limit."],
|
||||
},
|
||||
limitation: "Call graph stopped at the read limit.",
|
||||
},
|
||||
];
|
||||
it.each(coverageCases)(
|
||||
"keeps equal empty calls unknown for $label coverage",
|
||||
({ coverage, limitation }) => {
|
||||
const left = observe("a", dossier("return 0;", "0x1000"));
|
||||
const right = observe(
|
||||
"b",
|
||||
dossier("return 0;", "0x2000"),
|
||||
"rea-workflow",
|
||||
coverage,
|
||||
);
|
||||
const result = compareFunctions(left, right);
|
||||
const calls = result.dimensions.find(
|
||||
({ dimension }) => dimension === "calls",
|
||||
);
|
||||
expect(calls).toMatchObject({
|
||||
status: "unknown",
|
||||
conclusion_kind: "unresolved_branch",
|
||||
left_count: 0,
|
||||
right_count: 0,
|
||||
limitations: [`Right: ${limitation}`],
|
||||
});
|
||||
expect(result.status).toBe("unknown");
|
||||
},
|
||||
);
|
||||
|
||||
it("retains complete observations with execution diagnostics as comparable", () => {
|
||||
const coverage = {
|
||||
status: "complete" as const,
|
||||
diagnostics: ["Provider reported a non-call xref warning."],
|
||||
};
|
||||
const left = observe(
|
||||
"c",
|
||||
dossier("return 0;", "0x1000"),
|
||||
"rea-workflow",
|
||||
coverage,
|
||||
);
|
||||
const right = observe(
|
||||
"d",
|
||||
dossier("return 0;", "0x2000"),
|
||||
"rea-workflow",
|
||||
coverage,
|
||||
);
|
||||
expect(
|
||||
compareFunctions(left, right).dimensions.find(
|
||||
({ dimension }) => dimension === "calls",
|
||||
),
|
||||
).toMatchObject({ status: "unchanged", left_count: 0, right_count: 0 });
|
||||
});
|
||||
|
||||
it("keeps unresolved call sites unknown despite complete resolved-set coverage", () => {
|
||||
const left = observe("e", dossier("return 0;", "0x1000"));
|
||||
const rightDossier = functionDossierSchema.parse({
|
||||
...dossier("return 0;", "0x2000"),
|
||||
unresolved_calls: [
|
||||
{
|
||||
address: "0x2010",
|
||||
reason: "Indirect dispatch target could not be resolved.",
|
||||
},
|
||||
],
|
||||
});
|
||||
const right = observe("f", rightDossier);
|
||||
expect(
|
||||
compareFunctions(left, right).dimensions.find(
|
||||
({ dimension }) => dimension === "calls",
|
||||
),
|
||||
).toMatchObject({
|
||||
status: "unknown",
|
||||
limitations: [
|
||||
"Right: Unresolved call at 0x2010: Indirect dispatch target could not be resolved.",
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("function collection collation ties", () => {
|
||||
const composed = procedure("0x2000", "caf\u00e9");
|
||||
const decomposed = procedure("0x3000", "cafe\u0301");
|
||||
@@ -342,6 +439,7 @@ describe("function comparison ordering controls", () => {
|
||||
result: jsonValueSchema.parse(result),
|
||||
confidence: "derived",
|
||||
authority: "shipped-artifact",
|
||||
relationshipCoverage: { status: "complete" },
|
||||
},
|
||||
);
|
||||
const left = controlled("0x1000", false);
|
||||
@@ -525,7 +623,9 @@ describe("function comparison", () => {
|
||||
limitations: [expect.stringContaining("did not expose reference kinds")],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("function comparison identity and malformed CFG", () => {
|
||||
it("does not promote address-derived names through equal signatures", () => {
|
||||
const autoDossier = (base: "0x1000" | "0x2000") =>
|
||||
functionDossierSchema.parse({
|
||||
@@ -546,6 +646,22 @@ describe("function comparison", () => {
|
||||
comparison.dimensions.find(({ dimension }) => dimension === "identity"),
|
||||
).toMatchObject({ status: "unknown" });
|
||||
expect(comparison.status).toBe("unknown");
|
||||
const changedText = compareFunctions(
|
||||
observe("b", autoDossier("0x1000")),
|
||||
observe(
|
||||
"c",
|
||||
functionDossierSchema.parse({
|
||||
...autoDossier("0x2000"),
|
||||
pseudocode: "return 1;",
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(changedText.dimensions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ dimension: "pseudocode", status: "changed" }),
|
||||
]),
|
||||
);
|
||||
expect(changedText.status).toBe("unknown");
|
||||
});
|
||||
|
||||
it("rejects invalid CFG normalization and preserves newline differences", () => {
|
||||
|
||||
@@ -233,6 +233,18 @@ const compareCalls = (
|
||||
): FunctionDimension => {
|
||||
const leftValues = callProjection(left);
|
||||
const rightValues = callProjection(right);
|
||||
const coverageLimitations = [
|
||||
...left.callCoverageLimitations.map((reason) => `Left: ${reason}`),
|
||||
...right.callCoverageLimitations.map((reason) => `Right: ${reason}`),
|
||||
];
|
||||
if (coverageLimitations.length > 0)
|
||||
return unresolvedDimension({
|
||||
dimension: "calls",
|
||||
links: context.links,
|
||||
leftCount: leftValues.length,
|
||||
rightCount: rightValues.length,
|
||||
limitations: coverageLimitations,
|
||||
});
|
||||
if ([...leftValues, ...rightValues].some(({ name }) => isAutoName(name)))
|
||||
return unresolvedDimension({
|
||||
dimension: "calls",
|
||||
|
||||
@@ -69,15 +69,11 @@ export const overallStatus = (
|
||||
dimensions: readonly FunctionDimension[],
|
||||
match: FunctionComparisonResult["function_match"]["status"],
|
||||
): FunctionComparisonResult["status"] => {
|
||||
if (
|
||||
match === "ambiguous" ||
|
||||
dimensions.some(({ status }) => status === "unknown")
|
||||
)
|
||||
return "unknown";
|
||||
if (match === "mismatched") return "changed";
|
||||
return dimensions.some(({ status }) => status === "changed")
|
||||
? "changed"
|
||||
: "unchanged";
|
||||
if (match === "ambiguous") return "unknown";
|
||||
if (dimensions.some(({ status }) => status === "changed")) return "changed";
|
||||
if (dimensions.some(({ status }) => status === "unknown")) return "unknown";
|
||||
return "unchanged";
|
||||
};
|
||||
|
||||
export const summarize = (dimensions: readonly FunctionDimension[]) => ({
|
||||
|
||||
@@ -4,7 +4,7 @@ import { digestSchema } from "./../domain/digests.js";
|
||||
import { prefixedDigestSchema } from "./../domain/digests.js";
|
||||
|
||||
const comparisonStatusSchema = z.enum(["unchanged", "changed", "unknown"]);
|
||||
const dimensionNameSchema = z.enum([
|
||||
export const functionComparisonDimensionNameSchema = z.enum([
|
||||
"identity",
|
||||
"pseudocode",
|
||||
"assembly",
|
||||
@@ -29,7 +29,7 @@ const textDeltaSchema = z.object({
|
||||
});
|
||||
|
||||
const functionDimensionContextShape = {
|
||||
dimension: dimensionNameSchema,
|
||||
dimension: functionComparisonDimensionNameSchema,
|
||||
left_count: z.number().int().min(0).nullable(),
|
||||
right_count: z.number().int().min(0).nullable(),
|
||||
evidence_links: z.array(evidenceIdSchema).min(2),
|
||||
@@ -98,4 +98,6 @@ export type FunctionComparisonResult = z.infer<
|
||||
typeof functionComparisonResultSchema
|
||||
>;
|
||||
export type FunctionDimension = z.infer<typeof functionDimensionSchema>;
|
||||
export type DimensionName = z.infer<typeof dimensionNameSchema>;
|
||||
export type DimensionName = z.infer<
|
||||
typeof functionComparisonDimensionNameSchema
|
||||
>;
|
||||
@@ -1,6 +1,7 @@
|
||||
import { parseEvidence, type Evidence } from "./evidence.js";
|
||||
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
|
||||
import { functionDossierSchema, type FunctionDossier } from "./hopperValues.js";
|
||||
import { functionCallCoverageLimitations } from "./functionCallCoverage.js";
|
||||
|
||||
/** Complete function observation parsed from one analyze_function Evidence record. */
|
||||
export interface FunctionSnapshot {
|
||||
@@ -8,6 +9,7 @@ export interface FunctionSnapshot {
|
||||
readonly subject: NonNullable<Evidence["subject"]>;
|
||||
readonly dossier: FunctionDossier;
|
||||
readonly limitations: readonly string[];
|
||||
readonly callCoverageLimitations: readonly string[];
|
||||
}
|
||||
|
||||
/** Parse one complete analyze_function Evidence record for comparison workflows. */
|
||||
@@ -20,6 +22,10 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => {
|
||||
if (evidence.subject === null)
|
||||
throw new TypeError("Function comparison requires artifact-bound Evidence");
|
||||
const dossier = functionDossierSchema.parse(evidence.normalized_result);
|
||||
const callCoverageLimitations = functionCallCoverageLimitations(
|
||||
evidence.relationship_coverage,
|
||||
dossier,
|
||||
);
|
||||
return {
|
||||
evidence,
|
||||
subject: evidence.subject,
|
||||
@@ -27,5 +33,6 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => {
|
||||
limitations: [
|
||||
...new Set([...evidence.limitations, ...dossier.limitations]),
|
||||
].sort((left, right) => compareUnicodeCodePoints(left, right)),
|
||||
callCoverageLimitations,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,73 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createResidualUnknown } from "./residualUnknown.js";
|
||||
import { reconstructionClaimUnknowns } from "./reconstructionUnknowns.js";
|
||||
|
||||
const ev = (digit: string) => `ev_${digit.repeat(64)}`;
|
||||
const base = {
|
||||
severity: "high" as const,
|
||||
domain: "function-comparison",
|
||||
supporting_evidence_ids: [ev("a"), ev("b")],
|
||||
contradicting_evidence_ids: [],
|
||||
required_authority: null,
|
||||
required_confidence: "derived" as const,
|
||||
required_environment: null,
|
||||
recommended_probes: [],
|
||||
};
|
||||
|
||||
it("keeps explicit dependencies while excluding unrelated scoped unknowns", () => {
|
||||
const comparisonId = ev("a");
|
||||
const other = createResidualUnknown(
|
||||
{
|
||||
...base,
|
||||
question: "Other dimension",
|
||||
comparison_conclusion_scope: {
|
||||
comparison_evidence_id: comparisonId,
|
||||
dimensions: ["pseudocode"],
|
||||
},
|
||||
relationships: [],
|
||||
},
|
||||
ev("c"),
|
||||
null,
|
||||
);
|
||||
const calls = createResidualUnknown(
|
||||
{
|
||||
...base,
|
||||
question: "Call dimension",
|
||||
comparison_conclusion_scope: {
|
||||
comparison_evidence_id: comparisonId,
|
||||
dimensions: ["calls"],
|
||||
},
|
||||
relationships: [{ type: "depends-on", unknown_id: other.unknown_id }],
|
||||
},
|
||||
ev("d"),
|
||||
null,
|
||||
);
|
||||
const unrelated = createResidualUnknown(
|
||||
{
|
||||
...base,
|
||||
question: "Unrelated dimension",
|
||||
comparison_conclusion_scope: {
|
||||
comparison_evidence_id: comparisonId,
|
||||
dimensions: ["pseudocode"],
|
||||
},
|
||||
relationships: [],
|
||||
},
|
||||
ev("e"),
|
||||
null,
|
||||
);
|
||||
const selected = reconstructionClaimUnknowns(
|
||||
new Map([
|
||||
[calls.unknown_id, calls],
|
||||
[other.unknown_id, other],
|
||||
[unrelated.unknown_id, unrelated],
|
||||
]),
|
||||
comparisonId,
|
||||
[ev("b")],
|
||||
"calls",
|
||||
);
|
||||
expect(selected.map(({ unknown_id }) => unknown_id)).toEqual([
|
||||
calls.unknown_id,
|
||||
other.unknown_id,
|
||||
]);
|
||||
});
|
||||
@@ -28,13 +28,26 @@ export const reconstructionClaimUnknowns = (
|
||||
heads: ReadonlyMap<string, ResidualUnknown>,
|
||||
comparisonId: string,
|
||||
sourceIds: readonly string[],
|
||||
dimension?: string,
|
||||
): ResidualUnknown[] => {
|
||||
const appliesToClaim = (item: ResidualUnknown): boolean => {
|
||||
const scope = item.comparison_conclusion_scope;
|
||||
return (
|
||||
scope === undefined ||
|
||||
(scope.comparison_evidence_id === comparisonId &&
|
||||
(dimension === undefined ||
|
||||
scope.dimensions.some(
|
||||
(scopedDimension) => scopedDimension === dimension,
|
||||
)))
|
||||
);
|
||||
};
|
||||
const evidenceIds = new Set([comparisonId, ...sourceIds]);
|
||||
const relevantIds = new Set(
|
||||
[...heads.values()]
|
||||
.filter(
|
||||
(item) =>
|
||||
item.status !== "resolved" &&
|
||||
appliesToClaim(item) &&
|
||||
[
|
||||
...item.supporting_evidence_ids,
|
||||
...item.contradicting_evidence_ids,
|
||||
|
||||
@@ -69,6 +69,7 @@ const source = (
|
||||
result: dossier,
|
||||
confidence: "derived",
|
||||
authority: "shipped-artifact",
|
||||
relationshipCoverage: { status: "complete" },
|
||||
...observation,
|
||||
},
|
||||
);
|
||||
@@ -99,14 +100,15 @@ const comparison = (
|
||||
provider?: EvidenceProvider;
|
||||
target?: EvidenceSubjectTarget;
|
||||
} = {},
|
||||
resultInput = baselineResult,
|
||||
): Evidence =>
|
||||
createEvidence(identity.target, identity.provider ?? provider, {
|
||||
predicateType: "rea.function-comparison",
|
||||
operation: "compare_functions",
|
||||
parameters,
|
||||
result: jsonValueSchema.parse({
|
||||
...baselineResult,
|
||||
dimensions: baselineResult.dimensions.map((dimension) => ({
|
||||
...resultInput,
|
||||
dimensions: resultInput.dimensions.map((dimension) => ({
|
||||
...dimension,
|
||||
evidence_links: sources.map(({ evidence_id: id }) => id),
|
||||
})),
|
||||
@@ -195,6 +197,100 @@ describe("function reconstruction source parameters", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("function reconstruction call coverage", () => {
|
||||
it("rechecks old unchanged comparisons without affecting unrelated finite dimensions", () => {
|
||||
const incomplete = source("7", {
|
||||
relationshipCoverage: {
|
||||
status: "unknown",
|
||||
reason: "Call graph unavailable.",
|
||||
},
|
||||
});
|
||||
const incompleteSides = {
|
||||
left_evidence_id: left.evidence_id,
|
||||
right_evidence_id: incomplete.evidence_id,
|
||||
};
|
||||
const oldUnchanged = comparison(incompleteSides, {}, [left, incomplete]);
|
||||
for (const dimension of ["calls", "overall"] as const) {
|
||||
const result = admittedVerification(
|
||||
oldUnchanged,
|
||||
[left, incomplete],
|
||||
dimension,
|
||||
)();
|
||||
expect(result).toMatchObject({
|
||||
status: "unknown",
|
||||
summary: { total: 1, passed: 0, failed: 0, unknown: 1 },
|
||||
claims: {
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
dimension,
|
||||
status: "unknown",
|
||||
observed_status: "unchanged",
|
||||
limitations: expect.arrayContaining([
|
||||
`Source ${incomplete.evidence_id}: Call graph unavailable.`,
|
||||
]),
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
expect(
|
||||
admittedVerification(oldUnchanged, [left, incomplete], "pseudocode")(),
|
||||
).toMatchObject({
|
||||
status: "pass",
|
||||
claims: {
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
dimension: "pseudocode",
|
||||
status: "pass",
|
||||
observed_status: "unchanged",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a confirmed historical overall change as a failure despite unknown calls", () => {
|
||||
const changedDossier = jsonValueSchema.parse({
|
||||
...functionDossierSchema.parse(dossier),
|
||||
pseudocode: "return 1;",
|
||||
});
|
||||
const changedComplete = source("8", { result: changedDossier });
|
||||
const incomplete = source("9", {
|
||||
result: changedDossier,
|
||||
relationshipCoverage: {
|
||||
status: "unknown",
|
||||
reason: "Call graph unavailable.",
|
||||
},
|
||||
});
|
||||
const changedResult = compareFunctions(left, changedComplete);
|
||||
const oldChanged = comparison(
|
||||
{
|
||||
left_evidence_id: left.evidence_id,
|
||||
right_evidence_id: incomplete.evidence_id,
|
||||
},
|
||||
{},
|
||||
[left, incomplete],
|
||||
{},
|
||||
changedResult,
|
||||
);
|
||||
expect(oldChanged.normalized_result).toMatchObject({ status: "changed" });
|
||||
expect(
|
||||
admittedVerification(oldChanged, [left, incomplete], "overall")(),
|
||||
).toMatchObject({
|
||||
status: "fail",
|
||||
summary: { total: 1, passed: 0, failed: 1, unknown: 0 },
|
||||
claims: {
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
status: "fail",
|
||||
observed_status: "changed",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("function reconstruction source aliases", () => {
|
||||
it.each([
|
||||
["no source declarations", {}],
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
isProcessEvidenceProvider,
|
||||
} from "./process/processEvidenceProvider.js";
|
||||
import { functionDossierSchema } from "./hopperValues.js";
|
||||
import { functionCallCoverageLimitations } from "./functionCallCoverage.js";
|
||||
import {
|
||||
deriveProcessComparisonStatus,
|
||||
PROCESS_COMPARISON_DIMENSIONS,
|
||||
@@ -58,6 +59,11 @@ const providers = {
|
||||
},
|
||||
} as const;
|
||||
|
||||
interface FunctionSourceFacts {
|
||||
readonly evidence_id: string;
|
||||
readonly callCoverageLimitations: readonly string[];
|
||||
}
|
||||
|
||||
const acceptsPredicate = (
|
||||
expected: readonly string[],
|
||||
actual: string,
|
||||
@@ -137,14 +143,19 @@ const evaluateClaim = (
|
||||
throw new TypeError(`Missing source Evidence ${id}`);
|
||||
return source;
|
||||
});
|
||||
validateSourceKinds(claim, sources);
|
||||
const functionSourceFacts = readValidatedSourceFacts(claim, sources);
|
||||
const observed = observedStatus(claim, comparison);
|
||||
const limitations = authorityLimitations(claim, sources, sides);
|
||||
const relevantUnknowns = reconstructionClaimUnknowns(
|
||||
const limitations = [
|
||||
...authorityLimitations(claim, sources, sides),
|
||||
...sourceCallCoverageLimitations(claim, functionSourceFacts, observed),
|
||||
];
|
||||
const relatedUnknowns = reconstructionClaimUnknowns(
|
||||
heads,
|
||||
comparison.evidence_id,
|
||||
allSourceIds,
|
||||
claim.dimension,
|
||||
);
|
||||
const relevantUnknowns = relatedUnknowns;
|
||||
const status = classify(observed, limitations, relevantUnknowns);
|
||||
const displayedUnknowns = relevantUnknowns;
|
||||
return reconstructionClaimResultSchema.parse({
|
||||
@@ -168,10 +179,10 @@ const evaluateClaim = (
|
||||
});
|
||||
};
|
||||
|
||||
const validateSourceKinds = (
|
||||
const readValidatedSourceFacts = (
|
||||
claim: Claim,
|
||||
sources: readonly Evidence[],
|
||||
): void => {
|
||||
): readonly FunctionSourceFacts[] => {
|
||||
const expected =
|
||||
claim.kind === "behavioral"
|
||||
? ["capture_process_scenario", "rea.process-capture"]
|
||||
@@ -188,13 +199,22 @@ const validateSourceKinds = (
|
||||
throw new TypeError(
|
||||
"Comparison source Evidence has an unexpected observation type",
|
||||
);
|
||||
const functionSources: FunctionSourceFacts[] = [];
|
||||
for (const source of sources) {
|
||||
if (claim.kind === "behavioral")
|
||||
parseProcessCapture(source.normalized_result);
|
||||
else if (claim.kind === "structural-function")
|
||||
functionDossierSchema.parse(source.normalized_result);
|
||||
else artifactInventoryResultSchema.parse(source.normalized_result);
|
||||
else if (claim.kind === "structural-function") {
|
||||
const dossier = functionDossierSchema.parse(source.normalized_result);
|
||||
functionSources.push({
|
||||
evidence_id: source.evidence_id,
|
||||
callCoverageLimitations: functionCallCoverageLimitations(
|
||||
source.relationship_coverage,
|
||||
dossier,
|
||||
),
|
||||
});
|
||||
} else artifactInventoryResultSchema.parse(source.normalized_result);
|
||||
}
|
||||
return functionSources;
|
||||
};
|
||||
|
||||
const validateComparisonIdentity = (claim: Claim, evidence: Evidence): void => {
|
||||
@@ -317,6 +337,22 @@ const authorityLimitations = (
|
||||
return limitations;
|
||||
};
|
||||
|
||||
const sourceCallCoverageLimitations = (
|
||||
claim: Claim,
|
||||
functionSources: readonly FunctionSourceFacts[],
|
||||
observed: ObservedStatus,
|
||||
): string[] => {
|
||||
if (
|
||||
claim.kind !== "structural-function" ||
|
||||
(claim.dimension !== "calls" &&
|
||||
!(claim.dimension === "overall" && observed !== "changed"))
|
||||
)
|
||||
return [];
|
||||
return functionSources.flatMap(({ evidence_id, callCoverageLimitations }) =>
|
||||
callCoverageLimitations.map((reason) => `Source ${evidence_id}: ${reason}`),
|
||||
);
|
||||
};
|
||||
|
||||
const classify = (
|
||||
observed: ObservedStatus,
|
||||
limitations: readonly string[],
|
||||
|
||||
@@ -97,6 +97,49 @@ it("normalizes producer text before identity and revision authentication", () =>
|
||||
expect(residualUnknownSchema.parse(created)).toEqual(created);
|
||||
});
|
||||
|
||||
it("authenticates typed comparison scope and preserves it across revisions", () => {
|
||||
const comparisonId = `ev_${"c".repeat(64)}`;
|
||||
const scopedInput = recordUnknownInputSchema.parse({
|
||||
...input,
|
||||
domain: "function-comparison",
|
||||
supporting_evidence_ids: [comparisonId],
|
||||
comparison_conclusion_scope: {
|
||||
comparison_evidence_id: comparisonId,
|
||||
dimensions: ["calls"],
|
||||
},
|
||||
});
|
||||
const created = createResidualUnknown(scopedInput, mutationId, scopeDigest);
|
||||
const updated = updateResidualUnknown(
|
||||
created,
|
||||
{
|
||||
...input,
|
||||
unknown_id: created.unknown_id,
|
||||
expected_revision: 1,
|
||||
status: "investigating",
|
||||
severity: "high",
|
||||
supporting_evidence_ids: [comparisonId],
|
||||
required_confidence: "observed",
|
||||
recommended_probes: [],
|
||||
resolution: null,
|
||||
},
|
||||
mutationId,
|
||||
);
|
||||
expect(updated.comparison_conclusion_scope).toEqual({
|
||||
comparison_evidence_id: comparisonId,
|
||||
dimensions: ["calls"],
|
||||
});
|
||||
expect(residualUnknownSchema.parse(updated)).toEqual(updated);
|
||||
expect(
|
||||
residualUnknownSchema.safeParse({
|
||||
...created,
|
||||
comparison_conclusion_scope: {
|
||||
comparison_evidence_id: `ev_${"d".repeat(64)}`,
|
||||
dimensions: ["calls"],
|
||||
},
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps generated revisions subject to state and relationship validation", () => {
|
||||
const created = createResidualUnknown(input, mutationId, scopeDigest);
|
||||
const command = {
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
compositeKey,
|
||||
} from "./unicodeCodePointOrder.js";
|
||||
import { prefixedDigestSchema } from "./../domain/digests.js";
|
||||
import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js";
|
||||
|
||||
const evidenceIdSchema = prefixedDigestSchema("ev");
|
||||
const unknownIdSchema = prefixedDigestSchema("unk");
|
||||
@@ -23,6 +24,7 @@ const stableUnknownId = (
|
||||
question: string,
|
||||
domain: string,
|
||||
scopeDigest: string | null,
|
||||
comparisonConclusionScope?: z.infer<typeof comparisonConclusionScopeSchema>,
|
||||
): string =>
|
||||
`unk_${createHash("sha256")
|
||||
.update(
|
||||
@@ -30,6 +32,9 @@ const stableUnknownId = (
|
||||
domain: domain.trim(),
|
||||
question: question.replace(/\s+/gu, " ").trim(),
|
||||
scope_digest: scopeDigest,
|
||||
...(comparisonConclusionScope === undefined
|
||||
? {}
|
||||
: { comparison_conclusion_scope: comparisonConclusionScope }),
|
||||
}),
|
||||
)
|
||||
.digest("hex")}`;
|
||||
@@ -38,6 +43,23 @@ const probeSchema = z.object({
|
||||
operation: z.string().min(1),
|
||||
rationale: text,
|
||||
});
|
||||
const comparisonConclusionScopeSchema = z
|
||||
.object({
|
||||
comparison_evidence_id: evidenceIdSchema,
|
||||
dimensions: z
|
||||
.array(
|
||||
z.union([z.literal("overall"), functionComparisonDimensionNameSchema]),
|
||||
)
|
||||
.min(1),
|
||||
})
|
||||
.strict()
|
||||
.superRefine(({ dimensions }, context) => {
|
||||
if (!sameOrderedValues(dimensions, sortedUnique(dimensions)))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Comparison dimensions must be sorted and unique",
|
||||
});
|
||||
});
|
||||
const relationshipSchema = z.object({
|
||||
type: z.enum(["depends-on", "related-to", "contradicts"]),
|
||||
unknown_id: unknownIdSchema,
|
||||
@@ -78,6 +100,7 @@ const residualUnknownObjectSchema = z.object({
|
||||
]),
|
||||
severity: z.enum(["low", "medium", "high", "critical"]),
|
||||
domain: text,
|
||||
comparison_conclusion_scope: comparisonConclusionScopeSchema.exactOptional(),
|
||||
supporting_evidence_ids: z.array(evidenceIdSchema),
|
||||
contradicting_evidence_ids: z.array(evidenceIdSchema),
|
||||
required_authority: z
|
||||
@@ -97,6 +120,8 @@ const residualUnknownObjectSchema = z.object({
|
||||
mutation_evidence_ids: z.array(evidenceIdSchema).min(1),
|
||||
});
|
||||
|
||||
export type ResidualUnknown = z.infer<typeof residualUnknownSchema>;
|
||||
|
||||
type UnknownRevision = Omit<
|
||||
z.infer<typeof residualUnknownObjectSchema>,
|
||||
"revision_digest"
|
||||
@@ -105,7 +130,12 @@ type UnknownRevision = Omit<
|
||||
const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
|
||||
if (
|
||||
value.unknown_id !==
|
||||
stableUnknownId(value.question, value.domain, value.scope_digest)
|
||||
stableUnknownId(
|
||||
value.question,
|
||||
value.domain,
|
||||
value.scope_digest,
|
||||
value.comparison_conclusion_scope,
|
||||
)
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
@@ -153,6 +183,29 @@ const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
|
||||
code: "custom",
|
||||
message: "Evidence IDs must be unique",
|
||||
});
|
||||
if (
|
||||
value.comparison_conclusion_scope !== undefined &&
|
||||
![
|
||||
...value.supporting_evidence_ids,
|
||||
...value.contradicting_evidence_ids,
|
||||
].includes(value.comparison_conclusion_scope.comparison_evidence_id)
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message:
|
||||
"Scoped comparison Evidence must support or contradict the unknown",
|
||||
path: ["comparison_conclusion_scope"],
|
||||
});
|
||||
if (
|
||||
value.comparison_conclusion_scope !== undefined &&
|
||||
value.domain !== "function-comparison"
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message:
|
||||
"Comparison conclusion scope requires function-comparison domain",
|
||||
path: ["comparison_conclusion_scope"],
|
||||
});
|
||||
const contradictingEvidence = new Set(value.contradicting_evidence_ids);
|
||||
if (value.supporting_evidence_ids.some((id) => contradictingEvidence.has(id)))
|
||||
context.addIssue({
|
||||
@@ -170,7 +223,7 @@ const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
|
||||
});
|
||||
};
|
||||
|
||||
export const residualUnknownSchema = residualUnknownObjectSchema.superRefine(
|
||||
const residualUnknownValidatedSchema = residualUnknownObjectSchema.superRefine(
|
||||
(value, context) => {
|
||||
if (computeRevisionDigest(value) !== value.revision_digest)
|
||||
context.addIssue({
|
||||
@@ -182,11 +235,12 @@ export const residualUnknownSchema = residualUnknownObjectSchema.superRefine(
|
||||
},
|
||||
);
|
||||
|
||||
export const residualUnknownSchema = residualUnknownValidatedSchema;
|
||||
|
||||
const generatedRevisionSchema = residualUnknownObjectSchema
|
||||
.omit({ revision_digest: true })
|
||||
.superRefine(refineRevision);
|
||||
|
||||
export type ResidualUnknown = z.infer<typeof residualUnknownSchema>;
|
||||
export type UnknownStatus = ResidualUnknown["status"];
|
||||
|
||||
/** Input for deterministic creation of one residual unknown. */
|
||||
@@ -237,8 +291,21 @@ export const createResidualUnknown = (
|
||||
): ResidualUnknown => {
|
||||
const question = input.question.replace(/\s+/gu, " ").trim();
|
||||
const domain = input.domain.trim();
|
||||
const comparisonConclusionScope =
|
||||
input.comparison_conclusion_scope === undefined
|
||||
? undefined
|
||||
: {
|
||||
comparison_evidence_id:
|
||||
input.comparison_conclusion_scope.comparison_evidence_id,
|
||||
dimensions: input.comparison_conclusion_scope.dimensions,
|
||||
};
|
||||
const revision = {
|
||||
unknown_id: stableUnknownId(question, domain, scopeDigest),
|
||||
unknown_id: stableUnknownId(
|
||||
question,
|
||||
domain,
|
||||
scopeDigest,
|
||||
comparisonConclusionScope,
|
||||
),
|
||||
revision: 1,
|
||||
previous_revision_digest: null,
|
||||
scope_digest: scopeDigest,
|
||||
@@ -246,6 +313,9 @@ export const createResidualUnknown = (
|
||||
status:
|
||||
input.contradicting_evidence_ids.length > 0 ? "contradicted" : "open",
|
||||
domain,
|
||||
...(comparisonConclusionScope === undefined
|
||||
? {}
|
||||
: { comparison_conclusion_scope: comparisonConclusionScope }),
|
||||
...projectRevisionDetails(input),
|
||||
resolution: null,
|
||||
mutation_evidence_ids: [mutationEvidenceId],
|
||||
@@ -267,6 +337,9 @@ export const updateResidualUnknown = (
|
||||
question: current.question,
|
||||
status: input.status,
|
||||
domain: current.domain,
|
||||
...(current.comparison_conclusion_scope === undefined
|
||||
? {}
|
||||
: { comparison_conclusion_scope: current.comparison_conclusion_scope }),
|
||||
...projectRevisionDetails(input),
|
||||
resolution: input.resolution,
|
||||
mutation_evidence_ids: sortedUnique([
|
||||
@@ -307,7 +380,7 @@ const projectRevisionDetails = (input: RevisionDetailsInput) => ({
|
||||
});
|
||||
|
||||
const computeRevisionDigest = (
|
||||
value: Omit<ResidualUnknown, "revision_digest"> | ResidualUnknown,
|
||||
value: UnknownRevision | ResidualUnknown,
|
||||
): string => {
|
||||
const { revision_digest: _revisionDigest, ...semantic } =
|
||||
"revision_digest" in value ? value : { ...value, revision_digest: "" };
|
||||
@@ -323,6 +396,12 @@ const canonicalJson = (value: unknown): string => {
|
||||
|
||||
const sortedUnique = (values: readonly string[]): string[] =>
|
||||
[...new Set(values)].sort(compareUnicodeCodePoints);
|
||||
const sameOrderedValues = (
|
||||
left: readonly string[],
|
||||
right: readonly string[],
|
||||
): boolean =>
|
||||
left.length === right.length &&
|
||||
left.every((value, index) => value === right[index]);
|
||||
|
||||
const sortedRelationships = (
|
||||
values: readonly z.infer<typeof relationshipSchema>[],
|
||||
|
||||
@@ -223,11 +223,22 @@ describe("native IDA provider observations", () => {
|
||||
procedure: "main",
|
||||
});
|
||||
if (!dossier.ok) throw dossier.error;
|
||||
expect(dossier.value.relationshipCoverage).toEqual({
|
||||
expect(dossier.value.relationshipCoverage).toMatchObject({
|
||||
status: "unknown",
|
||||
reason:
|
||||
"Upstream xrefs do not establish direct caller relationships.",
|
||||
reason: more
|
||||
? expect.stringContaining(
|
||||
"Callees: Upstream callees capped results at 500; additional callees remain unknown.",
|
||||
)
|
||||
: expect.stringContaining(
|
||||
"Callers: Upstream xrefs do not establish direct caller relationships.",
|
||||
),
|
||||
});
|
||||
if (more)
|
||||
expect(dossier.value.relationshipCoverage).toMatchObject({
|
||||
reason: expect.stringContaining(
|
||||
"Callers: Upstream xrefs do not establish direct caller relationships.",
|
||||
),
|
||||
});
|
||||
expect(dossier.value.result).toMatchObject({
|
||||
callers: [],
|
||||
callees: [{ address: "0x2000", name: "puts" }],
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
} from "../domain/hopperValues.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import type { RelationshipCoverage } from "../domain/relationshipCoverage.js";
|
||||
import { combineFunctionRelationshipCoverage } from "../domain/functionCallCoverage.js";
|
||||
import { readCallGraphCallees, unavailableCallGraph } from "./IdaCallGraph.js";
|
||||
import type { IdaMcpConnection } from "./IdaMcpConnection.js";
|
||||
import type { IdaOperation } from "./IdaProviderCapabilities.js";
|
||||
@@ -70,33 +71,6 @@ interface IdaRelationships {
|
||||
readonly coverage: RelationshipCoverage;
|
||||
}
|
||||
|
||||
const combinedRelationshipCoverage = (
|
||||
callers: RelationshipCoverage,
|
||||
callees: RelationshipCoverage,
|
||||
): RelationshipCoverage => {
|
||||
const coverage = [callers, callees];
|
||||
const diagnostics = [
|
||||
...new Set(
|
||||
coverage.flatMap((item) =>
|
||||
"diagnostics" in item ? item.diagnostics : [],
|
||||
),
|
||||
),
|
||||
];
|
||||
const details = diagnostics.length > 0 ? { diagnostics } : {};
|
||||
const unknown = coverage.filter((item) => item.status === "unknown");
|
||||
if (unknown.length > 0)
|
||||
return {
|
||||
status: "unknown",
|
||||
reason: [...new Set(unknown.map((item) => item.reason))].join(" "),
|
||||
...details,
|
||||
};
|
||||
const limitations = coverage.flatMap((item) =>
|
||||
item.status === "partial" ? item.limitations : [],
|
||||
);
|
||||
return limitations.length > 0
|
||||
? { status: "partial", limitations, ...details }
|
||||
: { status: "complete", ...details };
|
||||
};
|
||||
/** The only lookup answers upstream reports for a name or address that is not a function. */
|
||||
const MISSING_PROCEDURE_ANSWERS: ReadonlySet<string> = new Set([
|
||||
"Not found",
|
||||
@@ -632,7 +606,7 @@ export class IdaOperationRunner {
|
||||
const pseudocode = await this.pseudocode(fn);
|
||||
const callers = await this.relationships(fn, "callers");
|
||||
const callees = await this.relationships(fn, "callees");
|
||||
this.relationshipCoverage = combinedRelationshipCoverage(
|
||||
this.relationshipCoverage = combineFunctionRelationshipCoverage(
|
||||
callers.coverage,
|
||||
callees.coverage,
|
||||
);
|
||||
|
||||
@@ -20,6 +20,7 @@ import { withAdmittedAnalysis } from "./analysisAdmission.js";
|
||||
import { runAdmittedToolOperation } from "./admittedToolOperation.js";
|
||||
|
||||
import { compareFunctions } from "../domain/functionComparison.js";
|
||||
import { uniqueSorted } from "../domain/canonicalOrdering.js";
|
||||
/** Register explicit Evidence-backed function comparison. */
|
||||
export const registerFunctionComparisonTool = (
|
||||
server: EvidenceMcpServer,
|
||||
@@ -102,12 +103,12 @@ export const registerFunctionComparisonTool = (
|
||||
const recorded = recordDerivedEvidence(
|
||||
session,
|
||||
evidence,
|
||||
functionUnknownInput({
|
||||
status: comparison.status,
|
||||
comparisonUnknownInput(
|
||||
comparison,
|
||||
leftIds,
|
||||
rightIds,
|
||||
comparisonEvidenceId: evidence.evidence_id,
|
||||
}),
|
||||
evidence.evidence_id,
|
||||
),
|
||||
);
|
||||
return server.delivery.toEvidenceToolResult(
|
||||
evidence,
|
||||
@@ -119,26 +120,63 @@ export const registerFunctionComparisonTool = (
|
||||
);
|
||||
};
|
||||
|
||||
const comparisonUnknownInput = (
|
||||
comparison: ReturnType<typeof compareFunctions>,
|
||||
leftIds: readonly string[],
|
||||
rightIds: readonly string[],
|
||||
comparisonEvidenceId: string,
|
||||
): RecordUnknownInput | undefined =>
|
||||
functionUnknownInput({
|
||||
status: comparison.status,
|
||||
dimensions: comparison.dimensions
|
||||
.filter(({ status }) =>
|
||||
comparison.status === "unknown"
|
||||
? status === "unknown"
|
||||
: status === "changed",
|
||||
)
|
||||
.map(({ dimension }) => dimension),
|
||||
leftIds,
|
||||
rightIds,
|
||||
comparisonEvidenceId,
|
||||
});
|
||||
|
||||
const functionUnknownInput = ({
|
||||
status,
|
||||
dimensions,
|
||||
leftIds,
|
||||
rightIds,
|
||||
comparisonEvidenceId,
|
||||
}: {
|
||||
status: ReturnType<typeof compareFunctions>["status"];
|
||||
dimensions: readonly ReturnType<
|
||||
typeof compareFunctions
|
||||
>["dimensions"][number]["dimension"][];
|
||||
leftIds: readonly string[];
|
||||
rightIds: readonly string[];
|
||||
comparisonEvidenceId: string;
|
||||
}): RecordUnknownInput | undefined => {
|
||||
if (status === "unchanged") return undefined;
|
||||
const supportingIds = [...new Set([...leftIds, ...rightIds])];
|
||||
const supportingIds = [
|
||||
...new Set([...leftIds, ...rightIds, comparisonEvidenceId]),
|
||||
];
|
||||
const contradictingIds = rightIds.filter((id) => !leftIds.includes(id));
|
||||
const contradictory = status === "changed";
|
||||
return {
|
||||
question: `Function comparison is ${status} (comparison ${comparisonEvidenceId})`,
|
||||
severity: status === "changed" ? "medium" : "high",
|
||||
domain: "function-comparison",
|
||||
supporting_evidence_ids: contradictory ? [...leftIds] : supportingIds,
|
||||
comparison_conclusion_scope: {
|
||||
comparison_evidence_id: comparisonEvidenceId,
|
||||
dimensions: uniqueSorted([
|
||||
...(status === "unknown" || dimensions.length === 0
|
||||
? ["overall" as const]
|
||||
: []),
|
||||
...dimensions,
|
||||
]),
|
||||
},
|
||||
supporting_evidence_ids: contradictory
|
||||
? [...leftIds, comparisonEvidenceId]
|
||||
: supportingIds,
|
||||
contradicting_evidence_ids: contradictory ? contradictingIds : [],
|
||||
required_authority: "shipped-artifact",
|
||||
required_confidence: "observed",
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
|
||||
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { functionDossierSchema } from "../../../src/domain/hopperValues.js";
|
||||
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
||||
import { createServer } from "../../../src/server/createServer.js";
|
||||
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
||||
|
||||
const sourceWithCoverage = (
|
||||
template: typeof FUNCTION_COMPARISON_EXAMPLE.left,
|
||||
observation: string,
|
||||
result: typeof template.normalized_result,
|
||||
relationshipCoverage:
|
||||
| { readonly status: "complete" }
|
||||
| { readonly status: "unknown"; readonly reason: string },
|
||||
) => {
|
||||
if (template.subject === null)
|
||||
throw new Error("Function comparison fixtures must be artifact-bound");
|
||||
return createEvidence(
|
||||
{
|
||||
path: `${template.subject.local_path}/${observation}`,
|
||||
sha256: template.subject.digest.sha256,
|
||||
format: template.subject.format,
|
||||
},
|
||||
template.provider,
|
||||
{
|
||||
operation: "analyze_function",
|
||||
parameters: { ...template.parameters, observation },
|
||||
result,
|
||||
confidence: template.confidence,
|
||||
authority: template.authority,
|
||||
relationshipCoverage,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const verify = async (
|
||||
call: (
|
||||
name: string,
|
||||
arguments_: Record<string, unknown>,
|
||||
) => Promise<{
|
||||
readonly isError?: boolean | undefined;
|
||||
readonly structuredContent?: unknown | undefined;
|
||||
}>,
|
||||
comparison: ReturnType<typeof parseEvidence>,
|
||||
dimensions: readonly ("calls" | "overall" | "pseudocode")[],
|
||||
) => {
|
||||
const response = await call("verify_reconstruction", {
|
||||
specification: {
|
||||
name: "Function call coverage verification",
|
||||
claims: dimensions.map((dimension) => ({
|
||||
kind: "structural-function",
|
||||
claim_id: dimension,
|
||||
title: `Check ${dimension}`,
|
||||
comparison_evidence_id: comparison.evidence_id,
|
||||
dimension,
|
||||
})),
|
||||
},
|
||||
});
|
||||
expect(response.isError, JSON.stringify(response.structuredContent)).not.toBe(
|
||||
true,
|
||||
);
|
||||
return parseEvidence(response.structuredContent);
|
||||
};
|
||||
|
||||
const assertChangedDespiteUnknownCalls = async (
|
||||
call: Parameters<typeof verify>[0],
|
||||
left: ReturnType<typeof parseEvidence>,
|
||||
): Promise<void> => {
|
||||
const changedRight = sourceWithCoverage(
|
||||
FUNCTION_COMPARISON_EXAMPLE.right,
|
||||
"coverage-changed-right",
|
||||
jsonValueSchema.parse({
|
||||
...functionDossierSchema.parse(
|
||||
FUNCTION_COMPARISON_EXAMPLE.right.normalized_result,
|
||||
),
|
||||
pseudocode: "return 1;",
|
||||
}),
|
||||
{ status: "unknown", reason: "Call graph unavailable." },
|
||||
);
|
||||
const changed = await call("compare_functions", {
|
||||
left,
|
||||
right: changedRight,
|
||||
});
|
||||
expect(changed.isError, JSON.stringify(changed.structuredContent)).not.toBe(
|
||||
true,
|
||||
);
|
||||
const changedComparison = parseEvidence(changed.structuredContent);
|
||||
expect(changedComparison.normalized_result).toMatchObject({
|
||||
status: "changed",
|
||||
dimensions: expect.arrayContaining([
|
||||
expect.objectContaining({ dimension: "calls", status: "unknown" }),
|
||||
expect.objectContaining({ dimension: "pseudocode", status: "changed" }),
|
||||
]),
|
||||
});
|
||||
const verification = await verify(call, changedComparison, ["overall"]);
|
||||
expect(verification.normalized_result).toMatchObject({
|
||||
status: "fail",
|
||||
summary: { total: 1, passed: 0, failed: 1, unknown: 0 },
|
||||
claims: {
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
claim_id: "overall",
|
||||
status: "fail",
|
||||
observed_status: "changed",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
it("keeps incomplete calls unknown through public comparison and verification while preserving confirmed changes", async () => {
|
||||
const session = createTestBinarySession(() => {
|
||||
throw new Error("Function comparison must not launch a provider");
|
||||
});
|
||||
const server = createServer({ kind: "session", session });
|
||||
const client = new Client({ name: "function-coverage-test", version: "1" });
|
||||
const [clientTransport, serverTransport] =
|
||||
InMemoryTransport.createLinkedPair();
|
||||
try {
|
||||
await server.connect(serverTransport);
|
||||
await client.connect(clientTransport);
|
||||
const left = sourceWithCoverage(
|
||||
FUNCTION_COMPARISON_EXAMPLE.left,
|
||||
"coverage-left",
|
||||
FUNCTION_COMPARISON_EXAMPLE.left.normalized_result,
|
||||
{ status: "complete" },
|
||||
);
|
||||
const right = sourceWithCoverage(
|
||||
FUNCTION_COMPARISON_EXAMPLE.right,
|
||||
"coverage-right",
|
||||
FUNCTION_COMPARISON_EXAMPLE.left.normalized_result,
|
||||
{ status: "unknown", reason: "Call graph unavailable." },
|
||||
);
|
||||
expect(left.subject).not.toBeNull();
|
||||
expect(right.subject).not.toBeNull();
|
||||
expect(left.limitations).not.toContain(
|
||||
"Artifact identity is unavailable for this observation.",
|
||||
);
|
||||
expect(right.limitations).not.toContain(
|
||||
"Artifact identity is unavailable for this observation.",
|
||||
);
|
||||
const compared = await client.callTool({
|
||||
name: "compare_functions",
|
||||
arguments: { left, right },
|
||||
});
|
||||
expect(
|
||||
compared.isError,
|
||||
JSON.stringify(compared.structuredContent),
|
||||
).not.toBe(true);
|
||||
const comparison = parseEvidence(compared.structuredContent);
|
||||
expect(session.evidenceById(left.evidence_id)?.subject).not.toBeNull();
|
||||
expect(session.evidenceById(right.evidence_id)?.subject).not.toBeNull();
|
||||
expect(comparison.normalized_result).toMatchObject({
|
||||
status: "unknown",
|
||||
summary: { unchanged: 7, changed: 0, unknown: 1 },
|
||||
dimensions: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
dimension: "calls",
|
||||
status: "unknown",
|
||||
conclusion_kind: "unresolved_branch",
|
||||
left_count: 0,
|
||||
right_count: 0,
|
||||
limitations: ["Right: Call graph unavailable."],
|
||||
evidence_links: [left.evidence_id, right.evidence_id],
|
||||
}),
|
||||
]),
|
||||
});
|
||||
expect(comparison.evidence_links).toEqual(
|
||||
expect.arrayContaining([left.evidence_id, right.evidence_id]),
|
||||
);
|
||||
|
||||
const unknowns = await client.callTool({
|
||||
name: "list_unknowns",
|
||||
arguments: { domain: "function-comparison" },
|
||||
});
|
||||
expect(unknowns.isError).not.toBe(true);
|
||||
expect(unknowns.structuredContent).toMatchObject({
|
||||
result: {
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
domain: "function-comparison",
|
||||
status: "open",
|
||||
supporting_evidence_ids: expect.arrayContaining([
|
||||
left.evidence_id,
|
||||
right.evidence_id,
|
||||
]),
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const verification = await verify(
|
||||
async (name, arguments_) =>
|
||||
client.callTool({ name, arguments: arguments_ }),
|
||||
comparison,
|
||||
["calls", "overall", "pseudocode"],
|
||||
);
|
||||
expect(verification.normalized_result).toMatchObject({
|
||||
status: "unknown",
|
||||
summary: { total: 3, passed: 1, failed: 0, unknown: 2 },
|
||||
claims: {
|
||||
items: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
claim_id: "calls",
|
||||
status: "unknown",
|
||||
observed_status: "unknown",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
claim_id: "overall",
|
||||
status: "unknown",
|
||||
observed_status: "unknown",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
claim_id: "pseudocode",
|
||||
status: "pass",
|
||||
observed_status: "unchanged",
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
|
||||
await assertChangedDespiteUnknownCalls(
|
||||
async (name, arguments_) =>
|
||||
client.callTool({ name, arguments: arguments_ }),
|
||||
left,
|
||||
);
|
||||
} finally {
|
||||
await Promise.allSettled([client.close(), server.close(), session.close()]);
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user