diff --git a/src/domain/reconstructionVerification.test.ts b/src/domain/reconstructionVerification.test.ts index 52fcd5a486..0103b7bf1d 100644 --- a/src/domain/reconstructionVerification.test.ts +++ b/src/domain/reconstructionVerification.test.ts @@ -615,3 +615,108 @@ it("keeps a valid comparison selected for the wrong claim kind as caller input", }); } }); + +describe("retained reconstruction source coverage", () => { + const omittedTerminalCapture = { + ...EMPTY_PROCESS_CAPTURE_EXAMPLE, + truncated: true, + truncation_details: { + ...EMPTY_PROCESS_CAPTURE_EXAMPLE.truncation_details, + raw_terminal: { + ...EMPTY_PROCESS_CAPTURE_EXAMPLE.truncation_details.raw_terminal, + observed_frames: 1, + observed_bytes: 1, + }, + }, + }; + + it("keeps old unchanged terminal and overall claims unknown while retaining unrelated claims", () => { + const left = source("1", "controlled-replay"); + const right = source("2", "controlled-replay", "observed", { + captureResult: omittedTerminalCapture, + }); + const comparison = processComparison(left, right, processResult()); + const bundle = createEvidenceBundle([left, right, comparison]); + const before = JSON.stringify(bundle); + const result = verifyReconstruction( + { + name: "Retained capture coverage", + claims: ["terminal", "overall", "exit", "interaction"].map( + (dimension) => ({ + kind: "behavioral", + claim_id: dimension, + title: dimension, + dimension, + comparison_evidence_id: comparison.evidence_id, + }), + ), + }, + bundle, + ); + expect( + result.claims.items.map(({ claim_id, status }) => [claim_id, status]), + ).toEqual([ + ["exit", "pass"], + ["interaction", "pass"], + ["overall", "unknown"], + ["terminal", "unknown"], + ]); + expect( + result.claims.items.find(({ claim_id }) => claim_id === "terminal") + ?.limitations, + ).toContain( + `Source ${right.evidence_id}: terminal observation coverage is incomplete.`, + ); + expect(JSON.stringify(bundle)).toBe(before); + }); + + it("retains a decisive changed claim despite omitted terminal observations", () => { + const left = source("1", "controlled-replay"); + const right = source("2", "controlled-replay", "observed", { + captureResult: omittedTerminalCapture, + }); + const comparison = processComparison(left, right, processResult("changed")); + const claims = ["terminal", "overall"].map((dimension) => ({ + kind: "behavioral", + claim_id: dimension, + title: dimension, + dimension, + comparison_evidence_id: comparison.evidence_id, + })); + expect( + verifyReconstruction( + { name: "Observed disagreement", claims }, + createEvidenceBundle([left, right, comparison]), + ), + ).toMatchObject({ + status: "fail", + summary: { failed: 2, unknown: 0 }, + }); + }); + + it.each(["status", "kind"] as const)( + "rejects a summary inconsistent with actual claim %s", + (field) => { + const left = source("1", "controlled-replay"); + const right = source("2", "controlled-replay"); + const comparison = processComparison(left, right, processResult()); + const result = verifyReconstruction( + behavioralSpec(comparison), + createEvidenceBundle([left, right, comparison]), + ); + const inconsistent = { + ...result, + claims: { + items: result.claims.items.map((claim) => + field === "status" + ? { ...claim, status: "fail", observed_status: "changed" } + : { ...claim, kind: "structural-function" }, + ), + }, + }; + expect( + reconstructionVerificationResultSchema.safeParse(inconsistent).success, + ).toBe(false); + }, + ); +}); diff --git a/src/domain/reconstructionVerification.ts b/src/domain/reconstructionVerification.ts index 7a1bc056fd..85fe3a4503 100644 --- a/src/domain/reconstructionVerification.ts +++ b/src/domain/reconstructionVerification.ts @@ -15,7 +15,16 @@ import { analysisInputErrorFromIssues } from "./inputIssueProjection.js"; import { readEvidencePayload } from "./evidenceValidation.js"; import { functionDossierSchema } from "./native/functionDossier.js"; import { assessFunctionCallCoverage } from "./functionCallCoverage.js"; -import { processCaptureSchema } from "./process/processCapture.js"; +import { + processCaptureSchema, + type UnverifiedProcessCapture, +} from "./process/processCapture.js"; +import { processSourceTruncated } from "./process/processCaptureCoverage.js"; +import { + PROCESS_COMPARISON_DIMENSIONS, + type ProcessComparisonDimension, +} from "./process/processComparison.js"; +import type { ProcessObservationSource } from "./process/processObservation.js"; import { digestCanonicalValue } from "./canonicalDigest.js"; import { reconstructionClaimResultSchema, @@ -45,6 +54,14 @@ interface FunctionSourceFacts { readonly callCoverageDiagnostics: readonly string[]; } +interface ValidatedSourceFacts { + readonly functions: readonly FunctionSourceFacts[]; + readonly processes: readonly { + readonly evidence_id: string; + readonly capture: UnverifiedProcessCapture; + }[]; +} + /** Verify only explicitly declared claims; no global equivalence is inferred. */ export const verifyReconstruction = ( specificationInput: unknown, @@ -144,17 +161,18 @@ const evaluateClaim = ( ); return source; }); - const functionSourceFacts = readValidatedSourceFacts(claim, sources); + const sourceFacts = readValidatedSourceFacts(claim, sources); const observation = comparisonObservation(claim, comparison); const observed = observation.status; const callCoverage = sourceCallCoverage( claim, - functionSourceFacts, + sourceFacts.functions, observation.hasIndependentFunctionChange === true, ); const blockers = [ ...authorityLimitations(claim, sources, sides), ...callCoverage.blockers, + ...sourceProcessCoverage(claim, sourceFacts.processes, observed), ]; const relevantUnknowns = reconstructionClaimUnknowns( heads, @@ -189,7 +207,7 @@ const evaluateClaim = ( const readValidatedSourceFacts = ( claim: Claim, sources: readonly Evidence[], -): readonly FunctionSourceFacts[] => { +): ValidatedSourceFacts => { const isExpectedType = (source: Evidence): boolean => { if (claim.kind === "behavioral") return ( @@ -216,6 +234,7 @@ const readValidatedSourceFacts = ( }, ); const functionSources: FunctionSourceFacts[] = []; + const processSources: ValidatedSourceFacts["processes"][number][] = []; for (const source of sources) { if (claim.kind === "behavioral") { const capture = readEvidencePayload( @@ -223,6 +242,10 @@ const readValidatedSourceFacts = ( source.normalized_result, ); if (!capture.ok) throw capture.error; + processSources.push({ + evidence_id: source.evidence_id, + capture: capture.value, + }); } else if (claim.kind === "structural-function") { const parsedDossier = readEvidencePayload( functionDossierSchema, @@ -256,7 +279,7 @@ const readValidatedSourceFacts = ( } } } - return functionSources; + return { functions: functionSources, processes: processSources }; }; const validateComparisonIdentity = ( @@ -404,6 +427,44 @@ const authorityLimitations = ( return limitations; }; +const processObservationSourceByDimension: Readonly< + Record +> = { + terminal: "terminal_rendered", + interaction: "interaction", + exit: "lifecycle", + filesystem: "filesystem", + process: "process", +}; + +const sourceProcessCoverage = ( + claim: Claim, + sources: ValidatedSourceFacts["processes"], + observed: ObservedStatus, +): string[] => { + if ( + claim.kind !== "behavioral" || + ["changed", "added", "removed", "contradiction"].includes(observed) + ) + return []; + const dimensions = + claim.dimension === "overall" + ? PROCESS_COMPARISON_DIMENSIONS + : [claim.dimension]; + return sources.flatMap(({ evidence_id, capture }) => + dimensions.flatMap((dimension) => + processSourceTruncated( + capture, + processObservationSourceByDimension[dimension], + ) + ? [ + `Source ${evidence_id}: ${dimension} observation coverage is incomplete.`, + ] + : [], + ), + ); +}; + const sourceCallCoverage = ( claim: Claim, functionSources: readonly FunctionSourceFacts[], diff --git a/src/domain/reconstructionVerificationSchemas.ts b/src/domain/reconstructionVerificationSchemas.ts index 1746456e3b..40003a25d1 100644 --- a/src/domain/reconstructionVerificationSchemas.ts +++ b/src/domain/reconstructionVerificationSchemas.ts @@ -118,9 +118,28 @@ export const reconstructionVerificationResultSchema = z }) .superRefine((result, context) => { const { summary } = result; + const claims = result.claims.items; const expectedStatus = summary.failed > 0 ? "fail" : summary.unknown > 0 ? "unknown" : "pass"; const issues = [ + [ + summary.passed !== + claims.filter(({ status }) => status === "pass").length || + summary.failed !== + claims.filter(({ status }) => status === "fail").length || + summary.unknown !== + claims.filter(({ status }) => status === "unknown").length, + "Claim status counts must summarize retained claims", + ["summary"], + ], + [ + summary.behavioral !== + claims.filter(({ kind }) => kind === "behavioral").length || + summary.structural !== + claims.filter(({ kind }) => kind !== "behavioral").length, + "Claim kind counts must summarize retained claims", + ["summary"], + ], [ summary.total !== summary.passed + summary.failed + summary.unknown, "Claim status counts must equal the summary total",