diff --git a/docs/reconstruction-coverage.md b/docs/reconstruction-coverage.md new file mode 100644 index 0000000000..d9912db92d --- /dev/null +++ b/docs/reconstruction-coverage.md @@ -0,0 +1,43 @@ +# Reconstruction coverage + +`rea evaluate-reconstruction-coverage coverage.json --json` evaluates an inline +coverage workspace and its selected `boundary_id`. MCP exposes the same input +through `evaluate_reconstruction_coverage`. Results contain a status, reasons, +recommended probes and Evidence references; the supplied bundle retains the +complete records. + +Evidence IDs and bundle manifests must authenticate before evaluation. A valid +record can still be insufficient proof: a passing verifier summary cannot +promote inferred Evidence into a replay observation. That case returns +`unknown`, while a missing or tampered record remains an input error. + +## Comparable verifier Evidence + +At least one record explicitly cited by the latest verifier observation must +support the declared contract authority: + +| Contract authority | Evidence source | Confidence | +| ------------------- | ----------------------------------------- | ----------------------- | +| `shipped-artifact` | `shipped-artifact` | `observed` or `derived` | +| `controlled-replay` | `controlled-replay` | `observed` | +| `live-observation` | `controlled-replay` or `external-service` | `observed` | +| `external` | `external-service` | `observed` | + +Contextual references can accompany qualifying proof. An unrelated observed +record elsewhere in the bundle cannot repair a result that cites only +inference. Older observed results cannot replace the newest weak result. +Equivalent observations at the same instant can share qualifying proof from +one of those observations; conflicting tied assessments remain unknown. + +## Package and integration proof + +Build, clean-install and runtime-routing claims require observed runtime +Evidence (`controlled-replay` or `external-service`). Package-content and +authority-independence audits can also use observed or derived +`shipped-artifact` Evidence. Each proof must cite its qualifying records and +cover the selected boundary's artifact digests. An inference alone establishes +neither a passing nor a failed package workflow. + +The evaluator preserves Evidence provenance and returns an actionable unknown +when qualification is unavailable. It does not change the cited records to +make their authority match a caller's declaration. diff --git a/docs/reconstruction-obligation-ledgers.md b/docs/reconstruction-obligation-ledgers.md index ffb0b9999c..e3e4b6f0dd 100644 --- a/docs/reconstruction-obligation-ledgers.md +++ b/docs/reconstruction-obligation-ledgers.md @@ -323,3 +323,6 @@ disposition but does not silently count it as verified. The tool returns every obligation inline. Its `ledger_id` and `closure_digest` identify the complete result, including summaries, reports, ownership and dependency graphs, Evidence links, and limitations. + +For named boundaries with explicit verifier contracts and package-proof kinds, +see [reconstruction coverage](reconstruction-coverage.md). diff --git a/src/domain/reconstructionCoverage.fixture.ts b/src/domain/reconstructionCoverage.fixture.ts index 2d1e37a19d..78d4b892be 100644 --- a/src/domain/reconstructionCoverage.fixture.ts +++ b/src/domain/reconstructionCoverage.fixture.ts @@ -16,6 +16,11 @@ const EVIDENCE_RECORDS = ["1", "2", "3", "4", "5"].map((character) => operation: `fixture-${character}`, parameters: {}, result: { character }, + confidence: "observed", + authority: + character === "3" || character === "4" + ? "controlled-replay" + : "shipped-artifact", }, ), ); diff --git a/src/domain/reconstructionCoverage.test.ts b/src/domain/reconstructionCoverage.test.ts index 628ad67c10..9a3d8c2b2e 100644 --- a/src/domain/reconstructionCoverage.test.ts +++ b/src/domain/reconstructionCoverage.test.ts @@ -14,6 +14,29 @@ import { const digest = (character: string): string => character.repeat(64); +it("preserves duplicate coverage owners instead of overwriting them in shared indexes", () => { + const workspace = completeReconstructionCoverageData(); + const owner = workspace.owners[0]; + if (owner === undefined) throw new Error("Expected fixture owner"); + const result = evaluateReconstructionClosure( + { + ...workspace, + owners: [owner, owner], + }, + "replacement.cli", + RECONSTRUCTION_COVERAGE_NOW, + ); + expect(result).toMatchObject({ + status: "partial", + reasons: expect.arrayContaining([ + expect.objectContaining({ + code: "owner-duplicate", + subject_id: "cli.help", + }), + ]), + }); +}); + describe("reconstruction coverage closure", () => { it("returns ready only when ownership, verification, package, and authority closure pass", () => { const workspace = completeReconstructionCoverageData(); diff --git a/src/domain/reconstructionCoverage.ts b/src/domain/reconstructionCoverage.ts index 04913d4ae3..715d0c6ba9 100644 --- a/src/domain/reconstructionCoverage.ts +++ b/src/domain/reconstructionCoverage.ts @@ -8,7 +8,11 @@ import { compositeKey, } from "./unicodeCodePointOrder.js"; -import { evidenceBundleSchema, parseEvidenceBundle } from "./evidenceBundle.js"; +import { + authenticateParsedEvidenceBundle, + evidenceBundleSchema, +} from "./evidenceBundle.js"; +import type { Evidence } from "./evidence.js"; import { evaluateReconstructionClaims, evaluateReconstructionPackageProofs, @@ -188,13 +192,37 @@ export type ReconstructionVerifierContract = z.infer< typeof verifierContractSchema >; +/** Owned coverage snapshot and its authenticated, unique lookup facts. */ +export interface ParsedReconstructionCoverage { + readonly coverage: ReconstructionCoverageData; + readonly evidenceById: ReadonlyMap; + readonly artifacts: ReadonlyMap< + string, + ReconstructionCoverageData["artifacts"][number] + >; + readonly surfaces: ReadonlyMap< + string, + ReconstructionCoverageData["surfaces"][number] + >; + readonly claims: ReadonlyMap< + string, + ReconstructionCoverageData["claims"][number] + >; + readonly ownersBySurface: ReadonlyMap< + string, + readonly ReconstructionCoverageData["owners"][number][] + >; +} + /** Parse inline coverage data and validate every evidence and authority link. */ export const createReconstructionCoverageData = ( input: unknown, -): ReconstructionCoverageData => { +): ReconstructionCoverageData => parseCoverageData(input).coverage; + +/** Retain authenticated Evidence facts alongside the validated coverage snapshot. */ +const parseCoverageData = (input: unknown): ParsedReconstructionCoverage => { const coverage = reconstructionCoverageDataSchema.parse(input); - validateCoverageReferences(coverage); - return coverage; + return { coverage, ...validateCoverageReferences(coverage) }; }; const closureReasonSchema = z.strictObject({ @@ -252,7 +280,6 @@ export type ReconstructionClosureResult = z.infer< typeof reconstructionClosureResultSchema >; -type CoverageData = z.infer; type ClosureReason = z.infer; /** Build a verifier contract whose digest prevents retrospective coverage edits. */ @@ -272,15 +299,15 @@ export const evaluateReconstructionClosure = ( boundaryId: string, nowEpochMs: number, ): ReconstructionClosureResult => { - const coverage = createReconstructionCoverageData(coverageInput); - const boundary = coverage.boundaries.find( + const parsed = parseCoverageData(coverageInput); + const boundary = parsed.coverage.boundaries.find( ({ boundary_id: id }) => id === boundaryId, ); if (boundary === undefined) throw new TypeError(`Unknown reconstruction boundary: ${boundaryId}`); const nowInstant = offsetDateTimeInstantFromEpochMilliseconds(nowEpochMs); const context: ReconstructionEvaluationContext = { - coverage, + ...parsed, boundary, nowEpochMs, nowInstant, @@ -306,21 +333,25 @@ export const evaluateReconstructionClosure = ( }); }; -const validateCoverageReferences = (workspace: CoverageData): void => { - const evidenceBundle = parseEvidenceBundle(workspace.evidence_bundle); - const evidenceIds = new Set( - evidenceBundle.records.map(({ evidence_id: id }) => id), +const validateCoverageReferences = ( + workspace: ReconstructionCoverageData, +): Omit => { + const evidenceBundle = authenticateParsedEvidenceBundle( + workspace.evidence_bundle, + ); + const evidenceById = new Map( + evidenceBundle.records.map((record) => [record.evidence_id, record]), ); const unknownIds = new Set( evidenceBundle.unknowns.map(({ unknown_id: id }) => id), ); - const artifacts = assertUnique(workspace.artifacts, "artifact_id"); - const surfaces = assertUnique(workspace.surfaces, "surface_id"); - const claims = assertUnique(workspace.claims, "claim_id"); - const contracts = assertUnique(workspace.verifier_contracts, "verifier_id"); - assertUnique(workspace.contradictions, "contradiction_id"); - assertUnique(workspace.package_proofs, "proof_id"); - assertUnique(workspace.boundaries, "boundary_id"); + const artifacts = indexUnique(workspace.artifacts, "artifact_id"); + const surfaces = indexUnique(workspace.surfaces, "surface_id"); + const claims = indexUnique(workspace.claims, "claim_id"); + const contracts = indexUnique(workspace.verifier_contracts, "verifier_id"); + indexUnique(workspace.contradictions, "contradiction_id"); + indexUnique(workspace.package_proofs, "proof_id"); + indexUnique(workspace.boundaries, "boundary_id"); for (const contract of workspace.verifier_contracts) { const { contract_sha256: contractSha256, ...base } = contract; if (contractSha256 !== digest(verifierContractBaseSchema.parse(base))) @@ -331,9 +362,17 @@ const validateCoverageReferences = (workspace: CoverageData): void => { for (const surface of workspace.surfaces) if (!artifacts.has(surface.artifact_id)) throw new TypeError(`Surface artifact is missing: ${surface.surface_id}`); - for (const owner of workspace.owners) + const ownersBySurface = new Map< + string, + ReconstructionCoverageData["owners"][number][] + >(); + for (const owner of workspace.owners) { if (!surfaces.has(owner.surface_id)) throw new TypeError(`Owner surface is missing: ${owner.surface_id}`); + const owners = ownersBySurface.get(owner.surface_id); + if (owners === undefined) ownersBySurface.set(owner.surface_id, [owner]); + else owners.push(owner); + } for (const claim of workspace.claims) if (claim.surface_ids.some((id) => !surfaces.has(id))) throw new TypeError(`Claim surface is missing: ${claim.claim_id}`); @@ -353,7 +392,7 @@ const validateCoverageReferences = (workspace: CoverageData): void => { ...workspace.package_proofs.flatMap(({ evidence_ids: ids }) => ids), ]; const danglingEvidence = referencedEvidenceIds.find( - (id) => !evidenceIds.has(id), + (id) => !evidenceById.has(id), ); if (danglingEvidence !== undefined) throw new TypeError( @@ -366,16 +405,20 @@ const validateCoverageReferences = (workspace: CoverageData): void => { throw new TypeError( `Coverage workspace residual unknown is missing: ${danglingUnknown}`, ); + return { evidenceById, artifacts, surfaces, claims, ownersBySurface }; }; -const assertUnique = , Key extends keyof Item>( +const indexUnique = , Key extends keyof Item>( items: readonly Item[], key: Key, -): Set => { - const values = items.map((item) => item[key]); - if (new Set(values).size !== values.length) - throw new TypeError(`Duplicate reconstruction coverage ${String(key)}`); - return new Set(values); +): ReadonlyMap => { + const values = new Map(); + for (const item of items) { + if (values.has(item[key])) + throw new TypeError(`Duplicate reconstruction coverage ${String(key)}`); + values.set(item[key], item); + } + return values; }; const closureStatus = ( diff --git a/src/domain/reconstructionCoverageEvaluation.ts b/src/domain/reconstructionCoverageEvaluation.ts index f8a56b261a..fb92ace929 100644 --- a/src/domain/reconstructionCoverageEvaluation.ts +++ b/src/domain/reconstructionCoverageEvaluation.ts @@ -2,7 +2,12 @@ import type { ReconstructionClosureResult, ReconstructionCoverageData, ReconstructionVerifierContract, + ParsedReconstructionCoverage, } from "./reconstructionCoverage.js"; +import { + packageProofHasComparableEvidence, + verifierHasComparableEvidence, +} from "./reconstructionCoverageEvidence.js"; import { compareOffsetDateTimeInstants, offsetDateTimeInstantFromEpochMilliseconds, @@ -35,8 +40,7 @@ const probeOperation = (code: ClosureReason["code"]): string => { type Boundary = ReconstructionCoverageData["boundaries"][number]; type ClosureReason = ReconstructionClosureResult["reasons"][number]; -export interface ReconstructionEvaluationContext { - readonly coverage: ReconstructionCoverageData; +export interface ReconstructionEvaluationContext extends ParsedReconstructionCoverage { readonly boundary: Boundary; readonly nowEpochMs: number; readonly nowInstant: OffsetDateTimeInstant; @@ -47,9 +51,8 @@ export interface ReconstructionEvaluationContext { export const evaluateReconstructionSurfaces = ( context: ReconstructionEvaluationContext, ): void => { - const surfaces = indexUnique(context.coverage.surfaces, "surface_id"); for (const surfaceId of context.boundary.required_surface_ids) { - const surface = surfaces.get(surfaceId); + const surface = context.surfaces.get(surfaceId); if (surface === undefined) { addReason( context, @@ -61,7 +64,7 @@ export const evaluateReconstructionSurfaces = ( } addEvidence(context, surface.evidence_ids); for (const dependencyId of surface.dependency_surface_ids) - if (!surfaces.has(dependencyId)) + if (!context.surfaces.has(dependencyId)) addReason( context, "dependency-surface-missing", @@ -76,9 +79,7 @@ const evaluateOwner = ( context: ReconstructionEvaluationContext, surfaceId: string, ): void => { - const owners = context.coverage.owners.filter( - ({ surface_id: id }) => id === surfaceId, - ); + const owners = context.ownersBySurface.get(surfaceId) ?? []; if (owners.length === 0) { addReason( context, @@ -150,10 +151,9 @@ const evaluateOwner = ( export const evaluateReconstructionClaims = ( context: ReconstructionEvaluationContext, ): void => { - const claims = indexUnique(context.coverage.claims, "claim_id"); - const indexes = createVerifierIndexes(context.coverage); + const indexes = createVerifierIndexes(context); for (const claimId of context.boundary.required_claim_ids) { - const claim = claims.get(claimId); + const claim = context.claims.get(claimId); if (claim === undefined) { addReason( context, @@ -214,8 +214,9 @@ interface VerifierIndexes { } const createVerifierIndexes = ( - coverage: ReconstructionCoverageData, + context: ReconstructionEvaluationContext, ): VerifierIndexes => { + const { coverage } = context; const contractsByClaim = new Map(); for (const contract of coverage.verifier_contracts) for (const claimId of new Set(contract.claim_ids)) { @@ -224,16 +225,6 @@ const createVerifierIndexes = ( contractsByClaim.set(claimId, contracts); } - const ownersBySurface = new Map< - string, - ReconstructionCoverageData["owners"][number][] - >(); - for (const owner of coverage.owners) { - const owners = ownersBySurface.get(owner.surface_id) ?? []; - owners.push(owner); - ownersBySurface.set(owner.surface_id, owners); - } - return { contractsByClaim, latestResultsByVerifier: latestVerifierObservationsByVerifier( @@ -247,9 +238,9 @@ const createVerifierIndexes = ( ownership.disposition === "implemented" ? [ownership.owner_sha256] : [], ), ), - surfaces: indexUnique(coverage.surfaces, "surface_id"), - artifacts: indexUnique(coverage.artifacts, "artifact_id"), - ownersBySurface, + surfaces: context.surfaces, + artifacts: context.artifacts, + ownersBySurface: context.ownersBySurface, }; }; @@ -277,10 +268,9 @@ const evaluateVerifierResult = ( ); const latest = latestResults[0]; if (latest === undefined) return; - const { result } = latest; if ( !latestResults.every((candidate) => - sameVerifierAssessment(result, candidate.result), + sameVerifierAssessment(latest.result, candidate.result), ) ) { addReason( @@ -291,7 +281,20 @@ const evaluateVerifierResult = ( ); return; } - if (!verifierResultIsCompatible(claim, contract, result, indexes)) { + const comparable = latestResults.find(({ result }) => + verifierHasComparableEvidence(result, contract, context.evidenceById), + ); + if (comparable === undefined) { + addReason( + context, + "verifier-result-incompatible", + claim.claim_id, + `Latest verifier observation has no cited Evidence with comparable ${contract.authority} authority and confidence.`, + ); + return; + } + const { result } = comparable; + if (!verifierResultMatchesContract(claim, contract, result, indexes)) { addReason( context, "verifier-result-incompatible", @@ -302,10 +305,10 @@ const evaluateVerifierResult = ( } const cutoffEpochMs = context.nowEpochMs - contract.max_age_ms; if ( - compareOffsetDateTimeInstants(latest.instant, context.nowInstant) > 0 || + compareOffsetDateTimeInstants(comparable.instant, context.nowInstant) > 0 || (cutoffEpochMs >= -Number.MAX_SAFE_INTEGER && compareOffsetDateTimeInstants( - latest.instant, + comparable.instant, offsetDateTimeInstantFromEpochMilliseconds(cutoffEpochMs), ) < 0) ) { @@ -333,7 +336,7 @@ const evaluateVerifierResult = ( ); }; -const verifierResultIsCompatible = ( +const verifierResultMatchesContract = ( claim: ReconstructionCoverageData["claims"][number], contract: ReconstructionVerifierContract, result: ReconstructionCoverageData["verifier_results"][number], @@ -415,20 +418,13 @@ export const evaluateReconstructionCoverageRisks = ( export const evaluateReconstructionPackageProofs = ( context: ReconstructionEvaluationContext, ): void => { - const surfaces = new Map( - context.coverage.surfaces.map((surface) => [surface.surface_id, surface]), - ); - const artifacts = new Map( - context.coverage.artifacts.map((artifact) => [ - artifact.artifact_id, - artifact, - ]), - ); const requiredArtifactDigests = context.boundary.required_surface_ids.flatMap( (surfaceId) => { - const surface = surfaces.get(surfaceId); + const surface = context.surfaces.get(surfaceId); const artifact = - surface === undefined ? undefined : artifacts.get(surface.artifact_id); + surface === undefined + ? undefined + : context.artifacts.get(surface.artifact_id); return artifact === undefined ? [] : [artifact.artifact_sha256]; }, ); @@ -436,30 +432,42 @@ export const evaluateReconstructionPackageProofs = ( const proofs = context.coverage.package_proofs.filter( (proof) => proof.kind === kind, ); + const applicable = ( + proof: ReconstructionCoverageData["package_proofs"][number], + ): boolean => + requiredArtifactDigests.every((digest) => + proof.artifact_sha256s.includes(digest), + ) && packageProofHasComparableEvidence(proof, context.evidenceById); const passing = proofs.find( - ({ artifact_sha256s: digests, status }) => - status === "pass" && - requiredArtifactDigests.every((digest) => digests.includes(digest)), + (proof) => proof.status === "pass" && applicable(proof), ); if (passing !== undefined) { addEvidence(context, passing.evidence_ids); continue; } - if (proofs.some(({ status }) => status === "fail")) + const failing = proofs.find( + (proof) => proof.status === "fail" && applicable(proof), + ); + if (failing !== undefined) { + addEvidence(context, failing.evidence_ids); addReason( context, "package-proof-failed", kind, "Required package or integration proof failed.", ); - else if (proofs.length > 0) + } else if (proofs.length > 0) { + addEvidence( + context, + proofs.flatMap(({ evidence_ids }) => evidence_ids), + ); addReason( context, "package-proof-unknown", kind, - "Required package or integration proof is not passing.", + "Required package or integration proof has no qualifying passing Evidence.", ); - else + } else addReason( context, "package-proof-missing", @@ -469,11 +477,6 @@ export const evaluateReconstructionPackageProofs = ( } }; -const indexUnique = , Key extends keyof Item>( - items: readonly Item[], - key: Key, -): ReadonlyMap => new Map(items.map((item) => [item[key], item])); - const addReason = ( context: ReconstructionEvaluationContext, code: ClosureReason["code"], diff --git a/src/domain/reconstructionCoverageEvidence.test.ts b/src/domain/reconstructionCoverageEvidence.test.ts new file mode 100644 index 0000000000..aea275a3d9 --- /dev/null +++ b/src/domain/reconstructionCoverageEvidence.test.ts @@ -0,0 +1,244 @@ +import { expect, it } from "vitest"; + +import { createEvidence, type Evidence } from "./evidence.js"; +import { createEvidenceBundle } from "./evidenceBundle.js"; +import { + completeReconstructionCoverageFixture, + RECONSTRUCTION_COVERAGE_NOW, +} from "./reconstructionCoverage.fixture.js"; +import { + createReconstructionCoverageData, + createReconstructionVerifierContract, + evaluateReconstructionClosure, + type ReconstructionCoverageData, +} from "./reconstructionCoverage.js"; + +const proof = ( + authority: Evidence["authority"], + confidence: Evidence["confidence"], +) => + createEvidence( + undefined, + { id: "proof-test", name: "Proof fixture", version: "1" }, + { + operation: "verify", + parameters: {}, + authority, + confidence, + result: + confidence === "observed" || + (authority === "shipped-artifact" && confidence === "derived") + ? { status: "pass" } + : { status: "unknown", note: "No replay was performed" }, + }, + ); + +const withProofs = (records: readonly Evidence[]) => { + const { workspace, result } = completeReconstructionCoverageFixture(); + return createReconstructionCoverageData({ + ...workspace, + evidence_bundle: createEvidenceBundle([ + ...workspace.evidence_bundle.records, + ...records, + ]), + verifier_results: [ + { + ...result, + evidence_ids: records.map(({ evidence_id }) => evidence_id), + }, + ], + }); +}; + +const withPackageProof = ( + kind: ReconstructionCoverageData["package_proofs"][number]["kind"], + record: Evidence, + status: ReconstructionCoverageData["package_proofs"][number]["status"] = "pass", +) => { + const { workspace } = completeReconstructionCoverageFixture(); + return createReconstructionCoverageData({ + ...workspace, + evidence_bundle: createEvidenceBundle([ + ...workspace.evidence_bundle.records, + record, + ]), + package_proofs: [ + { + proof_id: "selected", + kind, + status, + artifact_sha256s: ["1".repeat(64)], + evidence_ids: [record.evidence_id], + }, + ], + boundaries: workspace.boundaries.map((boundary) => ({ + ...boundary, + required_package_proof_kinds: [kind], + })), + }); +}; + +const evaluate = (coverage: ReconstructionCoverageData) => + evaluateReconstructionClosure( + coverage, + "replacement.cli", + RECONSTRUCTION_COVERAGE_NOW, + ); + +it.each([ + ["analyst-inference", "inferred"], + ["analyst-inference", "derived"], + ["analyst-inference", "observed"], + ["controlled-replay", "inferred"], + ["controlled-replay", "derived"], + ["shipped-artifact", "observed"], + ["external-service", "observed"], + ["historical-reference", "observed"], +] as const)( + "does not promote %s/%s into observed controlled replay", + (authority, confidence) => { + const record = proof(authority, confidence); + const coverage = withProofs([record]); + const result = evaluate(coverage); + expect(result).toMatchObject({ + status: "unknown", + reasons: [ + expect.objectContaining({ + code: "verifier-result-incompatible", + subject_id: "claim.cli.help", + }), + ], + evidence_ids: expect.arrayContaining([record.evidence_id]), + recommended_probes: [ + expect.objectContaining({ operation: "run_reconstruction_verifier" }), + ], + }); + expect( + coverage.evidence_bundle.records.find( + ({ evidence_id }) => evidence_id === record.evidence_id, + ), + ).toEqual(record); + expect(result.reasons[0]?.detail).toContain( + "controlled-replay authority and confidence", + ); + }, +); + +it("accepts cited observed replay alongside contextual inference without borrowing uncited proof", () => { + const weak = proof("analyst-inference", "inferred"); + const observed = proof("controlled-replay", "observed"); + expect(evaluate(withProofs([weak, observed]))).toMatchObject({ + status: "ready", + evidence_ids: expect.arrayContaining([ + weak.evidence_id, + observed.evidence_id, + ]), + }); + expect(evaluate(withProofs([weak])).status).toBe("unknown"); +}); + +it("does not fall back to older observed proof when the latest result only cites inference", () => { + const record = proof("analyst-inference", "inferred"); + const coverage = withProofs([record]); + const { result: previous } = completeReconstructionCoverageFixture(); + const latest = coverage.verifier_results[0]; + if (latest === undefined) throw new Error("Expected verifier result"); + const result = evaluate( + createReconstructionCoverageData({ + ...coverage, + verifier_results: [ + previous, + { ...latest, observed_at: "2026-07-16T11:30:00.000Z" }, + ], + }), + ); + expect(result.status).toBe("unknown"); + expect(result.evidence_ids).not.toContain(previous.evidence_ids[0]); +}); + +it.each([ + ["shipped-artifact", "shipped-artifact", "observed", "ready"], + ["shipped-artifact", "shipped-artifact", "derived", "ready"], + ["shipped-artifact", "shipped-artifact", "inferred", "unknown"], + ["live-observation", "external-service", "observed", "ready"], + ["live-observation", "controlled-replay", "observed", "ready"], + ["external", "external-service", "observed", "ready"], + ["external", "controlled-replay", "observed", "unknown"], +] as const)( + "checks %s authority against actual %s/%s Evidence", + (required, authority, confidence, status) => { + const coverage = withProofs([proof(authority, confidence)]); + const original = coverage.verifier_contracts[0]; + const result = coverage.verifier_results[0]; + if (original === undefined || result === undefined) + throw new Error("Expected verifier contract"); + const { contract_sha256: _digest, ...input } = original; + const contract = createReconstructionVerifierContract({ + ...input, + authority: required, + }); + const changed = createReconstructionCoverageData({ + ...coverage, + claims: coverage.claims.map((claim) => ({ + ...claim, + required_authority: required, + })), + verifier_contracts: [contract], + verifier_results: [ + { ...result, contract_sha256: contract.contract_sha256 }, + ], + }); + expect(evaluate(changed).status).toBe(status); + }, +); + +it.each([ + "build", + "clean-install", + "package-contents", + "runtime-routing", + "authority-independence", +] as const)("keeps %s unknown when only inferred Evidence is cited", (kind) => { + const record = proof("analyst-inference", "inferred"); + const coverage = withPackageProof(kind, record); + expect(evaluate(coverage)).toMatchObject({ + status: "unknown", + reasons: [ + expect.objectContaining({ + code: "package-proof-unknown", + subject_id: kind, + }), + ], + evidence_ids: expect.arrayContaining([record.evidence_id]), + }); +}); + +it.each([ + ["build", "shipped-artifact", "observed", "unknown"], + ["build", "external-service", "observed", "ready"], + ["runtime-routing", "controlled-replay", "observed", "ready"], + ["clean-install", "controlled-replay", "derived", "unknown"], + ["package-contents", "shipped-artifact", "derived", "ready"], + ["authority-independence", "shipped-artifact", "derived", "ready"], +] as const)( + "checks %s proof at the actual %s/%s boundary", + (kind, authority, confidence, status) => { + expect( + evaluate(withPackageProof(kind, proof(authority, confidence))).status, + ).toBe(status); + }, +); + +it("distinguishes observed package failure from an inferred failure claim", () => { + const inference = proof("analyst-inference", "inferred"); + expect(evaluate(withPackageProof("build", inference, "fail"))).toMatchObject({ + status: "unknown", + evidence_ids: expect.arrayContaining([inference.evidence_id]), + }); + const observed = proof("controlled-replay", "observed"); + expect(evaluate(withPackageProof("build", observed, "fail"))).toMatchObject({ + status: "failed", + reasons: [expect.objectContaining({ code: "package-proof-failed" })], + evidence_ids: expect.arrayContaining([observed.evidence_id]), + }); +}); diff --git a/src/domain/reconstructionCoverageEvidence.ts b/src/domain/reconstructionCoverageEvidence.ts new file mode 100644 index 0000000000..036fda121f --- /dev/null +++ b/src/domain/reconstructionCoverageEvidence.ts @@ -0,0 +1,64 @@ +import type { Evidence } from "./evidence.js"; +import type { + ReconstructionCoverageData, + ReconstructionVerifierContract, +} from "./reconstructionCoverage.js"; + +type Authority = ReconstructionVerifierContract["authority"]; +type PackageProof = ReconstructionCoverageData["package_proofs"][number]; +type VerifierResult = ReconstructionCoverageData["verifier_results"][number]; + +const evidenceAuthorities: Readonly< + Record +> = { + "shipped-artifact": ["shipped-artifact"], + "controlled-replay": ["controlled-replay"], + "live-observation": ["controlled-replay", "external-service"], + external: ["external-service"], +}; + +const packageAuthorities: Readonly< + Record +> = { + build: ["live-observation"], + "clean-install": ["live-observation"], + "package-contents": ["shipped-artifact", "live-observation"], + "runtime-routing": ["live-observation"], + "authority-independence": ["shipped-artifact", "live-observation"], +}; + +/** Qualify only the Evidence cited by this verifier observation. */ +export const verifierHasComparableEvidence = ( + result: VerifierResult, + contract: ReconstructionVerifierContract, + records: ReadonlyMap, +): boolean => + hasComparableEvidence(result.evidence_ids, [contract.authority], records); + +/** Runtime package claims need observations; static package audits can be derived. */ +export const packageProofHasComparableEvidence = ( + proof: PackageProof, + records: ReadonlyMap, +): boolean => + hasComparableEvidence( + proof.evidence_ids, + packageAuthorities[proof.kind], + records, + ); + +const hasComparableEvidence = ( + evidenceIds: readonly string[], + requirements: readonly Authority[], + records: ReadonlyMap, +): boolean => + evidenceIds.some((id) => { + const evidence = records.get(id); + if (evidence === undefined) return false; + return requirements.some( + (required) => + evidenceAuthorities[required].includes(evidence.authority) && + (evidence.confidence === "observed" || + (required === "shipped-artifact" && + evidence.confidence === "derived")), + ); + }); diff --git a/tests/boundary/mcp/reconstructionCoverageMcp.test.ts b/tests/boundary/mcp/reconstructionCoverageMcp.test.ts index 1e167b8b09..951ee00375 100644 --- a/tests/boundary/mcp/reconstructionCoverageMcp.test.ts +++ b/tests/boundary/mcp/reconstructionCoverageMcp.test.ts @@ -8,6 +8,8 @@ import { reconstructionCoverageEvidenceId, } from "../../../src/domain/reconstructionCoverage.fixture.js"; import { createReconstructionCoverageData } from "../../../src/domain/reconstructionCoverage.js"; +import { createEvidence } from "../../../src/domain/evidence.js"; +import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; @@ -43,6 +45,64 @@ describe("reconstruction coverage MCP", () => { const latest = coverage.verifier_results[0]; if (latest === undefined) throw new Error("Expected fixture verifier result"); + const inference = createEvidence( + undefined, + { id: "weak-proof", name: "Weak proof fixture", version: "1" }, + { + operation: "infer", + parameters: {}, + result: { status: "unknown", note: "No replay was performed" }, + authority: "analyst-inference", + confidence: "inferred", + }, + ); + const weakCoverage = createReconstructionCoverageData({ + ...coverage, + evidence_bundle: createEvidenceBundle([ + ...coverage.evidence_bundle.records, + inference, + ]), + verifier_results: [ + { ...latest, evidence_ids: [inference.evidence_id] }, + ], + }); + const weak = await client.callTool({ + name: "evaluate_reconstruction_coverage", + arguments: { coverage: weakCoverage, boundary_id: boundaryId }, + }); + expect(weak.isError).not.toBe(true); + expect(weak.structuredContent).toMatchObject({ + status: "unknown", + reasons: [ + expect.objectContaining({ code: "verifier-result-incompatible" }), + ], + evidence_ids: expect.arrayContaining([inference.evidence_id]), + }); + const tampered = await client.callTool({ + name: "evaluate_reconstruction_coverage", + arguments: { + boundary_id: boundaryId, + coverage: { + ...weakCoverage, + evidence_bundle: { + ...weakCoverage.evidence_bundle, + records: weakCoverage.evidence_bundle.records.map((record) => + record.evidence_id === inference.evidence_id + ? { + ...record, + authority: "controlled-replay", + confidence: "observed", + } + : record, + ), + }, + }, + }, + }); + expect(tampered.isError).toBe(true); + expect(parseMcpToolError(tampered)).toMatchObject({ + error: { code: "invalid_request" }, + }); const ambiguous = await client.callTool({ name: "evaluate_reconstruction_coverage", arguments: {