mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
fix(reconstruction): require comparable cited Evidence for coverage
Fixes #3357 Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
This commit is contained in:
9 files changed
+569
-81
No files matched your search
@@ -0,0 +1,43 @@
|
||||
# Reconstruction coverage
|
||||
|
||||
`rea evaluate-reconstruction-coverage coverage.json --json` evaluates an inline
|
||||
coverage workspace and its selected `boundary_id`. MCP exposes the same input
|
||||
through `evaluate_reconstruction_coverage`. Results contain a status, reasons,
|
||||
recommended probes and Evidence references; the supplied bundle retains the
|
||||
complete records.
|
||||
|
||||
Evidence IDs and bundle manifests must authenticate before evaluation. A valid
|
||||
record can still be insufficient proof: a passing verifier summary cannot
|
||||
promote inferred Evidence into a replay observation. That case returns
|
||||
`unknown`, while a missing or tampered record remains an input error.
|
||||
|
||||
## Comparable verifier Evidence
|
||||
|
||||
At least one record explicitly cited by the latest verifier observation must
|
||||
support the declared contract authority:
|
||||
|
||||
| Contract authority | Evidence source | Confidence |
|
||||
| ------------------- | ----------------------------------------- | ----------------------- |
|
||||
| `shipped-artifact` | `shipped-artifact` | `observed` or `derived` |
|
||||
| `controlled-replay` | `controlled-replay` | `observed` |
|
||||
| `live-observation` | `controlled-replay` or `external-service` | `observed` |
|
||||
| `external` | `external-service` | `observed` |
|
||||
|
||||
Contextual references can accompany qualifying proof. An unrelated observed
|
||||
record elsewhere in the bundle cannot repair a result that cites only
|
||||
inference. Older observed results cannot replace the newest weak result.
|
||||
Equivalent observations at the same instant can share qualifying proof from
|
||||
one of those observations; conflicting tied assessments remain unknown.
|
||||
|
||||
## Package and integration proof
|
||||
|
||||
Build, clean-install and runtime-routing claims require observed runtime
|
||||
Evidence (`controlled-replay` or `external-service`). Package-content and
|
||||
authority-independence audits can also use observed or derived
|
||||
`shipped-artifact` Evidence. Each proof must cite its qualifying records and
|
||||
cover the selected boundary's artifact digests. An inference alone establishes
|
||||
neither a passing nor a failed package workflow.
|
||||
|
||||
The evaluator preserves Evidence provenance and returns an actionable unknown
|
||||
when qualification is unavailable. It does not change the cited records to
|
||||
make their authority match a caller's declaration.
|
||||
@@ -323,3 +323,6 @@ disposition but does not silently count it as verified.
|
||||
The tool returns every obligation inline. Its `ledger_id` and `closure_digest`
|
||||
identify the complete result, including summaries, reports, ownership and
|
||||
dependency graphs, Evidence links, and limitations.
|
||||
|
||||
For named boundaries with explicit verifier contracts and package-proof kinds,
|
||||
see [reconstruction coverage](reconstruction-coverage.md).
|
||||
@@ -16,6 +16,11 @@ const EVIDENCE_RECORDS = ["1", "2", "3", "4", "5"].map((character) =>
|
||||
operation: `fixture-${character}`,
|
||||
parameters: {},
|
||||
result: { character },
|
||||
confidence: "observed",
|
||||
authority:
|
||||
character === "3" || character === "4"
|
||||
? "controlled-replay"
|
||||
: "shipped-artifact",
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
@@ -14,6 +14,29 @@ import {
|
||||
|
||||
const digest = (character: string): string => character.repeat(64);
|
||||
|
||||
it("preserves duplicate coverage owners instead of overwriting them in shared indexes", () => {
|
||||
const workspace = completeReconstructionCoverageData();
|
||||
const owner = workspace.owners[0];
|
||||
if (owner === undefined) throw new Error("Expected fixture owner");
|
||||
const result = evaluateReconstructionClosure(
|
||||
{
|
||||
...workspace,
|
||||
owners: [owner, owner],
|
||||
},
|
||||
"replacement.cli",
|
||||
RECONSTRUCTION_COVERAGE_NOW,
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
status: "partial",
|
||||
reasons: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
code: "owner-duplicate",
|
||||
subject_id: "cli.help",
|
||||
}),
|
||||
]),
|
||||
});
|
||||
});
|
||||
|
||||
describe("reconstruction coverage closure", () => {
|
||||
it("returns ready only when ownership, verification, package, and authority closure pass", () => {
|
||||
const workspace = completeReconstructionCoverageData();
|
||||
|
||||
@@ -8,7 +8,11 @@ import {
|
||||
compositeKey,
|
||||
} from "./unicodeCodePointOrder.js";
|
||||
|
||||
import { evidenceBundleSchema, parseEvidenceBundle } from "./evidenceBundle.js";
|
||||
import {
|
||||
authenticateParsedEvidenceBundle,
|
||||
evidenceBundleSchema,
|
||||
} from "./evidenceBundle.js";
|
||||
import type { Evidence } from "./evidence.js";
|
||||
import {
|
||||
evaluateReconstructionClaims,
|
||||
evaluateReconstructionPackageProofs,
|
||||
@@ -188,13 +192,37 @@ export type ReconstructionVerifierContract = z.infer<
|
||||
typeof verifierContractSchema
|
||||
>;
|
||||
|
||||
/** Owned coverage snapshot and its authenticated, unique lookup facts. */
|
||||
export interface ParsedReconstructionCoverage {
|
||||
readonly coverage: ReconstructionCoverageData;
|
||||
readonly evidenceById: ReadonlyMap<string, Evidence>;
|
||||
readonly artifacts: ReadonlyMap<
|
||||
string,
|
||||
ReconstructionCoverageData["artifacts"][number]
|
||||
>;
|
||||
readonly surfaces: ReadonlyMap<
|
||||
string,
|
||||
ReconstructionCoverageData["surfaces"][number]
|
||||
>;
|
||||
readonly claims: ReadonlyMap<
|
||||
string,
|
||||
ReconstructionCoverageData["claims"][number]
|
||||
>;
|
||||
readonly ownersBySurface: ReadonlyMap<
|
||||
string,
|
||||
readonly ReconstructionCoverageData["owners"][number][]
|
||||
>;
|
||||
}
|
||||
|
||||
/** Parse inline coverage data and validate every evidence and authority link. */
|
||||
export const createReconstructionCoverageData = (
|
||||
input: unknown,
|
||||
): ReconstructionCoverageData => {
|
||||
): ReconstructionCoverageData => parseCoverageData(input).coverage;
|
||||
|
||||
/** Retain authenticated Evidence facts alongside the validated coverage snapshot. */
|
||||
const parseCoverageData = (input: unknown): ParsedReconstructionCoverage => {
|
||||
const coverage = reconstructionCoverageDataSchema.parse(input);
|
||||
validateCoverageReferences(coverage);
|
||||
return coverage;
|
||||
return { coverage, ...validateCoverageReferences(coverage) };
|
||||
};
|
||||
|
||||
const closureReasonSchema = z.strictObject({
|
||||
@@ -252,7 +280,6 @@ export type ReconstructionClosureResult = z.infer<
|
||||
typeof reconstructionClosureResultSchema
|
||||
>;
|
||||
|
||||
type CoverageData = z.infer<typeof reconstructionCoverageDataSchema>;
|
||||
type ClosureReason = z.infer<typeof closureReasonSchema>;
|
||||
|
||||
/** Build a verifier contract whose digest prevents retrospective coverage edits. */
|
||||
@@ -272,15 +299,15 @@ export const evaluateReconstructionClosure = (
|
||||
boundaryId: string,
|
||||
nowEpochMs: number,
|
||||
): ReconstructionClosureResult => {
|
||||
const coverage = createReconstructionCoverageData(coverageInput);
|
||||
const boundary = coverage.boundaries.find(
|
||||
const parsed = parseCoverageData(coverageInput);
|
||||
const boundary = parsed.coverage.boundaries.find(
|
||||
({ boundary_id: id }) => id === boundaryId,
|
||||
);
|
||||
if (boundary === undefined)
|
||||
throw new TypeError(`Unknown reconstruction boundary: ${boundaryId}`);
|
||||
const nowInstant = offsetDateTimeInstantFromEpochMilliseconds(nowEpochMs);
|
||||
const context: ReconstructionEvaluationContext = {
|
||||
coverage,
|
||||
...parsed,
|
||||
boundary,
|
||||
nowEpochMs,
|
||||
nowInstant,
|
||||
@@ -306,21 +333,25 @@ export const evaluateReconstructionClosure = (
|
||||
});
|
||||
};
|
||||
|
||||
const validateCoverageReferences = (workspace: CoverageData): void => {
|
||||
const evidenceBundle = parseEvidenceBundle(workspace.evidence_bundle);
|
||||
const evidenceIds = new Set(
|
||||
evidenceBundle.records.map(({ evidence_id: id }) => id),
|
||||
const validateCoverageReferences = (
|
||||
workspace: ReconstructionCoverageData,
|
||||
): Omit<ParsedReconstructionCoverage, "coverage"> => {
|
||||
const evidenceBundle = authenticateParsedEvidenceBundle(
|
||||
workspace.evidence_bundle,
|
||||
);
|
||||
const evidenceById = new Map(
|
||||
evidenceBundle.records.map((record) => [record.evidence_id, record]),
|
||||
);
|
||||
const unknownIds = new Set(
|
||||
evidenceBundle.unknowns.map(({ unknown_id: id }) => id),
|
||||
);
|
||||
const artifacts = assertUnique(workspace.artifacts, "artifact_id");
|
||||
const surfaces = assertUnique(workspace.surfaces, "surface_id");
|
||||
const claims = assertUnique(workspace.claims, "claim_id");
|
||||
const contracts = assertUnique(workspace.verifier_contracts, "verifier_id");
|
||||
assertUnique(workspace.contradictions, "contradiction_id");
|
||||
assertUnique(workspace.package_proofs, "proof_id");
|
||||
assertUnique(workspace.boundaries, "boundary_id");
|
||||
const artifacts = indexUnique(workspace.artifacts, "artifact_id");
|
||||
const surfaces = indexUnique(workspace.surfaces, "surface_id");
|
||||
const claims = indexUnique(workspace.claims, "claim_id");
|
||||
const contracts = indexUnique(workspace.verifier_contracts, "verifier_id");
|
||||
indexUnique(workspace.contradictions, "contradiction_id");
|
||||
indexUnique(workspace.package_proofs, "proof_id");
|
||||
indexUnique(workspace.boundaries, "boundary_id");
|
||||
for (const contract of workspace.verifier_contracts) {
|
||||
const { contract_sha256: contractSha256, ...base } = contract;
|
||||
if (contractSha256 !== digest(verifierContractBaseSchema.parse(base)))
|
||||
@@ -331,9 +362,17 @@ const validateCoverageReferences = (workspace: CoverageData): void => {
|
||||
for (const surface of workspace.surfaces)
|
||||
if (!artifacts.has(surface.artifact_id))
|
||||
throw new TypeError(`Surface artifact is missing: ${surface.surface_id}`);
|
||||
for (const owner of workspace.owners)
|
||||
const ownersBySurface = new Map<
|
||||
string,
|
||||
ReconstructionCoverageData["owners"][number][]
|
||||
>();
|
||||
for (const owner of workspace.owners) {
|
||||
if (!surfaces.has(owner.surface_id))
|
||||
throw new TypeError(`Owner surface is missing: ${owner.surface_id}`);
|
||||
const owners = ownersBySurface.get(owner.surface_id);
|
||||
if (owners === undefined) ownersBySurface.set(owner.surface_id, [owner]);
|
||||
else owners.push(owner);
|
||||
}
|
||||
for (const claim of workspace.claims)
|
||||
if (claim.surface_ids.some((id) => !surfaces.has(id)))
|
||||
throw new TypeError(`Claim surface is missing: ${claim.claim_id}`);
|
||||
@@ -353,7 +392,7 @@ const validateCoverageReferences = (workspace: CoverageData): void => {
|
||||
...workspace.package_proofs.flatMap(({ evidence_ids: ids }) => ids),
|
||||
];
|
||||
const danglingEvidence = referencedEvidenceIds.find(
|
||||
(id) => !evidenceIds.has(id),
|
||||
(id) => !evidenceById.has(id),
|
||||
);
|
||||
if (danglingEvidence !== undefined)
|
||||
throw new TypeError(
|
||||
@@ -366,16 +405,20 @@ const validateCoverageReferences = (workspace: CoverageData): void => {
|
||||
throw new TypeError(
|
||||
`Coverage workspace residual unknown is missing: ${danglingUnknown}`,
|
||||
);
|
||||
return { evidenceById, artifacts, surfaces, claims, ownersBySurface };
|
||||
};
|
||||
|
||||
const assertUnique = <Item extends Record<Key, string>, Key extends keyof Item>(
|
||||
const indexUnique = <Item extends Record<Key, string>, Key extends keyof Item>(
|
||||
items: readonly Item[],
|
||||
key: Key,
|
||||
): Set<string> => {
|
||||
const values = items.map((item) => item[key]);
|
||||
if (new Set(values).size !== values.length)
|
||||
throw new TypeError(`Duplicate reconstruction coverage ${String(key)}`);
|
||||
return new Set(values);
|
||||
): ReadonlyMap<string, Item> => {
|
||||
const values = new Map<string, Item>();
|
||||
for (const item of items) {
|
||||
if (values.has(item[key]))
|
||||
throw new TypeError(`Duplicate reconstruction coverage ${String(key)}`);
|
||||
values.set(item[key], item);
|
||||
}
|
||||
return values;
|
||||
};
|
||||
|
||||
const closureStatus = (
|
||||
|
||||
@@ -2,7 +2,12 @@ import type {
|
||||
ReconstructionClosureResult,
|
||||
ReconstructionCoverageData,
|
||||
ReconstructionVerifierContract,
|
||||
ParsedReconstructionCoverage,
|
||||
} from "./reconstructionCoverage.js";
|
||||
import {
|
||||
packageProofHasComparableEvidence,
|
||||
verifierHasComparableEvidence,
|
||||
} from "./reconstructionCoverageEvidence.js";
|
||||
import {
|
||||
compareOffsetDateTimeInstants,
|
||||
offsetDateTimeInstantFromEpochMilliseconds,
|
||||
@@ -35,8 +40,7 @@ const probeOperation = (code: ClosureReason["code"]): string => {
|
||||
|
||||
type Boundary = ReconstructionCoverageData["boundaries"][number];
|
||||
type ClosureReason = ReconstructionClosureResult["reasons"][number];
|
||||
export interface ReconstructionEvaluationContext {
|
||||
readonly coverage: ReconstructionCoverageData;
|
||||
export interface ReconstructionEvaluationContext extends ParsedReconstructionCoverage {
|
||||
readonly boundary: Boundary;
|
||||
readonly nowEpochMs: number;
|
||||
readonly nowInstant: OffsetDateTimeInstant;
|
||||
@@ -47,9 +51,8 @@ export interface ReconstructionEvaluationContext {
|
||||
export const evaluateReconstructionSurfaces = (
|
||||
context: ReconstructionEvaluationContext,
|
||||
): void => {
|
||||
const surfaces = indexUnique(context.coverage.surfaces, "surface_id");
|
||||
for (const surfaceId of context.boundary.required_surface_ids) {
|
||||
const surface = surfaces.get(surfaceId);
|
||||
const surface = context.surfaces.get(surfaceId);
|
||||
if (surface === undefined) {
|
||||
addReason(
|
||||
context,
|
||||
@@ -61,7 +64,7 @@ export const evaluateReconstructionSurfaces = (
|
||||
}
|
||||
addEvidence(context, surface.evidence_ids);
|
||||
for (const dependencyId of surface.dependency_surface_ids)
|
||||
if (!surfaces.has(dependencyId))
|
||||
if (!context.surfaces.has(dependencyId))
|
||||
addReason(
|
||||
context,
|
||||
"dependency-surface-missing",
|
||||
@@ -76,9 +79,7 @@ const evaluateOwner = (
|
||||
context: ReconstructionEvaluationContext,
|
||||
surfaceId: string,
|
||||
): void => {
|
||||
const owners = context.coverage.owners.filter(
|
||||
({ surface_id: id }) => id === surfaceId,
|
||||
);
|
||||
const owners = context.ownersBySurface.get(surfaceId) ?? [];
|
||||
if (owners.length === 0) {
|
||||
addReason(
|
||||
context,
|
||||
@@ -150,10 +151,9 @@ const evaluateOwner = (
|
||||
export const evaluateReconstructionClaims = (
|
||||
context: ReconstructionEvaluationContext,
|
||||
): void => {
|
||||
const claims = indexUnique(context.coverage.claims, "claim_id");
|
||||
const indexes = createVerifierIndexes(context.coverage);
|
||||
const indexes = createVerifierIndexes(context);
|
||||
for (const claimId of context.boundary.required_claim_ids) {
|
||||
const claim = claims.get(claimId);
|
||||
const claim = context.claims.get(claimId);
|
||||
if (claim === undefined) {
|
||||
addReason(
|
||||
context,
|
||||
@@ -214,8 +214,9 @@ interface VerifierIndexes {
|
||||
}
|
||||
|
||||
const createVerifierIndexes = (
|
||||
coverage: ReconstructionCoverageData,
|
||||
context: ReconstructionEvaluationContext,
|
||||
): VerifierIndexes => {
|
||||
const { coverage } = context;
|
||||
const contractsByClaim = new Map<string, ReconstructionVerifierContract[]>();
|
||||
for (const contract of coverage.verifier_contracts)
|
||||
for (const claimId of new Set(contract.claim_ids)) {
|
||||
@@ -224,16 +225,6 @@ const createVerifierIndexes = (
|
||||
contractsByClaim.set(claimId, contracts);
|
||||
}
|
||||
|
||||
const ownersBySurface = new Map<
|
||||
string,
|
||||
ReconstructionCoverageData["owners"][number][]
|
||||
>();
|
||||
for (const owner of coverage.owners) {
|
||||
const owners = ownersBySurface.get(owner.surface_id) ?? [];
|
||||
owners.push(owner);
|
||||
ownersBySurface.set(owner.surface_id, owners);
|
||||
}
|
||||
|
||||
return {
|
||||
contractsByClaim,
|
||||
latestResultsByVerifier: latestVerifierObservationsByVerifier(
|
||||
@@ -247,9 +238,9 @@ const createVerifierIndexes = (
|
||||
ownership.disposition === "implemented" ? [ownership.owner_sha256] : [],
|
||||
),
|
||||
),
|
||||
surfaces: indexUnique(coverage.surfaces, "surface_id"),
|
||||
artifacts: indexUnique(coverage.artifacts, "artifact_id"),
|
||||
ownersBySurface,
|
||||
surfaces: context.surfaces,
|
||||
artifacts: context.artifacts,
|
||||
ownersBySurface: context.ownersBySurface,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -277,10 +268,9 @@ const evaluateVerifierResult = (
|
||||
);
|
||||
const latest = latestResults[0];
|
||||
if (latest === undefined) return;
|
||||
const { result } = latest;
|
||||
if (
|
||||
!latestResults.every((candidate) =>
|
||||
sameVerifierAssessment(result, candidate.result),
|
||||
sameVerifierAssessment(latest.result, candidate.result),
|
||||
)
|
||||
) {
|
||||
addReason(
|
||||
@@ -291,7 +281,20 @@ const evaluateVerifierResult = (
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!verifierResultIsCompatible(claim, contract, result, indexes)) {
|
||||
const comparable = latestResults.find(({ result }) =>
|
||||
verifierHasComparableEvidence(result, contract, context.evidenceById),
|
||||
);
|
||||
if (comparable === undefined) {
|
||||
addReason(
|
||||
context,
|
||||
"verifier-result-incompatible",
|
||||
claim.claim_id,
|
||||
`Latest verifier observation has no cited Evidence with comparable ${contract.authority} authority and confidence.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const { result } = comparable;
|
||||
if (!verifierResultMatchesContract(claim, contract, result, indexes)) {
|
||||
addReason(
|
||||
context,
|
||||
"verifier-result-incompatible",
|
||||
@@ -302,10 +305,10 @@ const evaluateVerifierResult = (
|
||||
}
|
||||
const cutoffEpochMs = context.nowEpochMs - contract.max_age_ms;
|
||||
if (
|
||||
compareOffsetDateTimeInstants(latest.instant, context.nowInstant) > 0 ||
|
||||
compareOffsetDateTimeInstants(comparable.instant, context.nowInstant) > 0 ||
|
||||
(cutoffEpochMs >= -Number.MAX_SAFE_INTEGER &&
|
||||
compareOffsetDateTimeInstants(
|
||||
latest.instant,
|
||||
comparable.instant,
|
||||
offsetDateTimeInstantFromEpochMilliseconds(cutoffEpochMs),
|
||||
) < 0)
|
||||
) {
|
||||
@@ -333,7 +336,7 @@ const evaluateVerifierResult = (
|
||||
);
|
||||
};
|
||||
|
||||
const verifierResultIsCompatible = (
|
||||
const verifierResultMatchesContract = (
|
||||
claim: ReconstructionCoverageData["claims"][number],
|
||||
contract: ReconstructionVerifierContract,
|
||||
result: ReconstructionCoverageData["verifier_results"][number],
|
||||
@@ -415,20 +418,13 @@ export const evaluateReconstructionCoverageRisks = (
|
||||
export const evaluateReconstructionPackageProofs = (
|
||||
context: ReconstructionEvaluationContext,
|
||||
): void => {
|
||||
const surfaces = new Map(
|
||||
context.coverage.surfaces.map((surface) => [surface.surface_id, surface]),
|
||||
);
|
||||
const artifacts = new Map(
|
||||
context.coverage.artifacts.map((artifact) => [
|
||||
artifact.artifact_id,
|
||||
artifact,
|
||||
]),
|
||||
);
|
||||
const requiredArtifactDigests = context.boundary.required_surface_ids.flatMap(
|
||||
(surfaceId) => {
|
||||
const surface = surfaces.get(surfaceId);
|
||||
const surface = context.surfaces.get(surfaceId);
|
||||
const artifact =
|
||||
surface === undefined ? undefined : artifacts.get(surface.artifact_id);
|
||||
surface === undefined
|
||||
? undefined
|
||||
: context.artifacts.get(surface.artifact_id);
|
||||
return artifact === undefined ? [] : [artifact.artifact_sha256];
|
||||
},
|
||||
);
|
||||
@@ -436,30 +432,42 @@ export const evaluateReconstructionPackageProofs = (
|
||||
const proofs = context.coverage.package_proofs.filter(
|
||||
(proof) => proof.kind === kind,
|
||||
);
|
||||
const applicable = (
|
||||
proof: ReconstructionCoverageData["package_proofs"][number],
|
||||
): boolean =>
|
||||
requiredArtifactDigests.every((digest) =>
|
||||
proof.artifact_sha256s.includes(digest),
|
||||
) && packageProofHasComparableEvidence(proof, context.evidenceById);
|
||||
const passing = proofs.find(
|
||||
({ artifact_sha256s: digests, status }) =>
|
||||
status === "pass" &&
|
||||
requiredArtifactDigests.every((digest) => digests.includes(digest)),
|
||||
(proof) => proof.status === "pass" && applicable(proof),
|
||||
);
|
||||
if (passing !== undefined) {
|
||||
addEvidence(context, passing.evidence_ids);
|
||||
continue;
|
||||
}
|
||||
if (proofs.some(({ status }) => status === "fail"))
|
||||
const failing = proofs.find(
|
||||
(proof) => proof.status === "fail" && applicable(proof),
|
||||
);
|
||||
if (failing !== undefined) {
|
||||
addEvidence(context, failing.evidence_ids);
|
||||
addReason(
|
||||
context,
|
||||
"package-proof-failed",
|
||||
kind,
|
||||
"Required package or integration proof failed.",
|
||||
);
|
||||
else if (proofs.length > 0)
|
||||
} else if (proofs.length > 0) {
|
||||
addEvidence(
|
||||
context,
|
||||
proofs.flatMap(({ evidence_ids }) => evidence_ids),
|
||||
);
|
||||
addReason(
|
||||
context,
|
||||
"package-proof-unknown",
|
||||
kind,
|
||||
"Required package or integration proof is not passing.",
|
||||
"Required package or integration proof has no qualifying passing Evidence.",
|
||||
);
|
||||
else
|
||||
} else
|
||||
addReason(
|
||||
context,
|
||||
"package-proof-missing",
|
||||
@@ -469,11 +477,6 @@ export const evaluateReconstructionPackageProofs = (
|
||||
}
|
||||
};
|
||||
|
||||
const indexUnique = <Item extends Record<Key, string>, Key extends keyof Item>(
|
||||
items: readonly Item[],
|
||||
key: Key,
|
||||
): ReadonlyMap<string, Item> => new Map(items.map((item) => [item[key], item]));
|
||||
|
||||
const addReason = (
|
||||
context: ReconstructionEvaluationContext,
|
||||
code: ClosureReason["code"],
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createEvidence, type Evidence } from "./evidence.js";
|
||||
import { createEvidenceBundle } from "./evidenceBundle.js";
|
||||
import {
|
||||
completeReconstructionCoverageFixture,
|
||||
RECONSTRUCTION_COVERAGE_NOW,
|
||||
} from "./reconstructionCoverage.fixture.js";
|
||||
import {
|
||||
createReconstructionCoverageData,
|
||||
createReconstructionVerifierContract,
|
||||
evaluateReconstructionClosure,
|
||||
type ReconstructionCoverageData,
|
||||
} from "./reconstructionCoverage.js";
|
||||
|
||||
const proof = (
|
||||
authority: Evidence["authority"],
|
||||
confidence: Evidence["confidence"],
|
||||
) =>
|
||||
createEvidence(
|
||||
undefined,
|
||||
{ id: "proof-test", name: "Proof fixture", version: "1" },
|
||||
{
|
||||
operation: "verify",
|
||||
parameters: {},
|
||||
authority,
|
||||
confidence,
|
||||
result:
|
||||
confidence === "observed" ||
|
||||
(authority === "shipped-artifact" && confidence === "derived")
|
||||
? { status: "pass" }
|
||||
: { status: "unknown", note: "No replay was performed" },
|
||||
},
|
||||
);
|
||||
|
||||
const withProofs = (records: readonly Evidence[]) => {
|
||||
const { workspace, result } = completeReconstructionCoverageFixture();
|
||||
return createReconstructionCoverageData({
|
||||
...workspace,
|
||||
evidence_bundle: createEvidenceBundle([
|
||||
...workspace.evidence_bundle.records,
|
||||
...records,
|
||||
]),
|
||||
verifier_results: [
|
||||
{
|
||||
...result,
|
||||
evidence_ids: records.map(({ evidence_id }) => evidence_id),
|
||||
},
|
||||
],
|
||||
});
|
||||
};
|
||||
|
||||
const withPackageProof = (
|
||||
kind: ReconstructionCoverageData["package_proofs"][number]["kind"],
|
||||
record: Evidence,
|
||||
status: ReconstructionCoverageData["package_proofs"][number]["status"] = "pass",
|
||||
) => {
|
||||
const { workspace } = completeReconstructionCoverageFixture();
|
||||
return createReconstructionCoverageData({
|
||||
...workspace,
|
||||
evidence_bundle: createEvidenceBundle([
|
||||
...workspace.evidence_bundle.records,
|
||||
record,
|
||||
]),
|
||||
package_proofs: [
|
||||
{
|
||||
proof_id: "selected",
|
||||
kind,
|
||||
status,
|
||||
artifact_sha256s: ["1".repeat(64)],
|
||||
evidence_ids: [record.evidence_id],
|
||||
},
|
||||
],
|
||||
boundaries: workspace.boundaries.map((boundary) => ({
|
||||
...boundary,
|
||||
required_package_proof_kinds: [kind],
|
||||
})),
|
||||
});
|
||||
};
|
||||
|
||||
const evaluate = (coverage: ReconstructionCoverageData) =>
|
||||
evaluateReconstructionClosure(
|
||||
coverage,
|
||||
"replacement.cli",
|
||||
RECONSTRUCTION_COVERAGE_NOW,
|
||||
);
|
||||
|
||||
it.each([
|
||||
["analyst-inference", "inferred"],
|
||||
["analyst-inference", "derived"],
|
||||
["analyst-inference", "observed"],
|
||||
["controlled-replay", "inferred"],
|
||||
["controlled-replay", "derived"],
|
||||
["shipped-artifact", "observed"],
|
||||
["external-service", "observed"],
|
||||
["historical-reference", "observed"],
|
||||
] as const)(
|
||||
"does not promote %s/%s into observed controlled replay",
|
||||
(authority, confidence) => {
|
||||
const record = proof(authority, confidence);
|
||||
const coverage = withProofs([record]);
|
||||
const result = evaluate(coverage);
|
||||
expect(result).toMatchObject({
|
||||
status: "unknown",
|
||||
reasons: [
|
||||
expect.objectContaining({
|
||||
code: "verifier-result-incompatible",
|
||||
subject_id: "claim.cli.help",
|
||||
}),
|
||||
],
|
||||
evidence_ids: expect.arrayContaining([record.evidence_id]),
|
||||
recommended_probes: [
|
||||
expect.objectContaining({ operation: "run_reconstruction_verifier" }),
|
||||
],
|
||||
});
|
||||
expect(
|
||||
coverage.evidence_bundle.records.find(
|
||||
({ evidence_id }) => evidence_id === record.evidence_id,
|
||||
),
|
||||
).toEqual(record);
|
||||
expect(result.reasons[0]?.detail).toContain(
|
||||
"controlled-replay authority and confidence",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("accepts cited observed replay alongside contextual inference without borrowing uncited proof", () => {
|
||||
const weak = proof("analyst-inference", "inferred");
|
||||
const observed = proof("controlled-replay", "observed");
|
||||
expect(evaluate(withProofs([weak, observed]))).toMatchObject({
|
||||
status: "ready",
|
||||
evidence_ids: expect.arrayContaining([
|
||||
weak.evidence_id,
|
||||
observed.evidence_id,
|
||||
]),
|
||||
});
|
||||
expect(evaluate(withProofs([weak])).status).toBe("unknown");
|
||||
});
|
||||
|
||||
it("does not fall back to older observed proof when the latest result only cites inference", () => {
|
||||
const record = proof("analyst-inference", "inferred");
|
||||
const coverage = withProofs([record]);
|
||||
const { result: previous } = completeReconstructionCoverageFixture();
|
||||
const latest = coverage.verifier_results[0];
|
||||
if (latest === undefined) throw new Error("Expected verifier result");
|
||||
const result = evaluate(
|
||||
createReconstructionCoverageData({
|
||||
...coverage,
|
||||
verifier_results: [
|
||||
previous,
|
||||
{ ...latest, observed_at: "2026-07-16T11:30:00.000Z" },
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(result.status).toBe("unknown");
|
||||
expect(result.evidence_ids).not.toContain(previous.evidence_ids[0]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["shipped-artifact", "shipped-artifact", "observed", "ready"],
|
||||
["shipped-artifact", "shipped-artifact", "derived", "ready"],
|
||||
["shipped-artifact", "shipped-artifact", "inferred", "unknown"],
|
||||
["live-observation", "external-service", "observed", "ready"],
|
||||
["live-observation", "controlled-replay", "observed", "ready"],
|
||||
["external", "external-service", "observed", "ready"],
|
||||
["external", "controlled-replay", "observed", "unknown"],
|
||||
] as const)(
|
||||
"checks %s authority against actual %s/%s Evidence",
|
||||
(required, authority, confidence, status) => {
|
||||
const coverage = withProofs([proof(authority, confidence)]);
|
||||
const original = coverage.verifier_contracts[0];
|
||||
const result = coverage.verifier_results[0];
|
||||
if (original === undefined || result === undefined)
|
||||
throw new Error("Expected verifier contract");
|
||||
const { contract_sha256: _digest, ...input } = original;
|
||||
const contract = createReconstructionVerifierContract({
|
||||
...input,
|
||||
authority: required,
|
||||
});
|
||||
const changed = createReconstructionCoverageData({
|
||||
...coverage,
|
||||
claims: coverage.claims.map((claim) => ({
|
||||
...claim,
|
||||
required_authority: required,
|
||||
})),
|
||||
verifier_contracts: [contract],
|
||||
verifier_results: [
|
||||
{ ...result, contract_sha256: contract.contract_sha256 },
|
||||
],
|
||||
});
|
||||
expect(evaluate(changed).status).toBe(status);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
"build",
|
||||
"clean-install",
|
||||
"package-contents",
|
||||
"runtime-routing",
|
||||
"authority-independence",
|
||||
] as const)("keeps %s unknown when only inferred Evidence is cited", (kind) => {
|
||||
const record = proof("analyst-inference", "inferred");
|
||||
const coverage = withPackageProof(kind, record);
|
||||
expect(evaluate(coverage)).toMatchObject({
|
||||
status: "unknown",
|
||||
reasons: [
|
||||
expect.objectContaining({
|
||||
code: "package-proof-unknown",
|
||||
subject_id: kind,
|
||||
}),
|
||||
],
|
||||
evidence_ids: expect.arrayContaining([record.evidence_id]),
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["build", "shipped-artifact", "observed", "unknown"],
|
||||
["build", "external-service", "observed", "ready"],
|
||||
["runtime-routing", "controlled-replay", "observed", "ready"],
|
||||
["clean-install", "controlled-replay", "derived", "unknown"],
|
||||
["package-contents", "shipped-artifact", "derived", "ready"],
|
||||
["authority-independence", "shipped-artifact", "derived", "ready"],
|
||||
] as const)(
|
||||
"checks %s proof at the actual %s/%s boundary",
|
||||
(kind, authority, confidence, status) => {
|
||||
expect(
|
||||
evaluate(withPackageProof(kind, proof(authority, confidence))).status,
|
||||
).toBe(status);
|
||||
},
|
||||
);
|
||||
|
||||
it("distinguishes observed package failure from an inferred failure claim", () => {
|
||||
const inference = proof("analyst-inference", "inferred");
|
||||
expect(evaluate(withPackageProof("build", inference, "fail"))).toMatchObject({
|
||||
status: "unknown",
|
||||
evidence_ids: expect.arrayContaining([inference.evidence_id]),
|
||||
});
|
||||
const observed = proof("controlled-replay", "observed");
|
||||
expect(evaluate(withPackageProof("build", observed, "fail"))).toMatchObject({
|
||||
status: "failed",
|
||||
reasons: [expect.objectContaining({ code: "package-proof-failed" })],
|
||||
evidence_ids: expect.arrayContaining([observed.evidence_id]),
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
import type { Evidence } from "./evidence.js";
|
||||
import type {
|
||||
ReconstructionCoverageData,
|
||||
ReconstructionVerifierContract,
|
||||
} from "./reconstructionCoverage.js";
|
||||
|
||||
type Authority = ReconstructionVerifierContract["authority"];
|
||||
type PackageProof = ReconstructionCoverageData["package_proofs"][number];
|
||||
type VerifierResult = ReconstructionCoverageData["verifier_results"][number];
|
||||
|
||||
const evidenceAuthorities: Readonly<
|
||||
Record<Authority, readonly Evidence["authority"][]>
|
||||
> = {
|
||||
"shipped-artifact": ["shipped-artifact"],
|
||||
"controlled-replay": ["controlled-replay"],
|
||||
"live-observation": ["controlled-replay", "external-service"],
|
||||
external: ["external-service"],
|
||||
};
|
||||
|
||||
const packageAuthorities: Readonly<
|
||||
Record<PackageProof["kind"], readonly Authority[]>
|
||||
> = {
|
||||
build: ["live-observation"],
|
||||
"clean-install": ["live-observation"],
|
||||
"package-contents": ["shipped-artifact", "live-observation"],
|
||||
"runtime-routing": ["live-observation"],
|
||||
"authority-independence": ["shipped-artifact", "live-observation"],
|
||||
};
|
||||
|
||||
/** Qualify only the Evidence cited by this verifier observation. */
|
||||
export const verifierHasComparableEvidence = (
|
||||
result: VerifierResult,
|
||||
contract: ReconstructionVerifierContract,
|
||||
records: ReadonlyMap<string, Evidence>,
|
||||
): boolean =>
|
||||
hasComparableEvidence(result.evidence_ids, [contract.authority], records);
|
||||
|
||||
/** Runtime package claims need observations; static package audits can be derived. */
|
||||
export const packageProofHasComparableEvidence = (
|
||||
proof: PackageProof,
|
||||
records: ReadonlyMap<string, Evidence>,
|
||||
): boolean =>
|
||||
hasComparableEvidence(
|
||||
proof.evidence_ids,
|
||||
packageAuthorities[proof.kind],
|
||||
records,
|
||||
);
|
||||
|
||||
const hasComparableEvidence = (
|
||||
evidenceIds: readonly string[],
|
||||
requirements: readonly Authority[],
|
||||
records: ReadonlyMap<string, Evidence>,
|
||||
): boolean =>
|
||||
evidenceIds.some((id) => {
|
||||
const evidence = records.get(id);
|
||||
if (evidence === undefined) return false;
|
||||
return requirements.some(
|
||||
(required) =>
|
||||
evidenceAuthorities[required].includes(evidence.authority) &&
|
||||
(evidence.confidence === "observed" ||
|
||||
(required === "shipped-artifact" &&
|
||||
evidence.confidence === "derived")),
|
||||
);
|
||||
});
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
reconstructionCoverageEvidenceId,
|
||||
} from "../../../src/domain/reconstructionCoverage.fixture.js";
|
||||
import { createReconstructionCoverageData } from "../../../src/domain/reconstructionCoverage.js";
|
||||
import { createEvidence } from "../../../src/domain/evidence.js";
|
||||
import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js";
|
||||
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
||||
import { createServer } from "../../../src/server/createServer.js";
|
||||
import { observed } from "../../fixtures/analysisExecution.js";
|
||||
@@ -43,6 +45,64 @@ describe("reconstruction coverage MCP", () => {
|
||||
const latest = coverage.verifier_results[0];
|
||||
if (latest === undefined)
|
||||
throw new Error("Expected fixture verifier result");
|
||||
const inference = createEvidence(
|
||||
undefined,
|
||||
{ id: "weak-proof", name: "Weak proof fixture", version: "1" },
|
||||
{
|
||||
operation: "infer",
|
||||
parameters: {},
|
||||
result: { status: "unknown", note: "No replay was performed" },
|
||||
authority: "analyst-inference",
|
||||
confidence: "inferred",
|
||||
},
|
||||
);
|
||||
const weakCoverage = createReconstructionCoverageData({
|
||||
...coverage,
|
||||
evidence_bundle: createEvidenceBundle([
|
||||
...coverage.evidence_bundle.records,
|
||||
inference,
|
||||
]),
|
||||
verifier_results: [
|
||||
{ ...latest, evidence_ids: [inference.evidence_id] },
|
||||
],
|
||||
});
|
||||
const weak = await client.callTool({
|
||||
name: "evaluate_reconstruction_coverage",
|
||||
arguments: { coverage: weakCoverage, boundary_id: boundaryId },
|
||||
});
|
||||
expect(weak.isError).not.toBe(true);
|
||||
expect(weak.structuredContent).toMatchObject({
|
||||
status: "unknown",
|
||||
reasons: [
|
||||
expect.objectContaining({ code: "verifier-result-incompatible" }),
|
||||
],
|
||||
evidence_ids: expect.arrayContaining([inference.evidence_id]),
|
||||
});
|
||||
const tampered = await client.callTool({
|
||||
name: "evaluate_reconstruction_coverage",
|
||||
arguments: {
|
||||
boundary_id: boundaryId,
|
||||
coverage: {
|
||||
...weakCoverage,
|
||||
evidence_bundle: {
|
||||
...weakCoverage.evidence_bundle,
|
||||
records: weakCoverage.evidence_bundle.records.map((record) =>
|
||||
record.evidence_id === inference.evidence_id
|
||||
? {
|
||||
...record,
|
||||
authority: "controlled-replay",
|
||||
confidence: "observed",
|
||||
}
|
||||
: record,
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(tampered.isError).toBe(true);
|
||||
expect(parseMcpToolError(tampered)).toMatchObject({
|
||||
error: { code: "invalid_request" },
|
||||
});
|
||||
const ambiguous = await client.callTool({
|
||||
name: "evaluate_reconstruction_coverage",
|
||||
arguments: {
|
||||
|
||||
Reference in new issue
Block a user