fix(reconstruction): resolve latest verifier observations without input-order bias

Group observations by exact normalized instant, retaining arbitrary fractional precision and offset equivalence. Coalesce equivalent assessments and all supporting Evidence; report conflicting latest assessments as unknown with verifier-result-ambiguous. Use the same instant comparison for freshness and future checks.

Replace first-input-wins coverage with permutation, provenance, offset, sub-millisecond ordering, and future-clock regressions, including the public MCP workflow. Validation: focused reconstruction/offset tests and MCP workflow passed; converted JSON Schema validation, typecheck/lint, and docs checks passed. Full suite and real-provider lanes were not run.
This commit is contained in:
morluto committed 2026-10-13 13:23:38 +08:00
1 parent 3c57e1a8b2
commit 1f7f808c4f
7 files changed
+402 -40

No files matched your search

+1 -1
View File
@@ -218,7 +218,7 @@ export const APPLICATION_TOOL_CONTRACTS = [
name: "evaluate_reconstruction_coverage",
...toolContractMetadata("evaluate_reconstruction_coverage"),
description:
"Evaluate inline evidence-backed reconstruction coverage against one named boundary. Missing ownership or inventory is partial; stale, weak, truncated, skipped, or unresolved proof is unknown; contradictions, failed proof, missing owners, and authority routing fail closed.",
"Evaluate inline evidence-backed reconstruction coverage against one named boundary. Missing ownership or inventory is partial; stale, weak, truncated, skipped, unresolved, or conflicting tied verifier proof is unknown; contradictions, failed proof, missing owners, and authority routing fail closed.",
kind: "application",
inputSchema: reconstructionCoverageEvaluationInputSchema,
outputSchema: reconstructionClosureResultSchema,
+83
View File
@@ -1,5 +1,11 @@
import { z } from "zod";
export interface OffsetDateTimeInstant {
readonly epochSecond: number;
/** Exact fractional second with insignificant trailing zeroes removed. */
readonly fraction: string;
}
/**
* ISO-8601 date-time that requires an explicit UTC offset.
*
@@ -19,3 +25,80 @@ export const offsetDateTimeSchema = z
/^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|(?:[+\x2d](?:[01]\d|2[0-3]):[0-5]\d)))$/u,
"Use an ISO-8601 date-time with an explicit UTC offset",
);
/** Parse an already validated offset timestamp without losing sub-millisecond precision. */
export const parseOffsetDateTimeInstant = (
value: string,
): OffsetDateTimeInstant => {
const match = /^(.*?)(?:\.(\d+))?(Z|[+\x2d]\d{2}:\d{2})$/u.exec(value);
if (match === null)
throw new TypeError("Expected a validated offset date-time.");
const [, dateTime, fractionalDigits, offset] = match;
if (dateTime === undefined || offset === undefined)
throw new TypeError("Expected a validated offset date-time.");
const wholeSecondMilliseconds = Date.parse(`${dateTime}${offset}`);
if (
!Number.isSafeInteger(wholeSecondMilliseconds) ||
wholeSecondMilliseconds % 1000 !== 0
)
throw new TypeError(
"Offset date-time is outside the supported time range.",
);
return {
epochSecond: wholeSecondMilliseconds / 1000,
fraction: fractionalDigits?.replace(/0+$/u, "") ?? "",
};
};
/** Convert a Date.now-style integer into the same exact comparison shape. */
export const offsetDateTimeInstantFromEpochMilliseconds = (
epochMilliseconds: number,
): OffsetDateTimeInstant => {
if (!Number.isFinite(epochMilliseconds))
throw new TypeError("Expected finite epoch milliseconds.");
if (Math.abs(epochMilliseconds) > Number.MAX_SAFE_INTEGER)
return { epochSecond: epochMilliseconds / 1000, fraction: "" };
const epochSecond = Math.floor(epochMilliseconds / 1000);
const millisecondsWithinSecond = epochMilliseconds - epochSecond * 1000;
return {
epochSecond,
fraction: fractionalSecondFromMilliseconds(millisecondsWithinSecond),
};
};
const fractionalSecondFromMilliseconds = (milliseconds: number): string => {
if (milliseconds === 0) return "";
const [coefficient = "", exponentText] = String(milliseconds)
.toLowerCase()
.split("e");
const exponent = exponentText === undefined ? 0 : Number(exponentText);
const point = coefficient.indexOf(".");
const decimalPosition =
(point < 0 ? coefficient.length : point) + exponent - 3;
const digits = coefficient.replace(".", "");
const fraction =
decimalPosition <= 0
? `0.${"0".repeat(-decimalPosition)}${digits}`
: decimalPosition < digits.length
? `${digits.slice(0, decimalPosition)}.${digits.slice(decimalPosition)}`
: `${digits}${"0".repeat(decimalPosition - digits.length)}`;
return (
fraction.includes(".") ? fraction.slice(fraction.indexOf(".") + 1) : ""
).replace(/0+$/u, "");
};
/** Compare instants exactly while treating absent fraction digits as zeroes. */
export const compareOffsetDateTimeInstants = (
left: OffsetDateTimeInstant,
right: OffsetDateTimeInstant,
): number => {
if (left.epochSecond !== right.epochSecond)
return left.epochSecond < right.epochSecond ? -1 : 1;
const length = Math.max(left.fraction.length, right.fraction.length);
for (let index = 0; index < length; index += 1) {
const leftDigit = left.fraction.charCodeAt(index) || 48;
const rightDigit = right.fraction.charCodeAt(index) || 48;
if (leftDigit !== rightDigit) return leftDigit < rightDigit ? -1 : 1;
}
return 0;
};
+167 -15
View File
@@ -286,14 +286,18 @@ describe("reconstruction coverage verifier selection", () => {
});
});
describe("reconstruction coverage verifier result ordering", () => {
it("preserves the first input result when verifier timestamps tie", () => {
describe("reconstruction coverage conflicting latest verifier observations", () => {
it("keeps conflicting tied assessments unknown regardless of input order", () => {
const workspace = completeReconstructionCoverageData();
const result = workspace.verifier_results[0];
const boundary = workspace.boundaries[0];
if (result === undefined || boundary === undefined)
throw new Error("Expected complete reconstruction coverage fixture");
const failed = { ...result, status: "fail" as const };
const failed = {
...result,
status: "fail" as const,
evidence_ids: [reconstructionCoverageEvidenceId("1")],
};
const firstPass = createReconstructionCoverageData({
...workspace,
verifier_results: [result, failed],
@@ -303,23 +307,109 @@ describe("reconstruction coverage verifier result ordering", () => {
verifier_results: [failed, result],
});
const firstPassResult = evaluateReconstructionClosure(
firstPass,
boundary.boundary_id,
RECONSTRUCTION_COVERAGE_NOW,
);
const firstFailResult = evaluateReconstructionClosure(
firstFail,
boundary.boundary_id,
RECONSTRUCTION_COVERAGE_NOW,
);
expect(firstPassResult).toEqual(firstFailResult);
expect(firstPassResult).toMatchObject({
status: "unknown",
reasons: [
expect.objectContaining({
code: "verifier-result-ambiguous",
subject_id: "claim.cli.help",
}),
],
});
expect(firstPassResult.evidence_ids).toEqual(
expect.arrayContaining([
reconstructionCoverageEvidenceId("3"),
reconstructionCoverageEvidenceId("1"),
]),
);
});
});
describe("reconstruction coverage equivalent latest verifier observations", () => {
it("coalesces equivalent equal-instant observations and retains all evidence", () => {
const workspace = completeReconstructionCoverageData();
const result = workspace.verifier_results[0];
const boundary = workspace.boundaries[0];
if (result === undefined || boundary === undefined)
throw new Error("Expected complete reconstruction coverage fixture");
const first = {
...result,
observed_at: "2026-07-16T11:00:00.123456789012345678900Z",
};
const equivalent = {
...first,
observed_at: "2026-07-16T06:00:00.1234567890123456789-05:00",
covered_claim_ids: [...result.covered_claim_ids].reverse(),
covered_dimensions: [...result.covered_dimensions].reverse(),
artifact_sha256s: [...result.artifact_sha256s].reverse(),
owner_sha256s: [...result.owner_sha256s].reverse(),
evidence_ids: [reconstructionCoverageEvidenceId("1")],
};
const forward = createReconstructionCoverageData({
...workspace,
verifier_results: [first, equivalent],
});
const reversed = createReconstructionCoverageData({
...workspace,
verifier_results: [equivalent, first],
});
const forwardResult = evaluateReconstructionClosure(
forward,
boundary.boundary_id,
RECONSTRUCTION_COVERAGE_NOW,
);
const reversedResult = evaluateReconstructionClosure(
reversed,
boundary.boundary_id,
RECONSTRUCTION_COVERAGE_NOW,
);
expect(forwardResult).toEqual(reversedResult);
expect(forwardResult.status).toBe("ready");
expect(forwardResult.evidence_ids).toEqual(
expect.arrayContaining([
reconstructionCoverageEvidenceId("3"),
reconstructionCoverageEvidenceId("1"),
]),
);
});
});
describe("reconstruction coverage verifier timestamp selection", () => {
it("lets a later unambiguous observation supersede an older tie", () => {
const workspace = completeReconstructionCoverageData();
const result = workspace.verifier_results[0];
const boundary = workspace.boundaries[0];
if (result === undefined || boundary === undefined)
throw new Error("Expected complete reconstruction coverage fixture");
const failed = { ...result, status: "fail" as const };
const latestPass = {
...result,
observed_at: "2026-07-16T11:30:00.000Z",
};
const changed = createReconstructionCoverageData({
...workspace,
verifier_results: [result, failed, latestPass],
});
expect(
evaluateReconstructionClosure(
firstPass,
boundary.boundary_id,
RECONSTRUCTION_COVERAGE_NOW,
).status,
).toBe("ready");
expect(
evaluateReconstructionClosure(
firstFail,
changed,
boundary.boundary_id,
RECONSTRUCTION_COVERAGE_NOW,
),
).toMatchObject({
status: "failed",
reasons: [expect.objectContaining({ code: "verifier-failed" })],
});
).toMatchObject({ status: "ready", reasons: [] });
});
it("orders offset-bearing verifier timestamps chronologically", () => {
@@ -352,6 +442,68 @@ describe("reconstruction coverage verifier result ordering", () => {
});
});
it("selects the later result when timestamps differ below millisecond precision", () => {
const workspace = completeReconstructionCoverageData();
const result = workspace.verifier_results[0];
if (result === undefined)
throw new Error("Expected fixture verifier result");
const changed = createReconstructionCoverageData({
...workspace,
verifier_results: [
{ ...result, observed_at: "2026-07-16T11:00:00.1231Z" },
{
...result,
observed_at: "2026-07-16T11:00:00.1239Z",
status: "fail",
},
],
});
expect(
evaluateReconstructionClosure(
changed,
"replacement.cli",
RECONSTRUCTION_COVERAGE_NOW,
),
).toMatchObject({
status: "failed",
reasons: [expect.objectContaining({ code: "verifier-failed" })],
});
});
});
describe("reconstruction coverage verifier future timestamps", () => {
it("treats an observation beyond millisecond-precision now as future", () => {
const workspace = completeReconstructionCoverageData();
const result = workspace.verifier_results[0];
if (result === undefined)
throw new Error("Expected fixture verifier result");
const changed = createReconstructionCoverageData({
...workspace,
verifier_results: [
{ ...result, observed_at: "2026-07-16T12:00:00.1239Z" },
],
});
expect(
evaluateReconstructionClosure(
changed,
"replacement.cli",
Date.parse("2026-07-16T12:00:00.123Z") + 0.5,
),
).toMatchObject({
status: "unknown",
reasons: [expect.objectContaining({ code: "verifier-result-stale" })],
});
});
it("rejects a non-finite evaluation clock", () => {
const workspace = completeReconstructionCoverageData();
expect(() =>
evaluateReconstructionClosure(workspace, "replacement.cli", Number.NaN),
).toThrow("Expected finite epoch milliseconds.");
});
it("rejects verifier observations from the future", () => {
const workspace = completeReconstructionCoverageData();
const result = workspace.verifier_results[0];
+8 -1
View File
@@ -19,7 +19,10 @@ import {
} from "./reconstructionCoverageEvaluation.js";
import { digestSchema } from "./../domain/digests.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
import { offsetDateTimeSchema } from "./offsetDateTime.js";
import {
offsetDateTimeInstantFromEpochMilliseconds,
offsetDateTimeSchema,
} from "./offsetDateTime.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const stableIdSchema = z
@@ -213,6 +216,7 @@ const closureReasonSchema = z.strictObject({
"verifier-result-missing",
"verifier-result-stale",
"verifier-result-incompatible",
"verifier-result-ambiguous",
"verifier-failed",
"verifier-unknown",
"active-unknown",
@@ -274,10 +278,12 @@ export const evaluateReconstructionClosure = (
);
if (boundary === undefined)
throw new TypeError(`Unknown reconstruction boundary: ${boundaryId}`);
const nowInstant = offsetDateTimeInstantFromEpochMilliseconds(nowEpochMs);
const context: ReconstructionEvaluationContext = {
coverage,
boundary,
nowEpochMs,
nowInstant,
reasons: [],
evidenceIds: new Set<string>(),
};
@@ -394,6 +400,7 @@ const closureStatus = (
[
"verifier-result-stale",
"verifier-result-incompatible",
"verifier-result-ambiguous",
"verifier-unknown",
"active-unknown",
"owner-path-unknown",
+46 -22
View File
@@ -3,6 +3,16 @@ import type {
ReconstructionCoverageData,
ReconstructionVerifierContract,
} from "./reconstructionCoverage.js";
import {
compareOffsetDateTimeInstants,
offsetDateTimeInstantFromEpochMilliseconds,
type OffsetDateTimeInstant,
} from "./offsetDateTime.js";
import {
latestVerifierObservationsByVerifier,
sameVerifierAssessment,
type VerifierObservation,
} from "./reconstructionCoverageVerifierResults.js";
export const recommendedReconstructionProbes = (
reasons: readonly ClosureReason[],
@@ -29,6 +39,7 @@ export interface ReconstructionEvaluationContext {
readonly coverage: ReconstructionCoverageData;
readonly boundary: Boundary;
readonly nowEpochMs: number;
readonly nowInstant: OffsetDateTimeInstant;
readonly reasons: ClosureReason[];
readonly evidenceIds: Set<string>;
}
@@ -184,7 +195,7 @@ interface VerifierIndexes {
>;
readonly latestResultsByVerifier: ReadonlyMap<
string,
ReconstructionCoverageData["verifier_results"][number]
readonly VerifierObservation[]
>;
readonly artifactDigests: ReadonlySet<string>;
readonly ownerDigests: ReadonlySet<string>;
@@ -213,20 +224,6 @@ const createVerifierIndexes = (
contractsByClaim.set(claimId, contracts);
}
const latestResultsByVerifier = new Map<
string,
ReconstructionCoverageData["verifier_results"][number]
>();
for (const result of coverage.verifier_results) {
const latest = latestResultsByVerifier.get(result.verifier_id);
// Stable descending sort used to choose the first input record on ties.
if (
latest === undefined ||
Date.parse(result.observed_at) > Date.parse(latest.observed_at)
)
latestResultsByVerifier.set(result.verifier_id, result);
}
const ownersBySurface = new Map<
string,
ReconstructionCoverageData["owners"][number][]
@@ -239,7 +236,9 @@ const createVerifierIndexes = (
return {
contractsByClaim,
latestResultsByVerifier,
latestResultsByVerifier: latestVerifierObservationsByVerifier(
coverage.verifier_results,
),
artifactDigests: new Set(
coverage.artifacts.map(({ artifact_sha256: value }) => value),
),
@@ -260,8 +259,10 @@ const evaluateVerifierResult = (
contract: ReconstructionVerifierContract,
indexes: VerifierIndexes,
): void => {
const result = indexes.latestResultsByVerifier.get(contract.verifier_id);
if (result === undefined) {
const latestResults = indexes.latestResultsByVerifier.get(
contract.verifier_id,
);
if (latestResults === undefined) {
addReason(
context,
"verifier-result-missing",
@@ -270,7 +271,26 @@ const evaluateVerifierResult = (
);
return;
}
addEvidence(context, result.evidence_ids);
addEvidence(
context,
latestResults.flatMap(({ result }) => result.evidence_ids),
);
const latest = latestResults[0];
if (latest === undefined) return;
const { result } = latest;
if (
!latestResults.every((candidate) =>
sameVerifierAssessment(result, candidate.result),
)
) {
addReason(
context,
"verifier-result-ambiguous",
claim.claim_id,
"Latest verifier observations have conflicting assessment content.",
);
return;
}
if (!verifierResultIsCompatible(claim, contract, result, indexes)) {
addReason(
context,
@@ -280,10 +300,14 @@ const evaluateVerifierResult = (
);
return;
}
const observedAt = Date.parse(result.observed_at);
const cutoffEpochMs = context.nowEpochMs - contract.max_age_ms;
if (
observedAt > context.nowEpochMs ||
context.nowEpochMs - observedAt > contract.max_age_ms
compareOffsetDateTimeInstants(latest.instant, context.nowInstant) > 0 ||
(cutoffEpochMs >= -Number.MAX_SAFE_INTEGER &&
compareOffsetDateTimeInstants(
latest.instant,
offsetDateTimeInstantFromEpochMilliseconds(cutoffEpochMs),
) < 0)
) {
addReason(
context,
@@ -0,0 +1,57 @@
import type { ReconstructionCoverageData } from "./reconstructionCoverage.js";
import { uniqueSorted } from "./canonicalOrdering.js";
import {
compareOffsetDateTimeInstants,
parseOffsetDateTimeInstant,
type OffsetDateTimeInstant,
} from "./offsetDateTime.js";
type VerifierResult = ReconstructionCoverageData["verifier_results"][number];
export interface VerifierObservation {
readonly result: VerifierResult;
readonly instant: OffsetDateTimeInstant;
}
/** Index each verifier's latest timestamp group without relying on array order. */
export const latestVerifierObservationsByVerifier = (
results: readonly VerifierResult[],
): ReadonlyMap<string, readonly VerifierObservation[]> => {
const latestByVerifier = new Map<string, VerifierObservation[]>();
for (const result of results) {
const observation = {
result,
instant: parseOffsetDateTimeInstant(result.observed_at),
};
const latest = latestByVerifier.get(result.verifier_id);
const latestInstant = latest?.[0]?.instant;
const comparison =
latestInstant === undefined
? 1
: compareOffsetDateTimeInstants(observation.instant, latestInstant);
if (latest === undefined || comparison > 0)
latestByVerifier.set(result.verifier_id, [observation]);
else if (comparison === 0) latest.push(observation);
}
return latestByVerifier;
};
/** Compare assessment content while excluding timestamp spelling and provenance. */
export const sameVerifierAssessment = (
left: VerifierResult,
right: VerifierResult,
): boolean =>
verifierAssessmentIdentity(left) === verifierAssessmentIdentity(right);
const verifierAssessmentIdentity = (result: VerifierResult): string =>
JSON.stringify({
verifier_id: result.verifier_id,
contract_sha256: result.contract_sha256,
status: result.status,
covered_claim_ids: uniqueSorted(result.covered_claim_ids),
covered_dimensions: uniqueSorted(result.covered_dimensions),
artifact_sha256s: uniqueSorted(result.artifact_sha256s),
owner_sha256s: uniqueSorted(result.owner_sha256s),
normalization_sha256: result.normalization_sha256,
repeats: result.repeats,
});
@@ -3,7 +3,10 @@ import { ok as resultOk } from "../../../src/domain/result.js";
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { describe, expect, it } from "vitest";
import { completeReconstructionCoverageData } from "../../../src/domain/reconstructionCoverage.fixture.js";
import {
completeReconstructionCoverageData,
reconstructionCoverageEvidenceId,
} from "../../../src/domain/reconstructionCoverage.fixture.js";
import { createReconstructionCoverageData } from "../../../src/domain/reconstructionCoverage.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { createServer } from "../../../src/server/createServer.js";
@@ -36,6 +39,42 @@ describe("reconstruction coverage MCP", () => {
boundary_id: boundaryId,
summary: { reasons: 0 },
});
const coverage = currentFixtureCoverage(boundaryId);
const latest = coverage.verifier_results[0];
if (latest === undefined)
throw new Error("Expected fixture verifier result");
const ambiguous = await client.callTool({
name: "evaluate_reconstruction_coverage",
arguments: {
coverage: createReconstructionCoverageData({
...coverage,
verifier_results: [
latest,
{
...latest,
status: "fail",
evidence_ids: [reconstructionCoverageEvidenceId("1")],
},
],
}),
boundary_id: boundaryId,
},
});
expect(ambiguous.isError).not.toBe(true);
expect(ambiguous.structuredContent).toMatchObject({
status: "unknown",
boundary_id: boundaryId,
reasons: [
expect.objectContaining({
code: "verifier-result-ambiguous",
subject_id: "claim.cli.help",
}),
],
evidence_ids: expect.arrayContaining([
reconstructionCoverageEvidenceId("3"),
reconstructionCoverageEvidenceId("1"),
]),
});
const unknown = await client.callTool({
name: "evaluate_reconstruction_coverage",
arguments: {