fix(reconstruction): verify visual screenshot comparisons (#12186)

* fix(reconstruction): admit visual screenshot comparisons

* fix(reconstruction): corroborate retained visual comparisons

* fix(reconstruction): bind visual claims to distinct captures

* fix(reconstruction): distinguish visual capture pairs

---------

Co-authored-by: Josh <joshuakrishna21@gmail.com>
Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-19 06:57:15 -05:00
1 parent e8d190e035
commit 94cbae6d0c
11 files changed
+811 -49

No files matched your search

+15
View File
@@ -83,3 +83,18 @@ Selection uses each unknown's latest revision, its comparison identity and
Evidence references. Explicit unknown dependencies remain relevant across
dimensions. Unknowns for another comparison and resolved heads do not become
blockers merely because their source Evidence overlaps.
## Visual reconstruction claims
A visual claim uses retained `compare_web_screenshots` Evidence and two
`capture_web_screenshot` records. Set `source_evidence.before_evidence_id` and
`source_evidence.after_evidence_id` on the claim to select each observation.
Both IDs are needed when the compared PNGs are byte-identical or a digest
matches multiple retained captures. Each selected record must match its side's
artifact digest. Separate pairs can reuse the same comparison Evidence, and
the verifier checks coverage for each pair. With unique digest matches, the
verifier can select the records automatically. It replays the PNG comparison at
the recorded threshold. Incomplete capture coverage, an unreadable image, a
dimension mismatch, or a comparison that
disagrees with the retained pixels leaves the claim `unknown`. Visual claims
are counted in `summary.visual`.
+1 -1
View File
@@ -224,7 +224,7 @@ export const SESSION_TOOL_CONTRACTS = [
session(
"verify_reconstruction",
evidenceEffects,
"Verify a finite typed behavioral and structural specification against a canonical Evidence bundle. Pass means every declared claim has complete comparable authority—not global source equivalence; changed claims fail and missing, limited, or unresolved evidence stays unknown.",
"Verify a finite typed behavioral, structural, and visual specification against a canonical Evidence bundle. Pass means every declared claim has complete comparable authority—not global source equivalence; changed claims fail and missing, limited, or unresolved evidence stays unknown.",
reconstructionVerificationInputSchema,
),
session(
+17 -7
View File
@@ -10,6 +10,7 @@ import {
readComparisonResult,
type ParsedComparisonEvidence,
} from "./comparisonEvidenceResult.js";
import type { ComparisonEvidenceOperation } from "./comparisonEvidenceIdentity.js";
import type { Evidence } from "./evidence.js";
import { evidenceInputSchema } from "./evidenceReference.js";
import { err, ok, type Result } from "./result.js";
@@ -28,6 +29,12 @@ import {
import { prefixedDigestSchema } from "./../domain/digests.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
type ChangedBehaviorComparisonOperation = Exclude<
ComparisonEvidenceOperation,
"compare_web_screenshots"
>;
type ChangedBehaviorComparison =
ParsedComparisonEvidence<ChangedBehaviorComparisonOperation>;
/** Input for aggregating existing comparison Evidence. */
export const changedBehaviorInputSchema = z.strictObject({
@@ -198,17 +205,20 @@ const parseComparisonEvidence = (
input: unknown,
fullInput: unknown,
index: number,
): ParsedComparisonEvidence => {
): ChangedBehaviorComparison => {
const parsedEvidence = readEvidence(input);
if (!parsedEvidence.ok) throw parsedEvidence.error;
const evidence = parsedEvidence.value;
if (!isComparisonEvidenceOperation(evidence.operation))
if (
!isComparisonEvidenceOperation(evidence.operation) ||
evidence.operation === "compare_web_screenshots"
)
throw callerInputError(
fullInput,
["comparisons", index],
"Changed-behavior analysis requires process, artifact, or function comparison Evidence",
);
const operation = evidence.operation;
const operation: ChangedBehaviorComparisonOperation = evidence.operation;
if (!hasComparisonEvidenceIdentity(evidence))
throw evidenceIntegrityError(
"Comparison Evidence operation, predicate, and provider disagree",
@@ -226,7 +236,7 @@ const parseComparisonEvidence = (
return result.value;
};
const findingsFor = (comparison: ParsedComparisonEvidence): Finding[] => {
const findingsFor = (comparison: ChangedBehaviorComparison): Finding[] => {
if (comparison.operation === "compare_process_captures")
return processFindings(comparison);
if (comparison.operation === "compare_functions")
@@ -236,7 +246,7 @@ const findingsFor = (comparison: ParsedComparisonEvidence): Finding[] => {
const processFindings = (
comparison: Extract<
ParsedComparisonEvidence,
ChangedBehaviorComparison,
{ readonly operation: "compare_process_captures" }
>,
): Finding[] => {
@@ -292,7 +302,7 @@ const unprovenEqualityReason = (
const functionFindings = (
comparison: Extract<
ParsedComparisonEvidence,
ChangedBehaviorComparison,
{ readonly operation: "compare_functions" }
>,
): Finding[] => {
@@ -333,7 +343,7 @@ const functionFindings = (
const artifactFindings = (
comparison: Extract<
ParsedComparisonEvidence,
ChangedBehaviorComparison,
{ readonly operation: "compare_artifacts" }
>,
): Finding[] => {
+19 -4
View File
@@ -38,6 +38,14 @@ const COMPARISON_EVIDENCE_IDENTITIES = {
predicate: "rea.process-comparison",
provider: PROCESS_PROVIDER,
},
compare_web_screenshots: {
predicate: "rea.web-screenshot-diff",
provider: {
id: "rea-cdp-browser",
name: "REA Chrome DevTools Protocol observation provider",
version: "2",
},
},
} as const;
export type ComparisonEvidenceOperation =
@@ -59,14 +67,21 @@ export const hasComparisonEvidenceIdentity = (evidence: Evidence): boolean => {
: evidence.provider.id === expected.provider.id &&
evidence.provider.name === expected.provider.name &&
evidence.provider.version === expected.provider.version;
const subjectMatches =
evidence.operation === "compare_web_screenshots"
? sameTargetIdentity(
evidence.subject.identity,
unknownTarget("not-observed"),
)
: sameTargetIdentity(
evidence.subject.identity,
unknownTarget("derived-from-linked-evidence"),
);
return (
evidence.predicate_type === expected.predicate &&
providerMatches &&
evidence.confidence === "derived" &&
evidence.authority === "analyst-inference" &&
sameTargetIdentity(
evidence.subject.identity,
unknownTarget("derived-from-linked-evidence"),
)
subjectMatches
);
};
+14
View File
@@ -14,6 +14,10 @@ import {
processCaptureComparisonSchema,
type ProcessCaptureComparison,
} from "./process/processComparison.js";
import {
webScreenshotDiffSchema,
type WebScreenshotDiff,
} from "./webScreenshot.js";
import { err, ok, type Result } from "./result.js";
import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js";
@@ -21,6 +25,7 @@ type ComparisonResults = {
readonly compare_artifacts: ArtifactComparisonResult;
readonly compare_functions: FunctionComparisonResult;
readonly compare_process_captures: ProcessCaptureComparison;
readonly compare_web_screenshots: WebScreenshotDiff;
};
/** One admitted comparison's envelope and its operation-specific parsed facts. */
@@ -101,6 +106,15 @@ export function readComparisonResult(
if (!links.ok) return links;
return ok({ operation, evidence, result: parsed.value });
}
if (operation === "compare_web_screenshots") {
const parsed = readEvidencePayload(
webScreenshotDiffSchema,
evidence.normalized_result,
{ within: ["normalized_result"] },
);
if (!parsed.ok) return parsed;
return ok({ operation, evidence, result: parsed.value });
}
const parsed = readEvidencePayload(
artifactComparisonResultSchema,
evidence.normalized_result,
+1
View File
@@ -361,6 +361,7 @@ const inflatePng = (compressed: Buffer, maxOutputLength: number): Buffer => {
"code" in cause &&
(cause.code === "Z_DATA_ERROR" ||
cause.code === "Z_BUF_ERROR" ||
cause.code === "Z_NEED_DICT" ||
cause.code === "ERR_BUFFER_TOO_LARGE")
)
throw new TypeError("Invalid PNG image data", { cause });
+11 -5
View File
@@ -141,8 +141,14 @@ const defaultProbe = (
operation: "analyze_function",
rationale: "Capture complete function dossiers under equal limits.",
}
: {
operation: "inspect_artifact",
rationale:
"Capture complete artifact inventories under equal limits.",
};
: kind === "visual"
? {
operation: "capture_web_screenshot",
rationale:
"Capture both visible viewports under the same browser scope.",
}
: {
operation: "inspect_artifact",
rationale:
"Capture complete artifact inventories under equal limits.",
};
@@ -547,6 +547,7 @@ describe("reconstruction verification integrity", () => {
unknown: 0,
behavioral: 0,
structural: 1,
visual: 0,
},
claims: { items: [claim] },
recommended_probes: [],
+247 -22
View File
@@ -1,4 +1,7 @@
import { isDeepStrictEqual } from "node:util";
import { evidenceFileTarget } from "./evidence.js";
import { z } from "zod";
import {
ArtifactInventoryCarrierError,
parseArtifactInventoryEvidence,
@@ -11,7 +14,11 @@ import { readComparisonResult } from "./comparisonEvidenceResult.js";
import type { Evidence } from "./evidence.js";
import { hasComparisonEvidenceIdentity } from "./comparisonEvidenceIdentity.js";
import { readEvidenceBundle } from "./evidenceBundle.js";
import { AnalysisInputError } from "./analysisErrorCore.js";
import {
AnalysisInputError,
AnalysisResourceConstraintError,
AnalysisUnsupportedTargetError,
} from "./analysisErrorCore.js";
import { EvidenceIntegrityError } from "./evidenceErrors.js";
import { analysisInputErrorFromIssues } from "./inputIssueProjection.js";
import { readEvidencePayload } from "./evidenceValidation.js";
@@ -29,7 +36,9 @@ import {
import type { ProcessObservationSource } from "./process/processObservation.js";
import { digestCanonicalValue } from "./canonicalDigest.js";
import {
countReconstructionClaimKinds,
reconstructionClaimResultSchema,
reconstructionClaimSelector,
reconstructionSpecificationSchema,
reconstructionVerificationResultSchema,
type ReconstructionClaim as Claim,
@@ -43,11 +52,14 @@ import {
reconstructionUnknownHeads,
} from "./reconstructionUnknowns.js";
import type { ResidualUnknown } from "./residualUnknown.js";
import { comparePngScreenshots } from "./pngVisualDiff.js";
import { webScreenshotSchema, type WebScreenshot } from "./webScreenshot.js";
const comparisonOperationsByClaimKind = {
behavioral: "compare_process_captures",
"structural-function": "compare_functions",
"structural-artifact": "compare_artifacts",
visual: "compare_web_screenshots",
} as const;
interface FunctionSourceFacts {
@@ -95,10 +107,38 @@ export const verifyReconstruction = (
);
const heads = reconstructionUnknownHeads(bundle.unknowns);
const artifactReplays = new Map<string, ArtifactReplay>();
const screenshots: ScreenshotAssessment = {
captures: specification.claims.some(({ kind }) => kind === "visual")
? indexScreenshotCaptures(records.values())
: new Map(),
replayBlockers: new Map(),
};
const visualSelectors = new Set<string>();
const results = specification.claims
.map((claim, index) =>
evaluateClaim(claim, index, records, heads, artifactReplays),
)
.map((claim, index) => {
const result = evaluateClaim(
claim,
index,
records,
heads,
artifactReplays,
screenshots,
);
if (claim.kind === "visual") {
// An inferred capture pair can resolve to the same selector as an explicit pair.
const selector = reconstructionClaimSelector(claim, [
result.left_evidence_ids[0]!,
result.right_evidence_ids[0]!,
]);
if (visualSelectors.has(selector))
throw claimComparisonInputError(
index,
"Claim comparison selectors must be unique",
);
visualSelectors.add(selector);
}
return result;
})
.sort((left, right) =>
compareUnicodeCodePoints(left.claim_id, right.claim_id),
);
@@ -124,8 +164,7 @@ export const verifyReconstruction = (
passed: count(results, "pass"),
failed,
unknown: unresolved,
behavioral: results.filter(({ kind }) => kind === "behavioral").length,
structural: results.filter(({ kind }) => kind !== "behavioral").length,
...countReconstructionClaimKinds(results),
},
claims: {
items: results,
@@ -145,17 +184,28 @@ const evaluateClaim = (
records: ReadonlyMap<string, Evidence>,
heads: ReadonlyMap<string, ResidualUnknown>,
artifactReplays: Map<string, ArtifactReplay>,
screenshots: ScreenshotAssessment,
): ClaimResult => {
const comparison = records.get(claim.comparison_evidence_id);
if (comparison === undefined)
throw missingSourceInputError(
throw claimComparisonInputError(
claimIndex,
`Missing comparison Evidence ${claim.comparison_evidence_id}`,
);
validateComparisonIdentity(claim, comparison, claimIndex);
const sides = comparisonSourceEvidenceSides(comparison);
const { sides, observation, visualBlockers } =
claim.kind === "visual"
? assessScreenshotComparison(claim, comparison, screenshots)
: {
sides: comparisonSourceEvidenceSides(comparison),
observation: comparisonObservation(claim, comparison),
visualBlockers: [],
};
const allSourceIds = uniqueSorted([...sides.left, ...sides.right]);
if (!sameSet(comparison.evidence_links, allSourceIds))
if (
claim.kind !== "visual" &&
!sameSet(comparison.evidence_links, allSourceIds)
)
throw new EvidenceIntegrityError(
"Comparison Evidence closure disagrees with its parameters",
{
@@ -163,17 +213,31 @@ const evaluateClaim = (
"Comparison Evidence closure disagrees with its parameters",
},
);
if (
claim.kind === "visual" &&
comparison.evidence_links.length > 0 &&
!sameSet(comparison.evidence_links, allSourceIds)
)
throw new EvidenceIntegrityError(
"Screenshot comparison Evidence links disagree with its artifact inputs",
{
userMessage:
"Screenshot comparison Evidence links disagree with its artifact inputs",
},
);
const sources = allSourceIds.map((id) => {
const source = records.get(id);
if (source === undefined)
throw missingSourceInputError(
throw claimComparisonInputError(
claimIndex,
`Missing source Evidence ${id}`,
);
return source;
});
const sourceFacts = readValidatedSourceFacts(claim, sources);
const observation = comparisonObservation(claim, comparison);
const sourceFacts =
claim.kind === "visual"
? { functions: [], processes: [] }
: readValidatedSourceFacts(claim, sources);
const observed = observation.status;
const callCoverage = sourceCallCoverage(
claim,
@@ -183,6 +247,7 @@ const evaluateClaim = (
const blockers = [
...authorityLimitations(claim, sources, sides),
...callCoverage.blockers,
...visualBlockers,
...sourceProcessCoverage(claim, sourceFacts.processes, observed),
...(claim.kind === "structural-artifact"
? assessArtifactSourceComparison(
@@ -224,6 +289,159 @@ const evaluateClaim = (
});
};
const screenshotComparisonParametersSchema = z.strictObject({
before_artifact_sha256: z.string().regex(/^[a-f0-9]{64}$/u),
after_artifact_sha256: z.string().regex(/^[a-f0-9]{64}$/u),
channel_threshold: z.int().min(0).max(255),
comparison_input_sha256: z
.string()
.regex(/^[a-f0-9]{64}$/u)
.optional(),
});
interface ScreenshotCapture {
readonly evidenceId: string;
readonly screenshot: WebScreenshot;
}
interface ScreenshotAssessment {
readonly captures: ReadonlyMap<string, readonly ScreenshotCapture[]>;
readonly replayBlockers: Map<string, string | null>;
}
const indexScreenshotCaptures = (records: Iterable<Evidence>) => {
const captures = new Map<string, ScreenshotCapture[]>();
for (const source of records) {
if (
source.operation !== "capture_web_screenshot" ||
source.predicate_type !== "rea.web-screenshot"
)
continue;
const parsed = readEvidencePayload(
webScreenshotSchema,
source.normalized_result,
{ within: ["normalized_result"] },
);
if (!parsed.ok) throw parsed.error;
const sha256 = parsed.value.artifact.sha256;
const capture = {
evidenceId: source.evidence_id,
screenshot: parsed.value,
};
const existing = captures.get(sha256);
if (existing === undefined) captures.set(sha256, [capture]);
else existing.push(capture);
}
return captures;
};
const assessScreenshotComparison = (
claim: Extract<Claim, { kind: "visual" }>,
comparison: Evidence,
{ captures, replayBlockers }: ScreenshotAssessment,
): {
sides: { left: string[]; right: string[] };
observation: ComparisonObservation;
visualBlockers: string[];
} => {
const parsed = readEvidencePayload(
screenshotComparisonParametersSchema,
comparison.parameters,
{ within: ["parameters"] },
);
if (!parsed.ok) throw parsed.error;
const sourceForArtifact = (
sha256: string,
side: "before" | "after",
evidenceId: string | undefined,
) => {
const matches = captures.get(sha256);
if (evidenceId !== undefined) {
const selected = matches?.find((item) => item.evidenceId === evidenceId);
if (selected !== undefined) return selected;
const message = `Screenshot comparison ${side} capture Evidence does not match its artifact digest or is missing from the bundle (${evidenceId}).`;
throw new EvidenceIntegrityError(message, { userMessage: message });
}
if (matches?.length !== 1) {
const message = `Screenshot comparison ${side} artifact does not identify exactly one captured Evidence record. Supply both capture IDs in source_evidence when the digest is ambiguous.`;
throw new EvidenceIntegrityError(message, { userMessage: message });
}
return matches[0]!;
};
const before = sourceForArtifact(
parsed.value.before_artifact_sha256,
"before",
claim.source_evidence?.before_evidence_id,
);
const after = sourceForArtifact(
parsed.value.after_artifact_sha256,
"after",
claim.source_evidence?.after_evidence_id,
);
const saved = readComparisonResult(comparison, "compare_web_screenshots");
if (!saved.ok) throw saved.error;
const { limitations, ...recorded } = saved.value.result;
const visualBlockers: string[] = [];
for (const { evidenceId, screenshot } of [before, after]) {
const { status, conditions } = screenshot.completeness;
if (
status !== "complete_within_window" ||
conditions.some((condition) => condition !== "complete_within_window")
)
visualBlockers.push(
`Source ${evidenceId}: Screenshot coverage is incomplete (${uniqueSorted([status, ...conditions]).join(", ")}).`,
);
}
// The comparison fixes both artifact digests and the threshold. Coverage
// stays specific to the selected captures, while the pixel replay is reusable.
let replayBlocker = replayBlockers.get(comparison.evidence_id);
if (replayBlocker === undefined) {
try {
const { limitations: _limitations, ...replayed } = comparePngScreenshots({
before: before.screenshot.artifact,
after: after.screenshot.artifact,
channel_threshold: parsed.value.channel_threshold,
});
replayBlocker = isDeepStrictEqual(recorded, replayed)
? null
: "Retained screenshot pixels do not corroborate the recorded comparison.";
} catch (cause) {
if (
!(cause instanceof TypeError) &&
!(cause instanceof AnalysisResourceConstraintError) &&
!(cause instanceof AnalysisUnsupportedTargetError)
)
throw cause;
replayBlocker = `Retained screenshot comparison could not be replayed: ${cause.message}`;
}
replayBlockers.set(comparison.evidence_id, replayBlocker);
}
if (replayBlocker !== null) visualBlockers.push(replayBlocker);
return {
sides: { left: [before.evidenceId], right: [after.evidenceId] },
observation: {
status:
recorded.status === "identical"
? "unchanged"
: recorded.status === "different"
? "changed"
: "unknown",
limitations: [
...limitations,
...[before, after].flatMap(({ evidenceId, screenshot }) =>
screenshot.limitations.map(
(detail) => `Source ${evidenceId}: ${detail}`,
),
),
...(recorded.status === "dimension_mismatch"
? ["Screenshot dimensions differ."]
: []),
],
},
visualBlockers,
};
};
const assessArtifactSourceComparison = (
sides: {
readonly left: readonly string[];
@@ -295,7 +513,7 @@ const assessArtifactSourceComparison = (
};
const readValidatedSourceFacts = (
claim: Claim,
claim: Exclude<Claim, { kind: "visual" }>,
sources: readonly Evidence[],
): ValidatedSourceFacts => {
const isExpectedType = (source: Evidence): boolean => {
@@ -381,7 +599,7 @@ const validateComparisonIdentity = (
): void => {
const expectedOperation = comparisonOperationsByClaimKind[claim.kind];
if (evidence.operation !== expectedOperation)
throw missingSourceInputError(
throw claimComparisonInputError(
claimIndex,
`Claim ${claim.claim_id} requires ${expectedOperation} Evidence; selected ${evidence.operation}`,
);
@@ -391,15 +609,17 @@ const validateComparisonIdentity = (
}
};
const comparisonObservation = (
claim: Claim,
evidence: Evidence,
): {
interface ComparisonObservation {
readonly status: ObservedStatus;
readonly limitations: readonly string[];
readonly hasIndependentFunctionChange?: boolean;
readonly selectedDimensions?: readonly string[];
} => {
}
const comparisonObservation = (
claim: Exclude<Claim, { kind: "visual" }>,
evidence: Evidence,
): ComparisonObservation => {
if (claim.kind === "behavioral") {
const parsed = readComparisonResult(evidence, "compare_process_captures");
if (!parsed.ok) throw parsed.error;
@@ -460,7 +680,7 @@ const comparisonObservation = (
return { status: result.status, limitations: [] };
};
const missingSourceInputError = (claimIndex: number, message: string) =>
const claimComparisonInputError = (claimIndex: number, message: string) =>
new AnalysisInputError("verify_reconstruction", { userMessage: message }, [
{
path: ["claims", claimIndex, "comparison_evidence_id"],
@@ -478,7 +698,11 @@ const authorityLimitations = (
},
): string[] => {
const requiredAuthority =
claim.kind === "behavioral" ? "controlled-replay" : "shipped-artifact";
claim.kind === "behavioral"
? "controlled-replay"
: claim.kind === "visual"
? "external-service"
: "shipped-artifact";
const limitations: string[] = [];
if (sides.left.some((id) => sides.right.includes(id)))
limitations.push(
@@ -497,7 +721,8 @@ const authorityLimitations = (
`Source Evidence lacks qualifying ${requiredAuthority} authority.`,
);
if (
claim.kind !== "behavioral" &&
(claim.kind === "structural-function" ||
claim.kind === "structural-artifact") &&
sources.some((source) => evidenceFileTarget(source) === undefined)
)
limitations.push("Structural source artifact identity is unavailable.");
@@ -0,0 +1,424 @@
import { describe, expect, it } from "vitest";
import { createEvidence, type Evidence } from "./evidence.js";
import { createEvidenceBundle } from "./evidenceBundle.js";
import {
createWebScreenshotArtifact,
webScreenshotSchema,
} from "./webScreenshot.js";
import { unknownTarget } from "./targetIdentity.js";
import { verifyReconstruction } from "./reconstructionVerification.js";
import { reconstructionVerificationResultSchema } from "./reconstructionVerificationSchemas.js";
const browserProvider = {
id: "rea-cdp-browser",
name: "REA Chrome DevTools Protocol observation provider",
version: "2",
} as const;
// The first two 1x1 images encode the same RGBA pixel with different compression.
const beforePng =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAEElEQVR4AQEFAPr/AAAAAP4BAwD/dUi/yAAAAABJRU5ErkJggg==";
const samePng =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNgYGD4BwABAwD/Xb6bfwAAAABJRU5ErkJggg==";
const differentPng =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNgZGT8DwABDQEDO4rAiwAAAABJRU5ErkJggg==";
const widerPng =
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR42mNgYGD4B8IABf0B/aLhVnkAAAAASUVORK5CYII=";
// This PNG's IDAT zlib stream requests a preset dictionary.
const dictionaryPng =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAEUlEQVR4uwwXAwhjYGBg+AcAAQMA/7BXXg0AAAAASUVORK5CYII=";
const unverifiedDocument = "The captured document could not be verified.";
const screenshot = (
bytes: string,
capturedAt: string,
{ limited = false, width = 1 } = {},
): Evidence =>
createEvidence(unknownTarget("not-observed"), browserProvider, {
predicateType: "rea.web-screenshot",
operation: "capture_web_screenshot",
parameters: {},
result: {
browser: {
product: "Chrome/130",
protocol_version: "1.3",
revision: "fixture",
user_agent: "fixture",
js_version: "fixture",
},
target: {
target_id: "fixture-page",
url: "https://example.test/",
origin: "https://example.test",
},
captured_at: capturedAt,
viewport: { width, height: 1 },
artifact: createWebScreenshotArtifact(Buffer.from(bytes, "base64")),
completeness: {
status: limited ? "attach_limited" : "complete_within_window",
conditions: [limited ? "attach_limited" : "complete_within_window"],
policy_filtered_sections: [],
attach_limited_sections: limited ? ["screenshot"] : [],
truncated_sections: [],
unavailable_sections: [],
excluded: [],
dropped_events: {
scripts: 0,
network_requests: 0,
console_events: 0,
websocket_connections: 0,
websocket_frames: 0,
webmcp_tools: 0,
timeline_events: 0,
total: 0,
},
},
limitations: limited ? [unverifiedDocument] : [],
},
confidence: "observed",
authority: "external-service",
});
const comparison = (
before: Evidence,
after: Evidence,
status: "identical" | "different" | "dimension_mismatch",
threshold = 0,
): Evidence =>
createEvidence(unknownTarget("not-observed"), browserProvider, {
predicateType: "rea.web-screenshot-diff",
operation: "compare_web_screenshots",
parameters: {
before_artifact_sha256: webScreenshotSchema.parse(
before.normalized_result,
).artifact.sha256,
after_artifact_sha256: webScreenshotSchema.parse(after.normalized_result)
.artifact.sha256,
channel_threshold: threshold,
},
result: {
before: { width: 1, height: 1 },
after: {
width: status === "dimension_mismatch" ? 2 : 1,
height: 1,
},
channel_threshold: threshold,
limitations: [],
status,
...(status === "identical"
? {
compared_pixels: 1,
changed_pixels: 0,
changed_ratio: 0,
maximum_channel_delta: threshold,
mean_absolute_channel_delta: threshold,
}
: status === "different"
? {
compared_pixels: 1,
changed_pixels: 1,
changed_ratio: 1,
maximum_channel_delta: 1,
mean_absolute_channel_delta: 1,
}
: {
compared_pixels: 0,
changed_pixels: null,
changed_ratio: null,
maximum_channel_delta: null,
mean_absolute_channel_delta: null,
}),
},
confidence: "derived",
authority: "analyst-inference",
});
const specification = (
evidence: Evidence,
sourceEvidence?: { before_evidence_id: string; after_evidence_id: string },
) => ({
name: "Visual reconstruction",
claims: [
{
kind: "visual" as const,
claim_id: "viewport",
title: "Viewport pixels remain equivalent",
comparison_evidence_id: evidence.evidence_id,
dimension: "overall" as const,
...(sourceEvidence === undefined
? {}
: { source_evidence: sourceEvidence }),
},
],
});
const verify = (comparisonEvidence: Evidence, sources: Evidence[]) =>
verifyReconstruction(
specification(comparisonEvidence),
createEvidenceBundle([...sources, comparisonEvidence]),
);
describe("visual reconstruction evidence", () => {
it("accepts a screenshot comparison and matches its source artifacts", () => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
const after = screenshot(samePng, "2026-01-01T00:00:01.000Z");
const result = verify(comparison(before, after, "identical"), [
before,
after,
]);
expect(result.claims.items).toMatchObject([
{ kind: "visual", status: "pass", observed_status: "unchanged" },
]);
expect(result.summary).toMatchObject({
total: 1,
behavioral: 0,
structural: 0,
visual: 1,
});
expect(
reconstructionVerificationResultSchema.safeParse({
...result,
summary: { ...result.summary, structural: 1, visual: 0 },
}).success,
).toBe(false);
});
it.each([
["different", "fail", "changed"],
["dimension_mismatch", "unknown", "unknown"],
] as const)(
"projects a %s screenshot diff into reconstruction status",
(diffStatus, status, observedStatus) => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
const mismatch = diffStatus === "dimension_mismatch";
const after = screenshot(
mismatch ? widerPng : differentPng,
"2026-01-01T00:00:01.000Z",
{ width: mismatch ? 2 : 1 },
);
const result = verify(comparison(before, after, diffStatus), [
before,
after,
]);
expect(result.claims.items).toMatchObject([
{ kind: "visual", status, observed_status: observedStatus },
]);
if (diffStatus === "dimension_mismatch")
expect(result.recommended_probes).toContainEqual(
expect.objectContaining({ operation: "capture_web_screenshot" }),
);
},
);
});
describe("visual reconstruction capture identity", () => {
it("rejects an ambiguous artifact-to-capture match", () => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
const duplicate = screenshot(beforePng, "2026-01-01T00:00:01.000Z");
const after = screenshot(samePng, "2026-01-01T00:00:02.000Z");
expect(() =>
verify(comparison(before, after, "identical"), [
before,
duplicate,
after,
]),
).toThrow(/does not identify exactly one captured Evidence record/u);
});
it.each([false, true])(
"checks capture pairs independently when they share one comparison (second pair limited: %s)",
(limited) => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
const after = screenshot(beforePng, "2026-01-01T00:00:01.000Z");
const repeatedBefore = screenshot(beforePng, "2026-01-01T00:00:02.000Z", {
limited,
});
const repeatedAfter = screenshot(beforePng, "2026-01-01T00:00:03.000Z");
const saved = comparison(before, after, "identical");
const spec = specification(saved, {
before_evidence_id: before.evidence_id,
after_evidence_id: after.evidence_id,
});
const second = specification(saved, {
before_evidence_id: repeatedBefore.evidence_id,
after_evidence_id: repeatedAfter.evidence_id,
}).claims[0]!;
const bundle = createEvidenceBundle([
repeatedAfter,
after,
saved,
before,
repeatedBefore,
]);
const result = verifyReconstruction(
{
...spec,
claims: [...spec.claims, { ...second, claim_id: "repeated" }],
},
bundle,
);
expect(result.claims.items).toMatchObject([
{
claim_id: "repeated",
status: limited ? "unknown" : "pass",
left_evidence_ids: [repeatedBefore.evidence_id],
right_evidence_ids: [repeatedAfter.evidence_id],
},
{
claim_id: "viewport",
status: "pass",
left_evidence_ids: [before.evidence_id],
right_evidence_ids: [after.evidence_id],
},
]);
},
);
it.each(["explicit", "inferred"] as const)(
"rejects the same capture pair when the second selector is %s",
(selection) => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
const after = screenshot(samePng, "2026-01-01T00:00:01.000Z");
const saved = comparison(before, after, "identical");
const sources = {
before_evidence_id: before.evidence_id,
after_evidence_id: after.evidence_id,
};
const spec = specification(saved, sources);
const second = specification(
saved,
selection === "explicit" ? sources : undefined,
).claims[0]!;
expect(() =>
verifyReconstruction(
{
...spec,
claims: [...spec.claims, { ...second, claim_id: "repeat" }],
},
createEvidenceBundle([before, after, saved]),
),
).toThrow(
expect.objectContaining({
issues: expect.arrayContaining([
expect.objectContaining({
message: "Claim comparison selectors must be unique",
}),
]),
}),
);
},
);
it("rejects an explicit screenshot source whose bytes disagree with that side", () => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
const after = screenshot(samePng, "2026-01-01T00:00:01.000Z");
const saved = comparison(before, after, "identical");
const spec = specification(saved, {
before_evidence_id: after.evidence_id,
after_evidence_id: before.evidence_id,
});
expect(() =>
verifyReconstruction(spec, createEvidenceBundle([before, after, saved])),
).toThrow(/before capture Evidence does not match its artifact digest/u);
});
});
describe("visual reconstruction coverage and corroboration", () => {
it("withholds a visual verdict when capture attribution is incomplete", () => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z", {
limited: true,
});
const after = screenshot(samePng, "2026-01-01T00:00:01.000Z");
const result = verify(comparison(before, after, "identical"), [
before,
after,
]);
expect(result.claims.items).toMatchObject([
{
status: "unknown",
observed_status: "unchanged",
limitations: expect.arrayContaining([
expect.stringMatching(
/Screenshot coverage is incomplete.*attach_limited/u,
),
expect.stringContaining(unverifiedDocument),
]),
},
]);
expect(result.recommended_probes).toContainEqual(
expect.objectContaining({ operation: "capture_web_screenshot" }),
);
});
it("requires the retained pixels to corroborate the reported comparison", () => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
const after = screenshot(differentPng, "2026-01-01T00:00:01.000Z");
const result = verify(comparison(before, after, "identical"), [
before,
after,
]);
expect(result.claims.items).toMatchObject([
{
status: "unknown",
observed_status: "unchanged",
limitations: [
"Retained screenshot pixels do not corroborate the recorded comparison.",
],
},
]);
expect(
verify(comparison(before, after, "identical", 1), [before, after]).status,
).toBe("pass");
});
it.each([
["invalid signature", "bm90LWEtcG5n", "Invalid PNG signature"],
["preset dictionary", dictionaryPng, "Invalid PNG image data"],
])(
"preserves other claims when retained PNG bytes cannot be decoded (%s)",
(_defect, bytes, reason) => {
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
const after = screenshot(bytes, "2026-01-01T00:00:01.000Z");
const independent = screenshot(samePng, "2026-01-01T00:00:02.000Z");
const unreadable = comparison(before, after, "identical");
const readable = comparison(before, independent, "identical");
const spec = specification(unreadable);
const result = verifyReconstruction(
{
...spec,
claims: [
...spec.claims,
{ ...specification(readable).claims[0], claim_id: "independent" },
],
},
createEvidenceBundle([
before,
after,
independent,
unreadable,
readable,
]),
);
expect(result.claims.items).toMatchObject([
{ claim_id: "independent", status: "pass" },
{
claim_id: "viewport",
status: "unknown",
limitations: [
`Retained screenshot comparison could not be replayed: ${reason}`,
],
},
]);
},
);
});
+61 -10
View File
@@ -1,5 +1,6 @@
import { z } from "zod";
import { compositeKey } from "./unicodeCodePointOrder.js";
import { IDENTIFIER_PATTERN } from "./stringPatterns.js";
import { digestSchema, prefixedDigestSchema } from "./../domain/digests.js";
import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js";
@@ -34,14 +35,47 @@ const artifactClaimSchema = z.object({
kind: z.literal("structural-artifact"),
dimension: z.literal("overall"),
});
const visualClaimSchema = z.object({
...commonClaim,
kind: z.literal("visual"),
dimension: z.literal("overall"),
source_evidence: z
.strictObject({
before_evidence_id: evidenceIdSchema,
after_evidence_id: evidenceIdSchema,
})
.optional()
.describe(
"Select the retained screenshot capture for each side. Supply both IDs when a digest matches multiple captures, including byte-identical before and after captures.",
),
});
const reconstructionClaimSchema = z.discriminatedUnion("kind", [
behavioralClaimSchema,
functionClaimSchema,
artifactClaimSchema,
visualClaimSchema,
]);
/** Finite, typed behavioral and structural specification. */
export const reconstructionClaimSelector = (
claim: ReconstructionClaim,
resolvedPair?: readonly [string, string],
): string => {
const selected = claim.kind === "visual" ? claim.source_evidence : undefined;
const pair =
resolvedPair ??
(selected === undefined
? []
: [selected.before_evidence_id, selected.after_evidence_id]);
return compositeKey([
claim.kind,
claim.comparison_evidence_id,
claim.dimension,
...pair,
]);
};
/** Finite, typed behavioral, structural, and visual specification. */
export const reconstructionSpecificationSchema = z
.object({
name: z.string().trim().min(1),
@@ -51,9 +85,8 @@ export const reconstructionSpecificationSchema = z
const ids = value.claims.map(({ claim_id: id }) => id);
if (new Set(ids).size !== ids.length)
context.addIssue({ code: "custom", message: "Claim IDs must be unique" });
const selectors = value.claims.map(
({ kind, comparison_evidence_id: id, dimension }) =>
`${kind}:${id}:${dimension}`,
const selectors = value.claims.map((claim) =>
reconstructionClaimSelector(claim),
);
if (new Set(selectors).size !== selectors.length)
context.addIssue({
@@ -69,7 +102,12 @@ export const reconstructionVerificationInputSchema = z.strictObject({
export const reconstructionClaimResultSchema = z.object({
claim_id: claimIdSchema,
kind: z.enum(["behavioral", "structural-function", "structural-artifact"]),
kind: z.enum([
"behavioral",
"structural-function",
"structural-artifact",
"visual",
]),
dimension: z.string().min(1),
status: verificationStatusSchema,
observed_status: z.enum([
@@ -89,6 +127,17 @@ export const reconstructionClaimResultSchema = z.object({
limitations: z.array(z.string()),
});
export const countReconstructionClaimKinds = (
claims: readonly Pick<ReconstructionClaimResult, "kind">[],
) => ({
behavioral: claims.filter(({ kind }) => kind === "behavioral").length,
structural: claims.filter(
({ kind }) =>
kind === "structural-function" || kind === "structural-artifact",
).length,
visual: claims.filter(({ kind }) => kind === "visual").length,
});
/** Evidence-cited result over every declared claim. */
export const reconstructionVerificationResultSchema = z
.object({
@@ -101,6 +150,7 @@ export const reconstructionVerificationResultSchema = z
unknown: z.int().min(0),
behavioral: z.int().min(0),
structural: z.int().min(0),
visual: z.int().min(0).default(0),
}),
claims: z.object({
items: z.array(reconstructionClaimResultSchema),
@@ -119,6 +169,7 @@ export const reconstructionVerificationResultSchema = z
.superRefine((result, context) => {
const { summary } = result;
const claims = result.claims.items;
const kinds = countReconstructionClaimKinds(claims);
const expectedStatus =
summary.failed > 0 ? "fail" : summary.unknown > 0 ? "unknown" : "pass";
const issues = [
@@ -133,10 +184,9 @@ export const reconstructionVerificationResultSchema = z
["summary"],
],
[
summary.behavioral !==
claims.filter(({ kind }) => kind === "behavioral").length ||
summary.structural !==
claims.filter(({ kind }) => kind !== "behavioral").length,
summary.behavioral !== kinds.behavioral ||
summary.structural !== kinds.structural ||
summary.visual !== kinds.visual,
"Claim kind counts must summarize retained claims",
["summary"],
],
@@ -146,7 +196,8 @@ export const reconstructionVerificationResultSchema = z
["summary", "total"],
],
[
summary.total !== summary.behavioral + summary.structural,
summary.total !==
summary.behavioral + summary.structural + summary.visual,
"Claim kind counts must equal the summary total",
["summary", "total"],
],