mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 13:12:26 +00:00
fix(reconstruction): verify visual screenshot comparisons (#12186)
* fix(reconstruction): admit visual screenshot comparisons * fix(reconstruction): corroborate retained visual comparisons * fix(reconstruction): bind visual claims to distinct captures * fix(reconstruction): distinguish visual capture pairs --------- Co-authored-by: Josh <joshuakrishna21@gmail.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
This commit is contained in:
11 files changed
+811
-49
No files matched your search
@@ -83,3 +83,18 @@ Selection uses each unknown's latest revision, its comparison identity and
|
||||
Evidence references. Explicit unknown dependencies remain relevant across
|
||||
dimensions. Unknowns for another comparison and resolved heads do not become
|
||||
blockers merely because their source Evidence overlaps.
|
||||
|
||||
## Visual reconstruction claims
|
||||
|
||||
A visual claim uses retained `compare_web_screenshots` Evidence and two
|
||||
`capture_web_screenshot` records. Set `source_evidence.before_evidence_id` and
|
||||
`source_evidence.after_evidence_id` on the claim to select each observation.
|
||||
Both IDs are needed when the compared PNGs are byte-identical or a digest
|
||||
matches multiple retained captures. Each selected record must match its side's
|
||||
artifact digest. Separate pairs can reuse the same comparison Evidence, and
|
||||
the verifier checks coverage for each pair. With unique digest matches, the
|
||||
verifier can select the records automatically. It replays the PNG comparison at
|
||||
the recorded threshold. Incomplete capture coverage, an unreadable image, a
|
||||
dimension mismatch, or a comparison that
|
||||
disagrees with the retained pixels leaves the claim `unknown`. Visual claims
|
||||
are counted in `summary.visual`.
|
||||
@@ -224,7 +224,7 @@ export const SESSION_TOOL_CONTRACTS = [
|
||||
session(
|
||||
"verify_reconstruction",
|
||||
evidenceEffects,
|
||||
"Verify a finite typed behavioral and structural specification against a canonical Evidence bundle. Pass means every declared claim has complete comparable authority—not global source equivalence; changed claims fail and missing, limited, or unresolved evidence stays unknown.",
|
||||
"Verify a finite typed behavioral, structural, and visual specification against a canonical Evidence bundle. Pass means every declared claim has complete comparable authority—not global source equivalence; changed claims fail and missing, limited, or unresolved evidence stays unknown.",
|
||||
reconstructionVerificationInputSchema,
|
||||
),
|
||||
session(
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
readComparisonResult,
|
||||
type ParsedComparisonEvidence,
|
||||
} from "./comparisonEvidenceResult.js";
|
||||
import type { ComparisonEvidenceOperation } from "./comparisonEvidenceIdentity.js";
|
||||
import type { Evidence } from "./evidence.js";
|
||||
import { evidenceInputSchema } from "./evidenceReference.js";
|
||||
import { err, ok, type Result } from "./result.js";
|
||||
@@ -28,6 +29,12 @@ import {
|
||||
import { prefixedDigestSchema } from "./../domain/digests.js";
|
||||
|
||||
const evidenceIdSchema = prefixedDigestSchema("ev");
|
||||
type ChangedBehaviorComparisonOperation = Exclude<
|
||||
ComparisonEvidenceOperation,
|
||||
"compare_web_screenshots"
|
||||
>;
|
||||
type ChangedBehaviorComparison =
|
||||
ParsedComparisonEvidence<ChangedBehaviorComparisonOperation>;
|
||||
|
||||
/** Input for aggregating existing comparison Evidence. */
|
||||
export const changedBehaviorInputSchema = z.strictObject({
|
||||
@@ -198,17 +205,20 @@ const parseComparisonEvidence = (
|
||||
input: unknown,
|
||||
fullInput: unknown,
|
||||
index: number,
|
||||
): ParsedComparisonEvidence => {
|
||||
): ChangedBehaviorComparison => {
|
||||
const parsedEvidence = readEvidence(input);
|
||||
if (!parsedEvidence.ok) throw parsedEvidence.error;
|
||||
const evidence = parsedEvidence.value;
|
||||
if (!isComparisonEvidenceOperation(evidence.operation))
|
||||
if (
|
||||
!isComparisonEvidenceOperation(evidence.operation) ||
|
||||
evidence.operation === "compare_web_screenshots"
|
||||
)
|
||||
throw callerInputError(
|
||||
fullInput,
|
||||
["comparisons", index],
|
||||
"Changed-behavior analysis requires process, artifact, or function comparison Evidence",
|
||||
);
|
||||
const operation = evidence.operation;
|
||||
const operation: ChangedBehaviorComparisonOperation = evidence.operation;
|
||||
if (!hasComparisonEvidenceIdentity(evidence))
|
||||
throw evidenceIntegrityError(
|
||||
"Comparison Evidence operation, predicate, and provider disagree",
|
||||
@@ -226,7 +236,7 @@ const parseComparisonEvidence = (
|
||||
return result.value;
|
||||
};
|
||||
|
||||
const findingsFor = (comparison: ParsedComparisonEvidence): Finding[] => {
|
||||
const findingsFor = (comparison: ChangedBehaviorComparison): Finding[] => {
|
||||
if (comparison.operation === "compare_process_captures")
|
||||
return processFindings(comparison);
|
||||
if (comparison.operation === "compare_functions")
|
||||
@@ -236,7 +246,7 @@ const findingsFor = (comparison: ParsedComparisonEvidence): Finding[] => {
|
||||
|
||||
const processFindings = (
|
||||
comparison: Extract<
|
||||
ParsedComparisonEvidence,
|
||||
ChangedBehaviorComparison,
|
||||
{ readonly operation: "compare_process_captures" }
|
||||
>,
|
||||
): Finding[] => {
|
||||
@@ -292,7 +302,7 @@ const unprovenEqualityReason = (
|
||||
|
||||
const functionFindings = (
|
||||
comparison: Extract<
|
||||
ParsedComparisonEvidence,
|
||||
ChangedBehaviorComparison,
|
||||
{ readonly operation: "compare_functions" }
|
||||
>,
|
||||
): Finding[] => {
|
||||
@@ -333,7 +343,7 @@ const functionFindings = (
|
||||
|
||||
const artifactFindings = (
|
||||
comparison: Extract<
|
||||
ParsedComparisonEvidence,
|
||||
ChangedBehaviorComparison,
|
||||
{ readonly operation: "compare_artifacts" }
|
||||
>,
|
||||
): Finding[] => {
|
||||
|
||||
@@ -38,6 +38,14 @@ const COMPARISON_EVIDENCE_IDENTITIES = {
|
||||
predicate: "rea.process-comparison",
|
||||
provider: PROCESS_PROVIDER,
|
||||
},
|
||||
compare_web_screenshots: {
|
||||
predicate: "rea.web-screenshot-diff",
|
||||
provider: {
|
||||
id: "rea-cdp-browser",
|
||||
name: "REA Chrome DevTools Protocol observation provider",
|
||||
version: "2",
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
export type ComparisonEvidenceOperation =
|
||||
@@ -59,14 +67,21 @@ export const hasComparisonEvidenceIdentity = (evidence: Evidence): boolean => {
|
||||
: evidence.provider.id === expected.provider.id &&
|
||||
evidence.provider.name === expected.provider.name &&
|
||||
evidence.provider.version === expected.provider.version;
|
||||
const subjectMatches =
|
||||
evidence.operation === "compare_web_screenshots"
|
||||
? sameTargetIdentity(
|
||||
evidence.subject.identity,
|
||||
unknownTarget("not-observed"),
|
||||
)
|
||||
: sameTargetIdentity(
|
||||
evidence.subject.identity,
|
||||
unknownTarget("derived-from-linked-evidence"),
|
||||
);
|
||||
return (
|
||||
evidence.predicate_type === expected.predicate &&
|
||||
providerMatches &&
|
||||
evidence.confidence === "derived" &&
|
||||
evidence.authority === "analyst-inference" &&
|
||||
sameTargetIdentity(
|
||||
evidence.subject.identity,
|
||||
unknownTarget("derived-from-linked-evidence"),
|
||||
)
|
||||
subjectMatches
|
||||
);
|
||||
};
|
||||
@@ -14,6 +14,10 @@ import {
|
||||
processCaptureComparisonSchema,
|
||||
type ProcessCaptureComparison,
|
||||
} from "./process/processComparison.js";
|
||||
import {
|
||||
webScreenshotDiffSchema,
|
||||
type WebScreenshotDiff,
|
||||
} from "./webScreenshot.js";
|
||||
import { err, ok, type Result } from "./result.js";
|
||||
import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js";
|
||||
|
||||
@@ -21,6 +25,7 @@ type ComparisonResults = {
|
||||
readonly compare_artifacts: ArtifactComparisonResult;
|
||||
readonly compare_functions: FunctionComparisonResult;
|
||||
readonly compare_process_captures: ProcessCaptureComparison;
|
||||
readonly compare_web_screenshots: WebScreenshotDiff;
|
||||
};
|
||||
|
||||
/** One admitted comparison's envelope and its operation-specific parsed facts. */
|
||||
@@ -101,6 +106,15 @@ export function readComparisonResult(
|
||||
if (!links.ok) return links;
|
||||
return ok({ operation, evidence, result: parsed.value });
|
||||
}
|
||||
if (operation === "compare_web_screenshots") {
|
||||
const parsed = readEvidencePayload(
|
||||
webScreenshotDiffSchema,
|
||||
evidence.normalized_result,
|
||||
{ within: ["normalized_result"] },
|
||||
);
|
||||
if (!parsed.ok) return parsed;
|
||||
return ok({ operation, evidence, result: parsed.value });
|
||||
}
|
||||
const parsed = readEvidencePayload(
|
||||
artifactComparisonResultSchema,
|
||||
evidence.normalized_result,
|
||||
|
||||
@@ -361,6 +361,7 @@ const inflatePng = (compressed: Buffer, maxOutputLength: number): Buffer => {
|
||||
"code" in cause &&
|
||||
(cause.code === "Z_DATA_ERROR" ||
|
||||
cause.code === "Z_BUF_ERROR" ||
|
||||
cause.code === "Z_NEED_DICT" ||
|
||||
cause.code === "ERR_BUFFER_TOO_LARGE")
|
||||
)
|
||||
throw new TypeError("Invalid PNG image data", { cause });
|
||||
|
||||
@@ -141,8 +141,14 @@ const defaultProbe = (
|
||||
operation: "analyze_function",
|
||||
rationale: "Capture complete function dossiers under equal limits.",
|
||||
}
|
||||
: {
|
||||
operation: "inspect_artifact",
|
||||
rationale:
|
||||
"Capture complete artifact inventories under equal limits.",
|
||||
};
|
||||
: kind === "visual"
|
||||
? {
|
||||
operation: "capture_web_screenshot",
|
||||
rationale:
|
||||
"Capture both visible viewports under the same browser scope.",
|
||||
}
|
||||
: {
|
||||
operation: "inspect_artifact",
|
||||
rationale:
|
||||
"Capture complete artifact inventories under equal limits.",
|
||||
};
|
||||
@@ -547,6 +547,7 @@ describe("reconstruction verification integrity", () => {
|
||||
unknown: 0,
|
||||
behavioral: 0,
|
||||
structural: 1,
|
||||
visual: 0,
|
||||
},
|
||||
claims: { items: [claim] },
|
||||
recommended_probes: [],
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
|
||||
import { evidenceFileTarget } from "./evidence.js";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
ArtifactInventoryCarrierError,
|
||||
parseArtifactInventoryEvidence,
|
||||
@@ -11,7 +14,11 @@ import { readComparisonResult } from "./comparisonEvidenceResult.js";
|
||||
import type { Evidence } from "./evidence.js";
|
||||
import { hasComparisonEvidenceIdentity } from "./comparisonEvidenceIdentity.js";
|
||||
import { readEvidenceBundle } from "./evidenceBundle.js";
|
||||
import { AnalysisInputError } from "./analysisErrorCore.js";
|
||||
import {
|
||||
AnalysisInputError,
|
||||
AnalysisResourceConstraintError,
|
||||
AnalysisUnsupportedTargetError,
|
||||
} from "./analysisErrorCore.js";
|
||||
import { EvidenceIntegrityError } from "./evidenceErrors.js";
|
||||
import { analysisInputErrorFromIssues } from "./inputIssueProjection.js";
|
||||
import { readEvidencePayload } from "./evidenceValidation.js";
|
||||
@@ -29,7 +36,9 @@ import {
|
||||
import type { ProcessObservationSource } from "./process/processObservation.js";
|
||||
import { digestCanonicalValue } from "./canonicalDigest.js";
|
||||
import {
|
||||
countReconstructionClaimKinds,
|
||||
reconstructionClaimResultSchema,
|
||||
reconstructionClaimSelector,
|
||||
reconstructionSpecificationSchema,
|
||||
reconstructionVerificationResultSchema,
|
||||
type ReconstructionClaim as Claim,
|
||||
@@ -43,11 +52,14 @@ import {
|
||||
reconstructionUnknownHeads,
|
||||
} from "./reconstructionUnknowns.js";
|
||||
import type { ResidualUnknown } from "./residualUnknown.js";
|
||||
import { comparePngScreenshots } from "./pngVisualDiff.js";
|
||||
import { webScreenshotSchema, type WebScreenshot } from "./webScreenshot.js";
|
||||
|
||||
const comparisonOperationsByClaimKind = {
|
||||
behavioral: "compare_process_captures",
|
||||
"structural-function": "compare_functions",
|
||||
"structural-artifact": "compare_artifacts",
|
||||
visual: "compare_web_screenshots",
|
||||
} as const;
|
||||
|
||||
interface FunctionSourceFacts {
|
||||
@@ -95,10 +107,38 @@ export const verifyReconstruction = (
|
||||
);
|
||||
const heads = reconstructionUnknownHeads(bundle.unknowns);
|
||||
const artifactReplays = new Map<string, ArtifactReplay>();
|
||||
const screenshots: ScreenshotAssessment = {
|
||||
captures: specification.claims.some(({ kind }) => kind === "visual")
|
||||
? indexScreenshotCaptures(records.values())
|
||||
: new Map(),
|
||||
replayBlockers: new Map(),
|
||||
};
|
||||
const visualSelectors = new Set<string>();
|
||||
const results = specification.claims
|
||||
.map((claim, index) =>
|
||||
evaluateClaim(claim, index, records, heads, artifactReplays),
|
||||
)
|
||||
.map((claim, index) => {
|
||||
const result = evaluateClaim(
|
||||
claim,
|
||||
index,
|
||||
records,
|
||||
heads,
|
||||
artifactReplays,
|
||||
screenshots,
|
||||
);
|
||||
if (claim.kind === "visual") {
|
||||
// An inferred capture pair can resolve to the same selector as an explicit pair.
|
||||
const selector = reconstructionClaimSelector(claim, [
|
||||
result.left_evidence_ids[0]!,
|
||||
result.right_evidence_ids[0]!,
|
||||
]);
|
||||
if (visualSelectors.has(selector))
|
||||
throw claimComparisonInputError(
|
||||
index,
|
||||
"Claim comparison selectors must be unique",
|
||||
);
|
||||
visualSelectors.add(selector);
|
||||
}
|
||||
return result;
|
||||
})
|
||||
.sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.claim_id, right.claim_id),
|
||||
);
|
||||
@@ -124,8 +164,7 @@ export const verifyReconstruction = (
|
||||
passed: count(results, "pass"),
|
||||
failed,
|
||||
unknown: unresolved,
|
||||
behavioral: results.filter(({ kind }) => kind === "behavioral").length,
|
||||
structural: results.filter(({ kind }) => kind !== "behavioral").length,
|
||||
...countReconstructionClaimKinds(results),
|
||||
},
|
||||
claims: {
|
||||
items: results,
|
||||
@@ -145,17 +184,28 @@ const evaluateClaim = (
|
||||
records: ReadonlyMap<string, Evidence>,
|
||||
heads: ReadonlyMap<string, ResidualUnknown>,
|
||||
artifactReplays: Map<string, ArtifactReplay>,
|
||||
screenshots: ScreenshotAssessment,
|
||||
): ClaimResult => {
|
||||
const comparison = records.get(claim.comparison_evidence_id);
|
||||
if (comparison === undefined)
|
||||
throw missingSourceInputError(
|
||||
throw claimComparisonInputError(
|
||||
claimIndex,
|
||||
`Missing comparison Evidence ${claim.comparison_evidence_id}`,
|
||||
);
|
||||
validateComparisonIdentity(claim, comparison, claimIndex);
|
||||
const sides = comparisonSourceEvidenceSides(comparison);
|
||||
const { sides, observation, visualBlockers } =
|
||||
claim.kind === "visual"
|
||||
? assessScreenshotComparison(claim, comparison, screenshots)
|
||||
: {
|
||||
sides: comparisonSourceEvidenceSides(comparison),
|
||||
observation: comparisonObservation(claim, comparison),
|
||||
visualBlockers: [],
|
||||
};
|
||||
const allSourceIds = uniqueSorted([...sides.left, ...sides.right]);
|
||||
if (!sameSet(comparison.evidence_links, allSourceIds))
|
||||
if (
|
||||
claim.kind !== "visual" &&
|
||||
!sameSet(comparison.evidence_links, allSourceIds)
|
||||
)
|
||||
throw new EvidenceIntegrityError(
|
||||
"Comparison Evidence closure disagrees with its parameters",
|
||||
{
|
||||
@@ -163,17 +213,31 @@ const evaluateClaim = (
|
||||
"Comparison Evidence closure disagrees with its parameters",
|
||||
},
|
||||
);
|
||||
if (
|
||||
claim.kind === "visual" &&
|
||||
comparison.evidence_links.length > 0 &&
|
||||
!sameSet(comparison.evidence_links, allSourceIds)
|
||||
)
|
||||
throw new EvidenceIntegrityError(
|
||||
"Screenshot comparison Evidence links disagree with its artifact inputs",
|
||||
{
|
||||
userMessage:
|
||||
"Screenshot comparison Evidence links disagree with its artifact inputs",
|
||||
},
|
||||
);
|
||||
const sources = allSourceIds.map((id) => {
|
||||
const source = records.get(id);
|
||||
if (source === undefined)
|
||||
throw missingSourceInputError(
|
||||
throw claimComparisonInputError(
|
||||
claimIndex,
|
||||
`Missing source Evidence ${id}`,
|
||||
);
|
||||
return source;
|
||||
});
|
||||
const sourceFacts = readValidatedSourceFacts(claim, sources);
|
||||
const observation = comparisonObservation(claim, comparison);
|
||||
const sourceFacts =
|
||||
claim.kind === "visual"
|
||||
? { functions: [], processes: [] }
|
||||
: readValidatedSourceFacts(claim, sources);
|
||||
const observed = observation.status;
|
||||
const callCoverage = sourceCallCoverage(
|
||||
claim,
|
||||
@@ -183,6 +247,7 @@ const evaluateClaim = (
|
||||
const blockers = [
|
||||
...authorityLimitations(claim, sources, sides),
|
||||
...callCoverage.blockers,
|
||||
...visualBlockers,
|
||||
...sourceProcessCoverage(claim, sourceFacts.processes, observed),
|
||||
...(claim.kind === "structural-artifact"
|
||||
? assessArtifactSourceComparison(
|
||||
@@ -224,6 +289,159 @@ const evaluateClaim = (
|
||||
});
|
||||
};
|
||||
|
||||
const screenshotComparisonParametersSchema = z.strictObject({
|
||||
before_artifact_sha256: z.string().regex(/^[a-f0-9]{64}$/u),
|
||||
after_artifact_sha256: z.string().regex(/^[a-f0-9]{64}$/u),
|
||||
channel_threshold: z.int().min(0).max(255),
|
||||
comparison_input_sha256: z
|
||||
.string()
|
||||
.regex(/^[a-f0-9]{64}$/u)
|
||||
.optional(),
|
||||
});
|
||||
|
||||
interface ScreenshotCapture {
|
||||
readonly evidenceId: string;
|
||||
readonly screenshot: WebScreenshot;
|
||||
}
|
||||
|
||||
interface ScreenshotAssessment {
|
||||
readonly captures: ReadonlyMap<string, readonly ScreenshotCapture[]>;
|
||||
readonly replayBlockers: Map<string, string | null>;
|
||||
}
|
||||
|
||||
const indexScreenshotCaptures = (records: Iterable<Evidence>) => {
|
||||
const captures = new Map<string, ScreenshotCapture[]>();
|
||||
for (const source of records) {
|
||||
if (
|
||||
source.operation !== "capture_web_screenshot" ||
|
||||
source.predicate_type !== "rea.web-screenshot"
|
||||
)
|
||||
continue;
|
||||
const parsed = readEvidencePayload(
|
||||
webScreenshotSchema,
|
||||
source.normalized_result,
|
||||
{ within: ["normalized_result"] },
|
||||
);
|
||||
if (!parsed.ok) throw parsed.error;
|
||||
const sha256 = parsed.value.artifact.sha256;
|
||||
const capture = {
|
||||
evidenceId: source.evidence_id,
|
||||
screenshot: parsed.value,
|
||||
};
|
||||
const existing = captures.get(sha256);
|
||||
if (existing === undefined) captures.set(sha256, [capture]);
|
||||
else existing.push(capture);
|
||||
}
|
||||
return captures;
|
||||
};
|
||||
|
||||
const assessScreenshotComparison = (
|
||||
claim: Extract<Claim, { kind: "visual" }>,
|
||||
comparison: Evidence,
|
||||
{ captures, replayBlockers }: ScreenshotAssessment,
|
||||
): {
|
||||
sides: { left: string[]; right: string[] };
|
||||
observation: ComparisonObservation;
|
||||
visualBlockers: string[];
|
||||
} => {
|
||||
const parsed = readEvidencePayload(
|
||||
screenshotComparisonParametersSchema,
|
||||
comparison.parameters,
|
||||
{ within: ["parameters"] },
|
||||
);
|
||||
if (!parsed.ok) throw parsed.error;
|
||||
const sourceForArtifact = (
|
||||
sha256: string,
|
||||
side: "before" | "after",
|
||||
evidenceId: string | undefined,
|
||||
) => {
|
||||
const matches = captures.get(sha256);
|
||||
if (evidenceId !== undefined) {
|
||||
const selected = matches?.find((item) => item.evidenceId === evidenceId);
|
||||
if (selected !== undefined) return selected;
|
||||
const message = `Screenshot comparison ${side} capture Evidence does not match its artifact digest or is missing from the bundle (${evidenceId}).`;
|
||||
throw new EvidenceIntegrityError(message, { userMessage: message });
|
||||
}
|
||||
if (matches?.length !== 1) {
|
||||
const message = `Screenshot comparison ${side} artifact does not identify exactly one captured Evidence record. Supply both capture IDs in source_evidence when the digest is ambiguous.`;
|
||||
throw new EvidenceIntegrityError(message, { userMessage: message });
|
||||
}
|
||||
return matches[0]!;
|
||||
};
|
||||
const before = sourceForArtifact(
|
||||
parsed.value.before_artifact_sha256,
|
||||
"before",
|
||||
claim.source_evidence?.before_evidence_id,
|
||||
);
|
||||
const after = sourceForArtifact(
|
||||
parsed.value.after_artifact_sha256,
|
||||
"after",
|
||||
claim.source_evidence?.after_evidence_id,
|
||||
);
|
||||
const saved = readComparisonResult(comparison, "compare_web_screenshots");
|
||||
if (!saved.ok) throw saved.error;
|
||||
const { limitations, ...recorded } = saved.value.result;
|
||||
const visualBlockers: string[] = [];
|
||||
for (const { evidenceId, screenshot } of [before, after]) {
|
||||
const { status, conditions } = screenshot.completeness;
|
||||
if (
|
||||
status !== "complete_within_window" ||
|
||||
conditions.some((condition) => condition !== "complete_within_window")
|
||||
)
|
||||
visualBlockers.push(
|
||||
`Source ${evidenceId}: Screenshot coverage is incomplete (${uniqueSorted([status, ...conditions]).join(", ")}).`,
|
||||
);
|
||||
}
|
||||
// The comparison fixes both artifact digests and the threshold. Coverage
|
||||
// stays specific to the selected captures, while the pixel replay is reusable.
|
||||
let replayBlocker = replayBlockers.get(comparison.evidence_id);
|
||||
if (replayBlocker === undefined) {
|
||||
try {
|
||||
const { limitations: _limitations, ...replayed } = comparePngScreenshots({
|
||||
before: before.screenshot.artifact,
|
||||
after: after.screenshot.artifact,
|
||||
channel_threshold: parsed.value.channel_threshold,
|
||||
});
|
||||
replayBlocker = isDeepStrictEqual(recorded, replayed)
|
||||
? null
|
||||
: "Retained screenshot pixels do not corroborate the recorded comparison.";
|
||||
} catch (cause) {
|
||||
if (
|
||||
!(cause instanceof TypeError) &&
|
||||
!(cause instanceof AnalysisResourceConstraintError) &&
|
||||
!(cause instanceof AnalysisUnsupportedTargetError)
|
||||
)
|
||||
throw cause;
|
||||
replayBlocker = `Retained screenshot comparison could not be replayed: ${cause.message}`;
|
||||
}
|
||||
replayBlockers.set(comparison.evidence_id, replayBlocker);
|
||||
}
|
||||
if (replayBlocker !== null) visualBlockers.push(replayBlocker);
|
||||
return {
|
||||
sides: { left: [before.evidenceId], right: [after.evidenceId] },
|
||||
observation: {
|
||||
status:
|
||||
recorded.status === "identical"
|
||||
? "unchanged"
|
||||
: recorded.status === "different"
|
||||
? "changed"
|
||||
: "unknown",
|
||||
limitations: [
|
||||
...limitations,
|
||||
...[before, after].flatMap(({ evidenceId, screenshot }) =>
|
||||
screenshot.limitations.map(
|
||||
(detail) => `Source ${evidenceId}: ${detail}`,
|
||||
),
|
||||
),
|
||||
...(recorded.status === "dimension_mismatch"
|
||||
? ["Screenshot dimensions differ."]
|
||||
: []),
|
||||
],
|
||||
},
|
||||
visualBlockers,
|
||||
};
|
||||
};
|
||||
|
||||
const assessArtifactSourceComparison = (
|
||||
sides: {
|
||||
readonly left: readonly string[];
|
||||
@@ -295,7 +513,7 @@ const assessArtifactSourceComparison = (
|
||||
};
|
||||
|
||||
const readValidatedSourceFacts = (
|
||||
claim: Claim,
|
||||
claim: Exclude<Claim, { kind: "visual" }>,
|
||||
sources: readonly Evidence[],
|
||||
): ValidatedSourceFacts => {
|
||||
const isExpectedType = (source: Evidence): boolean => {
|
||||
@@ -381,7 +599,7 @@ const validateComparisonIdentity = (
|
||||
): void => {
|
||||
const expectedOperation = comparisonOperationsByClaimKind[claim.kind];
|
||||
if (evidence.operation !== expectedOperation)
|
||||
throw missingSourceInputError(
|
||||
throw claimComparisonInputError(
|
||||
claimIndex,
|
||||
`Claim ${claim.claim_id} requires ${expectedOperation} Evidence; selected ${evidence.operation}`,
|
||||
);
|
||||
@@ -391,15 +609,17 @@ const validateComparisonIdentity = (
|
||||
}
|
||||
};
|
||||
|
||||
const comparisonObservation = (
|
||||
claim: Claim,
|
||||
evidence: Evidence,
|
||||
): {
|
||||
interface ComparisonObservation {
|
||||
readonly status: ObservedStatus;
|
||||
readonly limitations: readonly string[];
|
||||
readonly hasIndependentFunctionChange?: boolean;
|
||||
readonly selectedDimensions?: readonly string[];
|
||||
} => {
|
||||
}
|
||||
|
||||
const comparisonObservation = (
|
||||
claim: Exclude<Claim, { kind: "visual" }>,
|
||||
evidence: Evidence,
|
||||
): ComparisonObservation => {
|
||||
if (claim.kind === "behavioral") {
|
||||
const parsed = readComparisonResult(evidence, "compare_process_captures");
|
||||
if (!parsed.ok) throw parsed.error;
|
||||
@@ -460,7 +680,7 @@ const comparisonObservation = (
|
||||
return { status: result.status, limitations: [] };
|
||||
};
|
||||
|
||||
const missingSourceInputError = (claimIndex: number, message: string) =>
|
||||
const claimComparisonInputError = (claimIndex: number, message: string) =>
|
||||
new AnalysisInputError("verify_reconstruction", { userMessage: message }, [
|
||||
{
|
||||
path: ["claims", claimIndex, "comparison_evidence_id"],
|
||||
@@ -478,7 +698,11 @@ const authorityLimitations = (
|
||||
},
|
||||
): string[] => {
|
||||
const requiredAuthority =
|
||||
claim.kind === "behavioral" ? "controlled-replay" : "shipped-artifact";
|
||||
claim.kind === "behavioral"
|
||||
? "controlled-replay"
|
||||
: claim.kind === "visual"
|
||||
? "external-service"
|
||||
: "shipped-artifact";
|
||||
const limitations: string[] = [];
|
||||
if (sides.left.some((id) => sides.right.includes(id)))
|
||||
limitations.push(
|
||||
@@ -497,7 +721,8 @@ const authorityLimitations = (
|
||||
`Source Evidence lacks qualifying ${requiredAuthority} authority.`,
|
||||
);
|
||||
if (
|
||||
claim.kind !== "behavioral" &&
|
||||
(claim.kind === "structural-function" ||
|
||||
claim.kind === "structural-artifact") &&
|
||||
sources.some((source) => evidenceFileTarget(source) === undefined)
|
||||
)
|
||||
limitations.push("Structural source artifact identity is unavailable.");
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { createEvidence, type Evidence } from "./evidence.js";
|
||||
import { createEvidenceBundle } from "./evidenceBundle.js";
|
||||
import {
|
||||
createWebScreenshotArtifact,
|
||||
webScreenshotSchema,
|
||||
} from "./webScreenshot.js";
|
||||
import { unknownTarget } from "./targetIdentity.js";
|
||||
import { verifyReconstruction } from "./reconstructionVerification.js";
|
||||
import { reconstructionVerificationResultSchema } from "./reconstructionVerificationSchemas.js";
|
||||
|
||||
const browserProvider = {
|
||||
id: "rea-cdp-browser",
|
||||
name: "REA Chrome DevTools Protocol observation provider",
|
||||
version: "2",
|
||||
} as const;
|
||||
|
||||
// The first two 1x1 images encode the same RGBA pixel with different compression.
|
||||
const beforePng =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAEElEQVR4AQEFAPr/AAAAAP4BAwD/dUi/yAAAAABJRU5ErkJggg==";
|
||||
const samePng =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNgYGD4BwABAwD/Xb6bfwAAAABJRU5ErkJggg==";
|
||||
const differentPng =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNgZGT8DwABDQEDO4rAiwAAAABJRU5ErkJggg==";
|
||||
const widerPng =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR42mNgYGD4B8IABf0B/aLhVnkAAAAASUVORK5CYII=";
|
||||
|
||||
// This PNG's IDAT zlib stream requests a preset dictionary.
|
||||
const dictionaryPng =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAEUlEQVR4uwwXAwhjYGBg+AcAAQMA/7BXXg0AAAAASUVORK5CYII=";
|
||||
const unverifiedDocument = "The captured document could not be verified.";
|
||||
|
||||
const screenshot = (
|
||||
bytes: string,
|
||||
capturedAt: string,
|
||||
{ limited = false, width = 1 } = {},
|
||||
): Evidence =>
|
||||
createEvidence(unknownTarget("not-observed"), browserProvider, {
|
||||
predicateType: "rea.web-screenshot",
|
||||
operation: "capture_web_screenshot",
|
||||
parameters: {},
|
||||
result: {
|
||||
browser: {
|
||||
product: "Chrome/130",
|
||||
protocol_version: "1.3",
|
||||
revision: "fixture",
|
||||
user_agent: "fixture",
|
||||
js_version: "fixture",
|
||||
},
|
||||
target: {
|
||||
target_id: "fixture-page",
|
||||
url: "https://example.test/",
|
||||
origin: "https://example.test",
|
||||
},
|
||||
captured_at: capturedAt,
|
||||
viewport: { width, height: 1 },
|
||||
artifact: createWebScreenshotArtifact(Buffer.from(bytes, "base64")),
|
||||
completeness: {
|
||||
status: limited ? "attach_limited" : "complete_within_window",
|
||||
conditions: [limited ? "attach_limited" : "complete_within_window"],
|
||||
policy_filtered_sections: [],
|
||||
attach_limited_sections: limited ? ["screenshot"] : [],
|
||||
truncated_sections: [],
|
||||
unavailable_sections: [],
|
||||
excluded: [],
|
||||
dropped_events: {
|
||||
scripts: 0,
|
||||
network_requests: 0,
|
||||
console_events: 0,
|
||||
websocket_connections: 0,
|
||||
websocket_frames: 0,
|
||||
webmcp_tools: 0,
|
||||
timeline_events: 0,
|
||||
total: 0,
|
||||
},
|
||||
},
|
||||
limitations: limited ? [unverifiedDocument] : [],
|
||||
},
|
||||
confidence: "observed",
|
||||
authority: "external-service",
|
||||
});
|
||||
|
||||
const comparison = (
|
||||
before: Evidence,
|
||||
after: Evidence,
|
||||
status: "identical" | "different" | "dimension_mismatch",
|
||||
threshold = 0,
|
||||
): Evidence =>
|
||||
createEvidence(unknownTarget("not-observed"), browserProvider, {
|
||||
predicateType: "rea.web-screenshot-diff",
|
||||
operation: "compare_web_screenshots",
|
||||
parameters: {
|
||||
before_artifact_sha256: webScreenshotSchema.parse(
|
||||
before.normalized_result,
|
||||
).artifact.sha256,
|
||||
after_artifact_sha256: webScreenshotSchema.parse(after.normalized_result)
|
||||
.artifact.sha256,
|
||||
channel_threshold: threshold,
|
||||
},
|
||||
result: {
|
||||
before: { width: 1, height: 1 },
|
||||
after: {
|
||||
width: status === "dimension_mismatch" ? 2 : 1,
|
||||
height: 1,
|
||||
},
|
||||
channel_threshold: threshold,
|
||||
limitations: [],
|
||||
status,
|
||||
...(status === "identical"
|
||||
? {
|
||||
compared_pixels: 1,
|
||||
changed_pixels: 0,
|
||||
changed_ratio: 0,
|
||||
maximum_channel_delta: threshold,
|
||||
mean_absolute_channel_delta: threshold,
|
||||
}
|
||||
: status === "different"
|
||||
? {
|
||||
compared_pixels: 1,
|
||||
changed_pixels: 1,
|
||||
changed_ratio: 1,
|
||||
maximum_channel_delta: 1,
|
||||
mean_absolute_channel_delta: 1,
|
||||
}
|
||||
: {
|
||||
compared_pixels: 0,
|
||||
changed_pixels: null,
|
||||
changed_ratio: null,
|
||||
maximum_channel_delta: null,
|
||||
mean_absolute_channel_delta: null,
|
||||
}),
|
||||
},
|
||||
confidence: "derived",
|
||||
authority: "analyst-inference",
|
||||
});
|
||||
|
||||
const specification = (
|
||||
evidence: Evidence,
|
||||
sourceEvidence?: { before_evidence_id: string; after_evidence_id: string },
|
||||
) => ({
|
||||
name: "Visual reconstruction",
|
||||
claims: [
|
||||
{
|
||||
kind: "visual" as const,
|
||||
claim_id: "viewport",
|
||||
title: "Viewport pixels remain equivalent",
|
||||
comparison_evidence_id: evidence.evidence_id,
|
||||
dimension: "overall" as const,
|
||||
...(sourceEvidence === undefined
|
||||
? {}
|
||||
: { source_evidence: sourceEvidence }),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const verify = (comparisonEvidence: Evidence, sources: Evidence[]) =>
|
||||
verifyReconstruction(
|
||||
specification(comparisonEvidence),
|
||||
createEvidenceBundle([...sources, comparisonEvidence]),
|
||||
);
|
||||
|
||||
describe("visual reconstruction evidence", () => {
|
||||
it("accepts a screenshot comparison and matches its source artifacts", () => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
|
||||
const after = screenshot(samePng, "2026-01-01T00:00:01.000Z");
|
||||
const result = verify(comparison(before, after, "identical"), [
|
||||
before,
|
||||
after,
|
||||
]);
|
||||
|
||||
expect(result.claims.items).toMatchObject([
|
||||
{ kind: "visual", status: "pass", observed_status: "unchanged" },
|
||||
]);
|
||||
expect(result.summary).toMatchObject({
|
||||
total: 1,
|
||||
behavioral: 0,
|
||||
structural: 0,
|
||||
visual: 1,
|
||||
});
|
||||
expect(
|
||||
reconstructionVerificationResultSchema.safeParse({
|
||||
...result,
|
||||
summary: { ...result.summary, structural: 1, visual: 0 },
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["different", "fail", "changed"],
|
||||
["dimension_mismatch", "unknown", "unknown"],
|
||||
] as const)(
|
||||
"projects a %s screenshot diff into reconstruction status",
|
||||
(diffStatus, status, observedStatus) => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
|
||||
const mismatch = diffStatus === "dimension_mismatch";
|
||||
const after = screenshot(
|
||||
mismatch ? widerPng : differentPng,
|
||||
"2026-01-01T00:00:01.000Z",
|
||||
{ width: mismatch ? 2 : 1 },
|
||||
);
|
||||
const result = verify(comparison(before, after, diffStatus), [
|
||||
before,
|
||||
after,
|
||||
]);
|
||||
|
||||
expect(result.claims.items).toMatchObject([
|
||||
{ kind: "visual", status, observed_status: observedStatus },
|
||||
]);
|
||||
if (diffStatus === "dimension_mismatch")
|
||||
expect(result.recommended_probes).toContainEqual(
|
||||
expect.objectContaining({ operation: "capture_web_screenshot" }),
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("visual reconstruction capture identity", () => {
|
||||
it("rejects an ambiguous artifact-to-capture match", () => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
|
||||
const duplicate = screenshot(beforePng, "2026-01-01T00:00:01.000Z");
|
||||
const after = screenshot(samePng, "2026-01-01T00:00:02.000Z");
|
||||
|
||||
expect(() =>
|
||||
verify(comparison(before, after, "identical"), [
|
||||
before,
|
||||
duplicate,
|
||||
after,
|
||||
]),
|
||||
).toThrow(/does not identify exactly one captured Evidence record/u);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"checks capture pairs independently when they share one comparison (second pair limited: %s)",
|
||||
(limited) => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
|
||||
const after = screenshot(beforePng, "2026-01-01T00:00:01.000Z");
|
||||
const repeatedBefore = screenshot(beforePng, "2026-01-01T00:00:02.000Z", {
|
||||
limited,
|
||||
});
|
||||
const repeatedAfter = screenshot(beforePng, "2026-01-01T00:00:03.000Z");
|
||||
const saved = comparison(before, after, "identical");
|
||||
const spec = specification(saved, {
|
||||
before_evidence_id: before.evidence_id,
|
||||
after_evidence_id: after.evidence_id,
|
||||
});
|
||||
const second = specification(saved, {
|
||||
before_evidence_id: repeatedBefore.evidence_id,
|
||||
after_evidence_id: repeatedAfter.evidence_id,
|
||||
}).claims[0]!;
|
||||
const bundle = createEvidenceBundle([
|
||||
repeatedAfter,
|
||||
after,
|
||||
saved,
|
||||
before,
|
||||
repeatedBefore,
|
||||
]);
|
||||
|
||||
const result = verifyReconstruction(
|
||||
{
|
||||
...spec,
|
||||
claims: [...spec.claims, { ...second, claim_id: "repeated" }],
|
||||
},
|
||||
bundle,
|
||||
);
|
||||
expect(result.claims.items).toMatchObject([
|
||||
{
|
||||
claim_id: "repeated",
|
||||
status: limited ? "unknown" : "pass",
|
||||
left_evidence_ids: [repeatedBefore.evidence_id],
|
||||
right_evidence_ids: [repeatedAfter.evidence_id],
|
||||
},
|
||||
{
|
||||
claim_id: "viewport",
|
||||
status: "pass",
|
||||
left_evidence_ids: [before.evidence_id],
|
||||
right_evidence_ids: [after.evidence_id],
|
||||
},
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["explicit", "inferred"] as const)(
|
||||
"rejects the same capture pair when the second selector is %s",
|
||||
(selection) => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
|
||||
const after = screenshot(samePng, "2026-01-01T00:00:01.000Z");
|
||||
const saved = comparison(before, after, "identical");
|
||||
const sources = {
|
||||
before_evidence_id: before.evidence_id,
|
||||
after_evidence_id: after.evidence_id,
|
||||
};
|
||||
const spec = specification(saved, sources);
|
||||
const second = specification(
|
||||
saved,
|
||||
selection === "explicit" ? sources : undefined,
|
||||
).claims[0]!;
|
||||
|
||||
expect(() =>
|
||||
verifyReconstruction(
|
||||
{
|
||||
...spec,
|
||||
claims: [...spec.claims, { ...second, claim_id: "repeat" }],
|
||||
},
|
||||
createEvidenceBundle([before, after, saved]),
|
||||
),
|
||||
).toThrow(
|
||||
expect.objectContaining({
|
||||
issues: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
message: "Claim comparison selectors must be unique",
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects an explicit screenshot source whose bytes disagree with that side", () => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
|
||||
const after = screenshot(samePng, "2026-01-01T00:00:01.000Z");
|
||||
const saved = comparison(before, after, "identical");
|
||||
const spec = specification(saved, {
|
||||
before_evidence_id: after.evidence_id,
|
||||
after_evidence_id: before.evidence_id,
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
verifyReconstruction(spec, createEvidenceBundle([before, after, saved])),
|
||||
).toThrow(/before capture Evidence does not match its artifact digest/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("visual reconstruction coverage and corroboration", () => {
|
||||
it("withholds a visual verdict when capture attribution is incomplete", () => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z", {
|
||||
limited: true,
|
||||
});
|
||||
const after = screenshot(samePng, "2026-01-01T00:00:01.000Z");
|
||||
const result = verify(comparison(before, after, "identical"), [
|
||||
before,
|
||||
after,
|
||||
]);
|
||||
|
||||
expect(result.claims.items).toMatchObject([
|
||||
{
|
||||
status: "unknown",
|
||||
observed_status: "unchanged",
|
||||
limitations: expect.arrayContaining([
|
||||
expect.stringMatching(
|
||||
/Screenshot coverage is incomplete.*attach_limited/u,
|
||||
),
|
||||
expect.stringContaining(unverifiedDocument),
|
||||
]),
|
||||
},
|
||||
]);
|
||||
expect(result.recommended_probes).toContainEqual(
|
||||
expect.objectContaining({ operation: "capture_web_screenshot" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("requires the retained pixels to corroborate the reported comparison", () => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
|
||||
const after = screenshot(differentPng, "2026-01-01T00:00:01.000Z");
|
||||
const result = verify(comparison(before, after, "identical"), [
|
||||
before,
|
||||
after,
|
||||
]);
|
||||
|
||||
expect(result.claims.items).toMatchObject([
|
||||
{
|
||||
status: "unknown",
|
||||
observed_status: "unchanged",
|
||||
limitations: [
|
||||
"Retained screenshot pixels do not corroborate the recorded comparison.",
|
||||
],
|
||||
},
|
||||
]);
|
||||
expect(
|
||||
verify(comparison(before, after, "identical", 1), [before, after]).status,
|
||||
).toBe("pass");
|
||||
});
|
||||
it.each([
|
||||
["invalid signature", "bm90LWEtcG5n", "Invalid PNG signature"],
|
||||
["preset dictionary", dictionaryPng, "Invalid PNG image data"],
|
||||
])(
|
||||
"preserves other claims when retained PNG bytes cannot be decoded (%s)",
|
||||
(_defect, bytes, reason) => {
|
||||
const before = screenshot(beforePng, "2026-01-01T00:00:00.000Z");
|
||||
const after = screenshot(bytes, "2026-01-01T00:00:01.000Z");
|
||||
const independent = screenshot(samePng, "2026-01-01T00:00:02.000Z");
|
||||
const unreadable = comparison(before, after, "identical");
|
||||
const readable = comparison(before, independent, "identical");
|
||||
const spec = specification(unreadable);
|
||||
const result = verifyReconstruction(
|
||||
{
|
||||
...spec,
|
||||
claims: [
|
||||
...spec.claims,
|
||||
{ ...specification(readable).claims[0], claim_id: "independent" },
|
||||
],
|
||||
},
|
||||
createEvidenceBundle([
|
||||
before,
|
||||
after,
|
||||
independent,
|
||||
unreadable,
|
||||
readable,
|
||||
]),
|
||||
);
|
||||
|
||||
expect(result.claims.items).toMatchObject([
|
||||
{ claim_id: "independent", status: "pass" },
|
||||
{
|
||||
claim_id: "viewport",
|
||||
status: "unknown",
|
||||
limitations: [
|
||||
`Retained screenshot comparison could not be replayed: ${reason}`,
|
||||
],
|
||||
},
|
||||
]);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { compositeKey } from "./unicodeCodePointOrder.js";
|
||||
import { IDENTIFIER_PATTERN } from "./stringPatterns.js";
|
||||
import { digestSchema, prefixedDigestSchema } from "./../domain/digests.js";
|
||||
import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js";
|
||||
@@ -34,14 +35,47 @@ const artifactClaimSchema = z.object({
|
||||
kind: z.literal("structural-artifact"),
|
||||
dimension: z.literal("overall"),
|
||||
});
|
||||
const visualClaimSchema = z.object({
|
||||
...commonClaim,
|
||||
kind: z.literal("visual"),
|
||||
dimension: z.literal("overall"),
|
||||
source_evidence: z
|
||||
.strictObject({
|
||||
before_evidence_id: evidenceIdSchema,
|
||||
after_evidence_id: evidenceIdSchema,
|
||||
})
|
||||
.optional()
|
||||
.describe(
|
||||
"Select the retained screenshot capture for each side. Supply both IDs when a digest matches multiple captures, including byte-identical before and after captures.",
|
||||
),
|
||||
});
|
||||
|
||||
const reconstructionClaimSchema = z.discriminatedUnion("kind", [
|
||||
behavioralClaimSchema,
|
||||
functionClaimSchema,
|
||||
artifactClaimSchema,
|
||||
visualClaimSchema,
|
||||
]);
|
||||
|
||||
/** Finite, typed behavioral and structural specification. */
|
||||
export const reconstructionClaimSelector = (
|
||||
claim: ReconstructionClaim,
|
||||
resolvedPair?: readonly [string, string],
|
||||
): string => {
|
||||
const selected = claim.kind === "visual" ? claim.source_evidence : undefined;
|
||||
const pair =
|
||||
resolvedPair ??
|
||||
(selected === undefined
|
||||
? []
|
||||
: [selected.before_evidence_id, selected.after_evidence_id]);
|
||||
return compositeKey([
|
||||
claim.kind,
|
||||
claim.comparison_evidence_id,
|
||||
claim.dimension,
|
||||
...pair,
|
||||
]);
|
||||
};
|
||||
|
||||
/** Finite, typed behavioral, structural, and visual specification. */
|
||||
export const reconstructionSpecificationSchema = z
|
||||
.object({
|
||||
name: z.string().trim().min(1),
|
||||
@@ -51,9 +85,8 @@ export const reconstructionSpecificationSchema = z
|
||||
const ids = value.claims.map(({ claim_id: id }) => id);
|
||||
if (new Set(ids).size !== ids.length)
|
||||
context.addIssue({ code: "custom", message: "Claim IDs must be unique" });
|
||||
const selectors = value.claims.map(
|
||||
({ kind, comparison_evidence_id: id, dimension }) =>
|
||||
`${kind}:${id}:${dimension}`,
|
||||
const selectors = value.claims.map((claim) =>
|
||||
reconstructionClaimSelector(claim),
|
||||
);
|
||||
if (new Set(selectors).size !== selectors.length)
|
||||
context.addIssue({
|
||||
@@ -69,7 +102,12 @@ export const reconstructionVerificationInputSchema = z.strictObject({
|
||||
|
||||
export const reconstructionClaimResultSchema = z.object({
|
||||
claim_id: claimIdSchema,
|
||||
kind: z.enum(["behavioral", "structural-function", "structural-artifact"]),
|
||||
kind: z.enum([
|
||||
"behavioral",
|
||||
"structural-function",
|
||||
"structural-artifact",
|
||||
"visual",
|
||||
]),
|
||||
dimension: z.string().min(1),
|
||||
status: verificationStatusSchema,
|
||||
observed_status: z.enum([
|
||||
@@ -89,6 +127,17 @@ export const reconstructionClaimResultSchema = z.object({
|
||||
limitations: z.array(z.string()),
|
||||
});
|
||||
|
||||
export const countReconstructionClaimKinds = (
|
||||
claims: readonly Pick<ReconstructionClaimResult, "kind">[],
|
||||
) => ({
|
||||
behavioral: claims.filter(({ kind }) => kind === "behavioral").length,
|
||||
structural: claims.filter(
|
||||
({ kind }) =>
|
||||
kind === "structural-function" || kind === "structural-artifact",
|
||||
).length,
|
||||
visual: claims.filter(({ kind }) => kind === "visual").length,
|
||||
});
|
||||
|
||||
/** Evidence-cited result over every declared claim. */
|
||||
export const reconstructionVerificationResultSchema = z
|
||||
.object({
|
||||
@@ -101,6 +150,7 @@ export const reconstructionVerificationResultSchema = z
|
||||
unknown: z.int().min(0),
|
||||
behavioral: z.int().min(0),
|
||||
structural: z.int().min(0),
|
||||
visual: z.int().min(0).default(0),
|
||||
}),
|
||||
claims: z.object({
|
||||
items: z.array(reconstructionClaimResultSchema),
|
||||
@@ -119,6 +169,7 @@ export const reconstructionVerificationResultSchema = z
|
||||
.superRefine((result, context) => {
|
||||
const { summary } = result;
|
||||
const claims = result.claims.items;
|
||||
const kinds = countReconstructionClaimKinds(claims);
|
||||
const expectedStatus =
|
||||
summary.failed > 0 ? "fail" : summary.unknown > 0 ? "unknown" : "pass";
|
||||
const issues = [
|
||||
@@ -133,10 +184,9 @@ export const reconstructionVerificationResultSchema = z
|
||||
["summary"],
|
||||
],
|
||||
[
|
||||
summary.behavioral !==
|
||||
claims.filter(({ kind }) => kind === "behavioral").length ||
|
||||
summary.structural !==
|
||||
claims.filter(({ kind }) => kind !== "behavioral").length,
|
||||
summary.behavioral !== kinds.behavioral ||
|
||||
summary.structural !== kinds.structural ||
|
||||
summary.visual !== kinds.visual,
|
||||
"Claim kind counts must summarize retained claims",
|
||||
["summary"],
|
||||
],
|
||||
@@ -146,7 +196,8 @@ export const reconstructionVerificationResultSchema = z
|
||||
["summary", "total"],
|
||||
],
|
||||
[
|
||||
summary.total !== summary.behavioral + summary.structural,
|
||||
summary.total !==
|
||||
summary.behavioral + summary.structural + summary.visual,
|
||||
"Claim kind counts must equal the summary total",
|
||||
["summary", "total"],
|
||||
],
|
||||
|
||||
Reference in new issue
Block a user