feat(evidence): report comparison constraints, remaining issue counts and observed workflows as fields (#11078)

* feat(evidence): report comparison-source constraints, remaining issue counts and observed workflows as fields

* fix(evidence): locate snapshot bundle schema issues within the snapshot document

* test(evidence): check each comparison-input mismatch and terminal omission once (#11087)
This commit is contained in:
morluto authored and GitHub committed 2026-11-15 20:20:40 +08:00
1 parent de7d4dec83
commit 7dd24dc08c
17 files changed
+222 -91

No files matched your search

@@ -14,6 +14,7 @@ import {
} from "../../domain/evidenceErrors.js";
import { err, ok, type Result } from "../../domain/result.js";
import { describeValidationFailure } from "../../domain/validationFailure.js";
import { validationIssueOptions } from "../../domain/evidenceValidation.js";
import { isUnsupportedEvidenceFormat } from "../../domain/evidenceFormat.js";
import type { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
import {
@@ -87,7 +88,10 @@ export const readAnalysisSnapshot = async (
if (signal?.aborted === true)
return err(cancelledBy(READ_OPERATION, signal));
return err(
new EvidenceIntegrityError(snapshotValidationMessage(cause), { cause }),
new EvidenceIntegrityError(snapshotValidationMessage(cause), {
cause,
...validationIssueOptions(cause, []),
}),
);
}
};
@@ -156,7 +160,10 @@ export const writeAnalysisSnapshot = async (
encoded = serializeAnalysisSnapshot(snapshot);
} catch (cause: unknown) {
return err(
new EvidenceIntegrityError(snapshotValidationMessage(cause), { cause }),
new EvidenceIntegrityError(snapshotValidationMessage(cause), {
cause,
...validationIssueOptions(cause, []),
}),
);
}
return writeTextFile(encoded, path, overwrite, resources);
+17 -12
View File
@@ -14,7 +14,7 @@ import type { AnalysisErrorProjection } from "../../domain/analysisErrorProjecti
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import {
describeValidationFailure,
firstValidationIssuePath,
firstValidationIssue,
} from "../../domain/validationFailure.js";
import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js";
import {
@@ -35,6 +35,7 @@ import { ProcessCaptureError } from "../../process/capture/ProcessCaptureError.j
import {
createProcessCaptureEvidence,
createProcessScenarioPlanEvidence,
wrongWorkflowIssueFacts,
type ProcessCaptureEvidenceFailure,
} from "./ProcessEvidence.js";
import { prepareProcessScenarioPlan } from "./ProcessScenarioPreparation.js";
@@ -282,6 +283,7 @@ const projectCaptureFileFailure = (
"Capture evidence is not from the current process-capture workflow. Create new capture evidence, then try again.",
[side],
"invalid_value",
wrongWorkflowIssueFacts(failure),
);
case "unsupported-provider":
return unsupportedProcessCaptureProvider(side, failure);
@@ -305,15 +307,23 @@ const parseTraceSpecification = (input: unknown) => {
};
const invalidCaptureEvidence = (cause: unknown, side: "left" | "right") => {
const issuePath = firstValidationIssuePath(cause);
const issue = firstValidationIssue(cause);
return processInputFailure(
"compare_process_captures",
`Capture evidence is malformed (${describeValidationFailure(cause)}). Create new capture evidence, then try again.`,
[side],
"invalid_value",
issuePath === undefined
issue === undefined
? undefined
: { issue_path: ["normalized_result", ...issuePath] },
: {
observed: {
issue_path: ["normalized_result", ...issue.path],
...(issue.constraint === undefined
? {}
: { constraint: issue.constraint }),
remaining_issue_count: issue.remainingIssueCount,
},
},
);
};
@@ -344,7 +354,7 @@ const readJson = async (
["input_file"],
"invalid_value",
// Read and parse failures share one issue reason; name which one occurred.
{ file_reason: loaded.error.reason },
{ observed: { file_reason: loaded.error.reason } },
);
};
@@ -353,15 +363,10 @@ const processInputFailure = (
userMessage: string,
path: readonly (string | number)[],
reason: "invalid_value" | "missing_argument",
observed?: AnalysisInputIssue["observed"],
facts?: Pick<AnalysisInputIssue, "expected" | "observed">,
): AnalysisInputError =>
new AnalysisInputError(operation, { userMessage }, [
{
path,
reason,
message: userMessage,
...(observed === undefined ? {} : { observed }),
},
{ path, reason, message: userMessage, ...facts },
]);
const cliAnalysisError = (error: AnalysisError): ProcessCliErrorOutput => ({
@@ -1,11 +1,9 @@
import { processComparisonInputSchema } from "../../contracts/processComparisonContracts.js";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import {
describeValidationFailure,
firstValidationIssuePath,
} from "../../domain/validationFailure.js";
import { describeValidationFailure } from "../../domain/validationFailure.js";
import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
import { validationIssueOptions } from "../../domain/evidenceValidation.js";
import {
PROCESS_COMPARISON_DIMENSIONS,
type ProcessCaptureComparison,
@@ -19,6 +17,7 @@ import type { RecordUnknownInput } from "../../domain/residualUnknown.js";
import { err, ok, type Result } from "../../domain/result.js";
import {
parseProcessCaptureEvidence,
wrongWorkflowIssueFacts,
type ProcessCaptureEvidenceFailure,
} from "./ProcessEvidence.js";
import {
@@ -114,7 +113,7 @@ export const readProcessComparisonSource = (
new EvidenceIntegrityError(`${side} Evidence failed validation`, {
cause: failure.cause,
side,
...issuePathFact(failure.cause),
...validationIssueOptions(failure.cause, []),
userMessage: `The ${side} Evidence failed validation (${describeValidationFailure(failure.cause)}). Supply unmodified capture Evidence from capture_process_scenario.`,
}),
);
@@ -125,6 +124,7 @@ export const readProcessComparisonSource = (
path: [side],
reason: "invalid_value",
message: `Expected process capture Evidence (operation ${failure.expected.operation}, predicate ${failure.expected.predicate}); the ${side} Evidence has operation ${failure.actual.operation} and predicate ${failure.actual.predicate}.`,
...wrongWorkflowIssueFacts(failure),
},
]),
);
@@ -137,7 +137,7 @@ export const readProcessComparisonSource = (
{
cause: failure.cause,
side,
...issuePathFact(failure.cause, ["normalized_result"]),
...validationIssueOptions(failure.cause, ["normalized_result"]),
userMessage: `The ${side} Evidence has an invalid process capture result (${describeValidationFailure(failure.cause)}). Recreate the capture with capture_process_scenario.`,
},
),
@@ -145,17 +145,6 @@ export const readProcessComparisonSource = (
}
};
/** Issue paths are relative to the Evidence record, as for `readEvidence`. */
const issuePathFact = (
cause: unknown,
within: readonly (string | number)[] = [],
) => {
const issuePath = firstValidationIssuePath(cause);
return issuePath === undefined
? {}
: { issuePath: [...within, ...issuePath] };
};
/** Report the same provider admission failure at both public adapters. */
export const unsupportedProcessCaptureProvider = (
side: "left" | "right",
@@ -50,6 +50,20 @@ export type ProcessCaptureEvidenceFailure =
}
| { readonly kind: "invalid-capture"; readonly cause: z.ZodError };
/** The workflow a capture input must come from and the one it came from, as input-issue facts. */
export const wrongWorkflowIssueFacts = (
failure: Extract<ProcessCaptureEvidenceFailure, { kind: "wrong-workflow" }>,
) => ({
expected: {
operation: failure.expected.operation,
predicate_type: failure.expected.predicate,
},
observed: {
operation: failure.actual.operation,
predicate_type: failure.actual.predicate,
},
});
/** Authenticate capture Evidence before admitting its workflow, provider and payload. */
export const parseProcessCaptureEvidence = (
input: unknown,
+6
View File
@@ -804,6 +804,12 @@ const requestErrorDetails = (
...(error.issuePath === undefined
? {}
: { issue_path: [...error.issuePath] }),
...(error.constraint === undefined
? {}
: { constraint: error.constraint }),
...(error.remainingIssueCount === undefined
? {}
: { remaining_issue_count: error.remainingIssueCount }),
};
return undefined;
};
+26 -2
View File
@@ -277,6 +277,27 @@ const snapshotFileHeaderSchema = z.object({
evidence_bundle: z.unknown(),
});
/**
* The bundle is validated on its own, so locate its schema issues within the
* snapshot document, as entry issues are.
*/
const parseSnapshotBundle = <Bundle>(
value: unknown,
parse: (value: unknown) => Bundle,
): Bundle => {
try {
return parse(value);
} catch (cause: unknown) {
if (!(cause instanceof z.ZodError)) throw cause;
throw new z.ZodError(
cause.issues.map((issue) => ({
...issue,
path: ["evidence_bundle", ...issue.path],
})),
);
}
};
/** Reconstruct file entries only from fully authenticated matching Evidence. */
export const parsePreparedAnalysisSnapshot = (
input: unknown,
@@ -284,7 +305,10 @@ export const parsePreparedAnalysisSnapshot = (
workflowEntries: readonly PreparedSnapshotEntry[],
): AnalysisSnapshot => {
const header = snapshotFileHeaderSchema.parse(input);
const bundle = parseImmutableEvidenceBundle(header.evidence_bundle);
const bundle = parseSnapshotBundle(
header.evidence_bundle,
parseImmutableEvidenceBundle,
);
const providerRows = entries.map((handle) => {
const row = preparedEntries.get(handle);
if (row?.kind !== "provider")
@@ -712,7 +736,7 @@ const authenticateSnapshot = (
const parsed = analysisSnapshotSchema.parse(input);
const restored = restoreSnapshot(
parsed,
parseEvidenceBundle(parsed.evidence_bundle),
parseSnapshotBundle(parsed.evidence_bundle, parseEvidenceBundle),
parsed.entries.map((entry) => ({
kind: "provider",
entry,
+8
View File
@@ -24,6 +24,10 @@ export class EvidenceIntegrityError extends AnalysisError {
readonly side: "left" | "right" | undefined;
/** The first failing location inside that Evidence or its result, when schema-located. */
readonly issuePath: readonly (string | number)[] | undefined;
/** The constraint a schema refinement named on that first issue. */
readonly constraint: string | undefined;
/** Schema issues after the first, which the message only summarizes. */
readonly remainingIssueCount: number | undefined;
constructor(
message: string,
@@ -31,12 +35,16 @@ export class EvidenceIntegrityError extends AnalysisError {
readonly userMessage?: string;
readonly side?: "left" | "right";
readonly issuePath?: readonly (string | number)[];
readonly constraint?: string;
readonly remainingIssueCount?: number;
},
) {
super(message, options);
this.userMessage = options?.userMessage;
this.side = options?.side;
this.issuePath = options?.issuePath;
this.constraint = options?.constraint;
this.remainingIssueCount = options?.remainingIssueCount;
}
}
+23 -6
View File
@@ -4,7 +4,7 @@ import { parseEvidence, immutableEvidence, type Evidence } from "./evidence.js";
import { EvidenceIntegrityError } from "./evidenceErrors.js";
import {
describeValidationFailure,
firstValidationIssuePath,
firstValidationIssue,
} from "./validationFailure.js";
import { err, ok, type Result } from "./result.js";
@@ -74,19 +74,36 @@ export const readEvidencePayload = <Output>(
: err(integrityFailure(parsed.error, location));
};
/** The issue path is reported only when its position in the record is known. */
/**
* Schema issue facts for an integrity failure. The issue path is reported
* only when its position in the record (`within`) is known.
*/
export const validationIssueOptions = (
cause: unknown,
within?: readonly (string | number)[],
): {
readonly issuePath?: readonly (string | number)[];
readonly constraint?: string;
readonly remainingIssueCount?: number;
} => {
const issue = firstValidationIssue(cause);
if (issue === undefined) return {};
return {
...(within === undefined ? {} : { issuePath: [...within, ...issue.path] }),
...(issue.constraint === undefined ? {} : { constraint: issue.constraint }),
remainingIssueCount: issue.remainingIssueCount,
};
};
const integrityFailure = (
cause: z.ZodError,
location: EvidenceSourceLocation | undefined,
): EvidenceIntegrityError => {
const message = `Evidence validation failed ${describeValidationFailure(cause)}`;
const issuePath = firstValidationIssuePath(cause);
return new EvidenceIntegrityError(message, {
cause,
userMessage: message,
...(location?.side === undefined ? {} : { side: location.side }),
...(location?.within === undefined || issuePath === undefined
? {}
: { issuePath: [...location.within, ...issuePath] }),
...validationIssueOptions(cause, location?.within),
});
};
+1
View File
@@ -174,6 +174,7 @@ export const functionComparisonResultSchema = z
path: ["status"],
message:
"Function comparison status contradicts its dimensions or function match.",
params: { constraint: "status-matches-dimensions" },
});
});
+13 -4
View File
@@ -68,14 +68,17 @@ export const parseFunctionEvidence = (
const parsed = readEvidence(input, side);
if (!parsed.ok) return parsed;
const evidence = parsed.value;
const invalidSelection = (message: string, expected?: JsonValue) =>
const invalidSelection = (
message: string,
facts?: { readonly expected: JsonValue; readonly observed: JsonValue },
) =>
err(
new AnalysisInputError(selection.operation, { userMessage: message }, [
{
path: selection.path,
reason: "invalid_value",
message,
...(expected === undefined ? {} : { expected }),
...facts,
},
]),
);
@@ -93,8 +96,14 @@ export const parseFunctionEvidence = (
return invalidSelection(
`Function analysis requires analyze_function Evidence with the rea.analysis predicate; the ${subject} has operation ${evidence.operation} and predicate ${evidence.predicate_type}.`,
{
operation: "analyze_function",
predicate_type: ANALYSIS_EVIDENCE_PREDICATE,
expected: {
operation: "analyze_function",
predicate_type: ANALYSIS_EVIDENCE_PREDICATE,
},
observed: {
operation: evidence.operation,
predicate_type: evidence.predicate_type,
},
},
);
}
+1
View File
@@ -101,6 +101,7 @@ export const processCaptureComparisonSchema = z
code: "custom",
message: "Process comparison status contradicts its dimensions",
path: ["status"],
params: { constraint: "status-matches-dimensions" },
});
});
export type ProcessCaptureComparison = z.infer<
+27 -7
View File
@@ -1,14 +1,34 @@
import { z } from "zod";
/** The location `describeValidationFailure` names, when the cause is schema-located. */
export const firstValidationIssuePath = (
/**
* What `describeValidationFailure` states in prose, as facts a caller can act
* on without reading the description.
*/
export interface ValidationIssueFacts {
/** The first issue's location. */
readonly path: readonly (string | number)[];
/** The constraint a custom refinement named on the first issue. */
readonly constraint: string | undefined;
/** Further issues the description summarizes as "(N more issues)". */
readonly remainingIssueCount: number;
}
/** The first schema issue's facts, when the cause is a schema failure. */
export const firstValidationIssue = (
cause: unknown,
): readonly (string | number)[] | undefined => {
): ValidationIssueFacts | undefined => {
if (!(cause instanceof z.ZodError)) return undefined;
const first = cause.issues[0];
return first?.path.map((key) =>
typeof key === "number" ? key : String(key),
);
const [first, ...rest] = cause.issues;
if (first === undefined) return undefined;
const constraint: unknown =
first.code === "custom" ? first.params?.["constraint"] : undefined;
return {
path: first.path.map((key) =>
typeof key === "number" ? key : String(key),
),
constraint: typeof constraint === "string" ? constraint : undefined,
remainingIssueCount: rest.length,
};
};
/** Locate the first schema issue, or keep a thrown constraint message. */
@@ -55,14 +55,17 @@ cliTest(
code: "evidence_integrity_mismatch",
category: "integrity_mismatch",
retryable: false,
details: { reason: expect.stringContaining(`at ${field}:`) },
details: {
reason: expect.stringContaining(`at ${field}:`),
issue_path: field
.split(".")
.map((key) => (/^\d+$/u.test(key) ? Number(key) : key)),
},
});
expect(error).not.toHaveProperty("cause");
expect(error).not.toHaveProperty("stack");
if (field === "target")
expect(error.details?.reason).toMatch(
/ at target: .+ \(2 more issues\)$/,
);
expect(error.details?.remaining_issue_count).toBe(2);
}
// MCP adds open-session facts; the snapshot reason must match exactly.
expect(mcpError.details).toMatchObject(cliError.details ?? {});
@@ -58,12 +58,15 @@ describe.each([
code: "evidence_integrity_mismatch",
category: "integrity_mismatch",
retryable: false,
details: { reason: expect.stringContaining(`at ${field}:`) },
details: {
reason: expect.stringContaining(`at ${field}:`),
issue_path: field
.split(".")
.map((key) => (/^\d+$/u.test(key) ? Number(key) : key)),
},
});
if (field === "target")
expect(projected.details?.reason).toMatch(
/ at target: .+ \(2 more issues\)$/,
);
expect(projected.details?.remaining_issue_count).toBe(2);
expect(projected).not.toHaveProperty("cause");
expect(projected).not.toHaveProperty("stack");
expect(await readFile(path, "utf8")).toBe(encoded);
@@ -144,6 +147,39 @@ it("refuses a snapshot holding an earlier-format bundle with the recreate guidan
});
});
it("locates a bundle schema issue within the snapshot when reading and writing", async () => {
const root = await createTestTempDirectory("rea-snapshot-bundle-issue-");
const { providers: _providers, ...bundle } = createEvidenceBundle([]);
const snapshot = {
target: snapshotTarget(ANALYSIS_SNAPSHOT_TARGET),
binding: snapshotBinding(ANALYSIS_SNAPSHOT_PROFILE),
entries: [],
workflow_entries: [],
evidence_bundle: bundle,
};
const path = join(root, "snapshot.json");
await writeFile(path, JSON.stringify(snapshot));
const read = await readAnalysisSnapshot(path);
const written = await withArtifactResourceScope((resources) =>
writeAnalysisSnapshot(
snapshot as never,
join(root, "written.json"),
false,
resources,
),
);
for (const result of [read, written]) {
if (result.ok) throw new Error("Expected invalid bundle refusal");
expect(projectAnalysisError(result.error)).toMatchObject({
code: "evidence_integrity_mismatch",
details: {
reason: expect.stringContaining("at evidence_bundle.providers:"),
issue_path: ["evidence_bundle", "providers"],
},
});
}
});
it("retains serialization schema diagnostics without creating a snapshot file", async () => {
const root = await createTestTempDirectory("rea-snapshot-write-diagnostic-");
const path = join(root, "snapshot.json");
@@ -314,10 +314,9 @@ describe("function comparison MCP integration", () => {
});
describe("function comparison input-kind diagnostics", () => {
// Each mismatch is checked once; alternating sides covers both labels.
it.each([
{ side: "left", mismatch: "operation" },
{ side: "right", mismatch: "operation" },
{ side: "left", mismatch: "predicate" },
{ side: "right", mismatch: "predicate" },
] as const)(
"rejects inline $side Evidence with the wrong $mismatch as an input-selection error",
@@ -350,11 +349,14 @@ describe("function comparison input-kind diagnostics", () => {
{
path: [side],
reason: "invalid_value",
message: `Function analysis requires analyze_function Evidence with the rea.analysis predicate; the ${side} Evidence has operation ${wrong.operation} and predicate ${wrong.predicate_type}.`,
expected: {
operation: "analyze_function",
predicate_type: "rea.analysis",
},
observed: {
operation: wrong.operation,
predicate_type: wrong.predicate_type,
},
},
],
},
@@ -390,9 +392,7 @@ describe("function comparison input-kind diagnostics", () => {
issues: [
{
path: ["left"],
message: expect.stringContaining(
"the left Evidence has operation inventory_artifact",
),
observed: { operation: "inventory_artifact" },
},
],
},
@@ -531,11 +531,7 @@ it.each([
const rejected = await verification(inconsistent.evidence_id);
expect(parseMcpToolError(rejected).error).toMatchObject({
code: "evidence_integrity_mismatch",
details: {
reason: expect.stringContaining(
"Function comparison status contradicts its dimensions or function match",
),
},
details: { constraint: "status-matches-dimensions" },
});
expect(harness.session.evidenceById(inconsistent.evidence_id)).toEqual(
inconsistent,
@@ -110,14 +110,14 @@ const compareViaCli = async (
);
};
it.each(
(["left", "right"] as const).flatMap((missingWorker) =>
(["left", "right", "both", "neither"] as const).map((omittedRendered) => ({
missingWorker,
omittedRendered,
})),
),
)(
// Each omission is checked once, alternating which side lacks the worker: a
// one-sided omission on either side, both sides, and neither.
it.each([
{ missingWorker: "left", omittedRendered: "left" },
{ missingWorker: "right", omittedRendered: "right" },
{ missingWorker: "right", omittedRendered: "both" },
{ missingWorker: "left", omittedRendered: "neither" },
] as const)(
"preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output",
async ({ missingWorker, omittedRendered }) => {
const observation = (side: "left" | "right") => {
@@ -468,9 +468,11 @@ it("names the side and constraint when Evidence is not a usable capture", async
{
path: ["left"],
reason: "invalid_value",
message: expect.stringContaining(
"the left Evidence has operation other",
),
expected: {
operation: "capture_process_scenario",
predicate_type: "rea.process-capture",
},
observed: { operation: "other" },
},
],
},
@@ -552,7 +554,7 @@ it("names the side and constraint when Evidence is not a usable capture", async
issues: [
expect.objectContaining({
path: ["right"],
observed: { issue_path: issuePath },
observed: expect.objectContaining({ issue_path: issuePath }),
}),
],
},
@@ -586,9 +588,6 @@ it("keeps structural capture issues alongside semantic ones through MCP and the
},
},
});
// The count of remaining issues is only in the description.
const remaining = "(1 more issues)";
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left: capture, right: mixed },
@@ -596,23 +595,23 @@ it("keeps structural capture issues alongside semantic ones through MCP and the
expect(parseMcpToolError(response)).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
message: expect.stringContaining(remaining),
details: {
side: "right",
issue_path: ["normalized_result", "manifest", "completed_at"],
remaining_issue_count: 1,
},
},
});
expect(await compareViaCli(capture, mixed)).toMatchObject({
code: "invalid_request",
category: "invalid_input",
message: expect.stringContaining(remaining),
details: {
issues: [
{
path: ["right"],
observed: {
issue_path: ["normalized_result", "manifest", "completed_at"],
remaining_issue_count: 1,
},
},
],
@@ -101,11 +101,7 @@ it("rejects an inconsistent imported process comparison while retaining its Evid
const rejected = await verify(inconsistent.evidence_id, selected);
expect(parseMcpToolError(rejected).error).toMatchObject({
code: "evidence_integrity_mismatch",
details: {
reason: expect.stringContaining(
"Process comparison status contradicts its dimensions",
),
},
details: { constraint: "status-matches-dimensions" },
});
const failed = await verify(consistent.evidence_id, selected);
expect(failed.isError).not.toBe(true);