mirror of
https://github.com/morluto/rea.git
synced 2026-11-15 19:23:24 +00:00
feat(evidence): report comparison constraints, remaining issue counts and observed workflows as fields (#11078)
* feat(evidence): report comparison-source constraints, remaining issue counts and observed workflows as fields * fix(evidence): locate snapshot bundle schema issues within the snapshot document * test(evidence): check each comparison-input mismatch and terminal omission once (#11087)
This commit is contained in:
1 parent
de7d4dec83
commit
7dd24dc08c
17 files changed
+222
-91
No files matched your search
@@ -14,6 +14,7 @@ import {
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { describeValidationFailure } from "../../domain/validationFailure.js";
|
||||
import { validationIssueOptions } from "../../domain/evidenceValidation.js";
|
||||
import { isUnsupportedEvidenceFormat } from "../../domain/evidenceFormat.js";
|
||||
import type { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
|
||||
import {
|
||||
@@ -87,7 +88,10 @@ export const readAnalysisSnapshot = async (
|
||||
if (signal?.aborted === true)
|
||||
return err(cancelledBy(READ_OPERATION, signal));
|
||||
return err(
|
||||
new EvidenceIntegrityError(snapshotValidationMessage(cause), { cause }),
|
||||
new EvidenceIntegrityError(snapshotValidationMessage(cause), {
|
||||
cause,
|
||||
...validationIssueOptions(cause, []),
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -156,7 +160,10 @@ export const writeAnalysisSnapshot = async (
|
||||
encoded = serializeAnalysisSnapshot(snapshot);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError(snapshotValidationMessage(cause), { cause }),
|
||||
new EvidenceIntegrityError(snapshotValidationMessage(cause), {
|
||||
cause,
|
||||
...validationIssueOptions(cause, []),
|
||||
}),
|
||||
);
|
||||
}
|
||||
return writeTextFile(encoded, path, overwrite, resources);
|
||||
|
||||
@@ -14,7 +14,7 @@ import type { AnalysisErrorProjection } from "../../domain/analysisErrorProjecti
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
import {
|
||||
describeValidationFailure,
|
||||
firstValidationIssuePath,
|
||||
firstValidationIssue,
|
||||
} from "../../domain/validationFailure.js";
|
||||
import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js";
|
||||
import {
|
||||
@@ -35,6 +35,7 @@ import { ProcessCaptureError } from "../../process/capture/ProcessCaptureError.j
|
||||
import {
|
||||
createProcessCaptureEvidence,
|
||||
createProcessScenarioPlanEvidence,
|
||||
wrongWorkflowIssueFacts,
|
||||
type ProcessCaptureEvidenceFailure,
|
||||
} from "./ProcessEvidence.js";
|
||||
import { prepareProcessScenarioPlan } from "./ProcessScenarioPreparation.js";
|
||||
@@ -282,6 +283,7 @@ const projectCaptureFileFailure = (
|
||||
"Capture evidence is not from the current process-capture workflow. Create new capture evidence, then try again.",
|
||||
[side],
|
||||
"invalid_value",
|
||||
wrongWorkflowIssueFacts(failure),
|
||||
);
|
||||
case "unsupported-provider":
|
||||
return unsupportedProcessCaptureProvider(side, failure);
|
||||
@@ -305,15 +307,23 @@ const parseTraceSpecification = (input: unknown) => {
|
||||
};
|
||||
|
||||
const invalidCaptureEvidence = (cause: unknown, side: "left" | "right") => {
|
||||
const issuePath = firstValidationIssuePath(cause);
|
||||
const issue = firstValidationIssue(cause);
|
||||
return processInputFailure(
|
||||
"compare_process_captures",
|
||||
`Capture evidence is malformed (${describeValidationFailure(cause)}). Create new capture evidence, then try again.`,
|
||||
[side],
|
||||
"invalid_value",
|
||||
issuePath === undefined
|
||||
issue === undefined
|
||||
? undefined
|
||||
: { issue_path: ["normalized_result", ...issuePath] },
|
||||
: {
|
||||
observed: {
|
||||
issue_path: ["normalized_result", ...issue.path],
|
||||
...(issue.constraint === undefined
|
||||
? {}
|
||||
: { constraint: issue.constraint }),
|
||||
remaining_issue_count: issue.remainingIssueCount,
|
||||
},
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
@@ -344,7 +354,7 @@ const readJson = async (
|
||||
["input_file"],
|
||||
"invalid_value",
|
||||
// Read and parse failures share one issue reason; name which one occurred.
|
||||
{ file_reason: loaded.error.reason },
|
||||
{ observed: { file_reason: loaded.error.reason } },
|
||||
);
|
||||
};
|
||||
|
||||
@@ -353,15 +363,10 @@ const processInputFailure = (
|
||||
userMessage: string,
|
||||
path: readonly (string | number)[],
|
||||
reason: "invalid_value" | "missing_argument",
|
||||
observed?: AnalysisInputIssue["observed"],
|
||||
facts?: Pick<AnalysisInputIssue, "expected" | "observed">,
|
||||
): AnalysisInputError =>
|
||||
new AnalysisInputError(operation, { userMessage }, [
|
||||
{
|
||||
path,
|
||||
reason,
|
||||
message: userMessage,
|
||||
...(observed === undefined ? {} : { observed }),
|
||||
},
|
||||
{ path, reason, message: userMessage, ...facts },
|
||||
]);
|
||||
|
||||
const cliAnalysisError = (error: AnalysisError): ProcessCliErrorOutput => ({
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import { processComparisonInputSchema } from "../../contracts/processComparisonContracts.js";
|
||||
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
import {
|
||||
describeValidationFailure,
|
||||
firstValidationIssuePath,
|
||||
} from "../../domain/validationFailure.js";
|
||||
import { describeValidationFailure } from "../../domain/validationFailure.js";
|
||||
import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import { validationIssueOptions } from "../../domain/evidenceValidation.js";
|
||||
import {
|
||||
PROCESS_COMPARISON_DIMENSIONS,
|
||||
type ProcessCaptureComparison,
|
||||
@@ -19,6 +17,7 @@ import type { RecordUnknownInput } from "../../domain/residualUnknown.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import {
|
||||
parseProcessCaptureEvidence,
|
||||
wrongWorkflowIssueFacts,
|
||||
type ProcessCaptureEvidenceFailure,
|
||||
} from "./ProcessEvidence.js";
|
||||
import {
|
||||
@@ -114,7 +113,7 @@ export const readProcessComparisonSource = (
|
||||
new EvidenceIntegrityError(`${side} Evidence failed validation`, {
|
||||
cause: failure.cause,
|
||||
side,
|
||||
...issuePathFact(failure.cause),
|
||||
...validationIssueOptions(failure.cause, []),
|
||||
userMessage: `The ${side} Evidence failed validation (${describeValidationFailure(failure.cause)}). Supply unmodified capture Evidence from capture_process_scenario.`,
|
||||
}),
|
||||
);
|
||||
@@ -125,6 +124,7 @@ export const readProcessComparisonSource = (
|
||||
path: [side],
|
||||
reason: "invalid_value",
|
||||
message: `Expected process capture Evidence (operation ${failure.expected.operation}, predicate ${failure.expected.predicate}); the ${side} Evidence has operation ${failure.actual.operation} and predicate ${failure.actual.predicate}.`,
|
||||
...wrongWorkflowIssueFacts(failure),
|
||||
},
|
||||
]),
|
||||
);
|
||||
@@ -137,7 +137,7 @@ export const readProcessComparisonSource = (
|
||||
{
|
||||
cause: failure.cause,
|
||||
side,
|
||||
...issuePathFact(failure.cause, ["normalized_result"]),
|
||||
...validationIssueOptions(failure.cause, ["normalized_result"]),
|
||||
userMessage: `The ${side} Evidence has an invalid process capture result (${describeValidationFailure(failure.cause)}). Recreate the capture with capture_process_scenario.`,
|
||||
},
|
||||
),
|
||||
@@ -145,17 +145,6 @@ export const readProcessComparisonSource = (
|
||||
}
|
||||
};
|
||||
|
||||
/** Issue paths are relative to the Evidence record, as for `readEvidence`. */
|
||||
const issuePathFact = (
|
||||
cause: unknown,
|
||||
within: readonly (string | number)[] = [],
|
||||
) => {
|
||||
const issuePath = firstValidationIssuePath(cause);
|
||||
return issuePath === undefined
|
||||
? {}
|
||||
: { issuePath: [...within, ...issuePath] };
|
||||
};
|
||||
|
||||
/** Report the same provider admission failure at both public adapters. */
|
||||
export const unsupportedProcessCaptureProvider = (
|
||||
side: "left" | "right",
|
||||
|
||||
@@ -50,6 +50,20 @@ export type ProcessCaptureEvidenceFailure =
|
||||
}
|
||||
| { readonly kind: "invalid-capture"; readonly cause: z.ZodError };
|
||||
|
||||
/** The workflow a capture input must come from and the one it came from, as input-issue facts. */
|
||||
export const wrongWorkflowIssueFacts = (
|
||||
failure: Extract<ProcessCaptureEvidenceFailure, { kind: "wrong-workflow" }>,
|
||||
) => ({
|
||||
expected: {
|
||||
operation: failure.expected.operation,
|
||||
predicate_type: failure.expected.predicate,
|
||||
},
|
||||
observed: {
|
||||
operation: failure.actual.operation,
|
||||
predicate_type: failure.actual.predicate,
|
||||
},
|
||||
});
|
||||
|
||||
/** Authenticate capture Evidence before admitting its workflow, provider and payload. */
|
||||
export const parseProcessCaptureEvidence = (
|
||||
input: unknown,
|
||||
|
||||
@@ -804,6 +804,12 @@ const requestErrorDetails = (
|
||||
...(error.issuePath === undefined
|
||||
? {}
|
||||
: { issue_path: [...error.issuePath] }),
|
||||
...(error.constraint === undefined
|
||||
? {}
|
||||
: { constraint: error.constraint }),
|
||||
...(error.remainingIssueCount === undefined
|
||||
? {}
|
||||
: { remaining_issue_count: error.remainingIssueCount }),
|
||||
};
|
||||
return undefined;
|
||||
};
|
||||
|
||||
@@ -277,6 +277,27 @@ const snapshotFileHeaderSchema = z.object({
|
||||
evidence_bundle: z.unknown(),
|
||||
});
|
||||
|
||||
/**
|
||||
* The bundle is validated on its own, so locate its schema issues within the
|
||||
* snapshot document, as entry issues are.
|
||||
*/
|
||||
const parseSnapshotBundle = <Bundle>(
|
||||
value: unknown,
|
||||
parse: (value: unknown) => Bundle,
|
||||
): Bundle => {
|
||||
try {
|
||||
return parse(value);
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof z.ZodError)) throw cause;
|
||||
throw new z.ZodError(
|
||||
cause.issues.map((issue) => ({
|
||||
...issue,
|
||||
path: ["evidence_bundle", ...issue.path],
|
||||
})),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
/** Reconstruct file entries only from fully authenticated matching Evidence. */
|
||||
export const parsePreparedAnalysisSnapshot = (
|
||||
input: unknown,
|
||||
@@ -284,7 +305,10 @@ export const parsePreparedAnalysisSnapshot = (
|
||||
workflowEntries: readonly PreparedSnapshotEntry[],
|
||||
): AnalysisSnapshot => {
|
||||
const header = snapshotFileHeaderSchema.parse(input);
|
||||
const bundle = parseImmutableEvidenceBundle(header.evidence_bundle);
|
||||
const bundle = parseSnapshotBundle(
|
||||
header.evidence_bundle,
|
||||
parseImmutableEvidenceBundle,
|
||||
);
|
||||
const providerRows = entries.map((handle) => {
|
||||
const row = preparedEntries.get(handle);
|
||||
if (row?.kind !== "provider")
|
||||
@@ -712,7 +736,7 @@ const authenticateSnapshot = (
|
||||
const parsed = analysisSnapshotSchema.parse(input);
|
||||
const restored = restoreSnapshot(
|
||||
parsed,
|
||||
parseEvidenceBundle(parsed.evidence_bundle),
|
||||
parseSnapshotBundle(parsed.evidence_bundle, parseEvidenceBundle),
|
||||
parsed.entries.map((entry) => ({
|
||||
kind: "provider",
|
||||
entry,
|
||||
|
||||
@@ -24,6 +24,10 @@ export class EvidenceIntegrityError extends AnalysisError {
|
||||
readonly side: "left" | "right" | undefined;
|
||||
/** The first failing location inside that Evidence or its result, when schema-located. */
|
||||
readonly issuePath: readonly (string | number)[] | undefined;
|
||||
/** The constraint a schema refinement named on that first issue. */
|
||||
readonly constraint: string | undefined;
|
||||
/** Schema issues after the first, which the message only summarizes. */
|
||||
readonly remainingIssueCount: number | undefined;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
@@ -31,12 +35,16 @@ export class EvidenceIntegrityError extends AnalysisError {
|
||||
readonly userMessage?: string;
|
||||
readonly side?: "left" | "right";
|
||||
readonly issuePath?: readonly (string | number)[];
|
||||
readonly constraint?: string;
|
||||
readonly remainingIssueCount?: number;
|
||||
},
|
||||
) {
|
||||
super(message, options);
|
||||
this.userMessage = options?.userMessage;
|
||||
this.side = options?.side;
|
||||
this.issuePath = options?.issuePath;
|
||||
this.constraint = options?.constraint;
|
||||
this.remainingIssueCount = options?.remainingIssueCount;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { parseEvidence, immutableEvidence, type Evidence } from "./evidence.js";
|
||||
import { EvidenceIntegrityError } from "./evidenceErrors.js";
|
||||
import {
|
||||
describeValidationFailure,
|
||||
firstValidationIssuePath,
|
||||
firstValidationIssue,
|
||||
} from "./validationFailure.js";
|
||||
import { err, ok, type Result } from "./result.js";
|
||||
|
||||
@@ -74,19 +74,36 @@ export const readEvidencePayload = <Output>(
|
||||
: err(integrityFailure(parsed.error, location));
|
||||
};
|
||||
|
||||
/** The issue path is reported only when its position in the record is known. */
|
||||
/**
|
||||
* Schema issue facts for an integrity failure. The issue path is reported
|
||||
* only when its position in the record (`within`) is known.
|
||||
*/
|
||||
export const validationIssueOptions = (
|
||||
cause: unknown,
|
||||
within?: readonly (string | number)[],
|
||||
): {
|
||||
readonly issuePath?: readonly (string | number)[];
|
||||
readonly constraint?: string;
|
||||
readonly remainingIssueCount?: number;
|
||||
} => {
|
||||
const issue = firstValidationIssue(cause);
|
||||
if (issue === undefined) return {};
|
||||
return {
|
||||
...(within === undefined ? {} : { issuePath: [...within, ...issue.path] }),
|
||||
...(issue.constraint === undefined ? {} : { constraint: issue.constraint }),
|
||||
remainingIssueCount: issue.remainingIssueCount,
|
||||
};
|
||||
};
|
||||
|
||||
const integrityFailure = (
|
||||
cause: z.ZodError,
|
||||
location: EvidenceSourceLocation | undefined,
|
||||
): EvidenceIntegrityError => {
|
||||
const message = `Evidence validation failed ${describeValidationFailure(cause)}`;
|
||||
const issuePath = firstValidationIssuePath(cause);
|
||||
return new EvidenceIntegrityError(message, {
|
||||
cause,
|
||||
userMessage: message,
|
||||
...(location?.side === undefined ? {} : { side: location.side }),
|
||||
...(location?.within === undefined || issuePath === undefined
|
||||
? {}
|
||||
: { issuePath: [...location.within, ...issuePath] }),
|
||||
...validationIssueOptions(cause, location?.within),
|
||||
});
|
||||
};
|
||||
@@ -174,6 +174,7 @@ export const functionComparisonResultSchema = z
|
||||
path: ["status"],
|
||||
message:
|
||||
"Function comparison status contradicts its dimensions or function match.",
|
||||
params: { constraint: "status-matches-dimensions" },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -68,14 +68,17 @@ export const parseFunctionEvidence = (
|
||||
const parsed = readEvidence(input, side);
|
||||
if (!parsed.ok) return parsed;
|
||||
const evidence = parsed.value;
|
||||
const invalidSelection = (message: string, expected?: JsonValue) =>
|
||||
const invalidSelection = (
|
||||
message: string,
|
||||
facts?: { readonly expected: JsonValue; readonly observed: JsonValue },
|
||||
) =>
|
||||
err(
|
||||
new AnalysisInputError(selection.operation, { userMessage: message }, [
|
||||
{
|
||||
path: selection.path,
|
||||
reason: "invalid_value",
|
||||
message,
|
||||
...(expected === undefined ? {} : { expected }),
|
||||
...facts,
|
||||
},
|
||||
]),
|
||||
);
|
||||
@@ -93,8 +96,14 @@ export const parseFunctionEvidence = (
|
||||
return invalidSelection(
|
||||
`Function analysis requires analyze_function Evidence with the rea.analysis predicate; the ${subject} has operation ${evidence.operation} and predicate ${evidence.predicate_type}.`,
|
||||
{
|
||||
operation: "analyze_function",
|
||||
predicate_type: ANALYSIS_EVIDENCE_PREDICATE,
|
||||
expected: {
|
||||
operation: "analyze_function",
|
||||
predicate_type: ANALYSIS_EVIDENCE_PREDICATE,
|
||||
},
|
||||
observed: {
|
||||
operation: evidence.operation,
|
||||
predicate_type: evidence.predicate_type,
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ export const processCaptureComparisonSchema = z
|
||||
code: "custom",
|
||||
message: "Process comparison status contradicts its dimensions",
|
||||
path: ["status"],
|
||||
params: { constraint: "status-matches-dimensions" },
|
||||
});
|
||||
});
|
||||
export type ProcessCaptureComparison = z.infer<
|
||||
|
||||
@@ -1,14 +1,34 @@
|
||||
import { z } from "zod";
|
||||
|
||||
/** The location `describeValidationFailure` names, when the cause is schema-located. */
|
||||
export const firstValidationIssuePath = (
|
||||
/**
|
||||
* What `describeValidationFailure` states in prose, as facts a caller can act
|
||||
* on without reading the description.
|
||||
*/
|
||||
export interface ValidationIssueFacts {
|
||||
/** The first issue's location. */
|
||||
readonly path: readonly (string | number)[];
|
||||
/** The constraint a custom refinement named on the first issue. */
|
||||
readonly constraint: string | undefined;
|
||||
/** Further issues the description summarizes as "(N more issues)". */
|
||||
readonly remainingIssueCount: number;
|
||||
}
|
||||
|
||||
/** The first schema issue's facts, when the cause is a schema failure. */
|
||||
export const firstValidationIssue = (
|
||||
cause: unknown,
|
||||
): readonly (string | number)[] | undefined => {
|
||||
): ValidationIssueFacts | undefined => {
|
||||
if (!(cause instanceof z.ZodError)) return undefined;
|
||||
const first = cause.issues[0];
|
||||
return first?.path.map((key) =>
|
||||
typeof key === "number" ? key : String(key),
|
||||
);
|
||||
const [first, ...rest] = cause.issues;
|
||||
if (first === undefined) return undefined;
|
||||
const constraint: unknown =
|
||||
first.code === "custom" ? first.params?.["constraint"] : undefined;
|
||||
return {
|
||||
path: first.path.map((key) =>
|
||||
typeof key === "number" ? key : String(key),
|
||||
),
|
||||
constraint: typeof constraint === "string" ? constraint : undefined,
|
||||
remainingIssueCount: rest.length,
|
||||
};
|
||||
};
|
||||
|
||||
/** Locate the first schema issue, or keep a thrown constraint message. */
|
||||
|
||||
@@ -55,14 +55,17 @@ cliTest(
|
||||
code: "evidence_integrity_mismatch",
|
||||
category: "integrity_mismatch",
|
||||
retryable: false,
|
||||
details: { reason: expect.stringContaining(`at ${field}:`) },
|
||||
details: {
|
||||
reason: expect.stringContaining(`at ${field}:`),
|
||||
issue_path: field
|
||||
.split(".")
|
||||
.map((key) => (/^\d+$/u.test(key) ? Number(key) : key)),
|
||||
},
|
||||
});
|
||||
expect(error).not.toHaveProperty("cause");
|
||||
expect(error).not.toHaveProperty("stack");
|
||||
if (field === "target")
|
||||
expect(error.details?.reason).toMatch(
|
||||
/ at target: .+ \(2 more issues\)$/,
|
||||
);
|
||||
expect(error.details?.remaining_issue_count).toBe(2);
|
||||
}
|
||||
// MCP adds open-session facts; the snapshot reason must match exactly.
|
||||
expect(mcpError.details).toMatchObject(cliError.details ?? {});
|
||||
|
||||
@@ -58,12 +58,15 @@ describe.each([
|
||||
code: "evidence_integrity_mismatch",
|
||||
category: "integrity_mismatch",
|
||||
retryable: false,
|
||||
details: { reason: expect.stringContaining(`at ${field}:`) },
|
||||
details: {
|
||||
reason: expect.stringContaining(`at ${field}:`),
|
||||
issue_path: field
|
||||
.split(".")
|
||||
.map((key) => (/^\d+$/u.test(key) ? Number(key) : key)),
|
||||
},
|
||||
});
|
||||
if (field === "target")
|
||||
expect(projected.details?.reason).toMatch(
|
||||
/ at target: .+ \(2 more issues\)$/,
|
||||
);
|
||||
expect(projected.details?.remaining_issue_count).toBe(2);
|
||||
expect(projected).not.toHaveProperty("cause");
|
||||
expect(projected).not.toHaveProperty("stack");
|
||||
expect(await readFile(path, "utf8")).toBe(encoded);
|
||||
@@ -144,6 +147,39 @@ it("refuses a snapshot holding an earlier-format bundle with the recreate guidan
|
||||
});
|
||||
});
|
||||
|
||||
it("locates a bundle schema issue within the snapshot when reading and writing", async () => {
|
||||
const root = await createTestTempDirectory("rea-snapshot-bundle-issue-");
|
||||
const { providers: _providers, ...bundle } = createEvidenceBundle([]);
|
||||
const snapshot = {
|
||||
target: snapshotTarget(ANALYSIS_SNAPSHOT_TARGET),
|
||||
binding: snapshotBinding(ANALYSIS_SNAPSHOT_PROFILE),
|
||||
entries: [],
|
||||
workflow_entries: [],
|
||||
evidence_bundle: bundle,
|
||||
};
|
||||
const path = join(root, "snapshot.json");
|
||||
await writeFile(path, JSON.stringify(snapshot));
|
||||
const read = await readAnalysisSnapshot(path);
|
||||
const written = await withArtifactResourceScope((resources) =>
|
||||
writeAnalysisSnapshot(
|
||||
snapshot as never,
|
||||
join(root, "written.json"),
|
||||
false,
|
||||
resources,
|
||||
),
|
||||
);
|
||||
for (const result of [read, written]) {
|
||||
if (result.ok) throw new Error("Expected invalid bundle refusal");
|
||||
expect(projectAnalysisError(result.error)).toMatchObject({
|
||||
code: "evidence_integrity_mismatch",
|
||||
details: {
|
||||
reason: expect.stringContaining("at evidence_bundle.providers:"),
|
||||
issue_path: ["evidence_bundle", "providers"],
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("retains serialization schema diagnostics without creating a snapshot file", async () => {
|
||||
const root = await createTestTempDirectory("rea-snapshot-write-diagnostic-");
|
||||
const path = join(root, "snapshot.json");
|
||||
|
||||
@@ -314,10 +314,9 @@ describe("function comparison MCP integration", () => {
|
||||
});
|
||||
|
||||
describe("function comparison input-kind diagnostics", () => {
|
||||
// Each mismatch is checked once; alternating sides covers both labels.
|
||||
it.each([
|
||||
{ side: "left", mismatch: "operation" },
|
||||
{ side: "right", mismatch: "operation" },
|
||||
{ side: "left", mismatch: "predicate" },
|
||||
{ side: "right", mismatch: "predicate" },
|
||||
] as const)(
|
||||
"rejects inline $side Evidence with the wrong $mismatch as an input-selection error",
|
||||
@@ -350,11 +349,14 @@ describe("function comparison input-kind diagnostics", () => {
|
||||
{
|
||||
path: [side],
|
||||
reason: "invalid_value",
|
||||
message: `Function analysis requires analyze_function Evidence with the rea.analysis predicate; the ${side} Evidence has operation ${wrong.operation} and predicate ${wrong.predicate_type}.`,
|
||||
expected: {
|
||||
operation: "analyze_function",
|
||||
predicate_type: "rea.analysis",
|
||||
},
|
||||
observed: {
|
||||
operation: wrong.operation,
|
||||
predicate_type: wrong.predicate_type,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -390,9 +392,7 @@ describe("function comparison input-kind diagnostics", () => {
|
||||
issues: [
|
||||
{
|
||||
path: ["left"],
|
||||
message: expect.stringContaining(
|
||||
"the left Evidence has operation inventory_artifact",
|
||||
),
|
||||
observed: { operation: "inventory_artifact" },
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -531,11 +531,7 @@ it.each([
|
||||
const rejected = await verification(inconsistent.evidence_id);
|
||||
expect(parseMcpToolError(rejected).error).toMatchObject({
|
||||
code: "evidence_integrity_mismatch",
|
||||
details: {
|
||||
reason: expect.stringContaining(
|
||||
"Function comparison status contradicts its dimensions or function match",
|
||||
),
|
||||
},
|
||||
details: { constraint: "status-matches-dimensions" },
|
||||
});
|
||||
expect(harness.session.evidenceById(inconsistent.evidence_id)).toEqual(
|
||||
inconsistent,
|
||||
|
||||
@@ -110,14 +110,14 @@ const compareViaCli = async (
|
||||
);
|
||||
};
|
||||
|
||||
it.each(
|
||||
(["left", "right"] as const).flatMap((missingWorker) =>
|
||||
(["left", "right", "both", "neither"] as const).map((omittedRendered) => ({
|
||||
missingWorker,
|
||||
omittedRendered,
|
||||
})),
|
||||
),
|
||||
)(
|
||||
// Each omission is checked once, alternating which side lacks the worker: a
|
||||
// one-sided omission on either side, both sides, and neither.
|
||||
it.each([
|
||||
{ missingWorker: "left", omittedRendered: "left" },
|
||||
{ missingWorker: "right", omittedRendered: "right" },
|
||||
{ missingWorker: "right", omittedRendered: "both" },
|
||||
{ missingWorker: "left", omittedRendered: "neither" },
|
||||
] as const)(
|
||||
"preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output",
|
||||
async ({ missingWorker, omittedRendered }) => {
|
||||
const observation = (side: "left" | "right") => {
|
||||
@@ -468,9 +468,11 @@ it("names the side and constraint when Evidence is not a usable capture", async
|
||||
{
|
||||
path: ["left"],
|
||||
reason: "invalid_value",
|
||||
message: expect.stringContaining(
|
||||
"the left Evidence has operation other",
|
||||
),
|
||||
expected: {
|
||||
operation: "capture_process_scenario",
|
||||
predicate_type: "rea.process-capture",
|
||||
},
|
||||
observed: { operation: "other" },
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -552,7 +554,7 @@ it("names the side and constraint when Evidence is not a usable capture", async
|
||||
issues: [
|
||||
expect.objectContaining({
|
||||
path: ["right"],
|
||||
observed: { issue_path: issuePath },
|
||||
observed: expect.objectContaining({ issue_path: issuePath }),
|
||||
}),
|
||||
],
|
||||
},
|
||||
@@ -586,9 +588,6 @@ it("keeps structural capture issues alongside semantic ones through MCP and the
|
||||
},
|
||||
},
|
||||
});
|
||||
// The count of remaining issues is only in the description.
|
||||
const remaining = "(1 more issues)";
|
||||
|
||||
const response = await client.callTool({
|
||||
name: "compare_process_captures",
|
||||
arguments: { left: capture, right: mixed },
|
||||
@@ -596,23 +595,23 @@ it("keeps structural capture issues alongside semantic ones through MCP and the
|
||||
expect(parseMcpToolError(response)).toMatchObject({
|
||||
error: {
|
||||
code: "evidence_integrity_mismatch",
|
||||
message: expect.stringContaining(remaining),
|
||||
details: {
|
||||
side: "right",
|
||||
issue_path: ["normalized_result", "manifest", "completed_at"],
|
||||
remaining_issue_count: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(await compareViaCli(capture, mixed)).toMatchObject({
|
||||
code: "invalid_request",
|
||||
category: "invalid_input",
|
||||
message: expect.stringContaining(remaining),
|
||||
details: {
|
||||
issues: [
|
||||
{
|
||||
path: ["right"],
|
||||
observed: {
|
||||
issue_path: ["normalized_result", "manifest", "completed_at"],
|
||||
remaining_issue_count: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
@@ -101,11 +101,7 @@ it("rejects an inconsistent imported process comparison while retaining its Evid
|
||||
const rejected = await verify(inconsistent.evidence_id, selected);
|
||||
expect(parseMcpToolError(rejected).error).toMatchObject({
|
||||
code: "evidence_integrity_mismatch",
|
||||
details: {
|
||||
reason: expect.stringContaining(
|
||||
"Process comparison status contradicts its dimensions",
|
||||
),
|
||||
},
|
||||
details: { constraint: "status-matches-dimensions" },
|
||||
});
|
||||
const failed = await verify(consistent.evidence_id, selected);
|
||||
expect(failed.isError).not.toBe(true);
|
||||
|
||||
Reference in new issue
Block a user