test(evidence): distinguish legacy format from schema refusal (#13045)

* test(evidence): distinguish legacy format from schema refusal

* test(evidence): verify full format recovery guidance
This commit is contained in:
morluto authored and GitHub committed 2026-11-21 08:32:56 +00:00
1 parent 9fd8c8d903
commit 184e9c083c
4 files changed
+81 -103

No files matched your search

+9 -11
View File
@@ -101,18 +101,16 @@ describe("evidence bundle failure classification", () => {
parameters: {},
result: { observed: true },
});
const single = safeParseEvidenceBundle(record);
expect(single.ok).toBe(false);
if (single.ok) return;
expect(single.error.kind).toBe("single-record");
const tampered = safeParseEvidenceBundle({
...record,
normalized_result: { observed: false },
expect(safeParseEvidenceBundle(record)).toMatchObject({
ok: false,
error: { kind: "single-record" },
});
expect(tampered.ok).toBe(false);
if (tampered.ok) return;
expect(tampered.error.kind).toBe("invalid-bundle");
expect(
safeParseEvidenceBundle({
...record,
normalized_result: { observed: false },
}),
).toMatchObject({ ok: false, error: { kind: "invalid-bundle" } });
});
});
+37 -38
View File
@@ -1,10 +1,9 @@
import { unknownTarget } from "./targetIdentity.js";
import { expect, it } from "vitest";
import { createEvidence, parseEvidence } from "./evidence.js";
import { createEvidence } from "./evidence.js";
import {
createEvidenceBundle,
readEvidenceBundle,
safeParseEvidenceBundle,
} from "./evidenceBundle.js";
import {
@@ -12,8 +11,6 @@ import {
RECREATE_EVIDENCE_GUIDANCE,
} from "./evidenceFormat.js";
import { PRE_66_NULL_COVERAGE_EVIDENCE } from "./evidenceLegacy.fixture.js";
import { ParseFailureError } from "./parseFailure.js";
import { EvidenceIntegrityError } from "./evidenceErrors.js";
import { readEvidence, readImmutableEvidence } from "./evidenceValidation.js";
const record = createEvidence(
@@ -45,7 +42,6 @@ it.each([
if (parsed.ok) throw new Error("Expected the earlier format to be refused");
expect(parsed.error.kind).toBe("unsupported-format");
expect(parsed.error.message).toContain(RECREATE_EVIDENCE_GUIDANCE);
expect(parsed.error.cause).toBeInstanceOf(ParseFailureError);
expect(parsed.error.cause).toMatchObject({
failure: {
kind: "unsupported",
@@ -54,42 +50,45 @@ it.each([
undecodedContent: true,
},
});
const read = readEvidenceBundle(earlier, {
operation: "import_evidence_bundle",
path: ["evidence_bundle"],
});
expect(read).toMatchObject({
});
it("distinguishes earlier records from malformed unmarked input", () => {
const unsupported = {
kind: "unsupported",
format: "evidence-record",
field: "evidence_format",
undecodedContent: true,
};
const bundle = createEvidenceBundle([record]);
for (const [fixture, read] of [
[PRE_66_NULL_COVERAGE_EVIDENCE, readEvidence],
[{ ...record, evidence_format: 1 }, readImmutableEvidence],
] as const) {
const earlier = structuredClone(fixture);
expect(read(earlier)).toMatchObject({
ok: false,
error: {
_tag: "EvidenceIntegrityError",
userMessage: expect.stringContaining(RECREATE_EVIDENCE_GUIDANCE),
cause: { failure: unsupported },
},
});
expect(
safeParseEvidenceBundle({ ...bundle, records: [earlier] }),
).toMatchObject({
ok: false,
error: { kind: "unsupported-format", cause: { failure: unsupported } },
});
expect(earlier).toEqual(fixture);
}
expect(
readEvidence({ evidence_id: record.evidence_id }, { side: "left" }),
).toMatchObject({
ok: false,
error: {
_tag: "EvidenceIntegrityError",
userMessage: expect.stringContaining(RECREATE_EVIDENCE_GUIDANCE),
side: "left",
issuePath: ["evidence_format"],
},
});
});
it.each([
["a pre-6.6 record", PRE_66_NULL_COVERAGE_EVIDENCE],
["a revision 1 record", { ...record, evidence_format: 1 }],
])(
"refuses %s alone or inside a current bundle, with the recreate guidance",
(_name, fixture) => {
const earlier = structuredClone(fixture);
expect(() => parseEvidence(earlier)).toThrow(RECREATE_EVIDENCE_GUIDANCE);
for (const read of [readEvidence, readImmutableEvidence]) {
const refused = read(earlier);
if (refused.ok) throw new Error("Expected the earlier record refused");
expect(refused.error).toBeInstanceOf(EvidenceIntegrityError);
expect(refused.error.userMessage).toContain(RECREATE_EVIDENCE_GUIDANCE);
}
const parsed = safeParseEvidenceBundle({
...createEvidenceBundle([record]),
records: [earlier],
});
if (parsed.ok) throw new Error("Expected the earlier record to be refused");
expect(parsed.error.kind).toBe("unsupported-format");
expect(parsed.error.cause).toMatchObject({
failure: { format: "evidence-record", field: "evidence_format" },
});
expect(earlier).toEqual(fixture);
},
);
@@ -1,9 +1,6 @@
import { unknownTarget } from "../../../src/domain/targetIdentity.js";
import {
EVIDENCE_FORMAT_REVISION,
RECREATE_EVIDENCE_GUIDANCE,
} from "../../../src/domain/evidenceFormat.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { RECREATE_EVIDENCE_GUIDANCE } from "../../../src/domain/evidenceFormat.js";
import { PRE_66_NULL_COVERAGE_EVIDENCE } from "../../../src/domain/evidenceLegacy.fixture.js";
import {
access,
@@ -498,39 +495,33 @@ cliTest(
async ({ cli }) => {
const root = await createTestTempDirectory("rea-legacy-coverage-identity-");
const path = join(root, "legacy.json");
const record = PRE_66_NULL_COVERAGE_EVIDENCE;
// A current-format bundle carrying an earlier record is refused by the
// record's own format marker, before its ID is checked.
const bundle = {
evidence_format: EVIDENCE_FORMAT_REVISION,
artifacts: [],
providers: [record.provider],
environments: [],
scenarios: [],
captures: [],
unknowns: [],
records: [record],
await writeFile(
path,
JSON.stringify({
...createEvidenceBundle([]),
records: [PRE_66_NULL_COVERAGE_EVIDENCE],
}),
);
const refusal = {
code: "evidence_integrity_mismatch",
retryable: false,
message: expect.stringContaining(
"Evidence record predates format revision",
),
};
await writeFile(path, JSON.stringify(bundle));
const imported = await cli.run({
arguments: ["evidence-import", path, "--json"],
});
expect(imported.exitCode).toBe(1);
expect(imported.json).toMatchObject({
code: "evidence_integrity_mismatch",
retryable: false,
message: expect.stringContaining(RECREATE_EVIDENCE_GUIDANCE),
});
const call = await cli.sharedMcpCall?.();
if (call === undefined) throw new Error("Expected shared MCP session");
const response = await call("import_evidence_bundle", { path });
expect(response.isError).toBe(true);
expect(parseMcpToolError(response)).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
retryable: false,
for (const error of [imported.json, parseMcpToolError(response).error]) {
expect(error).toMatchObject(refusal);
expect(error).toMatchObject({
message: expect.stringContaining(RECREATE_EVIDENCE_GUIDANCE),
},
});
});
}
},
);
@@ -9,10 +9,7 @@ import { describe, expect, it } from "vitest";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js";
import {
EVIDENCE_FORMAT_REVISION,
RECREATE_EVIDENCE_GUIDANCE,
} from "../../../src/domain/evidenceFormat.js";
import { RECREATE_EVIDENCE_GUIDANCE } from "../../../src/domain/evidenceFormat.js";
import {
createEvidence,
evidenceSchema,
@@ -257,29 +254,22 @@ describe("reconstruction obligation ledger process provider admission", () => {
);
});
describe("reconstruction obligation ledger Evidence format", () => {
it.each([
["an unmarked", undefined],
["a later", EVIDENCE_FORMAT_REVISION + 1],
])(
"refuses %s Evidence format through the format owner, not the SDK",
async (_label, revision) => {
const { evidence_format: _current, ...unmarked } = createEvidenceBundle(
[],
it("reports unmarked and later Evidence bundle formats on one MCP connection", async () => {
const { evidence_format: revision, ...unmarked } = createEvidenceBundle([]);
await withLedgerClient(async (request) => {
for (const [bundle, reason] of [
[unmarked, "This Evidence bundle predates format revision"],
[{ ...unmarked, evidence_format: revision + 1 }, "is not format"],
] as const) {
const result = await request(bundle);
expect(result.isError).toBe(true);
const projected = parseMcpToolError(result);
expect(projected).toMatchObject(integrityError(reason));
expect(projected).toMatchObject(
integrityError(RECREATE_EVIDENCE_GUIDANCE),
);
const bundle =
revision === undefined
? unmarked
: { ...unmarked, evidence_format: revision };
await withLedgerClient(async (request) => {
const result = await request(bundle);
expect(result.isError).toBe(true);
expect(parseMcpToolError(result)).toMatchObject(
integrityError(RECREATE_EVIDENCE_GUIDANCE),
);
});
},
);
}
});
});
describe("reconstruction obligation ledger MCP parity", () => {