Files
reverse-engineering-anything/tests/conformanceTrustGate.test.ts
T
morluto 4ee30e4145 feat(conformance): add portable conformance export package format
Define a committed scenario/replay/shim/evidence package format for
portable process-fidelity conformance fixtures (#336). The package
format is versioned, path-independent, and consumable from a clean
checkout.

- conformancePackage.ts: top-level package schema with scenario
  manifests, replay plans, shim plans, expected evidence, and
  verifier contracts. Deterministic package ID computed from
  canonical JSON. Cross-reference validation ensures internal
  consistency.
- conformanceTrustGate.ts: trust gates that compare captured runs
  against committed conformance packages (#337). Classifies semantic
  diffs from timing/normalization noise. Unknown/truncated evidence
  is never treated as equivalence.
- conformanceReplay.ts: CI replay engine that runs each scenario,
  evaluates trust gates, and detects drift between captured and
  committed evidence (#338).

Closes #336, #337, #338
2026-07-28 05:40:17 +00:00

114 lines
3.5 KiB
TypeScript

import { describe, expect, it } from "vitest";
import type { VerifierContract } from "../src/domain/conformancePackage.js";
import {
compareDimension,
evaluateTrustGate,
isSemanticDimension,
isVolatileDimension,
} from "../src/domain/conformanceTrustGate.js";
const contract: VerifierContract = {
scenario_id: "s1",
dimensions: [
{ name: "exit_code", required: true, comparison: "exact" },
{ name: "stdout", required: true, comparison: "semantic" },
{ name: "timing", required: true, comparison: "fuzzy" },
],
timing_tolerance_ms: 100,
};
describe("conformance trust gates", () => {
it("classifies volatile dimensions correctly", () => {
expect(isVolatileDimension("timing")).toBe(true);
expect(isVolatileDimension("timestamp")).toBe(true);
expect(isVolatileDimension("pid")).toBe(true);
expect(isVolatileDimension("exit_code")).toBe(false);
});
it("classifies semantic dimensions correctly", () => {
expect(isSemanticDimension("exit_code")).toBe(true);
expect(isSemanticDimension("stdout")).toBe(true);
expect(isSemanticDimension("timing")).toBe(false);
});
it("matches identical values", () => {
const result = compareDimension("exit_code", 0, 0);
expect(result.status).toBe("match");
});
it("detects mismatches", () => {
const result = compareDimension("exit_code", 0, 1);
expect(result.status).toBe("mismatch");
});
it("ignores volatile dimensions", () => {
const result = compareDimension("timing", 100, 200);
expect(result.status).toBe("match");
expect(result.message).toContain("volatile");
});
it("handles truncated evidence", () => {
const result = compareDimension("exit_code", 0, null, {
truncated: true,
});
expect(result.status).toBe("truncated");
});
it("handles undefined values", () => {
const result = compareDimension("exit_code", undefined, undefined);
expect(result.status).toBe("unknown");
});
it("handles missing actual", () => {
const result = compareDimension("exit_code", 0, undefined);
expect(result.status).toBe("mismatch");
});
it("passes when all dimensions match", () => {
const result = evaluateTrustGate(
contract,
{ exit_code: 0, stdout: "hello" },
{ exit_code: 0, stdout: "hello" },
);
expect(result.verdict).toBe("pass");
});
it("fails when a required dimension mismatches", () => {
const result = evaluateTrustGate(
contract,
{ exit_code: 0, stdout: "hello" },
{ exit_code: 1, stdout: "hello" },
);
expect(result.verdict).toBe("fail");
expect(result.first_divergence).not.toBeNull();
expect(result.first_divergence!.dimension).toBe("exit_code");
});
it("reports unknown when both dimensions are undefined", () => {
const result = evaluateTrustGate(contract, {}, {});
// exit_code and stdout are both undefined -> unknown
// timing is volatile -> match
expect(result.verdict).toBe("unknown");
});
it("ignores volatile dimension differences", () => {
const result = evaluateTrustGate(
contract,
{ exit_code: 0, stdout: "hello", timing: 100 },
{ exit_code: 0, stdout: "hello", timing: 200 },
);
expect(result.verdict).toBe("pass");
});
it("truncated evidence is not treated as equivalence", () => {
const result = evaluateTrustGate(
contract,
{ exit_code: 0, stdout: "hello" },
{ exit_code: 0, stdout: "hello" },
{ truncated: true },
);
expect(result.verdict).toBe("fail");
});
});