mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
morluto
3ad465a666
Keep the maintained manual evaluator, fixture rubrics and semantic tests together under tests/evaluation/model-evals instead of shipping them in dist. Load the same typed owners from the manual runner and update the architecture and client guide. Grading/report contracts and executable logic are unchanged.
814 lines
26 KiB
TypeScript
814 lines
26 KiB
TypeScript
import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server";
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
|
import { createEvidence, type Evidence } from "../../../src/domain/evidence.js";
|
|
import type { JsonValue } from "../../../src/domain/jsonValue.js";
|
|
import { evaluateCodexEvents } from "./CodexAgentEval.js";
|
|
import {
|
|
agentFixtureAnswerInstructions,
|
|
agentFixtureClaims,
|
|
} from "./AgentEvaluationFixtures.js";
|
|
import { inspectManagedArtifactBytes } from "../../../src/dotnet/ManagedArtifactInspector.js";
|
|
import { inspectManagedMembersBytes } from "../../../src/dotnet/ManagedMemberInspector.js";
|
|
import {
|
|
buildManagedPeFixture,
|
|
managedPeFixtureTarget,
|
|
} from "../../../src/dotnet/ManagedPe.fixture.js";
|
|
import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js";
|
|
import { ToolResultDelivery } from "../../../src/server/toolResult.js";
|
|
|
|
const delivery = new ToolResultDelivery(STDIO_DEFAULT_MAX_BUFFER_SIZE);
|
|
|
|
const provider = { id: "fixture", name: "Fixture analysis", version: "1" };
|
|
const subject = {
|
|
path: "/targets/app.asar",
|
|
sha256: "a".repeat(64),
|
|
format: "asar",
|
|
};
|
|
const createObservation = (
|
|
result: JsonValue = {
|
|
fact: { present: true, channel: "profile:read" },
|
|
unrelated: 1,
|
|
},
|
|
path = subject.path,
|
|
): Evidence =>
|
|
createEvidence({ ...subject, path, format: "asar" }, provider, {
|
|
operation: "analyze_javascript_application",
|
|
parameters: { format: "asar" },
|
|
result,
|
|
confidence: "derived",
|
|
authority: "shipped-artifact",
|
|
});
|
|
const resultFor = (evidence: Evidence) =>
|
|
delivery.toEvidenceToolResult(
|
|
evidence,
|
|
toolContract(
|
|
evidence.operation === "inspect_managed_artifact"
|
|
? "inspect_managed_artifact"
|
|
: evidence.operation === "inspect_managed_members"
|
|
? "inspect_managed_members"
|
|
: "analyze_javascript_application",
|
|
),
|
|
undefined,
|
|
);
|
|
const delivered = (
|
|
evidence: Evidence,
|
|
overrides: Record<string, unknown> = {},
|
|
) => ({
|
|
type: "item.completed",
|
|
item: {
|
|
id: evidence.evidence_id,
|
|
type: "mcp_tool_call",
|
|
server: "rea",
|
|
tool: evidence.operation,
|
|
status: "completed",
|
|
result: resultFor(evidence),
|
|
...overrides,
|
|
},
|
|
});
|
|
const final = (text: string) => ({
|
|
type: "item.completed",
|
|
item: { type: "agent_message", text },
|
|
});
|
|
const claimFor = (
|
|
evidence: Evidence,
|
|
overrides: Record<string, unknown> = {},
|
|
) => ({
|
|
id: "bridge",
|
|
value: { present: true, channel: "profile:read" },
|
|
evidence_id: evidence.evidence_id,
|
|
confidence: "derived",
|
|
authority: "shipped-artifact",
|
|
...overrides,
|
|
});
|
|
const expected = (): FixtureClaimExpectation => ({
|
|
id: "bridge",
|
|
expectedValue: { present: true, channel: "profile:read" },
|
|
source: {
|
|
operation: "analyze_javascript_application",
|
|
confidence: "derived",
|
|
authority: "shipped-artifact",
|
|
subject: { path: subject.path, sha256: subject.sha256 },
|
|
parameters: { format: "asar" },
|
|
select: (value) =>
|
|
value !== null && typeof value === "object" && !Array.isArray(value)
|
|
? value.fact
|
|
: undefined,
|
|
},
|
|
});
|
|
const assess = (
|
|
events: readonly unknown[],
|
|
text: string,
|
|
fixtureClaims: readonly FixtureClaimExpectation[] = [expected()],
|
|
) =>
|
|
evaluateCodexEvents(
|
|
[...events, final(text)],
|
|
"analyze_javascript_application",
|
|
{
|
|
fixtureClaims,
|
|
},
|
|
);
|
|
const answer = (claims: readonly unknown[]) => JSON.stringify({ claims });
|
|
|
|
// Synthetic transcript cases exercise grading, not an SDK-built assembly oracle.
|
|
const managedTranscript = () => {
|
|
const bytes = buildManagedPeFixture({
|
|
typeNamespace: "AgentEval",
|
|
typeName: "Program",
|
|
methodName: "Main",
|
|
});
|
|
const target = managedPeFixtureTarget(bytes, "/targets/AgentEval.dll");
|
|
const artifact = inspectManagedArtifactBytes(bytes, target);
|
|
const members = inspectManagedMembersBytes(bytes, target);
|
|
const program = members.types[0];
|
|
if (program === undefined) throw new Error("Fixture type missing");
|
|
const result = {
|
|
...members,
|
|
types: [
|
|
program,
|
|
{
|
|
...program,
|
|
token: "0x02000002",
|
|
name: "Profile",
|
|
full_name: "AgentEval.Profile",
|
|
},
|
|
],
|
|
};
|
|
const rubric = agentFixtureClaims("managed", {
|
|
javascript: "",
|
|
javascriptSha256: "",
|
|
javascriptShapeLeft: "",
|
|
javascriptShapeRight: "",
|
|
notesElectron: "",
|
|
notesElectronSha256: "",
|
|
managed: target.path,
|
|
managedSha256: target.sha256,
|
|
});
|
|
if (rubric === undefined) throw new Error("Managed rubric missing");
|
|
const evidenceFor = (
|
|
operation: string,
|
|
value: JsonValue,
|
|
overrides: Record<string, unknown> = {},
|
|
) =>
|
|
createEvidence(target, provider, {
|
|
operation,
|
|
parameters: {},
|
|
result: value,
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
...overrides,
|
|
});
|
|
const artifactEvidence = evidenceFor("inspect_managed_artifact", artifact);
|
|
const memberEvidence = evidenceFor("inspect_managed_members", result);
|
|
const textFor = (evidence: Evidence) =>
|
|
answer([
|
|
{
|
|
id: "public_types",
|
|
value: ["AgentEval.Profile", "AgentEval.Program"],
|
|
evidence_id: evidence.evidence_id,
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
},
|
|
{
|
|
id: "entry_point",
|
|
value: { declaring_type: "AgentEval.Program", name: "Main" },
|
|
evidence_id: evidence.evidence_id,
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
},
|
|
]);
|
|
const grade = (
|
|
events = [delivered(artifactEvidence), delivered(memberEvidence)],
|
|
text = textFor(memberEvidence),
|
|
) =>
|
|
evaluateCodexEvents([...events, final(text)], "inspect_managed_artifact", {
|
|
fixtureClaims: rubric,
|
|
});
|
|
return {
|
|
artifact,
|
|
result,
|
|
artifactEvidence,
|
|
memberEvidence,
|
|
evidenceFor,
|
|
textFor,
|
|
grade,
|
|
};
|
|
};
|
|
|
|
describe("managed known-answer fixture grading", () => {
|
|
it("grades public types independent of row order and resolves the header token", () => {
|
|
expect(managedTranscript().grade().factualCorrectness).toBe("passed");
|
|
});
|
|
|
|
it("does not pin compiler-local entry tokens or MVIDs", () => {
|
|
const fixture = managedTranscript();
|
|
const artifact = structuredClone(fixture.artifact);
|
|
const members = structuredClone(fixture.result);
|
|
if (
|
|
artifact.pe.cli === null ||
|
|
artifact.module === null ||
|
|
members.module === null ||
|
|
members.methods[0] === undefined
|
|
)
|
|
throw new Error("Fixture identity missing");
|
|
artifact.pe.cli.entry_point.value = "0x06000009";
|
|
members.methods[0].token = "0x06000009";
|
|
const mvid = "11111111-2222-3333-4444-555555555555";
|
|
artifact.module.mvid = mvid;
|
|
members.module.mvid = mvid;
|
|
members.identity_scope.requires_mvid = mvid;
|
|
members.types.reverse();
|
|
const artifactEvidence = fixture.evidenceFor(
|
|
"inspect_managed_artifact",
|
|
artifact,
|
|
);
|
|
const memberEvidence = fixture.evidenceFor(
|
|
"inspect_managed_members",
|
|
members,
|
|
);
|
|
expect(
|
|
fixture.grade(
|
|
[delivered(artifactEvidence), delivered(memberEvidence)],
|
|
fixture.textFor(memberEvidence),
|
|
).factualCorrectness,
|
|
).toBe("passed");
|
|
});
|
|
|
|
it.each([
|
|
"wrong-token",
|
|
"native-rva",
|
|
"mvid",
|
|
"null-mvid",
|
|
"digest",
|
|
"partial",
|
|
])("rejects an unsupported artifact entry point: %s", (kind) => {
|
|
const fixture = managedTranscript();
|
|
const artifact = structuredClone(fixture.artifact);
|
|
if (artifact.pe.cli === null || artifact.module === null)
|
|
throw new Error("Fixture identity missing");
|
|
if (kind === "wrong-token")
|
|
artifact.pe.cli.entry_point.value = "0x06000002";
|
|
if (kind === "native-rva") artifact.pe.cli.entry_point.kind = "native-rva";
|
|
if (kind === "mvid")
|
|
artifact.module.mvid = "11111111-2222-3333-4444-555555555555";
|
|
if (kind === "null-mvid") artifact.module.mvid = null;
|
|
if (kind === "digest") artifact.artifact.sha256 = "b".repeat(64);
|
|
if (kind === "partial") artifact.coverage.state = "partial";
|
|
const evidence = fixture.evidenceFor("inspect_managed_artifact", artifact);
|
|
expect(
|
|
fixture.grade([delivered(evidence), delivered(fixture.memberEvidence)])
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
});
|
|
|
|
describe("managed inventory scope", () => {
|
|
it.each(["selection", "parameters", "both"])(
|
|
"rejects selected public-type inventories while retaining entry-point grounding: %s",
|
|
(kind) => {
|
|
const fixture = managedTranscript();
|
|
const typeNames = ["AgentEval.Profile", "AgentEval.Program"];
|
|
const members = {
|
|
...fixture.result,
|
|
selection:
|
|
kind === "parameters"
|
|
? null
|
|
: { type_names: typeNames, unmatched_type_names: [] },
|
|
};
|
|
const evidence = fixture.evidenceFor("inspect_managed_members", members, {
|
|
parameters: kind === "selection" ? {} : { type_names: typeNames },
|
|
});
|
|
expect(
|
|
fixture.grade(
|
|
[delivered(fixture.artifactEvidence), delivered(evidence)],
|
|
fixture.textFor(evidence),
|
|
),
|
|
).toMatchObject({
|
|
factualCorrectness: "failed",
|
|
factualAssessment: {
|
|
claims: [
|
|
{ id: "public_types", status: "failed" },
|
|
{ id: "entry_point", status: "passed" },
|
|
],
|
|
},
|
|
});
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("managed member grounding", () => {
|
|
it.each([
|
|
"extra-public-type",
|
|
"missing-public-type",
|
|
"duplicate-method",
|
|
"decoy-main",
|
|
"scope-digest",
|
|
"scope-mvid",
|
|
"null-scope",
|
|
"partial",
|
|
"unavailable",
|
|
])("rejects unsupported member facts: %s", (kind) => {
|
|
const fixture = managedTranscript();
|
|
const members = structuredClone(fixture.result);
|
|
const method = members.methods[0];
|
|
const type = members.types[0];
|
|
if (method === undefined || type === undefined)
|
|
throw new Error("Fixture members missing");
|
|
if (kind === "extra-public-type")
|
|
members.types.push({
|
|
...type,
|
|
name: "Extra",
|
|
full_name: "AgentEval.Extra",
|
|
});
|
|
if (kind === "missing-public-type") members.types.pop();
|
|
if (kind === "duplicate-method") members.methods.push(method);
|
|
if (kind === "decoy-main") {
|
|
method.name = "Other";
|
|
members.methods.push({ ...method, token: "0x06000002", name: "Main" });
|
|
}
|
|
if (kind === "scope-digest")
|
|
members.identity_scope.requires_artifact_sha256 = "b".repeat(64);
|
|
if (kind === "scope-mvid")
|
|
members.identity_scope.requires_mvid =
|
|
"11111111-2222-3333-4444-555555555555";
|
|
if (kind === "null-scope") members.identity_scope.requires_mvid = null;
|
|
if (kind === "partial") members.metadata.status = "partial";
|
|
if (kind === "unavailable") members.coverage.state = "unavailable";
|
|
const evidence = fixture.evidenceFor("inspect_managed_members", members);
|
|
expect(
|
|
fixture.grade(
|
|
[delivered(fixture.artifactEvidence), delivered(evidence)],
|
|
fixture.textFor(evidence),
|
|
).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("rejects distinct authenticated matching artifact deliveries", () => {
|
|
const fixture = managedTranscript();
|
|
const secondArtifact = fixture.evidenceFor(
|
|
"inspect_managed_artifact",
|
|
fixture.artifact,
|
|
{ parameters: { path: "/targets/AgentEval.dll" } },
|
|
);
|
|
expect(secondArtifact.evidence_id).not.toBe(
|
|
fixture.artifactEvidence.evidence_id,
|
|
);
|
|
const member = delivered(fixture.memberEvidence);
|
|
expect(
|
|
fixture.grade([delivered(secondArtifact), member]).factualCorrectness,
|
|
).toBe("passed");
|
|
expect(
|
|
fixture.grade([
|
|
delivered(fixture.artifactEvidence),
|
|
delivered(secondArtifact),
|
|
member,
|
|
]).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("accepts identical repeated artifact deliveries", () => {
|
|
const fixture = managedTranscript();
|
|
expect(
|
|
fixture.grade([
|
|
delivered(fixture.artifactEvidence),
|
|
delivered(fixture.artifactEvidence),
|
|
delivered(fixture.memberEvidence),
|
|
]).factualCorrectness,
|
|
).toBe("passed");
|
|
});
|
|
|
|
it("requires companion delivery and rejects conflicting or failed artifact deliveries", () => {
|
|
const fixture = managedTranscript();
|
|
const member = delivered(fixture.memberEvidence);
|
|
expect(fixture.grade([member]).factualCorrectness).toBe("failed");
|
|
expect(
|
|
fixture.grade([
|
|
delivered(fixture.artifactEvidence, { status: "failed" }),
|
|
member,
|
|
]).factualCorrectness,
|
|
).toBe("failed");
|
|
const conflicting = {
|
|
...fixture.artifactEvidence,
|
|
subject:
|
|
fixture.memberEvidence.subject === null
|
|
? null
|
|
: { ...fixture.memberEvidence.subject, local_path: "/other.dll" },
|
|
};
|
|
expect(
|
|
fixture.grade([
|
|
delivered(fixture.artifactEvidence),
|
|
delivered(conflicting),
|
|
member,
|
|
]).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("binds the digest, rejects fabricated values and cites the supporting operation", () => {
|
|
const fixture = managedTranscript();
|
|
expect(
|
|
fixture.grade(undefined, fixture.textFor(fixture.artifactEvidence))
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
expect(
|
|
fixture.grade(
|
|
undefined,
|
|
fixture.textFor(fixture.memberEvidence).replace('"Main"', '"Invented"'),
|
|
).factualCorrectness,
|
|
).toBe("failed");
|
|
const other = createEvidence(
|
|
{ path: "/targets/AgentEval.dll", sha256: "b".repeat(64), format: "pe" },
|
|
provider,
|
|
{
|
|
operation: "inspect_managed_members",
|
|
parameters: {},
|
|
result: fixture.result,
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
},
|
|
);
|
|
expect(
|
|
fixture.grade(
|
|
[delivered(fixture.artifactEvidence), delivered(other)],
|
|
fixture.textFor(other),
|
|
).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("keeps golden names out of instructions and preserves routing-only scenarios", () => {
|
|
const instructions = agentFixtureAnswerInstructions("managed");
|
|
expect(instructions).toContain("do not read target files directly");
|
|
expect(instructions).not.toMatch(/AgentEval|Profile|Program|Main/u);
|
|
expect(agentFixtureAnswerInstructions("native")).toBe("");
|
|
});
|
|
});
|
|
|
|
// Removing exact fixture comparison or accepted-source validation must break these regressions.
|
|
describe("known-answer fixture grading", () => {
|
|
it("passes closed claims against authenticated production MCP Evidence", () => {
|
|
const evidence = createObservation();
|
|
expect(
|
|
assess([delivered(evidence)], answer([claimFor(evidence)])),
|
|
).toMatchObject({
|
|
factualCorrectness: "passed",
|
|
factualAssessment: {
|
|
scope: "configured_fixture_claims",
|
|
status: "passed",
|
|
},
|
|
});
|
|
});
|
|
|
|
it("rejects fabricated prose despite expected terms and a produced Evidence ID", () => {
|
|
const evidence = createObservation();
|
|
const text = `Observed evidence ${evidence.evidence_id}: profileApi and profile:read do not exist; there is no preload or contextBridge. All communications use quantum teleportation.`;
|
|
expect(assess([delivered(evidence)], text).factualCorrectness).toBe(
|
|
"failed",
|
|
);
|
|
});
|
|
|
|
it.each([
|
|
{ present: false, channel: "profile:read" },
|
|
{ present: true, channel: "invented:channel" },
|
|
{ present: true, channel: "profile:read", fabricated: "teleportation" },
|
|
"profile:read does not exist",
|
|
])("rejects a wrong or contradictory value %j", (value) => {
|
|
const evidence = createObservation();
|
|
expect(
|
|
assess([delivered(evidence)], answer([claimFor(evidence, { value })]))
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("retains not_assessed without a fixture rubric", () => {
|
|
expect(
|
|
evaluateCodexEvents(
|
|
[final("Observed metadata remains unknown.")],
|
|
"analyze_javascript_application",
|
|
).factualCorrectness,
|
|
).toBe("not_assessed");
|
|
});
|
|
|
|
it.each([{ rubric: [] }, { rubric: [expected(), expected()] }])(
|
|
"fails closed for an empty or duplicate rubric",
|
|
({ rubric }) => {
|
|
const evidence = createObservation();
|
|
expect(
|
|
assess([delivered(evidence)], answer([claimFor(evidence)]), rubric)
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
},
|
|
);
|
|
|
|
it("rejects selector failures without crashing the evaluation", () => {
|
|
const evidence = createObservation();
|
|
const expectation = expected();
|
|
const rubric = [
|
|
{
|
|
...expectation,
|
|
source: {
|
|
...expectation.source,
|
|
select: () => {
|
|
throw new TypeError("Malformed result");
|
|
},
|
|
},
|
|
},
|
|
];
|
|
expect(
|
|
assess([delivered(evidence)], answer([claimFor(evidence)]), rubric)
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
});
|
|
|
|
describe("closed final claim manifests", () => {
|
|
it.each([
|
|
"missing",
|
|
"duplicate",
|
|
"unknown",
|
|
"extra-field",
|
|
"extra-top-level",
|
|
"surrounding-prose",
|
|
"missing-authority",
|
|
])("rejects %s claims", (kind) => {
|
|
const evidence = createObservation();
|
|
const claim = claimFor(evidence);
|
|
let text = answer([claim]);
|
|
if (kind === "missing") text = answer([]);
|
|
if (kind === "duplicate") text = answer([claim, claim]);
|
|
if (kind === "unknown")
|
|
text = answer([claim, { ...claim, id: "fabrication" }]);
|
|
if (kind === "extra-field")
|
|
text = answer([{ ...claim, explanation: "Also teleportation" }]);
|
|
if (kind === "extra-top-level")
|
|
text = JSON.stringify({
|
|
claims: [claim],
|
|
explanation: "Also teleportation",
|
|
});
|
|
if (kind === "surrounding-prose") text = `${text}\nAlso teleportation.`;
|
|
if (kind === "missing-authority")
|
|
text = answer([
|
|
{
|
|
id: claim.id,
|
|
value: claim.value,
|
|
evidence_id: claim.evidence_id,
|
|
confidence: claim.confidence,
|
|
},
|
|
]);
|
|
expect(assess([delivered(evidence)], text).factualCorrectness).toBe(
|
|
"failed",
|
|
);
|
|
});
|
|
|
|
it("rejects duplicate object members containing contradictory values", () => {
|
|
const evidence = createObservation();
|
|
const text = answer([claimFor(evidence)]).replace(
|
|
'"present":true',
|
|
'"present":false,"present":true',
|
|
);
|
|
expect(assess([delivered(evidence)], text).factualCorrectness).toBe(
|
|
"failed",
|
|
);
|
|
});
|
|
|
|
it("fails closed for a deeply nested generated answer", () => {
|
|
const text = `${"[".repeat(20_000)}null${"]".repeat(20_000)}`;
|
|
expect(assess([], text).factualCorrectness).toBe("failed");
|
|
});
|
|
|
|
it.each(["confidence", "authority"])("rejects an overstated %s", (field) => {
|
|
const evidence = createObservation();
|
|
const claim = claimFor(evidence, {
|
|
[field]: field === "confidence" ? "observed" : "controlled-replay",
|
|
});
|
|
expect(
|
|
assess([delivered(evidence)], answer([claim])).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
});
|
|
|
|
describe("successful Evidence grounding", () => {
|
|
it.each([
|
|
{ server: "other" },
|
|
{ server: null },
|
|
{ status: "failed" },
|
|
{ status: "in_progress" },
|
|
{ error: { code: "unavailable" } },
|
|
{ tool: "other_tool" },
|
|
])("rejects a foreign or unsuccessful completion %j", (overrides) => {
|
|
const evidence = createObservation();
|
|
expect(
|
|
assess([delivered(evidence, overrides)], answer([claimFor(evidence)]))
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("rejects IDs present only in arguments or unrelated metadata", () => {
|
|
const evidence = createObservation();
|
|
const event = delivered(evidence, {
|
|
arguments: { evidence },
|
|
result: {
|
|
structuredContent: { metadata: { evidence_id: evidence.evidence_id } },
|
|
},
|
|
});
|
|
expect(
|
|
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("rejects a tampered envelope even with a valid-looking Evidence ID", () => {
|
|
const evidence = createObservation();
|
|
const event = delivered(evidence, {
|
|
result: {
|
|
structuredContent: {
|
|
...evidence,
|
|
authority: "controlled-replay",
|
|
},
|
|
},
|
|
});
|
|
expect(
|
|
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it.each(["id", "result", "error"])(
|
|
"rejects inconsistent or failed MCP wrapper %s",
|
|
(field) => {
|
|
const evidence = createObservation();
|
|
const projection = {
|
|
evidence_id: evidence.evidence_id,
|
|
result: evidence.normalized_result,
|
|
evidence,
|
|
};
|
|
const event = delivered(evidence, {
|
|
result: {
|
|
structuredContent: {
|
|
...projection,
|
|
...(field === "id"
|
|
? { evidence_id: `ev_${"b".repeat(64)}` }
|
|
: field === "result"
|
|
? { result: { fabricated: true } }
|
|
: {}),
|
|
},
|
|
...(field === "error" ? { isError: true } : {}),
|
|
},
|
|
});
|
|
expect(
|
|
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
|
).toBe("failed");
|
|
},
|
|
);
|
|
|
|
it.each(["error", "contradictory-data"])(
|
|
"rejects conflicting outer result/output aliases containing %s",
|
|
(kind) => {
|
|
const evidence = createObservation();
|
|
const output =
|
|
kind === "error"
|
|
? { isError: true, structuredContent: { error: { code: "denied" } } }
|
|
: {
|
|
...resultFor(evidence),
|
|
structuredContent: {
|
|
...evidence,
|
|
normalized_result: {
|
|
fact: { present: false, channel: "profile:read" },
|
|
},
|
|
},
|
|
};
|
|
const event = delivered(evidence, { output });
|
|
expect(
|
|
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
|
).toBe("failed");
|
|
},
|
|
);
|
|
|
|
it("accepts JSON-equal separately allocated outer result/output aliases", () => {
|
|
const evidence = createObservation();
|
|
const event = delivered(evidence, { output: resultFor(evidence) });
|
|
expect(
|
|
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
|
).toBe("passed");
|
|
});
|
|
|
|
it("accepts the documented snake-case structured-content transcript representation", () => {
|
|
const evidence = createObservation();
|
|
const event = delivered(evidence, {
|
|
result: { structured_content: resultFor(evidence).structuredContent },
|
|
});
|
|
expect(
|
|
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
|
).toBe("passed");
|
|
});
|
|
|
|
it("grades a complete text-only production Evidence response", () => {
|
|
const evidence = createObservation();
|
|
const event = delivered(evidence, {
|
|
result: {
|
|
content: resultFor(evidence).content,
|
|
structured_content: null,
|
|
},
|
|
});
|
|
expect(
|
|
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
|
).toBe("passed");
|
|
});
|
|
|
|
it("rejects authentic unrelated Evidence containing the same value elsewhere", () => {
|
|
const evidence = createObservation({ unrelated: expected().expectedValue });
|
|
expect(
|
|
assess([delivered(evidence)], answer([claimFor(evidence)]))
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
});
|
|
|
|
describe("fixture identity and provenance binding", () => {
|
|
it("rejects Evidence for a different subject despite an equal supported value", () => {
|
|
const evidence = createObservation(undefined, "/targets/other.asar");
|
|
expect(
|
|
assess([delivered(evidence)], answer([claimFor(evidence)]))
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("rejects a wrong required source parameter", () => {
|
|
const evidence = createObservation();
|
|
const expectation = expected();
|
|
const rubric = [
|
|
{
|
|
...expectation,
|
|
source: { ...expectation.source, parameters: { format: "directory" } },
|
|
},
|
|
];
|
|
expect(
|
|
assess([delivered(evidence)], answer([claimFor(evidence)]), rubric)
|
|
.factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("rejects conflicting subject diagnostics for the same semantic Evidence ID", () => {
|
|
const evidence = createObservation();
|
|
const conflicting = {
|
|
...evidence,
|
|
subject:
|
|
evidence.subject === null
|
|
? null
|
|
: { ...evidence.subject, local_path: "/targets/other.asar" },
|
|
};
|
|
expect(
|
|
assess(
|
|
[delivered(evidence), delivered(conflicting)],
|
|
answer([claimFor(evidence)]),
|
|
).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
|
|
it("requires linked source Evidence to be delivered and match the configured artifact", () => {
|
|
const source = createObservation(undefined, "/targets/parser-v1");
|
|
const evidence = createEvidence({ ...subject, format: "asar" }, provider, {
|
|
operation: "analyze_javascript_application",
|
|
parameters: { format: "asar" },
|
|
result: { fact: expected().expectedValue },
|
|
confidence: "derived",
|
|
authority: "shipped-artifact",
|
|
evidenceLinks: [source.evidence_id],
|
|
});
|
|
const expectation = expected();
|
|
const rubric = [
|
|
{
|
|
...expectation,
|
|
source: {
|
|
...expectation.source,
|
|
linkedSubjects: [
|
|
{
|
|
operation: source.operation,
|
|
path: "/targets/parser-v1",
|
|
sha256: subject.sha256,
|
|
},
|
|
],
|
|
},
|
|
},
|
|
];
|
|
const text = answer([claimFor(evidence)]);
|
|
expect(assess([delivered(evidence)], text, rubric).factualCorrectness).toBe(
|
|
"failed",
|
|
);
|
|
expect(
|
|
assess([delivered(source), delivered(evidence)], text, rubric)
|
|
.factualCorrectness,
|
|
).toBe("passed");
|
|
expect(
|
|
assess([delivered(source), delivered(evidence)], text, [
|
|
{
|
|
...expectation,
|
|
source: {
|
|
...expectation.source,
|
|
linkedSubjects: [
|
|
{ operation: source.operation, path: "/targets/other-parser" },
|
|
],
|
|
},
|
|
},
|
|
]).factualCorrectness,
|
|
).toBe("failed");
|
|
});
|
|
});
|