mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
refactor(verification): share native client fixture assertions
This commit is contained in:
1 parent
0929751798
commit
f4cc6c85ef
13 files changed
+303
-372
No files matched your search
@@ -18,7 +18,10 @@ import {
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import {
|
||||
expandMcpSessionStatus,
|
||||
sortVerifierNames,
|
||||
@@ -90,10 +93,7 @@ await withVerifierCleanup(async () => {
|
||||
}
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
const schemaValidator = new Ajv2020({
|
||||
strict: false,
|
||||
validateFormats: false,
|
||||
@@ -249,33 +249,9 @@ await withVerifierCleanup(async () => {
|
||||
value.normalized_result?.graph?.nodes?.length > 0,
|
||||
);
|
||||
if (observed) {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(observed);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
const { evidence } = await assertClientJavaScriptFixtureEvidence(
|
||||
observed,
|
||||
target,
|
||||
);
|
||||
const bytes = JSON.stringify(evidence);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
@@ -5,14 +5,14 @@ import { homedir } from "node:os";
|
||||
import { isAbsolute, join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
@@ -80,10 +80,7 @@ await withVerifierCleanup(async () => {
|
||||
timeout: 30_000,
|
||||
stdin: "pipe",
|
||||
});
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
const environment = {
|
||||
PATH: process.env.PATH,
|
||||
NODE_OPTIONS: process.env.NODE_OPTIONS,
|
||||
@@ -246,43 +243,7 @@ await withVerifierCleanup(async () => {
|
||||
format: "directory",
|
||||
});
|
||||
}
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(last));
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shapes = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.filter(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(
|
||||
shapes.length > 0 &&
|
||||
shapes.every(
|
||||
(shape) => shape.return_shape_coverage?.projection_complete,
|
||||
),
|
||||
);
|
||||
const fields = shapes.flatMap((shape) =>
|
||||
shape.static_return_shapes.flatMap((item) => item.fields),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
);
|
||||
await assertClientJavaScriptFixtureEvidence(JSON.parse(last), target);
|
||||
evidenceBytes = Buffer.byteLength(last);
|
||||
await writeFile(join(lab, "native-evidence.json"), last);
|
||||
evidenceSeen = true;
|
||||
|
||||
@@ -15,7 +15,10 @@ import { compactPresentationForMcpInputSchemaProfile } from "../dist/config/mcpI
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
@@ -108,10 +111,7 @@ await withVerifierCleanup(async () => {
|
||||
}
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
const schemaValidator = new Ajv2020({
|
||||
strict: false,
|
||||
validateFormats: false,
|
||||
@@ -206,33 +206,9 @@ await withVerifierCleanup(async () => {
|
||||
containsText(body.messages, skillDirectory);
|
||||
const observed = extractEvidence(replies);
|
||||
if (observed) {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(observed);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
const { evidence } = await assertClientJavaScriptFixtureEvidence(
|
||||
observed,
|
||||
target,
|
||||
);
|
||||
const bytes = JSON.stringify(evidence);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
@@ -6,15 +6,15 @@ import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Ajv2020 } from "ajv/dist/2020.js";
|
||||
import { parse as parseYaml } from "yaml";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { pathExists } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
@@ -78,10 +78,7 @@ await withVerifierCleanup(async () => {
|
||||
stdin: "pipe",
|
||||
},
|
||||
);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
clientVersion = (
|
||||
await commands.exec(command, ["--version"], {
|
||||
env: environment,
|
||||
@@ -197,37 +194,9 @@ await withVerifierCleanup(async () => {
|
||||
for (const message of toolMessages) {
|
||||
const found = extractEvidence(message.content);
|
||||
if (found === undefined) continue;
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(found);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.find(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap(
|
||||
(entry) => entry.fields,
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
const { evidence } = await assertClientJavaScriptFixtureEvidence(
|
||||
found,
|
||||
target,
|
||||
);
|
||||
const text = JSON.stringify(evidence);
|
||||
await writeFile(join(lab, "evidence.json"), text);
|
||||
|
||||
@@ -17,7 +17,10 @@ import {
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs";
|
||||
|
||||
@@ -188,10 +191,7 @@ await withVerifierCleanup(async () => {
|
||||
);
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
const schemaValidator = new Ajv2020({
|
||||
strict: false,
|
||||
validateFormats: false,
|
||||
@@ -329,33 +329,9 @@ await withVerifierCleanup(async () => {
|
||||
);
|
||||
const observed = extractEvidence(responses);
|
||||
if (observed) {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(observed);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
const { evidence } = await assertClientJavaScriptFixtureEvidence(
|
||||
observed,
|
||||
target,
|
||||
);
|
||||
const bytes = JSON.stringify(evidence);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
@@ -5,14 +5,14 @@ import { homedir, tmpdir } from "node:os";
|
||||
import { isAbsolute, join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
@@ -95,7 +95,7 @@ await withVerifierCleanup(async () => {
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' +
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE +
|
||||
// A real analysis result large enough to exercise native MCP offloading.
|
||||
Array.from(
|
||||
{ length: 60 },
|
||||
@@ -135,42 +135,9 @@ await withVerifierCleanup(async () => {
|
||||
};
|
||||
};
|
||||
const verifyEvidence = async (bytes) => {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(bytes.toString("utf8")));
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shapes = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.filter(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(
|
||||
shapes.length > 0 &&
|
||||
shapes.every(
|
||||
(shape) => shape.return_shape_coverage?.projection_complete,
|
||||
),
|
||||
);
|
||||
const fields = shapes.flatMap((shape) =>
|
||||
shape.static_return_shapes.flatMap((item) => item.fields),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
const { evidence, shapes } = await assertClientJavaScriptFixtureEvidence(
|
||||
JSON.parse(bytes.toString("utf8")),
|
||||
target,
|
||||
);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
artifactBytes = bytes.length;
|
||||
|
||||
@@ -4,14 +4,14 @@ import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
@@ -66,10 +66,7 @@ await withVerifierCleanup(async () => {
|
||||
...(stickyHermes ? [mkdir(selectedProfile, { recursive: true })] : []),
|
||||
mkdir(target, { recursive: true }),
|
||||
]);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
HOME: account,
|
||||
@@ -195,32 +192,9 @@ await withVerifierCleanup(async () => {
|
||||
}
|
||||
};
|
||||
const validateEvidence = async (value) => {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(value);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
const { evidence } = await assertClientJavaScriptFixtureEvidence(
|
||||
value,
|
||||
target,
|
||||
);
|
||||
const serialized = JSON.stringify(evidence);
|
||||
await writeFile(join(lab, "evidence.json"), serialized);
|
||||
|
||||
@@ -4,14 +4,14 @@ import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { homedir, tmpdir } from "node:os";
|
||||
import { join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
@@ -85,10 +85,7 @@ await withVerifierCleanup(async () => {
|
||||
};
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
const analysisDevice = "xd://mcp__rea_analyze_javascript_application";
|
||||
const containsText = (value, text) => {
|
||||
if (typeof value === "string") return value.includes(text);
|
||||
@@ -171,37 +168,9 @@ await withVerifierCleanup(async () => {
|
||||
evidenceReply,
|
||||
"OMP must deliver the full analysis result to the next model turn",
|
||||
);
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(evidenceReply.content));
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.find(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap(
|
||||
(item) => item.fields,
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
const { evidence } = await assertClientJavaScriptFixtureEvidence(
|
||||
JSON.parse(evidenceReply.content),
|
||||
target,
|
||||
);
|
||||
const bytes = JSON.stringify(evidence);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
@@ -5,14 +5,14 @@ import { tmpdir } from "node:os";
|
||||
import { isAbsolute, join, relative, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import {
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
@@ -108,10 +108,7 @@ await withVerifierCleanup(async () => {
|
||||
}
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
const schemaValidator = new Ajv2020({
|
||||
strict: false,
|
||||
validateFormats: false,
|
||||
@@ -183,33 +180,9 @@ await withVerifierCleanup(async () => {
|
||||
);
|
||||
const observed = extractEvidence(replies);
|
||||
if (observed) {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(observed);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
const { evidence } = await assertClientJavaScriptFixtureEvidence(
|
||||
observed,
|
||||
target,
|
||||
);
|
||||
const bytes = JSON.stringify(evidence);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
@@ -15,12 +15,12 @@ import { createHash } from "node:crypto";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
|
||||
assertClientJavaScriptFixtureEvidence,
|
||||
} from "./verify/clients/javascript-fixture.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
@@ -191,10 +191,7 @@ await withVerifierCleanup(async () => {
|
||||
}
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target, { recursive: true });
|
||||
await writeFile(
|
||||
`${target}/app.js`,
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
await writeFile(`${target}/app.js`, CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
|
||||
const extractEvidence = (value) => {
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
@@ -257,35 +254,11 @@ await withVerifierCleanup(async () => {
|
||||
assert(artifactPath);
|
||||
const artifact = await readFile(artifactPath);
|
||||
artifactDigest = createHash("sha256").update(artifact).digest("hex");
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(artifact.toString("utf8")));
|
||||
const { evidence } = await assertClientJavaScriptFixtureEvidence(
|
||||
JSON.parse(artifact.toString("utf8")),
|
||||
target,
|
||||
);
|
||||
assert(extractEvidence(evidence));
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
);
|
||||
evidenceSeen = true;
|
||||
await writeFile(
|
||||
`${lab}/${client}-${mode}-evidence.json`,
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import type { z } from "zod";
|
||||
import type { JavaScriptApplicationAnalysisResult } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import type { toolContract } from "../../../src/contracts/toolContracts.js";
|
||||
|
||||
type FixtureContract = ReturnType<
|
||||
typeof toolContract<"analyze_javascript_application">
|
||||
>;
|
||||
type FixtureEvidence = Omit<
|
||||
z.output<FixtureContract["outputSchema"]>,
|
||||
"normalized_result"
|
||||
> & {
|
||||
readonly normalized_result: JavaScriptApplicationAnalysisResult;
|
||||
};
|
||||
type FixtureProperties =
|
||||
FixtureEvidence["normalized_result"]["graph"]["nodes"][number]["observations"][number]["properties"];
|
||||
|
||||
export const CLIENT_JAVASCRIPT_FIXTURE_SOURCE: string;
|
||||
export function assertClientJavaScriptFixtureEvidence(
|
||||
value: unknown,
|
||||
inputPath: string,
|
||||
): Promise<{ evidence: FixtureEvidence; shapes: FixtureProperties[] }>;
|
||||
@@ -0,0 +1,56 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { toolContract } from "../../../dist/contracts/toolContracts.js";
|
||||
import { assertJavaScriptApplicationInputPaths } from "../../lib/javascript-application-paths.mjs";
|
||||
|
||||
export const CLIENT_JAVASCRIPT_FIXTURE_SOURCE =
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n';
|
||||
|
||||
/** Validate the same analyst result across native client transports. */
|
||||
export const assertClientJavaScriptFixtureEvidence = async (
|
||||
value,
|
||||
inputPath,
|
||||
) => {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(value);
|
||||
assert(
|
||||
Array.isArray(evidence.normalized_result.graph?.nodes),
|
||||
"Client must return the full application graph",
|
||||
);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, inputPath);
|
||||
const shapes = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.filter(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(shapes.length > 0, "Client must return the fixture's result export");
|
||||
assert(
|
||||
shapes.every((shape) => shape.return_shape_coverage?.projection_complete),
|
||||
"Every fixture result observation must have complete return-shape coverage",
|
||||
);
|
||||
for (const shape of shapes) {
|
||||
const fields = shape.static_return_shapes.flatMap((item) => item.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
"Client must return the fixture's compatibility literal",
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
"Client must return the fixture's count literal",
|
||||
);
|
||||
}
|
||||
return { evidence, shapes };
|
||||
};
|
||||
Reference in new issue
Block a user