mirror of
https://github.com/morluto/rea.git
synced 2026-11-13 18:22:42 +00:00
fix(desktop): preserve own JSON keys in task grading (#10331)
* fix(desktop): preserve own state keys in task witnesses * test(desktop): preserve witness parse failure details * fix(desktop): retain own column keys in task conditions
This commit is contained in:
1 parent
617e4cbfb3
commit
cd96c5e9eb
5 files changed
+207
-21
No files matched your search
@@ -1,6 +1,7 @@
|
||||
import { isPortableBundleName } from "../portableBundleName.js";
|
||||
import { z } from "zod";
|
||||
|
||||
import { jsonRecordSchema } from "../jsonRecord.js";
|
||||
import { jsonValueSchema, type JsonValue } from "../jsonValue.js";
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
import { describeValidationFailure } from "../validationFailure.js";
|
||||
@@ -15,12 +16,10 @@ const line = z
|
||||
.refine((value) => !value.includes("\n"), "a line cannot hold a line break");
|
||||
const lines = z.array(line).min(1).max(100);
|
||||
const count = z.int().min(0);
|
||||
const where = z
|
||||
.record(z.string().min(1), jsonValueSchema)
|
||||
.refine(
|
||||
(columns) => Object.keys(columns).length > 0,
|
||||
"name at least one column",
|
||||
);
|
||||
const where = jsonRecordSchema(jsonValueSchema, z.string().min(1)).refine(
|
||||
(columns) => Object.keys(columns).length > 0,
|
||||
"name at least one column",
|
||||
);
|
||||
const comparison = {
|
||||
equals: count.optional(),
|
||||
at_least: count.optional(),
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { parseDesktopTask } from "./appTask.js";
|
||||
import { evaluateTaskChecks } from "./appTaskChecks.js";
|
||||
|
||||
it.each(["__proto__", "constructor", "ordinary"])(
|
||||
"retains the own row condition %s while grading every selected column",
|
||||
(column) => {
|
||||
const where = Object.fromEntries([
|
||||
["id", 1],
|
||||
[column, "wanted"],
|
||||
]);
|
||||
const common = { file: "db.sqlite", table: "rows", where };
|
||||
const parsed = parseDesktopTask({
|
||||
name: "find-row",
|
||||
prompt: "Find the row matching every selected column.",
|
||||
checks: [
|
||||
{ kind: "row_present", ...common },
|
||||
{ kind: "row_absent", ...common },
|
||||
{ kind: "row_count", ...common, equals: 0 },
|
||||
],
|
||||
});
|
||||
if (!parsed.ok) throw new Error(parsed.error);
|
||||
const first = parsed.value.checks[0];
|
||||
if (first?.kind !== "row_present") throw new Error("Expected a row check");
|
||||
expect(Object.keys(first.where)).toEqual(["id", column]);
|
||||
const views = new Map([
|
||||
[
|
||||
"db.sqlite",
|
||||
{
|
||||
format: "sqlite",
|
||||
view: {
|
||||
tables: {
|
||||
rows: {
|
||||
columns: ["id", column],
|
||||
primary_key: ["id"],
|
||||
rows: [[1, "other"]],
|
||||
truncated: false,
|
||||
},
|
||||
},
|
||||
tables_truncated: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
expect(
|
||||
evaluateTaskChecks(parsed.value.checks, views).map(
|
||||
({ passed }) => passed,
|
||||
),
|
||||
).toEqual([false, true, true]);
|
||||
expect(Object.keys(where)).toEqual(["id", column]);
|
||||
},
|
||||
);
|
||||
|
||||
it("accepts an own __proto__ condition as a nonempty row predicate", () => {
|
||||
const where = JSON.parse('{"__proto__":"wanted"}');
|
||||
const parsed = parseDesktopTask({
|
||||
name: "find-row",
|
||||
prompt: "Find the selected row.",
|
||||
checks: [{ kind: "row_present", file: "db.sqlite", table: "rows", where }],
|
||||
});
|
||||
if (!parsed.ok) throw new Error(parsed.error);
|
||||
const check = parsed.value.checks[0];
|
||||
if (check?.kind !== "row_present") throw new Error("Expected a row check");
|
||||
expect(Object.keys(check.where)).toEqual(["__proto__"]);
|
||||
expect(check.where["__proto__"]).toBe("wanted");
|
||||
});
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
type WitnessStageRecord,
|
||||
} from "./buildWitness.js";
|
||||
import type { ComputerAction } from "./computerActions.js";
|
||||
import { parseDesktopTask } from "./appTask.js";
|
||||
import { evaluateTaskChecks } from "./appTaskChecks.js";
|
||||
|
||||
type Logged = {
|
||||
readonly action: ComputerAction;
|
||||
@@ -351,6 +353,60 @@ describe("build witness over a long session", () => {
|
||||
});
|
||||
|
||||
describe("build witness state views", () => {
|
||||
it.each(["__proto__", "constructor", "ordinary"])(
|
||||
"grades the exact own state path %s instead of treating it as absent",
|
||||
(path) => {
|
||||
const view = JSON.parse('{"count":7,"__proto__":"retained value"}');
|
||||
const parts = [
|
||||
{
|
||||
kind: "data",
|
||||
data: {
|
||||
state: Object.fromEntries([[path, { format: "json", view }]]),
|
||||
},
|
||||
},
|
||||
];
|
||||
const task = parseDesktopTask({
|
||||
name: "check-state",
|
||||
prompt: "Check the selected file and its value.",
|
||||
checks: [
|
||||
{ kind: "file_present", file: path },
|
||||
{ kind: "file_absent", file: path },
|
||||
{ kind: "value_equals", file: path, pointer: "/count", value: 7 },
|
||||
],
|
||||
});
|
||||
const parsed = appStateViews(parts);
|
||||
if (!task.ok) throw new Error(task.error);
|
||||
if (!parsed.ok) throw new Error(parsed.error);
|
||||
expect([...parsed.value.keys()]).toEqual([path]);
|
||||
expect(parsed.value.get(path)).toEqual({
|
||||
format: "json",
|
||||
view,
|
||||
error: null,
|
||||
});
|
||||
expect(
|
||||
evaluateTaskChecks(task.value.checks, parsed.value).map(
|
||||
({ passed }) => passed,
|
||||
),
|
||||
).toEqual([true, false, true]);
|
||||
expect(Object.hasOwn(parts[0]!.data.state, path)).toBe(true);
|
||||
expect(Object.hasOwn(view, "__proto__")).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(
|
||||
["__proto__", "ordinary"].flatMap((path) => [
|
||||
{ path, entry: { format: 7, view: null } },
|
||||
{ path, entry: { format: "json" } },
|
||||
{ path, entry: { format: null, view: null, error: 7 } },
|
||||
]),
|
||||
)("rejects a malformed view under $path: $entry", ({ path, entry }) => {
|
||||
expect(
|
||||
appStateViews([
|
||||
{ kind: "data", data: { state: Object.fromEntries([[path, entry]]) } },
|
||||
]),
|
||||
).toMatchObject({ ok: false });
|
||||
});
|
||||
|
||||
it("reads the app's data/get state into the views the checks read, keeping each file's error", () => {
|
||||
expect(
|
||||
appStateViews([
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { jsonValueSchema } from "../jsonValue.js";
|
||||
import { jsonRecordSchema } from "../jsonRecord.js";
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
import type { TaskCheckOutcome, TaskFileView } from "./appTaskChecks.js";
|
||||
import type { ComputerAction } from "./computerActions.js";
|
||||
@@ -221,14 +222,17 @@ export const admitWitnessStage = (
|
||||
return ok(log.total);
|
||||
};
|
||||
|
||||
const fileViewSchema = z.looseObject({
|
||||
const fileViewSchema = z.object({
|
||||
format: z.string().nullable(),
|
||||
view: jsonValueSchema.nullable(),
|
||||
error: z.string().nullish(),
|
||||
error: z
|
||||
.string()
|
||||
.nullish()
|
||||
.transform((error) => error ?? null),
|
||||
});
|
||||
const statePartSchema = z.looseObject({
|
||||
kind: z.literal("data"),
|
||||
data: z.looseObject({ state: z.record(z.string(), fileViewSchema) }),
|
||||
data: z.looseObject({ state: jsonRecordSchema(fileViewSchema) }),
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -241,18 +245,7 @@ export const appStateViews = (
|
||||
for (const part of parts) {
|
||||
const parsed = statePartSchema.safeParse(part);
|
||||
if (parsed.success)
|
||||
return ok(
|
||||
new Map(
|
||||
Object.entries(parsed.data.data.state).map(([path, entry]) => [
|
||||
path,
|
||||
{
|
||||
format: entry.format,
|
||||
view: entry.view,
|
||||
error: entry.error ?? null,
|
||||
},
|
||||
]),
|
||||
),
|
||||
);
|
||||
return ok(new Map(Object.entries(parsed.data.data.state)));
|
||||
}
|
||||
return err(
|
||||
"The app's data/get returned no state views; witness a bundle exported by REA, whose app serves them.",
|
||||
|
||||
@@ -157,6 +157,78 @@ const expectUndecided = (
|
||||
for (const fact of [stage, ...facts]) expect(error.message).toContain(fact);
|
||||
};
|
||||
|
||||
it.each([
|
||||
{ kind: "file_present", passed: true },
|
||||
{ kind: "file_absent", passed: false },
|
||||
] as const)(
|
||||
"grades $kind against an own __proto__ state file over MCP",
|
||||
async ({ kind, passed }) => {
|
||||
const witness = await witnessed(
|
||||
0,
|
||||
{},
|
||||
{
|
||||
name: "selected-file",
|
||||
prompt: "Check the selected file.",
|
||||
checks: [{ kind, file: "__proto__" }],
|
||||
},
|
||||
);
|
||||
witness.app.state = JSON.parse(
|
||||
'{"__proto__":{"format":"json","view":{"count":7}}}',
|
||||
);
|
||||
const report = parsed(await witness.check({}));
|
||||
expect(report).toMatchObject({
|
||||
passed: passed ? 1 : 0,
|
||||
failed: passed ? 0 : 1,
|
||||
results: [{ id: "check-1", passed }],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([false, true])(
|
||||
"keeps own row conditions through MCP (only special column: %s)",
|
||||
async (onlySpecial) => {
|
||||
const where = onlySpecial
|
||||
? JSON.parse('{"__proto__":"other"}')
|
||||
: JSON.parse('{"id":1,"__proto__":"wanted"}');
|
||||
const common = { file: "db.sqlite", table: "rows", where };
|
||||
const witness = await witnessed(
|
||||
0,
|
||||
{},
|
||||
{
|
||||
name: "find-row",
|
||||
prompt: "Check every selected column of the row.",
|
||||
checks: [
|
||||
{ kind: "row_present", ...common },
|
||||
{ kind: "row_absent", ...common },
|
||||
{ kind: "row_count", ...common, equals: onlySpecial ? 1 : 0 },
|
||||
],
|
||||
},
|
||||
);
|
||||
witness.app.state = {
|
||||
"db.sqlite": {
|
||||
format: "sqlite",
|
||||
view: {
|
||||
tables: {
|
||||
rows: {
|
||||
columns: ["id", "__proto__"],
|
||||
primary_key: ["id"],
|
||||
rows: [[1, "other"]],
|
||||
truncated: false,
|
||||
},
|
||||
},
|
||||
tables_truncated: false,
|
||||
},
|
||||
truncated: false,
|
||||
},
|
||||
};
|
||||
const report = parsed(await witness.check({}));
|
||||
expect(report).toMatchObject({ passed: 2, failed: 1 });
|
||||
expect(report.results.map(({ passed }) => passed)).toEqual(
|
||||
onlySpecial ? [true, false, true] : [false, true, true],
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("witnesses a task: the seed fails, the edit passes, a near miss fails and so does the reset seed", async () => {
|
||||
const { app, computer, directory, check, act } = await witnessed();
|
||||
|
||||
|
||||
Reference in new issue
Block a user