fix(desktop): preserve own JSON keys in task grading (#10331)

* fix(desktop): preserve own state keys in task witnesses

* test(desktop): preserve witness parse failure details

* fix(desktop): retain own column keys in task conditions
This commit is contained in:
Kaoru0822 authored and GitHub committed 2026-11-13 06:07:06 -08:00
1 parent 617e4cbfb3
commit cd96c5e9eb
5 files changed
+207 -21

No files matched your search

+5 -6
View File
@@ -1,6 +1,7 @@
import { isPortableBundleName } from "../portableBundleName.js";
import { z } from "zod";
import { jsonRecordSchema } from "../jsonRecord.js";
import { jsonValueSchema, type JsonValue } from "../jsonValue.js";
import { err, ok, type Result } from "../result.js";
import { describeValidationFailure } from "../validationFailure.js";
@@ -15,12 +16,10 @@ const line = z
.refine((value) => !value.includes("\n"), "a line cannot hold a line break");
const lines = z.array(line).min(1).max(100);
const count = z.int().min(0);
const where = z
.record(z.string().min(1), jsonValueSchema)
.refine(
(columns) => Object.keys(columns).length > 0,
"name at least one column",
);
const where = jsonRecordSchema(jsonValueSchema, z.string().min(1)).refine(
(columns) => Object.keys(columns).length > 0,
"name at least one column",
);
const comparison = {
equals: count.optional(),
at_least: count.optional(),
+66
View File
@@ -0,0 +1,66 @@
import { expect, it } from "vitest";
import { parseDesktopTask } from "./appTask.js";
import { evaluateTaskChecks } from "./appTaskChecks.js";
it.each(["__proto__", "constructor", "ordinary"])(
"retains the own row condition %s while grading every selected column",
(column) => {
const where = Object.fromEntries([
["id", 1],
[column, "wanted"],
]);
const common = { file: "db.sqlite", table: "rows", where };
const parsed = parseDesktopTask({
name: "find-row",
prompt: "Find the row matching every selected column.",
checks: [
{ kind: "row_present", ...common },
{ kind: "row_absent", ...common },
{ kind: "row_count", ...common, equals: 0 },
],
});
if (!parsed.ok) throw new Error(parsed.error);
const first = parsed.value.checks[0];
if (first?.kind !== "row_present") throw new Error("Expected a row check");
expect(Object.keys(first.where)).toEqual(["id", column]);
const views = new Map([
[
"db.sqlite",
{
format: "sqlite",
view: {
tables: {
rows: {
columns: ["id", column],
primary_key: ["id"],
rows: [[1, "other"]],
truncated: false,
},
},
tables_truncated: false,
},
},
],
]);
expect(
evaluateTaskChecks(parsed.value.checks, views).map(
({ passed }) => passed,
),
).toEqual([false, true, true]);
expect(Object.keys(where)).toEqual(["id", column]);
},
);
it("accepts an own __proto__ condition as a nonempty row predicate", () => {
const where = JSON.parse('{"__proto__":"wanted"}');
const parsed = parseDesktopTask({
name: "find-row",
prompt: "Find the selected row.",
checks: [{ kind: "row_present", file: "db.sqlite", table: "rows", where }],
});
if (!parsed.ok) throw new Error(parsed.error);
const check = parsed.value.checks[0];
if (check?.kind !== "row_present") throw new Error("Expected a row check");
expect(Object.keys(check.where)).toEqual(["__proto__"]);
expect(check.where["__proto__"]).toBe("wanted");
});
+56
View File
@@ -8,6 +8,8 @@ import {
type WitnessStageRecord,
} from "./buildWitness.js";
import type { ComputerAction } from "./computerActions.js";
import { parseDesktopTask } from "./appTask.js";
import { evaluateTaskChecks } from "./appTaskChecks.js";
type Logged = {
readonly action: ComputerAction;
@@ -351,6 +353,60 @@ describe("build witness over a long session", () => {
});
describe("build witness state views", () => {
it.each(["__proto__", "constructor", "ordinary"])(
"grades the exact own state path %s instead of treating it as absent",
(path) => {
const view = JSON.parse('{"count":7,"__proto__":"retained value"}');
const parts = [
{
kind: "data",
data: {
state: Object.fromEntries([[path, { format: "json", view }]]),
},
},
];
const task = parseDesktopTask({
name: "check-state",
prompt: "Check the selected file and its value.",
checks: [
{ kind: "file_present", file: path },
{ kind: "file_absent", file: path },
{ kind: "value_equals", file: path, pointer: "/count", value: 7 },
],
});
const parsed = appStateViews(parts);
if (!task.ok) throw new Error(task.error);
if (!parsed.ok) throw new Error(parsed.error);
expect([...parsed.value.keys()]).toEqual([path]);
expect(parsed.value.get(path)).toEqual({
format: "json",
view,
error: null,
});
expect(
evaluateTaskChecks(task.value.checks, parsed.value).map(
({ passed }) => passed,
),
).toEqual([true, false, true]);
expect(Object.hasOwn(parts[0]!.data.state, path)).toBe(true);
expect(Object.hasOwn(view, "__proto__")).toBe(true);
},
);
it.each(
["__proto__", "ordinary"].flatMap((path) => [
{ path, entry: { format: 7, view: null } },
{ path, entry: { format: "json" } },
{ path, entry: { format: null, view: null, error: 7 } },
]),
)("rejects a malformed view under $path: $entry", ({ path, entry }) => {
expect(
appStateViews([
{ kind: "data", data: { state: Object.fromEntries([[path, entry]]) } },
]),
).toMatchObject({ ok: false });
});
it("reads the app's data/get state into the views the checks read, keeping each file's error", () => {
expect(
appStateViews([
+8 -15
View File
@@ -1,6 +1,7 @@
import { z } from "zod";
import { jsonValueSchema } from "../jsonValue.js";
import { jsonRecordSchema } from "../jsonRecord.js";
import { err, ok, type Result } from "../result.js";
import type { TaskCheckOutcome, TaskFileView } from "./appTaskChecks.js";
import type { ComputerAction } from "./computerActions.js";
@@ -221,14 +222,17 @@ export const admitWitnessStage = (
return ok(log.total);
};
const fileViewSchema = z.looseObject({
const fileViewSchema = z.object({
format: z.string().nullable(),
view: jsonValueSchema.nullable(),
error: z.string().nullish(),
error: z
.string()
.nullish()
.transform((error) => error ?? null),
});
const statePartSchema = z.looseObject({
kind: z.literal("data"),
data: z.looseObject({ state: z.record(z.string(), fileViewSchema) }),
data: z.looseObject({ state: jsonRecordSchema(fileViewSchema) }),
});
/**
@@ -241,18 +245,7 @@ export const appStateViews = (
for (const part of parts) {
const parsed = statePartSchema.safeParse(part);
if (parsed.success)
return ok(
new Map(
Object.entries(parsed.data.data.state).map(([path, entry]) => [
path,
{
format: entry.format,
view: entry.view,
error: entry.error ?? null,
},
]),
),
);
return ok(new Map(Object.entries(parsed.data.data.state)));
}
return err(
"The app's data/get returned no state views; witness a bundle exported by REA, whose app serves them.",
@@ -157,6 +157,78 @@ const expectUndecided = (
for (const fact of [stage, ...facts]) expect(error.message).toContain(fact);
};
it.each([
{ kind: "file_present", passed: true },
{ kind: "file_absent", passed: false },
] as const)(
"grades $kind against an own __proto__ state file over MCP",
async ({ kind, passed }) => {
const witness = await witnessed(
0,
{},
{
name: "selected-file",
prompt: "Check the selected file.",
checks: [{ kind, file: "__proto__" }],
},
);
witness.app.state = JSON.parse(
'{"__proto__":{"format":"json","view":{"count":7}}}',
);
const report = parsed(await witness.check({}));
expect(report).toMatchObject({
passed: passed ? 1 : 0,
failed: passed ? 0 : 1,
results: [{ id: "check-1", passed }],
});
},
);
it.each([false, true])(
"keeps own row conditions through MCP (only special column: %s)",
async (onlySpecial) => {
const where = onlySpecial
? JSON.parse('{"__proto__":"other"}')
: JSON.parse('{"id":1,"__proto__":"wanted"}');
const common = { file: "db.sqlite", table: "rows", where };
const witness = await witnessed(
0,
{},
{
name: "find-row",
prompt: "Check every selected column of the row.",
checks: [
{ kind: "row_present", ...common },
{ kind: "row_absent", ...common },
{ kind: "row_count", ...common, equals: onlySpecial ? 1 : 0 },
],
},
);
witness.app.state = {
"db.sqlite": {
format: "sqlite",
view: {
tables: {
rows: {
columns: ["id", "__proto__"],
primary_key: ["id"],
rows: [[1, "other"]],
truncated: false,
},
},
tables_truncated: false,
},
truncated: false,
},
};
const report = parsed(await witness.check({}));
expect(report).toMatchObject({ passed: 2, failed: 1 });
expect(report.results.map(({ passed }) => passed)).toEqual(
onlySpecial ? [true, false, true] : [false, true, true],
);
},
);
it("witnesses a task: the seed fails, the edit passes, a near miss fails and so does the reset seed", async () => {
const { app, computer, directory, check, act } = await witnessed();