From d2f9b8b7de6aeec86d0e818bf1d04cb43bbc22af Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 10 Nov 2026 02:26:28 -0800 Subject: [PATCH] feat(desktop): build several tasks from one capture, and measure each task's pass rate within a budget (#9216) * docs(desktop): clarify gzip scrub limits * fix(setup): own desktop build process cleanup * fix(doctor): compare desktop settings with resolved defaults * fix(desktop): refuse a build runtime whose export takes no task ends Several tasks pass --task-end to the build computer's export, so a runtime packed before it now fails at the start of the capture with the repack fix, as a runtime without synthetic seeds already did, rather than at export. Co-Authored-By: Claude Opus 5.5 * style(desktop): format the noisy-record draft files with the pinned oxfmt Co-Authored-By: Claude Opus 5.5 * test(desktop): give the root-array draft test its own describe The records describe grew past the 150-line lint limit with it. Co-Authored-By: Claude Opus 5.5 * fix(doctor): inspect client-recorded desktop runners * fix(doctor): reuse matching runner probes * fix(setup): propagate cancellation to desktop actions * fix(desktop): rewrite a download's launch keys written with spaces around = The desktop entry parser accepts `Exec = notes %F` and an indented `TryExec =notes`, so the install script now matches those keys the same way instead of leaving the original program in place. Adds the feature's tasks.md. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep why a measured run failed in its solve record A solve that returned an outcome dropped the runner's error, so a run that failed to deploy, solve or verify was retried with no reason recorded. The record, the measure result and difficulty.json now carry it. Co-Authored-By: Claude Opus 5.5 * fix(setup): stop Hopper on cancel, never report a cancelled setup ready, and keep a failed action's client setting - The Hopper install gets setup's signal: Linux already honoured it, and the macOS installer now stops its downloads and stops before mounting or copying. - runSetup checks the signal after the final doctor and close, so a cancellation there raises instead of returning ready. - A failed pack or agentenv install no longer writes its archive or runner to client configs, so the setting the client already had stays. Co-Authored-By: Claude Opus 5.5 * fix(setup): keep the completed setup result in a cancellation after cleanup A cancellation during the final doctor or host close carried only the applied actions, dropping each client's result and backup path and the doctor report. It now carries the whole result as setup_result, as doctor's cancellation carries its report. Co-Authored-By: Claude Opus 5.5 * test(desktop): move launch execution to boundary suite * fix(desktop): redact build schema failure output * refactor(desktop): normalize schema failure once * fix(desktop): omit unsafe schema parser snippets * fix(desktop): preserve draft change and selected path evidence * test(desktop): group retained dconf note regressions * fix(desktop): respect archive entry format in draft checks * test(desktop): separate unowned-file draft note cases * fix(desktop): preserve explicitly selected journal documents * fix(xml): avoid scanning selected ancestors per tag * fix(desktop): exclude covered text runs from draft counts * fix: preserve text when drafting desktop checks * fix: restore nested desktop state aliases * refactor: share UUID recognition across draft checks * fix: retain multiline XML attribute context * fix: retain relationship identifiers in task drafts * docs: clarify desktop draft identifier handling * test(desktop): type formatting run cases as pairs * fix(desktop): carry document format into selector evaluation * test(desktop): separate markup boundary regression group * fix(desktop): propagate formats through draft selectors * fix(desktop): switch selector format within archive entries * test(desktop): remove duplicate seed filename case * fix(setup): retain diagnostics when cancellation overlaps failure * refactor(setup): keep command cancellation at its owner * test(setup): verify cancellation at the application boundary * test(desktop): pass JSON format in exported index parity * docs(desktop): clarify task checks before scratch export * fix(desktop): respell selected paths in file URLs * test(desktop): group captured document spelling cases * test(desktop): run installer scripts at boundary * style(desktop): remove trailing blank line * fix(desktop): bound draft preview serialization * fix(desktop): restore selected state aliases * fix(desktop): respect entry format in draft noise * test(desktop): match exported alias evidence at its JSON boundary * fix(desktop): retain app state logs in exported seeds * fix: align desktop numeric seed comparisons * test(desktop): keep one draft exclusion fixture import * test(desktop): retain log eligibility across draft test groups * fix(doctor): resolve empty registered desktop runner * style: fix desktop launch test formatting * style: format native command path schema * fix(desktop): map selected external launch paths * style(desktop): format seed document regressions * test(desktop): group launch URL regressions * fix(desktop): omit redundant nested state aliases * fix(desktop): compare added free text as a whole * fix(desktop): check recompressed seeds as shipped * docs(desktop): clarify recovered seed format * test(desktop): group shipped gzip seed checks * fix: retain quoted identifier text in task drafts * fix: recognize Unicode XML in desktop state readers * fix(desktop): bound nested draft checks * fix(desktop): explain missing arrays in truncated views * fix(desktop): find XML type references in archives * test(desktop): remove duplicated capture assertions * fix(desktop): retain nested omitted text boundaries * style(desktop): format draft value imports * test(desktop): group model and draft value checks * fix(desktop): map captured launch paths once * refactor: remove unused whole-text guard state * refactor: stop passing unused text children * test: align free-text checks with whole-note policy * fix: validate recompressed desktop archive seeds * style(desktop): format archive seed regression * fix(desktop): retain placed-item sibling guards * fix(desktop): preserve archive paths and bound doctor reads * fix(doctor): refuse non-files before descriptor admission * fix(setup): track remaining cleanup resources across retries * test(desktop): specify XML format in heap pairing fixture * fix(setup): skip Hopper launch after cancelled copy * fix: preserve task measurement limits and cleanup * fix(desktop): retain reported solve sandbox cleanup * test(desktop): check retained log seed admission * fix(doctor): query agentenv for model endpoint * fix(doctor): preserve uncertainty for empty endpoint override * fix(desktop): redact runtime schema errors * fix(desktop): normalize captured file URLs * fix(desktop): report seeded files by capture path * fix(desktop): retain startup readiness failure stage * fix(desktop): retain JSON date checks * fix(desktop): preserve archive formatting checks * fix(desktop): preserve explicitly selected journal files * fix(desktop): report build command transport limits * docs(desktop): clarify unreadable dconf versions * test(desktop): remove duplicate history regression * fix(desktop): align XML names with exported reader * fix(desktop): create selected state alias targets * fix(desktop): inspect supported runtime archives * fix(desktop): keep ordinary text checks exact * fix(desktop): verify registered build host paths * fix(desktop): retain record identity in draft checks * test(desktop): preserve text reference numbers * fix(desktop): bound download install input reads * fix(desktop): match placed notes by whole text * test(desktop): match multiple task export results * fix(desktop): retain failed measurement file closes * fix(setup): retain completed actions on cancellation * test(setup): retain cancelled installer result * fix(desktop): scrub XML independently of reader compatibility * fix(desktop): preserve file aliases and reader diagnostics * style(desktop): format review fixes * fix(desktop): inspect archives with registered runner * test(desktop): verify registered archive interpreter * style(desktop): format capture review fixes * style(desktop): format review changes * feat(desktop): target agentenv-framework 0.9.1464 Desktop bundles, setup's --install-agentenv and the doctor's exact-version check now target 0.9.1464, which keeps an exported app's data/get call, holds the computer fixture above boot dialogs, leaves an unresolved setting's build unverified rather than failed, and mounts lock packages instead of copying them into the image. The web environment lane stays on 0.9.1431. Co-Authored-By: Claude Opus 5.5 * fix(desktop): retain per-solve sandbox cleanup roots * test(desktop): expect the solve cleanup helper among the shipped agentenv helpers Co-Authored-By: Claude Opus 5.5 * feat(desktop): clean up a stopped solve's sandboxes through agentenv 0.9.1482 agentenv-framework 0.9.1482 lists the local sandboxes under a root (local_sandbox_ids) and rebuilds one killed while it wrote its port map for teardown. The solve cleanup helper now terminates each sandbox agentenv lists, keeping its Docker and host-process checks, and reports anything else left under the root. Desktop bundles, setup's --install-agentenv and the doctor's version check target 0.9.1482. Co-Authored-By: Claude Opus 5.5 * fix(desktop): verify a killed solve's cleanup once its sandbox root is empty A solve's sandboxes are all local under its private root, and agentenv now keeps a sandbox's folder until its teardown is confirmed. So when the helper ends without a report and REA's cleanup empties that root, the cleanup is verified: a cancelled solve ends as a cancellation and a solve past its deadline as a failed run, rather than as cleanup the measure cannot vouch for. Co-Authored-By: Claude Opus 5.5 * feat(desktop): target agentenv-framework 0.9.1485 0.9.1485 keeps a local sandbox's folder until its teardown is confirmed, which the solve cleanup's verified result relies on. (0.9.1484 failed to publish.) Co-Authored-By: Claude Opus 5.5 * fix(desktop): record a solve stopped before it reports as failed, not as a refused request A helper killed from outside or stopped at its deadline left no report, so the measure refused with invalid input and told the caller to fix its model endpoint. A helper a signal ended now gives a failed solve of unknown cost, which the measure records and stops on; one that exits on its own without a report still ends the measure as a solve that could not run. Co-Authored-By: Claude Opus 5.5 * fix(desktop): check verifier behavior after selector changes * test(windows): verify rejected writes during cleanup by effect * fix(desktop): preserve selected alias target kinds in exports * fix(cli): retain caller capture paths in environment builds * fix(ghidra): classify oversized endpoint records consistently * test: align model and Evidence expectations with current contracts * fix(desktop): reconcile upstream sandbox cleanup recovery * fix(docs): link cleanup qualification as repository evidence --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../references/desktop-build.md | 150 ++++-- .../references/environments.md | 3 +- bridge/agentenv/solve.py | 121 +++++ bridge/agentenv/solve_cleanup.py | 63 +++ bridge/agentenv/test_solve_cleanup.py | 108 +++++ docs/desktop-application-export.md | 15 +- docs/desktop-application-tasks.md | 53 ++- docs/desktop-build-computer.md | 49 +- docs/desktop-build-witness.md | 89 +++- docs/specs/desktop-app-environments/README.md | 8 + .../desktop-app-environments/export/spec.md | 4 +- .../several-tasks/plan.md | 20 + .../several-tasks/spec.md | 53 +++ .../several-tasks/tasks.md | 8 + .../several-tasks/verification.md | 54 +++ .../cleanup-qualification-2026-11-10.json | 59 +++ .../task-difficulty/plan.md | 16 + .../task-difficulty/spec.md | 53 +++ .../task-difficulty/tasks.md | 8 + .../task-difficulty/verification.md | 90 ++++ .../desktop-app-environments/tasks/spec.md | 12 +- package.json | 2 + src/application/DesktopBuildDoctor.test.ts | 5 +- src/application/DesktopBuildDoctor.ts | 13 +- src/application/MacHopper.test.ts | 24 + src/application/Setup.desktopBuild.test.ts | 6 +- src/application/Setup.recovery.test.ts | 5 + .../desktop/DesktopApplicationExport.ts | 34 +- .../desktop/DesktopArchiveTask.test.ts | 4 +- .../desktop/DesktopBuildCapture.ts | 26 +- .../desktop/DesktopBuildComputers.ts | 71 ++- .../desktop/DesktopBuildExport.test.ts | 68 +-- src/application/desktop/DesktopBuildExport.ts | 68 ++- .../desktop/DesktopBuildOwnedWork.ts | 75 +++ .../desktop/DesktopBuildSolves.test.ts | 423 +++++++++++++++++ src/application/desktop/DesktopBuildSolves.ts | 430 ++++++++++++++++++ .../desktop/DesktopBuildSummary.ts | 188 +++++--- .../desktop/DesktopCheckpointEnds.test.ts | 6 +- .../desktop/DesktopExportIdentity.test.ts | 4 +- .../desktop/DesktopExportReaders.ts | 26 +- .../desktop/DesktopGzipDocument.test.ts | 6 +- .../desktop/DesktopSeveralTasks.test.ts | 314 +++++++++++++ .../desktop/DesktopTaskExport.test.ts | 32 +- src/application/desktop/DesktopTaskExport.ts | 375 ++++++++++----- .../desktop/desktopCapture.fixture.ts | 36 ++ src/cli/desktopCaptureCommands.ts | 9 +- src/contracts/desktopBuildMeasureContract.ts | 108 +++++ src/contracts/desktopBuildTaskSchemas.ts | 84 ++++ src/contracts/desktopBuildToolContracts.ts | 112 +++-- src/contracts/toolEffects.test.ts | 1 + src/contracts/toolEffects.ts | 9 + src/desktop/AgentEnvBridgeFiles.test.ts | 30 ++ src/desktop/AgentEnvComputerHelper.ts | 2 +- src/desktop/AgentEnvSolveHelper.test.ts | 265 +++++++++++ src/desktop/AgentEnvSolveHelper.ts | 354 ++++++++++++++ src/desktop/DesktopBuildDoctorFacts.test.ts | 140 +++++- src/desktop/DesktopBuildDoctorFacts.ts | 87 +++- src/desktop/DesktopBuildSetupHost.test.ts | 4 +- src/desktop/HostFileRead.ts | 3 + src/desktop/SolveSandboxRoot.test.ts | 25 + src/desktop/SolveSandboxRoot.ts | 133 ++++++ src/domain/analysisErrorBase.ts | 2 + src/domain/desktop/appExport.ts | 6 +- src/domain/desktop/appTask.ts | 2 +- src/domain/desktop/appTaskChecks.fixture.ts | 30 ++ src/domain/desktop/appTaskDraftItems.test.ts | 19 +- src/domain/desktop/appTaskDraftItems.ts | 33 +- src/domain/desktop/appTaskDraftJson.test.ts | 16 + .../desktop/appTaskDraftLineGuards.test.ts | 25 + src/domain/desktop/appTaskPythonPointer.ts | 5 +- src/domain/desktop/appTaskReaders.test.ts | 47 -- src/domain/desktop/taskDifficulty.test.ts | 194 ++++++++ src/domain/desktop/taskDifficulty.ts | 196 ++++++++ src/domain/desktop/viewPointer.ts | 7 +- src/ghidra/GhidraTransport.ts | 54 +-- src/server/registerDesktopBuildTools.ts | 8 +- tests/acceptance/modelArtifactCli.test.ts | 1 + .../application/buildExportGzipScans.test.ts | 2 +- tests/boundary/cli/cliToolCoverage.test.ts | 1 + tests/boundary/mcp/buildCaptureMcp.test.ts | 38 +- .../boundary/mcp/buildCaptureRetryMcp.test.ts | 8 +- .../mcp/buildCaptureStatePathsMcp.test.ts | 6 +- tests/boundary/mcp/buildMeasureMcp.test.ts | 225 +++++++++ tests/boundary/mcp/buildSecretsMcp.test.ts | 14 +- .../boundary/mcp/buildSeveralTasksMcp.test.ts | 137 ++++++ tests/boundary/mcp/buildWithheldMcp.test.ts | 7 +- .../providers/desktopTaskPython.test.ts | 2 +- .../providers/ida/calleeLookupBatch.test.ts | 11 +- tests/fixtures/buildCaptureFakes.ts | 37 +- tests/fixtures/fakeAgentEnvSolves.ts | 123 +++++ tests/fixtures/fakeBuildComputerRea.ts | 7 +- 91 files changed, 5278 insertions(+), 628 deletions(-) create mode 100644 bridge/agentenv/solve.py create mode 100644 bridge/agentenv/solve_cleanup.py create mode 100644 bridge/agentenv/test_solve_cleanup.py create mode 100644 docs/specs/desktop-app-environments/several-tasks/plan.md create mode 100644 docs/specs/desktop-app-environments/several-tasks/spec.md create mode 100644 docs/specs/desktop-app-environments/several-tasks/tasks.md create mode 100644 docs/specs/desktop-app-environments/several-tasks/verification.md create mode 100644 docs/specs/desktop-app-environments/task-difficulty/cleanup-qualification-2026-11-10.json create mode 100644 docs/specs/desktop-app-environments/task-difficulty/plan.md create mode 100644 docs/specs/desktop-app-environments/task-difficulty/spec.md create mode 100644 docs/specs/desktop-app-environments/task-difficulty/tasks.md create mode 100644 docs/specs/desktop-app-environments/task-difficulty/verification.md create mode 100644 src/application/desktop/DesktopBuildOwnedWork.ts create mode 100644 src/application/desktop/DesktopBuildSolves.test.ts create mode 100644 src/application/desktop/DesktopBuildSolves.ts create mode 100644 src/application/desktop/DesktopSeveralTasks.test.ts create mode 100644 src/contracts/desktopBuildMeasureContract.ts create mode 100644 src/contracts/desktopBuildTaskSchemas.ts create mode 100644 src/desktop/AgentEnvBridgeFiles.test.ts create mode 100644 src/desktop/AgentEnvSolveHelper.test.ts create mode 100644 src/desktop/AgentEnvSolveHelper.ts create mode 100644 src/desktop/SolveSandboxRoot.test.ts create mode 100644 src/desktop/SolveSandboxRoot.ts create mode 100644 src/domain/desktop/taskDifficulty.test.ts create mode 100644 src/domain/desktop/taskDifficulty.ts create mode 100644 tests/boundary/mcp/buildMeasureMcp.test.ts create mode 100644 tests/boundary/mcp/buildSeveralTasksMcp.test.ts create mode 100644 tests/fixtures/fakeAgentEnvSolves.ts diff --git a/.agents/skills/reverse-engineer-anything/references/desktop-build.md b/.agents/skills/reverse-engineer-anything/references/desktop-build.md index e5a8cc7e7a..e098e0ac4a 100644 --- a/.agents/skills/reverse-engineer-anything/references/desktop-build.md +++ b/.agents/skills/reverse-engineer-anything/references/desktop-build.md @@ -32,16 +32,16 @@ start without asking: 1. The app and the task, if either is unclear: what changes, in which document. 2. Whether they have a starting file of their own; otherwise you make one. -3. Whether they want solver runs after the witness: how many (the budget) - and with which model. -4. If so, whether they want to review the task before those runs. +3. Whether they want the tasks measured after the witness: with which + model, within which budget, and whether to harden a task until it lands + in the band. +4. If so, whether they want to review the tasks before those solves. When you cannot ask, as in a headless `claude -p` run, do not stop: take these defaults for whatever the request leaves open, say which you took in your first message and in the report, and go on. Install from the Ubuntu -archive, make a synthetic seed, skip the review, and make 3 solver runs -within $4 in all, with the model the request names or else the runner's -default. +archive, make a synthetic seed, skip the review, and measure each task +within $4 in all, with the model the request names, without hardening. ## When you are blocked @@ -113,10 +113,31 @@ the first source that has the app: to add a package. Ask the requester for its URL and SHA-256; do not fetch a vendor's download page yourself. List its `Depends` as `dependencies`, with any runtime piece it lacks: it installs with the network off. -- Flatpak, AppImage and snaps are not supported yet: say so and stop. An - archive package that only installs a snap, such as noble's `firefox`, - counts as a snap. A package with no build for this host's architecture - (some PPAs are amd64 only) cannot be installed either. +- Take an AppImage or a tarball only when no package has the app. Pin it by + the release file's URL for this host's architecture and the SHA-256 its + project publishes, such as a GitHub release asset's digest or the + release's `SHA256SUMS`; ask the requester when the project publishes none. + The computer downloads it when the bundle is put, refuses other bytes, and + unpacks it under `/opt/rea-apps/` with the network off; the + export downloads the same file. +- An AppImage installs its own desktop entry, so `desktop_entry` is the + `.desktop` file at the top of the image, such as FreeCAD's + `org.freecad.FreeCAD`. A wrong name fails the start with the entries the + image has: start again with one of them. The image brings its libraries, + so `extra_packages` is rarely needed; FreeCAD 1.1.4's needs none. +- A tarball has no entry of its own. `program` is the program's path inside + the archive, such as `firefox/firefox`; REA writes an entry named + `desktop_entry` that runs it with the files it opens, and `name` is what + the Applications menu shows. A tarball uses the computer's libraries: name + what its program lacks in `extra_packages`. gzip, bzip2 and xz archives + are read, zstd ones are not. +- Flatpak and snaps are not supported: say so and stop. Flatpak's sandbox + needs user namespaces the computer's container does not grant, so a + Flathub app installs but never opens; ask for the project's AppImage, + tarball or `.deb` when it ships one. An archive package that only installs + a snap, such as noble's `firefox`, counts as a snap. A package with no + build for this host's architecture (some PPAs are amd64 only) cannot be + installed either. - `desktop_entry` is the package's `.desktop` file name under `/usr/share/applications` without `.desktop`, such as `org.inkscape.Inkscape`. Add `window_class` when you know it; Inkscape's @@ -373,10 +394,22 @@ the first source that has the app: 4. Only once the seed is saved, `mark_build_checkpoint` with name `seed`. 5. Make the change the task asks for, as a user would, and save it. 6. Only once the change is saved, `mark_build_checkpoint` with name `end`. + For more tasks in this environment, make each next task's change on top + of the last as a user would, save it, and mark `end-`, such as + `end-add-bread`. Each task is checked at its own end, so its checks must + not read what a later task changes, and a solving agent starts each from + the seed, so a task must not need an earlier one done first. + Then, for each task you can, make a near miss of its change on top, a + plausible wrong edit such as a typo or a wrong number, save it and mark + `near-`: the export refuses a task whose near miss passes every + check. Take the near misses after the last end, since each later draft + starts from the end before it. 7. `stop_build_capture`. Read `draft.files` (each file that changed, with its net change and its `values`), `draft.task.checks` (suggested - checks), `draft.guards`, `draft.notes` and `limitations`. When `draft` - is null, read `draft_refusal` first: a missing `seed` or `end` + checks), `draft.guards`, `draft.notes` and `limitations`; each + `later_drafts` entry is the same for one later task, from the end + checkpoint before its own, so it shows only that task's change. When + `draft` is null, read `draft_refusal` first: a missing `seed` or `end` checkpoint means capturing again on a new build computer, while a failed draft command names its own cause to fix. If a declared secret was used, automatic drafting is withheld: write the task manually and export it. @@ -420,7 +453,8 @@ the first source that has the app: ## Write the task -Write `{"name", "prompt", "checks"}` from the draft and what you did. +Write `{"name", "prompt", "checks"}` from the draft and what you did, one +per task, each with its own name. - `name`: lowercase letters, digits and hyphens, not `smoke`. - `prompt`: what the requester wants, in their words and the app's terms, @@ -650,16 +684,22 @@ on a screenshot three times, and the check passes when most answers are yes. ## Export -`export_build_environment` with `task` and `directory`, the absolute path of -a new or empty folder here, such as `/rea-builds/poster/bundle`. The +`export_build_environment` with `tasks` and `directory`, the absolute path of +a new or empty folder here, such as `/rea-builds/poster/bundle`. Each +entry of `tasks` is `{"task": ..., "end_checkpoint": ...}`: leave +`end_checkpoint` out for the task checked at `end`, and give +`end-` for each later one. The export seeds what was saved at `seed`, replaces the computer's host name, `user@host` and home path (a bare user name, such as AbiWord's author -`user`, stays), and checks the task against `seed` and `end`. It refuses a task -unless at least one check fails on the seed and every check passes at the -end, and says why, such as a pointer that names nothing at the end. A +`user`, stays), and checks each task against `seed`, its own end and its +`near-` when you took one. It refuses a task +unless at least one check fails on the seed, every check passes at its +end and one fails at its near miss, and says which task and why, such as a +pointer that names nothing at the end. A check that the rest is untouched passes on the seed too, which is fine. On a refusal, fix the task and export again, into another folder if that one -is no longer empty. A refusal naming a secret's handle means a file of the +is no longer empty. The bundle's eval lists every task, so a run of the +bundle runs them all. A refusal naming a secret's handle means a file of the bundle holds that secret, most often an app's sign-in or licence state in the seed: do not work around it, but report it and stop. Keep `seed_files`, `scrubbed_files`, `unscrubbed_files` and `limitations` for the @@ -694,8 +734,9 @@ again from it. ## Witness -Witness in a fresh computer of the bundle, from the prompt, as a solving -agent would. Do not replay the build's actions or coordinates, and do not +Witness each task in its own fresh computer of the bundle, from the prompt, +as a solving agent would; a computer is witnessed once, so the steps below +repeat for each task, each with its own record folder. Do not replay the build's actions or coordinates, and do not check between stages to steer the edit: a solving agent never sees the checks. Give every `check_build_task` the task exactly as exported. @@ -742,21 +783,55 @@ result and decide: - `after_reset` passed: the reset did not restore the seed. Report it as a limitation. -Stop after three witnesses and report what still fails. +Stop after three witnesses of a task and report what still fails. -## Solver runs +## Measure -Only if the requester asked for them, or by the headless default above. If -they wanted to review the task first, show them its prompt and checks and -wait for their approval. Then run the bundle within the budget, one run at a -time, with `REA_AGENT_ENV_COMMAND`'s runner, or else `agent-env` on PATH: -`agent-env run --task --agent claude-code --model `. -The runner needs a model endpoint in your shell: `LITELLM_BASE_URL` and -`LITELLM_API_KEY`, or `base_url` and `api_key` under `[model]` in -`.agentenv/config.toml`. When a run says no model endpoint is set, tell the -person what to set and stop. Count runs reported `passed`. A run -`scored below 1` is a miss; one reported `failed` did not finish, so run it -again if the budget allows instead of counting it. Never run past the budget. +Only if the requester asked for it, or by the headless default above. If +they wanted to review the tasks first, show them each prompt and checks and +wait for their approval. Then `measure_build_task` each task with `bundle`, +`task`, `model` and `budget_usd`, its share of the budget. It runs the task +in fresh computers, 3 times, then 2 more unless they passed none or all, and +keeps a task that passes 1 to 3 of 5 (`band: "in_band"`). It stops before a +solve the largest cost so far would take past the budget, and writes the +record into the bundle's `difficulty.json` and README. The solver needs a +model endpoint where REA starts: `LITELLM_BASE_URL` and `LITELLM_API_KEY`, or +`base_url` and `api_key` under `[model]` in `.agentenv/config.toml`; when the +measure says a solve did not run for want of one, tell the person what to +set and stop. Read the result's `failures_per_check` and each solve's +`failed_checks` before calling a task hard: a check that fails in most failed +solves while the app shows the right change is brittle, so fix the check, +export and witness again, rather than count it as difficulty. + +## Harder tasks + +When the requester wants tasks worth evaluating or training on, harden a +task that came out `too_easy` and measure it again, within the budget. The +solver sees and fixes its slips on screen, so one edit with the value in the +prompt is solved almost every time; so far, on TreeLine with Sonnet 5.5, so +were one more dialog, a conditional move of seven records, and ten +corrections read from a second document and applied across the list. Try, +one at a time, measuring each: + +- A longer chain of exact edits than that, each a value the checks compare + whole, so one slip in the chain fails the task. +- Facts read in one view and used in another: values the solver must read + from a second document the app opens, or from another record, rather than + from the prompt, such as "the same year as The Quiet Glacier's". +- Distractors beside each target: records with near-identical names or + values, and a guard check on each that it stays as it was. +- Mistakes that do not show once saved: a value the app displays rounded or + formatted unlike what it stores, or an edit the app keeps only when its + field loses focus. + +Stop when the budget left cannot cover 3 solves of the hardened task, and +report the bands measured. + +Each hardened task stays sound: its checks still fail on the seed and pass on +its end, its near miss fails, and another valid way of doing it passes, so +check values, never the order or method the solver used. Put the second +document in the seed. Count records with `member_count` and check each +value by a selector that names its record, since no check joins two records. ## Close and report @@ -767,10 +842,10 @@ is left. Report: - the bundle path and the environment name; -- the task's name and prompt; +- each task's name, prompt and end checkpoint; - each check, and why it proves the edit and fails the near miss, and each reader's command; -- each witness stage with its result, the verdict, and the paths of +- each task's witness stages with their results, the verdict, and the paths of `witness.json` and the recording; - the seed's contents: `seed_files` and what each holds; - `scrubbed_files` and `unscrubbed_files`; an unscrubbed file still names @@ -778,7 +853,8 @@ Report: are gzip seeds whose headers were cleared, their content unchanged: give their count; - limitations: the capture's, the export's and anything you could not do; -- solver runs, if any: the model, the runs and the passes; +- each measured task's band, passes of finished solves, turns, cost and + failures per check, from `difficulty.json`; - the build time and the questions you asked. Never publish or share the bundle yourself. diff --git a/.agents/skills/reverse-engineer-anything/references/environments.md b/.agents/skills/reverse-engineer-anything/references/environments.md index 914c7eadfa..9b184c851f 100644 --- a/.agents/skills/reverse-engineer-anything/references/environments.md +++ b/.agents/skills/reverse-engineer-anything/references/environments.md @@ -381,7 +381,8 @@ Write the task as JSON with a `name`, the `prompt` and the `checks`, then run the task only when its checks fail on the seed and all pass on the captured end state. A refusal names the check: one that fails on the end state does not match what the user left; "every check already passes on the seed" means no -check captures the change. +check captures the change. For several tasks from one capture, repeat +`--task` and name each later task's end with `--task-end NAME=CHECKPOINT`. When the export refuses an application that no apt repository installs, ask the user for the `.deb` file they installed it from and capture with diff --git a/bridge/agentenv/solve.py b/bridge/agentenv/solve.py new file mode 100644 index 0000000000..1658e4c15f --- /dev/null +++ b/bridge/agentenv/solve.py @@ -0,0 +1,121 @@ +"""Run one solve of a bundle's task in a fresh agentenv computer for REA and print how it ended. + +REA runs ``solve.py `` with the agentenv runner's own interpreter, so +``agent_env`` imports. It runs the task once on this machine with ``run_bundle``, the solver deployed as the agent +and asked on the model, the endpoint taken from the environment or agentenv's config as a run takes it, and the +computer torn down as the run ends. It prints one JSON line: the run's outcome, instance id, seconds, each check's +result, the solver's turns and cost from its trajectory when that is a local file, and agentenv's version; or +``{"error": ...}`` when the task could not run. Everything else, agentenv's progress included, goes to stderr. It exits +0 after a run and 2 when none ran. +""" + +from __future__ import annotations + +import json +import os +import sys +from importlib import metadata +from pathlib import Path + +#: The most characters of a check's description or a run's error REA keeps. +MAX_TEXT = 400 + + +def _protocol(): + """A private copy of stdout for the JSON line; fd 1 then points at stderr, as do children that inherit it.""" + protocol = os.fdopen(os.dup(1), "w", encoding="utf-8") + os.dup2(2, 1) + sys.stdout = sys.stderr + return protocol + + +def _say(protocol, value: dict) -> None: + protocol.write(json.dumps(value) + "\n") + protocol.flush() + + +def _field(value, name: str): + """A field of agentenv's dataclasses, or of the dicts a stored context holds.""" + return value.get(name) if isinstance(value, dict) else getattr(value, name, None) + + +def _trajectory_end(context) -> tuple: + """The solver's turns and cost from the last record of its local trajectory, or None for each.""" + for response in reversed(_field(context, "prompt_responses") or []): + url = _field(response, "agent_trajectory_object_url") or "" + if not url.startswith("file://"): + continue + try: + records = json.loads(Path(url[len("file://"):]).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None, None + last = records[-1] if isinstance(records, list) and records and isinstance(records[-1], dict) else {} + turns, usd = last.get("num_turns"), last.get("total_cost_usd") + return (turns if isinstance(turns, int) else None, + float(usd) if isinstance(usd, (int, float)) and not isinstance(usd, bool) else None) + return None, None + + +def _checks(context, task: str) -> list: + """Each check's result as the task's verifier recorded it under the instance's verifications.""" + verifications = (_field(context, "metadata") or {}).get("verifications") or {} + found = verifications.get(task) or next(iter(verifications.values()), {}) + results = found.get("results") if isinstance(found, dict) else None + return [{"id": str(result.get("id")), "passed": result.get("result") is True, + "description": str(result.get("description", ""))[:MAX_TEXT]} + for result in results or [] if isinstance(result, dict)] + + +def _version() -> str | None: + try: + return metadata.version("agentenv-framework") + except metadata.PackageNotFoundError: + return None + + +def _cleanup(result) -> dict: + """Only a returned teardown report can confirm what agentenv released.""" + if result is None: + return {"unknown": True, "sandboxes": []} + unknown, sandboxes = False, [] + for run in result.runs: + report = getattr(run, "torn_down", None) + if report is None: + unknown = True + else: + sandboxes.extend({"id": item.sandbox_id, "type": item.sandbox_type} for item in report.still_up) + return {"unknown": unknown, "sandboxes": sandboxes} + + +def main(argv: list[str]) -> int: + protocol = _protocol() + bundle, task, model, agent = argv[1:5] + try: + from agent_env.bundle import run_bundle + except ImportError as error: + _say(protocol, {"error": f"agent_env does not import with {sys.executable}: {error}", + "cleanup": {"unknown": False, "sandboxes": []}}) + return 2 + try: + result = run_bundle(bundle, tasks=[task], model=model, agent=agent, sandbox="local", emulate=False, + on_progress=lambda line: print(line, file=sys.stderr, flush=True)) + except BaseException as error: # noqa: BLE001 - a signal's RunInterrupted too; run_bundle tore its run down + _say(protocol, {"error": f"{type(error).__name__}: {error}"[:MAX_TEXT * 4], + "cleanup": _cleanup(getattr(error, "result", None))}) + return 2 + run = result.runs[0] if result.runs else None + if run is None: + _say(protocol, {"error": "the bundle ran no task", "cleanup": _cleanup(result)}) + return 2 + turns, usd = _trajectory_end(run.context) + error = getattr(run, "error", None) + _say(protocol, {"outcome": run.outcome.value, "instance_id": run.instance_id, + "seconds": round(float(run.duration), 1), "checks": _checks(run.context, task), + "turns": turns, "usd": usd, "agentenv": _version(), + "error": None if error is None else f"{type(error).__name__}: {error}"[:MAX_TEXT], + "cleanup": _cleanup(result)}) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/bridge/agentenv/solve_cleanup.py b/bridge/agentenv/solve_cleanup.py new file mode 100644 index 0000000000..3398098624 --- /dev/null +++ b/bridge/agentenv/solve_cleanup.py @@ -0,0 +1,63 @@ +"""Recover local sandboxes under one stopped solve's private agentenv root.""" +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +import subprocess +import sys +from pathlib import Path + + +def _docker(*args: str) -> str: + command = os.environ.get("REA_DOCKER_COMMAND") or "docker" + result = subprocess.run([command, *args], capture_output=True, text=True, check=False) + if result.returncode: + raise RuntimeError(f"Docker {' '.join(args[:2])} failed: {result.stderr[-1000:]}") + return result.stdout.strip() + + +async def cleanup(root: Path) -> None: + # A helper has already stopped. agentenv lists the sandboxes under the root + # (AGENT_ENV_LOCAL_SANDBOX_DIR); anything else there remains owned for inspection. + if not any(root.iterdir()): + return + from agent_env.providers.sandbox_providers.local_docker.provider import ( + LocalSandboxProvider, local_sandbox_ids, remove_local_work_dir, + ) + for sandbox_id in local_sandbox_ids(): + await LocalSandboxProvider().terminate_sandbox(sandbox_id) + # The pinned provider reports incomplete teardown and retains the folder. + # Keep REA's independent Docker/Linux checks before releasing that owner. + if _docker("ps", "-aq", "--filter", f"name=^agent-{sandbox_id}$"): + raise RuntimeError(f"Sandbox {sandbox_id} still has its agent container") + if _docker("ps", "-aq", "--filter", f"label=agentenv.sandbox={sandbox_id}"): + raise RuntimeError(f"Sandbox {sandbox_id} still has labeled containers") + if _docker("network", "ls", "-q", "--filter", f"label=agentenv.sandbox={sandbox_id}"): + raise RuntimeError(f"Sandbox {sandbox_id} still has labeled networks") + if sys.platform != "linux": + raise RuntimeError(f"Cannot verify host processes for sandbox {sandbox_id} on {sys.platform}") + from agent_env.providers.sandbox_providers.local_docker.provider import _marked_pids + if _marked_pids(sandbox_id): + raise RuntimeError(f"Sandbox {sandbox_id} still has marked host processes") + if remove_local_work_dir(sandbox_id) is None: + raise RuntimeError(f"Could not confirm removal of sandbox {sandbox_id}'s work folder") + leftover = sorted(entry.name for entry in root.iterdir()) + if leftover: + raise RuntimeError(f"Unexpected sandbox root entries: {', '.join(leftover)}") + + +def main() -> int: + try: + with contextlib.redirect_stdout(sys.stderr): + asyncio.run(cleanup(Path(os.environ["AGENT_ENV_LOCAL_SANDBOX_DIR"]))) + print(json.dumps({"ok": True})) + return 0 + except Exception as error: + print(json.dumps({"ok": False, "error": f"{type(error).__name__}: {error}"})) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bridge/agentenv/test_solve_cleanup.py b/bridge/agentenv/test_solve_cleanup.py new file mode 100644 index 0000000000..d0ebdd453b --- /dev/null +++ b/bridge/agentenv/test_solve_cleanup.py @@ -0,0 +1,108 @@ +"""Exercise cleanup decisions without a live agentenv provider.""" +import asyncio +import tempfile +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +from bridge.agentenv import solve_cleanup + + +class SolveCleanupTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.folder = self.root / "agent-env-local-a1b2c3d4-abcdef12" + self.folder.mkdir() + self.terminated = [] + self.module = types.ModuleType("agent_env.providers.sandbox_providers.local_docker.provider") + test = self + + class Provider: + async def terminate_sandbox(self, sandbox_id): + test.terminated.append(sandbox_id) + + self.module.LocalSandboxProvider = Provider + self.module.local_sandbox_ids = self._listed + self.module.remove_local_work_dir = lambda sandbox_id: self._remove(sandbox_id) + self.module._marked_pids = lambda _sandbox_id: [] + self.patch_module = patch.dict("sys.modules", {self.module.__name__: self.module}) + self.patch_module.start() + self.addCleanup(self.patch_module.stop) + + def _listed(self): + # As agentenv lists them: `agent-env--` folders, whatever the id's shape. + found = set() + for entry in self.root.iterdir(): + sandbox_id, _, suffix = entry.name.removeprefix("agent-env-").rpartition("-") + if entry.name.startswith("agent-env-") and sandbox_id and suffix and entry.is_dir(): + found.add(sandbox_id) + return sorted(found) + + def _remove(self, sandbox_id): + folder = next(self.root.glob(f"agent-env-{sandbox_id}-*")) + folder.rmdir() + return folder + + def run_cleanup(self): + asyncio.run(solve_cleanup.cleanup(self.root)) + + def test_terminates_owned_sandbox_before_removing_its_folder(self): + with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): + self.run_cleanup() + self.assertEqual(self.terminated, ["local-a1b2c3d4"]) + self.assertFalse(self.folder.exists()) + + def test_retains_folder_when_docker_reports_container_then_retries(self): + with patch.object(solve_cleanup, "_docker", return_value="still-running"), patch.object(solve_cleanup.sys, "platform", "linux"): + with self.assertRaisesRegex(RuntimeError, "still has its agent container"): + self.run_cleanup() + self.assertTrue(self.folder.exists()) + with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): + self.run_cleanup() + self.assertFalse(self.folder.exists()) + self.assertEqual(len(self.terminated), 2) + + def test_partial_cleanup_and_late_folder_are_retried(self): + later = self.root / "agent-env-local-f1223344-abcdef12" + later.mkdir() + def docker(*args): + return "still-running" if "local-f1223344" in " ".join(args) else "" + with patch.object(solve_cleanup, "_docker", side_effect=docker), patch.object(solve_cleanup.sys, "platform", "linux"): + with self.assertRaisesRegex(RuntimeError, "still has its agent container"): + self.run_cleanup() + self.assertTrue(later.exists()) + self.assertFalse(self.folder.exists()) + with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): + self.run_cleanup() + self.assertEqual(list(self.root.iterdir()), []) + + def test_macos_keeps_folder_when_host_process_cleanup_cannot_be_verified(self): + with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "darwin"): + with self.assertRaisesRegex(RuntimeError, "Cannot verify host processes"): + self.run_cleanup() + self.assertTrue(self.folder.exists()) + + def test_cleans_up_each_sandbox_agentenv_lists_whatever_its_id_looks_like(self): + self.folder.rmdir() + listed = self.root / "agent-env-local-a1b2c3d4e5-abcdef12" + listed.mkdir() + with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): + self.run_cleanup() + self.assertEqual(self.terminated, ["local-a1b2c3d4e5"]) + self.assertFalse(listed.exists()) + + def test_unexpected_entry_remains_owned(self): + self.folder.rmdir() + (self.root / "unexpected").mkdir() + with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): + with self.assertRaisesRegex(RuntimeError, "Unexpected sandbox root entries: unexpected"): + self.run_cleanup() + self.assertEqual(self.terminated, []) + self.assertTrue((self.root / "unexpected").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/desktop-application-export.md b/docs/desktop-application-export.md index ec0e136bc7..1aac6f86e9 100644 --- a/docs/desktop-application-export.md +++ b/docs/desktop-application-export.md @@ -20,8 +20,10 @@ part (`org.xfce.mousepad` gives `mousepad`), or else the whole id without a trailing version (`glabels-3.0` gives `glabels`), with an `app-` prefix when that name would not be a portable bundle name (for example all digits). The output directory must be new or empty. `--task` adds a task an agent wrote for -the capture, once REA has validated it against the capture; see -[desktop application tasks](desktop-application-tasks.md). `--package-url` +the capture, once REA has validated it against the capture; repeat it for +several tasks, each checked at its own end checkpoint, which +`--task-end NAME=CHECKPOINT` names; see +[desktop application tasks](desktop-application-tasks.md#several-tasks). `--package-url` says where the computer downloads the package of an application installed from a `.deb` file; see [Applications from a .deb file](#applications-from-a-deb-file). `--download-install FILE` says which AppImage or tarball a build computer @@ -70,7 +72,7 @@ Evidence; the bundle itself is complete. | `envs//install-download.sh` | With `--download-install`: the run script that unpacks the downloaded AppImage or tarball | | `envs//applications/` | With a tarball's `--download-install`: the desktop entry REA wrote for its program | | `tasks/smoke.json` | A task that deploys the app | -| `tasks/.json` | With `--task`: the agent's task, its verifier in `artifacts/` and the eval in `evals/` | +| `tasks/.json` | With `--task`: each of the agent's tasks, its verifier in `artifacts/` and the eval in `evals/` | | `README.md` | What was exported, from which capture, and what to keep in mind | The class file is `_app.py` with hyphens as underscores, for example @@ -178,7 +180,9 @@ The class declares: unless that spelling is outside the captured home, as when the home itself is a symbolic link. When a selected link names an application state directory, the bundle seeds the application's state path and restores the selected - alias at boot. Selections nested under that alias are seeded under the same + alias at boot. Targets recorded as directories are created even when the seed + has no files there; file targets and targets of unknown kind are not turned + into directories. Selections nested under that alias are seeded under the same state path. An existing incompatible alias stops setup without replacing it. When two files map to one home path, the export uses the one present before the capture, else the first by path, and a limitation names the other. - Data hooks, because declaring any hook switches off agentenv's default data @@ -599,5 +603,4 @@ when a DOCTYPE names a DTD. Known differences remain: Unless it was [made for the task](#a-seed-made-for-the-task), the seed is the user's real data. Keep the bundle local and do not publish it. -The class was checked against agentenv-framework 0.9.1464 with -`python -m agentenv_protocol.computer._declare` and `agent-env run --dry-run`. +The class targets agentenv-framework 0.9.1485. diff --git a/docs/desktop-application-tasks.md b/docs/desktop-application-tasks.md index 1ab4c9942f..9ccb2247a3 100644 --- a/docs/desktop-application-tasks.md +++ b/docs/desktop-application-tasks.md @@ -330,7 +330,7 @@ operation `describe_desktop_application_task` and predicate type - An element placed by its own position, holding numeric `x` and `y` as a like sibling does, as Xournal++ places each `` note on a layer, is an item of its own, never a run of the others' text: one added whole - gets a `text_contains` of its text on the element holding the items, + gets a `member_count` matching its whole text with whitespace collapsed, which passes wherever it is placed, and that list's `member_count`. - Noise leaves the suggestions when the file's change holds anything else, a change only position reaches included: view state, which a tag @@ -444,7 +444,7 @@ operation `describe_desktop_application_task` and predicate type a line with no text of its own, a command such as `\end_layout`, tags alone such as `` or ``, or a key alone such as a RedNotebook day's `24:`, nor a line that repeats; an item placed by its own position - that an edit added, a `text_contains` of each of up to four kept items' + that an edit added, a `member_count` matching each of up to four kept items' text, nearest first. Where members were only edited, a `member_count` of the innermost list holding the edited element that an edit goes through by a member with like siblings, a record rather than a @@ -682,6 +682,9 @@ rather than by its position: - `[>PATH=VALUE]` names the member that holds `VALUE` at a path of one to three `/`-separated steps inside it. `>` reads XML and JSON differently: + For XML, `[>text()=VALUE]` compares the member's own whole text, with + whitespace collapsed, to `VALUE`. In JSON, `text()` remains a field name. + | Form | In an XML element | In a JSON record | | -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | | `[>CHILD=VALUE]` | a child element `CHILD` whose text, every text node in it with each run of whitespace collapsed to one space and none at either end, is `VALUE` | the string field `CHILD`, compared whole | @@ -882,9 +885,7 @@ It refuses a task whose every check passes on the seed, and a check that fails on the end state, with that check's reason. Only a failure on what the seed's view shows counts: a task whose checks fail on the seed only because a view could not be read or may have left out what they look for, such as a line past -the 5,000 a text view keeps, is refused, naming each failure. A capture yields -one task, so the option is taken once. The result's `task` names the task, its -number of checks and those that fail on what the seed shows. A refused task +the 5,000 a text view keeps, is refused, naming each failure. A refused task writes no bundle. The bundle then holds, besides the app and `tasks/smoke.json`: @@ -893,7 +894,7 @@ The bundle then holds, besides the app and `tasks/smoke.json`: | ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | | `tasks/.json` | Deploy the app, deploy the agent and its skills, record the screen, prompt the agent, run the verifier, stop recording | | `artifacts/-check/verify.py` | Reads the app's `data/get` through the computer and evaluates the checks, one result per check | -| `evals/.toml` | The task, so a run of the bundle runs it; the smoke task runs with `--task smoke` | +| `evals/.toml` | Every task, so a run of the bundle runs them all; the smoke task runs with `--task smoke` | The verifier finds the app among the computer's child environments, calls its `data/get` under the run's grant of the computer's control token, and reads @@ -901,12 +902,48 @@ the `state` data part. It reads again until the app's files settle, as [grading once the app has saved](desktop-application-export.md#grading-once-the-app-has-saved) describes, and each result's `settled` says whether they did, and `read_cut` whether the bound cut a read still running. It raises when the app cannot be reached, so the run -fails rather than scoring the agent. The README lists the task, its prompt, its +fails rather than scoring the agent. The README lists each task, its prompt, its checks and the recipe below. +### Several tasks + +One capture can yield several tasks: repeat `--task`, one file per task, each +with its own name. Do each task's change on top of the last one in the same +capture and take a checkpoint once it is saved; the first task's end is the +end checkpoint, and `--task-end NAME=CHECKPOINT`, repeatable, names another +task's. Each task is checked at its own end, so a task whose check fails there +is refused, naming the task and the checkpoint, even when it passes at the +last one; its checks should not read what a later task changes. A checkpoint +named `near-NAME`, when the capture took one, is that task's near miss: a +plausible wrong edit on top of its change, such as a typo. The export refuses +the task unless one of its checks fails on what the near miss's views show. + +```sh +rea export-desktop-application ./notes-capture ./notes-bundle \ + --seed-checkpoint seed --end-checkpoint end \ + --task add-eggs.json --task add-bread.json --task-end add-bread=end-add-bread +``` + +Two tasks of one name, a `--task-end` naming none of the tasks or not in +`NAME=CHECKPOINT` form, and a checkpoint the capture did not take once at or +after the seed are refused. The result's `tasks` lists each task: its name, +its number of checks, its end checkpoint, those that fail on what the seed +shows, and its near miss's checkpoint, or null, with those that fail there. +`describe-desktop-application-task` drafts a later task from the end before +it: `--seed-checkpoint end --end-checkpoint end-add-bread`. + +A task's readers run at the seed as the bundle ships it, at its own end and +at its near miss. A screen-judged task ships the seed's screenshot and its own +end's as the judge's examples, and is refused when a file shows its change +since the end before it, or the seed for the first; its near miss is left to +the witness, so its `near_checkpoint` is null. + ## Solve rate -REA does not run agents. To measure how often an agent solves the task, run it +REA's CLI runs no agent. On a build host, the MCP tool `measure_build_task` +measures a bundle's task for a model, records it in the bundle's +`difficulty.json`, and keeps the task in the band when it passes 1 to 3 of 5; +see [measuring a task](desktop-build-witness.md#measure). By hand, run the task k times, three by default, and count the runs whose summary line says `passed`: diff --git a/docs/desktop-build-computer.md b/docs/desktop-build-computer.md index 1b95dd62e0..43a6d6db48 100644 --- a/docs/desktop-build-computer.md +++ b/docs/desktop-build-computer.md @@ -37,14 +37,14 @@ recorded instead. It records `REA_AGENT_ENV_COMMAND`, `REA_DOCKER_COMMAND` and `REA_BUILD_COMPUTER_REA_ARCHIVE` as absolute paths in that client's REA registration, keeping a value the registration already has other than an archive setup packed for an earlier REA version, and refreshes the skill. -`--install-agentenv` runs `uv tool install agentenv-framework==0.9.1464` +`--install-agentenv` runs `uv tool install agentenv-framework==0.9.1485` when `uv` is on `PATH`, and otherwise prints the command. Setup never installs Docker or Node.js and never runs `sudo`. Restart the client afterwards. Doctor is read-only and checks, in order: a Linux amd64 or arm64 host, Node.js, procps `ps`, the Docker CLI, a Docker daemon reachable without -`sudo`, 20 GB free where Docker keeps images, agentenv-framework 0.9.1464, +`sudo`, 20 GB free where Docker keeps images, agentenv-framework 0.9.1485, the runtime archive of this REA version, the build settings in the registration or the environment, and the skill. Each failure names its fix. It also reports, without counting them against `healthy`, whether a model @@ -67,9 +67,9 @@ rest of this page is the same setup by hand. - [Docker Engine](https://docs.docker.com/engine/install/) and its [post-install steps](https://docs.docker.com/engine/install/linux-postinstall/): add yourself to the `docker` group, then log in again. -- agentenv-framework 0.9.1464, the version bundles target. Install +- agentenv-framework 0.9.1485, the version bundles target. Install [uv](https://docs.astral.sh/uv/getting-started/installation/) first, then - run `uv tool install agentenv-framework==0.9.1464`. Set + run `uv tool install agentenv-framework==0.9.1485`. Set `REA_AGENT_ENV_COMMAND` to the absolute path of the resulting `agent-env` program, normally `$HOME/.local/bin/agent-env`; `command -v agent-env` shows the path when its install directory is on your `PATH`. Installing it @@ -126,11 +126,11 @@ setup wrote, such as in `~/.claude.json`: } ``` -Solver runs are `agent-env run` commands the agent runs in its shell, and -agentenv refuses them without a model endpoint: export `LITELLM_BASE_URL` and -`LITELLM_API_KEY` in the shell that starts `claude`, whose Bash tool inherits -them, or set `base_url` and `api_key` under `[model]` in -`.agentenv/config.toml`. +`measure_build_task` runs solves through agentenv, which refuses them without +a model endpoint: export `LITELLM_BASE_URL` and `LITELLM_API_KEY` in the shell +that starts `claude`, whose REA server inherits them, or set `base_url` and +`api_key` under `[model]` in `.agentenv/config.toml`. REA passes them on to +agentenv and never reads the key. Claude Code asks before each MCP tool call its permission settings do not allow, and a build makes dozens of `act_on_build_computer` calls. You can @@ -142,13 +142,13 @@ A headless run, `claude -p`, ends when the skill asks its upfront questions, and refuses the tool calls its permission settings do not allow. Put the answers in the request: the install source when the app is not in the Ubuntu archive, a synthetic seed or the file to start from, whether to review the -task before solver runs, and the solver runs' count, model and budget, or +task before it is measured, and the model and budget to measure it with, or none: ```bash claude -p 'Build me a Gnote environment where the agent adds olive oil to the -shopping list note. Make a synthetic note. Then make 2 solver runs with -, at most $4 in all, without a review first.' +shopping list note. Make a synthetic note. Then measure it with , +at most $4 in all, without a review first.' ``` ## Start a computer @@ -190,9 +190,32 @@ Applications menu entry can open the app. | `archive` | `package` from the Ubuntu 24.04 archive, on amd64 and arm64, with its optional `extra_packages` beside it | | `apt_repository` | `package` from the repository at `uri` with `suite` and `components`, trusting the key file `key_file` | | `package_file` | The `.deb` at `url`, checked against `sha256`, for one `architecture`, after its apt `dependencies` are in | +| `appimage` | The AppImage at `url`, checked against `sha256`, for one `architecture`, extracted with its own entry | +| `tarball` | The archive at `url`, checked against `sha256`, for one `architecture`, with an entry running `program` | Build steps run with the network off, so a `.deb`'s dependencies must be -listed. `extra_packages` names up to 16 more archive packages, by the same +listed. + +An AppImage or a tarball is downloaded by agentenv when the bundle is put, +refused unless its SHA-256 is the one named, and unpacked by a root build +step, `install-download.sh`, under `/opt/rea-apps/`; the +export writes the same step. An AppImage is extracted with its own +`--appimage-extract`, so no FUSE is needed, and the step installs the image's +own `.desktop` under `/usr/local/share/applications` with +each `Exec` program replaced by the image's `AppRun`, after any `env` prefix +and its assignments, without `TryExec`, and its icon; an image without that entry fails the build, naming the entries it +has. A tarball is read with Python's `tarfile`, since the computer has no +`xz` program, so gzip, bzip2 and xz archives work. Its `data` filter refuses +links that point outside the folder, absolute ones included, and special +files, so such an archive fails the build. Its `program` is a path +inside the archive, and REA writes the entry: `name` in the menu, or the +entry's last part, and `program` with the files it opens (`%F`). An +Electron app's command is rewritten after this step, on the entry it +installed. An image or tarball holds no Debian package, so the capture +records its package as `unknown`; the export checks instead that the +captured program lies under the folder the download unpacks to. The +downloaded file stays in the image's layer under the app's folder beside +what it unpacks, so a 721 MB AppImage that unpacks to 3.0 GB adds both. `extra_packages` names up to 16 more archive packages, by the same name rules as `package`, for a runtime piece the app needs that its package does not install: apt here skips Recommends, so MyPaint 2.0.1 exits at launch without `librsvg2-common`, GTK's SVG icon loader, and Freeplane 1.7.10 needs diff --git a/docs/desktop-build-witness.md b/docs/desktop-build-witness.md index 25e415aed7..1ee38ba382 100644 --- a/docs/desktop-build-witness.md +++ b/docs/desktop-build-witness.md @@ -7,13 +7,14 @@ it writes, and then witnesses the task in a fresh computer of that bundle. REA runs its own capture, task draft and export inside the build computer and grades the witness with the same checks the exported verifier runs. -| Tool | What it does | -| -------------------------- | ------------------------------------------------------------------------------------------- | -| `start_build_capture` | Downloads the package lists as the computer's user and launches the app under REA's capture | -| `mark_build_checkpoint` | Takes a named checkpoint and returns the capture's counts and the files that changed | -| `stop_build_capture` | Stops the capture and returns its checkpoints and the task draft between `seed` and `end` | -| `export_build_environment` | Exports the bundle with the agent's task inside the computer and copies it to a folder here | -| `check_build_task` | Grades a task against the app's live state; with a `stage`, files the check in the witness | +| Tool | What it does | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `start_build_capture` | Downloads the package lists as the computer's user and launches the app under REA's capture | +| `mark_build_checkpoint` | Takes a named checkpoint and returns the capture's counts and the files that changed | +| `stop_build_capture` | Stops the capture and returns its checkpoints and the task drafts, from `seed` to `end` and on to each `end-` | +| `export_build_environment` | Exports the bundle with the agent's tasks inside the computer and copies it to a folder here | +| `check_build_task` | Grades a task against the app's live state; with a `stage`, files the check in the witness | +| `measure_build_task` | Runs a bundle's task in fresh computers for a model and records how hard it is in the bundle | Like the computer tools, these are MCP-only and act on a `computer_id` from `start_build_computer`. @@ -108,9 +109,16 @@ See [desktop builds](https://github.com/morluto/rea/blob/main/.agents/skills/rev environment nor package), and checks that hold after the change and not on the seed, from the suggestions, `values` and any `guards` worth keeping. -7. `export_build_environment` with the task and a new or empty `directory`. +7. `export_build_environment` with the tasks and a new or empty `directory`. + Each task is `{"task", "end_checkpoint"}`, its end `end` by default; a + later task's change made on top of the last, and marked `end-`, + is checked there, and `stop_build_capture`'s `later_drafts` drafts it from + the end before it. A checkpoint `near-` taken after a near miss + of that task's change must fail one of its checks. An end the capture did + not take is refused before anything runs. The export runs inside the computer with `--seed-checkpoint seed - --end-checkpoint end`, so it seeds what was saved at `seed`, scrubs the + --end-checkpoint end` and each task's `--task-end`, so it seeds what was + saved at `seed`, scrubs the computer's host name and user from it, and refuses a task whose checks all pass on the seed. It also passes `--synthetic-seed`, since the agent made the seed for the task, so the README does not call it the user's real @@ -222,9 +230,11 @@ Then, with the task exactly as exported: `check_build_task` reads the app's `data/get`, takes the views it returns beside the files and evaluates each check with REA's evaluator, the one the export validates tasks with and the exported verifier mirrors, one result per -check. Once the files settle it runs the task's readers in the computer through +check. Once the files settle it runs the task's +[readers](specs/desktop-app-environments/file-readers/spec.md) in the computer through `computer-exec/v1`, as the verifier does, and a check through a reader reads -that reader's view. A task of screen judges reads no files: REA takes a screenshot and runs the bundle's own judge on it +that reader's view. A task of [screen judges](specs/desktop-app-environments/screen-judge/spec.md) +reads no files: REA takes a screenshot and runs the bundle's own judge on it with the agentenv runner's Python and model endpoint. Like the verifier, it grades only once the app's files settle, since an app such as Gnote saves a few seconds after the last key. It reads `data/get` every 0.5 s and grades once every file's bytes have held for 3 s, counted @@ -294,7 +304,62 @@ folder something else wrote into, the witness keeps its stages: check the stage that ended it again, with the same task, and REA writes the record without checking again. Once it is saved, the same check returns the saved record again, so a record saved as its call was cancelled is not lost. A -computer is witnessed once; witness again from a fresh computer. +computer is witnessed once; witness again from a fresh computer. A bundle of +several tasks is witnessed one task per fresh computer: the seed stage +refuses a task whose steps and verifier differ from the bundle's for that +task's name, so each task's record shows that task alone from the seed. + +## Measure + +`measure_build_task` takes a bundle folder, a task's name, a `model`, an +`agent` (`claude-code` by default) and `budget_usd`. It runs the task through +REA's helper, `bridge/agentenv/solve.py`, with the agentenv runner's Python on +this host, one solve at a time, each in a fresh computer the run tears down: +the bundle's own steps, with the agent deployed as the solver on the model. +The model endpoint is the one agentenv reads where REA starts, +`LITELLM_BASE_URL` and `LITELLM_API_KEY` or `[model]` in +`.agentenv/config.toml`; REA holds no model client and never reads the key. + +Each solve uses a private agentenv sandbox root. If the helper stops before +reporting cleanup, REA terminates each sandbox agentenv lists under that root +(`local_sandbox_ids`), including one killed while it recorded its ports, and +keeps the root when cleanup fails. agentenv keeps a sandbox's folder until its +teardown is confirmed, so once that root is empty the solve's cleanup is +verified, report or not. On macOS, REA cannot verify host processes left by a +recovered sandbox. + +| Solves finished | Passes | Band | Next | +| --------------- | --------- | ---------- | ------------- | +| 3 | 0 | `too_hard` | stop | +| 3 | 3 | `too_easy` | stop | +| 3 | 1 or 2 | | 2 more solves | +| 5 | 1, 2 or 3 | `in_band` | stop | +| 5 | 4 | `too_easy` | stop | + +A solve agentenv reports `failed`, `unscored` or `cancelled` did not finish: +it is run again, at most twice, and not counted, though its cost is. Before +each solve the measure stops, `stopped: "budget"`, when the largest cost so +far would take the total past `budget_usd`; a solve's cost is known only when +it ends, from Claude Code's last trajectory record, so the first solve always +runs and a solve that reports no cost stops the measure. A solve whose helper +is stopped before it reports, by the solve's deadline or killed from outside, +counts as `failed` with no known cost, so the measure records it and stops. A +solve that could not run at all, such as for want of a model endpoint, ends the measure with +its error and records nothing. + +The result and `difficulty.json`, at the bundle's root, keep per task: the +model, agent and time; the band and why it stopped; `k`, the finished solves, +the passes and unfinished solves; the cost in all and the budget; each +finished solve's turns; `failures_per_check`, how many finished solves failed +each check; each solve's outcome, turns, cost, seconds, failed checks, +instance id and the error agentenv reported for it; `task_sha256`, the +SHA-256 of the task's steps file followed by its verifier, so a record whose +task changed since shows it is stale; and +REA's and agentenv's versions. Measuring a task again replaces its record and +keeps the others. A `difficulty.json` or README REA cannot read or parse is refused +before any solve and left as it is, and measures of one bundle write one at a +time, so neither loses the other's record. The README gets a Measured difficulty section with one +line per task, which agentenv ignores like the file. See the [specification](specs/desktop-app-environments/build-witness/spec.md) for scope and verification. diff --git a/docs/specs/desktop-app-environments/README.md b/docs/specs/desktop-app-environments/README.md index 44d1d5b106..cdfdd851a7 100644 --- a/docs/specs/desktop-app-environments/README.md +++ b/docs/specs/desktop-app-environments/README.md @@ -72,6 +72,14 @@ The planned sequence was one PR per step: result, log, record or exported bundle holds; sign-in and licence state withheld from the seed and supplied at run time, seeded only with the person's recorded consent; known handle values are checked explicitly. +19. [Several tasks](several-tasks/spec.md): one capture yields several + tasks, each checked at its own end checkpoint and, when one was taken, + failing at its near miss; the eval lists them all and each is witnessed + on its own fresh computer. +20. [Task difficulty](task-difficulty/spec.md): a build tool solves a + bundle's task in fresh computers for a model, 3 times and 2 more unless + they passed none or all, within a budget, and records its band, cost, + turns and failures per check in the bundle. Version 1 covers Linux with X11, applications whose state lives in files or SQLite under the home directory, and real applications from public packages. diff --git a/docs/specs/desktop-app-environments/export/spec.md b/docs/specs/desktop-app-environments/export/spec.md index c6af4338de..c0c79d73d1 100644 --- a/docs/specs/desktop-app-environments/export/spec.md +++ b/docs/specs/desktop-app-environments/export/spec.md @@ -43,8 +43,8 @@ state through `data/add`, `data/get` and `data/reset`. captured home path respelled when seeded. Baseline dconf keys of the application are compiled into the user's dconf database with `dconf-cli`, without D-Bus. -- The README and the result state the agentenv-framework release the class was - checked against (0.9.1464), and that the seed is real user data, unless +- The README and the result state the agentenv-framework release the class + targets (0.9.1485), and that the seed is real user data, unless `--synthetic-seed` says it was made for the task. ## Non-goals diff --git a/docs/specs/desktop-app-environments/several-tasks/plan.md b/docs/specs/desktop-app-environments/several-tasks/plan.md new file mode 100644 index 0000000000..2f2b8bb9d7 --- /dev/null +++ b/docs/specs/desktop-app-environments/several-tasks/plan.md @@ -0,0 +1,20 @@ +# Plan + +1. Export: `admitDesktopTasks` in `DesktopTaskExport.ts` reads every task + file, resolves each task's end and near miss among the capture's + checkpoints with `selectStateEnds`, evaluates the checks at the seed, its + end and its near miss, and renders every task with one eval. +2. CLI: `--task-end` on `export-desktop-application`; the request carries + it to the export, and the result reports `tasks`. +3. Build: `DesktopBuildExport` writes one file per task and passes + `--task-end`; `DesktopBuildCapture` refuses an untaken end; + `DesktopBuildSummary` drafts each later end from the end before it. +4. Contracts: `export_build_environment` takes and reports `tasks`; + `stop_build_capture` reports `later_drafts`; `mark_build_checkpoint` + names `end-` and `near-`. +5. Docs: the building guide, the tasks, export and build-witness guides, + and this spec. +6. Tests first: the export of two tasks from one capture with a near miss, + and its refusals; the build tools' later draft, export and refusal + through the MCP with the real CLI's outputs; then TreeLine on the arm64 + VM with three tasks in one capture, each witnessed. diff --git a/docs/specs/desktop-app-environments/several-tasks/spec.md b/docs/specs/desktop-app-environments/several-tasks/spec.md new file mode 100644 index 0000000000..7da922ce86 --- /dev/null +++ b/docs/specs/desktop-app-environments/several-tasks/spec.md @@ -0,0 +1,53 @@ +# Several tasks + +## Outcome + +One capture yields an environment with several tasks. The building agent +makes each task's change on top of the last in one capture and marks its own +end checkpoint, and may mark a near miss of any task. The export checks each +task against the seed, its own end and its near miss, writes every task's +steps and verifier, and lists them all in the bundle's eval. Each task is +then witnessed on its own fresh computer of the bundle. + +## Requirements + +- `export-desktop-application` takes `--task` once per task and + `--task-end NAME=CHECKPOINT`, repeatable, for each task checked at a + checkpoint other than `--end-checkpoint`. It refuses two tasks of one name, + an end not in `NAME=CHECKPOINT` form or naming none of the tasks, two ends + for one task, and a checkpoint the capture does not hold once at or after + the seed. +- Each task's checks are evaluated at the seed, as the bundle ships it, and + at its own end, and the existing rules hold per task: a check failing at + its end, every check passing on the seed, or no check failing on what the + seed's views show refuses the task. Each refusal names the task and the + end checkpoint it was checked at. +- A checkpoint `near-NAME` is task NAME's near miss. When the capture took + it, at or after the seed, the export refuses the task unless a check fails + on what the near miss's views show; a failure for want of a whole or + readable view does not count. +- The result's `tasks` lists, per task, its name, number of checks, end + checkpoint, the checks failing on the seed, and the near miss's checkpoint + or null with the checks failing there. The bundle holds every task's + `tasks/NAME.json` and `artifacts/NAME-check/verify.py`, one + `evals/ENV.toml` naming them in the order given, and a README section per + task saying where it was checked. +- `export_build_environment` takes `tasks`, at most 8, each + `{task, end_checkpoint}`, the end `end` by default, and refuses an end the + stopped capture did not take before running anything in the computer. It + writes each task to its own file in the computer and passes each task's + end as `--task-end`. Its result's `tasks` is the export's. +- `stop_build_capture` returns `later_drafts`: one per checkpoint named + `end-` taken after `end`, drafted with + `describe-desktop-application-task` from the end checkpoint before it, so + it shows only that task's change, with the draft or the reason there is + none. +- `check_build_task` needs no change: on a computer of the bundle it takes + any task the bundle holds, by its rendered steps and verifier, and a + computer is witnessed once, so each task is witnessed on its own computer. + +## Non-goals + +- No reset inside a capture: later tasks are made on top of earlier ones, + and their checks should not read what a later task changes. +- No measured difficulty; solves stay outside REA here. diff --git a/docs/specs/desktop-app-environments/several-tasks/tasks.md b/docs/specs/desktop-app-environments/several-tasks/tasks.md new file mode 100644 index 0000000000..64a9a5a10c --- /dev/null +++ b/docs/specs/desktop-app-environments/several-tasks/tasks.md @@ -0,0 +1,8 @@ +# Tasks + +- [x] Record the outcome, requirements and non-goals. +- [x] Export several tasks from one capture, each at its own end and near miss, with one eval listing them all. +- [x] Take each task's end with `--task-end`, and refuse duplicate names, malformed or unknown ends, and untaken checkpoints. +- [x] Export several tasks from a build computer and draft each later task from the end before it. +- [x] Document the flow in the building guide and the tasks, export and build-witness guides. +- [x] Add the tests and build TreeLine with three tasks on the arm64 VM, witnessing each. diff --git a/docs/specs/desktop-app-environments/several-tasks/verification.md b/docs/specs/desktop-app-environments/several-tasks/verification.md new file mode 100644 index 0000000000..e3b967d314 --- /dev/null +++ b/docs/specs/desktop-app-environments/several-tasks/verification.md @@ -0,0 +1,54 @@ +# Verification + +## Tests + +- `src/application/desktop/DesktopSeveralTasks.test.ts` exports two tasks + from one capture, `add-eggs` at `end` and `add-bread` at `end-add-bread`, + with a near miss of the eggs at `near-add-eggs`, and checks the result's + `tasks`, both tasks' files, the eval naming both and the README's two + sections. It refuses `add-bread` checked at `end`, before its change; a + near miss every check of `add-bread` passes; two tasks of one name; an end + for a task not given; an end without `=`; and an untaken checkpoint. Each + of the seven failed on the code before this change, which refused a + second task file. +- `tests/boundary/mcp/buildSeveralTasksMcp.test.ts` runs the build tools + against the real CLI's outputs for that capture: `stop_build_capture` + returns one later draft, from `end` to `end-add-bread`, and + `export_build_environment` writes both tasks to the computer, passes each + end as `--task-end` and reports both; an untaken end is refused before the + export runs. Both failed before the change. + +## TreeLine, arm64 + +On the arm64 Ubuntu 24.04 build host with agentenv-framework 0.9.1431, one +capture of TreeLine 3.1.5 from the Ubuntu archive, through REA's MCP tools: + +- Seed: the Book List template saved as `Documents/books.trln`, one category + `SF Books` with two authors and two books. +- `rename-category` at `end`: the category renamed `Science Fiction`. +- `set-foundation-year` at `end-set-foundation-year`: Foundation's copyright + 1951 to 1952. +- `add-asimov-book` at `end-add-asimov-book`: I, Robot added under Isaac + Asimov with copyright 1950. +- Near misses, after the last end: Foundation 1925 and I, Robot 1905, marked + `near-set-foundation-year` and `near-add-asimov-book`, then the category + `Science Fictoin`, marked `near-rename-category`. + +`stop_build_capture` drafted the first task from `seed` to `end` and the two +later ones each from the end before it, each showing only its own change: +the category's name, Foundation's copyright, and the new book with Asimov's +second child. `export_build_environment` kept all three, each failing on the +seed and at its near miss, and wrote one eval: +`tasks = ["rename-category", "set-foundation-year", "add-asimov-book"]`. + +Each task was then witnessed on its own fresh computer of the bundle, from +its prompt: the seed failed a check, the edit passed every check, a near +miss (`Science Fictoin`, 1925, 1905) failed one and the reset seed failed +again, so all three records, `witness-rename-category`, +`witness-set-foundation-year` and `witness-add-asimov-book`, hold. +`agent-env run --dry-run` of the bundle plans the eval's three tasks. No +solver ran. + +Not tested: a near miss whose only failures come from a view that could not +be read or is cut short, which the export refuses like such a seed, and an +`end-` taken before `end`, which gets no later draft. diff --git a/docs/specs/desktop-app-environments/task-difficulty/cleanup-qualification-2026-11-10.json b/docs/specs/desktop-app-environments/task-difficulty/cleanup-qualification-2026-11-10.json new file mode 100644 index 0000000000..63318b3a34 --- /dev/null +++ b/docs/specs/desktop-app-environments/task-difficulty/cleanup-qualification-2026-11-10.json @@ -0,0 +1,59 @@ +{ + "framework": "0.9.1485", + "platform": "linux", + "python": "3.12.16", + "docker_server": "29.1.3", + "cases": [ + { + "stage": "allocation", + "helper_exit": -9, + "cleanup": "verified", + "private_root_removed": true, + "owned_containers": 0, + "owned_networks": 0, + "marked_processes": 0, + "foreign_container_preserved": true, + "failure_retained_folder": false, + "elapsed_seconds": 1.217 + }, + { + "stage": "active", + "helper_exit": -9, + "cleanup": "verified", + "private_root_removed": true, + "owned_containers": 0, + "owned_networks": 0, + "marked_processes": 0, + "foreign_container_preserved": true, + "failure_retained_folder": false, + "elapsed_seconds": 0.968 + }, + { + "stage": "refusal-retry", + "helper_exit": -9, + "cleanup": "verified", + "private_root_removed": true, + "owned_containers": 0, + "owned_networks": 0, + "marked_processes": 0, + "foreign_container_preserved": true, + "failure_retained_folder": true, + "elapsed_seconds": 1.516 + } + ], + "limitations": [ + "Uses actual published LocalSandbox/LocalSandboxProvider and REA's cleanup helper with real Docker resources; no paid model or complete run_bundle solve.", + "Linux process qualification runs inside the isolated test container's process namespace." + ], + "image_repo_digests": [ + "python@sha256:5e2bdb1b0d542d5039cffb8181c65930093949194a65c365dfb1b0fe13a4fa5b" + ], + "framework_wheel_sha256": "14768f69fa9c48c3e4aec2224161747d3643e77a711848e9079dfd62aa5fcca9", + "rea_cleanup_helper_sha256": "789ebeb1de83764a9a274616289e57800ac4631559fc00f8664f262d1893333d", + "observed_at_utc": "2026-11-10T08:43:41.806755+00:00", + "upstream": [ + "https://github.com/scaleapi/agentenv-framework/issues/546", + "https://github.com/scaleapi/agentenv-framework/pull/555", + "https://github.com/scaleapi/agentenv-framework/pull/561" + ] +} diff --git a/docs/specs/desktop-app-environments/task-difficulty/plan.md b/docs/specs/desktop-app-environments/task-difficulty/plan.md new file mode 100644 index 0000000000..042a7a5e54 --- /dev/null +++ b/docs/specs/desktop-app-environments/task-difficulty/plan.md @@ -0,0 +1,16 @@ +# Plan + +1. Domain: `taskDifficulty.ts` holds the band, the next-solve decision under + the budget, the record and its README section. +2. Helper: `bridge/agentenv/solve.py` runs one solve with `run_bundle` and + reports it; `AgentEnvSolveHelper.ts` spawns it as an owned process with the + runner's Python and reads its report. +3. Application: `DesktopBuildSolves.ts` measures a task and writes + `difficulty.json` and the README; `DesktopBuildOwnedWork.ts` holds the work + closing REA stops, now shared by a computer's start and a measure. +4. MCP: the contract, its effects, registration with heartbeat progress, and + the CLI-parity list. +5. Docs: the building guide's Measure and Harder tasks sections, the witness, + tasks and build computer guides, and this spec. +6. Tests first: the band and budget rules, and the tool through the MCP with + a stand-in agentenv; then TreeLine tasks on the arm64 VM. diff --git a/docs/specs/desktop-app-environments/task-difficulty/spec.md b/docs/specs/desktop-app-environments/task-difficulty/spec.md new file mode 100644 index 0000000000..a6bfcbb5c0 --- /dev/null +++ b/docs/specs/desktop-app-environments/task-difficulty/spec.md @@ -0,0 +1,53 @@ +# Task difficulty + +## Outcome + +The building agent can tell how hard a bundle's task is for a model, and +harden it until it lands in a band worth evaluating or training on. One MCP +tool solves the task in fresh computers, adaptively and within a budget, and +records the result in the bundle. + +## Requirements + +- `measure_build_task` takes `bundle` (an absolute folder on the host), + `task`, `model`, `agent` (`claude-code` by default) and `budget_usd`. It + refuses a task whose `tasks/NAME.json` or `artifacts/NAME-check/verify.py` + the bundle lacks, before any solve. +- Each solve runs `bridge/agentenv/solve.py` with the agentenv runner's + Python, which calls `run_bundle` for the one task with the model and agent, + `sandbox="local"` and `emulate=False`, and prints one JSON line: the run's + outcome, instance id, seconds, each check's result from the instance's + verifications, the solver's turns and cost from the last record of its + local trajectory, and agentenv's version; or the error that kept it from + running. The endpoint is whatever agentenv reads where REA starts; REA holds + no model client and passes the environment on unread. +- Solves run one at a time. The band: 3 finished solves with 0 passes is + `too_hard` and with 3 is `too_easy`; otherwise 2 more, and 1 to 3 of 5 is + `in_band`, 4 `too_easy`. A solve that did not finish (`failed`, `unscored`, + `cancelled`) is run again, at most twice, and not counted. Before each solve + the measure stops when the largest cost so far would take the total past the + budget, and after a solve that reports no cost. +- The result and `difficulty.json` at the bundle's root keep, per task: model, + agent, time, band, why it stopped, finished solves (`k`), passes, + unfinished solves, cost and budget, each finished solve's turns, failures + per check, each solve, the task's digest (SHA-256 of its steps file then its + verifier) and REA's and agentenv's versions. A task measured again replaces + its own record. The README's Measured difficulty section lists each task. +- A solve that could not run ends the measure with its error and writes + nothing. A `difficulty.json` or README that exists but cannot be read + or parsed is refused before any solve and again before writing, and left + as it is; record writes to one bundle run one at a time. +- A helper still running after the stop grace has its process group cleared + before REA reads its report, and that read waits at most 5 s. A helper whose + cleanup fails raises a cleanup error naming its process group, and REA keeps + it so closing retries the cleanup. Cancelling, or REA closing its build computers, stops the running + solve, whose run tears its computer down. +- The building guide tells the agent how to measure, how to read failures per + check for brittle checks, and how to harden a task that is too easy while + keeping it sound. + +## Non-goals + +- No parallel solves, no other model than the solver's, and no judge: the + verifier's checks decide every solve. +- No cost cap inside a solve: the budget is checked between solves. diff --git a/docs/specs/desktop-app-environments/task-difficulty/tasks.md b/docs/specs/desktop-app-environments/task-difficulty/tasks.md new file mode 100644 index 0000000000..ad7911118e --- /dev/null +++ b/docs/specs/desktop-app-environments/task-difficulty/tasks.md @@ -0,0 +1,8 @@ +# Tasks + +- [x] Record the outcome, requirements and non-goals. +- [x] Decide the band and the budget stop from the solves so far. +- [x] Run a solve in a fresh computer through REA's helper and read its report. +- [x] Record the measure in `difficulty.json` and the README. +- [x] Document measuring and hardening in the building guide. +- [x] Add the tests, measure TreeLine tasks on the arm64 VM and harden one. diff --git a/docs/specs/desktop-app-environments/task-difficulty/verification.md b/docs/specs/desktop-app-environments/task-difficulty/verification.md new file mode 100644 index 0000000000..5187b594e3 --- /dev/null +++ b/docs/specs/desktop-app-environments/task-difficulty/verification.md @@ -0,0 +1,90 @@ +# Verification + +## Tests + +- `src/domain/desktop/taskDifficulty.test.ts`: the band for 0 and 3 of 3, + 1 and 2 of 3, 1, 3 and 4 of 5 and an unfinished solve beside finished ones; + 3 solves, 2 more, then a stop; an unfinished solve run again at most twice; + the stop before a solve the largest cost so far would take past the budget, + and after one of unknown cost; the record's counts, cost, turns and + failures per check; and the README section replaced in place. +- `tests/boundary/mcp/buildMeasureMcp.test.ts` calls `measure_build_task` + through the MCP with a stand-in agentenv whose `run_bundle` answers solves + in order: 1 of 3 then 2 more keeps the task at 2 of 5, each solve run once + in the bundle with the model, `claude-code`, `sandbox="local"`, + `emulate=False` and the endpoint REA started with, and `difficulty.json` + and the README record it with the task's digest; 3 of 3 stops as too easy + after running an unfinished solve again; the budget stops before a third + solve; a task the bundle lacks is refused before any solve. All of them + failed before the change: the module and the tool did not exist. After + review, the unfinished solve's error, `RuntimeError: deploy`, is kept in + the result and `difficulty.json`; that test failed first, since the record + dropped every solve's error. + +## TreeLine, arm64 + +On the arm64 Ubuntu 24.04 build host with agentenv-framework 0.9.1431, REA's +MCP server started with `LITELLM_BASE_URL` and `LITELLM_API_KEY`, each task +measured with `anthropic/claude-sonnet-5-5`: + +| Bundle | Task | Budget | Solves | Passes | Band | Turns | Cost | +| -------- | --------------------- | ------ | ------ | ------ | ---------------------- | ----------- | ----- | +| `d-tl3` | `rename-category` | $2.50 | 2 | 2 | undecided, budget stop | 16, 20 | $2.02 | +| `d-tl3` | `set-foundation-year` | $2.50 | 3 | 3 | too easy | 17, 18, 22 | $1.89 | +| `d-tl3` | `add-asimov-book` | $2.50 | 3 | 3 | too easy | 17, 17, 17 | $1.64 | +| `d-hard` | `apply-corrections` | $9 | 3 | 3 | too easy | 106, 61, 74 | $5.14 | + +`rename-category` stopped before a third solve because the largest cost so +far, $1.06, would have taken $2.02 past $2.50. `apply-corrections` hardens +the 15-book reading list with a second document the solver opens in TreeLine, +`Documents/corrections.trln`, listing ten corrections to apply: eight years +and two authors across all three categories, one of them "the same year as +The Quiet Glacier's", with guards on three books left alone. Checked offline +against the app's own views, it fails on the seed, passes when applied, and +fails each near miss: a correction left out, an author without its hyphen, +the cross-reference given another book's new year, and an untouched book +changed. Every solve opened the corrections in a second window and applied +all ten; one typed a year into the tree, saw the stray node and undid it. + +No task landed in the band: the four measures cost $10.69 of the $15 the +validation allowed, and the $4.31 left could not cover 3 solves of another +hardened task at the $1.50 to $2.07 a solve `apply-corrections` cost. Each +solve's computer was torn down by its run. `difficulty.json` and the README +of each bundle list the tasks measured. + +## Forced-stop cleanup qualification, 2026-11-10 + +The original missing-identity review concern is addressed by the private root +owned by each `SolveSandboxRoot` and the provider guarantees shipped in the +pinned `agentenv-framework==0.9.1485`. +[Upstream #555](https://github.com/scaleapi/agentenv-framework/pull/555) exposes +`local_sandbox_ids()` for that root; +[#561](https://github.com/scaleapi/agentenv-framework/pull/561) makes incomplete +termination raise, so the teardown owner retains the work folder for retry. +[Issue #546](https://github.com/scaleapi/agentenv-framework/issues/546) is closed. +The published wheel was inspected, including folder creation before resource +startup, provider termination, and task teardown's folder-removal ordering. + +The [qualification record](https://github.com/morluto/rea/blob/main/docs/specs/desktop-app-environments/task-difficulty/cleanup-qualification-2026-11-10.json) records the +wheel digest and observations. In an isolated Linux arm64 container with the +published provider, Python 3.12 and Docker Desktop's Linux daemon: + +- A helper created a provider-owned folder and was killed with SIGKILL before + returning any sandbox report. REA's cleanup helper discovered and removed it. +- Another helper started a real named/labeled Docker container, labeled network, + and detached Linux process carrying the sandbox marker. After its process + group was killed, REA's cleanup helper removed all three and emptied the root. +- A controlled Docker removal refusal made cleanup fail while preserving the + work folder and container. Removing the refusal and retrying that same root + removed the resources and folder. +- A separate unrelated container stayed running through every case and was + removed by the qualification's own final cleanup. + +The probe invoked the production `solve_cleanup.py` and actual published +`LocalSandbox`/`LocalSandboxProvider` with real Docker resources. It did not run +a paid model or a complete `run_bundle` solve. Linux process checks were inside +the qualification container's process namespace; this does not qualify native +macOS host-process cleanup. Existing conservative REA checks remain in place. +Controlled TypeScript cleanup/report/cancellation regressions also passed: +22 tests across `AgentEnvSolveHelper`, `SolveSandboxRoot`, and +`DesktopBuildSolves`. diff --git a/docs/specs/desktop-app-environments/tasks/spec.md b/docs/specs/desktop-app-environments/tasks/spec.md index 78bf2e40b7..65a718ac79 100644 --- a/docs/specs/desktop-app-environments/tasks/spec.md +++ b/docs/specs/desktop-app-environments/tasks/spec.md @@ -2,7 +2,8 @@ ## Outcome -Each capture yields one agentenv task in its exported bundle. An agent writes +Each capture yields one agentenv task in its exported bundle, or several, +as [several tasks](../several-tasks/spec.md) describes. An agent writes the task's prompt and declarative checks from the change the capture recorded; REA validates the task against the capture and keeps it only if its checks fail on the seed and pass on the captured end state. The bundle runs @@ -69,7 +70,8 @@ never runs code the agent wrote. - `export-desktop-application --task FILE` evaluates the checks on the state view's first versions (the seed) and last versions (the end), refuses a task whose every check passes on the seed and a check that fails at the end, - and refuses more than one task file. It writes `tasks/NAME.json` (deploy_env, + and, before [several tasks](../several-tasks/spec.md), refused more than + one task file. It writes `tasks/NAME.json` (deploy_env, deploy_agent, add_skills, start_computer_recording, prompt_agent, env_outcome_verifier, end_computer_recording), `artifacts/NAME-check/verify.py` and `evals/ENV.toml` naming the task, and a @@ -83,10 +85,10 @@ never runs code the agent wrote. ## Non-goals -- No REA command that runs agents or measures a solve rate; the user guide - gives the `agent-env run` recipe. +- No REA CLI command that runs agents; the user guide gives the + `agent-env run` recipe, and the build tools measure a solve rate with + [task difficulty](../task-difficulty/spec.md). - No checks over dconf keys, which `data/get` serves as binary, or over files outside the owned paths. - No timestamp detection beyond ISO 8601 and `asctime` date-times; other volatile values are left to the author. -- No more than one task per capture. diff --git a/package.json b/package.json index bddc10d2ea..2313ac1d20 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,8 @@ "bridge/mitmproxy/capture.py", "bridge/agentenv/capture.py", "bridge/agentenv/computer.py", + "bridge/agentenv/solve.py", + "bridge/agentenv/solve_cleanup.py", "scripts/rea.mjs", "scripts/lib/cli-process-environment.mjs", "scripts/rea-android-workspace-guardian.mjs", diff --git a/src/application/DesktopBuildDoctor.test.ts b/src/application/DesktopBuildDoctor.test.ts index fb49f8093e..04e7ee13c7 100644 --- a/src/application/DesktopBuildDoctor.test.ts +++ b/src/application/DesktopBuildDoctor.test.ts @@ -14,7 +14,7 @@ const ready: DesktopBuildFacts = { status: "found", path: "/home/ada/.local/bin/agent-env", source: "path", - version: "0.9.1464", + version: "0.9.1485", }, archive: { path: "/home/ada/.cache/rea/build-computer/rea-6.8.0.tar.xz", @@ -201,7 +201,7 @@ describe("desktop build tool checks", () => { ); expect(check.classification).toBe("missing_dependency"); expect(check.remediation).toContain( - "uv tool install agentenv-framework==0.9.1464", + "uv tool install agentenv-framework==0.9.1485", ); }); @@ -336,6 +336,7 @@ describe("desktop build registration settings", () => { ); expect(check.detail).toContain(Object.values(fact)[0]![0]); expect(check.details).toMatchObject({ registrations: [fact] }); + expect(check.remediation).toContain("client's REA registration"); } }); it("names the registration's missing settings, never their values", async () => { diff --git a/src/application/DesktopBuildDoctor.ts b/src/application/DesktopBuildDoctor.ts index 5d8abf241e..9e7cd78d18 100644 --- a/src/application/DesktopBuildDoctor.ts +++ b/src/application/DesktopBuildDoctor.ts @@ -362,6 +362,13 @@ const registrationCheck = ( unresolved.length > 0 || (unchecked?.length ?? 0) > 0, ); const repair = failing[0]?.client ?? unknown[0]?.client; + const remediation = failing.some( + ({ differing }) => (differing?.length ?? 0) > 0, + ) + ? "Correct or remove the failed settings in the client's REA registration, then restart the client and run doctor again." + : unknown.length > 0 + ? "Check the unverified settings in the client's REA registration, then restart the client and run doctor again." + : `Run ${setupCommand(repair === undefined ? clients : [repair])}, then restart the client.`; return result( "registration", gaps.length > 0 && failing.length === 0 && unknown.length === 0, @@ -371,7 +378,7 @@ const registrationCheck = ( { classification: failing.length === 0 && gaps.length > 0 ? "unknown" : "config_drift", - remediation: `Run ${setupCommand(repair === undefined ? clients : [repair])}, then restart the client.`, + remediation, details: { registrations: gaps }, }, ); @@ -398,12 +405,12 @@ const registrationDetail = (gap: { ...(gap.differing === undefined ? [] : [ - `${gap.client}'s registration records different values for ${gap.differing.join(", ")}.`, + `${gap.client}'s recorded ${gap.differing.join(", ")} failed verification.`, ]), ...(gap.unchecked === undefined ? [] : [ - `REA did not check ${gap.client}'s recorded ${gap.unchecked.join(", ")} against the paths used in this terminal.`, + `REA could not verify ${gap.client}'s recorded ${gap.unchecked.join(", ")}.`, ]), ]; return details.length > 0 diff --git a/src/application/MacHopper.test.ts b/src/application/MacHopper.test.ts index ba3853fc1f..771bb4e792 100644 --- a/src/application/MacHopper.test.ts +++ b/src/application/MacHopper.test.ts @@ -589,3 +589,27 @@ it("stops before mounting when setup is cancelled as the package downloads", asy expect(host.mounts).toBe(0); expect(host.installs).toBe(0); }); + +it("retains the completed installation but skips opening when cancelled during its copy", async () => { + const host = new FakeMacHopperHost(); + const controller = new AbortController(); + const install = host.installBundle; + host.installBundle = async (...args) => { + const result = await install(...args); + controller.abort(new Error("stop setup")); + return result; + }; + + await expect( + installMacHopper({ signal: controller.signal }, host), + ).resolves.toMatchObject({ + status: "installed", + launcherPath: + "/home/user/Applications/Hopper Disassembler.app/Contents/MacOS/hopper", + openFailure: "Setup was cancelled before Hopper was opened", + }); + expect(host.installs).toBe(1); + expect(host.opens).toBe(0); + expect(host.unmounts).toBe(1); + expect(host.cleanups).toBe(1); +}); diff --git a/src/application/Setup.desktopBuild.test.ts b/src/application/Setup.desktopBuild.test.ts index e438925253..f7903a7da2 100644 --- a/src/application/Setup.desktopBuild.test.ts +++ b/src/application/Setup.desktopBuild.test.ts @@ -25,7 +25,7 @@ const desktopHost = (facts: Partial = {}) => { selected: [], archive: ARCHIVE, archivePacked: false, - agentEnvVersion: "0.9.1464", + agentEnvVersion: "0.9.1485", recorded: {}, ...facts, }), @@ -244,7 +244,7 @@ describe("rea setup --desktop-build", () => { expect(result.plannedActions[0]).toMatchObject({ id: "install_agentenv", - commands: ["uv tool install agentenv-framework==0.9.1464"], + commands: ["uv tool install agentenv-framework==0.9.1485"], }); expect(calls).toEqual(["install /usr/local/bin/uv", `pack ${ARCHIVE}`]); expect(host.configuredProviderEnvironments[0]).toMatchObject({ @@ -266,7 +266,7 @@ describe("rea setup --desktop-build", () => { expect(calls).toEqual([`pack ${ARCHIVE}`]); expect(result.status).toBe("needs_human"); expect(result.remediation).toContain( - "uv tool install agentenv-framework==0.9.1464", + "uv tool install agentenv-framework==0.9.1485", ); }); diff --git a/src/application/Setup.recovery.test.ts b/src/application/Setup.recovery.test.ts index 3c39067745..8a2e3d988c 100644 --- a/src/application/Setup.recovery.test.ts +++ b/src/application/Setup.recovery.test.ts @@ -948,6 +948,11 @@ describe("setup cancellation during installation and cleanup", () => { expect(received).toBe(controller.signal); expect(failure).toBeInstanceOf(AnalysisCancelledError); expect(failure).toMatchObject({ cause: reason }); + expect((failure as AnalysisCancelledError).diagnostics).toMatchObject({ + setup_progress: { + hopperOutcome: { status: "failed", code: "setup_cancelled" }, + }, + }); expect(host.configurations).toBe(0); }); diff --git a/src/application/desktop/DesktopApplicationExport.ts b/src/application/desktop/DesktopApplicationExport.ts index 7aa0751bcf..e1c5f5430b 100644 --- a/src/application/desktop/DesktopApplicationExport.ts +++ b/src/application/desktop/DesktopApplicationExport.ts @@ -69,7 +69,11 @@ import { exportLimitations } from "./DesktopExportLimitations.js"; import { withRetainedContent } from "./DesktopRetainedContent.js"; import { planLaunch, seedModes, seedVersions } from "./DesktopExportLaunch.js"; import { selectStateEnds } from "./DesktopStateVersions.js"; -import { admitDesktopTask, type AdmittedTask } from "./DesktopTaskExport.js"; +import { + admitDesktopTasks, + type AdmittedTasks, + type TaskSummary, +} from "./DesktopTaskExport.js"; import { taskReads } from "./DesktopExportReaders.js"; const OPERATION = "export_desktop_application"; @@ -82,18 +86,21 @@ export interface DesktopExportRequest extends WithheldRequest { readonly capture: string; readonly directory: string; readonly name: string | undefined; - /** Task files to validate and add; a capture yields at most one. */ + /** Task files to validate and add, each checked at its own end. */ readonly tasks: readonly string[]; + /** `NAME=CHECKPOINT` for each task checked at a checkpoint other than the end checkpoint. */ + readonly taskEnds?: readonly string[] | undefined; /** Where the computer downloads the captured package file from, for an application installed from one. */ readonly packageUrl?: string | undefined; /** Packages installed beside the application, such as a runtime piece a build request named. */ readonly extraPackages?: readonly string[] | undefined; /** A JSON file naming the AppImage or tarball a build computer unpacked the application from. */ readonly downloadInstall?: string | undefined; + /** The folder a build computer keeps a screenshot of each checkpoint in, a screen judge's examples. */ readonly screens?: string | undefined; /** The checkpoint the seed is taken at; the baseline when omitted. */ readonly seedCheckpoint?: string | undefined; - /** The checkpoint a task's end state is read at; the last when omitted. */ + /** The checkpoint a task's end state is read at unless it names its own; the last when omitted. */ readonly endCheckpoint?: string | undefined; /** The seed was made for the task, as in a build computer, rather than taken from the user's own data. */ readonly syntheticSeed?: boolean | undefined; @@ -119,7 +126,7 @@ export interface DesktopExportResult { readonly dconf_groups: number; readonly files: readonly string[]; readonly limitations: readonly string[]; - readonly task: AdmittedTask["summary"] | null; + readonly tasks: readonly TaskSummary[]; /** Files withheld from the seed, each a secret setting a deploy takes from its environment variable. */ readonly withheld_files: readonly WithheldFile[]; /** Withheld files seeded all the same, with the person's consent. */ @@ -295,7 +302,7 @@ const planExport = async ( /** The bundle's generated files, the class module flagged when its dconf keys hold the identity, and its README, made once the seeds' scrub is known. */ const bundleText = ( plan: DesktopAppExportPlan, - admitted: AdmittedTask | undefined, + admitted: AdmittedTasks | undefined, dconfHoldsIdentity: boolean, consent: string | undefined, ) => { @@ -379,9 +386,17 @@ const runExport = async ( checkpoints, signal, }); - const admitted = await admitDesktopTask( - request.tasks, - { ...plan, files, identity, ends, ...reads, seeded: new Set(seeds.keys()) }, + const admitted = await admitDesktopTasks( + { paths: request.tasks, ends: request.taskEnds ?? [] }, + { + ...plan, + files, + identity, + ends, + checkpoints, + ...reads, + seeded: new Set(seeds.keys()), + }, signal, ); const copy = await withheldCopy(request, plan, held.captured, signal); @@ -432,7 +447,7 @@ const runExport = async ( ...copies.map((copy) => copy.path), ], limitations, - task: admitted?.summary ?? null, + tasks: admitted?.summaries ?? [], withheld_files: plan.withheld ?? [], consented_files: plan.consentedFiles ?? [], }; @@ -468,6 +483,7 @@ export const exportDesktopApplication = async ( capture_evidence_id: captureEvidenceId, named: request.name !== undefined, tasks: request.tasks.length, + task_ends: [...(request.taskEnds ?? [])], package_url: request.packageUrl !== undefined, download_install: request.downloadInstall !== undefined, extra_packages: [...(request.extraPackages ?? [])], diff --git a/src/application/desktop/DesktopArchiveTask.test.ts b/src/application/desktop/DesktopArchiveTask.test.ts index 7588c8c3eb..119d88aa11 100644 --- a/src/application/desktop/DesktopArchiveTask.test.ts +++ b/src/application/desktop/DesktopArchiveTask.test.ts @@ -183,7 +183,7 @@ it.each([ if (accepted) { if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { failing_on_seed: ["check-1"] }, + tasks: [{ failing_on_seed: ["check-1"] }], }); } else { expect(result).toBeInstanceOf(Error); @@ -407,7 +407,7 @@ describe("paragraphs split into runs", () => { ]); if (whole.result instanceof Error) throw whole.result; expect(whole.result.normalized_result).toMatchObject({ - task: { failing_on_seed: ["check-1", "check-2"] }, + tasks: [{ failing_on_seed: ["check-1", "check-2"] }], }); const first = await exportTask(folder, [ diff --git a/src/application/desktop/DesktopBuildCapture.ts b/src/application/desktop/DesktopBuildCapture.ts index 9652200690..ee5515930c 100644 --- a/src/application/desktop/DesktopBuildCapture.ts +++ b/src/application/desktop/DesktopBuildCapture.ts @@ -132,6 +132,12 @@ const schemaCommandError = (exitCode: number | null, detail: string) => remediationAction: `Inspect the command error. If the runtime is outdated, ${repackRuntime}`, }); +/** Export options a build passes, as the runtime's schema names them, with what each supports. */ +const BUILD_EXPORT_OPTIONS = { + syntheticSeed: "synthetic seeds", + taskEnd: "task ends", +} as const; + /** Check the installed runtime's advertised export contract before capturing the user's work. */ const checkExportRuntime = async ( computer: ControlledComputer, @@ -193,11 +199,14 @@ const checkExportRuntime = async ( "The build computer's REA returned an invalid export schema.", ), ); - if (!("syntheticSeed" in schema.data.options.properties)) + const unsupported = Object.entries(BUILD_EXPORT_OPTIONS) + .filter(([name]) => !(name in schema.data.options.properties)) + .map(([, what]) => what); + if (unsupported.length > 0) return err( refused( START, - "The REA runtime in this build computer does not support synthetic seeds required by build export.", + `The REA runtime in this build computer does not support ${unsupported.join(" or ")} required by build export.`, repackRuntime, ), ); @@ -614,6 +623,9 @@ export class DesktopBuildCapture { signal: AbortSignal, ): Promise> { const run = this.#run; + const ends = input.tasks.map( + ({ end_checkpoint: end }) => end ?? END_CHECKPOINT, + ); if ( run?.stopped === undefined || !run.checkpoints.has(SEED_CHECKPOINT) || @@ -626,6 +638,15 @@ export class DesktopBuildCapture { "Capture with start_build_capture, mark seed and end, then call stop_build_capture.", ), ); + const untaken = ends.filter((end) => !run.checkpoints.has(end)); + if (untaken.length > 0) + return err( + refused( + EXPORT, + `The capture took no checkpoint named ${untaken.join(" or ")}; its checkpoints are ${[...run.checkpoints].join(", ")}.`, + "Give each task the end checkpoint marked once its change was saved.", + ), + ); const signIn = [SIGN_IN_START, SIGN_IN_END].filter((name) => run.checkpoints.has(name), ); @@ -677,6 +698,7 @@ export class DesktopBuildCapture { keep: this.secrets.keep, signIn: signIn.length === 2, consents: [...this.#consents], + checkpoints: run.checkpoints, }, input, signal, diff --git a/src/application/desktop/DesktopBuildComputers.ts b/src/application/desktop/DesktopBuildComputers.ts index d15e04b01f..b063442f52 100644 --- a/src/application/desktop/DesktopBuildComputers.ts +++ b/src/application/desktop/DesktopBuildComputers.ts @@ -49,6 +49,12 @@ import { } from "./DesktopBuildActionLog.js"; import { seededLockFiles, writeRequestBundle } from "./DesktopBuildBundle.js"; import { DesktopBuildCapture } from "./DesktopBuildCapture.js"; +import { DesktopBuildOwnedWork } from "./DesktopBuildOwnedWork.js"; +import { + measureInFreshComputers, + type MeasuredTask, + type MeasureRequest, +} from "./DesktopBuildSolves.js"; import { DesktopBuildSecrets, settingsEnvironment, @@ -62,6 +68,7 @@ import { ScreenJudgeProcess } from "../../desktop/ScreenJudgeProcess.js"; const START = "start_build_computer"; const ACT = "act_on_build_computer"; const CLOSE = "close_build_computer"; +const MEASURE = "measure_build_task"; /** A result with the driver's content blocks, delivered after REA's own. */ export interface ForwardedResult { @@ -112,12 +119,6 @@ export interface BuildComputerSteps { readonly turns: DesktopBuildTurns; } -/** A start still in progress, which shutdown cancels and joins. */ -interface PendingStart { - readonly settled: Promise; - readonly stop: AbortController; -} - const unknownComputer = (operation: string) => new AnalysisInputError(operation, { userMessage: @@ -384,10 +385,9 @@ const startFailure = ( */ export class DesktopBuildComputers { readonly #computers = new Map(); - readonly #starts = new Set(); + readonly #work = new DesktopBuildOwnedWork(); readonly #secrets: DesktopBuildSecrets; readonly #shared: SharedSettings; - #closed = false; constructor( private readonly environment: Readonly, @@ -422,25 +422,30 @@ export class DesktopBuildComputers { AnalysisError > > { - if (this.#closed) - return err( - new AnalysisCancelledError(START, { - cause: new Error("REA is closing its build computers"), - }), - ); - const stop = new AbortController(); - const settled = this.#start( - input, - AbortSignal.any([signal, stop.signal]), - progress, + return this.#work.run(START, signal, (stopping) => + this.#start(input, stopping, progress), + ); + } + + /** Measure a bundle's task in fresh computers of its own; closing REA stops it, tearing the solve's computer down. */ + measure( + input: MeasureRequest, + signal: AbortSignal, + progress?: ProgressReporter, + ): Promise> { + return this.#work.run(MEASURE, signal, (stopping) => + measureInFreshComputers( + input, + { + environment: this.environment, + defaults: this.defaults, + retainCleanup: (owner) => this.#work.retain(owner), + resources: this.#work.resources, + }, + stopping, + progress, + ), ); - const pending = { settled, stop }; - this.#starts.add(pending); - try { - return await settled; - } finally { - this.#starts.delete(pending); - } } async #start( @@ -731,17 +736,11 @@ export class DesktopBuildComputers { /** Cancel and join the starts in progress, then bring every held computer down, as the server closes. */ async closeAll(): Promise { - this.#closed = true; - const starts = [...this.#starts]; - for (const start of starts) - start.stop.abort(new Error("REA is closing its build computers")); - await Promise.allSettled(starts.map((start) => start.settled)); - const results = await Promise.all( + const failures = await this.#work.close(); + for (const result of await Promise.all( [...this.#computers.keys()].map((id) => this.close(id)), - ); - const failures = results.flatMap((result) => - result.ok ? [] : [result.error], - ); + )) + if (!result.ok) failures.push(result.error); try { await this.#secrets.close(); } catch (cause: unknown) { diff --git a/src/application/desktop/DesktopBuildExport.test.ts b/src/application/desktop/DesktopBuildExport.test.ts index a3f6d12c3b..ce7098dcb1 100644 --- a/src/application/desktop/DesktopBuildExport.test.ts +++ b/src/application/desktop/DesktopBuildExport.test.ts @@ -43,19 +43,23 @@ it.each([4, 8])( }, { directory: "/unused", - task: { - name: "t", - prompt: "Task", - checks: [ - { - kind: "lines_present", - file: "f0.bin", - lines: ["x"], - reader: "r0", + tasks: [ + { + task: { + name: "t", + prompt: "Task", + checks: [ + { + kind: "lines_present", + file: "f0.bin", + lines: ["x"], + reader: "r0", + }, + ], + readers, }, - ], - readers, - }, + }, + ], }, new AbortController().signal, ); @@ -103,25 +107,29 @@ it.each([ }, { directory: "/unused", - task: { - name: "t", - prompt: "Task", - checks: [ - { - kind: "lines_present", - file: "f0.bin", - lines: ["x"], - reader: "r0", + tasks: [ + { + task: { + name: "t", + prompt: "Task", + checks: [ + { + kind: "lines_present", + file: "f0.bin", + lines: ["x"], + reader: "r0", + }, + ], + readers: timeouts.map((timeout_s, index) => ({ + id: `r${String(index)}`, + file: `f${String(index)}.bin`, + argv: ["cat", "{file}"], + output: "lines" as const, + timeout_s, + })), }, - ], - readers: timeouts.map((timeout_s, index) => ({ - id: `r${String(index)}`, - file: `f${String(index)}.bin`, - argv: ["cat", "{file}"], - output: "lines" as const, - timeout_s, - })), - }, + }, + ], }, new AbortController().signal, ); diff --git a/src/application/desktop/DesktopBuildExport.ts b/src/application/desktop/DesktopBuildExport.ts index 032462c743..67c9588298 100644 --- a/src/application/desktop/DesktopBuildExport.ts +++ b/src/application/desktop/DesktopBuildExport.ts @@ -13,6 +13,7 @@ import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import type { desktopTaskSchema } from "../../domain/desktop/appTask.js"; import { counted } from "../../domain/desktop/appExportUnowned.js"; +import { END_CHECKPOINT } from "../../domain/desktop/buildCapture.js"; import { SIGN_IN_END, SIGN_IN_START, @@ -58,7 +59,7 @@ const exportResultSchema = z.looseObject({ recompressed_files: z.array(z.string()).default([]), unscrubbed_files: z.array(z.string()), limitations: z.array(z.string()), - task: jsonValueSchema, + tasks: z.array(jsonValueSchema), // An export from before withheld files were listed has none. withheld_files: z .array( @@ -72,9 +73,15 @@ const exportResultSchema = z.looseObject({ consented_files: z.array(z.string()).default([]), }); -/** What the agent asks the export for: its task, the folder here, optionally the environment's name, files to withhold, and whether to seed withheld state with the person's consent. */ -export interface BuildExportRequest { +/** One task the export takes, with the checkpoint its end state is read at. */ +interface BuildExportTask { readonly task: z.output; + readonly end_checkpoint?: string | undefined; +} + +/** What the agent asks the export for: its tasks, the folder here, optionally the environment's name, files to withhold, and whether to seed withheld state with the person's consent. */ +export interface BuildExportRequest { + readonly tasks: readonly BuildExportTask[]; readonly directory: string; readonly name?: string | undefined; readonly withhold?: readonly string[] | undefined; @@ -94,7 +101,7 @@ export type ExportedBuild = { readonly environment: string; readonly files: number; readonly bytes: number; - readonly task: JsonValue; + readonly tasks: JsonValue[]; readonly seed_files: string[]; readonly scrubbed_files: string[]; readonly recompressed_files: string[]; @@ -219,7 +226,7 @@ const writeBundle = async ( ); }; -/** The export's arguments past the capture and the bundle: the task, the ends, the install, and withheld state with its consents. */ +/** The export's arguments past the capture and the bundle: the tasks and each one's end, the ends, the install, and withheld state with its consents. */ const exportArguments = ( capture: { readonly folder: string; @@ -231,14 +238,17 @@ const exportArguments = ( }, request: BuildExportRequest, files: { - readonly task: string; + readonly tasks: readonly string[]; readonly consent: string; readonly withheld: string; readonly download: string; }, ): string[] => [ - "--task", - files.task, + ...files.tasks.flatMap((path) => ["--task", path]), + ...request.tasks.flatMap(({ task, end_checkpoint: end }) => [ + "--task-end", + `${task.name}=${end ?? END_CHECKPOINT}`, + ]), ...ENDS, // The building agent made the seed in this computer for the task. "--synthetic-seed", @@ -324,13 +334,16 @@ interface ExportCapture { /** Whether both sign-in checkpoints were marked, and the consents the person gave. */ readonly signIn: boolean; readonly consents: readonly BuildConsent[]; + /** The checkpoints the capture took, which say whether a task has a near miss its readers also read. */ + readonly checkpoints?: ReadonlySet | undefined; } /** - * Export a stopped capture in the computer with the agent's task, seeded at - * `seed` and checked at `end`, into a new folder of the work folder, then - * bring the bundle back and write it into the caller's folder. A task the - * export refuses is refused with the export's own reason. + * Export a stopped capture in the computer with the agent's tasks, seeded at + * `seed` and each checked at its own end, `end` by default, into a new + * folder of the work folder, then bring the bundle back and write it into + * the caller's folder. A task the export refuses is refused with the + * export's own reason. */ export const exportBuild = async ( computer: ControlledComputer, @@ -342,13 +355,17 @@ export const exportBuild = async ( const { folder } = capture; const attempt = String(capture.attempt); const bundle = `${folder}/bundle-${attempt}`; - const task = `${folder}/task-${attempt}.json`; + const taskFiles = Object.fromEntries( + request.tasks.map(({ task }, index) => [ + `${folder}/task-${attempt}-${String(index + 1)}.json`, + `${JSON.stringify(task, null, 2)}\n`, + ]), + ); const consent = `${folder}/consent-${attempt}.json`; const withheld = `${bundle}-withheld`; const stdout = `${folder}/export-${attempt}.json`; const downloadPath = `${folder}/download-install-${attempt}.json`; const download = capture.downloadInstall; - const taskFile = `${JSON.stringify(request.task, null, 2)}\n`; const completed: WithheldReport[] = []; let published: | { @@ -364,23 +381,30 @@ export const exportBuild = async ( `${folder}/capture`, bundle, ...exportArguments(capture, request, { - task, + tasks: Object.keys(taskFiles), consent, withheld, download: downloadPath, }), ]; + // Each task's readers run at its end and the seed, and at its near miss when the capture took one. const timeoutS = BUILD_REA_TIMEOUT_S + - 2 * - (request.task.readers ?? []).reduce( - (sum, reader) => sum + readerTimeout(reader) + READER_RUNNER_SLACK_S, - 0, - ); + request.tasks.reduce( + (total, { task }) => + total + + (capture.checkpoints?.has(`near-${task.name}`) === true ? 3 : 2) * + (task.readers ?? []).reduce( + (sum, reader) => + sum + readerTimeout(reader) + READER_RUNNER_SLACK_S, + 0, + ), + 0, + ); const options = { stdout, files: { - [task]: taskFile, + ...taskFiles, ...(download === undefined ? {} : { [downloadPath]: `${JSON.stringify(download, null, 2)}\n` }), @@ -435,7 +459,7 @@ export const exportBuild = async ( environment: result.environment, files: written.value.files, bytes: written.value.bytes, - task: result.task, + tasks: result.tasks, seed_files: result.seed_files, scrubbed_files: result.scrubbed_files, recompressed_files: result.recompressed_files, diff --git a/src/application/desktop/DesktopBuildOwnedWork.ts b/src/application/desktop/DesktopBuildOwnedWork.ts new file mode 100644 index 0000000000..f3cffe51df --- /dev/null +++ b/src/application/desktop/DesktopBuildOwnedWork.ts @@ -0,0 +1,75 @@ +import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { err, type Result } from "../../domain/result.js"; +import type { ProviderProcessSupervisor } from "../../process/ProviderProcess.js"; + +/** Work in progress REA must stop and await when it closes its build computers. */ +interface Pending { + readonly settled: Promise; + readonly stop: AbortController; +} + +/** A helper process whose cleanup failed, kept so closing can retry it. */ +type RetainedOwner = Pick; + +/** Long work the build tools own, such as a computer's start or a measure's solves, which closing stops and awaits. */ +export class DesktopBuildOwnedWork { + readonly resources = new ArtifactResourceScope(); + readonly #pending = new Set(); + readonly #retained = new Set(); + #closed = false; + + /** Run work that closing stops, refused once closing has begun. */ + async run( + operation: string, + signal: AbortSignal, + work: (stopping: AbortSignal) => Promise>, + ): Promise> { + if (this.#closed) + return err( + new AnalysisCancelledError(operation, { + cause: new Error("REA is closing its build computers"), + }), + ); + const stop = new AbortController(); + const settled = work(AbortSignal.any([signal, stop.signal])); + const pending = { settled, stop }; + this.#pending.add(pending); + try { + return await settled; + } finally { + this.#pending.delete(pending); + } + } + + /** Keep a helper whose cleanup could not be confirmed, for closing to retry. */ + retain(owner: RetainedOwner): void { + this.#retained.add(owner); + } + + /** Refuse new work, stop what runs and wait for it, then retry kept cleanups; an error for each still failing. */ + async close(): Promise { + this.#closed = true; + const pending = [...this.#pending]; + for (const work of pending) + work.stop.abort(new Error("REA is closing its build computers")); + await Promise.allSettled(pending.map((work) => work.settled)); + const failures: Error[] = []; + try { + await this.resources.retryCleanup(); + } catch (cause) { + failures.push(cause instanceof Error ? cause : new Error(String(cause))); + } + for (const owner of this.#retained) { + const stopped = await owner.stop(); + if (stopped.status === "incomplete") { + failures.push(new Error(stopped.reason)); + continue; + } + owner.dispose(); + this.#retained.delete(owner); + } + return failures; + } +} diff --git a/src/application/desktop/DesktopBuildSolves.test.ts b/src/application/desktop/DesktopBuildSolves.test.ts new file mode 100644 index 0000000000..0cbcf77fc9 --- /dev/null +++ b/src/application/desktop/DesktopBuildSolves.test.ts @@ -0,0 +1,423 @@ +import { + guestFileAnswerCeiling, + guestFileAnswerWorkingBytes, +} from "../../desktop/AgentEnvComputer.js"; +import { OwnedStagedOutput } from "../../filesystem/OwnedStagedOutput.js"; +import { DesktopBuildOwnedWork } from "./DesktopBuildOwnedWork.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { mkdir, readFile, truncate, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; +import * as hostFiles from "../../desktop/HostFileRead.js"; +import { RegularFileCleanupFailure } from "../../filesystem/OwnedRegularFile.js"; +import { + SolveCancelledError, + SolveCleanupError, +} from "../../desktop/AgentEnvSolveHelper.js"; + +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import type { SolveReport } from "../../desktop/AgentEnvSolveHelper.js"; +import { measureBuildTask, type Solver } from "./DesktopBuildSolves.js"; + +/** A bundle holding stand-in steps and verifiers for these tasks, and a README. */ +const bundleWith = async (...tasks: string[]) => { + const bundle = await createTestTempDirectory("measured-"); + for (const task of tasks) { + await mkdir(join(bundle, "tasks"), { recursive: true }); + await mkdir(join(bundle, "artifacts", `${task}-check`), { + recursive: true, + }); + await writeFile(join(bundle, "tasks", `${task}.json`), `["${task}"]\n`); + await writeFile( + join(bundle, "artifacts", `${task}-check`, "verify.py"), + `# ${task}\n`, + ); + } + await writeFile(join(bundle, "README.md"), "# App\n"); + return bundle; +}; + +const passed: SolveReport = { + outcome: "passed", + instance_id: null, + seconds: 1, + checks: [{ id: "year", passed: true, description: "year" }], + turns: 10, + usd: 0.5, + agentenv: "0.9.1431", + error: null, +}; + +/** A solver that passes every solve, counting its calls. */ +const counting = () => { + const calls: string[] = []; + const solver: Solver = async (request) => { + calls.push(request.task); + await new Promise((resolve) => setTimeout(resolve, 1)); + return passed; + }; + return { calls, solver }; +}; + +const request = (bundle: string, task: string) => ({ + bundle, + task, + model: "m", + agent: "claude-code", + budget_usd: 10, +}); + +describe("measureBuildTask's record of a bundle", () => { + it("reports a host capacity refusal rather than claiming the task is missing", async () => { + const bundle = await bundleWith("large"); + await truncate( + join(bundle, "tasks", "large.json"), + guestFileAnswerWorkingBytes(guestFileAnswerCeiling()) + 1, + ); + const { calls, solver } = counting(); + const result = await measureBuildTask( + request(bundle, "large"), + solver, + new AbortController().signal, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected capacity refusal"); + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "capacity", + }); + expect(calls).toEqual([]); + }); + + it("measures an exported task larger than 4 MiB", async () => { + const bundle = await bundleWith("large"); + await writeFile( + join(bundle, "tasks", "large.json"), + JSON.stringify(["x".repeat(4 * 1024 * 1024)]), + ); + const { calls, solver } = counting(); + const result = await measureBuildTask( + request(bundle, "large"), + solver, + new AbortController().signal, + ); + expect(result.ok).toBe(true); + expect(calls).toHaveLength(3); + }); + + it("keeps both records when two tasks of one bundle are measured at once", async () => { + const bundle = await bundleWith("add-eggs", "add-bread"); + const { solver } = counting(); + const signal = new AbortController().signal; + + const results = await Promise.all([ + measureBuildTask(request(bundle, "add-eggs"), solver, signal), + measureBuildTask(request(bundle, "add-bread"), solver, signal), + ]); + + for (const result of results) expect(result.ok).toBe(true); + const recorded = JSON.parse( + await readFile(join(bundle, "difficulty.json"), "utf8"), + ) as { tasks: Record }; + expect(Object.keys(recorded.tasks).sort()).toEqual([ + "add-bread", + "add-eggs", + ]); + const readme = await readFile(join(bundle, "README.md"), "utf8"); + expect(readme).toContain("`add-eggs`"); + expect(readme).toContain("`add-bread`"); + }); + + it.each([ + ["not JSON", "{not json"], + ["records it cannot read", '{"tasks": {"add-bread": {"model": 1}}}'], + ])( + "refuses a difficulty file holding %s before any solve, and leaves it as it was", + async (_label, content) => { + const bundle = await bundleWith("add-eggs"); + await writeFile(join(bundle, "difficulty.json"), content); + const { calls, solver } = counting(); + + const result = await measureBuildTask( + request(bundle, "add-eggs"), + solver, + new AbortController().signal, + ); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error.userMessage).toContain("difficulty.json"); + expect(calls).toEqual([]); + expect(await readFile(join(bundle, "difficulty.json"), "utf8")).toBe( + content, + ); + expect(await readFile(join(bundle, "README.md"), "utf8")).toBe("# App\n"); + }, + ); +}); + +it("retains a failed staged write and retries its cleanup on build shutdown", async () => { + const bundle = await bundleWith("add-eggs"); + const work = new DesktopBuildOwnedWork(); + const primary = new Error("publication failed"); + const cleanup = new Error("cleanup failed"); + const close = OwnedStagedOutput.prototype.close; + let owner: OwnedStagedOutput | undefined; + let attempts = 0; + const publication = vi + .spyOn(OwnedStagedOutput.prototype, "assertReadyForPublication") + .mockRejectedValueOnce(primary); + const release = vi + .spyOn(OwnedStagedOutput.prototype, "close") + .mockImplementation(async function (this: OwnedStagedOutput) { + owner = this; + if (++attempts === 1) throw cleanup; + await close.call(this); + }); + try { + const result = await measureBuildTask( + request(bundle, "add-eggs"), + counting().solver, + new AbortController().signal, + undefined, + work.resources, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failure"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanup?.reason).toContain("cleanup failed"); + expect(result.error.cause).toMatchObject({ + outcome: { ok: false, error: primary }, + }); + expect(result.error.partialObservation).toMatchObject({ + solves: [{}, {}, {}], + }); + expect(await work.close()).toEqual([]); + expect(result.error.cleanupIncomplete).toBe(false); + expect(attempts).toBe(2); + } finally { + publication.mockRestore(); + release.mockRestore(); + await owner?.close(); + } +}); + +it("retains completed paid solves when a later helper cannot be cleaned up", async () => { + const bundle = await bundleWith("add-eggs"); + let calls = 0; + const solver: Solver = async () => { + calls += 1; + if (calls === 1) return passed; + throw new SolveCleanupError( + passed, + ["process group 123"], + { reason: "cleanup incomplete" }, + "measure_build_task", + ); + }; + const result = await measureBuildTask( + request(bundle, "add-eggs"), + solver, + new AbortController().signal, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cleanup failure"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(projectAnalysisError(result.error)).toMatchObject({ + details: { + partial_observation: { + kind: "partial-task-measurement", + solves: [{ usd: 0.5 }, { usd: 0.5 }], + }, + }, + }); + expect(result.error.partialObservation).toMatchObject({ + kind: "partial-task-measurement", + bundle, + task: "add-eggs", + solves: [ + { outcome: "passed", turns: 10, usd: 0.5 }, + { outcome: "passed", turns: 10, usd: 0.5 }, + ], + }); +}); + +it("returns a typed failure if difficulty preflight encounters an I/O error", async () => { + const bundle = await bundleWith("add-eggs"); + const original = hostFiles.readHostFile; + const spy = vi + .spyOn(hostFiles, "readHostFile") + .mockImplementation(async (path, ...rest) => { + if (path.endsWith("difficulty.json")) + throw Object.assign(new Error("disk failure"), { code: "EIO" }); + return original(path, ...rest); + }); + try { + const { calls, solver } = counting(); + const result = await measureBuildTask( + request(bundle, "add-eggs"), + solver, + new AbortController().signal, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected I/O failure"); + expect(result.error).toMatchObject({ + _tag: "ProviderAdapterError", + cause: expect.objectContaining({ code: "EIO" }), + }); + expect(calls).toEqual([]); + } finally { + spy.mockRestore(); + } +}); + +it.each([ + ["tasks/add-eggs.json", 1, false], + ["difficulty.json", 1, false], + ["README.md", 2, true], +] as const)( + "retains the failed close of %s, including cancellation during finalization", + async (suffix, occurrence, cancel) => { + const bundle = await bundleWith("add-eggs"); + const work = new DesktopBuildOwnedWork(); + const controller = new AbortController(); + const reason = new Error("cancel during close"); + const original = hostFiles.readHostFile; + let reads = 0; + let closes = 0; + const spy = vi + .spyOn(hostFiles, "readHostFile") + .mockImplementation(async (path, ...rest) => { + if (path.endsWith(suffix) && ++reads === occurrence) { + if (cancel) controller.abort(reason); + throw hostFiles.hostCloseFailure( + new RegularFileCleanupFailure({ + path, + owner: { + close: async () => { + if (++closes === 1) throw new Error("still busy"); + }, + } as never, + outcome: { kind: "completed", value: Buffer.from("read") }, + cleanupCause: new Error("close failed"), + ...(cancel ? { completionCancellation: { reason } } : {}), + }), + ); + } + return original(path, ...rest); + }); + try { + const result = await measureBuildTask( + request(bundle, "add-eggs"), + counting().solver, + controller.signal, + undefined, + work.resources, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cleanup failure"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain(join(bundle, suffix)); + if (cancel) { + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.partialObservation).toMatchObject({ + kind: "partial-task-measurement", + solves: [{}, {}, {}], + }); + } + expect(await work.close()).toHaveLength(1); + expect(result.error.cleanupIncomplete).toBe(true); + expect(await work.close()).toEqual([]); + expect(result.error.cleanupIncomplete).toBe(false); + } finally { + spy.mockRestore(); + await work.close(); + } + }, +); + +it("keeps an input read failure primary when cancellation also arrives during close", async () => { + const bundle = await bundleWith("add-eggs"); + const work = new DesktopBuildOwnedWork(); + const controller = new AbortController(); + const original = hostFiles.readHostFile; + const spy = vi + .spyOn(hostFiles, "readHostFile") + .mockImplementation(async (path, ...rest) => { + if (!path.endsWith("tasks/add-eggs.json")) return original(path, ...rest); + controller.abort(new Error("cancel")); + throw hostFiles.hostCloseFailure( + new RegularFileCleanupFailure({ + path, + owner: { close: async () => {} } as never, + outcome: { kind: "failed", cause: new Error("read failed") }, + cleanupCause: new Error("close failed"), + }), + ); + }); + try { + const result = await measureBuildTask( + request(bundle, "add-eggs"), + counting().solver, + controller.signal, + undefined, + work.resources, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failure"); + expect(result.error._tag).toBe("ProviderAdapterError"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(await work.close()).toEqual([]); + expect(result.error.cleanupIncomplete).toBe(false); + } finally { + spy.mockRestore(); + await work.close(); + } +}); + +it("retains a solve when cancellation arrives after the helper report", async () => { + const bundle = await bundleWith("add-eggs"); + const reason = { stage: "final cleanup" }; + const result = await measureBuildTask( + request(bundle, "add-eggs"), + async () => { + throw new SolveCancelledError(passed, "measure_build_task", reason); + }, + new AbortController().signal, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + partialObservation: { + kind: "partial-task-measurement", + solves: [{ turns: 10, usd: 0.5 }], + }, + }); +}); + +it("retains completed solves when the destination becomes unreadable before publication", async () => { + const bundle = await bundleWith("add-eggs"); + let calls = 0; + const result = await measureBuildTask( + request(bundle, "add-eggs"), + async () => { + calls += 1; + if (calls === 3) + await writeFile(join(bundle, "difficulty.json"), "invalid"); + return passed; + }, + new AbortController().signal, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected refusal"); + expect(result.error.partialObservation).toMatchObject({ + kind: "partial-task-measurement", + solves: [{ usd: 0.5 }, { usd: 0.5 }, { usd: 0.5 }], + }); + expect(await readFile(join(bundle, "difficulty.json"), "utf8")).toBe( + "invalid", + ); +}); diff --git a/src/application/desktop/DesktopBuildSolves.ts b/src/application/desktop/DesktopBuildSolves.ts new file mode 100644 index 0000000000..0a08b6e839 --- /dev/null +++ b/src/application/desktop/DesktopBuildSolves.ts @@ -0,0 +1,430 @@ +import { createHash } from "node:crypto"; +import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { + ArtifactFileCleanupFailure, + bindArtifactFileCleanup, +} from "../../artifacts/ArtifactFileOperation.js"; +import { RegularFileCapacityError } from "../../filesystem/readStableFileBytes.js"; +import { + guestFileAnswerCeiling, + guestFileAnswerWorkingBytes, +} from "../../desktop/AgentEnvComputer.js"; +import { writeTextFile } from "../JsonFiles.js"; +import { join, resolve } from "node:path"; + +import { z } from "zod"; + +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisResourceConstraintError, +} from "../../domain/analysisErrorCore.js"; +import { + difficultyRecord, + difficultySection, + nextSolve, + withDifficultySection, + type DifficultyRecord, + type SolveRecord, + type StopReason, +} from "../../domain/desktop/taskDifficulty.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { safeParseUtf8Json } from "../../domain/safeJson.js"; +import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js"; +import { + runSolveHelper, + SolveCleanupError, + SolveCancelledError, + type SolveReport, + type SolveCleanupOwner, +} from "../../desktop/AgentEnvSolveHelper.js"; +import { + resolveBuildComputerEnvironment, + type BuildComputerDefaults, +} from "../../desktop/BuildComputerEnvironment.js"; +import { readHostFile } from "../../desktop/HostFileRead.js"; +import { PRODUCT_IDENTITY } from "../../identity.js"; +import { + silentProgressReporter, + type ProgressReporter, +} from "../ProgressReporter.js"; +import { errorText } from "./DesktopBuildCommands.js"; + +const MEASURE = "measure_build_task"; +/** The file at the bundle's root that records each measured task. */ +const DIFFICULTY_FILE = "difficulty.json"; +const README_FILE = "README.md"; +/** The most bytes REA reads of a task's steps, verifier, README or difficulty file. */ +const FILE_BYTES = 4 * 1024 * 1024; + +/** What the agent asks the measure for. */ +export interface MeasureRequest { + readonly bundle: string; + readonly task: string; + readonly model: string; + readonly agent: string; + readonly budget_usd: number; +} + +/** One solve of the task in a fresh computer. */ +export type Solver = ( + request: Pick, + signal: AbortSignal, +) => Promise; + +/** What the measure reports: the record it wrote and where. */ +export type MeasuredTask = Omit & { + readonly bundle: string; + readonly task: string; + readonly solves: DifficultyRecord["solves"]; + readonly difficulty_file: string; +}; + +const refused = (userMessage: string, remediationAction: string) => + new AnalysisInputError(MEASURE, { + userMessage, + remediationAction, + conformsToInputContract: true, + }); + +/** The records a difficulty file holds, each read only as far as the README names it. */ +const recordsSchema = z.looseObject({ + tasks: z.record( + z.string(), + z.looseObject({ + model: z.string(), + passes: z.int(), + k: z.int(), + band: z.enum(["in_band", "too_easy", "too_hard", "undecided"]), + usd: z.number(), + }), + ), +}); + +/** The task's steps and verifier as the export wrote them, digested, or why the bundle has none. */ +const taskDigest = async ( + bundle: string, + task: string, + signal: AbortSignal, +): Promise> => { + const paths = [`tasks/${task}.json`, `artifacts/${task}-check/verify.py`]; + const hash = createHash("sha256"); + const maximum = guestFileAnswerWorkingBytes(guestFileAnswerCeiling()); + for (const path of paths) { + try { + const file = await readHostFile( + join(bundle, path), + maximum, + signal, + true, + ); + if (!Buffer.isBuffer(file)) + return err( + refused( + `The bundle at ${bundle} holds no readable ${path}.`, + "Name a task the bundle's export wrote, with its steps and verifier.", + ), + ); + hash.update(file); + } catch (cause: unknown) { + if (cause instanceof RegularFileCapacityError) + return err( + new AnalysisResourceConstraintError( + MEASURE, + "capacity", + "Task file exceeds the host read budget", + { max_bytes: maximum, path: cause.path }, + { cause }, + ), + ); + throw cause; + } + } + return ok(hash.digest("hex")); +}; + +/** One finished report as the record keeps it. */ +const solveRecord = ( + report: Extract, +): SolveRecord => ({ + outcome: report.outcome, + turns: report.turns, + usd: report.usd, + seconds: report.seconds, + failed_checks: + report.outcome === "passed" || report.outcome === "scored below 1" + ? report.checks.filter((check) => !check.passed).map((check) => check.id) + : [], + instance_id: report.instance_id, + error: report.error, +}); + +/** The bundle's difficulty records and README as they stand. */ +interface BundleRecords { + readonly tasks: Readonly< + Record["tasks"][string]> + >; + readonly readme: string; +} + +/** The bundle's records, a missing file read as none; a file REA cannot read or parse is refused, so it is never replaced. */ +const readRecords = async ( + bundle: string, + signal: AbortSignal, +): Promise> => { + const unreadable = (file: string, why: string) => + err( + refused( + `The bundle's ${file} ${why}, so REA would replace records it cannot keep.`, + `Repair or move ${join(bundle, file)} and measure again.`, + ), + ); + const records = await readHostFile( + join(bundle, DIFFICULTY_FILE), + FILE_BYTES, + signal, + ); + const readme = await readHostFile( + join(bundle, README_FILE), + FILE_BYTES, + signal, + ); + if (records === "unreadable") + return unreadable(DIFFICULTY_FILE, "cannot be read"); + if (readme === "unreadable") return unreadable(README_FILE, "cannot be read"); + let tasks: BundleRecords["tasks"] = {}; + if (records !== "missing") { + const json = safeParseUtf8Json(records); + const parsed = json.ok ? recordsSchema.safeParse(json.value) : undefined; + if (parsed?.success !== true) + return unreadable(DIFFICULTY_FILE, "is not a difficulty file REA reads"); + tasks = parsed.data.tasks; + } + return ok({ + tasks, + readme: readme === "missing" ? "" : readme.toString("utf8"), + }); +}; + +/** The record writes under way, by bundle, so a measure merges with the one before it rather than racing it. */ +const recording = new Map>(); + +/** Run `write` after every earlier write to the same bundle by this REA. */ +const afterEarlierWrites = async ( + bundle: string, + write: () => Promise, +): Promise => { + const key = resolve(bundle); + const earlier = recording.get(key) ?? Promise.resolve(); + const mine = earlier.then(write, write); + const settled = mine.catch(() => undefined); + recording.set(key, settled); + try { + return await mine; + } finally { + if (recording.get(key) === settled) recording.delete(key); + } +}; + +/** Record the task's measure in the bundle's difficulty file, beside other tasks', and in its README. */ +const recordMeasure = ( + bundle: string, + task: string, + record: DifficultyRecord, + signal: AbortSignal, + resources: ArtifactResourceScope, +): Promise> => + afterEarlierWrites(bundle, async () => { + const current = await readRecords(bundle, signal); + if (!current.ok) return current; + const tasks = { ...current.value.tasks, [task]: record }; + const path = join(bundle, DIFFICULTY_FILE); + const saved = await writeTextFile( + `${JSON.stringify({ tasks }, null, 2)}\n`, + path, + true, + resources, + ); + if (!saved.ok) return saved; + const readme = await writeTextFile( + withDifficultySection(current.value.readme, difficultySection(tasks)), + join(bundle, README_FILE), + true, + resources, + ); + if (!readme.ok) return readme; + return ok(path); + }); + +/** + * Measure a bundle's task: solve it in fresh computers, 3 times, then 2 more + * unless they passed none or all, stopping before a solve the budget cannot + * cover, and record the band, cost, turns and failures per check in the + * bundle's difficulty file and README. + */ +export const measureBuildTask = async ( + request: MeasureRequest, + solver: Solver, + signal: AbortSignal, + progress: ProgressReporter = silentProgressReporter, + resources: ArtifactResourceScope = new ArtifactResourceScope(), +): Promise> => { + const solves: SolveRecord[] = []; + const retainSolves = (failure: AnalysisError): AnalysisError => { + if (solves.length > 0) + failure.retainPartialObservation({ + kind: "partial-task-measurement", + bundle: request.bundle, + task: request.task, + solves: [...solves], + }); + return failure; + }; + let agentenv: string | null = null; + let stopped: StopReason = "decided"; + try { + const digest = await taskDigest(request.bundle, request.task, signal); + if (!digest.ok) return digest; + // Refuse records REA could not keep before any solve is paid for. + const records = await readRecords(request.bundle, signal); + if (!records.ok) return records; + for (;;) { + const decision = nextSolve(solves, request.budget_usd); + if (!decision.next) { + stopped = decision.stopped; + break; + } + await progress + .report({ + phase: MEASURE, + completed: solves.length, + total: null, + message: `solve ${String(solves.length + 1)} of ${request.task} with ${request.model}`, + }) + .catch(() => undefined); + const report = await solver(request, signal); + if (!("outcome" in report)) + return err( + retainSolves( + refused( + `A solve of ${request.task} did not run: ${report.error}`, + "Fix what the error names, such as the model endpoint agentenv needs (LITELLM_BASE_URL and LITELLM_API_KEY where REA starts, or [model] in .agentenv/config.toml), and measure again.", + ), + ), + ); + agentenv = report.agentenv ?? agentenv; + solves.push(solveRecord(report)); + } + const record = difficultyRecord({ + model: request.model, + agent: request.agent, + solves, + budgetUsd: request.budget_usd, + stopped, + taskSha256: digest.value, + versions: { rea: PRODUCT_IDENTITY.packageVersion, agentenv }, + measuredAt: new Date().toISOString(), + }); + const file = await recordMeasure( + request.bundle, + request.task, + record, + signal, + resources, + ); + if (!file.ok) return err(retainSolves(file.error)); + const { versions: _versions, ...reported } = record; + return ok({ + bundle: request.bundle, + task: request.task, + ...reported, + difficulty_file: file.value, + }); + } catch (cause: unknown) { + if ( + (cause instanceof SolveCleanupError || + cause instanceof SolveCancelledError) && + cause.report !== undefined && + "outcome" in cause.report + ) + solves.push(solveRecord(cause.report)); + if (cause instanceof ArtifactFileCleanupFailure) { + const cleanup = cause.cleanup ?? { + reason: "an input handle could not be closed", + resources: [cause.fileCleanupFailure.path], + }; + for (const lease of cause.cleanupLeases ?? [cause.cleanupLease]) + resources.retain({ + kind: "lease", + resource: cause.fileCleanupFailure.path, + lease, + }); + const cancelled = + cause.cancellation !== undefined || + (signal.aborted && + cause.fileCleanupFailure.outcome.kind === "completed"); + const failure = cancelled + ? new AnalysisCancelledError(MEASURE, { + cause: + cause.cancellation === undefined + ? signal.reason + : cause.cancellation.reason, + cleanup, + }) + : new ProviderAdapterError(AGENTENV_PROVIDER, MEASURE, { + userMessage: `Measuring ${request.task} could not close an input file.`, + cause, + cleanup, + }); + bindArtifactFileCleanup(cause, failure); + return err(retainSolves(failure)); + } + const failure = + cause instanceof AnalysisError + ? cause + : signal.aborted + ? new AnalysisCancelledError(MEASURE, { cause: signal.reason }) + : new ProviderAdapterError(AGENTENV_PROVIDER, MEASURE, { + userMessage: `Measuring ${request.task} failed after ${String(solves.length)} solve(s): ${errorText(cause)}`, + remediationAction: + "Check the bundle files and host I/O error, then retry or retain the completed solve results.", + cause, + }); + return err(retainSolves(failure)); + } +}; + +/** Measure a task with each solve run by REA's helper in the build host's agentenv, with the build settings' defaults. */ +export const measureInFreshComputers = async ( + request: MeasureRequest, + host: { + readonly environment: Readonly; + readonly defaults: BuildComputerDefaults; + readonly retainCleanup: (owner: SolveCleanupOwner) => void; + readonly resources: ArtifactResourceScope; + }, + signal: AbortSignal, + progress?: ProgressReporter, +): Promise> => { + const environment = await resolveBuildComputerEnvironment( + host.environment, + process.platform, + host.defaults, + ); + return measureBuildTask( + request, + (solve, turn) => + runSolveHelper({ + ...solve, + environment, + signal: turn, + operation: MEASURE, + retainCleanup: host.retainCleanup, + }), + signal, + progress, + host.resources, + ); +}; diff --git a/src/application/desktop/DesktopBuildSummary.ts b/src/application/desktop/DesktopBuildSummary.ts index e3f46e7d9f..472973d61f 100644 --- a/src/application/desktop/DesktopBuildSummary.ts +++ b/src/application/desktop/DesktopBuildSummary.ts @@ -20,12 +20,7 @@ import { } from "../../domain/jsonValue.js"; import { err, ok, type Result } from "../../domain/result.js"; import type { ControlledComputer } from "../../desktop/BuildComputerController.js"; -import { - ENDS, - readOutput, - reportedFailure, - runRea, -} from "./DesktopBuildCommands.js"; +import { readOutput, reportedFailure, runRea } from "./DesktopBuildCommands.js"; const STOP = "stop_build_capture"; @@ -127,7 +122,39 @@ const draftNote = ({ ...(error === undefined ? {} : { error }), }); -/** A stopped capture: its checkpoints, what it found of the app and the draft between seed and end. */ +/** A task draft between two checkpoints. */ +type Draft = { + readonly seed_checkpoint: string; + readonly end_checkpoint: string; + readonly files: { + readonly file: string; + readonly role: string; + readonly change: JsonValue; + /** The values the change added, changed and removed, at the pointers a check names them by. */ + readonly values: DraftValue[]; + readonly values_cut: boolean; + }[]; + readonly task: { + readonly name: string; + readonly prompt: string; + readonly checks: Readonly>[]; + }; + /** Checks that the rest stays as it was, outside the task until an author adds them. */ + readonly guards: Readonly>[]; + /** Why the draft has no files, or why a file's suggestions are thin. */ + readonly notes: DraftNote[]; + /** Files the sign-in window changed, which the draft leaves out and the export withholds. */ + readonly withheld_files: string[]; +}; + +/** A later task's draft, from the end checkpoint before its own, or why there is none. */ +type LaterDraft = { + readonly end_checkpoint: string; + readonly draft: Draft | null; + readonly draft_refusal?: string; +}; + +/** A stopped capture: its checkpoints, what it found of the app, the draft between seed and end and those of later tasks. */ export type StoppedCapture = { readonly end_reason: string; readonly checkpoints: { @@ -143,57 +170,41 @@ export type StoppedCapture = { readonly sources: string[]; }; readonly limitations: string[]; - readonly draft: { - readonly seed_checkpoint: string; - readonly end_checkpoint: string; - readonly files: { - readonly file: string; - readonly role: string; - readonly change: JsonValue; - /** The values the change added, changed and removed, at the pointers a check names them by. */ - readonly values: DraftValue[]; - readonly values_cut: boolean; - }[]; - readonly task: { - readonly name: string; - readonly prompt: string; - readonly checks: Readonly>[]; - }; - /** Checks that the rest stays as it was, outside the task until an author adds them. */ - readonly guards: Readonly>[]; - /** Why the draft has no files, or why a file's suggestions are thin. */ - readonly notes: DraftNote[]; - /** Files the sign-in window changed, which the draft leaves out and the export withholds. */ - readonly withheld_files: string[]; - } | null; + readonly draft: Draft | null; readonly draft_refusal?: string; + /** One draft per later task's end checkpoint, `end-`, from the end checkpoint before it. */ + readonly later_drafts: LaterDraft[]; }; -/** A draft cannot safely be handed off after this server has used a secret. */ -export const secretDraftRefusal = { +/** One draft withheld because this server has used a declared secret. */ +const withheldDraft = { draft: null, draft_refusal: "Automatic task drafting is unavailable after this server used a declared secret. The capture remains available; write the task manually and export it.", } as const; -/** The task draft between `seed` and `end`, or why there is none. */ -const draftTask = async ( +/** A draft cannot safely be handed off after this server has used a secret: neither the first task's nor a later one's. */ +export const secretDraftRefusal: Pick< + StoppedCapture, + "draft" | "draft_refusal" | "later_drafts" +> = { ...withheldDraft, later_drafts: [] }; + +/** The task draft between two checkpoints, or why there is none; none once this server has used a declared secret. */ +const draftBetween = async ( computer: ControlledComputer, folder: string, - checkpoints: ReadonlySet, + ends: { + readonly from: string; + readonly to: string; + readonly file: string; + /** Whether the capture marked both sign-in checkpoints, whose files the draft leaves out. */ + readonly signIn: boolean; + }, hasDeclaredSecret: () => boolean, signal: AbortSignal, -): Promise> => { - if (hasDeclaredSecret()) return secretDraftRefusal; - const missing = [SEED_CHECKPOINT, END_CHECKPOINT].filter( - (name) => !checkpoints.has(name), - ); - if (missing.length > 0) - return { - draft: null, - draft_refusal: `No draft: the capture has no ${missing.join(" or ")} checkpoint; capture again and mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the edit is.`, - }; - const stdout = `${folder}/draft.json`; +): Promise> => { + if (hasDeclaredSecret()) return withheldDraft; + const stdout = `${folder}/${ends.file}`; const withheld = Symbol("secret draft withheld"); let bytes: number; try { @@ -209,8 +220,11 @@ const draftTask = async ( [ "describe-desktop-application-task", `${folder}/capture`, - ...ENDS, - ...(checkpoints.has(SIGN_IN_START) && checkpoints.has(SIGN_IN_END) + "--seed-checkpoint", + ends.from, + "--end-checkpoint", + ends.to, + ...(ends.signIn ? ["--withhold-from", SIGN_IN_START, "--withhold-to", SIGN_IN_END] : []), "--json", @@ -219,20 +233,20 @@ const draftTask = async ( ); bytes = result.stdoutBytes; } catch (cause: unknown) { - if (cause === withheld) return secretDraftRefusal; - if (hasDeclaredSecret() && !signal.aborted) return secretDraftRefusal; + if (cause === withheld) return withheldDraft; + if (hasDeclaredSecret() && !signal.aborted) return withheldDraft; throw cause; } - if (hasDeclaredSecret()) return secretDraftRefusal; + if (hasDeclaredSecret()) return withheldDraft; const output = await readOutput(computer, stdout, bytes, signal); - if (hasDeclaredSecret()) return secretDraftRefusal; + if (hasDeclaredSecret()) return withheldDraft; if (output.kind === "failure") return { draft: null, draft_refusal: output.failure.message }; const draft = draftResultSchema.parse(output.result); return { draft: { - seed_checkpoint: SEED_CHECKPOINT, - end_checkpoint: END_CHECKPOINT, + seed_checkpoint: ends.from, + end_checkpoint: ends.to, files: draft.files.map(({ file, role, change, values, values_cut }) => ({ file, role, @@ -252,6 +266,68 @@ const draftTask = async ( }; }; +/** Whether a checkpoint ends a later task: `end-`. */ +const endsLaterTask = (name: string): boolean => + name.startsWith(`${END_CHECKPOINT}-`); + +/** + * The draft between `seed` and `end`, or why there is none, and one draft + * per later task's end checkpoint taken after `end`, each from the end + * checkpoint before it, so it shows only that task's change; none once this + * server has used a declared secret. + */ +const draftTasks = async ( + computer: ControlledComputer, + folder: string, + checkpoints: readonly string[], + hasDeclaredSecret: () => boolean, + signal: AbortSignal, +): Promise< + Pick +> => { + if (hasDeclaredSecret()) return secretDraftRefusal; + const signIn = + checkpoints.includes(SIGN_IN_START) && checkpoints.includes(SIGN_IN_END); + const missing = [SEED_CHECKPOINT, END_CHECKPOINT].filter( + (name) => !checkpoints.includes(name), + ); + if (missing.length > 0) + return { + draft: null, + draft_refusal: `No draft: the capture has no ${missing.join(" or ")} checkpoint; capture again and mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the edit is.`, + later_drafts: [], + }; + const first = await draftBetween( + computer, + folder, + { from: SEED_CHECKPOINT, to: END_CHECKPOINT, file: "draft.json", signIn }, + hasDeclaredSecret, + signal, + ); + const later: LaterDraft[] = []; + let from: string = END_CHECKPOINT; + for (const name of checkpoints.slice(checkpoints.indexOf(END_CHECKPOINT))) { + if (!endsLaterTask(name)) continue; + later.push({ + end_checkpoint: name, + ...(await draftBetween( + computer, + folder, + { + from, + to: name, + file: `draft-${String(later.length + 2)}.json`, + signIn, + }, + hasDeclaredSecret, + signal, + )), + }); + from = name; + } + return { ...first, later_drafts: later }; +}; + /** * What a capture that has exited recorded, read from its own Evidence: the * checkpoints it holds, which the draft and the export trust over the @@ -305,10 +381,10 @@ export const summarizeCapture = async ( sources: [...installed.sources], }, limitations: [...recorded.limitations], - ...(await draftTask( + ...(await draftTasks( computer, capture.folder, - checkpoints, + recorded.state.checkpoints.map((checkpoint) => checkpoint.name), hasDeclaredSecret, signal, )), diff --git a/src/application/desktop/DesktopCheckpointEnds.test.ts b/src/application/desktop/DesktopCheckpointEnds.test.ts index c5102faa8d..9182d18850 100644 --- a/src/application/desktop/DesktopCheckpointEnds.test.ts +++ b/src/application/desktop/DesktopCheckpointEnds.test.ts @@ -121,7 +121,7 @@ describe("exportDesktopApplication between named checkpoints", () => { expect(result.normalized_result).toMatchObject({ checkpoints: { seed: "seed", end: "end" }, seed_files: [".config/notes/settings.ini", NOTE], - task: { name: "add-eggs", failing_on_seed: ["eggs"] }, + tasks: [{ name: "add-eggs", failing_on_seed: ["eggs"] }], }); const seed = join(directory, "envs/notes/seed"); expect(await readFile(join(seed, NOTE), "utf8")).toBe(SAVED_NOTE); @@ -157,7 +157,7 @@ describe("exportDesktopApplication between named checkpoints", () => { if (atExit.result instanceof Error) throw atExit.result; expect(atExit.result.normalized_result).toMatchObject({ checkpoints: { seed: "seed", end: "exit" }, - task: { failing_on_seed: ["eggs", "wrap"] }, + tasks: [{ failing_on_seed: ["eggs", "wrap"] }], }); }); @@ -184,7 +184,7 @@ describe("exportDesktopApplication between named checkpoints", () => { if (between.result instanceof Error) throw between.result; expect(between.result.normalized_result).toMatchObject({ checkpoints: { seed: "seed", end: "end" }, - task: { failing_on_seed: ["eggs"] }, + tasks: [{ failing_on_seed: ["eggs"] }], }); expect( (fromBaseline.result as { userMessage: string }).userMessage, diff --git a/src/application/desktop/DesktopExportIdentity.test.ts b/src/application/desktop/DesktopExportIdentity.test.ts index f1e26513b7..4fa65cb802 100644 --- a/src/application/desktop/DesktopExportIdentity.test.ts +++ b/src/application/desktop/DesktopExportIdentity.test.ts @@ -232,7 +232,7 @@ describe("exportDesktopApplication of a capture that names its host", () => { if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { failing_on_seed: ["note"] }, + tasks: [{ failing_on_seed: ["note"] }], }); }); @@ -299,7 +299,7 @@ describe("exportDesktopApplication where names, syntax or an unparsed value spel if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { failing_on_seed: ["check-1"] }, + tasks: [{ failing_on_seed: ["check-1"] }], }); }, ); diff --git a/src/application/desktop/DesktopExportReaders.ts b/src/application/desktop/DesktopExportReaders.ts index 014fd7df5a..2524428ca3 100644 --- a/src/application/desktop/DesktopExportReaders.ts +++ b/src/application/desktop/DesktopExportReaders.ts @@ -45,12 +45,12 @@ export interface ReaderSources { readonly screens: string | undefined; } -/** The screenshot a build computer took at the seed or the end checkpoint, too large to ship, or undefined when there is none. */ +/** The screenshot a build computer took at the seed or at a checkpoint's position, a task's end, too large to ship, or undefined when there is none. */ const screenAt = async ( sources: ReaderSources, - end: keyof StateEnds, + at: "seed" | number, ): Promise => { - const name = sources.checkpoints[sources.ends[end]]; + const name = sources.checkpoints[at === "seed" ? sources.ends.seed : at]; if (sources.screens === undefined || name === undefined) return undefined; const path = join(sources.screens, `${name}.jpg`); const found = await stat(path).catch(() => undefined); @@ -58,15 +58,15 @@ const screenAt = async ( return found.size > SCREEN_EXAMPLE_MAX_BYTES ? "too-large" : readFile(path); }; -/** A reader's file at the seed, as the bundle ships it, or at the end, as retained. */ +/** A reader's file at the seed, as the bundle ships it, or at a checkpoint's position, a task's end or near miss, as retained. */ const readerFile = async ( sources: ReaderSources, content: RetainedContent, file: string, - end: keyof StateEnds, + at: "seed" | number, ): Promise => { - const { copies, files, ends, identity, signal } = sources; - if (end === "seed") { + const { copies, files, checkpoints, identity, signal } = sources; + if (at === "seed") { const copy = copies.find(({ seed }) => seed?.home === file); return copy === undefined ? { kind: "absent" } @@ -76,12 +76,12 @@ const readerFile = async ( }; } const path = files.get(file); - const version = path === undefined ? undefined : versionAt(path, ends.end); + const version = path === undefined ? undefined : versionAt(path, at); if (version?.present !== true) return { kind: "absent" }; if (version.sha256 === null) return { kind: "error", - error: `~/${file} was not retained at the end checkpoint, so there is nothing to read`, + error: `~/${file} was not retained at the ${checkpoints[at] ?? "end"} checkpoint, so there is nothing to read`, }; const read = await content.read(version.sha256, signal); return read.ok @@ -92,19 +92,19 @@ const readerFile = async ( }; }; -/** Each reader's view of its file at one end, run on this host, where the export runs: in the build computer, beside the app. */ +/** Each reader's view of its file at the seed or at a checkpoint's position, run on this host, where the export runs: in the build computer, beside the app. */ const exportReaderViews = (sources: ReaderSources) => ( readers: readonly DesktopTaskReader[], - end: keyof StateEnds, + at: "seed" | number, ): Promise> => withRetainedContent( sources.capture, (content) => readerViewsOfBytes( readers, - (reader) => readerFile(sources, content, reader.file, end), + (reader) => readerFile(sources, content, reader.file, at), (argv, timeoutS) => runReaderHere(argv, timeoutS, sources.signal), ), sources.maxBytes, @@ -146,5 +146,5 @@ const readsOf = async (tasks: number, sources: ReaderSources) => ({ sources.maxBytes, ), readerViews: exportReaderViews(sources), - screens: (end: keyof StateEnds) => screenAt(sources, end), + screens: (at: "seed" | number) => screenAt(sources, at), }); diff --git a/src/application/desktop/DesktopGzipDocument.test.ts b/src/application/desktop/DesktopGzipDocument.test.ts index 90345904b0..4c85d78cba 100644 --- a/src/application/desktop/DesktopGzipDocument.test.ts +++ b/src/application/desktop/DesktopGzipDocument.test.ts @@ -197,7 +197,7 @@ describe("documents saved gzip-compressed", () => { expect(result.normalized_result).toMatchObject({ scrubbed_files: [SEED], unscrubbed_files: [], - task: { failing_on_seed: ["rent"] }, + tasks: [{ failing_on_seed: ["rent"] }], }); const shipped = await readFile(join(directory, SEED)); expect([...shipped.subarray(0, 4)]).toEqual([0x1f, 0x8b, 8, 0]); @@ -299,7 +299,7 @@ describe("gzip seeds as shipped", () => { const { result, directory } = await exportBudget(folder); if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { failing_on_seed: ["rent"] }, + tasks: [{ failing_on_seed: ["rent"] }], scrubbed_files: [SEED], }); const shipped = await readFile(join(directory, SEED)); @@ -322,7 +322,7 @@ describe("gzip seeds as shipped", () => { if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { failing_on_seed: ["rent"] }, + tasks: [{ failing_on_seed: ["rent"] }], }); const shipped = await readFile(join(directory, SEED)); expect(openStateDocument(BUDGET, shipped)).toMatchObject({ diff --git a/src/application/desktop/DesktopSeveralTasks.test.ts b/src/application/desktop/DesktopSeveralTasks.test.ts new file mode 100644 index 0000000000..00ceaa4c79 --- /dev/null +++ b/src/application/desktop/DesktopSeveralTasks.test.ts @@ -0,0 +1,314 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { exportDesktopApplication } from "./DesktopApplicationExport.js"; +import { + BREAD_NOTE, + checkpoint, + EDITED_NOTE, + NEAR_EGGS_NOTE, + readers, + seedEndCapture, + severalEndsCapture, + writeCaptureFolder, +} from "./desktopCapture.fixture.js"; +import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js"; + +const NOTE = "Documents/today.txt"; + +const ADD_EGGS = { + name: "add-eggs", + prompt: "In Notes, add eggs to today's list and save it.", + checks: [ + { id: "eggs", kind: "lines_present", file: NOTE, lines: ["eggs"] }, + { id: "no-bread-yet", kind: "lines_absent", file: NOTE, lines: ["bread"] }, + ], +}; +const ADD_BREAD = { + name: "add-bread", + prompt: "In Notes, add bread to today's list and save it.", + checks: [ + { id: "bread", kind: "lines_present", file: NOTE, lines: ["bread"] }, + ], +}; + +/** Export the two-task capture, its near miss at checkpoint `near`, with these tasks and end checkpoints. */ +const exportSeveral = async ( + tasks: readonly unknown[], + taskEnds: readonly string[], + near = "near-add-eggs", +) => exportCapture(severalEndsCapture(near), tasks, taskEnds); + +/** Export `recorded` with these tasks and end checkpoints, and a screenshot of each checkpoint `shots` names, as a build computer keeps them. */ +const exportCapture = async ( + recorded: DesktopApplicationCapture, + tasks: readonly unknown[], + taskEnds: readonly string[], + shots: Readonly> = {}, +) => { + const scratch = await createTestTempDirectory("desktop-tasks-"); + const screens = join(scratch, "screens"); + await mkdir(screens); + for (const [name, text] of Object.entries(shots)) + await writeFile(join(screens, `${name}.jpg`), text); + const paths = await Promise.all( + tasks.map(async (task, index) => { + const path = join(scratch, `task-${String(index)}.json`); + await writeFile(path, JSON.stringify(task)); + return path; + }), + ); + const directory = join(scratch, "bundle"); + const result = await exportDesktopApplication( + { + capture: await writeCaptureFolder(recorded, undefined, [ + EDITED_NOTE, + BREAD_NOTE, + NEAR_EGGS_NOTE, + ]), + directory, + name: undefined, + tasks: paths, + taskEnds, + seedCheckpoint: "seed", + endCheckpoint: "end", + screens, + }, + readers, + ); + return { result, directory }; +}; + +const exists = (path: string) => + access(path).then( + () => true, + () => false, + ); + +const refusal = (result: unknown) => { + expect(result).toBeInstanceOf(Error); + return (result as Error & { userMessage: string }).userMessage; +}; + +describe("exportDesktopApplication with several tasks", () => { + it("checks each task at its own end checkpoint and its near miss, and adds every task with the eval listing them all", async () => { + const { result, directory } = await exportSeveral( + [ADD_EGGS, ADD_BREAD], + ["add-bread=end-add-bread"], + ); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + tasks: [ + { + name: "add-eggs", + checks: 2, + end_checkpoint: "end", + failing_on_seed: ["eggs"], + near_checkpoint: "near-add-eggs", + failing_on_near: ["eggs", "no-bread-yet"], + }, + { + name: "add-bread", + checks: 1, + end_checkpoint: "end-add-bread", + failing_on_seed: ["bread"], + near_checkpoint: null, + failing_on_near: [], + }, + ], + }); + expect(await readFile(join(directory, "evals/notes.toml"), "utf8")).toBe( + 'tasks = ["add-eggs", "add-bread"]\n', + ); + for (const name of ["add-eggs", "add-bread"]) { + expect(await exists(join(directory, `tasks/${name}.json`))).toBe(true); + expect( + await exists(join(directory, `artifacts/${name}-check/verify.py`)), + ).toBe(true); + } + const readme = await readFile(join(directory, "README.md"), "utf8"); + expect(readme).toContain("## Task `add-eggs`"); + expect(readme).toContain("## Task `add-bread`"); + expect(readme).toContain("--task add-bread --agent claude-code"); + }); + + it("refuses a task checked at an end before its change, naming the task and that checkpoint, and writes no bundle", async () => { + const { result, directory } = await exportSeveral( + [ADD_EGGS, ADD_BREAD], + [], + ); + + const message = refusal(result); + expect(message).toContain('"add-bread"'); + expect(message).toContain("end"); + expect(message).toContain('Check "bread" fails on the captured end state'); + expect(await exists(directory)).toBe(false); + }); + + it("refuses a task its near- checkpoint passes, since the near miss must fail a check", async () => { + const { result, directory } = await exportSeveral( + [ADD_EGGS, ADD_BREAD], + ["add-bread=end-add-bread"], + "near-add-bread", + ); + + const message = refusal(result); + expect(message).toContain('"add-bread"'); + expect(message).toContain("near-add-bread"); + expect(await exists(directory)).toBe(false); + }); + + it.each([ + [ + "two tasks of one name", + [ADD_EGGS, ADD_EGGS], + [], + 'Two tasks are named "add-eggs"', + ], + [ + "an end for a task not given", + [ADD_EGGS], + ["add-bread=end-add-bread"], + '"add-bread"', + ], + [ + "an end without its checkpoint", + [ADD_EGGS], + ["add-eggs"], + "NAME=CHECKPOINT", + ], + [ + "an end the capture did not take", + [ADD_EGGS], + ["add-eggs=end-add-milk"], + '"end-add-milk"', + ], + ])( + "refuses %s and writes no bundle", + async (_label, tasks, ends, message) => { + const { result, directory } = await exportSeveral(tasks, ends); + + expect(refusal(result)).toContain(message); + expect(await exists(directory)).toBe(false); + }, + ); +}); + +/** A reader that prints the note's lines, as an app's headless mode would print a file REA reads as binary. */ +const LINES_READER = { + id: "lines", + file: NOTE, + argv: [ + "python3", + "-c", + "import sys; [print('REA ' + line) for line in open(sys.argv[1]).read().split()]", + "{file}", + ], + output: "lines", +}; + +const SHOW_LIST = { + name: "show-list", + prompt: "In Notes, show the shopping list.", + checks: [ + { + id: "shown", + kind: "screen_judge", + question: "Does the window show the shopping list?", + }, + ], +}; + +/** Eggs added by `end`, then the list shown on screen, which no file records, by `end-show-list`. */ +const shownAfterEggsCapture = (): DesktopApplicationCapture => { + const base = seedEndCapture(); + const { checkpoints } = base.state; + return { + ...base, + state: { + ...base.state, + checkpoints: [ + ...checkpoints.slice(0, 3), + checkpoint("end-show-list", "request", [], []), + ...checkpoints.slice(3), + ], + }, + }; +}; + +describe("exportDesktopApplication with several tasks and readers or screen judges", () => { + it("runs a later task's readers at its own end checkpoint", async () => { + const readBread = { + ...ADD_BREAD, + checks: [ + { + id: "bread", + kind: "lines_present", + file: NOTE, + reader: "lines", + lines: ["bread"], + }, + ], + readers: [LINES_READER], + }; + const { result } = await exportSeveral( + [ADD_EGGS, readBread], + ["add-bread=end-add-bread"], + ); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + tasks: [ + { name: "add-eggs" }, + { + name: "add-bread", + end_checkpoint: "end-add-bread", + failing_on_seed: ["bread"], + }, + ], + }); + }); + + it("judges a later task's screen from the seed to its own end, past a file the task before it changed", async () => { + const { result, directory } = await exportCapture( + shownAfterEggsCapture(), + [ADD_EGGS, SHOW_LIST], + ["show-list=end-show-list"], + { seed: "seed shot", "end-show-list": "shown shot" }, + ); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + tasks: [ + { name: "add-eggs", end_checkpoint: "end" }, + { + name: "show-list", + end_checkpoint: "end-show-list", + failing_on_seed: [], + }, + ], + }); + const verifier = await readFile( + join(directory, "artifacts/show-list-check/verify.py"), + "utf8", + ); + expect(verifier).toContain(Buffer.from("seed shot").toString("base64")); + expect(verifier).toContain(Buffer.from("shown shot").toString("base64")); + }); + + it("refuses a later screen judge whose own change shows in a file, naming the task", async () => { + const { result, directory } = await exportSeveral( + [ADD_EGGS, { ...SHOW_LIST, name: "add-bread" }], + ["add-bread=end-add-bread"], + ); + + const message = refusal(result); + expect(message).toContain('"add-bread"'); + expect(message).toContain("between end and end-add-bread"); + expect(await exists(directory)).toBe(false); + }); +}); diff --git a/src/application/desktop/DesktopTaskExport.test.ts b/src/application/desktop/DesktopTaskExport.test.ts index bedc9d0e56..03cb266ca3 100644 --- a/src/application/desktop/DesktopTaskExport.test.ts +++ b/src/application/desktop/DesktopTaskExport.test.ts @@ -286,7 +286,7 @@ describe("exportDesktopApplication with a task's seed", () => { ); if (kept instanceof Error) throw kept; expect(kept.normalized_result).toMatchObject({ - task: { failing_on_seed: ["row"] }, + tasks: [{ failing_on_seed: ["row"] }], }); }); @@ -312,7 +312,7 @@ describe("exportDesktopApplication with a task's seed", () => { if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { checks: 2, failing_on_seed: ["notes"] }, + tasks: [{ checks: 2, failing_on_seed: ["notes"] }], }); }); @@ -363,11 +363,16 @@ describe("exportDesktopApplication with a task", () => { if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { - name: "note-milk", - checks: 3, - failing_on_seed: ["milk", "check-2", "check-3"], - }, + tasks: [ + { + name: "note-milk", + checks: 3, + end_checkpoint: "edited", + failing_on_seed: ["milk", "check-2", "check-3"], + near_checkpoint: null, + failing_on_near: [], + }, + ], }); const steps = JSON.parse( await readFile(join(directory, "tasks/note-milk.json"), "utf8"), @@ -405,8 +410,11 @@ describe("exportDesktopApplication with a task", () => { it.each([ [ - [task([]), task([])], - "2 task files were given; a capture yields one task.", + [ + task([{ kind: "lines_present", file: NOTE, lines: ["milk"] }]), + task([{ kind: "lines_present", file: NOTE, lines: ["milk"] }]), + ], + 'Two tasks are named "note-milk"', ], ["{", "is not valid UTF-8 JSON"], ["\uFEFF{}", "Save the file as UTF-8 without a byte-order mark"], @@ -503,7 +511,7 @@ describe("exportDesktopApplication with a task's readers", () => { if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { name: "note-milk", checks: 1, failing_on_seed: ["version"] }, + tasks: [{ name: "note-milk", checks: 1, failing_on_seed: ["version"] }], }); const verifier = await readFile( join(directory, "artifacts/note-milk-check/verify.py"), @@ -720,7 +728,7 @@ describe("exportDesktopApplication with a screen-judged task", () => { ); if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { name: "note-shown", failing_on_seed: [] }, + tasks: [{ name: "note-shown", failing_on_seed: [] }], }); }); @@ -729,7 +737,7 @@ describe("exportDesktopApplication with a screen-judged task", () => { if (result instanceof Error) throw result; expect(result.normalized_result).toMatchObject({ - task: { name: "note-shown", checks: 1, failing_on_seed: [] }, + tasks: [{ name: "note-shown", checks: 1, failing_on_seed: [] }], }); expect( (result.normalized_result as { limitations: string[] }).limitations.some( diff --git a/src/application/desktop/DesktopTaskExport.ts b/src/application/desktop/DesktopTaskExport.ts index db4b22dc88..c5ba066193 100644 --- a/src/application/desktop/DesktopTaskExport.ts +++ b/src/application/desktop/DesktopTaskExport.ts @@ -9,6 +9,7 @@ import { import { evaluateTaskChecks, versionView, + type TaskCheckOutcome, type TaskFileView, } from "../../domain/desktop/appTaskChecks.js"; import { SETTLE_TIMING } from "../../domain/desktop/appSettle.js"; @@ -48,6 +49,7 @@ import { isPortableBundleName } from "../environment/environmentBundle.js"; import { readJsonFile } from "../JsonFiles.js"; import { desktopOperationFailure } from "./DesktopCaptureRefusals.js"; import { refuse } from "./DesktopExportInstall.js"; +import { selectStateEnds } from "./DesktopStateVersions.js"; const OPERATION = "export_desktop_application"; const REWRITE = "Fix the task file and export again."; @@ -62,38 +64,48 @@ type TaskTarget = Pick< > & { readonly files: ReadonlyMap; readonly identity: HostIdentity; + /** The seed, and the end a task without its own is checked at. */ readonly ends: StateEnds; + /** The capture's checkpoint names, by position. */ + readonly checkpoints: readonly string[]; /** Views of the text seeds the export rewrites, as it ships them. */ readonly seedViews: ReadonlyMap; /** Home paths the bundle actually ships, including files that are not text. */ readonly seeded: ReadonlySet; - /** Each reader's view of its file, by reader id: at the seed as the bundle ships it, or at the end. */ + /** Each reader's view of its file, by reader id: at the seed as the bundle ships it, or at a checkpoint's position. */ readonly readerViews: ( readers: readonly DesktopTaskReader[], - end: keyof StateEnds, + at: "seed" | number, ) => Promise>; - /** The screenshot a build computer took at the seed or the end, too large to ship, or undefined when there is none. */ + /** The screenshot a build computer took at the seed or at a checkpoint's position, too large to ship, or undefined when there is none. */ readonly screens: ( - end: keyof StateEnds, + at: "seed" | number, ) => Promise; }; -/** A task the capture validated: the files it adds to the bundle, its README section and its summary. */ -export interface AdmittedTask { +/** What the export says of one task it kept. */ +export interface TaskSummary { + readonly name: string; + readonly checks: number; + readonly end_checkpoint: string; + readonly failing_on_seed: string[]; + /** The checkpoint of the task's near miss, `near-`, when the capture took one. */ + readonly near_checkpoint: string | null; + readonly failing_on_near: string[]; +} + +/** The tasks the capture validated: the files they add to the bundle, their README sections and their summaries. */ +export interface AdmittedTasks { readonly files: readonly { readonly path: string; readonly content: string; }[]; readonly readme: string; - /** What the export says about the task, such as a prompt that does not name the app. */ + /** What the export says about the tasks, such as a prompt that does not name the app. */ readonly limitations: readonly string[]; - /** Whether the task judges the screen, whose app then resets by relaunching when it owns no files. */ + /** Whether a task judges the screen, whose app then resets by relaunching when it owns no files. */ readonly screenJudged?: boolean; - readonly summary: { - readonly name: string; - readonly checks: number; - readonly failing_on_seed: string[]; - }; + readonly summaries: readonly TaskSummary[]; } /** Why the task file could not be read, with the reader's detail and its error as the cause. */ @@ -142,10 +154,11 @@ const readTaskFile = async ( throw primary ?? typed(loaded.error); }; -/** The task's views of every file at its seed, as the bundle ships it, and at its end. */ +/** The task's views of every file at its seed, as the bundle ships it, at its end and at its near miss. */ interface TaskViews { readonly seed: ReadonlyMap; readonly end: ReadonlyMap; + readonly near: ReadonlyMap | undefined; } /** What a capture's identity check reads of a value: the captured home, or the host's name or user, if it names one. */ @@ -246,14 +259,14 @@ export const shippedSeedCheckView = ( captured: TaskFileView | undefined, ): TaskFileView | undefined => (seeded ? (shipped ?? captured) : undefined); -/** The captured views at a checkpoint, before export omissions and rewrites. */ +/** The captured views at a checkpoint's position, before export omissions and rewrites. */ const capturedViewsAt = ( target: TaskTarget, - end: keyof StateEnds, + at: number, ): ReadonlyMap => { const views = new Map(); for (const [home, path] of target.files) { - const version = versionAt(path, target.ends[end]); + const version = versionAt(path, at); const view = version === undefined ? undefined : versionView(version); if (view !== undefined) views.set(home, view); } @@ -294,7 +307,7 @@ ${task.readers.map((reader) => `- \`${reader.id}\` reads \`${reader.file}\` as $ const taskReadme = ( task: DesktopTask, app: string, - failingOnSeed: readonly string[], + summary: TaskSummary, ): string => ` ## Task \`${task.name}\` @@ -313,8 +326,12 @@ Checks, each of which must pass: ${task.checks.map(({ id, ...check }) => `- \`${id}\`: \`${JSON.stringify(check)}\``).join("\n")} ${readersReadme(task)} -REA kept the task because its checks fail on the seed (${failingOnSeed.map((id) => `\`${id}\``).join(", ")}) and all -pass on the captured end state. +REA kept the task because its checks fail on the seed (${ids(summary.failing_on_seed)}) and all +pass at the captured end state, checkpoint \`${summary.end_checkpoint}\`${ + summary.near_checkpoint === null + ? "" + : `, and its near miss, checkpoint \`${summary.near_checkpoint}\`, fails ${ids(summary.failing_on_near)}` +}. To measure a solve rate, run the task k times (3 by default) with the agent and model to measure, and count the runs whose summary line says \`passed\`; a run that says \`failed\` did not finish and is run again rather than counted: @@ -322,17 +339,21 @@ whose summary line says \`passed\`; a run that says \`failed\` did not finish an agent-env run --task ${task.name} --agent claude-code --model MODEL `; -/** Refuse a check or reader this app's computer cannot serve, naming it. */ +const ids = (checks: readonly string[]) => + checks.map((id) => `\`${id}\``).join(", "); + +/** Refuse a check or reader this app's computer cannot serve, naming it after `lead`, which names the task. */ const refuseTargetProblems = ( task: DesktopTask, target: TaskTarget, views: TaskViews, + lead: string, ): void => { for (const check of task.checks) { const problem = targetProblem(check, target, views); if (problem !== undefined) throw refuse( - `Check ${JSON.stringify(check.id)} ${problem}.`, + `${lead}Check ${JSON.stringify(check.id)} ${problem}.`, "task", REWRITE, ); @@ -341,22 +362,21 @@ const refuseTargetProblems = ( const problem = readerProblem(reader, target); if (problem !== undefined) throw refuse( - `Reader ${JSON.stringify(reader.id)} ${problem}.`, + `${lead}Reader ${JSON.stringify(reader.id)} ${problem}.`, "task", REWRITE, ); } }; -/** The readers' views at the end, as retained, and at the seed, as the bundle ships it: the app's own headless mode on those bytes. */ -const readersAtEnds = async ( +/** The readers' views at a checkpoint's position, as retained, or at the seed, as the bundle ships it: the app's own headless mode on those bytes. */ +const readersAt = async ( task: DesktopTask, target: TaskTarget, -): Promise => { + at: "seed" | number, +): Promise> => { const readers = task.readers ?? []; - if (readers.length === 0) return { seed: new Map(), end: new Map() }; - const end = await target.readerViews(readers, "end"); - return { seed: await target.readerViews(readers, "seed"), end }; + return readers.length === 0 ? new Map() : target.readerViews(readers, at); }; /** The README's section on a screen-judged task: its questions and how its judge is configured. */ @@ -388,35 +408,58 @@ To measure a solve rate, run the task k times (3 by default) and count the runs agent-env run --task ${task.name} --agent claude-code --model MODEL `; -/** Admit a screen-judged task: no file shows the change, and the build's seed and end screenshots ship as the judge's examples. */ +/** One task the export kept: its summary, README section, verifier, and what the export says about it. */ +interface AdmittedTask { + readonly summary: TaskSummary; + readonly readme: string; + readonly verifier: string; + readonly limitations: readonly string[]; + readonly screenJudged: boolean; +} + +/** Where a task is checked: its end, and the checkpoint its change was made on top of, the seed or the task end before it. */ +interface TaskAt { + readonly end: number; + readonly before: number; +} + +/** + * Admit a screen-judged task: no file shows its change since the checkpoint + * it was made on top of, and the build's screenshots at the seed and at the + * task's end ship as the judge's examples. + */ const admitScreenTask = async ( task: DesktopTask, target: TaskTarget, - views: TaskViews, + at: TaskAt, + lead: string, ): Promise => { // Only a file the app owns could take a check instead; the export refuses the rest. const owned = (all: ReadonlyMap) => new Map([...all].filter(([file]) => isWithinAny(file, target.paths))); - const changed = readableChanges(owned(views.seed), owned(views.end)); + const changed = readableChanges( + owned(capturedViewsAt(target, at.before)), + owned(capturedViewsAt(target, at.end)), + ); if (changed.length > 0) throw refuse( - `A screen_judge judges state no file shows, but ${changed.join(", ")} changed between the seed and the end in a way a check can read.`, + `${lead}A screen_judge judges state no file shows, but ${changed.join(", ")} changed between ${at.before === target.ends.seed ? "the seed and the end" : `${target.checkpoints[at.before] ?? "the task before"} and ${target.checkpoints[at.end] ?? "its end"}`} in a way a check can read.`, "task", `Check what changed in those files instead; ${DRAFT}.`, ); const [seed, end] = await Promise.all([ target.screens("seed"), - target.screens("end"), + target.screens(at.end), ]); if (seed === undefined || end === undefined) throw refuse( - `A screen_judge needs the screenshots a build computer takes at the seed and end checkpoints, its judge's examples, and the capture has no ${seed === undefined ? "seed" : "end"} screenshot.`, + `${lead}A screen_judge needs the screenshots a build computer takes at the seed and end checkpoints, its judge's examples, and the capture has no ${seed === undefined ? "seed" : "end"} screenshot.`, "task", "Capture the app in a build computer, marking seed and end there, and export from it.", ); if (seed === "too-large" || end === "too-large") throw refuse( - `The ${seed === "too-large" ? "seed" : "end"} screenshot is larger than the ${String(SCREEN_EXAMPLE_MAX_BYTES / 1024 / 1024)} MiB a judge's example may take.`, + `${lead}The ${seed === "too-large" ? "seed" : "end"} screenshot is larger than the ${String(SCREEN_EXAMPLE_MAX_BYTES / 1024 / 1024)} MiB a judge's example may take.`, "task", "Capture again with a simpler screen at that checkpoint.", ); @@ -429,32 +472,24 @@ const admitScreenTask = async ( summary: { name: task.name, checks: task.checks.length, + end_checkpoint: target.checkpoints[at.end] ?? "baseline", failing_on_seed: [], + near_checkpoint: null, + failing_on_near: [], }, limitations: [ ...(namesApp(task.prompt, target) ? [] : [ - `The task's prompt does not name the app (${target.title}), so start it with "In ${target.title}, ".`, + `The prompt of task ${task.name} does not name the app (${target.title}), so start it with "In ${target.title}, ".`, ]), - "The export judges no screen, so it did not check the screen_judge questions on the seed and the end: the witness does, running the bundle's own judge on a fresh screenshot at each stage, which needs a model endpoint.", + `The export judges no screen, so it did not check task ${task.name}'s screen_judge questions on the seed, the end or a near miss: the witness does, running the bundle's own judge on a fresh screenshot at each stage, which needs a model endpoint.`, ], readme: screenReadme(task, target.name), - files: [ - { - path: `tasks/${task.name}.json`, - content: renderTaskSteps(task, target.name), - }, - { - path: `artifacts/${taskVerifierArtifact(task)}/verify.py`, - content: renderTaskVerifier(task, target.name, examples), - }, - { path: `evals/${target.name}.toml`, content: renderTaskEval([task]) }, - ], + verifier: renderTaskVerifier(task, target.name, examples), }; }; -/** Read, validate and render the task a caller passed with the export, if any. */ /** Whether a prompt names the app by its title, environment or package, in any letter case. */ export const namesApp = ( prompt: string, @@ -473,49 +508,78 @@ export const namesApp = ( }); }; -export const admitDesktopTask = async ( - paths: readonly string[], +/** The checkpoint a task's near miss is taken at. */ +const nearCheckpoint = (task: string): string => `near-${task}`; + +/** Each task's own end checkpoint, from `NAME=CHECKPOINT`, by task name. */ +const taskEndsOf = ( + given: readonly string[], + names: ReadonlySet, +): ReadonlyMap => { + const ends = new Map(); + for (const entry of given) { + const at = entry.indexOf("="); + const name = entry.slice(0, at); + const checkpoint = entry.slice(at + 1); + if (at <= 0 || checkpoint === "") + throw refuse( + `The task end ${JSON.stringify(entry)} is not NAME=CHECKPOINT.`, + "task_end", + "Give each task end as the task's name, =, and the checkpoint it is checked at.", + ); + if (!names.has(name) || ends.has(name)) + throw refuse( + `The task end ${JSON.stringify(entry)} names ${JSON.stringify(name)}, ${ends.has(name) ? "whose end is already given" : "which is none of the tasks given"}.`, + "task_end", + "Give one end for each task, by the name in its task file.", + ); + ends.set(name, checkpoint); + } + return ends; +}; + +/** A checkpoint's position, refused unless the capture took it once, at or after the seed. */ +const positionOf = (target: TaskTarget, checkpoint: string): number => + selectStateEnds( + target.checkpoints, + { seed: target.checkpoints[target.ends.seed], end: checkpoint }, + OPERATION, + ).end; + +/** One task validated at the seed, at its own end and, when the capture took one, at its near miss. */ +const admitTask = async ( + task: DesktopTask, + at: TaskAt, target: TaskTarget, - signal: AbortSignal | undefined, -): Promise => { - if (paths.length > 1) - throw refuse( - `${String(paths.length)} task files were given; a capture yields one task.`, - "task", - "Pass --task once, with the task written for this capture.", - ); - const [path] = paths; - if (path === undefined) return undefined; - const parsed = parseDesktopTask( - await readTaskFile(path, signal), - isPortableBundleName, - ); - if (!parsed.ok) - throw refuse( - `The task file ${path} is refused: ${parsed.error}`, - "task", - REWRITE, - ); - const task = parsed.value; - const captured: TaskViews = { - seed: capturedViewsAt(target, "seed"), - end: capturedViewsAt(target, "end"), - }; +): Promise => { + const { end } = at; + const endName = target.checkpoints[end] ?? "baseline"; + const nearName = nearCheckpoint(task.name); + const near = target.checkpoints.includes(nearName) + ? positionOf(target, nearName) + : undefined; const views: TaskViews = { - seed: shippedSeedViews(target, captured.seed), - end: captured.end, + seed: shippedSeedViews(target, capturedViewsAt(target, target.ends.seed)), + end: capturedViewsAt(target, end), + near: near === undefined ? undefined : capturedViewsAt(target, near), }; - refuseTargetProblems(task, target, views); - if (isScreenTask(task)) return admitScreenTask(task, target, captured); - const read = await readersAtEnds(task, target); - const end = evaluateTaskChecks(task.checks, views.end, read.end); - const failing = end.find((outcome) => !outcome.passed); + const lead = `Task ${JSON.stringify(task.name)}, checked at ${endName}: `; + refuseTargetProblems(task, target, views, lead); + if (isScreenTask(task)) return admitScreenTask(task, target, at, lead); + const read = { + end: await readersAt(task, target, end), + seed: await readersAt(task, target, "seed"), + near: near === undefined ? undefined : await readersAt(task, target, near), + }; + const failing = evaluateTaskChecks(task.checks, views.end, read.end).find( + (outcome) => !outcome.passed, + ); if (failing !== undefined) { const check = task.checks.find(({ id }) => id === failing.id); throw refuse( - `Check ${JSON.stringify(failing.id)} fails on the captured end state: ${failing.description}${check === undefined ? "" : readerRead(check, read.end)}.`, + `${lead}Check ${JSON.stringify(failing.id)} fails on the captured end state: ${failing.description}${check === undefined ? "" : readerRead(check, read.end)}.`, "task", - `Write checks that the captured end state passes; ${DRAFT}.`, + `Write checks that the captured end state passes, or check the task at the checkpoint taken once its change was saved; ${DRAFT}.`, ); } const seed = evaluateTaskChecks(task.checks, views.seed, read.seed).filter( @@ -523,44 +587,137 @@ export const admitDesktopTask = async ( ); if (seed.length === 0) throw refuse( - "Every check already passes on the seed, so the task would be done before the agent starts.", + `${lead}Every check already passes on the seed, so the task would be done before the agent starts.`, "task", "Add a check that the change the user made is needed for.", ); // Only a failure on what the seed's view shows proves the task is not done yet. - const failingOnSeed = seed - .filter((outcome) => outcome.undecided !== true) - .map((outcome) => outcome.id); + const failingOnSeed = decidedFailures(seed); if (failingOnSeed.length === 0) throw refuse( - `No check fails on what the seed's views show: each failure there may come from a view that could not be read or is cut short (${seed - .map(({ id, description }) => `${JSON.stringify(id)}: ${description}`) - .join("; ")}).`, + `${lead}No check fails on what the seed's views show: each failure there may come from a view that could not be read or is cut short (${undecided(seed)}).`, "task", `Add a check that fails on what the seed's views hold, such as a line or row of a file read whole; ${DRAFT}.`, ); + const missed = + views.near === undefined + ? [] + : evaluateTaskChecks(task.checks, views.near, read.near).filter( + (outcome) => !outcome.passed, + ); + const failingOnNear = decidedFailures(missed); + if (views.near !== undefined && failingOnNear.length === 0) + throw refuse( + missed.length === 0 + ? `${lead}Every check passes at ${nearName}, its near miss, so the checks would accept that almost-right change.` + : `${lead}No check fails on what ${nearName}'s views show, its near miss: each failure there may come from a view that could not be read or is cut short (${undecided(missed)}).`, + "task", + `Tighten the checks so the near miss fails one, or take ${nearName} again on a near miss the task's checks are meant to catch.`, + ); + const summary: TaskSummary = { + name: task.name, + checks: task.checks.length, + end_checkpoint: endName, + failing_on_seed: failingOnSeed, + near_checkpoint: near === undefined ? null : nearName, + failing_on_near: failingOnNear, + }; return { - summary: { - name: task.name, - checks: task.checks.length, - failing_on_seed: failingOnSeed, - }, + summary, + screenJudged: false, limitations: namesApp(task.prompt, target) ? [] : [ - `The task's prompt does not name the app (${target.title}): a solving agent's own shell cannot see the computer's files, and the app's name sends it to the screen, so start the prompt with "In ${target.title}, ".`, + `The prompt of task ${task.name} does not name the app (${target.title}): a solving agent's own shell cannot see the computer's files, and the app's name sends it to the screen, so start the prompt with "In ${target.title}, ".`, ], - readme: taskReadme(task, target.name, failingOnSeed), + readme: taskReadme(task, target.name, summary), + verifier: renderTaskVerifier(task, target.name), + }; +}; + +const decidedFailures = (failed: readonly TaskCheckOutcome[]): string[] => + failed + .filter((outcome) => outcome.undecided !== true) + .map((outcome) => outcome.id); + +const undecided = (failed: readonly TaskCheckOutcome[]): string => + failed + .map(({ id, description }) => `${JSON.stringify(id)}: ${description}`) + .join("; "); + +/** The checkpoint a task's change was made on top of: the latest of the seed and the other tasks' ends before its own. */ +const changedFrom = ( + end: number, + seed: number, + ends: readonly number[], +): number => Math.max(seed, ...ends.filter((other) => other < end)); + +/** Read, validate and render the tasks a caller passed with the export, each at its own end checkpoint; undefined for none. */ +export const admitDesktopTasks = async ( + given: { + readonly paths: readonly string[]; + readonly ends: readonly string[]; + }, + target: TaskTarget, + signal: AbortSignal | undefined, +): Promise => { + const tasks: DesktopTask[] = []; + for (const path of given.paths) { + const parsed = parseDesktopTask( + await readTaskFile(path, signal), + isPortableBundleName, + ); + if (!parsed.ok) + throw refuse( + `The task file ${path} is refused: ${parsed.error}`, + "task", + REWRITE, + ); + if (tasks.some((task) => task.name === parsed.value.name)) + throw refuse( + `Two tasks are named ${JSON.stringify(parsed.value.name)}; the bundle names each task's files after it.`, + "task", + "Give each task its own name.", + ); + tasks.push(parsed.value); + } + const ends = taskEndsOf(given.ends, new Set(tasks.map((task) => task.name))); + if (tasks.length === 0) return undefined; + const positions = tasks.map((task) => { + const named = ends.get(task.name); + return named === undefined ? target.ends.end : positionOf(target, named); + }); + const admitted: AdmittedTask[] = []; + // One at a time: a task's readers run the app's own headless mode. + for (const [index, task] of tasks.entries()) { + const end = positions[index] as number; + admitted.push( + await admitTask( + task, + { end, before: changedFrom(end, target.ends.seed, positions) }, + target, + ), + ); + } + return { + summaries: admitted.map(({ summary }) => summary), + ...(admitted.some(({ screenJudged }) => screenJudged) + ? { screenJudged: true } + : {}), + limitations: admitted.flatMap(({ limitations }) => limitations), + readme: admitted.map(({ readme }) => readme).join(""), files: [ - { - path: `tasks/${task.name}.json`, - content: renderTaskSteps(task, target.name), - }, - { - path: `artifacts/${taskVerifierArtifact(task)}/verify.py`, - content: renderTaskVerifier(task, target.name), - }, - { path: `evals/${target.name}.toml`, content: renderTaskEval([task]) }, + ...tasks.flatMap((task, index) => [ + { + path: `tasks/${task.name}.json`, + content: renderTaskSteps(task, target.name), + }, + { + path: `artifacts/${taskVerifierArtifact(task)}/verify.py`, + content: (admitted[index] as AdmittedTask).verifier, + }, + ]), + { path: `evals/${target.name}.toml`, content: renderTaskEval(tasks) }, ], }; }; diff --git a/src/application/desktop/desktopCapture.fixture.ts b/src/application/desktop/desktopCapture.fixture.ts index 97f260ede5..da4ab47e0d 100644 --- a/src/application/desktop/desktopCapture.fixture.ts +++ b/src/application/desktop/desktopCapture.fixture.ts @@ -215,6 +215,42 @@ export const seedEndCapture = (): DesktopApplicationCapture => { }; }; +/** The note once bread is added after the eggs, and with the eggs then misspelled. */ +export const BREAD_NOTE = "milk\neggs\nbread\n"; +export const NEAR_EGGS_NOTE = "milk\negs\nbread\n"; + +/** + * Two tasks in one capture: eggs added by `end`, bread on top by + * `end-add-bread`, then the eggs misspelled by checkpoint `near`, before the + * stop. Write it with `EDITED_NOTE`, `BREAD_NOTE` and `NEAR_EGGS_NOTE` as extra content. + */ +export const severalEndsCapture = (near: string): DesktopApplicationCapture => { + const base = seedEndCapture(); + const { checkpoints } = base.state; + return { + ...base, + state: { + ...base.state, + checkpoints: [ + ...checkpoints.slice(0, 3), + checkpoint( + "end-add-bread", + "request", + [change(NOTE, { before: EDITED_NOTE, after: BREAD_NOTE })], + [retained(NOTE, BREAD_NOTE)], + ), + checkpoint( + near, + "request", + [change(NOTE, { before: BREAD_NOTE, after: NEAR_EGGS_NOTE })], + [retained(NOTE, NEAR_EGGS_NOTE)], + ), + ...checkpoints.slice(3), + ], + }, + }; +}; + export const writeCaptureFolder = async ( result: DesktopApplicationCapture, operation = DESKTOP_OPERATION, diff --git a/src/cli/desktopCaptureCommands.ts b/src/cli/desktopCaptureCommands.ts index 7c74ced004..5ed15fed04 100644 --- a/src/cli/desktopCaptureCommands.ts +++ b/src/cli/desktopCaptureCommands.ts @@ -326,7 +326,13 @@ const registerDesktopExportCommand = ( .array(localPathStringSchema) .optional() .describe( - "Task file to validate against the capture and add with its verifier and eval; a capture yields one", + "Task file to validate against the capture and add with its verifier and eval; repeat for several tasks, each checked at its own end", + ), + taskEnd: z + .array(z.string()) + .optional() + .describe( + "NAME=CHECKPOINT: the checkpoint task NAME's end state is read at, instead of --end-checkpoint; a checkpoint near-NAME, when taken, must fail one of its checks", ), packageUrl: z .string() @@ -382,6 +388,7 @@ const registerDesktopExportCommand = ( tasks: (options.task ?? []).map((path) => absoluteCallerPath(path), ), + taskEnds: options.taskEnd, packageUrl: options.packageUrl, downloadInstall: options.downloadInstall === undefined diff --git a/src/contracts/desktopBuildMeasureContract.ts b/src/contracts/desktopBuildMeasureContract.ts new file mode 100644 index 0000000000..11878e5eb3 --- /dev/null +++ b/src/contracts/desktopBuildMeasureContract.ts @@ -0,0 +1,108 @@ +import { z } from "zod"; + +import { absoluteLocalPathStringSchema } from "../domain/localPath.js"; +import type { ToolContract } from "./toolContractTypes.js"; +import { toolContractMetadata } from "./toolEffects.js"; + +const strings = z.array(z.string()); +const band = z + .enum(["in_band", "too_easy", "too_hard", "undecided"]) + .describe( + "in_band: it passed 1 to 3 of 5; too_easy: 3 of 3 or 4 of 5; too_hard: 0 of 3; undecided: the measure stopped before deciding.", + ); +const solve = z.strictObject({ + outcome: z.enum([ + "passed", + "scored below 1", + "unscored", + "failed", + "cancelled", + ]), + turns: z.int().nonnegative().nullable(), + usd: z.number().nonnegative().nullable(), + seconds: z.number().nonnegative(), + failed_checks: strings, + instance_id: z.string().nullable(), + error: z + .string() + .nullable() + .describe( + "The error agentenv reported for the run, such as why it failed to deploy, solve or verify; null when it reported none.", + ), +}); + +/** The tool that runs a bundle's task in fresh computers and records how hard it is. */ +export const MEASURE_BUILD_TASK_CONTRACT = { + name: "measure_build_task", + ...toolContractMetadata("measure_build_task"), + kind: "application", + description: + "Measure how hard an exported bundle's task is for a model: run it in fresh agentenv computers on this host, each solve the bundle's own steps with the agent deployed as the solver on the model, 3 times, then 2 more unless they passed none or all, and keep it in the band when it passes 1 to 3 of 5. A solve that did not finish is run again, at most twice, and not counted. Before each solve REA stops if the largest cost so far would take the total past budget_usd; a solve's cost is known only when it ends, so the first always runs, and a solve that reports no cost stops the measure. The solver calls the model endpoint agentenv reads where REA starts: LITELLM_BASE_URL and LITELLM_API_KEY, or [model] in .agentenv/config.toml; REA holds no model client and never reads the key. It writes difficulty.json at the bundle's root, each measured task's record by name, and a Measured difficulty section in its README, and returns the record: the band, passes, finished solves (k), turns, cost, failures per check and each solve. Solves run one at a time and take minutes each; REA reports progress meanwhile.", + inputSchema: z.strictObject({ + bundle: absoluteLocalPathStringSchema.describe( + "The exported bundle's folder on this host.", + ), + task: z + .string() + .min(1) + .describe("The task's name, as its tasks/.json names it."), + model: z + .string() + .min(1) + .describe( + "The model the solver runs on, such as anthropic/claude-sonnet-5-5.", + ), + agent: z + .string() + .min(1) + .default("claude-code") + .describe( + "The agent the solver runs as; claude-code reports each solve's turns and cost.", + ), + budget_usd: z + .number() + .positive() + .max(1000) + .describe("The most the measure may spend on solves, in US dollars."), + }), + outputSchema: z.strictObject({ + bundle: z.string(), + task: z.string(), + model: z.string(), + agent: z.string(), + measured_at: z.string(), + band, + stopped: z + .enum(["decided", "budget", "unfinished", "cost_unknown"]) + .describe( + "decided: the band is known; budget: the next solve could pass budget_usd; unfinished: 3 solves did not finish; cost_unknown: a solve reported no cost.", + ), + k: z.int().nonnegative().describe("The solves that finished."), + passes: z.int().nonnegative(), + unfinished: z.int().nonnegative(), + usd: z.number().nonnegative(), + budget_usd: z.number().positive(), + turns: z.array(z.int().nonnegative().nullable()), + failures_per_check: z + .record(z.string(), z.int().positive()) + .describe("How many finished solves failed each check."), + solves: z.array(solve), + task_sha256: z + .string() + .describe( + "The SHA-256 of the task's steps file followed by its verifier, so a changed task shows the record is stale.", + ), + difficulty_file: z.string(), + }), + examples: [ + { + title: "Measure a task with Sonnet within $8", + input: { + bundle: "/tmp/mousepad-bundle", + task: "add-eggs", + model: "anthropic/claude-sonnet-5-5", + budget_usd: 8, + }, + }, + ], +} as const satisfies ToolContract; diff --git a/src/contracts/desktopBuildTaskSchemas.ts b/src/contracts/desktopBuildTaskSchemas.ts new file mode 100644 index 0000000000..43f85990d0 --- /dev/null +++ b/src/contracts/desktopBuildTaskSchemas.ts @@ -0,0 +1,84 @@ +import { z } from "zod"; + +import { draftNoteSchema } from "../domain/desktop/appTaskDraftNotes.js"; +import { draftValueSchema } from "../domain/desktop/appTaskDraftValues.js"; +import { + jsonObjectSchema, + jsonValueSchema, + type JsonValue, +} from "../domain/jsonValue.js"; +import { draftWithheldField } from "./desktopBuildSecretsContract.js"; + +const strings = z.array(z.string()); + +/** The task the build tools' examples write. */ +export const EXAMPLE_TASK: Readonly> = { + name: "add-eggs", + prompt: + "In Mousepad, add eggs to the shopping list in Documents/todo.txt and save it.", + checks: [ + { kind: "lines_present", file: "Documents/todo.txt", lines: ["eggs"] }, + ], +}; + +/** A later task of the examples, checked at its own end. */ +export const EXAMPLE_LATER_TASK: Readonly> = { + task: { + name: "add-bread", + prompt: + "In Mousepad, add bread to the shopping list in Documents/todo.txt and save it.", + checks: [ + { kind: "lines_present", file: "Documents/todo.txt", lines: ["bread"] }, + ], + }, + end_checkpoint: "end-add-bread", +}; + +/** The most tasks one export takes. */ +export const MAX_BUILD_TASKS = 8; +export const draftSchema = z.strictObject({ + seed_checkpoint: z.string(), + end_checkpoint: z.string(), + files: z.array( + z.strictObject({ + file: z.string(), + role: z.string(), + change: jsonValueSchema, + values: z + .array(draftValueSchema) + .describe( + "The values the change added, changed and removed, ready to copy into a check: what a check can name first, then what only position reaches, then noise; at most 20 entries and 8 KB a file and 24 KB a draft. Only XML files, archives, JSON and key files have values: a text file's or SQLite database's change already lists its lines and rows.", + ), + values_cut: z.boolean().describe("Whether a limit left values out."), + }), + ), + task: z.strictObject({ + name: z.string(), + prompt: z.string(), + checks: z.array(jsonObjectSchema), + }), + guards: z + .array(jsonObjectSchema) + .describe( + "Optional checks that the rest keeps its seed value, which the end holds too: a field on like siblings, one that varies among them before a default they all share; the text an addition kept; a text file's lines beside its new one. Add those worth keeping to the task's checks.", + ), + notes: z + .array(draftNoteSchema) + .describe( + "Why the draft has no files, or why a file's suggested checks are thin; empty when neither.", + ), + withheld_files: draftWithheldField, +}); +export const exportedTaskSchema = z.strictObject({ + name: z.string(), + checks: z.int().positive(), + end_checkpoint: z.string(), + failing_on_seed: strings, + near_checkpoint: z + .string() + .nullable() + .describe( + "The task's near miss, near-, when the capture took one; null when it did not.", + ), + failing_on_near: strings, +}); diff --git a/src/contracts/desktopBuildToolContracts.ts b/src/contracts/desktopBuildToolContracts.ts index 7362c94907..61e9c4bd26 100644 --- a/src/contracts/desktopBuildToolContracts.ts +++ b/src/contracts/desktopBuildToolContracts.ts @@ -13,6 +13,14 @@ import { import { draftNoteSchema } from "../domain/desktop/appTaskDraftNotes.js"; import { draftValueSchema } from "../domain/desktop/appTaskDraftValues.js"; import { WITNESS_STAGES } from "../domain/desktop/buildWitness.js"; +import { MEASURE_BUILD_TASK_CONTRACT } from "./desktopBuildMeasureContract.js"; +import { + draftSchema, + EXAMPLE_LATER_TASK, + EXAMPLE_TASK, + exportedTaskSchema, + MAX_BUILD_TASKS, +} from "./desktopBuildTaskSchemas.js"; import { COMPUTER_ACTIONS, computerActionInputSchema, @@ -22,11 +30,7 @@ import { desktopCheckpointSchema, MAX_ACKNOWLEDGED_FILES, } from "../domain/desktop/desktopCapture.js"; -import { - jsonObjectSchema, - jsonValueSchema, - type JsonValue, -} from "../domain/jsonValue.js"; +import { jsonObjectSchema } from "../domain/jsonValue.js"; import { absoluteLocalPathStringSchema } from "../domain/localPath.js"; import type { ToolContract } from "./toolContractTypes.js"; import * as secrets from "./desktopBuildSecretsContract.js"; @@ -39,14 +43,6 @@ const driverResult = jsonObjectSchema "The driver's structured result, such as frame_id, width and height; null when it returned none.", ); const EXAMPLE_ID = "00000000-0000-4000-8000-000000000001"; -const EXAMPLE_TASK: Readonly> = { - name: "add-eggs", - prompt: - "In Mousepad, add eggs to the shopping list in Documents/todo.txt and save it.", - checks: [ - { kind: "lines_present", file: "Documents/todo.txt", lines: ["eggs"] }, - ], -}; const checkResult = z.strictObject({ id: z.string(), passed: z.boolean(), @@ -274,7 +270,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ name: "mark_build_checkpoint", ...toolContractMetadata("mark_build_checkpoint"), kind: "application", - description: `Take a named checkpoint of the app's state in the capture under way and wait for the capture to acknowledge it, with its counts (entries walked, paths changed since the checkpoint before, files retained) and up to ${String(MAX_ACKNOWLEDGED_FILES)} of the files that changed, those a window title named first, then those under state_paths and those the app held open, caches last, so you can see which file a save wrote. Mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the change is; a name already taken is refused.`, + description: `Take a named checkpoint of the app's state in the capture under way and wait for the capture to acknowledge it, with its counts (entries walked, paths changed since the checkpoint before, files retained) and up to ${String(MAX_ACKNOWLEDGED_FILES)} of the files that changed, those a window title named first, then those under state_paths and those the app held open, caches last, so you can see which file a save wrote. Mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the change is; a name already taken is refused. For several tasks, make each later task's change on top and mark ${END_CHECKPOINT}- once it is saved; then, for any task, make a near miss of its change, save it and mark near-, which the export requires to fail one of that task's checks.`, inputSchema: z.strictObject({ computer_id: computerId, name: buildCheckpointNameSchema, @@ -316,7 +312,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ name: "stop_build_capture", ...toolContractMetadata("stop_build_capture"), kind: "application", - description: `Stop the capture: the app is closed and the capture finishes inside the computer. Returns its checkpoints, the package and sources it found, its limitations and, when ${SEED_CHECKPOINT} and ${END_CHECKPOINT} were marked and this server has not used a declared secret, the task draft between them: the files that changed under the paths the bundle owns, each with its net change and, for XML files, archives, JSON and key files, the values it added, changed and removed (before and after, at the pointers a check names them by); suggested checks; and guards, optional checks that the rest stays as it was. Notes say why a draft is empty (nothing changed; the changes lie outside the owned paths, listed with their roles; the app saved its change only in dconf, which no check reads; a dconf database changed that REA could not read; or REA recognized no document: capture again with state_paths naming the file, or make a change the app saves in a file) or why a file's suggestions are thin (only position names the change, it is all view state, its text is too long, or its view is cut or unreadable). If the server has used a declared secret, automatic drafting is withheld before it runs; write the task manually and give it to export_build_environment. Otherwise write the task from the draft, its prompt in the user's words.`, + description: `Stop the capture: the app is closed and the capture finishes inside the computer. Returns its checkpoints, the package and sources it found, its limitations and, when ${SEED_CHECKPOINT} and ${END_CHECKPOINT} were marked and this server has not used a declared secret, the task draft between them, with one draft in later_drafts per later task's ${END_CHECKPOINT}- checkpoint, from the end checkpoint before it: the files that changed under the paths the bundle owns, each with its net change and, for XML files, archives, JSON and key files, the values it added, changed and removed (before and after, at the pointers a check names them by); suggested checks; and guards, optional checks that the rest stays as it was. Notes say why a draft is empty (nothing changed; the changes lie outside the owned paths, listed with their roles; the app saved its change only in dconf, which no check reads; a dconf database changed that REA could not read; or REA recognized no document: capture again with state_paths naming the file, or make a change the app saves in a file) or why a file's suggestions are thin (only position names the change, it is all view state, its text is too long, or its view is cut or unreadable). If the server has used a declared secret, automatic drafting is withheld before it runs; write the tasks manually and give them to export_build_environment. Otherwise write each task from its draft, its prompt in the user's words, and give them to export_build_environment.`, inputSchema: z.strictObject({ computer_id: computerId }), outputSchema: z.strictObject({ computer_id: z.uuid(), @@ -336,44 +332,19 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ sources: strings, }), limitations: strings, - draft: z - .strictObject({ - seed_checkpoint: z.string(), - end_checkpoint: z.string(), - files: z.array( - z.strictObject({ - file: z.string(), - role: z.string(), - change: jsonValueSchema, - values: z - .array(draftValueSchema) - .describe( - "The values the change added, changed and removed, ready to copy into a check: what a check can name first, then what only position reaches, then noise; at most 20 entries and 8 KB a file and 24 KB a draft. Only XML files, archives, JSON and key files have values: a text file's or SQLite database's change already lists its lines and rows.", - ), - values_cut: z - .boolean() - .describe("Whether a limit left values out."), - }), - ), - task: z.strictObject({ - name: z.string(), - prompt: z.string(), - checks: z.array(jsonObjectSchema), - }), - guards: z - .array(jsonObjectSchema) - .describe( - "Optional checks that the rest keeps its seed value, which the end holds too: a field on like siblings, one that varies among them before a default they all share; the text an addition kept; a text file's lines beside its new one. Add those worth keeping to the task's checks.", - ), - notes: z - .array(draftNoteSchema) - .describe( - "Why the draft has no files, or why a file's suggested checks are thin; empty when neither.", - ), - withheld_files: secrets.draftWithheldField, - }) - .nullable(), + draft: draftSchema.nullable(), draft_refusal: z.string().optional(), + later_drafts: z + .array( + z.strictObject({ + end_checkpoint: z.string(), + draft: draftSchema.nullable(), + draft_refusal: z.string().optional(), + }), + ) + .describe( + `One draft per later task's ${END_CHECKPOINT}- checkpoint taken after ${END_CHECKPOINT}, from the end checkpoint before it, so it shows only that task's change; empty for one task.`, + ), }), examples: [ { title: "Stop and draft the task", input: { computer_id: EXAMPLE_ID } }, @@ -383,10 +354,25 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ name: "export_build_environment", ...toolContractMetadata("export_build_environment"), kind: "application", - description: `Export the stopped capture as an agentenv computer app bundle with the given task, inside the computer, seeded at ${SEED_CHECKPOINT} and checked at ${END_CHECKPOINT}, then copy the bundle to a new or empty folder on this host, every file checked against its digest. A bundle whose file names or contents hold a secret from REA's build-secrets folder, as it is, in UTF-16, hex, base64, URL encoding or inside a gzip file, is refused and written nowhere, the error naming the secrets' handles and the files. ${secrets.WITHHELD_EXPORT_DESCRIPTION} A task the export refuses, such as a check that already passes on the seed, is an error naming why. Witness the bundle next: start_build_computer from the folder and check_build_task.`, + description: `Export the stopped capture as an agentenv computer app bundle with the given tasks, inside the computer, seeded at ${SEED_CHECKPOINT}, each task checked at its own end checkpoint, ${END_CHECKPOINT} by default, and at its near miss near- when the capture took one, then copy the bundle to a new or empty folder on this host, every file checked against its digest. The bundle's eval lists every task. A bundle whose file names or contents hold a secret from REA's build-secrets folder, as it is, in UTF-16, hex, base64, URL encoding or inside a gzip file, is refused and written nowhere, the error naming the secrets' handles and the files. ${secrets.WITHHELD_EXPORT_DESCRIPTION} A task the export refuses, such as a check that already passes on the seed, or a near miss every check passes, is an error naming the task and why, and no bundle is written. Witness each task next, each on a fresh computer: start_build_computer from the folder and check_build_task.`, inputSchema: z.strictObject({ computer_id: computerId, - task: desktopTaskSchema, + tasks: z + .array( + z.strictObject({ + task: desktopTaskSchema, + end_checkpoint: buildCheckpointNameSchema + .optional() + .describe( + `The checkpoint marked once this task's change was saved: ${END_CHECKPOINT} by default, ${END_CHECKPOINT}- for a later task.`, + ), + }), + ) + .min(1) + .max(MAX_BUILD_TASKS) + .describe( + "The tasks, each with its own name; the first is usually the one checked at end.", + ), directory: absoluteLocalPathStringSchema.describe( "A new or empty folder on this host for the bundle.", ), @@ -403,8 +389,11 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ environment: z.string(), files: z.int().nonnegative(), bytes: z.int().nonnegative(), - // The export's task summary: its name, checks and those failing on the seed. - task: jsonValueSchema, + tasks: z + .array(exportedTaskSchema) + .describe( + "Each task the export kept: its checks, its end checkpoint, those failing on the seed and those failing at its near miss.", + ), seed_files: strings, scrubbed_files: strings, // gzip seeds compressed again only to clear their headers, their documents as captured. @@ -418,7 +407,15 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ title: "Export the bundle with the task", input: { computer_id: EXAMPLE_ID, - task: EXAMPLE_TASK, + tasks: [{ task: EXAMPLE_TASK }], + directory: "/tmp/mousepad-bundle", + }, + }, + { + title: "Export two tasks, the second checked at its own end", + input: { + computer_id: EXAMPLE_ID, + tasks: [{ task: EXAMPLE_TASK }, EXAMPLE_LATER_TASK], directory: "/tmp/mousepad-bundle", }, }, @@ -496,4 +493,5 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ }, ], }, + MEASURE_BUILD_TASK_CONTRACT, ] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/toolEffects.test.ts b/src/contracts/toolEffects.test.ts index 58f8a3a0cb..15a57d7164 100644 --- a/src/contracts/toolEffects.test.ts +++ b/src/contracts/toolEffects.test.ts @@ -49,6 +49,7 @@ describe("tool effects", () => { "install_adb_package", "instrument_with_frida", "load_frida_script", + "measure_build_task", "observe_native_calls", "push_adb_file", "resume_frida_session", diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index bfdd0036fb..7e79ab262d 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -804,6 +804,15 @@ const TOOL_EFFECT_AUDIT: Readonly> = { mutatesSession: true, idempotent: false, }), + // Runs the bundle's task in fresh computers and writes its record into the bundle. + measure_build_task: effects({ + writesFilesystem: true, + launchesProcess: true, + accessesNetwork: true, + changesUiState: true, + executesForeignCode: true, + idempotent: false, + }), close_build_computer: effects({ mutatesSession: true, launchesProcess: true, diff --git a/src/desktop/AgentEnvBridgeFiles.test.ts b/src/desktop/AgentEnvBridgeFiles.test.ts new file mode 100644 index 0000000000..736e9520bb --- /dev/null +++ b/src/desktop/AgentEnvBridgeFiles.test.ts @@ -0,0 +1,30 @@ +import { readdir, readFile } from "node:fs/promises"; + +import { expect, it } from "vitest"; + +const ROOT = new URL("../../", import.meta.url); + +it("ships every agentenv helper the desktop code launches in the package", async () => { + const files = ( + JSON.parse(await readFile(new URL("package.json", ROOT), "utf8")) as { + files: string[]; + } + ).files; + const sources = await readdir(new URL("src/desktop/", ROOT)); + const launched = new Set(); + for (const source of sources.filter( + (name) => name.endsWith(".ts") && !name.endsWith(".test.ts"), + )) + for (const [, helper] of ( + await readFile(new URL(`src/desktop/${source}`, ROOT), "utf8") + ).matchAll(/"\.\.\/\.\.\/(bridge\/agentenv\/[\w.-]+\.py)"/gu)) + launched.add(helper as string); + + expect([...launched].sort()).toEqual([ + "bridge/agentenv/capture.py", + "bridge/agentenv/computer.py", + "bridge/agentenv/solve.py", + "bridge/agentenv/solve_cleanup.py", + ]); + for (const helper of launched) expect(files).toContain(helper); +}); diff --git a/src/desktop/AgentEnvComputerHelper.ts b/src/desktop/AgentEnvComputerHelper.ts index f5c8e02301..b16f3688c6 100644 --- a/src/desktop/AgentEnvComputerHelper.ts +++ b/src/desktop/AgentEnvComputerHelper.ts @@ -134,7 +134,7 @@ const sandboxFilters = (sandbox: string): readonly string[] => [ ]; /** Remove the containers of one sandbox REA's helper named, by their exact name and label; undefined once removed. */ -const removeSandbox = async ( +export const removeSandbox = async ( environment: Readonly, sandbox: string, ): Promise => { diff --git a/src/desktop/AgentEnvSolveHelper.test.ts b/src/desktop/AgentEnvSolveHelper.test.ts new file mode 100644 index 0000000000..4a24e714d8 --- /dev/null +++ b/src/desktop/AgentEnvSolveHelper.test.ts @@ -0,0 +1,265 @@ +import { + access, + chmod, + mkdir, + readdir, + rmdir, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { fakeAgentEnvSolves } from "../../tests/fixtures/fakeAgentEnvSolves.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { cleanupOwnedProcessGroup } from "../process/ProcessOwnership.js"; +import { spawnOwnedProviderProcess } from "../process/ProviderProcess.js"; +import { + runSolveHelper, + type SolveCleanupOwner, +} from "./AgentEnvSolveHelper.js"; + +const SOLVE = { outcome: "passed" as const, checks: {}, turns: 3, usd: 0.1 }; + +const until = async (path: string) => { + for (;;) { + if ( + await access(path).then( + () => true, + () => false, + ) + ) + return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } +}; + +const options = async (environment: Readonly>) => ({ + bundle: await createTestTempDirectory("solve-bundle-"), + task: "add-eggs", + model: "m", + agent: "claude-code", + environment: { ...process.env, ...environment }, + operation: "measure_build_task", +}); + +describe("runSolveHelper", () => { + it("stops a cancelled solve whose helper ignores SIGTERM within its stop grace", async () => { + const runner = await fakeAgentEnvSolves([SOLVE]); + const environment = { ...runner.environment, FAKE_SOLVE_HANGS: "1" }; + const stop = new AbortController(); + const retained: SolveCleanupOwner[] = []; + const started = until( + join(runner.environment.FAKE_STATE ?? "", "solves.jsonl"), + ).then(() => stop.abort(new Error("cancelled by the test"))); + const begun = Date.now(); + + const solved = runSolveHelper({ + ...(await options(environment)), + signal: stop.signal, + retainCleanup: (owner) => retained.push(owner), + timing: { solveMs: 60_000, stopMs: 300 }, + }); + + // agentenv keeps a sandbox's folder until its teardown is confirmed, so the + // emptied private root shows nothing is left up: only the cancellation remains. + const failure: unknown = await solved.then( + () => undefined, + (error: unknown) => error, + ); + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toBe("cancelled by the test"); + expect(failure).not.toHaveProperty("cleanupIncomplete", true); + await started; + expect(retained).toHaveLength(0); + expect(Date.now() - begun).toBeLessThan(15_000); + }, 30_000); + + it("records a helper stopped at its deadline without a report as a failed solve of unknown cost, its cleanup known", async () => { + const runner = await fakeAgentEnvSolves([SOLVE]); + const environment = { ...runner.environment, FAKE_SOLVE_HANGS: "1" }; + const retained: SolveCleanupOwner[] = []; + + const report = await runSolveHelper({ + ...(await options(environment)), + signal: new AbortController().signal, + retainCleanup: (owner) => retained.push(owner), + timing: { solveMs: 1_000, stopMs: 300 }, + }); + + expect(report).toMatchObject({ + outcome: "failed", + usd: null, + turns: null, + checks: [], + cleanup: { unknown: false, sandboxes: [] }, + }); + expect(report.error).toMatch(/stopped \(SIG[A-Z]+\) without a report/u); + expect(retained).toHaveLength(0); + }, 30_000); + + it("keeps a helper whose cleanup failed, so closing can retry it, and reports what is left", async () => { + const runner = await fakeAgentEnvSolves([SOLVE]); + const retained: SolveCleanupOwner[] = []; + let cleanups = 0; + + const solved = runSolveHelper({ + ...(await options(runner.environment)), + signal: new AbortController().signal, + retainCleanup: (owner) => retained.push(owner), + spawn: async (spawnOptions) => { + const owned = await spawnOwnedProviderProcess(spawnOptions); + return { + ...owned, + cleanup: async () => { + cleanups += 1; + const cleaned = await cleanupOwnedProcessGroup(owned.ownership); + return cleanups === 1 + ? { cleaned: false as const, reason: "a refusal by the test" } + : cleaned; + }, + }; + }, + }); + + const failure = await solved.catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProviderCleanupError); + expect(failure).toMatchObject({ + report: { outcome: "passed", turns: 3, usd: 0.1 }, + diagnostics: { solve_report: { outcome: "passed", turns: 3, usd: 0.1 } }, + }); + expect(retained).toHaveLength(1); + expect(await retained[0]?.stop()).toMatchObject({ + status: "verified-cleanup", + }); + retained[0]?.dispose(); + }, 30_000); +}); + +it("retains a reported sandbox on failed removal, then retries that exact sandbox", async () => { + const runner = await fakeAgentEnvSolves([SOLVE]); + const root = await createTestTempDirectory("solve-sandbox-cleanup-"); + const docker = join(root, "docker"); + const permit = join(root, "permit"); + await writeFile( + docker, + `#!/bin/sh\nif [ "$1" = ps ]; then echo owned-container; exit 0; fi\n[ -f "$FAKE_CLEANUP_PERMIT" ]\n`, + ); + await chmod(docker, 0o755); + const retained: SolveCleanupOwner[] = []; + const failure = await runSolveHelper({ + ...(await options({ + ...runner.environment, + REA_DOCKER_COMMAND: docker, + FAKE_CLEANUP_PERMIT: permit, + FAKE_SOLVE_STILL_UP: '["local-owned"]', + })), + signal: new AbortController().signal, + retainCleanup: (owner) => retained.push(owner), + }).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProviderCleanupError); + if (!(failure instanceof ProviderCleanupError)) throw failure; + expect(failure).toMatchObject({ + cleanupResources: ["agentenv sandbox local-owned"], + report: { outcome: "passed", turns: 3, usd: 0.1 }, + }); + expect(retained).toHaveLength(1); + await writeFile(permit, "allowed"); + expect(await retained[0]?.stop()).toMatchObject({ + status: "verified-cleanup", + }); + expect(failure.cleanupIncomplete).toBe(false); + expect(failure.cleanupResources).toEqual([]); + retained[0]?.dispose(); +}, 30_000); + +it("retains a completed report if cancellation arrives during final cleanup", async () => { + const runner = await fakeAgentEnvSolves([SOLVE]); + const controller = new AbortController(); + const reason = { stage: "final cleanup" }; + const failure = await runSolveHelper({ + ...(await options(runner.environment)), + signal: controller.signal, + retainCleanup: () => undefined, + spawn: async (spawnOptions) => { + const owned = await spawnOwnedProviderProcess(spawnOptions); + return { + ...owned, + cleanup: async () => { + const result = await cleanupOwnedProcessGroup(owned.ownership); + controller.abort(reason); + return result; + }, + }; + }, + }).catch((cause: unknown) => cause); + expect(failure).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + report: { outcome: "passed", turns: 3, usd: 0.1 }, + }); +}, 30_000); + +it("retains an unrecognized sandbox root entry and retries after it is repaired", async () => { + const runner = await fakeAgentEnvSolves([SOLVE]); + const parent = await createTestTempDirectory("solve-owned-roots-"); + const retained: SolveCleanupOwner[] = []; + const failure = await runSolveHelper({ + ...(await options({ + ...runner.environment, + AGENT_ENV_LOCAL_SANDBOX_DIR: parent, + FAKE_SOLVE_ROOT_ENTRY: "unexpected", + })), + signal: new AbortController().signal, + retainCleanup: (owner) => retained.push(owner), + }).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProviderCleanupError); + expect(retained).toHaveLength(1); + const roots = await readdir(parent); + expect(roots).toHaveLength(1); + const root = join(parent, roots[0] ?? ""); + expect(await readdir(root)).toEqual(["unexpected"]); + await rmdir(join(root, "unexpected")); + expect(await retained[0]?.stop()).toMatchObject({ + status: "verified-cleanup", + }); + expect(await readdir(parent)).toEqual([]); + if (!(failure instanceof ProviderCleanupError)) throw failure; + expect(failure.cleanupIncomplete).toBe(false); + retained[0]?.dispose(); +}, 30_000); + +it("retains root cleanup and the spawn failure when admission fails", async () => { + const runner = await fakeAgentEnvSolves([SOLVE]); + const parent = await createTestTempDirectory("solve-spawn-root-"); + const retained: SolveCleanupOwner[] = []; + const failure = await runSolveHelper({ + ...(await options({ + ...runner.environment, + AGENT_ENV_LOCAL_SANDBOX_DIR: parent, + })), + signal: new AbortController().signal, + retainCleanup: (owner) => retained.push(owner), + spawn: async (spawnOptions) => { + const root = spawnOptions.env?.AGENT_ENV_LOCAL_SANDBOX_DIR; + if (root === undefined) throw new Error("missing root"); + await mkdir(join(root, "unexpected")); + throw new Error("admission failed"); + }, + }).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProviderCleanupError); + if (!(failure instanceof ProviderCleanupError)) throw failure; + expect(failure.diagnostics).toMatchObject({ + spawn_error: "admission failed", + }); + expect(retained).toHaveLength(1); + const roots = await readdir(parent); + await rmdir(join(parent, roots[0] ?? "", "unexpected")); + expect(await retained[0]?.stop()).toMatchObject({ + status: "verified-cleanup", + }); + expect(failure.cleanupIncomplete).toBe(false); + expect(failure.cleanupResources).toEqual([]); + retained[0]?.dispose(); +}, 30_000); diff --git a/src/desktop/AgentEnvSolveHelper.ts b/src/desktop/AgentEnvSolveHelper.ts new file mode 100644 index 0000000000..40866c2f8d --- /dev/null +++ b/src/desktop/AgentEnvSolveHelper.ts @@ -0,0 +1,354 @@ +import { randomUUID } from "node:crypto"; +import { fileURLToPath } from "node:url"; + +import { z } from "zod"; + +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { jsonValueSchema } from "../domain/jsonValue.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { readBoundedControlLines } from "../process/BoundedControlLines.js"; +import { cleanupOwnedProcessGroup } from "../process/ProcessOwnership.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../process/ProviderProcess.js"; +import { SolveSandboxRoot } from "./SolveSandboxRoot.js"; +import { + AGENTENV_PROVIDER, + agentEnvInterpreter, + removeSandbox, +} from "./AgentEnvComputerHelper.js"; + +const HELPER = fileURLToPath( + new URL("../../bridge/agentenv/solve.py", import.meta.url), +); +const REPORT_BYTES = 256 * 1024; +const STDERR_BYTES = 16 * 1024; +/** The longest one solve may take, its deploy and teardown included, before REA stops it. */ +const SOLVE_MS = 45 * 60_000; +/** How long a stopped solve has to tear its computer down. */ +const STOP_MS = 120_000; +const POLL_MS = 1_000; +/** How long REA waits for the report once the helper's process group is cleared. */ +const REPORT_DRAIN_MS = 5_000; + +const cleanupSchema = z.strictObject({ + unknown: z.boolean(), + sandboxes: z.array( + z.strictObject({ id: z.string().min(1), type: z.string().nullable() }), + ), +}); +const reportSchema = z.union([ + z.strictObject({ + outcome: z.enum([ + "passed", + "scored below 1", + "unscored", + "failed", + "cancelled", + ]), + instance_id: z.string().nullable(), + seconds: z.number().nonnegative(), + checks: z.array( + z.strictObject({ + id: z.string(), + passed: z.boolean(), + description: z.string(), + }), + ), + turns: z.int().nonnegative().nullable(), + usd: z.number().nonnegative().nullable(), + agentenv: z.string().nullable(), + error: z.string().nullable(), + cleanup: cleanupSchema.optional(), + }), + z.strictObject({ error: z.string(), cleanup: cleanupSchema.optional() }), +]); + +/** What the helper printed: how the solve ended, or why no solve ran. */ +export type SolveReport = z.infer; +export type SolveCleanupOwner = Pick< + ProviderProcessSupervisor, + "stop" | "dispose" +>; + +/** Failed cleanup retains both its owner and any report the helper completed. */ +export class SolveCleanupError extends ProviderCleanupError { + constructor( + readonly report: SolveReport | undefined, + resources: readonly string[], + diagnostics: Readonly< + Record + >, + operation: string, + unknown = false, + ) { + super( + AGENTENV_PROVIDER, + resources, + { + ...diagnostics, + ...(report === undefined + ? {} + : { solve_report: jsonValueSchema.parse(report) }), + }, + { + operation, + remediationAction: unknown + ? "REA could not verify cleanup of every sandbox under this solve's private root. Closing REA retries cleanup of the retained root; inspect its agentenv records and Docker containers for anything still running." + : "Closing REA retries this cleanup; check that no agentenv run of the bundle is still up with docker ps.", + }, + ); + } +} + +/** A completed report remains available when cancellation arrives during teardown. */ +export class SolveCancelledError extends AnalysisCancelledError { + constructor( + readonly report: SolveReport, + operation: string, + reason: unknown, + ) { + super(operation, { + cause: reason, + diagnostics: { solve_report: jsonValueSchema.parse(report) }, + }); + } +} + +const parseReport = (line: string): SolveReport => { + try { + const parsed = reportSchema.safeParse(JSON.parse(line)); + if (parsed.success) return parsed.data; + } catch { + // Reported below as a line that is not a report. + } + return { + error: "REA's solve helper wrote something other than its report", + cleanup: { unknown: true, sandboxes: [] }, + }; +}; + +/** How long a solve may run, and how long a stopped one has to exit before its process group is cleared. */ +interface SolveTiming { + readonly solveMs: number; + readonly stopMs: number; +} + +/** + * One solve of a bundle's task in a fresh computer, through REA's helper run + * with the agentenv runner's Python; the solver's model endpoint is the one + * agentenv reads from the environment or its config. Cancelling it stops the + * helper, which tears its computer down, and a helper still running after + * the stop grace has its process group cleared. A helper whose cleanup could + * not be confirmed goes to `retainCleanup`, so closing REA can retry it. + */ +export const runSolveHelper = async (options: { + readonly bundle: string; + readonly task: string; + readonly model: string; + readonly agent: string; + readonly environment: Readonly; + readonly signal: AbortSignal; + readonly operation: string; + readonly retainCleanup: (owner: SolveCleanupOwner) => void; + readonly spawn?: typeof spawnOwnedProviderProcess; + readonly timing?: SolveTiming; +}): Promise => { + const { environment, signal } = options; + const timing = options.timing ?? { solveMs: SOLVE_MS, stopMs: STOP_MS }; + const interpreter = await agentEnvInterpreter(environment, options.operation); + const sandboxRoot = await SolveSandboxRoot.create(interpreter, environment); + let spawned: Awaited>; + try { + spawned = await (options.spawn ?? spawnOwnedProviderProcess)({ + command: interpreter, + arguments: [ + HELPER, + options.bundle, + options.task, + options.model, + options.agent, + ], + runId: randomUUID(), + expectedCommand: null, + hostEnvironment: environment, + env: { + PYTHONDONTWRITEBYTECODE: "1", + AGENT_ENV_LOCAL_SANDBOX_DIR: sandboxRoot.path, + }, + signal, + }); + } catch (cause: unknown) { + const cleanup = await sandboxRoot.stop(); + if (cleanup.status === "incomplete") { + const failure = new SolveCleanupError( + undefined, + [`sandbox root ${sandboxRoot.path}`], + { + reason: cleanup.reason, + spawn_error: cause instanceof Error ? cause.message : String(cause), + }, + options.operation, + ); + failure.bindCleanupResourceOwner({ + get cleanupResources() { + return sandboxRoot.removed + ? [] + : [`sandbox root ${sandboxRoot.path}`]; + }, + }); + options.retainCleanup({ + async stop() { + const result = await sandboxRoot.stop(); + if (result.status === "verified-cleanup") failure.resolveCleanup(); + return result; + }, + dispose: () => sandboxRoot.dispose(), + }); + throw failure; + } + throw cause; + } + const supervisor = new ProviderProcessSupervisor( + { + ...spawned, + ownsProcessLifetime: true, + cleanup: + spawned.cleanup ?? (() => cleanupOwnedProcessGroup(spawned.ownership)), + }, + { + captureStdout: false, + maxDiagnosticBytes: { stdout: 0, stderr: STDERR_BYTES }, + diagnosticRetention: "tail", + }, + ); + let report: SolveReport | undefined; + const reading = (async () => { + const stdout = spawned.process.stdout; + if (stdout === null) return; + for await (const line of readBoundedControlLines(stdout, REPORT_BYTES)) + report ??= line.ok + ? parseReport(line.value) + : { + error: "REA's solve helper's report is too long", + cleanup: { unknown: true, sandboxes: [] }, + }; + })().catch(() => undefined); + let stopped: Awaited>; + const begun = Date.now(); + try { + const deadline = begun + timing.solveMs; + while ( + !signal.aborted && + Date.now() < deadline && + !(await supervisor.waitForExit(POLL_MS)) + ); + if (supervisor.observedExit() === undefined) { + spawned.process.kill("SIGTERM"); + await supervisor.waitForExit(timing.stopMs); + } + } finally { + // Clearing the process group closes stdout, so the report is read only after it, and only so long. + stopped = await supervisor.stop(); + await Promise.race([ + reading, + new Promise((resolve) => setTimeout(resolve, REPORT_DRAIN_MS)), + ]); + } + let processPending = stopped.status === "incomplete"; + const pending = new Map( + (report?.cleanup?.sandboxes ?? []).map((item) => [item.id, item.type]), + ); + let unknown = report?.cleanup?.unknown ?? true; + const resources = () => [ + ...(processPending + ? [`process group ${String(spawned.ownership.processGroupId)}`] + : []), + ...[...pending.keys()].map((id) => `agentenv sandbox ${id}`), + ...(!sandboxRoot.removed ? [`sandbox root ${sandboxRoot.path}`] : []), + ...(unknown ? ["agentenv sandbox identity unknown"] : []), + ]; + let cleanupError: SolveCleanupError | undefined; + const cleanupOwner: SolveCleanupOwner = { + async stop() { + if (processPending) { + const result = await supervisor.stop(); + if (result.status === "incomplete") return result; + processPending = false; + } + const failures: string[] = []; + for (const [id, type] of pending) { + if (type !== null && type !== "local") { + failures.push(`Sandbox ${id} uses unsupported provider ${type}.`); + continue; + } + const failure = await removeSandbox(environment, id); + if (failure === undefined) pending.delete(id); + else failures.push(failure); + } + const rootCleanup = await sandboxRoot.stop(); + if (rootCleanup.status === "incomplete") + failures.push(rootCleanup.reason); + if (failures.length > 0) + return { status: "incomplete", reason: failures.join(" ") }; + // The solve's sandboxes are all local under its private root, and agentenv keeps a sandbox's folder until its teardown is confirmed, so an emptied root leaves none up. + if (sandboxRoot.removed) unknown = false; + if (!unknown) cleanupError?.resolveCleanup(); + return { status: "verified-cleanup" }; + }, + dispose: () => { + supervisor.dispose(); + sandboxRoot.dispose(); + }, + }; + const cleanup = processPending ? stopped : await cleanupOwner.stop(); + if (cleanup.status === "incomplete" || unknown) { + if (cleanup.status === "incomplete") options.retainCleanup(cleanupOwner); + const reason = + cleanup.status === "incomplete" + ? cleanup.reason + : "The helper ended before identifying every sandbox, and REA could not empty its sandbox root."; + cleanupError = new SolveCleanupError( + report, + resources(), + { + reason, + leader_pid: spawned.ownership.leaderPid, + process_group_id: spawned.ownership.processGroupId, + helper_stderr: supervisor.snapshot().stderr.text, + }, + options.operation, + unknown, + ); + cleanupError.bindCleanupResourceOwner({ + get cleanupResources() { + return resources(); + }, + }); + throw cleanupError; + } + if (signal.aborted && report !== undefined) + throw new SolveCancelledError(report, options.operation, signal.reason); + signal.throwIfAborted(); + if (report !== undefined) return report; + const exit = supervisor.observedExit(); + const stderr = supervisor.snapshot().stderr.text.slice(-2000); + // A helper a signal ended, REA's deadline or an outside kill, was stopped mid-solve; one that exited without a report never ran one. + if (exit?.signal != null) + return { + outcome: "failed", + instance_id: null, + seconds: (Date.now() - begun) / 1000, + checks: [], + turns: null, + usd: null, + agentenv: null, + error: `REA's solve helper was stopped (${exit.signal}) without a report: ${stderr}`, + cleanup: { unknown, sandboxes: [] }, + }; + return { + error: `REA's solve helper ended (exit ${String(exit?.code)}) without a report: ${stderr}`, + cleanup: { unknown, sandboxes: [] }, + }; +}; diff --git a/src/desktop/DesktopBuildDoctorFacts.test.ts b/src/desktop/DesktopBuildDoctorFacts.test.ts index 55726b8aa0..034387b15e 100644 --- a/src/desktop/DesktopBuildDoctorFacts.test.ts +++ b/src/desktop/DesktopBuildDoctorFacts.test.ts @@ -36,7 +36,7 @@ const configRunner = (path: string) => case "$1 $2" in "config debug") printf '{"search_path":[{"path":"%s","exists":true}]}' "$AGENT_ENV_CONFIG" ;; "config explain") printf '%s' "$FAKE_EXPLANATION" ;; - *) echo 'agent-env 0.9.1464' ;; + *) echo 'agent-env 0.9.1485' ;; esac`, ); @@ -90,7 +90,7 @@ it.skipIf(process.platform === "win32")( run: async (_command, args) => { if (args[0] === "--version") return { - stdout: "agent-env 0.9.1464", + stdout: "agent-env 0.9.1485", stderr: "", stdoutBytes: 19, stderrBytes: 0, @@ -268,7 +268,7 @@ it.skipIf(process.platform === "win32")( throw Object.assign(new Error("timed out"), { code: "ETIMEDOUT" }); } return { - stdout: "agent-env 0.9.1464", + stdout: "agent-env 0.9.1485", stderr: "", stdoutBytes: 19, stderrBytes: 0, @@ -290,7 +290,7 @@ it.skipIf(process.platform === "win32")( async () => { const { root, bin, home } = await buildHost("rea-desktop-empty-runner-"); const runner = join(bin, "agent-env"); - await program(runner, 'echo "agent-env 0.9.1464"'); + await program(runner, 'echo "agent-env 0.9.1485"'); const environment = { HOME: home, USERPROFILE: home, @@ -319,7 +319,7 @@ it.skipIf(process.platform === "win32")( ).toBe(runner); const facts = await readFacts(environment, ["claude_code"]); expect(facts.registrations[0]).toMatchObject({ - runner: { status: "current", version: "0.9.1464" }, + runner: { status: "current", version: "0.9.1485" }, }); expect(facts.registrations[0]?.differing ?? []).not.toContain( "REA_AGENT_ENV_COMMAND", @@ -369,7 +369,7 @@ it.skipIf(process.platform === "win32")( await program(join(bin, "docker"), "exit 1"); await program( join(bin, "agent-env"), - 'if [ "$1" = "--version" ]; then printf x >> "$REA_TEST_RUNNER_LOG"; fi; echo "agent-env 0.9.1464"', + 'if [ "$1" = "--version" ]; then printf x >> "$REA_TEST_RUNNER_LOG"; fi; echo "agent-env 0.9.1485"', ); const environment = { HOME: home, @@ -455,6 +455,132 @@ it.skipIf(process.platform === "win32")( }, ); +it.skipIf(process.platform === "win32")( + "verifies a client's custom docker and runtime archive without requiring the terminal's paths", + async () => { + const { root, bin, home } = await buildHost("rea-doctor-custom-paths-"); + const docker = join(root, "custom-docker"); + await program( + docker, + 'if [ "$1" = info ]; then echo /tmp; else exit 0; fi', + ); + await program( + join(bin, "docker"), + 'if [ "$1" = info ]; then echo /tmp; else exit 0; fi', + ); + const python = (await execute("which", ["python3"])).stdout.trim(); + const clientPython = join(root, "python-client"); + const pythonCalls = join(root, "python-calls"); + await program( + clientPython, + `printf '%s\\n' "$1" >> "${pythonCalls}"\nexec "${python}" "$@"`, + ); + await writeFile( + join(bin, "agent-env"), + `#!${clientPython}\nprint("agent-env 0.9.1431")\n`, + { mode: 0o755 }, + ); + const packed = join(root, "packed"); + await mkdir(join(packed, "runtime", "scripts"), { recursive: true }); + await writeFile(join(packed, "runtime", "scripts", "rea.mjs"), ""); + await writeFile( + join(packed, "runtime", "package.json"), + JSON.stringify({ version: PRODUCT_IDENTITY.packageVersion }), + ); + const archive = join(root, "custom.tar.gz"); + await execute("tar", ["-czf", archive, "-C", packed, "runtime"]); + await writeFile( + join(home, ".claude.json"), + JSON.stringify({ + mcpServers: { + rea: { + command: process.execPath, + args: ["/opt/rea/scripts/rea.mjs", "mcp"], + env: { + REA_DOCKER_COMMAND: docker, + REA_BUILD_COMPUTER_REA_ARCHIVE: archive, + REA_AGENT_ENV_COMMAND: join(bin, "agent-env"), + }, + }, + }, + }), + ); + const facts = await readFacts( + { + HOME: home, + USERPROFILE: home, + PATH: `${bin}:/usr/bin:/bin`, + XDG_CACHE_HOME: join(root, "cache"), + }, + ["claude_code"], + ); + expect(facts.registrations[0]).toMatchObject({ + registered: true, + unrecorded: [], + unresolved: [], + }); + expect(facts.registrations[0]?.differing ?? []).toEqual([]); + expect(facts.registrations[0]?.unchecked ?? []).toEqual([]); + expect((await readFile(pythonCalls, "utf8")).split("\n")).toContain("-I"); + expect(JSON.stringify(facts.registrations)).not.toContain(root); + const registration = (runner: string) => + JSON.stringify({ + mcpServers: { + rea: { + command: process.execPath, + args: ["/opt/rea/scripts/rea.mjs", "mcp"], + env: { + REA_DOCKER_COMMAND: docker, + REA_BUILD_COMPUTER_REA_ARCHIVE: archive, + REA_AGENT_ENV_COMMAND: runner, + }, + }, + }, + }); + const clientFacts = () => + readFacts( + { + HOME: home, + USERPROFILE: home, + PATH: `${bin}:/usr/bin:/bin`, + XDG_CACHE_HOME: join(root, "cache"), + }, + ["claude_code"], + ); + await writeFile( + join(home, ".claude.json"), + registration(join(root, "missing-runner")), + ); + expect((await clientFacts()).registrations[0]?.unchecked).toContain( + "REA_BUILD_COMPUTER_REA_ARCHIVE", + ); + const wrapper = join(root, "wrapper"); + await program(wrapper, 'echo "agent-env 0.9.1431"'); + await writeFile(join(home, ".claude.json"), registration(wrapper)); + expect((await clientFacts()).registrations[0]?.unchecked).toContain( + "REA_BUILD_COMPUTER_REA_ARCHIVE", + ); + await writeFile( + join(home, ".claude.json"), + registration(join(bin, "agent-env")), + ); + await rm(archive); + expect( + ( + await readFacts( + { + HOME: home, + USERPROFILE: home, + PATH: `${bin}:/usr/bin:/bin`, + XDG_CACHE_HOME: join(root, "cache"), + }, + ["claude_code"], + ) + ).registrations[0]?.differing, + ).toContain("REA_BUILD_COMPUTER_REA_ARCHIVE"); + }, +); + it.skipIf(process.platform === "win32")( "reads a refused daemon and names unrecorded settings without their values", async () => { @@ -580,7 +706,7 @@ it("recognizes a gzip runtime archive with a different top folder", async () => const python = (await execute("which", ["python3"])).stdout.trim(); const runner = join(bin, "agent-env"); - await writeFile(runner, `#!${python}\nprint('agent-env 0.9.1464')\n`, { + await writeFile(runner, `#!${python}\nprint('agent-env 0.9.1485')\n`, { mode: 0o755, }); const withoutSystemPython = await readFacts( diff --git a/src/desktop/DesktopBuildDoctorFacts.ts b/src/desktop/DesktopBuildDoctorFacts.ts index 6ba882e8a3..e17ff20220 100644 --- a/src/desktop/DesktopBuildDoctorFacts.ts +++ b/src/desktop/DesktopBuildDoctorFacts.ts @@ -77,7 +77,14 @@ export const readDesktopBuildFacts = async ( versions.set(path, pending); return pending; }; - const archive = await archiveFact(input, agentEnv); + const archive = await archiveFact(input, async () => + agentEnv.status === "found" + ? agentEnvInterpreter( + { ...input.environment, REA_AGENT_ENV_COMMAND: agentEnv.path }, + "desktop_build_doctor", + ).catch(() => "python3") + : "python3", + ); const chosenCommand = ( name: BuildComputerSetting, command: DesktopBuildCommand, @@ -249,7 +256,7 @@ const agentEnvVersion = async ( /** The setup-packed archive is named by its version; another is read for its package version. */ const archiveFact = async ( input: DesktopBuildFactsInput, - agentEnv: DesktopBuildCommand, + pythonForArchive: () => Promise, ): Promise => { const selected = input.environment.REA_BUILD_COMPUTER_REA_ARCHIVE; const source = @@ -264,13 +271,14 @@ const archiveFact = async ( ); if (!file) return { path, source, status: "missing" }; if (source === "cache") return { path, source, status: "current" }; - const python = - agentEnv.status === "found" - ? await agentEnvInterpreter( - { ...input.environment, REA_AGENT_ENV_COMMAND: agentEnv.path }, - "desktop_build_doctor", - ).catch(() => "python3") - : "python3"; + const python = await pythonForArchive(); + if (python === undefined) + return { + path, + source, + status: "unknown", + detail: `REA could not identify the selected runner's Python to inspect ${path}.`, + }; const packed = await attempt(input, python, [ "-I", "-c", @@ -314,7 +322,7 @@ const registrationFacts = async ( versionFor: (path: string) => Promise, ): Promise => { const selected = input.clientIds.length === 0 ? undefined : input.clientIds; - const recordedRunners = new Map(); + const recorded = new Map>(); const statuses = await input.registrations( selected, Object.fromEntries( @@ -325,11 +333,11 @@ const registrationFacts = async ( ), undefined, (client, name, value) => { - if ( - name === "REA_AGENT_ENV_COMMAND" && - (selected === undefined || selected.includes(client)) - ) - recordedRunners.set(client, value); + if (selected === undefined || selected.includes(client)) { + const settings = recorded.get(client) ?? new Map(); + settings.set(name, value); + recorded.set(client, settings); + } }, ); const registered = statuses.filter(({ command }) => command.length > 0); @@ -337,8 +345,8 @@ const registrationFacts = async ( return Promise.all( considered.map(async (client) => { const status = registered.find((entry) => entry.client === client); - const recordedRunner = - status === undefined ? undefined : recordedRunners.get(client); + const values = status === undefined ? undefined : recorded.get(client); + const recordedRunner = values?.get("REA_AGENT_ENV_COMMAND"); const effectiveRunner = recordedRunner === undefined ? undefined @@ -376,14 +384,43 @@ const registrationFacts = async ( effectiveRunner === undefined || effectiveRunner !== selectedSettings.REA_AGENT_ENV_COMMAND, ); - const compared = differing.filter( - (name) => selectedSettings[name as BuildComputerSetting] !== undefined, - ); - const unchecked = differing.filter( - (name) => - selectedSettings[name as BuildComputerSetting] === undefined && - !(name === "REA_AGENT_ENV_COMMAND" && runner !== undefined), - ); + const compared: string[] = []; + const unchecked: string[] = []; + for (const name of differing) { + if (name === "REA_AGENT_ENV_COMMAND") { + if (runner !== undefined) compared.push(name); + else unchecked.push(name); + continue; + } + const value = values?.get(name); + if (value === undefined) { + unchecked.push(name); + continue; + } + const environment = { ...input.environment, [name]: value }; + if (name === "REA_DOCKER_COMMAND") { + const command = await settingCommand(environment, name, "docker"); + if (command.status !== "found") compared.push(name); + else if ( + (await dockerDaemon({ ...input, environment }, command.path)) + .status !== "reachable" + ) + unchecked.push(name); + } else if (name === "REA_BUILD_COMPUTER_REA_ARCHIVE") { + const archive = await archiveFact( + { ...input, environment }, + async () => + effectiveRunner === undefined || runner?.status === "invalid" + ? undefined + : agentEnvInterpreter( + { ...environment, REA_AGENT_ENV_COMMAND: effectiveRunner }, + "desktop_build_doctor", + ).catch(() => undefined), + ); + if (archive.status === "unknown") unchecked.push(name); + else if (archive.status !== "current") compared.push(name); + } else unchecked.push(name); + } return { client, registered: status !== undefined, diff --git a/src/desktop/DesktopBuildSetupHost.test.ts b/src/desktop/DesktopBuildSetupHost.test.ts index 60cb71d921..6cacc80857 100644 --- a/src/desktop/DesktopBuildSetupHost.test.ts +++ b/src/desktop/DesktopBuildSetupHost.test.ts @@ -137,7 +137,7 @@ it.skipIf(process.platform === "win32").each([ await mkdir(home); await mkdir(owned, { recursive: true }); await program(join(bin, "docker"), "exit 0"); - await program(join(bin, "agent-env"), 'echo "agent-env 0.9.1464"'); + await program(join(bin, "agent-env"), 'echo "agent-env 0.9.1485"'); await program( join(bin, "uv"), `[ "$1 $2 $3" = "tool dir --bin" ] && echo ${join(home, ".local/bin")}`, @@ -186,7 +186,7 @@ it.skipIf(process.platform === "win32").each([ ), }, archivePacked: false, - agentEnvVersion: "0.9.1464", + agentEnvVersion: "0.9.1485", uv: { path: join(bin, "uv"), agentEnv: join(home, ".local/bin/agent-env"), diff --git a/src/desktop/HostFileRead.ts b/src/desktop/HostFileRead.ts index 29132261b9..b513dea6d4 100644 --- a/src/desktop/HostFileRead.ts +++ b/src/desktop/HostFileRead.ts @@ -44,6 +44,7 @@ export const readHostFile = async ( path: string, maximumBytes: number, signal: AbortSignal | undefined, + preserveCapacity = false, ): Promise => { const options = { symlinks: "follow" as const, @@ -56,6 +57,8 @@ export const readHostFile = async ( } catch (cause: unknown) { if (cause instanceof RegularFileCleanupFailure) throw hostCloseFailure(cause); + if (preserveCapacity && cause instanceof RegularFileCapacityError) + throw cause; signal?.throwIfAborted(); if (!isRefusedRead(cause)) throw cause; return (cause as NodeJS.ErrnoException).code === "ENOENT" diff --git a/src/desktop/SolveSandboxRoot.test.ts b/src/desktop/SolveSandboxRoot.test.ts new file mode 100644 index 0000000000..1674e9ffa9 --- /dev/null +++ b/src/desktop/SolveSandboxRoot.test.ts @@ -0,0 +1,25 @@ +import { execFileSync } from "node:child_process"; +import { readdir } from "node:fs/promises"; +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { SolveSandboxRoot } from "./SolveSandboxRoot.js"; + +it("joins overlapping cleanup requests for one owned root", async () => { + const parent = await createTestTempDirectory("solve-root-overlap-"); + const python = execFileSync( + "python3", + ["-c", "import sys; print(sys.executable)"], + { encoding: "utf8" }, + ).trim(); + const root = await SolveSandboxRoot.create(python, { + ...process.env, + AGENT_ENV_LOCAL_SANDBOX_DIR: parent, + }); + const results = await Promise.all([root.stop(), root.stop()]); + expect(results).toEqual([ + { status: "verified-cleanup" }, + { status: "verified-cleanup" }, + ]); + expect(await readdir(parent)).toEqual([]); +}); diff --git a/src/desktop/SolveSandboxRoot.ts b/src/desktop/SolveSandboxRoot.ts new file mode 100644 index 0000000000..b15cf22c52 --- /dev/null +++ b/src/desktop/SolveSandboxRoot.ts @@ -0,0 +1,133 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rmdir } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; + +import { cleanupOwnedProcessGroup } from "../process/ProcessOwnership.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../process/ProviderProcess.js"; + +const CLEANUP = fileURLToPath( + new URL("../../bridge/agentenv/solve_cleanup.py", import.meta.url), +); +const resultSchema = z.discriminatedUnion("ok", [ + z.object({ ok: z.literal(true) }), + z.object({ ok: z.literal(false), error: z.string() }), +]); + +type RootStop = + | { status: "verified-cleanup" } + | { status: "incomplete"; reason: string }; + +/** Owns a single solve's agentenv sandbox root and any cleanup process. */ +export class SolveSandboxRoot { + #cleanup: ProviderProcessSupervisor | undefined; + #removed = false; + #stopping: Promise | undefined; + private constructor( + readonly path: string, + private readonly interpreter: string, + private readonly environment: Readonly, + ) {} + + static async create( + interpreter: string, + environment: Readonly, + ): Promise { + const parent = + environment.AGENT_ENV_LOCAL_SANDBOX_DIR ?? + join(environment.HOME ?? homedir(), ".agent-env-sandboxes"); + await mkdir(parent, { recursive: true, mode: 0o700 }); + return new SolveSandboxRoot( + await mkdtemp(join(parent, "rea-solve-")), + interpreter, + environment, + ); + } + + get removed(): boolean { + return this.#removed; + } + + stop(): Promise { + if (this.#stopping !== undefined) return this.#stopping; + const running = this.#stop(); + this.#stopping = running; + void running.finally(() => { + if (this.#stopping === running) this.#stopping = undefined; + }); + return running; + } + + async #stop(): Promise { + if (this.#removed) return { status: "verified-cleanup" }; + try { + if (this.#cleanup !== undefined) { + const stopped = await this.#cleanup.stop(); + if (stopped.status === "incomplete") return stopped; + this.#cleanup = undefined; + } + const spawned = await spawnOwnedProviderProcess({ + command: this.interpreter, + arguments: [CLEANUP], + runId: randomUUID(), + expectedCommand: null, + hostEnvironment: this.environment, + env: { + AGENT_ENV_LOCAL_SANDBOX_DIR: this.path, + PYTHONDONTWRITEBYTECODE: "1", + }, + }); + const supervisor = new ProviderProcessSupervisor( + { + ...spawned, + ownsProcessLifetime: true, + cleanup: + spawned.cleanup ?? + (() => cleanupOwnedProcessGroup(spawned.ownership)), + }, + { + maxDiagnosticBytes: { stdout: 16 * 1024, stderr: 16 * 1024 }, + diagnosticRetention: "tail", + }, + ); + this.#cleanup = supervisor; + const exited = await supervisor.waitForExit(120_000); + const stopped = await supervisor.stop(); + if (stopped.status === "incomplete") return stopped; + this.#cleanup = undefined; + const snapshot = supervisor.snapshot(); + if (!exited || snapshot.diagnosticTruncated || snapshot.exitCode !== 0) + return { + status: "incomplete", + reason: + `Sandbox cleanup did not complete: ${snapshot.stderr.text} ${snapshot.stdout.text}`.trim(), + }; + const result = resultSchema.safeParse(JSON.parse(snapshot.stdout.text)); + if (!result.success || !result.data.ok) + return { + status: "incomplete", + reason: + result.success && !result.data.ok + ? result.data.error + : "Sandbox cleanup returned an invalid result.", + }; + await rmdir(this.path); + this.#removed = true; + return { status: "verified-cleanup" }; + } catch (error: unknown) { + return { + status: "incomplete", + reason: error instanceof Error ? error.message : String(error), + }; + } + } + + dispose(): void { + this.#cleanup?.dispose(); + } +} diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index 61aa0c2ec3..5571feb424 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -1,3 +1,4 @@ +import type { PartialTaskMeasurement } from "./desktop/taskDifficulty.js"; import type { AppleIpsPartialObservation } from "./native/appleIps.js"; import type { PlaywrightTrace } from "./playwrightTrace/playwrightTrace.js"; import type { GraphqlContract } from "./apiContract/graphqlContract.js"; @@ -488,6 +489,7 @@ import type { WebArtifactAcquisition } from "./webArtifactAcquisition.js"; /** Provider-neutral evidence collected before a typed analysis failure. */ export type AnalysisPartialObservation = + | PartialTaskMeasurement | { readonly kind: "omitted-observation"; readonly reason: diff --git a/src/domain/desktop/appExport.ts b/src/domain/desktop/appExport.ts index 7f0b4a5b79..94df518516 100644 --- a/src/domain/desktop/appExport.ts +++ b/src/domain/desktop/appExport.ts @@ -46,8 +46,8 @@ import { roleFacts, stateDirectoryOf } from "./stateRoles.js"; import { shellQuote } from "./shellQuote.js"; export { shellQuote } from "./shellQuote.js"; -/** agentenv-framework release the exported class was checked against. */ -export const DESKTOP_EXPORT_FRAMEWORK_VERSION = "0.9.1464"; +/** agentenv-framework release targeted by the exported class. */ +export const DESKTOP_EXPORT_FRAMEWORK_VERSION = "0.9.1485"; /** Everything the export renders, decided from a capture and its state view. */ export interface DesktopAppExportPlan { @@ -756,7 +756,7 @@ export const renderReadme = ( `# ${plan.title} computer app Exported by REA's \`export-desktop-application\` from a capture of the \`${plan.aptPackage}\` ${plan.download === undefined ? "package" : downloadKind(plan.download)} -(capture Evidence \`${plan.captureEvidenceId}\`). It was checked against agentenv-framework +(capture Evidence \`${plan.captureEvidenceId}\`). It targets agentenv-framework ${DESKTOP_EXPORT_FRAMEWORK_VERSION}. - \`envs/${plan.name}/\` is a computer app on the \`ubuntu-24.04\` base with the \`reference\` driver, for diff --git a/src/domain/desktop/appTask.ts b/src/domain/desktop/appTask.ts index 7932ca7e68..f81717f747 100644 --- a/src/domain/desktop/appTask.ts +++ b/src/domain/desktop/appTask.ts @@ -97,7 +97,7 @@ const checkSchema = z.discriminatedUnion("kind", [ .string() .optional() .describe( - "A selector of one or more [KEY=VALUE], [@KEY=VALUE] or [>PATH=VALUE] conditions that a member must meet to count, where PATH is up to three /-separated steps inside the member; every member counts without it, text nodes of an XML element's children included.", + "A selector of one or more [KEY=VALUE], [@KEY=VALUE] or [>PATH=VALUE] conditions that a member must meet to count, where PATH is up to three /-separated steps inside the member. For XML, [>text()=VALUE] compares the member's whole text with whitespace collapsed. Every member counts without a selector, text nodes of an XML element's children included.", ), ...comparison, ...readerField, diff --git a/src/domain/desktop/appTaskChecks.fixture.ts b/src/domain/desktop/appTaskChecks.fixture.ts index 5a31daf4dc..c14d97f55b 100644 --- a/src/domain/desktop/appTaskChecks.fixture.ts +++ b/src/domain/desktop/appTaskChecks.fixture.ts @@ -264,6 +264,16 @@ export const CHECK_VIEWS: Readonly> = { truncated: false, }, }, + "Documents/placed-notes.xml": { + format: "xml", + view: { + root: element("layer", {}, [ + element("text", { x: "1", y: "2" }, ["Buy eggs"]), + element("text", { x: "3", y: "4" }, ["Do not Buy eggs"]), + ]), + truncated: false, + }, + }, "Documents/flyer.sla": { format: "xml", view: { @@ -1347,6 +1357,26 @@ const ledger = "Documents/ledger.json"; /** Members named by a child element's text, its whitespace collapsed, alone or with other conditions. */ const CHILD_TEXT_CASES: readonly (readonly [DesktopTaskCheckInput, boolean])[] = [ + [ + { + kind: "member_count", + file: "Documents/placed-notes.xml", + pointer: "/root/children", + matching: "[name=text][>text()=Buy eggs]", + equals: 1, + }, + true, + ], + [ + { + kind: "member_count", + file: "Documents/placed-notes.xml", + pointer: "/root/children", + matching: "[name=text][>text()=eggs]", + at_least: 1, + }, + false, + ], [ { kind: "member_count", diff --git a/src/domain/desktop/appTaskDraftItems.test.ts b/src/domain/desktop/appTaskDraftItems.test.ts index 679fddb6ee..9441adb141 100644 --- a/src/domain/desktop/appTaskDraftItems.test.ts +++ b/src/domain/desktop/appTaskDraftItems.test.ts @@ -33,10 +33,11 @@ describe("draftTaskFiles for items placed by their own position", () => { expect(file.suggested).toEqual([ { - kind: "text_contains", + kind: "member_count", file: FILE, - pointer: LAYER, - value: "Quiz on Friday: bring a calculator", + pointer: `${LAYER}/children`, + matching: "[name=text][>text()=Quiz on Friday: bring a calculator]", + at_least: 1, }, { kind: "member_count", @@ -48,10 +49,11 @@ describe("draftTaskFiles for items placed by their own position", () => { ]); expect(file.guards).toEqual( NOTES.map(([, value]) => ({ - kind: "text_contains", + kind: "member_count", file: FILE, - pointer: LAYER, - value, + pointer: `${LAYER}/children`, + matching: `[name=text][>text()=${value}]`, + at_least: 1, })), ); expect( @@ -70,6 +72,11 @@ describe("draftTaskFiles for items placed by their own position", () => { ), ).toBe(true); expect(passing(FILE, xml(before), file.suggested)).toEqual([false, false]); + const wrong = notebook(PNG("CCCC"), ...NOTES, [ + "156.5", + "Do not Quiz on Friday: bring a calculator", + ]); + expect(passing(FILE, xml(wrong), file.suggested)[0]).toBe(false); expect(file.suggested.map(pointerOf)).not.toContain( "/root/children/[name=preview]", ); diff --git a/src/domain/desktop/appTaskDraftItems.ts b/src/domain/desktop/appTaskDraftItems.ts index 1a5a11ba63..c97fe008b1 100644 --- a/src/domain/desktop/appTaskDraftItems.ts +++ b/src/domain/desktop/appTaskDraftItems.ts @@ -1,5 +1,5 @@ import type { JsonValue } from "../jsonValue.js"; -import type { DesktopTaskCheckInput } from "./appTask.js"; +import { collapseWhitespace, type DesktopTaskCheckInput } from "./appTask.js"; import { isStamp } from "./appTaskDraftNoise.js"; import { elementText, isXmlElement, resolvePointer } from "./viewPointer.js"; @@ -75,14 +75,14 @@ const siblingTexts = ( const list = resolvePointer(view, `${owner}/children`, "xml"); return (Array.isArray(list) ? list : []).flatMap((sibling, index) => sibling !== item && isXmlElement(sibling) && sibling.name === item.name - ? [{ text: elementText(sibling).trim(), index }] + ? [{ text: collapseWhitespace(elementText(sibling)), index }] : [], ); }; /** * The check of an item placed by its own position and added whole: a - * `text_contains` of its text on the element holding the items, which holds + * count of items with its whole text on the element holding them, which holds * wherever the user places it, and guards of the text of up to four items * both ends hold, nearest first; undefined when that element is no check's, * or the text is short, a time, past the limit, or there at the seed. @@ -101,7 +101,7 @@ export const placedItemCheck = ( const owner = placeOf(pointer); const at = owner === undefined ? undefined : context.name(owner); const item = resolvePointer(ends.final, pointer, "xml"); - const value = isXmlElement(item) ? elementText(item).trim() : ""; + const value = isXmlElement(item) ? collapseWhitespace(elementText(item)) : ""; const seed = at === undefined ? undefined : resolvePointer(ends.baseline, at, "xml"); if ( @@ -112,7 +112,10 @@ export const placedItemCheck = ( value.length < MIN_TEXT || value.length > context.chars || isStamp(value) || - elementText(seed).includes(value) + value.includes("][") || + siblingTexts(ends.baseline, owner, item).some( + (entry) => entry.text === value, + ) ) return undefined; const kept = new Set( @@ -121,7 +124,10 @@ export const placedItemCheck = ( const list = resolvePointer(ends.final, `${owner}/children`, "xml"); const here = Array.isArray(list) ? list.indexOf(item) : 0; const guards = siblingTexts(ends.final, owner, item) - .filter(({ text }) => text.length >= MIN_TEXT && kept.has(text)) + .filter( + ({ text }) => + text.length >= MIN_TEXT && !text.includes("][") && kept.has(text), + ) .toSorted( (left, right) => Math.abs(left.index - here) - Math.abs(right.index - here), @@ -129,14 +135,21 @@ export const placedItemCheck = ( .slice(0, MAX_KEPT) .toSorted((left, right) => left.index - right.index) .map(({ text }) => ({ - kind: "text_contains" as const, + kind: "member_count" as const, file, - pointer: at, - value: text, + pointer: `${at}/children`, + matching: `[name=${item.name}][>text()=${text}]`, + at_least: 1, })); return { element: pointer, - check: { kind: "text_contains", file, pointer: at, value }, + check: { + kind: "member_count", + file, + pointer: `${at}/children`, + matching: `[name=${item.name}][>text()=${value}]`, + at_least: 1, + }, guards, }; }; diff --git a/src/domain/desktop/appTaskDraftJson.test.ts b/src/domain/desktop/appTaskDraftJson.test.ts index 7b3f3b2520..d304a22420 100644 --- a/src/domain/desktop/appTaskDraftJson.test.ts +++ b/src/domain/desktop/appTaskDraftJson.test.ts @@ -50,6 +50,22 @@ const passing = (view: JsonValue, checks: readonly DesktopTaskCheckInput[]) => new Map([[FILE, { format: "json" as const, view }]]), ).map(({ passed }) => passed); +it("guards the count of a top-level JSON array when a member is edited", () => { + const before = [ + { name: "one", done: false }, + { name: "two", done: false }, + ]; + const after = [{ name: "one", done: true }, before[1]!]; + const file = drafted(before, after); + expect(file.guards).toContainEqual({ + kind: "member_count", + file: FILE, + pointer: "", + equals: 2, + }); + expect(passing(after, file.guards).every(Boolean)).toBe(true); +}); + describe("draftTaskFiles for a root JSON array of records", () => { it("counts additions to a root array and guards edits to its named members", () => { const before = [ diff --git a/src/domain/desktop/appTaskDraftLineGuards.test.ts b/src/domain/desktop/appTaskDraftLineGuards.test.ts index 21da7634e9..395595d2ca 100644 --- a/src/domain/desktop/appTaskDraftLineGuards.test.ts +++ b/src/domain/desktop/appTaskDraftLineGuards.test.ts @@ -26,6 +26,31 @@ const drafted = (before: string, after: string) => { }; describe("draftTaskFiles for lines holding values the app generates", () => { + it("checks a reference number in ordinary text", () => { + const line = "Reference: ABC_123456"; + const file = drafted("# Instructions", `# Instructions\n${line}`); + expect(file.suggested).toEqual([ + { kind: "lines_present", file: FILE, lines: [line] }, + ]); + const checks = file.suggested.map((check, index) => ({ + ...check, + id: `c${index}`, + })); + expect( + evaluateTaskChecks( + checks, + new Map([ + [ + FILE, + { + format: "text", + view: textView("# Instructions\nReference: WRONG"), + }, + ], + ]), + ).map(({ passed }) => passed), + ).toEqual([false]); + }); it("keeps an unquoted assignment in ordinary prose exact", () => { const line = "Set created=2025550123 and save"; const file = drafted("# Instructions", `# Instructions\n${line}`); diff --git a/src/domain/desktop/appTaskPythonPointer.ts b/src/domain/desktop/appTaskPythonPointer.ts index 9f019a2648..7e4bbcbde7 100644 --- a/src/domain/desktop/appTaskPythonPointer.ts +++ b/src/domain/desktop/appTaskPythonPointer.ts @@ -97,7 +97,10 @@ def _holds_inside(member, path: str, value: str, form) -> bool: a JSON record the string at those keys.""" steps = path.split("/") if form == "xml": - return _is_element(member) and _element_holds(member, steps, value) + return _is_element(member) and ( + _collapse_whitespace(_element_text(member)) == value if steps == ["text()"] + else _element_holds(member, steps, value) + ) found = member for step in steps: if not isinstance(found, dict) or step not in found: diff --git a/src/domain/desktop/appTaskReaders.test.ts b/src/domain/desktop/appTaskReaders.test.ts index a661812d12..31585aaebc 100644 --- a/src/domain/desktop/appTaskReaders.test.ts +++ b/src/domain/desktop/appTaskReaders.test.ts @@ -1,10 +1,7 @@ -import { createHash } from "node:crypto"; - import { describe, expect, it } from "vitest"; import { parseDesktopTask } from "./appTask.js"; import { evaluateTaskChecks, type TaskFileView } from "./appTaskChecks.js"; -import { renderTaskVerifier } from "./appTaskPython.js"; import { PYTHON_READER_RUNNER, readerAnswerView, @@ -286,47 +283,3 @@ describe("the reader runner's command and answer", () => { ); }); }); - -describe("a verifier without readers", () => { - it("renders byte for byte as the selector parent before readers", () => { - const rendered = renderTaskVerifier( - { - name: "add-milk", - prompt: "In Mousepad, add buy milk to my to-do list and save it.", - checks: [ - { - id: "milk", - kind: "lines_present", - file: "Documents/todo.txt", - lines: ["buy milk"], - }, - { - id: "price", - kind: "value_equals", - file: ".config/notes/settings.json", - pointer: "/size", - value: 12.5, - tolerance: 0.001, - }, - { - id: "rows", - kind: "member_count", - file: "Documents/todo.xml", - pointer: "/root/children", - matching: "[name=item]", - at_least: 2, - }, - ], - }, - "mousepad", - ); - - expect(rendered).not.toContain("computer-exec"); - // Digest of the no-reader verifier rendered by the #9185 selector parent - // (61ab3f0c4). A task without readers must stay byte-for-byte that artifact. - // Older evaluator bodies remain subject to sameTaskVerifier's equivalence rules. - expect(createHash("sha256").update(rendered).digest("hex")).toBe( - "5de3e07500eab060a35e211a97e471e8e7942ab9e4b60e6ddb7df7851fa9cf00", - ); - }); -}); diff --git a/src/domain/desktop/taskDifficulty.test.ts b/src/domain/desktop/taskDifficulty.test.ts new file mode 100644 index 0000000000..9eff8482cd --- /dev/null +++ b/src/domain/desktop/taskDifficulty.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, it } from "vitest"; + +import { + bandOf, + difficultyRecord, + difficultySection, + nextSolve, + withDifficultySection, + type SolveRecord, +} from "./taskDifficulty.js"; + +const solve = ( + outcome: SolveRecord["outcome"], + fields: Partial = {}, +): SolveRecord => ({ + outcome, + turns: 20, + usd: 1, + seconds: 60, + failed_checks: outcome === "scored below 1" ? ["year"] : [], + instance_id: null, + error: null, + ...fields, +}); +const pass = solve("passed"); +const miss = solve("scored below 1"); +const unfinished = solve("failed", { turns: null, usd: 0.2 }); + +describe("the adaptive band", () => { + it.each([ + ["0 of 3", "too_hard", [miss, miss, miss]], + ["3 of 3", "too_easy", [pass, pass, pass]], + ["1 of 3", "undecided", [pass, miss, miss]], + ["2 of 3", "undecided", [pass, pass, miss]], + ["1 of 5", "in_band", [pass, miss, miss, miss, miss]], + ["3 of 5", "in_band", [pass, pass, miss, pass, miss]], + ["4 of 5", "too_easy", [pass, pass, miss, pass, pass]], + [ + "2 of 3 beside an unfinished run", + "undecided", + [pass, unfinished, pass, miss], + ], + ] as const)("places %s as %s", (_label, band, solves) => { + expect(bandOf(solves)).toBe(band); + }); + + it("asks for 3 solves, stops at 0 or 3 of 3, asks for 2 more otherwise and stops at 5", () => { + expect(nextSolve([], 10)).toEqual({ next: true }); + expect(nextSolve([pass, miss], 10)).toEqual({ next: true }); + expect(nextSolve([pass, pass, pass], 10)).toEqual({ + next: false, + stopped: "decided", + }); + expect(nextSolve([pass, miss, miss], 10)).toEqual({ next: true }); + expect(nextSolve([pass, miss, miss, pass], 10)).toEqual({ next: true }); + expect(nextSolve([pass, miss, miss, pass, miss], 10)).toEqual({ + next: false, + stopped: "decided", + }); + }); + + it("runs an unfinished solve again, at most twice, and does not count it", () => { + expect(nextSolve([pass, unfinished, pass], 10)).toEqual({ next: true }); + expect(nextSolve([unfinished, unfinished, pass], 10)).toEqual({ + next: true, + }); + expect(nextSolve([unfinished, unfinished, unfinished], 10)).toEqual({ + next: false, + stopped: "unfinished", + }); + }); + + it("admits an exactly affordable solve without admitting one over budget", () => { + const tenth = solve("passed", { usd: 0.1 }); + expect(nextSolve([tenth, tenth], 0.3)).toEqual({ next: true }); + expect(nextSolve([tenth, tenth], 0.299999)).toEqual({ + next: false, + stopped: "budget", + }); + }); + + it("stops before a solve that the largest cost so far would take past the budget", () => { + const costly = solve("passed", { usd: 1.5 }); + expect(nextSolve([costly, miss], 4)).toEqual({ next: true }); + expect(nextSolve([costly, miss], 3.9)).toEqual({ + next: false, + stopped: "budget", + }); + expect(nextSolve([solve("passed", { usd: null })], 10)).toEqual({ + next: false, + stopped: "cost_unknown", + }); + }); +}); + +describe("the difficulty record", () => { + it("counts passes, failures per check and cost over finished solves, and turns each", () => { + const record = difficultyRecord({ + model: "anthropic/claude-sonnet-5-5", + agent: "claude-code", + solves: [ + pass, + solve("scored below 1", { failed_checks: ["year", "kept"] }), + unfinished, + solve("scored below 1", { failed_checks: ["year"], turns: 31 }), + pass, + miss, + ], + budgetUsd: 8, + stopped: "decided", + taskSha256: "ab".repeat(32), + versions: { rea: "6.8.0", agentenv: "0.9.1431" }, + measuredAt: "2026-11-07T01:00:00.000Z", + }); + + expect(record).toMatchObject({ + band: "in_band", + k: 5, + passes: 2, + unfinished: 1, + usd: 5.2, + turns: [20, 20, 31, 20, 20], + failures_per_check: { year: 3, kept: 1 }, + stopped: "decided", + task_sha256: "ab".repeat(32), + versions: { rea: "6.8.0", agentenv: "0.9.1431" }, + }); + expect(record.solves).toHaveLength(6); + }); + + it("counts failures of checks named like an object's own members as plain numbers", () => { + const record = difficultyRecord({ + model: "m", + agent: "claude-code", + solves: [ + solve("scored below 1", { + failed_checks: ["constructor", "__proto__"], + }), + solve("scored below 1", { failed_checks: ["constructor"] }), + ], + budgetUsd: 8, + stopped: "budget", + taskSha256: "ef".repeat(32), + versions: { rea: "6.8.0", agentenv: null }, + measuredAt: "2026-11-07T01:00:00.000Z", + }); + + expect(Object.entries(record.failures_per_check)).toEqual([ + ["constructor", 2], + ["__proto__", 1], + ]); + expect(JSON.parse(JSON.stringify(record)).failures_per_check).toEqual( + JSON.parse('{"constructor": 2, "__proto__": 1}'), + ); + }); + + it("renders one README line per task and replaces the section it wrote before", () => { + const base = { + agent: "claude-code", + budgetUsd: 8, + stopped: "decided" as const, + taskSha256: "cd".repeat(32), + versions: { rea: "6.8.0", agentenv: "0.9.1431" }, + measuredAt: "2026-11-07T01:00:00.000Z", + }; + const section = difficultySection({ + "add-eggs": difficultyRecord({ + ...base, + model: "m", + solves: [pass, pass, pass], + }), + "add-bread": difficultyRecord({ + ...base, + model: "m", + solves: [pass, miss, miss, miss, pass], + }), + }); + expect(section).toContain("## Measured difficulty"); + expect(section).toContain("`add-eggs`: m passed 3 of 3, too easy"); + expect(section).toContain("`add-bread`: m passed 2 of 5, in the band"); + + const readme = "# App\n\n## Task `add-eggs`\n\nText.\n"; + const once = withDifficultySection(readme, section); + expect(withDifficultySection(once, section)).toBe(once); + expect(once.startsWith(readme)).toBe(true); + const later = withDifficultySection( + `${once}\n## After\n\nMore.\n`, + "## Measured difficulty\n\nNew.\n", + ); + expect(later).toContain("New."); + expect(later).not.toContain("too easy"); + expect(later).toContain("## After\n\nMore.\n"); + }); +}); diff --git a/src/domain/desktop/taskDifficulty.ts b/src/domain/desktop/taskDifficulty.ts new file mode 100644 index 0000000000..34ba494185 --- /dev/null +++ b/src/domain/desktop/taskDifficulty.ts @@ -0,0 +1,196 @@ +/** How one solve ended, as agentenv reports a run: only `passed` and `scored below 1` finished. */ +export type SolveOutcome = + | "passed" + | "scored below 1" + | "unscored" + | "failed" + | "cancelled"; + +/** One solve of a task in a fresh computer. */ +export type SolveRecord = { + readonly outcome: SolveOutcome; + readonly turns: number | null; + readonly usd: number | null; + readonly seconds: number; + /** The checks that failed, for a solve that finished. */ + readonly failed_checks: string[]; + readonly instance_id: string | null; + /** The error agentenv reported for the run, such as why it did not finish. */ + readonly error: string | null; +}; + +/** Completed paid solves retained when measurement stops before publication. */ +export interface PartialTaskMeasurement { + readonly kind: "partial-task-measurement"; + readonly bundle: string; + readonly task: string; + readonly solves: readonly SolveRecord[]; +} + +/** Where the finished solves place a task: kept in the band, or outside it either way. */ +export type Band = "in_band" | "too_easy" | "too_hard" | "undecided"; + +/** Why the measure stopped asking for solves. */ +export type StopReason = "decided" | "budget" | "unfinished" | "cost_unknown"; + +/** Solves asked first; a task that passes none or all of them is decided. */ +const FIRST_SOLVES = 3; +/** Solves in all for a task the first ones leave undecided. */ +const ALL_SOLVES = 5; +/** The most passes of all the solves a task in the band has. */ +const MOST_BAND_PASSES = 3; +/** Solves that did not finish, run again, before the measure stops. */ +const MAX_UNFINISHED = 2; + +const finished = (solve: SolveRecord): boolean => + solve.outcome === "passed" || solve.outcome === "scored below 1"; + +const counts = (solves: readonly SolveRecord[]) => { + const done = solves.filter(finished); + return { + done: done.length, + passes: done.filter((solve) => solve.outcome === "passed").length, + }; +}; + +/** The band the finished solves place the task in: 0 or 3 of 3 decide it, else 1 to 3 of 5 keep it. */ +export const bandOf = (solves: readonly SolveRecord[]): Band => { + const { done, passes } = counts(solves); + if (done >= ALL_SOLVES) + return passes <= MOST_BAND_PASSES ? "in_band" : "too_easy"; + if (done >= FIRST_SOLVES && passes === 0) return "too_hard"; + if (done >= FIRST_SOLVES && passes === done) return "too_easy"; + return "undecided"; +}; + +/** The total cost of the solves, unknown costs as none. */ +const spent = (solves: readonly SolveRecord[]): number => + solves.reduce((total, solve) => total + (solve.usd ?? 0), 0); + +/** + * Whether to run another solve: not once the band is decided, after too many + * unfinished solves, after a solve of unknown cost, or when the largest cost + * so far would take the total past the budget. + */ +export const nextSolve = ( + solves: readonly SolveRecord[], + budgetUsd: number, +): + | { readonly next: true } + | { readonly next: false; readonly stopped: StopReason } => { + if (bandOf(solves) !== "undecided") + return { next: false, stopped: "decided" }; + if (solves.filter((solve) => !finished(solve)).length > MAX_UNFINISHED) + return { next: false, stopped: "unfinished" }; + if (solves.some((solve) => solve.usd === null)) + return { next: false, stopped: "cost_unknown" }; + const largest = Math.max(0, ...solves.map((solve) => solve.usd ?? 0)); + const projected = spent(solves) + largest; + const rounding = + Number.EPSILON * Math.max(projected, budgetUsd) * (solves.length + 2); + return projected - budgetUsd > rounding + ? { next: false, stopped: "budget" } + : { next: true }; +}; + +/** What a bundle records of one task's measure in `difficulty.json`. */ +export type DifficultyRecord = { + readonly model: string; + readonly agent: string; + readonly measured_at: string; + readonly band: Band; + readonly stopped: StopReason; + /** Finished solves, which the band counts. */ + readonly k: number; + readonly passes: number; + readonly unfinished: number; + readonly usd: number; + readonly budget_usd: number; + /** Each finished solve's turns, in order. */ + readonly turns: (number | null)[]; + readonly failures_per_check: Record; + readonly solves: SolveRecord[]; + /** The SHA-256 of the task's steps file followed by its verifier, so a changed task shows the record is stale. */ + readonly task_sha256: string; + readonly versions: { readonly rea: string; readonly agentenv: string | null }; +}; + +const cents = (usd: number): number => Math.round(usd * 1e6) / 1e6; + +export const difficultyRecord = (input: { + readonly model: string; + readonly agent: string; + readonly solves: readonly SolveRecord[]; + readonly budgetUsd: number; + readonly stopped: StopReason; + readonly taskSha256: string; + readonly versions: DifficultyRecord["versions"]; + readonly measuredAt: string; +}): DifficultyRecord => { + const done = input.solves.filter(finished); + // A Map, so a check named like an object's own member, such as constructor, counts as any other. + const failures = new Map(); + for (const solve of done) + for (const check of solve.failed_checks) + failures.set(check, (failures.get(check) ?? 0) + 1); + return { + model: input.model, + agent: input.agent, + measured_at: input.measuredAt, + band: bandOf(input.solves), + stopped: input.stopped, + k: done.length, + passes: counts(input.solves).passes, + unfinished: input.solves.length - done.length, + usd: cents(spent(input.solves)), + budget_usd: input.budgetUsd, + turns: done.map((solve) => solve.turns), + failures_per_check: Object.fromEntries(failures), + solves: [...input.solves], + task_sha256: input.taskSha256, + versions: input.versions, + }; +}; + +const BAND_WORDS: Readonly> = { + in_band: "in the band", + too_easy: "too easy", + too_hard: "too hard", + undecided: "undecided", +}; + +const DIFFICULTY_HEADING = "## Measured difficulty"; + +/** What the README names of a task's record. */ +export type DifficultySummary = Pick< + DifficultyRecord, + "model" | "passes" | "k" | "band" | "usd" +>; + +/** The README section naming each measured task's model, passes, band and cost. */ +export const difficultySection = ( + records: Readonly>, +): string => + `${DIFFICULTY_HEADING} + +\`difficulty.json\` records each task's solves in fresh computers, by REA's \`measure_build_task\`: 3 solves, 2 more +unless they pass none or all, and a task is in the band when it passes 1 to 3 of 5. + +${Object.entries(records) + .map( + ([name, record]) => + `- \`${name}\`: ${record.model} passed ${String(record.passes)} of ${String(record.k)}, ${BAND_WORDS[record.band]}, $${record.usd.toFixed(2)} in all.`, + ) + .join("\n")} +`; + +/** The README with its measured-difficulty section replaced, or added at the end. */ +export const withDifficultySection = ( + readme: string, + section: string, +): string => { + const start = readme.indexOf(`\n${DIFFICULTY_HEADING}\n`); + if (start === -1) return `${readme.replace(/\n*$/u, "\n")}\n${section}`; + const next = readme.indexOf("\n## ", start + 1); + return `${readme.slice(0, start + 1)}${section}${next === -1 ? "" : readme.slice(next)}`; +}; diff --git a/src/domain/desktop/viewPointer.ts b/src/domain/desktop/viewPointer.ts index e29525c8fd..ca090d26e2 100644 --- a/src/domain/desktop/viewPointer.ts +++ b/src/domain/desktop/viewPointer.ts @@ -133,7 +133,12 @@ const holdsInside = ( ) => { const steps = insideSteps(key) ?? []; if (format === "xml") - return isXmlElement(member) && elementHolds(member, steps, value); + return ( + isXmlElement(member) && + (steps.length === 1 && steps[0] === "text()" + ? collapseWhitespace(elementText(member)) === value + : elementHolds(member, steps, value)) + ); let found: JsonValue | undefined = member; for (const step of steps) found = isRecord(found) ? own(found, step) : undefined; diff --git a/src/ghidra/GhidraTransport.ts b/src/ghidra/GhidraTransport.ts index 0d515f73df..6f06f01897 100644 --- a/src/ghidra/GhidraTransport.ts +++ b/src/ghidra/GhidraTransport.ts @@ -95,26 +95,10 @@ export const observeGhidraEndpoint = async ( return { path: endpoint.path }; } if (process.platform === "win32") { - let encoded: string; - try { - encoded = readWindowsPrivateRuntimeFile( - endpoint.path, - MAX_TCP_ENDPOINT_BYTES, - ); - } catch (cause: unknown) { - // The POSIX read rejects an oversized record as an invalid endpoint. - // The Windows lease reports the same bound as a reader limit. - if ( - cause instanceof ArtifactReaderFailure && - cause.reason === "limit" - ) - throw endpointFailure( - endpoint, - "Ghidra TCP endpoint is invalid", - cause, - ); - throw cause; - } + const encoded = readWindowsPrivateRuntimeFile( + endpoint.path, + MAX_TCP_ENDPOINT_BYTES, + ); assertArtifactReadActive(options.signal); return parseTcpEndpoint(endpoint, encoded); } @@ -126,13 +110,9 @@ export const observeGhidraEndpoint = async ( const initial = await file.stat({ bigint: true }); assertArtifactReadActive(options.signal); if (initial.size > BigInt(MAX_TCP_ENDPOINT_BYTES)) - throw endpointFailure( - endpoint, - "Ghidra TCP endpoint is invalid", - new ArtifactReaderFailure( - "limit", - `Ghidra endpoint record exceeds its compact publication bound (${initial.size} > ${MAX_TCP_ENDPOINT_BYTES} bytes)`, - ), + throw new ArtifactReaderFailure( + "limit", + `Ghidra endpoint record exceeds its compact publication bound (${initial.size} > ${MAX_TCP_ENDPOINT_BYTES} bytes)`, ); const input = readArtifactFileChunks(file, { end: initial.size, @@ -196,14 +176,12 @@ const observationFailure = ( cause: unknown, ): GhidraSessionError => { const { cause: primary, origin } = artifactFilePrimaryFacts(cause); - // The native reader rejects expanded TCP records before JSON parsing. - // Keep that protocol refusal distinct from an unrelated observation failure. - const observationMessage = + // Classify a bounded endpoint record without discarding its owned cleanup. + const invalidRecord = endpoint.transport === "authenticated-loopback-tcp" && - cause instanceof ArtifactReaderFailure && - cause.reason === "limit" - ? "Ghidra TCP endpoint is invalid" - : "Ghidra endpoint observation failed"; + origin === "owned-or-classified" && + primary instanceof ArtifactReaderFailure && + primary.reason === "limit"; let failure = origin === "owned-or-classified" && primary instanceof GhidraSessionError ? primary @@ -214,7 +192,13 @@ const observationFailure = ( }))("cancelled", "Ghidra endpoint observation was cancelled", cause, { cancellation: { reason: artifactFileCancellationCause(cause) }, }) - : endpointFailure(endpoint, observationMessage, cause); + : endpointFailure( + endpoint, + invalidRecord + ? "Ghidra TCP endpoint is invalid" + : "Ghidra endpoint observation failed", + cause, + ); if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) { const cleanup = new ProviderCleanupError( "ghidra", diff --git a/src/server/registerDesktopBuildTools.ts b/src/server/registerDesktopBuildTools.ts index 1f0fc1dfb6..4163bb56ce 100644 --- a/src/server/registerDesktopBuildTools.ts +++ b/src/server/registerDesktopBuildTools.ts @@ -173,7 +173,7 @@ export const registerDesktopBuildTools = ( server: EvidenceMcpServer, computers: Pick< DesktopBuildComputers, - "start" | "act" | "close" | "steps" | "redaction" + "start" | "act" | "close" | "steps" | "measure" | "redaction" >, logger: Logger, ): void => { @@ -220,6 +220,12 @@ export const registerDesktopBuildTools = ( steps.checkTask(input, context.mcpReq.signal, progress), ), ); + // Solves run one at a time for minutes each, so progress keeps the client waiting. + registerPlain(tool("measure_build_task"), (input, context) => + withHeartbeat(context, "measure_build_task", (progress) => + computers.measure(input, context.mcpReq.signal, progress), + ), + ); registerPlain(tool("record_build_consent"), (input, context) => steps.recordConsent(input, context.mcpReq.signal), ); diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts index 4ebf0d9a54..23e766f398 100644 --- a/tests/acceptance/modelArtifactCli.test.ts +++ b/tests/acceptance/modelArtifactCli.test.ts @@ -527,6 +527,7 @@ cliTest( category: "unsupported_target", details: { constraint: "file_format_rejected", + path, reason: "Declared extent is outside the file", }, }); diff --git a/tests/boundary/application/buildExportGzipScans.test.ts b/tests/boundary/application/buildExportGzipScans.test.ts index 7f8222b2f8..80d0ec49a3 100644 --- a/tests/boundary/application/buildExportGzipScans.test.ts +++ b/tests/boundary/application/buildExportGzipScans.test.ts @@ -59,7 +59,7 @@ it("decompresses a gzip seed file once to check it for known secrets and its siz gunzipped.length = 0; const refused = await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory }, + arguments: { computer_id: id, tasks: [{ task: TASK }], directory }, }); const { error } = parseMcpToolError(refused as CallToolResult); expect(error.code).toBe("invalid_request"); diff --git a/tests/boundary/cli/cliToolCoverage.test.ts b/tests/boundary/cli/cliToolCoverage.test.ts index f6e931b711..2dfe90d1ee 100644 --- a/tests/boundary/cli/cliToolCoverage.test.ts +++ b/tests/boundary/cli/cliToolCoverage.test.ts @@ -104,6 +104,7 @@ const MCP_TOOLS_WITHOUT_DEDICATED_CLI = Object.freeze([ "stop_build_capture", "export_build_environment", "check_build_task", + "measure_build_task", "record_build_consent", ]); diff --git a/tests/boundary/mcp/buildCaptureMcp.test.ts b/tests/boundary/mcp/buildCaptureMcp.test.ts index 4338ecee5e..5a6432f5b0 100644 --- a/tests/boundary/mcp/buildCaptureMcp.test.ts +++ b/tests/boundary/mcp/buildCaptureMcp.test.ts @@ -121,7 +121,7 @@ it("captures the app inside the computer, acknowledges checkpoints, drafts the t ( await callOk(client, "export_build_environment", { computer_id: id, - task: TASK, + tasks: [{ task: TASK }], directory, }) ).structuredContent, @@ -139,7 +139,7 @@ it("captures the app inside the computer, acknowledges checkpoints, drafts the t environment: "notes", files: (await files(directory)).length, seed_files: [".config/notes/settings.ini", "Documents/today.txt"], - task: { name: "add-eggs", failing_on_seed: [expect.any(String)] }, + tasks: [{ name: "add-eggs", failing_on_seed: [expect.any(String)] }], }); expect(await rea.notes("tasks.jsonl")).toEqual([TASK]); @@ -209,7 +209,7 @@ it("keeps a screenshot of each checkpoint in the computer, which the export take await callOk(client, "stop_build_capture", { computer_id: id }); await callOk(client, "export_build_environment", { computer_id: id, - task: TASK, + tasks: [{ task: TASK }], directory: join(await createTestTempDirectory("build-screens-"), "out"), }); @@ -350,6 +350,26 @@ it("refuses an older build runtime before capturing the app", async () => { ); }); +it("refuses a build runtime whose export takes no task ends before capturing the app", async () => { + const { rea, client } = await fakes({ + env: { FAKE_REA_EXPORT_WITHOUT: "taskEnd" }, + }); + const id = await startBuildComputer(client, REQUEST); + const { error } = parseMcpToolError( + await client.callTool({ + name: "start_build_capture", + arguments: { computer_id: id }, + }), + ); + expect(error.message).toContain("does not support task ends"); + expect(error.message).not.toContain("synthetic seeds"); + expect(JSON.stringify(error)).toContain("pack:build-computer-runtime"); + const calls = (await rea.notes("calls.jsonl")) as { argv: string[] }[]; + expect(calls.map(({ argv }) => argv[0])).not.toContain( + "capture-desktop-application", + ); +}); + it("reports an unsuccessful runtime schema command without diagnosing its cause", async () => { const { rea, client } = await fakes({ env: { FAKE_REA_SCHEMA_FAILURE: "1" }, @@ -406,12 +426,14 @@ it("exports a task larger than the old 96 KiB refusal and keeps its prompt", asy ( await callOk(client, "export_build_environment", { computer_id: id, - task, + tasks: [{ task }], directory: join(await createTestTempDirectory("build-out-"), "b"), }) ).structuredContent, ); - expect(exported.task).toMatchObject({ name: task.name }); + expect(exported.tasks).toEqual([ + expect.objectContaining({ name: task.name }), + ]); expect(await rea.notes("tasks.jsonl")).toEqual([task]); const calls = (await rea.notes("calls.jsonl")) as { argv: string[] }[]; expect(calls.map(({ argv }) => argv[0])).toContain( @@ -421,7 +443,9 @@ it("exports a task larger than the old 96 KiB refusal and keeps its prompt", asy name: "export_build_environment", arguments: { computer_id: id, - task: { ...task, prompt: "a".repeat(LINUX_ARGUMENT_VECTOR_BYTES) }, + tasks: [ + { task: { ...task, prompt: "a".repeat(LINUX_ARGUMENT_VECTOR_BYTES) } }, + ], directory: join(await createTestTempDirectory("build-out-"), "b"), }, }); @@ -466,7 +490,7 @@ it("refuses capture steps out of order and a capture of a bundle's computer", as ( await call("export_build_environment", { computer_id: id, - task: TASK, + tasks: [{ task: TASK }], directory: join(await createTestTempDirectory("build-out-"), "b"), }) ).error.message, diff --git a/tests/boundary/mcp/buildCaptureRetryMcp.test.ts b/tests/boundary/mcp/buildCaptureRetryMcp.test.ts index e92f68e156..a64d03249b 100644 --- a/tests/boundary/mcp/buildCaptureRetryMcp.test.ts +++ b/tests/boundary/mcp/buildCaptureRetryMcp.test.ts @@ -113,7 +113,7 @@ it("finishes a stop whose draft reply was lost when stop is asked again, and rep expect(await rea.notes("stopped.jsonl")).toHaveLength(1); const exported = await call(client, "export_build_environment", { computer_id: id, - task: BUILD_TASK, + tasks: [{ task: BUILD_TASK }], directory: join(await createTestTempDirectory("build-out-"), "bundle"), }); expect(exported.isError, JSON.stringify(exported)).not.toBe(true); @@ -249,7 +249,7 @@ it("returns the bundle an export saved when the computer closes while it commits }); const exported = await call(client, "export_build_environment", { computer_id: id, - task: BUILD_TASK, + tasks: [{ task: BUILD_TASK }], directory, }); expect(exported.isError, JSON.stringify(exported)).not.toBe(true); @@ -285,7 +285,7 @@ const captureRequest = async ( ( await callOk(client, "export_build_environment", { computer_id: id, - task: BUILD_TASK, + tasks: [{ task: BUILD_TASK }], directory, }) ).structuredContent, @@ -481,7 +481,7 @@ it("brings back a bundle whose file list outgrows a command's 1 MiB of output", ( await callOk(client, "export_build_environment", { computer_id: id, - task: BUILD_TASK, + tasks: [{ task: BUILD_TASK }], directory, }) ).structuredContent, diff --git a/tests/boundary/mcp/buildCaptureStatePathsMcp.test.ts b/tests/boundary/mcp/buildCaptureStatePathsMcp.test.ts index 6baa3cd7e6..a9a8352609 100644 --- a/tests/boundary/mcp/buildCaptureStatePathsMcp.test.ts +++ b/tests/boundary/mcp/buildCaptureStatePathsMcp.test.ts @@ -84,7 +84,7 @@ const exportTo = async ( name: "export_build_environment", arguments: { computer_id: id, - task: TASK, + tasks: [{ task: TASK }], directory: join(await createTestTempDirectory("build-out-"), "notes"), }, }); @@ -107,7 +107,9 @@ it("makes a file named in state_paths the app's document, which the draft covers (await exportTo(client, id)).structuredContent, ); expect(exported.seed_files).toContain(DOCUMENT); - expect(exported.task).toMatchObject({ name: TASK.name }); + expect(exported.tasks).toEqual([ + expect.objectContaining({ name: TASK.name, end_checkpoint: "end" }), + ]); }); it("says why a draft has no files when the app's document is not recognized, and the export names state_paths", async () => { diff --git a/tests/boundary/mcp/buildMeasureMcp.test.ts b/tests/boundary/mcp/buildMeasureMcp.test.ts new file mode 100644 index 0000000000..a3aa66e4c6 --- /dev/null +++ b/tests/boundary/mcp/buildMeasureMcp.test.ts @@ -0,0 +1,225 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { isPortableBundleName } from "../../../src/application/environment/environmentBundle.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseDesktopTask } from "../../../src/domain/desktop/appTask.js"; +import { + renderTaskSteps, + renderTaskVerifier, + taskVerifierArtifact, +} from "../../../src/domain/desktop/appTaskPython.js"; +import { callOk, connectBuildServer } from "../../fixtures/buildComputerMcp.js"; +import { fakeAgentEnvSolves } from "../../fixtures/fakeAgentEnvSolves.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const TASK = { + name: "set-year", + prompt: "In Notes, set the year to 1952 and save it.", + checks: [ + { + id: "year", + kind: "lines_present", + file: "Documents/n.txt", + lines: ["1952"], + }, + { + id: "kept", + kind: "lines_present", + file: "Documents/n.txt", + lines: ["milk"], + }, + ], +}; +const MODEL = "anthropic/claude-sonnet-5-5"; +const README = "# Notes computer app\n\n## Task `set-year`\n\nThe task.\n"; + +/** A bundle holding TASK as the export writes it, with its README. */ +const bundleWithTask = async () => { + const parsed = parseDesktopTask(TASK, isPortableBundleName); + if (!parsed.ok) throw new Error(parsed.error); + const bundle = await createTestTempDirectory("measured-bundle-"); + const steps = renderTaskSteps(parsed.value, "notes"); + const verifier = renderTaskVerifier(parsed.value, "notes"); + await mkdir(join(bundle, "tasks")); + await mkdir(join(bundle, "artifacts", taskVerifierArtifact(parsed.value)), { + recursive: true, + }); + await writeFile(join(bundle, "tasks", "set-year.json"), steps); + await writeFile( + join(bundle, "artifacts", taskVerifierArtifact(parsed.value), "verify.py"), + verifier, + ); + await writeFile(join(bundle, "README.md"), README); + const digest = createHash("sha256") + .update(steps) + .update(verifier) + .digest("hex"); + return { bundle, digest }; +}; + +const passed = (usd = 1, turns = 20) => ({ + outcome: "passed" as const, + checks: { year: true, kept: true }, + turns, + usd, +}); +const missed = (check: "year" | "kept", usd = 1) => ({ + outcome: "scored below 1" as const, + checks: { year: check !== "year", kept: check !== "kept" }, + turns: 30, + usd, +}); + +const measure = async ( + solves: Parameters[0], + budget = 10, + task = TASK.name, +) => { + const runner = await fakeAgentEnvSolves(solves); + const client = await connectBuildServer({ + ...runner.environment, + LITELLM_BASE_URL: "http://127.0.0.1:9/fake-endpoint", + }); + const { bundle, digest } = await bundleWithTask(); + const call = () => + client.callTool({ + name: "measure_build_task", + arguments: { bundle, task, model: MODEL, budget_usd: budget }, + }); + return { runner, client, bundle, digest, call }; +}; + +it("measures a task in fresh computers: 1 of 3 asks for 2 more, 2 of 5 keeps it, and the bundle records it", async () => { + const { runner, client, bundle, digest } = await measure([ + passed(), + missed("year"), + missed("year"), + passed(1.5, 24), + missed("kept"), + ]); + + const measured = toolContract("measure_build_task").outputSchema.parse( + ( + await callOk(client, "measure_build_task", { + bundle, + task: TASK.name, + model: MODEL, + budget_usd: 10, + }) + ).structuredContent, + ); + + expect(measured).toMatchObject({ + task: "set-year", + model: MODEL, + agent: "claude-code", + band: "in_band", + stopped: "decided", + k: 5, + passes: 2, + unfinished: 0, + usd: 5.5, + turns: [20, 30, 30, 24, 30], + failures_per_check: { year: 2, kept: 1 }, + }); + const solves = (await runner.notes("solves.jsonl")) as Record< + string, + unknown + >[]; + expect(solves).toHaveLength(5); + for (const solve of solves) + expect(solve).toEqual({ + root: bundle, + tasks: ["set-year"], + model: MODEL, + agent: "claude-code", + sandbox: "local", + emulate: false, + endpoint: "http://127.0.0.1:9/fake-endpoint", + }); + const recorded = JSON.parse( + await readFile(join(bundle, "difficulty.json"), "utf8"), + ) as { tasks: Record> }; + expect(recorded.tasks["set-year"]).toMatchObject({ + model: MODEL, + band: "in_band", + k: 5, + passes: 2, + usd: 5.5, + failures_per_check: { year: 2, kept: 1 }, + task_sha256: digest, + versions: { rea: expect.any(String) }, + }); + const readme = await readFile(join(bundle, "README.md"), "utf8"); + expect(readme.startsWith(README)).toBe(true); + expect(readme).toContain("## Measured difficulty"); + expect(readme).toContain(`\`set-year\`: ${MODEL} passed 2 of 5, in the band`); +}); + +it("stops at 3 of 3 as too easy, running an unfinished solve again without counting it and keeping why it failed", async () => { + const { runner, bundle, call } = await measure([ + { outcome: "failed", checks: {}, turns: 3, usd: 0.1 }, + passed(), + passed(), + passed(), + ]); + + const measured = toolContract("measure_build_task").outputSchema.parse( + (await call()).structuredContent, + ); + + expect(measured).toMatchObject({ + band: "too_easy", + k: 3, + passes: 3, + unfinished: 1, + }); + expect(measured.solves.map((solve) => solve.error)).toEqual([ + "RuntimeError: deploy", + null, + null, + null, + ]); + const recorded = JSON.parse( + await readFile(join(bundle, "difficulty.json"), "utf8"), + ) as { tasks: Record }; + expect(recorded.tasks["set-year"]?.solves[0]?.error).toBe( + "RuntimeError: deploy", + ); + expect(await runner.notes("solves.jsonl")).toHaveLength(4); +}); + +it("stops before a solve the largest cost so far would take past the budget, leaving the band undecided", async () => { + const { runner, call } = await measure( + [passed(0.9), missed("year", 0.9), passed(0.9)], + 2, + ); + + const measured = toolContract("measure_build_task").outputSchema.parse( + (await call()).structuredContent, + ); + + expect(measured).toMatchObject({ + band: "undecided", + stopped: "budget", + k: 2, + usd: 1.8, + budget_usd: 2, + }); + expect(await runner.notes("solves.jsonl")).toHaveLength(2); +}); + +it("refuses a task the bundle does not hold before running a solve", async () => { + const { runner, call } = await measure([passed()], 10, "add-bread"); + + const { error } = parseMcpToolError(await call()); + + expect(error.code).toBe("invalid_request"); + expect(error.message).toContain("add-bread"); + expect(await runner.notes("solves.jsonl")).toEqual([]); +}); diff --git a/tests/boundary/mcp/buildSecretsMcp.test.ts b/tests/boundary/mcp/buildSecretsMcp.test.ts index 934762e7d7..52121f4a5d 100644 --- a/tests/boundary/mcp/buildSecretsMcp.test.ts +++ b/tests/boundary/mcp/buildSecretsMcp.test.ts @@ -234,7 +234,7 @@ it("refuses to write a bundle that holds a handle's value, naming the files but const directory = join(await createTestTempDirectory("build-out-"), "notes"); const refused = await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory }, + arguments: { computer_id: id, tasks: [{ task: TASK }], directory }, }); const { error } = parseMcpToolError(refused as CallToolResult); expect(error.code).toBe("invalid_request"); @@ -258,7 +258,7 @@ it("refuses export if a declared handle cannot be checked", async () => { const directory = join(await createTestTempDirectory("build-out-"), "notes"); const result = await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory }, + arguments: { computer_id: id, tasks: [{ task: TASK }], directory }, }); const { error } = parseMcpToolError(result); expect(error.code).toBe("invalid_request"); @@ -290,7 +290,7 @@ it("refuses a bundle containing a declared form-encoded secret", async () => { const directory = join(await createTestTempDirectory("build-out-"), "notes"); const result = await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory }, + arguments: { computer_id: id, tasks: [{ task: TASK }], directory }, }); const { error } = parseMcpToolError(result as CallToolResult); expect(error.message).toContain("master"); @@ -331,7 +331,7 @@ it.each(["utf8", "utf16le", "utf16be"] as const)( ); const result = await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory }, + arguments: { computer_id: id, tasks: [{ task: TASK }], directory }, }); const error = parseMcpToolError(result).error; expect(error.code).toBe("invalid_request"); @@ -359,7 +359,7 @@ it("preserves unmarked token-shaped data in seeded UTF-16 and gzip files", async const directory = join(await createTestTempDirectory("build-out-"), "notes"); const refused = await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory }, + arguments: { computer_id: id, tasks: [{ task: TASK }], directory }, }); expect(refused.isError, JSON.stringify(refused)).not.toBe(true); const plain = await readFile( @@ -400,7 +400,7 @@ it("refuses a gzip file that expands past what REA checks for known secrets", as const directory = join(await createTestTempDirectory("build-out-"), "notes"); const refused = await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory }, + arguments: { computer_id: id, tasks: [{ task: TASK }], directory }, }); const { error } = parseMcpToolError(refused as CallToolResult); expect(error.code).toBe("invalid_request"); @@ -451,7 +451,7 @@ it("withholds the automatic draft before running it after a declared secret", as ); const exported = await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory }, + arguments: { computer_id: id, tasks: [{ task: TASK }], directory }, }); expect(exported.isError, JSON.stringify(exported)).not.toBe(true); }); diff --git a/tests/boundary/mcp/buildSeveralTasksMcp.test.ts b/tests/boundary/mcp/buildSeveralTasksMcp.test.ts new file mode 100644 index 0000000000..db7a1d6ff8 --- /dev/null +++ b/tests/boundary/mcp/buildSeveralTasksMcp.test.ts @@ -0,0 +1,137 @@ +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { + BREAD_NOTE, + NEAR_EGGS_NOTE, + severalEndsCapture, +} from "../../../src/application/desktop/desktopCapture.fixture.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { + BUILD_REQUEST as REQUEST, + BUILD_TASK as ADD_EGGS, + buildCaptureFakes as fakes, +} from "../../fixtures/buildCaptureFakes.js"; +import { callOk, startBuildComputer } from "../../fixtures/buildComputerMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const ADD_BREAD = { + name: "add-bread", + prompt: "Add bread to today's list and save it.", + checks: [ + { kind: "lines_present", file: "Documents/today.txt", lines: ["bread"] }, + ], +}; +const TASKS = [ + { task: ADD_EGGS, end_checkpoint: "end" }, + { task: ADD_BREAD, end_checkpoint: "end-add-bread" }, +]; + +/** A build computer whose capture took seed, end, end-add-bread and near-add-eggs, stopped. */ +const stoppedSeveral = async () => { + const built = await fakes({ + recorded: severalEndsCapture("near-add-eggs"), + extraContent: [BREAD_NOTE, NEAR_EGGS_NOTE], + tasks: TASKS, + }); + const id = await startBuildComputer(built.client, REQUEST); + await callOk(built.client, "start_build_capture", { computer_id: id }); + for (const name of ["seed", "end", "end-add-bread", "near-add-eggs"]) + await callOk(built.client, "mark_build_checkpoint", { + computer_id: id, + name, + }); + const stopped = toolContract("stop_build_capture").outputSchema.parse( + (await callOk(built.client, "stop_build_capture", { computer_id: id })) + .structuredContent, + ); + return { ...built, id, stopped }; +}; + +it("drafts each later task from the end before it, and exports every task checked at its own end", async () => { + const { rea, client, id, stopped } = await stoppedSeveral(); + + expect(stopped.draft).toMatchObject({ + seed_checkpoint: "seed", + end_checkpoint: "end", + }); + expect(stopped.later_drafts).toEqual([ + expect.objectContaining({ + end_checkpoint: "end-add-bread", + draft: expect.objectContaining({ + seed_checkpoint: "end", + end_checkpoint: "end-add-bread", + }), + }), + ]); + const directory = join(await createTestTempDirectory("build-out-"), "notes"); + const exported = toolContract("export_build_environment").outputSchema.parse( + ( + await callOk(client, "export_build_environment", { + computer_id: id, + tasks: TASKS, + directory, + }) + ).structuredContent, + ); + + expect(exported.tasks).toEqual([ + expect.objectContaining({ name: "add-eggs", end_checkpoint: "end" }), + expect.objectContaining({ + name: "add-bread", + end_checkpoint: "end-add-bread", + }), + ]); + expect(await rea.notes("tasks.jsonl")).toEqual([ADD_EGGS, ADD_BREAD]); + const calls = (await rea.notes("calls.jsonl")) as { argv: string[] }[]; + expect(calls.map(({ argv }) => argv[0])).toEqual([ + "export-desktop-application", + "capture-desktop-application", + "describe-desktop-application-task", + "describe-desktop-application-task", + "export-desktop-application", + ]); + expect(calls[3]?.argv).toEqual( + expect.arrayContaining([ + "--seed-checkpoint", + "end", + "--end-checkpoint", + "end-add-bread", + ]), + ); + expect(calls[4]?.argv).toEqual( + expect.arrayContaining([ + "--task-end", + "add-eggs=end", + "--task-end", + "add-bread=end-add-bread", + ]), + ); +}); + +it("refuses an export naming an end checkpoint the capture did not take, before running the export", async () => { + const { rea, client, id } = await stoppedSeveral(); + + const refused = parseMcpToolError( + await client.callTool({ + name: "export_build_environment", + arguments: { + computer_id: id, + tasks: [TASKS[0], { task: ADD_BREAD, end_checkpoint: "end-bread" }], + directory: join(await createTestTempDirectory("build-out-"), "notes"), + }, + }), + ); + + expect(refused.error.code).toBe("invalid_request"); + expect(refused.error.message).toContain("end-bread"); + const calls = (await rea.notes("calls.jsonl")) as { argv: string[] }[]; + expect( + calls.some( + ({ argv }) => + argv[0] === "export-desktop-application" && !argv.includes("--schema"), + ), + ).toBe(false); +}); diff --git a/tests/boundary/mcp/buildWithheldMcp.test.ts b/tests/boundary/mcp/buildWithheldMcp.test.ts index d28230f81c..861521f40e 100644 --- a/tests/boundary/mcp/buildWithheldMcp.test.ts +++ b/tests/boundary/mcp/buildWithheldMcp.test.ts @@ -107,7 +107,12 @@ const exportTo = async ( const directory = join(await createTestTempDirectory("build-out-"), "notes"); const result = (await client.callTool({ name: "export_build_environment", - arguments: { computer_id: id, task: TASK, directory, ...fields }, + arguments: { + computer_id: id, + tasks: [{ task: TASK }], + directory, + ...fields, + }, })) as CallToolResult; return { result, directory }; }; diff --git a/tests/boundary/providers/desktopTaskPython.test.ts b/tests/boundary/providers/desktopTaskPython.test.ts index 65b468bc13..f273f35b05 100644 --- a/tests/boundary/providers/desktopTaskPython.test.ts +++ b/tests/boundary/providers/desktopTaskPython.test.ts @@ -257,7 +257,7 @@ const linesPresent = (...lines: string[]): DesktopTaskCheck => ({ const graded = ({ results }: VerifierOutput) => results.map(({ result, settled, read_cut }) => [result, settled, read_cut]); -it("evaluates every kind of check without computer execution in the exported verifier as REA does", async () => { +it("evaluates every kind of file check without computer-exec as REA does", async () => { const checks: DesktopTaskCheck[] = CHECK_CASES.map(([check], index) => ({ ...check, id: `check-${String(index + 1)}`, diff --git a/tests/boundary/providers/ida/calleeLookupBatch.test.ts b/tests/boundary/providers/ida/calleeLookupBatch.test.ts index e9e1bd52ad..ac078e2faf 100644 --- a/tests/boundary/providers/ida/calleeLookupBatch.test.ts +++ b/tests/boundary/providers/ida/calleeLookupBatch.test.ts @@ -1,7 +1,7 @@ import { rm } from "node:fs/promises"; import { afterEach, describe, expect, it } from "vitest"; import { z } from "zod"; -import { IDA_CALLEE_DISCOVERY } from "../../../../src/ida/IdaCallGraph.js"; +import { UNCLASSIFIED_CALL_SITE_LIMITATION } from "../../../../src/ida/IdaCallGraph.js"; import { IdaOperationRunner } from "../../../../src/ida/IdaOperationRunner.js"; import { IdaSessionClient } from "../../../../src/ida/IdaSessionClient.js"; import { createIdaTarget, RecordingIdaMcp } from "../../../fixtures/idaMcp.js"; @@ -12,13 +12,12 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }); }); const queriesSchema = z.object({ queries: z.array(z.string()) }); -/** - * An uncapped page is complete as a function-edge list and still reports - * computed call sites as unknown discovery, which is what the runner emits. - */ +/** An uncapped inventory still cannot establish every possible callee. */ const uncappedCalleeCoverage = { status: "complete", - discovery: IDA_CALLEE_DISCOVERY, + discovery: { + callees: { status: "unknown", reason: UNCLASSIFIED_CALL_SITE_LIMITATION }, + }, }; const addresses = Array.from( { length: 20 }, diff --git a/tests/fixtures/buildCaptureFakes.ts b/tests/fixtures/buildCaptureFakes.ts index 662d7043d5..1e42449238 100644 --- a/tests/fixtures/buildCaptureFakes.ts +++ b/tests/fixtures/buildCaptureFakes.ts @@ -33,7 +33,7 @@ export const BUILD_REQUEST = { }; const ENDS = ["--seed-checkpoint", "seed", "--end-checkpoint", "end"]; -/** The app's package as the capture records it, the URL the export downloads its file from, if it needs one, and the packages it installs beside the app. */ +/** The app's package as the capture records it, the URL the export downloads its file from, if it needs one, the download it was installed from, if any, and the packages it installs beside the app. */ export interface CapturedInstall { readonly package?: DesktopPackage; readonly packageUrl?: string; @@ -41,10 +41,16 @@ export interface CapturedInstall { readonly extraPackages?: readonly string[]; } +/** A task the export takes with the checkpoint it is checked at. */ +export interface BuildTaskAtEnd { + readonly task: unknown; + readonly end_checkpoint: string; +} + /** * What the real CLI writes for a capture made between seed and end, by - * default `seedEndCapture`: the draft, and the export with BUILD_TASK, which - * must succeed unless `exportRefused`. + * default `seedEndCapture`: the draft, and the export with BUILD_TASK, or + * `tasks` each at its end checkpoint, which must succeed unless `exportRefused`. */ const realCliOutputs = async ( install: CapturedInstall | undefined, @@ -52,10 +58,12 @@ const realCliOutputs = async ( { exportRefused = false, extraContent = [], + tasks = [{ task: BUILD_TASK, end_checkpoint: "end" }], exportArgs = [], }: { exportRefused?: boolean; extraContent?: readonly string[]; + tasks?: readonly BuildTaskAtEnd[]; exportArgs?: readonly string[]; } = {}, ) => { @@ -70,8 +78,13 @@ const realCliOutputs = async ( [EDITED_NOTE, ...extraContent], ); const work = await createTestTempDirectory("build-cli-"); - const task = join(work, "task.json"); - await writeFile(task, JSON.stringify(BUILD_TASK)); + const taskFiles = await Promise.all( + tasks.map(async ({ task }, index) => { + const path = join(work, `task-${String(index)}.json`); + await writeFile(path, JSON.stringify(task)); + return path; + }), + ); const bundle = join(work, "bundle"); const download = join(work, "download-install.json"); if (install?.downloadInstall !== undefined) @@ -94,8 +107,13 @@ const realCliOutputs = async ( "export-desktop-application", capture, bundle, - "--task", - task, + ...taskFiles.flatMap((path) => ["--task", path]), + ...(tasks.length === 1 + ? [] + : tasks.flatMap(({ task, end_checkpoint }) => [ + "--task-end", + `${(task as { name: string }).name}=${end_checkpoint}`, + ])), ...ENDS, "--synthetic-seed", ...(install?.packageUrl === undefined @@ -138,7 +156,8 @@ const realCliOutputs = async ( * package, the archive's `notes`, and names the packages the real export * installs beside it; `screenshotDelaysMs` delays the computer's * screenshots in order. `recorded` replaces what the capture records, with - * `exportRefused` when its export refuses BUILD_TASK, and `extraContent` the + * `exportRefused` when its export refuses BUILD_TASK, or `tasks` in its + * place, and `extraContent` the * retained bytes it holds beyond the note's; `selected` is what it records * when given exactly the `--state-path` of `path`, under the home; * `serverEnv` adds to the REA server's environment, and `exportArgs` to the @@ -153,6 +172,7 @@ export const buildCaptureFakes = async ( readonly recorded?: DesktopApplicationCapture; readonly exportRefused?: boolean; readonly extraContent?: readonly string[]; + readonly tasks?: readonly BuildTaskAtEnd[]; readonly selected?: { readonly path: string; readonly recorded: DesktopApplicationCapture; @@ -168,6 +188,7 @@ export const buildCaptureFakes = async ( ...(options.extraContent === undefined ? {} : { extraContent: options.extraContent }), + ...(options.tasks === undefined ? {} : { tasks: options.tasks }), ...(options.exportArgs === undefined ? {} : { exportArgs: options.exportArgs }), diff --git a/tests/fixtures/fakeAgentEnvSolves.ts b/tests/fixtures/fakeAgentEnvSolves.ts new file mode 100644 index 0000000000..03f00818b8 --- /dev/null +++ b/tests/fixtures/fakeAgentEnvSolves.ts @@ -0,0 +1,123 @@ +import { execFileSync } from "node:child_process"; +import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { createTestTempDirectory } from "./temporaryDirectory.js"; + +/** + * A stand-in `agent_env.bundle.run_bundle` for solves: each call takes the + * next of `FAKE_SOLVES`, a JSON list of `{outcome, checks, turns, usd}`, + * writes the solver's trajectory as Claude Code's last record holds turns and + * cost, and returns a run shaped like agentenv's, its checks under the + * instance's verifications. Each call notes its arguments and whether a + * model endpoint reached it; `FAKE_SOLVE_RAISES` makes it raise instead, and + * `FAKE_SOLVE_HANGS` makes it ignore SIGTERM and never return. + */ +const FAKE_SOLVE_MODULE = String.raw` +import enum +import json +import os +import signal +import time +import types + +STATE = os.environ["FAKE_STATE"] + + +class Outcome(enum.Enum): + PASSED = "passed" + BELOW_ONE = "scored below 1" + FAILED = "failed" + + +def _note(name, value): + with open(os.path.join(STATE, name), "a") as handle: + handle.write(json.dumps(value) + "\n") + + +def run_bundle(root, *, tasks=(), model=None, agent=None, sandbox=None, emulate=True, on_progress=None): + counter = os.path.join(STATE, "count") + n = int(open(counter).read()) if os.path.exists(counter) else 0 + open(counter, "w").write(str(n + 1)) + _note("solves.jsonl", {"root": str(root), "tasks": list(tasks), "model": model, "agent": agent, + "sandbox": sandbox, "emulate": emulate, + "endpoint": os.environ.get("LITELLM_BASE_URL")}) + if os.environ.get("FAKE_SOLVE_RAISES"): + raise RuntimeError(os.environ["FAKE_SOLVE_RAISES"]) + if os.environ.get("FAKE_SOLVE_ROOT_ENTRY"): + os.mkdir(os.path.join(os.environ["AGENT_ENV_LOCAL_SANDBOX_DIR"], os.environ["FAKE_SOLVE_ROOT_ENTRY"])) + if os.environ.get("FAKE_SOLVE_HANGS"): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + time.sleep(3600) + on_progress("deploying the computer") + planned = json.loads(os.environ["FAKE_SOLVES"])[n] + trajectory = os.path.join(STATE, f"trajectory-{n}.json") + with open(trajectory, "w") as handle: + json.dump([{"type": "system"}, {"type": "result", "num_turns": planned["turns"], + "total_cost_usd": planned["usd"]}], handle) + results = [{"id": check, "result": passed, "description": f"{check} read", "settled": True} + for check, passed in planned["checks"].items()] + context = types.SimpleNamespace( + instance_id=f"inst-{n}", + metadata={"verifications": {tasks[0]: {"score": 1.0 if planned["outcome"] == "passed" else 0.0, + "results": results}}}, + prompt_responses=[types.SimpleNamespace(agent_trajectory_object_url="file://" + trajectory)], + ) + still_up = [types.SimpleNamespace(sandbox_id=id, sandbox_type="local") + for id in json.loads(os.environ.get("FAKE_SOLVE_STILL_UP", "[]"))] + run = types.SimpleNamespace(outcome=Outcome(planned["outcome"]), context=context, instance_id=f"inst-{n}", + duration=42.0, error=None if planned["outcome"] != "failed" else RuntimeError("deploy"), + torn_down=types.SimpleNamespace(still_up=still_up)) + return types.SimpleNamespace(runs=(run,)) +`; + +export interface FakeAgentEnvSolves { + readonly environment: Readonly>; + notes(name: string): Promise; +} + +/** A runner whose agentenv answers solves from `solves`, in order. */ +export const fakeAgentEnvSolves = async ( + solves: readonly { + readonly outcome: "passed" | "scored below 1" | "failed"; + readonly checks: Readonly>; + readonly turns: number; + readonly usd: number; + }[], +): Promise => { + const state = await createTestTempDirectory("fake-agentenv-solves-"); + const python = execFileSync( + "python3", + ["-c", "import sys; print(sys.executable)"], + { encoding: "utf8" }, + ).trim(); + await mkdir(join(state, "site", "agent_env"), { recursive: true }); + const runner = join(state, "agent-env"); + await writeFile(runner, `#!${python}\n# Stand-in runner.\n`); + await chmod(runner, 0o755); + await writeFile(join(state, "site", "agent_env", "__init__.py"), ""); + await writeFile( + join(state, "site", "agent_env", "bundle.py"), + FAKE_SOLVE_MODULE, + ); + return { + environment: { + REA_AGENT_ENV_COMMAND: runner, + PYTHONPATH: join(state, "site"), + FAKE_STATE: state, + FAKE_SOLVES: JSON.stringify(solves), + }, + notes: async (name) => { + const path = join(state, name); + try { + await access(path); + } catch { + return []; + } + return (await readFile(path, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as unknown); + }, + }; +}; diff --git a/tests/fixtures/fakeBuildComputerRea.ts b/tests/fixtures/fakeBuildComputerRea.ts index 68bd9cb5f8..bed7642228 100644 --- a/tests/fixtures/fakeBuildComputerRea.ts +++ b/tests/fixtures/fakeBuildComputerRea.ts @@ -133,10 +133,13 @@ elif command == "export-desktop-application": schema = json.load(handle) if os.environ.get("FAKE_REA_OLD_EXPORT"): schema["options"]["properties"].pop("syntheticSeed", None) + for name in os.environ.get("FAKE_REA_EXPORT_WITHOUT", "").split(","): + schema["options"]["properties"].pop(name, None) print(json.dumps(schema)) sys.exit(0) - with open(option(args, "--task")) as handle: - note("tasks.jsonl", json.load(handle)) + for path in options(args, "--task"): + with open(path) as handle: + note("tasks.jsonl", json.load(handle)) with open(output(args[0], "EXPORT")) as handle: printed = handle.read() if "code" not in json.loads(printed):