From fe2985a0b4c3b7210109690c1efeb099780983aa Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 9 Nov 2026 07:09:52 -0800 Subject: [PATCH] feat(desktop): check apps whose files REA can't read, through the app's own headless mode or a screen judge (#9217) * fix(desktop): preserve document URLs and value formats * fix(desktop): bound added checks and omit noisy text guards * fix(desktop): exclude unseeded locks from task checks * fix(desktop): preserve draft bounds and launch URLs * fix(desktop): render draft values using their source format * fix(desktop): preserve launch URLs and JSON draft values * fix(desktop): recognize captured file URL documents * fix(desktop): render captured file URLs for runtime home * fix(desktop): distinguish phone numbers from save metadata * fix(desktop): preserve launch URLs and user text * fix(desktop): preserve captured URLs and user text * fix(desktop): preserve local file URL authority * fix(desktop): preserve local file URL authority * fix(desktop): preserve local file URL authority * fix(desktop): render captured file URLs at launch * fix(desktop): classify numeric timestamps using field context * fix(desktop): classify numeric timestamps using field context * fix(desktop): classify numeric timestamps using field context * fix(desktop): retain context for generated draft values * fix(desktop): retain context for generated draft values * fix(desktop): retain context for generated draft values * test(desktop): cover generated IDs in text guards * test(desktop): cover generated IDs in text guards * test(desktop): cover generated IDs in text guards * fix(desktop): isolate launcher URL encoding * fix(desktop): isolate launcher URL encoding * fix(desktop): isolate launcher URL encoding * fix(desktop): isolate launcher URL encoding * fix(desktop): retain clean text in noisy record fields * fix(desktop): classify added line values in one pass * fix(desktop): apply XML quote rules to text drafts * fix(desktop): distinguish XML attributes from quoted text * fix(desktop): preserve parsed XML line context * Format appTaskChecks.ts with the pinned oxfmt Co-Authored-By: Claude Opus 5.5 * fix(desktop): retain partial XML syntax facts * fix(desktop): recognize markdown autolinks in drafts * fix(desktop): reuse export exclusions in task drafts * fix(desktop): report unchecked client build paths * docs(desktop): clarify gzip scrub limits * fix(desktop): reuse export exclusions in task drafts * docs(desktop): keep the judge command on one line Co-Authored-By: Claude Opus 5.5 * docs(desktop): clarify gzip scrub limits * fix(setup): own desktop build process cleanup * fix(doctor): compare desktop settings with resolved defaults * style(desktop): format the noisy-record draft files with the pinned oxfmt Co-Authored-By: Claude Opus 5.5 * test(desktop): give the root-array draft test its own describe The records describe grew past the 150-line lint limit with it. Co-Authored-By: Claude Opus 5.5 * fix(doctor): inspect client-recorded desktop runners * fix(doctor): reuse matching runner probes * fix(setup): propagate cancellation to desktop actions * fix(setup): stop Hopper on cancel, never report a cancelled setup ready, and keep a failed action's client setting - The Hopper install gets setup's signal: Linux already honoured it, and the macOS installer now stops its downloads and stops before mounting or copying. - runSetup checks the signal after the final doctor and close, so a cancellation there raises instead of returning ready. - A failed pack or agentenv install no longer writes its archive or runner to client configs, so the setting the client already had stays. Co-Authored-By: Claude Opus 5.5 * fix(setup): keep the completed setup result in a cancellation after cleanup A cancellation during the final doctor or host close carried only the applied actions, dropping each client's result and backup path and the doctor report. It now carries the whole result as setup_result, as doctor's cancellation carries its report. Co-Authored-By: Claude Opus 5.5 * test(desktop): move launch execution to boundary suite * fix(desktop): redact build schema failure output * refactor(desktop): normalize schema failure once * fix(desktop): omit unsafe schema parser snippets * fix(desktop): preserve draft change and selected path evidence * test(desktop): group retained dconf note regressions * fix(desktop): respect archive entry format in draft checks * test(desktop): separate unowned-file draft note cases * fix(desktop): preserve explicitly selected journal documents * fix(xml): avoid scanning selected ancestors per tag * fix(desktop): exclude covered text runs from draft counts * fix: preserve text when drafting desktop checks * fix: restore nested desktop state aliases * refactor: share UUID recognition across draft checks * fix: retain multiline XML attribute context * fix: retain relationship identifiers in task drafts * docs: clarify desktop draft identifier handling * test(desktop): type formatting run cases as pairs * fix(desktop): carry document format into selector evaluation * test(desktop): separate markup boundary regression group * fix(desktop): propagate formats through draft selectors * fix(desktop): switch selector format within archive entries * test(desktop): remove duplicate seed filename case * fix(setup): retain diagnostics when cancellation overlaps failure * refactor(setup): keep command cancellation at its owner * test(setup): verify cancellation at the application boundary * test(desktop): pass JSON format in exported index parity * docs(desktop): clarify task checks before scratch export * fix(desktop): respell selected paths in file URLs * test(desktop): group captured document spelling cases * test(desktop): pin reader-free verifier to selector parent * fix(desktop): bound draft preview serialization * fix(desktop): restore selected state aliases * fix(desktop): respect entry format in draft noise * test(desktop): match exported alias evidence at its JSON boundary * fix(desktop): retain app state logs in exported seeds * fix: align desktop numeric seed comparisons * test(desktop): keep one draft exclusion fixture import * test(desktop): retain log eligibility across draft test groups * fix(doctor): resolve empty registered desktop runner * test(desktop): check retained log seed admission * style: fix desktop launch test formatting * style: format native command path schema * fix(desktop): map selected external launch paths * style(desktop): format seed document regressions * test(desktop): group launch URL regressions * fix(desktop): omit redundant nested state aliases * fix(desktop): compare added free text as a whole * fix(desktop): check recompressed seeds as shipped * docs(desktop): clarify recovered seed format * test(desktop): group shipped gzip seed checks * fix: retain quoted identifier text in task drafts * fix: recognize Unicode XML in desktop state readers * fix(desktop): bound nested draft checks * fix(desktop): explain missing arrays in truncated views * fix(desktop): find XML type references in archives * test(desktop): remove duplicated capture assertions * fix(desktop): retain nested omitted text boundaries * style(desktop): format draft value imports * test(desktop): group model and draft value checks * fix(desktop): map captured launch paths once * refactor: remove unused whole-text guard state * refactor: stop passing unused text children * test: align free-text checks with whole-note policy * fix: validate recompressed desktop archive seeds * style(desktop): format archive seed regression * fix(desktop): retain placed-item sibling guards * fix(desktop): preserve archive paths and bound doctor reads * fix(doctor): refuse non-files before descriptor admission * fix(setup): track remaining cleanup resources across retries * test(desktop): specify XML format in heap pairing fixture * fix(desktop): compare screen task views by value * fix(desktop): reject inexact reader numbers * fix(desktop): budget export for both reader passes * fix(desktop): bound reader runner argument size * test(desktop): verify chunked reader payloads and parent output * fix(doctor): query agentenv for model endpoint * style(desktop): format reader payload assertion * fix(doctor): preserve uncertainty for empty endpoint override * fix(desktop): redact runtime schema errors * fix(desktop): normalize captured file URLs * fix(desktop): report seeded files by capture path * fix(desktop): retain startup readiness failure stage * fix(desktop): retain JSON date checks * fix(desktop): preserve archive formatting checks * fix(desktop): preserve explicitly selected journal files * fix(desktop): report build command transport limits * docs(desktop): clarify unreadable dconf versions * test(desktop): remove duplicate history regression * fix(desktop): align XML names with exported reader * fix(desktop): create selected state alias targets * fix(desktop): inspect supported runtime archives * fix(desktop): retain record identity in draft checks * fix(desktop): align timestamp selector digits * fix(desktop): scrub XML independently of reader compatibility * fix(desktop): preserve file aliases and reader diagnostics * fix(desktop): retain remote reader cleanup ownership * style(desktop): format capture review fixes * fix(desktop): retain reader leases across verifier reloads * test(desktop): overlap reader cleanup retries * fix(desktop): run long exports within exec limits --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../references/desktop-build.md | 139 ++++- .../references/environments.md | 5 +- bridge/agentenv/capture.py | 184 ++++++- bridge/agentenv/test_capture.py | 168 +++++++ docs/desktop-build-witness.md | 7 +- docs/specs/desktop-app-environments/README.md | 6 + .../file-readers/plan.md | 18 + .../file-readers/spec.md | 78 +++ .../file-readers/tasks.md | 9 + .../file-readers/verification.md | 77 +++ .../screen-judge/plan.md | 16 + .../screen-judge/spec.md | 53 ++ .../screen-judge/tasks.md | 9 + .../screen-judge/verification.md | 58 +++ .../desktop/DesktopApplicationExport.ts | 35 +- .../desktop/DesktopBuildCapture.ts | 45 +- .../DesktopBuildComputers.cleanup.test.ts | 149 ++++++ .../desktop/DesktopBuildComputers.ts | 118 ++++- .../desktop/DesktopBuildExport.test.ts | 115 +++++ src/application/desktop/DesktopBuildExport.ts | 97 ++-- .../desktop/DesktopBuildWitness.test.ts | 4 +- .../desktop/DesktopBuildWitness.ts | 122 ++--- .../desktop/DesktopBuildWitnessBundle.test.ts | 50 ++ .../desktop/DesktopBuildWitnessBundle.ts | 157 ++++++ .../desktop/DesktopBuildWitnessRecord.ts | 12 +- src/application/desktop/DesktopExportFiles.ts | 13 + .../desktop/DesktopExportReaders.ts | 140 ++++++ .../desktop/DesktopRemoteTaskReaders.test.ts | 188 +++++++ .../desktop/DesktopRemoteTaskReaders.ts | 300 +++++++++++ .../DesktopRetainedContent.cleanup.test.ts | 72 +++ .../desktop/DesktopRetainedContent.ts | 32 +- .../desktop/DesktopTaskExport.test.ts | 389 +++++++++++++- src/application/desktop/DesktopTaskExport.ts | 300 +++++++++-- .../desktop/DesktopTaskReaderCleanup.ts | 133 +++++ .../desktop/DesktopTaskReaders.test.ts | 168 +++++++ src/application/desktop/DesktopTaskReaders.ts | 125 +++++ src/cli/desktopCaptureCommands.ts | 63 ++- src/contracts/desktopBuildToolContracts.ts | 6 +- src/contracts/toolEffects.test.ts | 2 + src/contracts/toolEffects.ts | 6 +- src/desktop/AgentEnvComputer.ts | 76 ++- src/desktop/BuildComputerController.ts | 23 +- src/desktop/BuildComputerLongRuns.test.ts | 326 ++++++++++++ src/desktop/BuildComputerLongRuns.ts | 293 +++++++++++ src/desktop/ComputerHomeFiles.ts | 82 +++ .../ScreenJudgeProcess.integration.test.ts | 89 ++++ src/desktop/ScreenJudgeProcess.test.ts | 325 ++++++++++++ src/desktop/ScreenJudgeProcess.ts | 248 +++++++++ src/desktop/TaskReaderProcess.ts | 42 ++ src/domain/desktop/appExport.test.ts | 11 + src/domain/desktop/appExport.ts | 16 +- .../desktop/appExportInstallLimitations.ts | 5 + src/domain/desktop/appExportPython.ts | 30 ++ src/domain/desktop/appExportPythonJson.ts | 37 +- src/domain/desktop/appTask.ts | 186 ++++++- src/domain/desktop/appTaskChecks.ts | 20 +- .../desktop/appTaskDraftFileNotes.test.ts | 10 + src/domain/desktop/appTaskDraftFileNotes.ts | 2 +- src/domain/desktop/appTaskPython.ts | 33 +- src/domain/desktop/appTaskReaders.test.ts | 331 ++++++++++++ src/domain/desktop/appTaskReaders.ts | 475 ++++++++++++++++++ src/domain/desktop/appTaskScreen.test.ts | 134 +++++ src/domain/desktop/appTaskScreen.ts | 207 ++++++++ src/domain/desktop/pythonExactJsonNumber.ts | 37 ++ src/process/OwnedWorkerRuntime.ts | 19 +- src/process/RetainedCommandCleanup.ts | 64 +++ tests/boundary/mcp/buildCaptureMcp.test.ts | 34 ++ tests/boundary/mcp/buildWitnessMcp.test.ts | 200 +++++++- .../desktopLongBuildController.test.ts | 108 ++++ .../desktopScreenJudgePython.test.ts | 225 +++++++++ .../providers/desktopTaskArrayIndex.test.ts | 18 + .../providers/desktopTaskPython.test.ts | 419 ++++++++++++++- .../providers/desktopTaskReaderRunner.test.ts | 367 ++++++++++++++ tests/fixtures/buildComputerMcp.ts | 4 +- tests/fixtures/fakeAgentEnvComputer.ts | 23 +- 75 files changed, 7815 insertions(+), 372 deletions(-) create mode 100644 docs/specs/desktop-app-environments/file-readers/plan.md create mode 100644 docs/specs/desktop-app-environments/file-readers/spec.md create mode 100644 docs/specs/desktop-app-environments/file-readers/tasks.md create mode 100644 docs/specs/desktop-app-environments/file-readers/verification.md create mode 100644 docs/specs/desktop-app-environments/screen-judge/plan.md create mode 100644 docs/specs/desktop-app-environments/screen-judge/spec.md create mode 100644 docs/specs/desktop-app-environments/screen-judge/tasks.md create mode 100644 docs/specs/desktop-app-environments/screen-judge/verification.md create mode 100644 src/application/desktop/DesktopBuildComputers.cleanup.test.ts create mode 100644 src/application/desktop/DesktopBuildExport.test.ts create mode 100644 src/application/desktop/DesktopBuildWitnessBundle.test.ts create mode 100644 src/application/desktop/DesktopBuildWitnessBundle.ts create mode 100644 src/application/desktop/DesktopExportReaders.ts create mode 100644 src/application/desktop/DesktopRemoteTaskReaders.test.ts create mode 100644 src/application/desktop/DesktopRemoteTaskReaders.ts create mode 100644 src/application/desktop/DesktopRetainedContent.cleanup.test.ts create mode 100644 src/application/desktop/DesktopTaskReaderCleanup.ts create mode 100644 src/application/desktop/DesktopTaskReaders.test.ts create mode 100644 src/application/desktop/DesktopTaskReaders.ts create mode 100644 src/desktop/BuildComputerLongRuns.test.ts create mode 100644 src/desktop/BuildComputerLongRuns.ts create mode 100644 src/desktop/ComputerHomeFiles.ts create mode 100644 src/desktop/ScreenJudgeProcess.integration.test.ts create mode 100644 src/desktop/ScreenJudgeProcess.test.ts create mode 100644 src/desktop/ScreenJudgeProcess.ts create mode 100644 src/desktop/TaskReaderProcess.ts create mode 100644 src/domain/desktop/appTaskReaders.test.ts create mode 100644 src/domain/desktop/appTaskReaders.ts create mode 100644 src/domain/desktop/appTaskScreen.test.ts create mode 100644 src/domain/desktop/appTaskScreen.ts create mode 100644 src/domain/desktop/pythonExactJsonNumber.ts create mode 100644 src/process/RetainedCommandCleanup.ts create mode 100644 tests/boundary/providers/desktopLongBuildController.test.ts create mode 100644 tests/boundary/providers/desktopScreenJudgePython.test.ts create mode 100644 tests/boundary/providers/desktopTaskReaderRunner.test.ts diff --git a/.agents/skills/reverse-engineer-anything/references/desktop-build.md b/.agents/skills/reverse-engineer-anything/references/desktop-build.md index 0cee836a3b..683dcadc17 100644 --- a/.agents/skills/reverse-engineer-anything/references/desktop-build.md +++ b/.agents/skills/reverse-engineer-anything/references/desktop-build.md @@ -14,7 +14,9 @@ stop. For the whole build: - Drive the app only with `act_on_build_computer`. Never script it, run - commands in the computer or write its files another way. + commands in the computer or write its files another way. The one command + you write for the computer is a task's reader, which REA runs to check a + file; it never makes the seed or the edit. - Ask the upfront questions once, then work on your own. Pause only when blocked. - The seed's content is synthetic. Never put real personal data in it. @@ -353,7 +355,9 @@ the first source that has the app: "Compress diagram files" is unticked, which REA reads as plain XML. A format REA reads as binary, such as one compressed with xz, has no view: save in a format the app offers that REA reads, then reopen the file once and check that a plain Ctrl+S - keeps that format. + keeps that format. When the task is about the app's own binary + format, such as GIMP's `.xcf` or Blender's `.blend`, keep it and check + it through a reader (see Read a binary file with a reader). 4. Only once the seed is saved, `mark_build_checkpoint` with name `seed`. 5. Make the change the task asks for, as a user would, and save it. 6. Only once the change is saved, `mark_build_checkpoint` with name `end`. @@ -365,7 +369,9 @@ the first source that has the app: draft command names its own cause to fix. When `draft.files` is empty, `draft.notes` says why: - `no_change`: nothing was saved between `seed` and `end`. Capture again - on a new build computer, and save before marking `end`. + on a new build computer, and save before marking `end`. When the app + keeps the task's state only on screen, such as a calculator's result, + judge the screen instead (see Judge the screen). - `unowned_changes` with `no_document`: the app saved your document where REA did not count it as the app's. Its `files` list what changed, with their roles; the document is among them. Close the computer and @@ -374,7 +380,7 @@ the first source that has the app: - `uncheckable_changes`: the app saved the change in dconf, as gedit does its preferences, and no check reads dconf. Saving again does not help: choose a change the app saves in a file, and capture again on a new - build computer. + build computer, or judge the screen (see Judge the screen). - `unreadable_changes`: a dconf database changed but REA could not read it; `error` says why. This is not evidence the app saved there: report the error rather than changing the task. @@ -394,7 +400,9 @@ the first source that has the app: - `long_text`: the element's text is too long to check whole; check a phrase in it with `text_contains`. - `cut_view` or `unreadable_view`: REA's view holds only part of the - file, or none a check can read, such as a binary format (step 3). + file, or none a check can read, such as a binary format (step 3). For + a binary file, read it with a reader (see Read a binary file with a + reader). ## Write the task @@ -508,6 +516,124 @@ Write `{"name", "prompt", "checks"}` from the draft and what you did. The kinds and pointer rules are in [the desktop tasks section](environments.md#pick-the-checks). +### Read a binary file with a reader + +When the draft notes `unreadable_view` on a binary document, add a reader to +the task: the app's own headless command, which REA runs in the computer on a +private copy of the file, and whose output a check reads instead of the file. +Write it in the app's scripting mode, print each fact on a line starting +with `REA ` (with the space), and name the copy `{file}`: + +- GIMP `.xcf`, layer names, as `lines`: + + ```json + { + "id": "layers", + "file": "Documents/poster.xcf", + "argv": ["gimp-console-2.10", "-i", "-d", "-f", "-b", "-"], + "stdin": "(let ((image (car (gimp-file-load RUN-NONINTERACTIVE \"{file}\" \"{file}\")))) (for-each (lambda (layer) (display \"\\nREA layer \") (display (car (gimp-item-get-name layer)))) (vector->list (cadr (gimp-image-get-layers image)))) (newline))\n(gimp-quit 0)\n", + "output": "lines" + } + ``` + + checked with `{"kind": "lines_present", "file": "Documents/poster.xcf", "reader": "layers", "lines": ["layer Summer Sale"]}`. + Script-Fu's prompt `ts> ` starts a line, so print a newline before each + marked line, as `\nREA` does. A text layer's text is + `(car (gimp-text-layer-get-text layer))`. + +- Blender `.blend`, objects with their type and location, as `json`: + + ```json + { + "id": "objects", + "file": "Documents/scene.blend", + "argv": [ + "blender", + "-b", + "--factory-startup", + "{file}", + "--python-expr", + "import bpy, json; print('REA ' + json.dumps({o.name: {'type': o.type, 'location': [round(v, 4) for v in o.location]} for o in bpy.data.objects}, sort_keys=True))" + ], + "output": "json" + } + ``` + + checked with `{"kind": "value_equals", "file": "Documents/scene.blend", "reader": "objects", "pointer": "/Crate/location/0", "value": 2, "tolerance": 0.001}`. + Print every value a check compares as a number or string a check can + name, rounded where the app stores floats. + Blender 4 shows its Quick Setup splash at every launch until its + preferences are saved: with `.config/blender` in `state_paths`, confirm + Quick Setup with Continue, save the preferences (Edit → Preferences, then + Save Preferences in the menu at its bottom left), and read a probe + checkpoint's `changed_files` for `.config/blender/4.0/config/userpref.blend` + before `seed`, so solves start on the scene. Blender often takes the first click on a + menu or a dialog's button only to focus it: click again. + +- A format LibreOffice opens but REA cannot read, such as a legacy `.xls`, + converted with `--convert-to`: + + ```json + { + "id": "cells", + "file": "Documents/budget.xls", + "argv": [ + "sh", + "-c", + "soffice --headless --convert-to csv --outdir out \"$1\" >/dev/null && sed 's/^/REA /' out/*.csv", + "sh", + "{file}" + ], + "output": "lines", + "timeout_s": 120 + } + ``` + + Pass `{file}` to `sh -c` as an argument, as here, rather than inside the + script. + +The reader runs without the display, in a fresh home with the app's default +settings, so use the app's headless or batch mode; set `display: true` only +for an app that has none. It must exit 0 within `timeout_s` (60 s by +default, at most 300) and print at most 1 MiB. Pick facts the change sets and +the near miss gets wrong, such as a name or a number, not the app's whole +output. + +Give the task the reader in `readers`, and each check through it `reader` +and the reader's `file`. `json` readers serve `value_equals`, `value_absent` +and `member_count`; `lines` readers `lines_present`, `lines_absent` and +`line_count`. The export runs the reader in the build computer on the seed +and the end: when a check through it fails on the end, the refusal shows +what the reader printed, and a reader that fails shows its error and the end +of its stderr. Fix the reader or the check from that and export again. + +### Judge the screen + +Only when no file shows the change, as when the draft notes `no_change` +because the app keeps its state on screen, write the task's checks as +screen judges: `{"id": "shown", "kind": "screen_judge", "question": "Does the +calculator's display show 8.568?"}`. A model, the judge, answers the question +on a screenshot three times, and the check passes when most answers are yes. + +- REA keeps a screenshot at each checkpoint. Mark `seed` with the screen + showing the state before the task, where the answer is no, and `end` with + the change showing, where it is yes: the bundle ships both as the judge's + examples. +- Ask about what the change makes visible, with exact values, so the near + miss is a no: a result, a label, a selected item. Avoid what a window + moved or a cursor blinking can change. +- The prompt asks to leave the result showing, since the judge sees only + the screen when the agent stops. +- A task with screen judges holds only screen judges and no readers. The + export refuses one when a file shows the change, and checks none of its + questions: the witness does, running the bundle's own judge with the + runner's model endpoint (`LITELLM_BASE_URL` and `LITELLM_API_KEY`, or + `[model]` in `.agentenv/config.toml`) on a fresh screenshot at each stage. + Without an endpoint the witness's check fails: say what to set and stop. +- The judge's model is `[model.roles] judge`, else `[model] default`, else + `anthropic/claude-sonnet-5-5`. `agent-env run --judge-model` does not + reach it. Each grade costs about $0.02. + ## Export `export_build_environment` with `task` and `directory`, the absolute path of @@ -623,7 +749,8 @@ Report: - the bundle path and the environment name; - the task's name and prompt; -- each check, and why it proves the edit and fails the near miss; +- each check, and why it proves the edit and fails the near miss, and each + reader's command; - each witness stage with its result, the verdict, and the paths of `witness.json` and the recording; - the seed's contents: `seed_files` and what each holds; diff --git a/.agents/skills/reverse-engineer-anything/references/environments.md b/.agents/skills/reverse-engineer-anything/references/environments.md index c503bbdc6d..914c7eadfa 100644 --- a/.agents/skills/reverse-engineer-anything/references/environments.md +++ b/.agents/skills/reverse-engineer-anything/references/environments.md @@ -367,7 +367,10 @@ as `"5.0000000000000000"`, with a number. Check what the user wrote in a document with `text_equals` on its paragraph element, not `value_equals` on a text node: apps such as AbiWord and Writer split a paragraph into runs as they save, so its first text node holds only part of it; use `text_contains` for a -value within a sentence, such as a time. The +value within a sentence, such as a time. A binary document, such as GIMP's +`.xcf`, has no view: give the task a `readers` entry, the app's own headless +command printing `REA ` lines, and a check naming it in `reader`, as +[desktop builds](desktop-build.md#read-a-binary-file-with-a-reader) shows. The [desktop application tasks](https://github.com/morluto/rea/blob/main/docs/desktop-application-tasks.md) guide has the exact rules. diff --git a/bridge/agentenv/capture.py b/bridge/agentenv/capture.py index 67679331e4..732efe5b70 100644 --- a/bridge/agentenv/capture.py +++ b/bridge/agentenv/capture.py @@ -290,6 +290,187 @@ def run(arguments: dict) -> int: "stderr": done.stderr[-TAIL_BYTES:].decode("utf-8", "replace")}) +# The controller is sent with python -c, so a detached supervisor needs its +# own source. It alone holds the child process handle and signals its group. +LONG_RUN = r''' +import json, os, signal, subprocess, sys, threading, time +job = sys.argv[1] +result = {} +child = None +drainer = None +stderr_tail, stderr_errors = bytearray(), [] + +def drain(stream): + try: + while chunk := stream.read1(65536): + stderr_tail.extend(chunk) + del stderr_tail[:-4096] + except OSError as error: + stderr_errors.append(str(error)) + finally: + try: + stream.close() + except OSError as error: + stderr_errors.append(str(error)) + +def timed_out_marker(): + try: + open(os.path.join(job, "timed_out"), "x").close() + except FileExistsError: + pass + +try: + request = json.loads("".join(sys.argv[2:])) + if os.path.exists(os.path.join(job, "cancel")): + result = {"absent": True} + else: + with open(request["stdout"], "wb") as out: + child = subprocess.Popen(request["argv"], cwd=request["home"], stdin=subprocess.DEVNULL, + stdout=out, stderr=subprocess.PIPE, start_new_session=True) + drainer = threading.Thread(target=drain, args=(child.stderr,), daemon=True) + drainer.start() + deadline = time.monotonic() + request["timeout_s"] + interrupted = False + timed_out = False + while child.poll() is None: + timed_out = timed_out or time.monotonic() >= deadline + if not interrupted and (timed_out or os.path.exists(os.path.join(job, "cancel"))): + interrupted = True + if timed_out: + timed_out_marker() + try: + os.killpg(child.pid, signal.SIGINT) + except OSError: + pass + time.sleep(0.2) + # A descendant can still hold stderr after the leader exits. Do + # not publish completion until that stream has actually closed. + while drainer.is_alive(): + drainer.join(0.2) + if time.monotonic() >= deadline: + timed_out = True + timed_out_marker() + result = {"exit_code": child.returncode, "stdout_bytes": os.path.getsize(request["stdout"]), + "timed_out": timed_out, "cancelled": interrupted and not timed_out, + "stderr": bytes(stderr_tail).decode("utf-8", "replace")} + if stderr_errors: + result["error"] = f"The REA command's stderr could not be read: {stderr_errors[0]}" +except Exception as error: + result["error"] = f"The REA command supervisor failed: {error}" + # Keep the child handle until it exits, including when cancellation arrives + # after an unexpected supervisor error. + while child is not None and child.poll() is None: + if os.path.exists(os.path.join(job, "cancel")): + try: + os.killpg(child.pid, signal.SIGINT) + except OSError: + pass + time.sleep(0.2) + if drainer is not None and drainer.ident is not None: + drainer.join() + if child is not None: + result.setdefault("exit_code", child.returncode) + result.setdefault("stderr", bytes(stderr_tail).decode("utf-8", "replace")) + try: + result.setdefault("stdout_bytes", os.path.getsize(request["stdout"])) + except OSError: + pass +if child is None or child.poll() is not None: + temporary = os.path.join(job, "result.tmp") + with open(temporary, "w", encoding="utf-8") as out: + json.dump(result, out) + os.replace(temporary, os.path.join(job, "result.json")) +''' + + +def run_begin(arguments: dict) -> int: + """Start one long REA command in its own capture-owned job folder.""" + rea = shutil.which("rea") + if rea is None: + return _say({"error": "REA is not installed in this computer"}) + job = os.path.abspath(arguments["job"]) + os.makedirs(os.path.dirname(job), exist_ok=True) + try: + os.mkdir(job, 0o700) + except FileExistsError: + return _say({"refused": "the command job folder is already used"}) + process = None + try: + with open(os.path.join(job, "owner"), "x", encoding="ascii") as out: + out.write(arguments["token"]) + for path, text in arguments.get("files", {}).items(): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + handle.write(text) + os.makedirs(os.path.dirname(arguments["stdout"]), exist_ok=True) + request = json.dumps({"argv": [rea, *arguments["argv"]], "stdout": arguments["stdout"], + "home": os.path.expanduser("~"), "timeout_s": arguments["timeout_s"]}) + chunks = [request[i:i + 60000] for i in range(0, len(request), 60000)] + with open(os.path.join(job, "supervisor.log"), "wb") as log: + process = subprocess.Popen([sys.executable, "-I", "-c", LONG_RUN, job, *chunks], cwd=os.path.expanduser("~"), + env=_environment(os.path.abspath(arguments["work"])), stdin=subprocess.DEVNULL, + stdout=log, stderr=log, start_new_session=True) + except (OSError, ValueError) as error: + stage = "did not start" if process is None else "started but its setup did not finish" + result = {"error": f"The REA command supervisor {stage}: {error}"} + if process is None: + temporary = os.path.join(job, "result.tmp") + with open(temporary, "w", encoding="utf-8") as out: + json.dump(result, out) + os.replace(temporary, os.path.join(job, "result.json")) + return _say(result) + return _say({"started": True}) + + +def _run_result(job: str): + try: + with open(os.path.join(job, "result.json"), encoding="utf-8") as source: + return json.load(source) + except FileNotFoundError: + return None + + +def _owns_run(job: str, token: str) -> bool: + try: + with open(os.path.join(job, "owner"), encoding="ascii") as owner: + return owner.read() == token + except OSError: + return False + + +def run_poll(arguments: dict) -> int: + job = os.path.abspath(arguments["job"]) + if not _owns_run(job, arguments["token"]): + return _say({"error": "the command job ownership is not confirmed"}) + deadline = time.monotonic() + arguments.get("timeout_s", 30) + while (result := _run_result(job)) is None and time.monotonic() < deadline: + time.sleep(POLL_S) + if result is None: + pending = {"pending": True, "timed_out": os.path.exists(os.path.join(job, "timed_out"))} + progress = _tail(os.path.join(job, "supervisor.log")) + return _say({**pending, **({"progress": progress} if progress else {})}) + return _say({"finished": True, **result}) + + +def run_cancel(arguments: dict) -> int: + job = os.path.abspath(arguments["job"]) + os.makedirs(os.path.dirname(job), exist_ok=True) + try: + os.mkdir(job, 0o700) + except FileExistsError: + pass + else: + # A late begin cannot launch into a folder cancellation claimed. + with open(os.path.join(job, "owner"), "x", encoding="ascii") as owner: + owner.write(arguments["token"]) + return _say({"absent": True}) + if not _owns_run(job, arguments["token"]): + return _say({"error": "the command job ownership is not confirmed"}) + with open(os.path.join(job, "cancel"), "a", encoding="utf-8"): + pass + return run_poll(arguments) + + def listing(arguments: dict) -> int: """Every regular file under a folder, by its path relative to the home, with its size and SHA-256. @@ -328,7 +509,8 @@ def listing(arguments: dict) -> int: return _say({"listing_bytes": os.path.getsize(output)}) -COMMANDS = {"start": start, "mark": mark, "stop": stop, "run": run, "list": listing} +COMMANDS = {"start": start, "mark": mark, "stop": stop, "run": run, "list": listing, + "run-begin": run_begin, "run-poll": run_poll, "run-cancel": run_cancel} if __name__ == "__main__": sys.exit(COMMANDS[sys.argv[1]](json.loads("".join(sys.argv[2:])))) diff --git a/bridge/agentenv/test_capture.py b/bridge/agentenv/test_capture.py index 8f190edfbc..7fed0437e0 100644 --- a/bridge/agentenv/test_capture.py +++ b/bridge/agentenv/test_capture.py @@ -1,12 +1,17 @@ """Exercise the capture controller's filesystem inventory boundary.""" import json +import io +from contextlib import redirect_stdout import os from pathlib import Path import subprocess import sys import tempfile import unittest +from unittest.mock import patch + +from bridge.agentenv import capture CONTROLLER = Path(__file__).with_name("capture.py") @@ -60,5 +65,168 @@ class CaptureListingTest(unittest.TestCase): self.assertFalse(output.exists()) +@unittest.skipIf(sys.platform == "win32", "desktop build computers use Linux") +class LongRunTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.bin = self.root / "bin" + self.bin.mkdir() + rea = self.bin / "rea" + rea.write_text( + f"#!{sys.executable}\n" + "import os, signal, subprocess, sys, time\n" + "signal.signal(signal.SIGINT, lambda *_: sys.exit(130))\n" + "if len(sys.argv) > 1 and sys.argv[1] == 'wait':\n" + " open(sys.argv[2], 'w').close()\n" + " time.sleep(30)\n" + "if len(sys.argv) > 1 and sys.argv[1] == 'ignore':\n" + " signal.signal(signal.SIGINT, signal.SIG_IGN)\n" + " open(sys.argv[2], 'w').close()\n" + " while not os.path.exists(sys.argv[3]): time.sleep(0.02)\n" + "if len(sys.argv) > 1 and sys.argv[1] == 'stderr':\n" + " sys.stderr.write('x' * 100000 + 'TAIL')\n" + "if len(sys.argv) > 1 and sys.argv[1] == 'hold':\n" + " subprocess.Popen([sys.executable, '-c', 'import os,sys,time\\nwhile not os.path.exists(sys.argv[1]): time.sleep(0.02)', sys.argv[2]], stdout=subprocess.DEVNULL, stderr=sys.stderr, start_new_session=True)\n" + " sys.exit(0)\n" + "print(os.environ.get('AGENTENV_CONTROL_TOKEN', 'no token'))\n" + ) + rea.chmod(0o755) + self.environment = {**os.environ, "HOME": str(self.root), "PATH": f"{self.bin}{os.pathsep}{os.environ.get('PATH', '')}", + "AGENTENV_CONTROL_TOKEN": "not-for-the-child"} + self.job = self.root / "job" + self.output = self.root / "out.json" + + def command(self, name, **values): + result = subprocess.run([sys.executable, str(CONTROLLER), name, json.dumps(values)], + env=self.environment, text=True, capture_output=True, timeout=10, check=True) + return json.loads(result.stdout) + + def begin(self, argv, timeout=60): + return self.command("run-begin", job=str(self.job), work=str(self.root), stdout=str(self.output), + token="owner-token", argv=argv, files={}, timeout_s=timeout) + + def test_runs_and_reports_its_output(self): + self.assertEqual(self.begin([]), {"started": True}) + result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5) + self.assertEqual(result["exit_code"], 0) + self.assertEqual(self.output.read_text().strip(), "no token") + + def test_passes_large_command_without_persisting_its_arguments(self): + self.assertEqual(self.begin(["done", *("x" * 4000 for _ in range(20))]), {"started": True}) + result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5) + self.assertEqual(result["exit_code"], 0) + self.assertFalse((self.job / "request.json").exists()) + + def test_keeps_only_a_bounded_stderr_tail(self): + self.assertEqual(self.begin(["stderr"]), {"started": True}) + result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5) + self.assertEqual(len(result["stderr"]), 4096) + self.assertTrue(result["stderr"].endswith("TAIL")) + + def test_cancels_a_running_child_and_confirms_exit(self): + ready = self.root / "ready" + self.assertEqual(self.begin(["wait", str(ready)]), {"started": True}) + import time + deadline = time.monotonic() + 5 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(0.02) + self.assertTrue(ready.exists()) + result = self.command("run-cancel", job=str(self.job), token="owner-token", timeout_s=5) + self.assertTrue(result["cancelled"]) + self.assertEqual(result["exit_code"], 130) + + def test_cancel_before_late_start_claims_the_job(self): + self.assertEqual(self.command("run-cancel", job=str(self.job), token="owner-token", timeout_s=1), {"absent": True}) + self.assertIn("already used", self.begin([])["refused"]) + self.assertFalse(self.output.exists()) + + def test_cancel_refuses_a_folder_without_the_matching_owner(self): + self.job.mkdir() + (self.job / "owner").write_text("someone-else") + result = self.command("run-cancel", job=str(self.job), token="owner-token", timeout_s=1) + self.assertIn("ownership is not confirmed", result["error"]) + self.assertFalse((self.job / "cancel").exists()) + + def test_reports_supervisor_output_if_completion_is_unconfirmed(self): + self.job.mkdir() + (self.job / "owner").write_text("owner-token") + (self.job / "supervisor.log").write_text("supervisor failed") + result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=0) + self.assertEqual(result["progress"], "supervisor failed") + self.assertTrue(result["pending"]) + + def test_reports_a_timed_out_child_that_has_not_stopped(self): + ready, release = self.root / "ready", self.root / "release" + self.assertEqual(self.begin(["ignore", str(ready), str(release)], timeout=3), {"started": True}) + try: + result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=4) + self.assertTrue(result["pending"]) + self.assertTrue(result["timed_out"]) + self.assertTrue(ready.exists()) + finally: + release.touch() + self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5) + + def test_waits_for_an_inherited_stderr_pipe_after_the_child_exits(self): + release = self.root / "release" + self.assertEqual(self.begin(["hold", str(release)], timeout=3), {"started": True}) + try: + result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=4) + self.assertTrue(result["pending"]) + self.assertTrue(result["timed_out"]) + finally: + release.touch() + result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5) + self.assertTrue(result["finished"]) + + def test_failure_closing_supervisor_log_after_launch_does_not_claim_exit(self): + ready = self.root / "ready" + original_open = open + original_popen = subprocess.Popen + supervisors = [] + + class FailingClose: + def __init__(self, file): + self.file = file + + def __enter__(self): + return self.file + + def __exit__(self, *args): + self.file.close() + raise OSError("log close failed") + + def opening(path, *args, **kwargs): + file = original_open(path, *args, **kwargs) + return FailingClose(file) if str(path).endswith("supervisor.log") else file + + def spawning(*args, **kwargs): + process = original_popen(*args, **kwargs) + supervisors.append(process) + def cleanup(): + if process.poll() is None: + (self.job / "cancel").touch() + process.wait(timeout=5) + self.addCleanup(cleanup) + return process + + arguments = {"job": str(self.job), "work": str(self.root), "stdout": str(self.output), + "token": "owner-token", "argv": ["wait", str(ready)], "files": {}, "timeout_s": 60} + with patch.dict(os.environ, self.environment, clear=True), patch("builtins.open", side_effect=opening), patch.object(capture.subprocess, "Popen", side_effect=spawning), redirect_stdout(io.StringIO()) as output: + capture.run_begin(arguments) + self.assertIn("started but its setup did not finish", json.loads(output.getvalue())["error"]) + import time + deadline = time.monotonic() + 5 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(0.02) + self.assertTrue(ready.exists()) + self.assertFalse((self.job / "result.json").exists()) + result = self.command("run-cancel", job=str(self.job), token="owner-token", timeout_s=5) + self.assertTrue(result["cancelled"]) + supervisors[0].wait(timeout=5) + + if __name__ == "__main__": unittest.main() diff --git a/docs/desktop-build-witness.md b/docs/desktop-build-witness.md index 4a75dd8891..09b6bd1655 100644 --- a/docs/desktop-build-witness.md +++ b/docs/desktop-build-witness.md @@ -222,7 +222,12 @@ Then, with the task exactly as exported: `check_build_task` reads the app's `data/get`, takes the views it returns beside the files and evaluates each check with REA's evaluator, the one the export validates tasks with and the exported verifier mirrors, one result per -check. Like the verifier, it grades only once the app's files settle, since an +check. Once the files settle it runs the task's +[readers](desktop-application-tasks.md#readers) in the computer through +`computer-exec/v1`, as the verifier does, and a check through a reader reads +that reader's view. A task of [screen judges](desktop-application-tasks.md#screen-judges) +reads no files: REA takes a screenshot and runs the bundle's own judge on it +with the agentenv runner's Python and model endpoint. Like the verifier, it grades only once the app's files settle, since an app such as Gnote saves a few seconds after the last key. It reads `data/get` every 0.5 s and grades once every file's bytes have held for 3 s, counted from the read that returned them. 15 s after the first read began, it grades diff --git a/docs/specs/desktop-app-environments/README.md b/docs/specs/desktop-app-environments/README.md index 3a638c2d49..24cfe49230 100644 --- a/docs/specs/desktop-app-environments/README.md +++ b/docs/specs/desktop-app-environments/README.md @@ -61,6 +61,12 @@ The planned sequence was one PR per step: offers, from their project's AppImage or tarball pinned by URL and SHA-256, unpacked under `/opt/rea-apps` by the build computer and the export alike, starting with FreeCAD. +16. [Readers](file-readers/spec.md): checks of a file REA reads as binary, + such as GIMP's `.xcf` or Blender's `.blend`, through the app's own + headless command, run in the computer on a private copy of the file. +17. [Screen judges](screen-judge/spec.md): checks of state no file shows, + such as a calculator's result, by a model judging a screenshot against + the build's labelled seed and end screenshots. Version 1 covers Linux with X11, applications whose state lives in files or SQLite under the home directory, and real applications from public packages. diff --git a/docs/specs/desktop-app-environments/file-readers/plan.md b/docs/specs/desktop-app-environments/file-readers/plan.md new file mode 100644 index 0000000000..2ab4b82e3b --- /dev/null +++ b/docs/specs/desktop-app-environments/file-readers/plan.md @@ -0,0 +1,18 @@ +# Plan + +1. Schema: `readers` and the `reader` field in `appTask.ts`, with + `readersProblem` for ids, files, `{file}`, kinds by output and unread + readers; `DesktopTask.readers` only when the file has some. +2. Runner: `PYTHON_READER_RUNNER`, `readerCommand`, `readerArguments` and + `readerAnswerView` in `appTaskReaders.ts`. +3. Evaluators: `evaluateTaskChecks` takes the readers' views by id, and the + verifier's `evaluate_with_readers` does, leaving `evaluate` as it was. +4. Verifier: `appTaskPython.ts` renders `READER_RUNNER`, `READERS`, + `_reader_views` and `evaluate_with_readers` only for a task with readers, + so a task without renders as before. +5. Application: `readTaskReaderViews` and `readerViewsOfBytes` in + `DesktopTaskReaders.ts`; `runReaderHere` in `src/desktop/TaskReaderProcess.ts`; + the witness runs readers through `AgentEnvComputer.exec`; the export runs + them on `shippedSeedBytes` and the end's retained content. +6. Docs: the tasks reference, the witness page, the build skill's recipes for + GIMP, Blender and LibreOffice, and the draft note. diff --git a/docs/specs/desktop-app-environments/file-readers/spec.md b/docs/specs/desktop-app-environments/file-readers/spec.md new file mode 100644 index 0000000000..751bb80bd8 --- /dev/null +++ b/docs/specs/desktop-app-environments/file-readers/spec.md @@ -0,0 +1,78 @@ +# Readers: checking files REA cannot read + +## Outcome + +A task can check a file REA reads as binary, such as GIMP's `.xcf` or +Blender's `.blend`, through a reader: the app's own headless command, run in +the computer on a private copy of the file, whose marked output becomes a +view that the existing check kinds read. The export validates such checks on +the seed and the end, the witness checks them live, and the exported verifier +grades them in the solving agent's computer, all through one runner. + +## Requirements + +- Task file: an optional `readers` array of 1 to 8 + `{id, file, argv, stdin?, output, timeout_s?, display?}`. `id` is lowercase + letters, digits and hyphens and unique; `file` is a path under the home; + `argv` holds 1 to 64 non-empty strings; `stdin` at most 64 KiB; `output` + is `json` or `lines`; `timeout_s` is 1 to 300, 60 when omitted; `display` + is false when omitted. The argv or the stdin names the file as `{file}`. +- Checks: `lines_present`, `lines_absent` and `line_count` take an optional + `reader` naming a `lines` reader; `value_equals`, `value_absent` and + `member_count` one naming a `json` reader. The check's `file` is the + reader's. Other kinds take none. Every reader is read by a check. The parser + refuses each breach with the check or reader it names. +- Runner: one Python program, embedded in REA and in each verifier that needs + it, run as `python3 -I -c ` in the computer's session. + It answers with one JSON line: `{"view"}`, `{"absent": true}` when the file + is not there, or `{"error"}`. It copies the file into a new temporary folder, + runs the command there with `{file}` replaced by the copy's path in argv and + stdin, `HOME` and the four `XDG_` folders in that folder, without + `DISPLAY`, `WAYLAND_DISPLAY` or `XAUTHORITY` unless `display` is true, and + never with `DBUS_SESSION_BUS_ADDRESS`, `SESSION_MANAGER`, + `XDG_RUNTIME_DIR` or any `AGENTENV_` variable. Past `timeout_s` it kills + the command's whole session. It deletes the folder afterwards. +- View: each output line starting with `REA ` is a line of the view, without + the marker. A `lines` reader's view is a text view (`lines`, `line_count`, + `truncated` past 5,000 lines); a `json` reader's is the marked lines joined + by line feeds, read as one JSON value, refusing NaN, infinities and + integers past 2^53. A command that cannot start, exits other than 0, runs + past its timeout, prints more than 1 MiB, prints no marked line or prints a + `json` view that does not parse gives an error view, with the last 1 KiB of + its stderr; so does an answer past 960 KiB. +- Evaluators: REA's and the verifier's read a check with `reader` from the + reader's view, by id, instead of the file's. A missing reader view reads as + an absent file; an error view fails the check as undecided, worded as an + unreadable file's, in both. +- Export: the export runs each reader on this host, in the build computer, + on the file's bytes at the seed as the bundle ships them and at the end + checkpoint as retained; a file not retained at the end gives an error view. + It refuses a reader of a file outside the owned paths or naming the + captured home or the host's identity. A check through a reader that fails + on the end adds what the reader printed to the refusal. +- Witness: once the app's files settle, `check_build_task` runs the readers + through the computer's `computer-exec/v1`, and a runner that does not + answer fails the check call as a failed read of the app's state. +- Verifier: a task with readers holds the runner once and each reader's + arguments; after its files settle it runs each through `computer-exec/v1` + under the run's grant, with a request id of its own and the reader's + timeout plus 30 s, and raises when the computer offers no such route or + the runner does not answer, so the run fails rather than scoring the agent. + It grades through `evaluate_with_readers`, which only such a verifier holds: + a task without readers renders the verifier byte for byte as before, so a + bundle exported earlier witnesses as the same task. +- Cleanup: the witness and verifier own each remote reader's temporary copy + until the command finishes and its folder is removed. They report failed + cleanup with its folder and retain it for retry; an interrupted command is + not assumed to have stopped. The witness retries on check or close, and + the verifier on a later call in the same process. +- Draft: a file REA reads as binary is noted as checkable through a reader. + +## Non-goals + +- Drafting checks from a reader's output: the building agent writes the + reader and its checks; the export's refusals show what it read. +- XML output from a reader, and text checks through one. +- Shipping readers maintained by REA for known formats; the skill gives + recipes. +- A screen judge for state no file holds; that is the next step. diff --git a/docs/specs/desktop-app-environments/file-readers/tasks.md b/docs/specs/desktop-app-environments/file-readers/tasks.md new file mode 100644 index 0000000000..b870a14968 --- /dev/null +++ b/docs/specs/desktop-app-environments/file-readers/tasks.md @@ -0,0 +1,9 @@ +# Tasks + +- [x] Record the outcome, requirements and non-goals. +- [x] Add readers to the task schema and the parser. +- [x] Write the runner and read its answers. +- [x] Read checks through readers in both evaluators and the verifier. +- [x] Run readers in the export, on the seed and the end, and in the witness. +- [x] Document readers and give the build skill recipes. +- [x] Build GIMP and Blender environments with readers on an arm64 host, witness them and run solves. diff --git a/docs/specs/desktop-app-environments/file-readers/verification.md b/docs/specs/desktop-app-environments/file-readers/verification.md new file mode 100644 index 0000000000..a7f22239a7 --- /dev/null +++ b/docs/specs/desktop-app-environments/file-readers/verification.md @@ -0,0 +1,77 @@ +# Verification + +Run on 2026-11-06 on top of the integration of the desktop build stack, on +macOS with Node v22.23.1 and Python 3, and on an arm64 Ubuntu 24.04 build +host with agentenv-framework 0.9.1431. + +## Tests + +- `src/domain/desktop/appTaskReaders.test.ts`: the parser keeps readers and + refuses an undefined reader, a check whose file is not its reader's, a + reader naming no `{file}`, repeated ids, an unread reader, a reader file + with a leading slash, a kind its output does not serve and a text check + with a reader; the evaluator reads a check through its reader rather than + the file's binary view, fails it undecided with the reader's error, and + as decided on an absent file; the runner's command and answers. +- `tests/boundary/providers/desktopTaskReaderRunner.test.ts`: the runner, + run with `python3` as computer-exec runs it, gives the reader a private + copy in a fresh home without `DISPLAY`, the session bus or the control + token, keeps the display only when asked, reads a path in place of the + home's file, keeps only marked lines, answers absent without running the + reader, reports a non-zero exit, no marked line, a view that is not JSON, + an inexact number, more than 1 MiB and a command that cannot start, and + kills a reader and what it started past its timeout. +- `tests/boundary/providers/desktopTaskPython.test.ts`: the exported + verifier runs each reader through `computer-exec/v1` with the runner's + command, a fresh request id and the reader's timeout plus 30 s, grades + every reader check as REA's evaluator does, with the same descriptions, and + raises when the computer offers no `computer-exec/v1` or the runner does + not answer. +- `src/application/desktop/DesktopTaskExport.test.ts`: the export runs a + reader on the seed as shipped and on the end, keeps the task and ships the + reader in the verifier and the README; it refuses a check that fails on + the end with what the reader printed, a reader that exits non-zero with + its error, a task that already passes on the seed and a reader outside + the owned paths. +- `tests/boundary/mcp/buildWitnessMcp.test.ts`: a witness of a task whose + check reads a binary file through a reader, run by the fake computer's + computer-exec, holds at all four stages and returns no control token. +- `src/domain/desktop/appTaskDraftFileNotes.test.ts`: a binary file's draft + note points at readers. +- `src/domain/desktop/appTaskReaders.test.ts` also pins the SHA-256 of a + reader-less task's verifier as the tree before readers renders it, + `91821c46…`; the rendering with `evaluate`'s reader argument failed it. + +Each failed on the unchanged code: the reader module did not exist, the +schema refused `reader` and `readers`, and the note named no reader. + +## Real apps + +On the arm64 host, each through `start_build_computer`, the app's own +interface, `start_build_capture`, `seed` and `end` checkpoints, +`stop_build_capture` (whose draft noted the file as `unreadable_view` and +pointed at readers), `export_build_environment` and a witness in a fresh +computer of the bundle: + +| App | Document | Task | Reader | Export | Witness | +| ------------- | ----------------------- | ------------------------------------- | --------------------------------------- | --------------------------------- | ---------------------------- | +| GIMP 2.10.36 | `Documents/poster.xcf` | rename the Headline layer Summer Sale | Script-Fu through `gimp-console-2.10` | kept; 2 of 3 checks fail on seed | held; each stage about 5.6 s | +| Blender 4.0.2 | `Documents/scene.blend` | move the Crate to x = 2 m | `blender -b --factory-startup` with bpy | kept; 1 of 3 checks fails on seed | held; each stage about 4.4 s | + +A stage's time is the 3 s the app's files must hold plus the reader: about +2.5 s for GIMP and 1.4 s for Blender. The near misses were Summer Sail and +x = 2.5 m. + +Then three solves of each bundle with Claude Code on Claude Sonnet 5.5, +through `agent-env run`, each graded by the exported verifier, which ran the +reader through the computer's `computer-exec/v1`: + +| Task | Passed | Verifier step | Cost | +| ----------------------- | ------ | ------------- | ----- | +| `rename-headline-layer` | 3 of 3 | 5.6 to 5.7 s | $1.94 | +| `move-crate` | 3 of 3 | 4.5 s | $2.31 | + +Blender 4.0.2 writes its preferences only when they are saved, so its seed +held no `userpref.blend` and every boot showed the Quick Setup splash, which +the witness dismissed with Continue; the solves passed with it there. The +build skill now saves Blender's preferences before `seed`. diff --git a/docs/specs/desktop-app-environments/screen-judge/plan.md b/docs/specs/desktop-app-environments/screen-judge/plan.md new file mode 100644 index 0000000000..0ea1183be6 --- /dev/null +++ b/docs/specs/desktop-app-environments/screen-judge/plan.md @@ -0,0 +1,16 @@ +# Plan + +1. Schema: `screen_judge` in `appTask.ts`, with `screenProblem` keeping such + tasks to screen judges alone. +2. Judge: `appTaskScreen.ts` renders the screen verifier, finds the files a + check could read change in (`readableChanges`) and the examples a bundle + holds (`screenExamplesOf`); `renderTaskVerifier` dispatches to it. +3. Build computer: `DesktopBuildCapture.mark` keeps a screenshot through + `AgentEnvComputer.addHomeFiles`; the export takes `--screens`. +4. Export: `admitScreenTask` in `DesktopTaskExport.ts`; the relaunch reset in + `ownlessBody` of `appExportPython.ts`. +5. Witness: `judgeScreenTask` and `bundleTaskDifference` in + `DesktopBuildWitnessBundle.ts`, with `screenJudgeProcess` in + `src/desktop/ScreenJudgeProcess.ts`. +6. Docs: the tasks reference, the witness page, the build skill and the + doctor's endpoint check. diff --git a/docs/specs/desktop-app-environments/screen-judge/spec.md b/docs/specs/desktop-app-environments/screen-judge/spec.md new file mode 100644 index 0000000000..c702f04882 --- /dev/null +++ b/docs/specs/desktop-app-environments/screen-judge/spec.md @@ -0,0 +1,53 @@ +# Screen judges: checking state no file shows + +## Outcome + +A task can check what an app keeps only on screen, such as a calculator's +result, through a screen judge: a model answers a yes-or-no question on a +screenshot of the computer, calibrated by two labelled screenshots from the +build. The export ships the judge in the bundle, the witness runs that same +judge on fresh screenshots, and the verifier runs it at grading time. REA +calls no model itself. + +## Requirements + +- Task file: a check `{id?, kind: "screen_judge", question}`, the question + 1 to 2,000 characters after trimming. A task with one holds only screen + judges and no readers. +- Build computer: at each acknowledged checkpoint REA takes a JPEG + screenshot (quality 70) through the driver and writes it under the home at + `/screens/.jpg` with `data/add`; a screenshot + that fails leaves the checkpoint as it is. The export runs with + `--screens /screens`. +- Export: a screen-judged task is refused when an owned file changed between + the seed and the end in a way a check could read (readable or absent at + both ends, and different), naming the files, and when the seed's or the + end's screenshot is missing. It judges nothing: `failing_on_seed` is empty + and a limitation says the witness checks the questions. `verify.py` holds + the questions, the seed's screenshot as the failing example and the end's + as the passing one. An app that owns no files gets a reset that only + relaunches it, so the witness can take `after_reset`. +- Verifier: once the agent stops it waits 1 s, takes a JPEG screenshot + through the computer's `computer_screenshot` tool and asks each question 3 + times; a question passes when at least 2 answers are yes. Each vote shows + the instructions, the two labelled examples and the screenshot, and asks + for `{verdict, reason}` by JSON schema; a failed call is asked again, up to + 3 times, then the verifier raises. The model is `[model.roles] judge`, else + `[model] default`, else `anthropic/claude-sonnet-5-5`, through agentenv's + `resolve_model_call`; without an endpoint it raises, so the run fails + rather than scoring the agent. `agent-env run --judge-model` does not reach + it. +- Witness: for a screen-judged task, `check_build_task` waits 1 s, takes a + screenshot, and runs the bundle's `verify.py judge ` + with the agentenv runner's Python and REA's environment; it refuses on a + computer not started from the bundle, and a judge that fails fails the + check call. On a bundle computer it compares the bundle's verifier with the + task's using the examples the bundle holds. +- Doctor: the model endpoint check names screen-judged tasks as needing it. + +## Non-goals + +- Judging the screen in the export or drafting questions. +- Mixing screen judges with file checks or readers in one task. +- Passing `--judge-model` through; agentenv gives an outcome verifier no run + overrides. diff --git a/docs/specs/desktop-app-environments/screen-judge/tasks.md b/docs/specs/desktop-app-environments/screen-judge/tasks.md new file mode 100644 index 0000000000..b576dc01c3 --- /dev/null +++ b/docs/specs/desktop-app-environments/screen-judge/tasks.md @@ -0,0 +1,9 @@ +# Tasks + +- [x] Record the outcome, requirements and non-goals. +- [x] Add `screen_judge` to the task schema and the evaluator. +- [x] Render the judge into the verifier, with the build's examples. +- [x] Keep a screenshot at each checkpoint and admit screen-judged tasks in the export. +- [x] Run the bundle's judge in the witness. +- [x] Document screen judges. +- [x] Build a GNOME Calculator environment on an arm64 host, witness it and run solves. diff --git a/docs/specs/desktop-app-environments/screen-judge/verification.md b/docs/specs/desktop-app-environments/screen-judge/verification.md new file mode 100644 index 0000000000..5b8fd8c91f --- /dev/null +++ b/docs/specs/desktop-app-environments/screen-judge/verification.md @@ -0,0 +1,58 @@ +# Verification + +Run on 2026-11-06 on top of the readers commit, on macOS with Node v22.23.1 +and Python 3, and on an arm64 Ubuntu 24.04 build host with +agentenv-framework 0.9.1431. + +## Tests + +- `src/domain/desktop/appTaskScreen.test.ts`: the parser keeps a task of + screen judges and refuses one beside a file check, with readers, with a + blank question or with a `file`; REA's evaluator leaves a screen judge + undecided; the rendered verifier screenshots the computer, names the + judge's role, default model and votes, says `--judge-model` does not reach + it, and holds the examples `screenExamplesOf` reads back; `readableChanges` + counts only changes a check could read. +- `tests/boundary/providers/desktopScreenJudgePython.test.ts`: the verifier, + run with stand-ins for the MCP client, litellm and agentenv's config, + takes one screenshot, asks three votes each showing the seed's and the + end's examples before the screenshot, passes on two yes and fails on two + no, asks the `[model.roles] judge` model, asks a failed call again, raises + when no vote answers or no endpoint is set, and judges a given screenshot + when run as a script. +- `src/application/desktop/DesktopTaskExport.test.ts`: the export ships the + seed's and the end's screenshots in order with an empty `failing_on_seed` + and the witness limitation, and refuses a task whose capture shows a + readable file change or lacks a screenshot. +- `tests/boundary/mcp/buildCaptureMcp.test.ts`: marking `seed` and `end` + writes `screens/seed.jpg` and `screens/end.jpg` in the computer, and the + export is given `--screens`. +- `tests/boundary/mcp/buildWitnessMcp.test.ts`: a screen-judged witness runs + the bundle's verifier with the runner's Python at each stage, three votes a + stage, each with three images, and holds. +- `src/domain/desktop/appExport.test.ts`: an app owning no files gets a + relaunch-only reset only for a screen-judged task. +- `src/application/DesktopBuildDoctor.test.ts`: the endpoint check names + screen judges. + +Each failed on the readers commit before the change. + +## Real app + +GNOME Calculator 46.3 on the arm64 host. The capture's draft noted +`no_change` and `no_document`. The task `tip-amount` asks to work out 18% of +47.60 and leave the result showing, with one question, "Does the +calculator's display show the result 8.568?". The export kept it in 1.8 s +with the 39 KB seed and 41 KB end screenshots as examples, and gave the app, +which owns no files, a relaunch-only reset. + +| Stage | Screen | Judge | Held | Time | +| ------------- | -------------------------------- | ----------- | ---- | ------ | +| `seed` | empty display | no, 3 of 3 | yes | 10.7 s | +| `after_edit` | 47.6×0.18 = 8.568 | yes, 3 of 3 | yes | 9.7 s | +| `near_miss` | 47.6×0.17 = 8.092 | no, 3 of 3 | yes | 9.7 s | +| `after_reset` | empty display after the relaunch | no, 3 of 3 | yes | 9.8 s | + +Three solves with Claude Code on Claude Sonnet 5.5 all passed, the judge +saying yes in 3 of 3 votes each time; the verifier step took 8.1 to 9.1 s and +the solves cost $1.72, besides about $0.02 of judge calls per grade. diff --git a/src/application/desktop/DesktopApplicationExport.ts b/src/application/desktop/DesktopApplicationExport.ts index 623e534a67..7e507b292c 100644 --- a/src/application/desktop/DesktopApplicationExport.ts +++ b/src/application/desktop/DesktopApplicationExport.ts @@ -59,7 +59,6 @@ import { } from "./DesktopOutputTree.js"; import { retainedCopies, - shippedSeedViews, writeFiles, type SeedVersion, } from "./DesktopExportFiles.js"; @@ -67,6 +66,7 @@ import { withRetainedContent } from "./DesktopRetainedContent.js"; import { planLaunch, seedModes, seedVersions } from "./DesktopExportLaunch.js"; import { selectStateEnds } from "./DesktopStateVersions.js"; import { admitDesktopTask, type AdmittedTask } from "./DesktopTaskExport.js"; +import { taskReads } from "./DesktopExportReaders.js"; const OPERATION = "export_desktop_application"; /** Largest seed file the export copies; the state view reads only 64 MiB. */ @@ -91,6 +91,8 @@ export interface DesktopExportRequest { readonly endCheckpoint?: string | undefined; /** The seed was made for the task, as in a build computer, rather than taken from the user's own data. */ readonly syntheticSeed?: boolean | undefined; + /** The folder a build computer keeps a screenshot of each checkpoint in, `.jpg`, for a screen judge's examples. */ + readonly screens?: string | undefined; } /** What the export wrote. */ @@ -301,7 +303,10 @@ const bundleText = ( text(`${environment}/env.toml`, renderEnvToml(plan)), text( `${environment}/${appModuleFile(plan.name)}`, - renderAppClass(plan), + renderAppClass({ + ...plan, + relaunchOnReset: admitted?.screenJudged === true, + }), dconfHoldsIdentity, ), text(`${environment}/SKILL.md`, renderSkill(plan)), @@ -405,25 +410,17 @@ const runExport = async ( download: await readDownloadInstall(request.downloadInstall, signal), }); const copies = retainedCopies(plan, seeds); - // A task's seed checks read the seeds as the bundle ships them, scrubbed. - const seedViews = - request.tasks.length === 0 - ? new Map() - : await withRetainedContent( - request.capture, - (content) => shippedSeedViews(content, copies, identity, signal), - SEED_MAX_BYTES, - ); + const reads = await taskReads(request, SEED_MAX_BYTES, { + copies, + files, + ends, + identity, + checkpoints, + signal, + }); const admitted = await admitDesktopTask( request.tasks, - { - ...plan, - files, - identity, - ends, - seedViews, - seeded: new Set(seeds.keys()), - }, + { ...plan, files, identity, ends, ...reads, seeded: new Set(seeds.keys()) }, signal, ); const { generated, readme } = bundleText(plan, admitted, dconf.holdsIdentity); diff --git a/src/application/desktop/DesktopBuildCapture.ts b/src/application/desktop/DesktopBuildCapture.ts index 659b729c94..4fc9211185 100644 --- a/src/application/desktop/DesktopBuildCapture.ts +++ b/src/application/desktop/DesktopBuildCapture.ts @@ -20,7 +20,9 @@ import { import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { err, ok, type Result } from "../../domain/result.js"; import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js"; +import type { AgentEnvComputer } from "../../desktop/AgentEnvComputer.js"; import type { ControlledComputer } from "../../desktop/BuildComputerController.js"; +import { BuildComputerLongRuns } from "../../desktop/BuildComputerLongRuns.js"; import { endedCapture, notAnswered, @@ -47,6 +49,43 @@ import { const START = "start_build_capture"; const MARK = "mark_build_checkpoint"; + +/** The computer routes a capture uses: REA's controller, and the screenshot it keeps of each checkpoint. */ +type CaptureComputer = ControlledComputer & + Pick; + +/** A checkpoint's screenshot as the export takes a screen judge's examples: JPEG, small enough for a judge's prompt. */ +const CHECKPOINT_SHOT = { format: "jpeg", quality: 70 } as const; + +/** + * Keep a screenshot of the checkpoint just taken in the capture's screens + * folder, where the export finds a screen judge's examples. A screenshot + * that fails leaves the checkpoint as it is; only cancellation propagates. + */ +const keepScreenshot = async ( + computer: CaptureComputer, + folder: string, + name: string, + signal: AbortSignal, +): Promise => { + try { + const shot = await computer.callTool( + "computer_screenshot", + CHECKPOINT_SHOT, + signal, + ); + const image = shot.content.find((block) => block.type === "image")?.data; + if (shot.isError || typeof image !== "string") return; + await computer.addHomeFiles( + new Map([ + [`${folder}/screens/${name}.jpg`, Buffer.from(image, "base64")], + ]), + signal, + ); + } catch (cause: unknown) { + if (signal.aborted) throw cause; + } +}; const STOP = "stop_build_capture"; const EXPORT = "export_build_environment"; /** The start's step, for a capture that ended before the app's window opened. */ @@ -226,6 +265,7 @@ const unreleasedError = ( * can be asked again after a lost answer. */ export class DesktopBuildCapture { + readonly longRuns: BuildComputerLongRuns; #count = 0; #run: CaptureRun | undefined; readonly #entry: string | undefined; @@ -236,9 +276,10 @@ export class DesktopBuildCapture { readonly #extraPackages: readonly string[]; constructor( - private readonly computer: ControlledComputer, + private readonly computer: CaptureComputer, request: BuildRequest | undefined, ) { + this.longRuns = new BuildComputerLongRuns(computer); this.#entry = request?.desktop_entry; this.#packageFile = request?.install.kind === "package_file" @@ -441,6 +482,7 @@ export class DesktopBuildCapture { ), ); run.checkpoints.add(name); + await keepScreenshot(this.computer, run.folder, name, signal); const { kind: _kind, changed_files: changed, @@ -548,6 +590,7 @@ export class DesktopBuildCapture { }, input, signal, + this.longRuns, ); } } diff --git a/src/application/desktop/DesktopBuildComputers.cleanup.test.ts b/src/application/desktop/DesktopBuildComputers.cleanup.test.ts new file mode 100644 index 0000000000..a4ebd1cb20 --- /dev/null +++ b/src/application/desktop/DesktopBuildComputers.cleanup.test.ts @@ -0,0 +1,149 @@ +import { expect, it, vi } from "vitest"; +import { AgentEnvComputer } from "../../desktop/AgentEnvComputer.js"; +import { ComputerHelper } from "../../desktop/AgentEnvComputerHelper.js"; +import { ScreenJudgeProcess } from "../../desktop/ScreenJudgeProcess.js"; +import { BuildComputerLongRuns } from "../../desktop/BuildComputerLongRuns.js"; +import { DesktopBuildComputers } from "./DesktopBuildComputers.js"; +import { DesktopRemoteTaskReaders } from "./DesktopRemoteTaskReaders.js"; + +it.each([false, true])( + "releases the computer even if judge cleanup fails (computer also fails: %s)", + async (computerFails) => { + const computerClose = vi + .fn() + .mockResolvedValueOnce( + computerFails + ? { + verified: false, + reason: "computer failed", + resources: ["sandbox"], + } + : { verified: true, teardown: "agentenv" }, + ) + .mockResolvedValue({ verified: true, teardown: "agentenv" }); + const judgeClose = vi + .spyOn(ScreenJudgeProcess.prototype, "close") + .mockResolvedValueOnce({ + verified: false, + reason: "judge failed", + resources: ["judge"], + }) + .mockResolvedValue({ verified: true, teardown: "agentenv" }); + if (computerFails) + judgeClose.mockResolvedValueOnce({ + verified: false, + reason: "judge still failed", + resources: ["judge"], + }); + const spawn = vi + .spyOn(ComputerHelper, "spawn") + .mockResolvedValue({} as never); + const connect = vi.spyOn(AgentEnvComputer, "connect").mockResolvedValue({ + computer: { close: computerClose, sandboxId: "sandbox" }, + frame: { structured: null, content: [] }, + } as never); + const computers = new DesktopBuildComputers({ + REA_AGENT_ENV_COMMAND: "/runner", + REA_DOCKER_COMMAND: "/docker", + }); + try { + const started = await computers.start( + { source: { kind: "bundle", path: "/bundle" } }, + new AbortController().signal, + ); + if (!started.ok) throw started.error; + const id = started.value.value.computer_id; + const first = await computers.close(id); + expect(first.ok).toBe(false); + if (first.ok) throw new Error("Expected incomplete cleanup"); + expect(first.error.cleanupResources).toEqual( + expect.arrayContaining( + computerFails ? ["judge", "sandbox"] : ["judge"], + ), + ); + expect(computerClose).toHaveBeenCalledTimes(1); + const second = await computers.close(id); + if (computerFails) { + expect(second.ok).toBe(false); + expect(first.error.cleanupResources).toEqual(["judge"]); + expect(first.error.cleanupIncomplete).toBe(true); + expect((await computers.close(id)).ok).toBe(true); + } else expect(second.ok).toBe(true); + expect(first.error.cleanupIncomplete).toBe(false); + expect(first.error.cleanupResources).toEqual([]); + expect(computerClose).toHaveBeenCalledTimes(computerFails ? 2 : 1); + expect(judgeClose).toHaveBeenCalledTimes(computerFails ? 3 : 2); + } finally { + await computers.closeAll().catch(() => computers.closeAll()); + spawn.mockRestore(); + connect.mockRestore(); + judgeClose.mockRestore(); + } + }, +); + +it("retains the computer while command and reader cleanup fail, but still releases the judge", async () => { + const computerClose = vi + .fn() + .mockResolvedValue({ verified: true, teardown: "agentenv" }); + const commands = vi + .spyOn(BuildComputerLongRuns.prototype, "release") + .mockResolvedValueOnce({ + verified: false, + reason: "command failed", + resources: ["job"], + }) + .mockResolvedValue({ verified: true, teardown: "agentenv" }); + const readers = vi + .spyOn(DesktopRemoteTaskReaders.prototype, "release") + .mockResolvedValueOnce({ + verified: false, + reason: "reader failed", + resources: ["reader"], + }) + .mockResolvedValue({ verified: true, teardown: "agentenv" }); + const judge = vi + .spyOn(ScreenJudgeProcess.prototype, "close") + .mockResolvedValueOnce({ + verified: false, + reason: "judge failed", + resources: ["judge"], + }) + .mockResolvedValue({ verified: true, teardown: "agentenv" }); + const spawn = vi + .spyOn(ComputerHelper, "spawn") + .mockResolvedValue({} as never); + const connect = vi.spyOn(AgentEnvComputer, "connect").mockResolvedValue({ + computer: { close: computerClose, sandboxId: "sandbox" }, + frame: { structured: null, content: [] }, + } as never); + const computers = new DesktopBuildComputers({ + REA_AGENT_ENV_COMMAND: "/runner", + REA_DOCKER_COMMAND: "/docker", + }); + try { + const started = await computers.start( + { source: { kind: "bundle", path: "/bundle" } }, + new AbortController().signal, + ); + if (!started.ok) throw started.error; + const id = started.value.value.computer_id; + const first = await computers.close(id); + if (first.ok) throw new Error("expected incomplete cleanup"); + expect(first.error.cleanupResources).toEqual( + expect.arrayContaining(["job", "reader", "judge"]), + ); + expect(computerClose).not.toHaveBeenCalled(); + expect(judge).toHaveBeenCalledTimes(1); + expect((await computers.close(id)).ok).toBe(true); + expect(first.error.cleanupIncomplete).toBe(false); + expect(computerClose).toHaveBeenCalledTimes(1); + } finally { + await computers.closeAll().catch(() => computers.closeAll()); + commands.mockRestore(); + readers.mockRestore(); + judge.mockRestore(); + spawn.mockRestore(); + connect.mockRestore(); + } +}); diff --git a/src/application/desktop/DesktopBuildComputers.ts b/src/application/desktop/DesktopBuildComputers.ts index 0534c98a35..5b0ee563c0 100644 --- a/src/application/desktop/DesktopBuildComputers.ts +++ b/src/application/desktop/DesktopBuildComputers.ts @@ -51,6 +51,7 @@ import { DesktopBuildCapture } from "./DesktopBuildCapture.js"; import type { BuildSettle } from "./DesktopBuildSettle.js"; import { DesktopBuildTurns } from "./DesktopBuildTurns.js"; import { DesktopBuildWitness } from "./DesktopBuildWitness.js"; +import { ScreenJudgeProcess } from "../../desktop/ScreenJudgeProcess.js"; const START = "start_build_computer"; const ACT = "act_on_build_computer"; @@ -90,6 +91,10 @@ interface HeldComputer { readonly steps: BuildComputerSteps | undefined; /** Every action, step and close on the computer takes its turn here. */ readonly turns: DesktopBuildTurns; + readonly cleanup: { + resources: readonly string[]; + errors: Set>; + }; closing: boolean; } @@ -206,30 +211,79 @@ const liveComputer = ( computer: AgentEnvComputer, workspace: PrivateRuntimeRoot | undefined, source: BuildSource, - settle: BuildSettle, + grading: { readonly settle: BuildSettle; readonly judge: ScreenJudgeProcess }, ): HeldComputer => { const actions = new BuildComputerActionLog(); const turns = new DesktopBuildTurns(); + const capture = new DesktopBuildCapture( + computer, + source.kind === "request" ? source : undefined, + ); + const witness = new DesktopBuildWitness( + computer, + actions, + grading.settle, + source.kind === "bundle" + ? { path: source.path, judge: grading.judge.judge } + : undefined, + ); + let closedComputer: Extract | undefined; + const attempt = async ( + release: () => Promise, + ): Promise => { + try { + return await release(); + } catch (cause: unknown) { + return { + verified: false, + reason: cause instanceof Error ? cause.message : String(cause), + resources: [], + }; + } + }; return { computer, - release: () => computer.close(), + release: async () => { + const [commands, readers, judge] = await Promise.all([ + attempt(() => capture.longRuns.release()), + attempt(() => witness.readers.release()), + attempt(() => grading.judge.close()), + ]); + if (!commands.verified || !readers.verified) { + const failed = [commands, readers, judge].filter( + (result) => !result.verified, + ); + return { + verified: false, + reason: failed.map((result) => result.reason).join("; "), + resources: [...new Set(failed.flatMap((result) => result.resources))], + }; + } + const computerResult = + closedComputer === undefined + ? await attempt(() => computer.close()) + : closedComputer; + if (computerResult.verified) closedComputer = computerResult; + const failed = [judge, computerResult].filter( + (result) => !result.verified, + ); + if (failed.length === 0 && computerResult.verified) return computerResult; + return { + verified: false, + reason: failed.map((result) => result.reason).join("; "), + resources: [...new Set(failed.flatMap((result) => result.resources))], + }; + }, workspace, actions, steps: { computer, - capture: new DesktopBuildCapture( - computer, - source.kind === "request" ? source : undefined, - ), - witness: new DesktopBuildWitness( - computer, - actions, - settle, - source.kind === "bundle" ? source.path : undefined, - ), + capture, + witness, turns, }, turns, + cleanup: { resources: [], errors: new Set() }, closing: false, }; }; @@ -418,7 +472,11 @@ export class DesktopBuildComputers { ); this.#computers.set( id, - liveComputer(computer, target.workspace, source, this.settle), + liveComputer(computer, target.workspace, source, { + settle: this.settle, + // The resolved environment, which names a runner found on PATH. + judge: new ScreenJudgeProcess(environment), + }), ); return ok({ value: { @@ -445,6 +503,7 @@ export class DesktopBuildComputers { actions: new BuildComputerActionLog(), steps: undefined, turns: new DesktopBuildTurns(), + cleanup: { resources: [], errors: new Set() }, closing: true, }); const release = await this.#retire(id); @@ -565,16 +624,29 @@ export class DesktopBuildComputers { held.closing = true; // Tear down only once the call it cancels has ended; the calls waiting are refused. const release = await held.turns.close(() => this.#retire(id)); - return release.verified - ? ok({ computer_id: id, teardown: release.teardown }) - : err( - new ProviderCleanupError( - AGENTENV_PROVIDER, - release.resources, - { reason: release.reason }, - { operation: CLOSE, remediationAction: release.reason }, - ), - ); + held.cleanup.resources = release.verified ? [] : release.resources; + if (release.verified) { + for (const reference of held.cleanup.errors) + reference.deref()?.resolveCleanup(); + held.cleanup.errors.clear(); + return ok({ computer_id: id, teardown: release.teardown }); + } + const failure = new ProviderCleanupError( + AGENTENV_PROVIDER, + release.resources, + { reason: release.reason }, + { operation: CLOSE, remediationAction: release.reason }, + ); + failure.bindCleanupResourceOwner({ + get cleanupResources() { + return held.cleanup.resources; + }, + }); + for (const reference of held.cleanup.errors) + if (reference.deref() === undefined) + held.cleanup.errors.delete(reference); + held.cleanup.errors.add(new WeakRef(failure)); + return err(failure); } /** Release a held computer and its bundle folder; it stays held while either is unconfirmed. */ diff --git a/src/application/desktop/DesktopBuildExport.test.ts b/src/application/desktop/DesktopBuildExport.test.ts new file mode 100644 index 0000000000..111ce94d51 --- /dev/null +++ b/src/application/desktop/DesktopBuildExport.test.ts @@ -0,0 +1,115 @@ +import { expect, it } from "vitest"; + +import { exportBuild } from "./DesktopBuildExport.js"; + +it.each([4, 8])( + "gives the export time for both passes over %i readers", + async (count) => { + let passed: { timeout_s?: number } | undefined; + let routeTimeout: number | undefined; + const computer = { + exec: (argv: readonly string[], timeout: number) => { + if (argv[4] === "run-begin") + passed = JSON.parse(argv.slice(5).join("")) as { timeout_s?: number }; + routeTimeout = Math.max(routeTimeout ?? 0, timeout); + return Promise.resolve({ + exit_code: 0, + stdout: '{"refused":"stopped for test"}\n', + stderr: "", + truncated: false, + timed_out: false, + }); + }, + homeFiles: () => Promise.resolve(new Map()), + redacted: (text: string) => text, + }; + const readers = Array.from({ length: count }, (_, index) => ({ + id: `r${String(index)}`, + file: `f${String(index)}.bin`, + argv: ["cat", "{file}"], + output: "lines" as const, + timeout_s: 300, + })); + await exportBuild( + computer, + { folder: ".rea-build/c", attempt: 0, extraPackages: [] }, + { + directory: "/unused", + task: { + name: "t", + prompt: "Task", + checks: [ + { + kind: "lines_present", + file: "f0.bin", + lines: ["x"], + reader: "r0", + }, + ], + readers, + }, + }, + new AbortController().signal, + ); + expect(passed?.timeout_s).toBeGreaterThanOrEqual( + 1800 + 2 * count * (300 + 30), + ); + // The computer-exec/v1 protocol accepts no single call over 3,600 s. + expect(routeTimeout).toBeLessThanOrEqual(3600); + }, +); + +it.each([ + [[260, 260, 260], "run", 3600], + [[261, 260, 260], "run-begin", 90], +] as const)( + "keeps the computer-exec deadline within its protocol limit", + async (timeouts, expected, routeTimeout) => { + let command: string | undefined; + let observedTimeout: number | undefined; + const computer = { + exec: async (argv: readonly string[], timeout: number) => { + command = argv[4]; + observedTimeout = timeout; + return { + exit_code: 0, + stdout: '{"refused":"stop for test"}\n', + stderr: "", + timed_out: false, + truncated: false, + }; + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + }; + await exportBuild( + computer, + { folder: ".rea-build/c", attempt: 0, extraPackages: [] }, + { + directory: "/unused", + task: { + name: "t", + prompt: "Task", + checks: [ + { + kind: "lines_present", + file: "f0.bin", + lines: ["x"], + reader: "r0", + }, + ], + readers: timeouts.map((timeout_s, index) => ({ + id: `r${String(index)}`, + file: `f${String(index)}.bin`, + argv: ["cat", "{file}"], + output: "lines" as const, + timeout_s, + })), + }, + }, + new AbortController().signal, + ); + expect(command).toBe(expected); + expect(observedTimeout).toBe(routeTimeout); + }, +); diff --git a/src/application/desktop/DesktopBuildExport.ts b/src/application/desktop/DesktopBuildExport.ts index c7fdf5a9eb..6a89c2e844 100644 --- a/src/application/desktop/DesktopBuildExport.ts +++ b/src/application/desktop/DesktopBuildExport.ts @@ -2,8 +2,14 @@ import { Readable } from "node:stream"; import { z } from "zod"; -import { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { DownloadInstall } from "../../domain/desktop/appDownloadInstall.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + READER_RUNNER_SLACK_S, + readerTimeout, +} from "../../domain/desktop/appTaskReaders.js"; +import { BUILD_REA_TIMEOUT_S } from "../../desktop/BuildComputerController.js"; +import { BuildComputerLongRuns } from "../../desktop/BuildComputerLongRuns.js"; import type { desktopTaskSchema } from "../../domain/desktop/appTask.js"; import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js"; import { err, ok, type Result } from "../../domain/result.js"; @@ -20,6 +26,7 @@ import { refused, reportedFailure, runRea, + WORK, } from "./DesktopBuildCommands.js"; import { writeOutputTree, @@ -106,21 +113,6 @@ const writeBundle = async ( ); }; -/** The `--download-install` file the export reads beside the task, and its flag; neither without a download. */ -const downloadInput = ( - path: string, - install: DownloadInstall | undefined, -): { - readonly argv: readonly string[]; - readonly files: Readonly>; -} => - install === undefined - ? { argv: [], files: {} } - : { - argv: ["--download-install", path], - files: { [path]: `${JSON.stringify(install, null, 2)}\n` }, - }; - /** * Export a stopped capture in the computer with the agent's task, seeded at * `seed` and checked at `end`, into a new folder of the work folder, then @@ -134,50 +126,65 @@ export const exportBuild = async ( readonly attempt: number; /** The URL a computer downloads the app's `.deb` from, when the capture says apt does not install it. */ readonly packageUrl?: string; - /** The AppImage or tarball the request unpacked the app from, which the bundle downloads too. */ readonly downloadInstall?: DownloadInstall; /** The packages the request named beside the app, which the bundle installs beside it too. */ readonly extraPackages: readonly string[]; }, request: BuildExportRequest, signal: AbortSignal, + longRuns: BuildComputerLongRuns = new BuildComputerLongRuns(computer), ): Promise> => { const { folder } = capture; const attempt = String(capture.attempt); const bundle = `${folder}/bundle-${attempt}`; const task = `${folder}/task-${attempt}.json`; const stdout = `${folder}/export-${attempt}.json`; - const download = downloadInput( - `${folder}/download-install-${attempt}.json`, - capture.downloadInstall, - ); + const downloadPath = `${folder}/download-install-${attempt}.json`; + const download = capture.downloadInstall; const taskFile = `${JSON.stringify(request.task, null, 2)}\n`; try { - const { stdoutBytes: bytes } = await runRea( - computer, - [ - "export-desktop-application", - `${folder}/capture`, - bundle, - "--task", - task, - ...ENDS, - // The building agent made the seed in this computer for the task. - "--synthetic-seed", - ...(request.name === undefined ? [] : ["--name", request.name]), - ...(capture.packageUrl === undefined - ? [] - : ["--package-url", capture.packageUrl]), - ...download.argv, - ...capture.extraPackages.flatMap((name) => ["--extra-package", name]), - "--json", - ], - { - stdout, - files: { [task]: taskFile, ...download.files }, - signal, + const argv = [ + "export-desktop-application", + `${folder}/capture`, + bundle, + "--task", + task, + ...ENDS, + // The building agent made the seed in this computer for the task. + "--synthetic-seed", + "--screens", + `${folder}/screens`, + ...(request.name === undefined ? [] : ["--name", request.name]), + ...(capture.packageUrl === undefined + ? [] + : ["--package-url", capture.packageUrl]), + ...(download === undefined ? [] : ["--download-install", downloadPath]), + ...capture.extraPackages.flatMap((name) => ["--extra-package", name]), + "--json", + ]; + // Export reads each declared reader at both checkpoints. + const timeoutS = + BUILD_REA_TIMEOUT_S + + 2 * + (request.task.readers ?? []).reduce( + (sum, reader) => sum + readerTimeout(reader) + READER_RUNNER_SLACK_S, + 0, + ); + const options = { + stdout, + files: { + [task]: taskFile, + ...(download === undefined + ? {} + : { [downloadPath]: `${JSON.stringify(download, null, 2)}\n` }), }, - ); + timeoutS, + signal, + }; + const { stdoutBytes: bytes } = + timeoutS + 60 <= 3600 + ? await runRea(computer, argv, options) + : await longRuns.run(argv, { ...options, work: WORK }); const output = await readOutput(computer, stdout, bytes, signal); if (output.kind === "failure") return err( diff --git a/src/application/desktop/DesktopBuildWitness.test.ts b/src/application/desktop/DesktopBuildWitness.test.ts index c1d2b06f41..95436a58fb 100644 --- a/src/application/desktop/DesktopBuildWitness.test.ts +++ b/src/application/desktop/DesktopBuildWitness.test.ts @@ -39,6 +39,8 @@ const computer = { startRecording: () => Promise.reject(new Error("unstarted")), endRecording: () => Promise.reject(new Error("unended")), recordingSegments: () => Promise.reject(new Error("unfetched")), + exec: () => Promise.reject(new Error("unrun")), + callTool: () => Promise.reject(new Error("uncalled")), } as WitnessedComputer; const witnessAt = async (bundle: string) => @@ -46,7 +48,7 @@ const witnessAt = async (bundle: string) => computer, new BuildComputerActionLog(), SETTLE_TIMING, - bundle, + { path: bundle }, ); describe("bundle task reads", () => { diff --git a/src/application/desktop/DesktopBuildWitness.ts b/src/application/desktop/DesktopBuildWitness.ts index 034ad48787..94430a4ab0 100644 --- a/src/application/desktop/DesktopBuildWitness.ts +++ b/src/application/desktop/DesktopBuildWitness.ts @@ -1,7 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; import { readdir } from "node:fs/promises"; -import { join } from "node:path"; -import { isDeepStrictEqual } from "node:util"; import { z } from "zod"; @@ -17,12 +15,7 @@ import { evaluateTaskChecks, type TaskCheckOutcome, } from "../../domain/desktop/appTaskChecks.js"; -import { - renderTaskSteps, - renderTaskVerifier, - sameTaskVerifier, - taskVerifierArtifact, -} from "../../domain/desktop/appTaskPython.js"; +import { isScreenTask } from "../../domain/desktop/appTaskScreen.js"; import { admitWitnessStage, appStateViews, @@ -35,18 +28,12 @@ import { type WitnessStageRecord, } from "../../domain/desktop/buildWitness.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; -import { - guestFileAnswerCeiling, - guestFileAnswerWorkingBytes, -} from "../../desktop/AgentEnvComputer.js"; import { err, ok, type Result } from "../../domain/result.js"; -import { safeParseUtf8Json } from "../../domain/safeJson.js"; import type { AgentEnvComputer, RecordingManifest, } from "../../desktop/AgentEnvComputer.js"; import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js"; -import { readHostFile } from "../../desktop/HostFileRead.js"; import { isPortableBundleName } from "../environment/environmentBundle.js"; import { silentProgressReporter, @@ -56,6 +43,13 @@ import type { ReadonlyActionLog } from "./DesktopBuildActionLog.js"; import { errorText } from "./DesktopBuildCommands.js"; import { desktopOperationFailure } from "./DesktopCaptureRefusals.js"; import { settledAppData, type BuildSettle } from "./DesktopBuildSettle.js"; +import { DesktopRemoteTaskReaders } from "./DesktopRemoteTaskReaders.js"; +import { + bundleTaskDifference, + judgeScreenTask, + type Graded, + type WitnessBundle, +} from "./DesktopBuildWitnessBundle.js"; import { writeWitnessRecord, type EndedWitness, @@ -69,7 +63,12 @@ const FRESH = /** The computer routes the witness reads. */ export type WitnessedComputer = Pick< AgentEnvComputer, - "appData" | "startRecording" | "endRecording" | "recordingSegments" + | "appData" + | "startRecording" + | "endRecording" + | "recordingSegments" + | "exec" + | "callTool" >; type WitnessSummary = { @@ -81,13 +80,6 @@ type WitnessSummary = { readonly record: WitnessRecord | null; }; -/** A task's results on the app's views, whether its files had settled, and whether the bound cut a read. */ -type Graded = { - readonly results: TaskCheckOutcome[]; - readonly settled: boolean; - readonly read_cut: boolean; -}; - /** One check of the task against the app's live state, and the witness stage it filed, if any. */ export type CheckedTask = { readonly task: string; @@ -119,51 +111,6 @@ const usableFolder = async (directory: string): Promise => (cause as NodeJS.ErrnoException | undefined)?.code === "ENOENT", ); -/** - * The most bytes REA reads of each of the bundle's task files. A task the - * export accepts has no smaller file cap, and the steps file holds the whole - * prompt. The bound is the same working budget as a guest file answer: a - * quarter of the heap, divided by the copies a read and a parse hold. - */ -const bundleFileBytes = () => - guestFileAnswerWorkingBytes(guestFileAnswerCeiling()); -const stepsSchema = z.array( - z.looseObject({ type: z.string(), env_id: z.string().optional() }), -); - -/** Why the bundle's steps and verifier for the task are not what the export writes for it; undefined when they are. */ -const bundleTaskDifference = async ( - bundle: string, - task: DesktopTask, - signal: AbortSignal, -): Promise => { - const stepsPath = `tasks/${task.name}.json`; - const verifierPath = `artifacts/${taskVerifierArtifact(task)}/verify.py`; - const [steps, verifier] = await Promise.all( - [stepsPath, verifierPath].map((path) => - readHostFile(join(bundle, path), bundleFileBytes(), signal), - ), - ); - if (!Buffer.isBuffer(steps) || !Buffer.isBuffer(verifier)) - return `the bundle at ${bundle} has no readable ${Buffer.isBuffer(steps) ? verifierPath : stepsPath}.`; - const parsed = safeParseUtf8Json(steps); - const listed = parsed.ok ? parsed.value : undefined; - const app = stepsSchema - .safeParse(listed) - .data?.find((step) => step.type === "deploy_env")?.env_id; - if ( - app === undefined || - !isDeepStrictEqual(listed, JSON.parse(renderTaskSteps(task, app))) - ) - return `the bundle's ${stepsPath} differs from this task's steps.`; - return sameTaskVerifier( - renderTaskVerifier(task, app), - verifier.toString("utf8"), - ) - ? undefined - : `the bundle's ${verifierPath} differs from this task's checks.`; -}; - /** * A witness from its seed on: `open` while stages are checked, `saving` * once a stage ended it and until its record is written, which a repeat of @@ -242,6 +189,7 @@ export const retryPublishedCleanup = async ( export class DesktopBuildWitness { #witness: OpenWitness | undefined; + readonly readers: DesktopRemoteTaskReaders; /** * The request id of this computer's one witness recording, fixed before any * start is sent: a seed retried after a lost reply asks for the same id, @@ -254,8 +202,13 @@ export class DesktopBuildWitness { private readonly computer: WitnessedComputer, private readonly actions: ReadonlyActionLog, private readonly settle: BuildSettle = SETTLE_TIMING, - private readonly bundle?: string, - ) {} + private readonly bundle?: WitnessBundle, + ) { + this.readers = new DesktopRemoteTaskReaders( + (argv, timeout, signal, requestId) => + this.computer.exec(argv, timeout, signal, requestId), + ); + } async check( input: { @@ -297,12 +250,14 @@ export class DesktopBuildWitness { return filed.ok ? ok({ ...checked, witness: filed.value }) : filed; } - /** The task's checks against the app's views once its files settle, one result per check. */ + /** The task's checks against the app's views once its files settle, and its readers' views read then, one result per check. */ async #grade( task: DesktopTask, signal: AbortSignal, progress: ProgressReporter, ): Promise> { + if (isScreenTask(task)) + return judgeScreenTask(this.computer, this.bundle, task, signal); try { const read = await settledAppData( (turn) => this.computer.appData("data/get", turn), @@ -314,18 +269,19 @@ export class DesktopBuildWitness { .catch(() => undefined), ); const views = appStateViews(read.parts); - return views.ok - ? ok({ - results: [...evaluateTaskChecks(task.checks, views.value)], - settled: read.settled, - read_cut: read.readCut, - }) - : err( - refused( - views.error, - "Start the computer from a bundle export_build_environment wrote.", - ), - ); + if (!views.ok) + return err( + refused( + views.error, + "Start the computer from a bundle export_build_environment wrote.", + ), + ); + const readers = await this.readers.read(task.readers ?? [], signal); + return ok({ + results: [...evaluateTaskChecks(task.checks, views.value, readers)], + settled: read.settled, + read_cut: read.readCut, + }); } catch (cause: unknown) { return err( cause instanceof AnalysisError @@ -560,7 +516,7 @@ export class DesktopBuildWitness { let differs: string | undefined; if (this.bundle !== undefined) { try { - differs = await bundleTaskDifference(this.bundle, task, signal); + differs = await bundleTaskDifference(this.bundle.path, task, signal); } catch (cause: unknown) { // A cancelled read, an I/O failure, or a handle that would not close // stays a Result. A cleanup lease stays bound to that error. diff --git a/src/application/desktop/DesktopBuildWitnessBundle.test.ts b/src/application/desktop/DesktopBuildWitnessBundle.test.ts new file mode 100644 index 0000000000..3572cea573 --- /dev/null +++ b/src/application/desktop/DesktopBuildWitnessBundle.test.ts @@ -0,0 +1,50 @@ +import { expect, it } from "vitest"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { judgeScreenTask } from "./DesktopBuildWitnessBundle.js"; + +it("preserves earlier judge outcomes when a later check fails cleanup", async () => { + const failure = new ProviderCleanupError( + "agentenv", + ["child"], + { reason: "cleanup failed" }, + { operation: "check_build_task" }, + ); + const result = await judgeScreenTask( + { + callTool: async () => ({ + isError: false, + content: [{ type: "image", data: "aW1hZ2U=" }], + structured: null, + }), + }, + { + path: "/bundle", + judge: async (_verifier, check) => { + if (check.id === "second") throw failure; + return { id: check.id, passed: true, description: "first completed" }; + }, + }, + { + name: "test", + prompt: "test", + checks: [ + { kind: "screen_judge", id: "first", question: "First?" }, + { kind: "screen_judge", id: "second", question: "Second?" }, + ], + }, + new AbortController().signal, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cleanup failure"); + expect(result.error).toBe(failure); + expect(projectAnalysisError(result.error)).toMatchObject({ + details: { + diagnostics: { + completed_judge_checks: [ + { id: "first", passed: true, description: "first completed" }, + ], + }, + }, + }); +}); diff --git a/src/application/desktop/DesktopBuildWitnessBundle.ts b/src/application/desktop/DesktopBuildWitnessBundle.ts new file mode 100644 index 0000000000..4f5111ce58 --- /dev/null +++ b/src/application/desktop/DesktopBuildWitnessBundle.ts @@ -0,0 +1,157 @@ +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { isDeepStrictEqual } from "node:util"; + +import { z } from "zod"; + +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; +import type { DesktopTask } from "../../domain/desktop/appTask.js"; +import type { TaskCheckOutcome } from "../../domain/desktop/appTaskChecks.js"; +import { + renderTaskSteps, + renderTaskVerifier, + sameTaskVerifier, + taskVerifierArtifact, +} from "../../domain/desktop/appTaskPython.js"; +import { + SCREEN_SETTLE_MS, + isScreenTask, + screenExamplesOf, + type ScreenJudge, +} from "../../domain/desktop/appTaskScreen.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { safeParseUtf8Json } from "../../domain/safeJson.js"; +import { + guestFileAnswerCeiling, + guestFileAnswerWorkingBytes, + type AgentEnvComputer, +} from "../../desktop/AgentEnvComputer.js"; +import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js"; +import { readHostFile } from "../../desktop/HostFileRead.js"; +import { errorText } from "./DesktopBuildCommands.js"; + +const CHECK = "check_build_task"; + +/** The bundle folder a witnessed computer was started from, and the judge that runs its screen judges. */ +export interface WitnessBundle { + readonly path: string; + readonly judge?: ScreenJudge; +} + +/** A task's results on the app's state, whether its files had settled, and whether the bound cut a read. */ +export type Graded = { + readonly results: TaskCheckOutcome[]; + readonly settled: boolean; + readonly read_cut: boolean; +}; + +/** + * The most bytes REA reads of each of the bundle's task files. A task the + * export accepts has no smaller file cap, and the steps file holds the whole + * prompt. The bound is the same working budget as a guest file answer: a + * quarter of the heap, divided by the copies a read and a parse hold. + */ +const bundleFileBytes = () => + guestFileAnswerWorkingBytes(guestFileAnswerCeiling()); +const stepsSchema = z.array( + z.looseObject({ type: z.string(), env_id: z.string().optional() }), +); + +/** Why the bundle's steps and verifier for the task are not what the export writes for it; undefined when they are. */ +export const bundleTaskDifference = async ( + bundle: string, + task: DesktopTask, + signal: AbortSignal, +): Promise => { + const stepsPath = `tasks/${task.name}.json`; + const verifierPath = `artifacts/${taskVerifierArtifact(task)}/verify.py`; + const [steps, verifier] = await Promise.all( + [stepsPath, verifierPath].map((path) => + readHostFile(join(bundle, path), bundleFileBytes(), signal), + ), + ); + if (!Buffer.isBuffer(steps) || !Buffer.isBuffer(verifier)) + return `the bundle at ${bundle} has no readable ${Buffer.isBuffer(steps) ? verifierPath : stepsPath}.`; + const parsed = safeParseUtf8Json(steps); + const listed = parsed.ok ? parsed.value : undefined; + const app = stepsSchema + .safeParse(listed) + .data?.find((step) => step.type === "deploy_env")?.env_id; + if ( + app === undefined || + !isDeepStrictEqual(listed, JSON.parse(renderTaskSteps(task, app))) + ) + return `the bundle's ${stepsPath} differs from this task's steps.`; + const source = verifier.toString("utf8"); + const differs = `the bundle's ${verifierPath} differs from this task's checks.`; + if (!isScreenTask(task)) + return sameTaskVerifier(renderTaskVerifier(task, app), source) + ? undefined + : differs; + // A screen judge's examples are the build's screenshots, which only the bundle holds. + const examples = screenExamplesOf(source); + if (examples === undefined) + return `the bundle's ${verifierPath} holds no screen judge examples.`; + return source === renderTaskVerifier(task, app, examples) + ? undefined + : differs; +}; + +/** A screen-judged task's checks on a fresh screenshot, each answered by the bundle's own judge. */ +export const judgeScreenTask = async ( + computer: Pick, + bundle: WitnessBundle | undefined, + task: DesktopTask, + signal: AbortSignal, +): Promise> => { + const judge = bundle?.judge; + if (bundle === undefined || judge === undefined) + return err( + new AnalysisInputError(CHECK, { + userMessage: + "A screen_judge is checked by the bundle's own judge, so it needs a computer started from the bundle.", + remediationAction: + "Start a computer from the exported bundle with start_build_computer and check it there.", + conformsToInputContract: true, + }), + ); + const results: TaskCheckOutcome[] = []; + try { + await delay(SCREEN_SETTLE_MS, undefined, { signal }); + const shot = await computer.callTool( + "computer_screenshot", + { format: "jpeg", quality: 70 }, + signal, + ); + const image = shot.content.find((block) => block.type === "image")?.data; + if (shot.isError || typeof image !== "string") + throw new Error("the computer took no screenshot"); + const screen = Buffer.from(image, "base64"); + const verifier = join( + bundle.path, + "artifacts", + taskVerifierArtifact(task), + "verify.py", + ); + for (const check of task.checks) + if (check.kind === "screen_judge") + results.push(await judge(verifier, check, screen, signal)); + return ok({ results, settled: true, read_cut: false }); + } catch (cause: unknown) { + return err( + cause instanceof AnalysisError + ? results.length === 0 + ? cause + : cause.retainDiagnostics({ completed_judge_checks: results }) + : new ProviderAdapterError(AGENTENV_PROVIDER, CHECK, { + userMessage: `The screen judge could not check the screen: ${errorText(cause)}`, + cause, + ...(results.length === 0 + ? {} + : { diagnostics: { completed_judge_checks: results } }), + }), + ); + } +}; diff --git a/src/application/desktop/DesktopBuildWitnessRecord.ts b/src/application/desktop/DesktopBuildWitnessRecord.ts index 7095c15756..1837cddf27 100644 --- a/src/application/desktop/DesktopBuildWitnessRecord.ts +++ b/src/application/desktop/DesktopBuildWitnessRecord.ts @@ -4,6 +4,7 @@ import { Readable } from "node:stream"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { DesktopTask } from "../../domain/desktop/appTask.js"; +import { isScreenTask } from "../../domain/desktop/appTaskScreen.js"; import type { WitnessStage, WitnessStageRecord, @@ -236,6 +237,15 @@ const unsettledLimitation = ( ]; }; +/** How the witness reached its check results, by the kind of task. */ +const howChecked = (task: DesktopTask): string => { + if (isScreenTask(task)) + return "Checks took a screenshot of the computer and ran the bundle's own verifier as its judge, with the agentenv runner's Python and model endpoint, three votes a question; the judge sent each screenshot and the build's two examples to that model."; + return task.readers === undefined + ? "Checks read the app's data/get views with REA's evaluator, as the exported verifier does; the verifier itself did not run." + : "Checks read the app's data/get views and the task's readers' views, the readers run in the computer through computer-exec/v1, with REA's evaluator, as the exported verifier does; the verifier itself did not run."; +}; + /** How a witness write ended: saved, retryable, or published with cleanup still open. */ export type WitnessWrite = | { readonly kind: "saved"; readonly record: WitnessRecord } @@ -304,7 +314,7 @@ export const writeWitnessRecord = async ( confidence: "observed", authority: "controlled-replay", limitations: [ - "Checks read the app's data/get views with REA's evaluator, as the exported verifier does; the verifier itself did not run.", + howChecked(witness.task), ...("error" in recording ? [`The recording is missing: ${recording.error}`] : []), diff --git a/src/application/desktop/DesktopExportFiles.ts b/src/application/desktop/DesktopExportFiles.ts index 41183ca681..6bd5ddd1d7 100644 --- a/src/application/desktop/DesktopExportFiles.ts +++ b/src/application/desktop/DesktopExportFiles.ts @@ -116,6 +116,19 @@ const shippedSeed = ( }; }; +/** A seed's bytes as the bundle ships them, for a task's reader to read at the seed. */ +export const shippedSeedBytes = async ( + content: RetainedContent, + copy: RetainedCopy, + identity: HostIdentity, + signal: AbortSignal | undefined, +): Promise => { + const bytes = await retainedBytes(content, copy, signal); + return copy.seed === undefined + ? bytes + : (shippedSeed(bytes, copy.seed, identity).bytes ?? bytes); +}; + /** * The view of each rewritten seed, including a formerly unreadable gzip * seed, as the bundle ships it: the computer starts from these bytes. diff --git a/src/application/desktop/DesktopExportReaders.ts b/src/application/desktop/DesktopExportReaders.ts new file mode 100644 index 0000000000..a5c1eda76d --- /dev/null +++ b/src/application/desktop/DesktopExportReaders.ts @@ -0,0 +1,140 @@ +import { readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; + +import type { DesktopTaskReader } from "../../domain/desktop/appTask.js"; +import { SCREEN_EXAMPLE_MAX_BYTES } from "../../domain/desktop/appTaskScreen.js"; +import type { TaskFileView } from "../../domain/desktop/appTaskChecks.js"; +import { + versionAt, + type DesktopStatePath, + type StateEnds, +} from "../../domain/desktop/desktopState.js"; +import type { HostIdentity } from "../../domain/desktop/hostIdentity.js"; +import { runReaderHere } from "../../desktop/TaskReaderProcess.js"; +import { + shippedSeedBytes, + shippedSeedViews, + type RetainedCopy, +} from "./DesktopExportFiles.js"; +import { + withRetainedContent, + type RetainedContent, +} from "./DesktopRetainedContent.js"; +import { + readerViewsOfBytes, + type ReaderFileBytes, +} from "./DesktopTaskReaders.js"; +import type { DesktopExportRequest } from "./DesktopApplicationExport.js"; + +/** What the export reads a reader's file from: the retained content, the seeds it ships and each file's versions. */ +export interface ReaderSources { + readonly capture: string; + readonly copies: readonly RetainedCopy[]; + readonly files: ReadonlyMap; + readonly ends: StateEnds; + readonly identity: HostIdentity; + readonly maxBytes: number; + readonly signal: AbortSignal | undefined; + /** The capture's checkpoint names, and the folder a build computer keeps a screenshot of each in, if any. */ + readonly checkpoints: readonly string[]; + readonly screens: string | undefined; +} + +/** The screenshot a build computer took at the seed or the end checkpoint, too large to ship, or undefined when there is none. */ +const screenAt = async ( + sources: ReaderSources, + end: keyof StateEnds, +): Promise => { + const name = sources.checkpoints[sources.ends[end]]; + if (sources.screens === undefined || name === undefined) return undefined; + const path = join(sources.screens, `${name}.jpg`); + const found = await stat(path).catch(() => undefined); + if (found?.isFile() !== true) return undefined; + return found.size > SCREEN_EXAMPLE_MAX_BYTES ? "too-large" : readFile(path); +}; + +/** A reader's file at the seed, as the bundle ships it, or at the end, as retained. */ +const readerFile = async ( + sources: ReaderSources, + content: RetainedContent, + file: string, + end: keyof StateEnds, +): Promise => { + const { copies, files, ends, identity, signal } = sources; + if (end === "seed") { + const copy = copies.find(({ seed }) => seed?.home === file); + return copy === undefined + ? { kind: "absent" } + : { + kind: "bytes", + bytes: await shippedSeedBytes(content, copy, identity, signal), + }; + } + const path = files.get(file); + const version = path === undefined ? undefined : versionAt(path, ends.end); + if (version?.present !== true) return { kind: "absent" }; + if (version.sha256 === null) + return { + kind: "error", + error: `~/${file} was not retained at the end checkpoint, so there is nothing to read`, + }; + const read = await content.read(version.sha256, signal); + return read.ok + ? { kind: "bytes", bytes: read.value } + : { + kind: "error", + error: `the retained content of ~/${file} ${read.error}`, + }; +}; + +/** Each reader's view of its file at one end, run on this host, where the export runs: in the build computer, beside the app. */ +const exportReaderViews = + (sources: ReaderSources) => + ( + readers: readonly DesktopTaskReader[], + end: keyof StateEnds, + ): Promise> => + withRetainedContent( + sources.capture, + (content) => + readerViewsOfBytes( + readers, + (reader) => readerFile(sources, content, reader.file, end), + (argv, timeoutS) => runReaderHere(argv, timeoutS, sources.signal), + ), + sources.maxBytes, + ); + +/** What a task's checks read beyond the state view: the text seeds' views as the bundle ships them, its readers' and its screenshots. */ +export const taskReads = async ( + request: Pick, + maxBytes: number, + found: Omit, +) => { + const sources = { + ...found, + capture: request.capture, + screens: request.screens, + maxBytes, + }; + return readsOf(request.tasks.length, sources); +}; + +const readsOf = async (tasks: number, sources: ReaderSources) => ({ + seedViews: + tasks === 0 + ? new Map() + : await withRetainedContent( + sources.capture, + (content) => + shippedSeedViews( + content, + sources.copies, + sources.identity, + sources.signal, + ), + sources.maxBytes, + ), + readerViews: exportReaderViews(sources), + screens: (end: keyof StateEnds) => screenAt(sources, end), +}); diff --git a/src/application/desktop/DesktopRemoteTaskReaders.test.ts b/src/application/desktop/DesktopRemoteTaskReaders.test.ts new file mode 100644 index 0000000000..f2aac76297 --- /dev/null +++ b/src/application/desktop/DesktopRemoteTaskReaders.test.ts @@ -0,0 +1,188 @@ +import { expect, it } from "vitest"; + +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { PYTHON_READER_CLEANUP } from "../../domain/desktop/appTaskReaders.js"; +import { DesktopRemoteTaskReaders } from "./DesktopRemoteTaskReaders.js"; + +const reader = { + id: "note", + file: "Documents/note", + argv: ["reader", "{file}"], + output: "json" as const, +}; +const success = { + exit_code: 0, + timed_out: false, + stdout: JSON.stringify({ view: { format: "json", view: { value: 7 } } }), + stderr: "", +}; + +it("retains a completed view and retries an owned folder when cleanup fails", async () => { + let cleanups = 0; + const owner = new DesktopRemoteTaskReaders(async (argv) => { + if (argv[3] !== PYTHON_READER_CLEANUP) return success; + cleanups += 1; + return cleanups === 1 + ? { ...success, exit_code: 1, stderr: "permission denied" } + : success; + }); + const failure = await owner + .read([reader], new AbortController().signal) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ProviderCleanupError); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected cleanup failure"); + expect(failure.diagnostics).toMatchObject({ + completed_reader_views: { note: { format: "json", view: { value: 7 } } }, + }); + expect(owner.cleanupResources).toHaveLength(2); + expect(await owner.release()).toMatchObject({ verified: true }); + expect(owner.cleanupResources).toEqual([]); + expect(failure.cleanupIncomplete).toBe(false); +}); + +it("waits for an interrupted reader before removing its folder", async () => { + const controller = new AbortController(); + const calls: { requestId: string | undefined; cleanup: boolean }[] = []; + let finish: ((value: typeof success) => void) | undefined; + const owner = new DesktopRemoteTaskReaders( + async (argv, _timeout, _signal, requestId) => { + const cleanup = argv[3] === PYTHON_READER_CLEANUP; + calls.push({ requestId, cleanup }); + return cleanup + ? success + : new Promise((resolve) => { + finish = resolve; + }); + }, + ); + const read = owner + .read([reader], controller.signal) + .catch((error: unknown) => error); + await Promise.resolve(); + controller.abort(new Error("cancelled")); + const failure = await read; + expect(failure).toBeInstanceOf(ProviderCleanupError); + expect(owner.cleanupResources).toHaveLength(2); + const release = owner.release(); + await Promise.resolve(); + expect(calls.map(({ cleanup }) => cleanup)).toEqual([false]); + if (finish === undefined) throw new Error("reader did not start"); + finish(success); + expect(await release).toMatchObject({ verified: true }); + expect(calls.map(({ cleanup }) => cleanup)).toEqual([false, true]); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected cleanup failure"); + expect(failure.diagnostics).toMatchObject({ + completed_reader_views: { note: { view: { value: 7 } } }, + }); +}); + +it("preserves a completed view when cancellation arrives during successful cleanup", async () => { + const controller = new AbortController(); + const owner = new DesktopRemoteTaskReaders(async (argv) => { + if (argv[3] === PYTHON_READER_CLEANUP) + controller.abort(new Error("cancelled")); + return success; + }); + const failure = await owner + .read([reader], controller.signal) + .catch((error: unknown) => error); + if (!(failure instanceof AnalysisCancelledError)) + throw new Error("expected cancellation"); + expect(failure.diagnostics).toMatchObject({ + completed_reader_views: { note: { view: { value: 7 } } }, + }); + expect(owner.cleanupResources).toEqual([]); +}); + +it("keeps cancellation and a completed view when final cleanup also fails", async () => { + const controller = new AbortController(); + let fail = true; + const owner = new DesktopRemoteTaskReaders(async (argv) => { + if (argv[3] !== PYTHON_READER_CLEANUP) return success; + if (fail) { + fail = false; + controller.abort(new Error("cancelled")); + return { ...success, exit_code: 1, stderr: "denied" }; + } + return success; + }); + const failure = await owner + .read([reader], controller.signal) + .catch((error: unknown) => error); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected cleanup failure"); + expect(failure.cleanupPrimary).toBeInstanceOf(AnalysisCancelledError); + expect(failure.diagnostics).toMatchObject({ + completed_reader_views: { note: { view: { value: 7 } } }, + }); + expect(await owner.release()).toMatchObject({ verified: true }); +}); + +it("retains earlier views when a later reader's cleanup fails", async () => { + let cleanup = 0; + const owner = new DesktopRemoteTaskReaders(async (argv) => { + if (argv[3] !== PYTHON_READER_CLEANUP) return success; + cleanup += 1; + return cleanup === 2 + ? { ...success, exit_code: 1, stderr: "denied" } + : success; + }); + const failure = await owner + .read([reader, { ...reader, id: "second" }], new AbortController().signal) + .catch((error: unknown) => error); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected cleanup failure"); + expect(failure.diagnostics).toMatchObject({ + completed_reader_views: { + note: { view: { value: 7 } }, + second: { view: { value: 7 } }, + }, + }); + expect(await owner.release()).toMatchObject({ verified: true }); +}); + +it("retains an unknown transport outcome without starting the reader again", async () => { + let calls = 0; + const owner = new DesktopRemoteTaskReaders(async () => { + calls += 1; + throw new Error("connection lost"); + }); + await expect( + owner.read([reader], new AbortController().signal), + ).rejects.toBeInstanceOf(ProviderCleanupError); + expect(await owner.release()).toMatchObject({ verified: false }); + expect(calls).toBe(1); + expect(owner.cleanupResources).toHaveLength(2); +}); + +it("retains a timed-out runner's folder because its separate reader session may still run", async () => { + let calls = 0; + const owner = new DesktopRemoteTaskReaders(async () => { + calls += 1; + return { ...success, exit_code: null, timed_out: true }; + }); + await expect( + owner.read([reader], new AbortController().signal), + ).rejects.toBeInstanceOf(ProviderCleanupError); + expect(await owner.release()).toMatchObject({ verified: false }); + expect(calls).toBe(1); + expect(owner.cleanupResources).toHaveLength(2); +}); + +it("does not start a second cleanup while a timed-out cleanup may still be running", async () => { + let calls = 0; + const owner = new DesktopRemoteTaskReaders(async (argv) => { + if (argv[3] !== PYTHON_READER_CLEANUP) return success; + calls += 1; + return { ...success, exit_code: null, timed_out: true }; + }); + await expect( + owner.read([reader], new AbortController().signal), + ).rejects.toBeInstanceOf(ProviderCleanupError); + expect(await owner.release()).toMatchObject({ verified: false }); + expect(calls).toBe(1); + expect(owner.cleanupResources).toHaveLength(2); +}); diff --git a/src/application/desktop/DesktopRemoteTaskReaders.ts b/src/application/desktop/DesktopRemoteTaskReaders.ts new file mode 100644 index 0000000000..c172cfec10 --- /dev/null +++ b/src/application/desktop/DesktopRemoteTaskReaders.ts @@ -0,0 +1,300 @@ +import { randomUUID } from "node:crypto"; + +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import type { DesktopTaskReader } from "../../domain/desktop/appTask.js"; +import type { TaskFileView } from "../../domain/desktop/appTaskChecks.js"; +import { + PYTHON_READER_CLEANUP, + READER_RUNNER_SLACK_S, + readerAnswerView, + readerCommand, + readerTimeout, + type ReaderCommandResult, + type ReaderLease, +} from "../../domain/desktop/appTaskReaders.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { + AGENTENV_PROVIDER, + type ComputerRelease, +} from "../../desktop/AgentEnvComputerHelper.js"; +import { ABORTED, waitForAbortable } from "../../process/AbortablePromise.js"; + +type Exec = ( + argv: readonly string[], + timeoutS: number, + signal?: AbortSignal, + requestId?: string, +) => Promise; + +interface PendingReader { + readonly lease: ReaderLease; + readonly readerId: string; + readonly views: Map; + readonly absent: string[]; + readonly argv: readonly string[]; + readonly timeout: number; + readonly requestId: string; + settled: boolean; + command?: Promise; + uncertain?: unknown; + cleanupRequestId?: string; + cleanupUncertain?: string; +} + +/** Owns remote reader folders and joins interrupted commands before removing them. */ +export class DesktopRemoteTaskReaders { + readonly #pending = new Set(); + readonly #failures = new Set>(); + + constructor(private readonly exec: Exec) {} + + get cleanupResources(): readonly string[] { + return [...this.#pending].flatMap(({ lease }) => [ + lease.path, + `${lease.path}.owner`, + ]); + } + + async read( + readers: readonly DesktopTaskReader[], + signal: AbortSignal, + ): Promise> { + const views = new Map(); + const absent: string[] = []; + const prior = await this.release(); + if (!prior.verified) + throw this.#failure( + prior.reason, + this.#facts(views, absent), + undefined, + signal.aborted ? this.#cancelled(signal, views, absent) : undefined, + ); + for (const reader of readers) { + if (signal.aborted) throw this.#cancelled(signal, views, absent); + const token = randomUUID(); + const lease = { path: `/tmp/rea-reader-${token}`, token }; + const pending: PendingReader = { + lease, + readerId: reader.id, + views, + absent, + argv: readerCommand(reader, undefined, undefined, lease), + timeout: readerTimeout(reader) + READER_RUNNER_SLACK_S, + requestId: `rea-reader-${token}`, + settled: false, + }; + this.#pending.add(pending); + let result: ReaderCommandResult; + try { + pending.command = this.exec( + pending.argv, + pending.timeout, + undefined, + pending.requestId, + ); + const observed = await waitForAbortable(pending.command, signal); + if (observed === ABORTED) throw signal.reason; + result = observed; + if (result.timed_out) { + pending.uncertain = new Error( + "The remote runner timed out; its reader may still be running.", + ); + delete pending.command; + throw pending.uncertain; + } + pending.settled = true; + } catch (cause: unknown) { + if (!signal.aborted) { + pending.uncertain = cause; + delete pending.command; + } + throw this.#failure( + "The reader command has not been confirmed finished.", + this.#facts(views, absent), + cause, + signal.aborted ? this.#cancelled(signal, views, absent) : undefined, + ); + } + let failure: unknown; + try { + if (result.exit_code !== 0 || result.timed_out) + throw new Error( + `REA's reader runner for ${reader.id} ${result.timed_out ? "timed out" : `exited ${String(result.exit_code)}`}: ${result.stderr.slice(-1024) || result.stdout.slice(-1024)}`, + ); + const view = readerAnswerView(result.stdout); + if (view === undefined) absent.push(reader.id); + else views.set(reader.id, view); + } catch (cause: unknown) { + failure = cause; + } + const cleanup = await this.release(); + if (!cleanup.verified) + throw this.#failure( + cleanup.reason, + this.#facts(views, absent), + failure, + signal.aborted ? this.#cancelled(signal, views, absent) : undefined, + ); + if (signal.aborted) throw this.#cancelled(signal, views, absent); + if (failure !== undefined) throw failure; + } + return views; + } + + /** Join a possibly interrupted command, then remove only the folder it owns. */ + async release(): Promise { + const errors: string[] = []; + for (const pending of this.#pending) { + try { + if (!pending.settled) { + if (pending.command === undefined) + throw ( + pending.uncertain ?? + new Error("the reader command's outcome is unknown") + ); + try { + const result = await pending.command; + if (result.timed_out) + throw new Error( + "The remote runner timed out; its reader may still be running.", + ); + if (result.exit_code === 0 && !result.timed_out) { + try { + const view = readerAnswerView(result.stdout); + if (view === undefined) pending.absent.push(pending.readerId); + else pending.views.set(pending.readerId, view); + for (const reference of this.#failures) + reference + .deref() + ?.retainDiagnostics( + this.#facts(pending.views, pending.absent), + ); + } catch (cause: unknown) { + for (const reference of this.#failures) + reference.deref()?.retainDiagnostics({ + late_reader_failure: + cause instanceof Error ? cause.message : String(cause), + }); + } + } else { + for (const reference of this.#failures) + reference.deref()?.retainDiagnostics({ + late_reader_failure: result.timed_out + ? "The reader timed out." + : `The reader exited ${String(result.exit_code)}: ${result.stderr.slice(-1024)}`, + }); + } + } catch (cause: unknown) { + pending.uncertain = cause; + delete pending.command; + throw cause; + } + pending.settled = true; + delete pending.command; + } + if (pending.cleanupUncertain !== undefined) + throw new Error(pending.cleanupUncertain); + pending.cleanupRequestId ??= `rea-reader-cleanup-${randomUUID()}`; + const result = await this.exec( + [ + "python3", + "-I", + "-c", + PYTHON_READER_CLEANUP, + JSON.stringify(pending.lease), + ], + READER_RUNNER_SLACK_S, + undefined, + pending.cleanupRequestId, + ); + if (result.timed_out) { + pending.cleanupUncertain = + "The remote cleanup timed out and may still be running."; + throw new Error(pending.cleanupUncertain); + } + if (result.exit_code !== 0) { + delete pending.cleanupRequestId; + throw new Error( + result.stderr.slice(-1024) || + "the cleanup command did not complete", + ); + } + this.#pending.delete(pending); + } catch (cause: unknown) { + errors.push(cause instanceof Error ? cause.message : String(cause)); + } + } + if (errors.length > 0) + return { + verified: false, + reason: errors.join("; "), + resources: this.cleanupResources, + }; + for (const failure of this.#failures) failure.deref()?.resolveCleanup(); + this.#failures.clear(); + return { verified: true, teardown: "agentenv" }; + } + + #facts( + views: ReadonlyMap, + absent: readonly string[], + ): Readonly> { + return { + completed_reader_views: Object.fromEntries( + [...views].map(([id, view]) => [ + id, + { + format: view.format, + view: view.view, + ...(view.error === undefined ? {} : { error: view.error }), + ...(view.compression === undefined + ? {} + : { compression: view.compression }), + }, + ]), + ), + completed_reader_absent: [...absent], + }; + } + + #cancelled( + signal: AbortSignal, + views: ReadonlyMap, + absent: readonly string[], + ) { + return new AnalysisCancelledError("check_build_task", { + cause: signal.reason, + }).retainDiagnostics(this.#facts(views, absent)); + } + + #failure( + reason: string, + facts: Readonly>, + cause?: unknown, + primary?: AnalysisCancelledError, + ) { + const failure = new ProviderCleanupError( + AGENTENV_PROVIDER, + this.cleanupResources, + { + reason, + reader_requests: [...this.#pending].map(({ requestId, lease }) => ({ + request_id: requestId, + folder: lease.path, + })), + ...facts, + }, + { + operation: "check_build_task", + cause, + ...(primary === undefined ? {} : { cleanupPrimary: primary }), + remediationAction: + "Retry the check or close the build computer to retry cleanup. If the command outcome is unknown, inspect the reported computer and confirm the reader has stopped before removing its folder.", + }, + ); + failure.bindCleanupResourceOwner(this); + this.#failures.add(new WeakRef(failure)); + return failure; + } +} diff --git a/src/application/desktop/DesktopRetainedContent.cleanup.test.ts b/src/application/desktop/DesktopRetainedContent.cleanup.test.ts new file mode 100644 index 0000000000..49109c7add --- /dev/null +++ b/src/application/desktop/DesktopRetainedContent.cleanup.test.ts @@ -0,0 +1,72 @@ +import { expect, it, vi } from "vitest"; +import type { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +import { processWorkerCleanup } from "../../process/OwnedWorkerRuntime.js"; +import type { ProviderProcessSupervisor } from "../../process/ProviderProcess.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { + RetainedContent, + withRetainedContent, +} from "./DesktopRetainedContent.js"; +import { retainReaderCleanup } from "./DesktopTaskReaderCleanup.js"; + +it("retains both reader and capture-file cleanup until both owners release", async () => { + const folder = await createTestTempDirectory("reader-mixed-cleanup-"); + const process = { + stop: vi.fn().mockResolvedValue({ status: "stopped" }), + dispose: vi.fn(), + } as unknown as ProviderProcessSupervisor; + const original = retainReaderCleanup( + folder, + new OwnedCommandFailure( + "cancelled", + "cancelled", + null, + "running", + { cancellation: { reason: "stop" } }, + ["reader-run"], + process, + ), + { views: new Map(), absent: [] }, + ); + if (original === undefined) throw new Error("Expected retained reader"); + const close = vi + .fn() + .mockRejectedValueOnce(new Error("file close failed")) + .mockResolvedValue(undefined); + let fileLease: AnalysisCleanupLease | undefined; + const spy = vi + .spyOn(RetainedContent.prototype, "release") + .mockImplementation(async (scope) => { + const attempt = await scope.release({ + kind: "private-runtime-root", + resource: "capture-file", + root: { path: "capture-file", close }, + }); + if (attempt.kind !== "failed") + throw new Error("Expected failed file close"); + fileLease = attempt.lease; + return [attempt]; + }); + try { + const combined: unknown = await withRetainedContent("unused", async () => { + throw original; + }).catch((cause: unknown) => cause); + expect(combined).toBeInstanceOf(AnalysisError); + const failure = combined as AnalysisError; + expect(failure.cleanupResources).toEqual( + expect.arrayContaining(["reader-run", folder, "capture-file"]), + ); + await processWorkerCleanup.retry(); + expect(failure.cleanupIncomplete).toBe(true); + if (fileLease === undefined) throw new Error("Missing retained file lease"); + await fileLease.release(); + expect(failure.cleanupIncomplete).toBe(false); + expect(close).toHaveBeenCalledTimes(2); + } finally { + spy.mockRestore(); + await processWorkerCleanup.retry(); + await fileLease?.release(); + } +}); diff --git a/src/application/desktop/DesktopRetainedContent.ts b/src/application/desktop/DesktopRetainedContent.ts index 8d689c1fd6..a24bbddead 100644 --- a/src/application/desktop/DesktopRetainedContent.ts +++ b/src/application/desktop/DesktopRetainedContent.ts @@ -1,11 +1,14 @@ import { createHash } from "node:crypto"; import { join } from "node:path"; +import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js"; import { ArtifactResourceScope, type ArtifactCleanupAttempt, } from "../../artifacts/ArtifactResourceScope.js"; import { err, ok, type Result } from "../../domain/result.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; import { RegularFileCleanupFailure, @@ -120,8 +123,33 @@ export const withRetainedContent = async ( } const resources = new ArtifactResourceScope(); const [first, ...rest] = await content.release(resources); - if (first !== undefined) - throw resources.cleanupFailure(outcome, [first, ...rest]); + if (first !== undefined) { + const failures = [first, ...rest] as const; + if ( + outcome.kind === "failed" && + outcome.cause instanceof ProviderCleanupError + ) { + const fileFailure = resources.cleanupFailure( + { kind: "completed" }, + failures, + ); + const fileCleanup = new ProviderCleanupError( + "rea-desktop", + fileFailure.cleanup?.resources ?? [], + { reason: fileFailure.cleanup?.reason ?? fileFailure.message }, + { operation: outcome.cause.operation, cause: fileFailure }, + ); + bindArtifactFileCleanup(fileFailure, fileCleanup); + const combined = analysisErrorWithCleanupFailure( + outcome.cause, + fileCleanup, + outcome.cause.operation, + ); + combined.bindCleanupDependencies([outcome.cause, fileCleanup]); + throw combined; + } + throw resources.cleanupFailure(outcome, failures); + } if (outcome.kind === "failed") throw outcome.cause; return outcome.value; }; diff --git a/src/application/desktop/DesktopTaskExport.test.ts b/src/application/desktop/DesktopTaskExport.test.ts index 79fcb6f122..bedc9d0e56 100644 --- a/src/application/desktop/DesktopTaskExport.test.ts +++ b/src/application/desktop/DesktopTaskExport.test.ts @@ -1,4 +1,4 @@ -import { access, readFile, writeFile } from "node:fs/promises"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -9,11 +9,13 @@ import { exportDesktopApplication } from "./DesktopApplicationExport.js"; import { namesApp, shippedSeedCheckView } from "./DesktopTaskExport.js"; import { draftDesktopApplicationTask } from "./DesktopTaskDraft.js"; import { + HOME, NOTE as NOTE_PATH, capture, change, readers, retained, + seedEndCapture, writeCaptureFolder, } from "./desktopCapture.fixture.js"; @@ -128,6 +130,87 @@ const longNoteCapture = () => { }; }; +/** The seed-and-end capture with the app's log, which the seed leaves out, holding `error A` at the seed and not at the end. */ +const LOG = `${HOME}/.config/notes/notes.log`; +const loggedCapture = () => { + const base = seedEndCapture(); + return { + ...base, + state: { + ...base.state, + checkpoints: base.state.checkpoints.map((checkpoint) => + checkpoint.name === "seed" || checkpoint.name === "end" + ? { + ...checkpoint, + changes: [ + ...checkpoint.changes, + checkpoint.name === "seed" + ? change(LOG, { after: "error A\n" }, { state: true }) + : change( + LOG, + { before: "error A\n", after: "ok\n" }, + { state: true }, + ), + ], + retained: [ + ...checkpoint.retained, + retained( + LOG, + checkpoint.name === "seed" ? "error A\n" : "ok\n", + ), + ], + } + : checkpoint, + ), + }, + }; +}; + +describe("exportDesktopApplication with the seed it ships", () => { + it("checks a retained application log against the seed the bundle ships", async () => { + const scratch = await createTestTempDirectory("desktop-task-"); + const path = join(scratch, "task.json"); + await writeFile( + path, + JSON.stringify( + task([ + { + id: "no-error", + kind: "lines_absent", + file: ".config/notes/notes.log", + lines: ["error A"], + }, + ]), + ), + ); + const directory = join(scratch, "bundle"); + + const result = await exportDesktopApplication( + { + capture: await writeCaptureFolder(loggedCapture(), undefined, [ + "milk\neggs\n", + "error A\n", + "ok\n", + ]), + directory, + name: undefined, + tasks: [path], + seedCheckpoint: "seed", + endCheckpoint: "end", + }, + readers, + ); + + if (result instanceof Error) throw result; + expect( + await readFile( + join(directory, "envs/notes/seed/.config/notes/notes.log"), + "utf8", + ), + ).toBe("error A\n"); + }); +}); + describe("exportDesktopApplication with a task's seed", () => { it("refuses a task whose only seed failure misses a line its cut view may have left out", async () => { const scratch = await createTestTempDirectory("desktop-task-"); @@ -387,6 +470,310 @@ describe("exportDesktopApplication with a task", () => { }); }); +/** A reader of the notes database's last field, which the edit changes from v1 to v2, as an app's headless mode would print it. */ +const VERSION_READER = { + id: "version", + file: DATABASE, + argv: [ + "python3", + "-c", + "import sys; print('REA version ' + open(sys.argv[1], 'rb').read().split(b'\\0')[-1].decode())", + "{file}", + ], + output: "lines", +}; +const readerTask = (lines: string[], reader: object = VERSION_READER) => ({ + ...task([ + { + id: "version", + kind: "lines_present", + file: DATABASE, + reader: "version", + lines, + }, + ]), + readers: [reader], +}); + +describe("exportDesktopApplication with a task's readers", () => { + it("runs a reader on the seed as the bundle ships it and on the end, keeps the task and ships the reader", async () => { + const { result, directory } = await exportWithTasks( + readerTask(["version v2"]), + ); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + task: { name: "note-milk", checks: 1, failing_on_seed: ["version"] }, + }); + const verifier = await readFile( + join(directory, "artifacts/note-milk-check/verify.py"), + "utf8", + ); + expect(verifier).toContain('"reader": "version"'); + const readerLine = verifier.match( + /^\s*(\{"id": "version", "arguments": .*\}),$/mu, + ); + expect(readerLine).not.toBeNull(); + const shippedReader = JSON.parse(readerLine?.[1] ?? "null") as { + arguments: string[]; + }; + expect(shippedReader.arguments.length).toBeGreaterThan(0); + expect( + shippedReader.arguments.every((part) => typeof part === "string"), + ).toBe(true); + expect(JSON.parse(shippedReader.arguments.join(""))).toMatchObject({ + id: "version", + path: DATABASE, + argv: VERSION_READER.argv, + output: VERSION_READER.output, + }); + const readme = await readFile(join(directory, "README.md"), "utf8"); + expect(readme).toContain("Readers"); + expect(readme).toContain("`version` reads `.local/share/notes/notes.db`"); + }); + + it.each([ + [ + readerTask(["version v3"]), + 'Check "version" fails on the captured end state', + 'reader version printed: "version v2"', + ], + [ + readerTask(["version v2"], { + ...VERSION_READER, + argv: ["sh", "-c", "echo 'cannot open {file}' >&2; exit 4"], + }), + "the reader exited 4", + "cannot open", + ], + [ + { + ...readerTask([]), + checks: [ + { + kind: "lines_absent", + file: DATABASE, + reader: "version", + lines: ["version v3"], + }, + ], + }, + "Every check already passes on the seed", + "", + ], + ])( + "refuses %j with what its reader read and writes no bundle", + async (input, message, detail) => { + const { result, directory } = await exportWithTasks(input); + + expect(result).toBeInstanceOf(Error); + const { userMessage } = result as Error & { userMessage: string }; + expect(userMessage).toContain(message); + expect(userMessage).toContain(detail); + expect(await exists(directory)).toBe(false); + }, + ); + + it("refuses a reader of a file outside the app's owned paths", async () => { + const { result } = await exportWithTasks({ + ...task([ + { + kind: "lines_present", + file: ".cache/notes/x", + reader: "version", + lines: ["x"], + }, + ]), + readers: [{ ...VERSION_READER, file: ".cache/notes/x" }], + }); + expect(result).toBeInstanceOf(Error); + expect((result as Error & { userMessage: string }).userMessage).toContain( + "not under the app's owned paths", + ); + }); +}); + +/** The notes capture with nothing changed between its checkpoints, as an app that keeps its state on screen leaves it. */ +const screenOnlyCapture = () => { + const base = capture(); + return { + ...base, + state: { + ...base.state, + checkpoints: base.state.checkpoints.map((checkpoint) => + checkpoint.trigger === "baseline" + ? checkpoint + : { ...checkpoint, changes: [], retained: [] }, + ), + }, + }; +}; + +const SCREEN_TASK = { + name: "note-shown", + prompt: "In Notes, show the shopping list.", + checks: [ + { + id: "shown", + kind: "screen_judge", + question: "Does the window show the shopping list?", + }, + ], +}; + +/** Export `task` from a capture with a screenshot of each named checkpoint, as a build computer keeps them. */ +const exportScreenTask = async ( + task: unknown, + base = screenOnlyCapture(), + shots: Readonly> = { + baseline: "seed shot", + edited: "end shot", + }, +) => { + const scratch = await createTestTempDirectory("desktop-screen-"); + const screens = join(scratch, "screens"); + await mkdir(screens); + for (const [name, text] of Object.entries(shots)) + await writeFile(join(screens, `${name}.jpg`), text); + const path = join(scratch, "task.json"); + await writeFile(path, JSON.stringify(task)); + const directory = join(scratch, "bundle"); + const result = await exportDesktopApplication( + { + capture: await writeCaptureFolder(base), + directory, + name: undefined, + tasks: [path], + screens, + }, + readers, + ); + return { result, directory }; +}; + +/** The screen-only capture where a file outside the app's owned paths, a desktop's recent-files list, changed. */ +const unownedChangeCapture = () => { + const base = screenOnlyCapture(); + const recent = NOTE_PATH.replace( + "Documents/today.txt", + ".local/share/recently-used.xbel", + ); + return { + ...base, + state: { + ...base.state, + checkpoints: base.state.checkpoints.map((checkpoint) => + checkpoint.trigger === "baseline" + ? checkpoint + : { + ...checkpoint, + changes: [change(recent, { after: "milk\n" })], + retained: [retained(recent, "milk\n")], + }, + ), + }, + }; +}; + +describe("exportDesktopApplication with a screen-judged task", () => { + it.each([false, true])( + "compares captured log views rather than seed omissions (changed: %s)", + async (changed) => { + const base = screenOnlyCapture(); + const log = `${HOME}/.config/notes/notes.log`; + const before = "milk\n"; + const after = changed ? "[ui]\nwrap=true\n" : before; + const captureWithLog = { + ...base, + state: { + ...base.state, + checkpoints: base.state.checkpoints.map((checkpoint) => + checkpoint.trigger === "baseline" + ? { + ...checkpoint, + retained: [...checkpoint.retained, retained(log, before)], + } + : changed + ? { + ...checkpoint, + changes: [change(log, { before, after }, { state: true })], + retained: [retained(log, after)], + } + : checkpoint, + ), + }, + }; + const { result } = await exportScreenTask(SCREEN_TASK, captureWithLog); + if (changed) { + expect(result).toBeInstanceOf(AnalysisInputError); + expect((result as AnalysisInputError).userMessage).toContain( + "a check can read", + ); + } else if (result instanceof Error) throw result; + }, + ); + + it("lets a file outside the app's owned paths change without refusing the screen judge", async () => { + const { result } = await exportScreenTask( + SCREEN_TASK, + unownedChangeCapture(), + ); + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + task: { name: "note-shown", failing_on_seed: [] }, + }); + }); + + it("ships the seed's screenshot as the judge's failing example and the end's as its passing one, and leaves judging to the witness", async () => { + const { result, directory } = await exportScreenTask(SCREEN_TASK); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + task: { name: "note-shown", checks: 1, failing_on_seed: [] }, + }); + expect( + (result.normalized_result as { limitations: string[] }).limitations.some( + (line) => line.includes("witness") && line.includes("judge"), + ), + ).toBe(true); + const verifier = await readFile( + join(directory, "artifacts/note-shown-check/verify.py"), + "utf8", + ); + expect(verifier).toContain(Buffer.from("seed shot").toString("base64")); + expect(verifier).toContain(Buffer.from("end shot").toString("base64")); + expect( + verifier.indexOf(Buffer.from("seed shot").toString("base64")), + ).toBeLessThan( + verifier.indexOf(Buffer.from("end shot").toString("base64")), + ); + const readme = await readFile(join(directory, "README.md"), "utf8"); + expect(readme).toContain("Does the window show the shopping list?"); + expect(readme).toContain("--judge-model"); + }); + + it.each([ + [SCREEN_TASK, capture(), undefined, "a check can read"], + [SCREEN_TASK, screenOnlyCapture(), { baseline: "seed shot" }, "screenshot"], + [ + SCREEN_TASK, + screenOnlyCapture(), + { baseline: "x".repeat(2 * 1024 * 1024 + 1), edited: "end shot" }, + "larger than the 2 MiB", + ], + ])( + "refuses a screen judge where a file shows the change or a screenshot is missing (%#)", + async (task, base, shots, message) => { + const { result, directory } = await exportScreenTask(task, base, shots); + expect(result).toBeInstanceOf(Error); + expect((result as Error & { userMessage: string }).userMessage).toContain( + message, + ); + expect(await exists(directory)).toBe(false); + }, + ); +}); + describe("exportDesktopApplication with a task's prompt", () => { it("names a prompt that does not name the app in its limitations, since a solver's shell cannot see the computer", async () => { const checks = [ diff --git a/src/application/desktop/DesktopTaskExport.ts b/src/application/desktop/DesktopTaskExport.ts index 320c1b2a5d..c2e3e35702 100644 --- a/src/application/desktop/DesktopTaskExport.ts +++ b/src/application/desktop/DesktopTaskExport.ts @@ -4,6 +4,7 @@ import { parseDesktopTask, type DesktopTask, type DesktopTaskCheck, + type DesktopTaskReader, } from "../../domain/desktop/appTask.js"; import { evaluateTaskChecks, @@ -18,6 +19,13 @@ import { taskVerifierArtifact, } from "../../domain/desktop/appTaskPython.js"; import type { DesktopAppExportPlan } from "../../domain/desktop/appExport.js"; +import { + DEFAULT_JUDGE_MODEL, + SCREEN_EXAMPLE_MAX_BYTES, + SCREEN_JUDGE_VOTES, + isScreenTask, + readableChanges, +} from "../../domain/desktop/appTaskScreen.js"; import { STATE_PATH_INPUTS } from "../../domain/desktop/buildCapture.js"; import { versionAt, @@ -59,6 +67,15 @@ type TaskTarget = Pick< readonly seedViews: ReadonlyMap; /** Home paths the bundle actually ships, including files that are not text. */ readonly seeded: ReadonlySet; + /** Each reader's view of its file, by reader id: at the seed as the bundle ships it, or at the end. */ + readonly readerViews: ( + readers: readonly DesktopTaskReader[], + end: keyof StateEnds, + ) => Promise>; + /** The screenshot a build computer took at the seed or the end, too large to ship, or undefined when there is none. */ + readonly screens: ( + end: keyof StateEnds, + ) => Promise; }; /** A task the capture validated: the files it adds to the bundle, its README section and its summary. */ @@ -70,6 +87,8 @@ export interface AdmittedTask { readonly readme: string; /** What the export says about the task, such as a prompt that does not name the app. */ readonly limitations: readonly string[]; + /** Whether the task judges the screen, whose app then resets by relaunching when it owns no files. */ + readonly screenJudged?: boolean; readonly summary: { readonly name: string; readonly checks: number; @@ -129,6 +148,50 @@ interface TaskViews { readonly end: ReadonlyMap; } +/** What a capture's identity check reads of a value: the captured home, or the host's name or user, if it names one. */ +const identityProblem = ( + values: readonly string[], + target: TaskTarget, +): string | undefined => { + for (const value of values) { + if (value.includes(target.capturedHome)) + return `names the captured home ${target.capturedHome}, which the computer's home replaces`; + const named = identityLeftIn(value, target.identity); + if (named !== undefined) + return `names ${named}, the capture host's name or user, which the export removes from the seed`; + } + return undefined; +}; + +/** Why a reader cannot run in this app's computer, or undefined. */ +const readerProblem = ( + reader: DesktopTaskReader, + target: TaskTarget, +): string | undefined => + isWithinAny(reader.file, target.paths) + ? identityProblem([...reader.argv, reader.stdin ?? ""], target) + : `reads ${reader.file}, which is not under the app's owned paths (${target.paths.join(", ") || "none"}); data/get serves only those. For the bundle to own it, capture again naming it with ${STATE_PATH_INPUTS}`; + +/** What a reader read for a failing check, for the author to fix the check from, or nothing. */ +const readerRead = ( + check: DesktopTaskCheck, + readers: ReadonlyMap, +): string => { + if (!("reader" in check) || check.reader === undefined) return ""; + const read = readers.get(check.reader); + if (read === undefined || read.view === null) return ""; + const view = read.view; + const lines = + read.format === "text" && + typeof view === "object" && + !Array.isArray(view) && + Array.isArray(view.lines) + ? view.lines.map((line) => JSON.stringify(line)).join(", ") + : JSON.stringify(view); + const shown = lines.length > 600 ? `${lines.slice(0, 600)}…` : lines; + return `; reader ${check.reader} printed: ${shown}`; +}; + /** Every string in a value, member names included. */ const valueStrings = (value: JsonValue | undefined): string[] => { if (typeof value === "string") return [value]; @@ -146,18 +209,23 @@ const targetProblem = ( target: TaskTarget, views: TaskViews, ): string | undefined => { + if (check.kind === "screen_judge") + return identityProblem([check.question], target); if (!isWithinAny(check.file, target.paths)) return `names ${check.file}, which is not under the app's owned paths (${target.paths.join(", ") || "none"}); data/get serves only those. For the bundle to own it, capture again naming it with ${STATE_PATH_INPUTS}`; // What the check compares and names; its field names, kind and id label it. const { id: _id, kind: _kind, ...fields } = check; - for (const value of Object.values(fields).flatMap(valueStrings)) { - if (value.includes(target.capturedHome)) - return `names the captured home ${target.capturedHome}, which the computer's home replaces`; - const named = identityLeftIn(value, target.identity); - if (named !== undefined) - return `names ${named}, the capture host's name or user, which the export removes from the seed`; - } - if (!isPointerCheck(check)) return undefined; + const named = identityProblem( + Object.values(fields).flatMap(valueStrings), + target, + ); + if (named !== undefined) return named; + // A reader's view is JSON or lines whatever the file's own format. + if ( + !isPointerCheck(check) || + ("reader" in check && check.reader !== undefined) + ) + return undefined; return [views.seed, views.end].some( (at) => at.get(check.file)?.format === "text", ) @@ -176,31 +244,51 @@ export const shippedSeedCheckView = ( captured: TaskFileView | undefined, ): TaskFileView | undefined => (seeded ? (shipped ?? captured) : undefined); -/** Views of every file at the end checkpoint, or at the seed as the bundle ships it. */ -const viewsAt = ( +/** The captured views at a checkpoint, before export omissions and rewrites. */ +const capturedViewsAt = ( target: TaskTarget, end: keyof StateEnds, ): ReadonlyMap => { const views = new Map(); for (const [home, path] of target.files) { const version = versionAt(path, target.ends[end]); - // A file not retained at the seed is not seeded, so the computer starts without it. - if (end === "seed" && version?.present === true && version.sha256 === null) - continue; - const captured = version === undefined ? undefined : versionView(version); - const view = - end === "seed" - ? shippedSeedCheckView( - target.seeded.has(home), - target.seedViews.get(home), - captured, - ) - : captured; + const view = version === undefined ? undefined : versionView(version); if (view !== undefined) views.set(home, view); } return views; }; +/** Apply the bundle's omissions and rewritten files to its captured seed. */ +const shippedSeedViews = ( + target: TaskTarget, + captured: ReadonlyMap, +): ReadonlyMap => { + const views = new Map(); + for (const [home, path] of target.files) { + const version = versionAt(path, target.ends.seed); + // A file not retained at the seed is not seeded, so the computer starts without it. + if (version?.present === true && version.sha256 === null) continue; + const view = shippedSeedCheckView( + target.seeded.has(home), + target.seedViews.get(home), + captured.get(home), + ); + if (view !== undefined) views.set(home, view); + } + return views; +}; + +/** The README's list of a task's readers, if it has any. */ +const readersReadme = (task: DesktopTask): string => + task.readers === undefined + ? "" + : ` +Readers, each run in the computer on a private copy of its file in a fresh home, without the display unless it asks +for it; the lines it prints starting with \`REA \` are its view: + +${task.readers.map((reader) => `- \`${reader.id}\` reads \`${reader.file}\` as ${reader.output}: \`${JSON.stringify(reader.argv)}\`${reader.stdin === undefined ? "" : `, with stdin \`${JSON.stringify(reader.stdin)}\``}`).join("\n")} +`; + const taskReadme = ( task: DesktopTask, app: string, @@ -209,7 +297,7 @@ const taskReadme = ( ## Task \`${task.name}\` \`tasks/${task.name}.json\` deploys the app, has the agent work through the computer from the prompt below, and grades -the outcome with \`artifacts/${taskVerifierArtifact(task)}/verify.py\`, which reads the app's \`data/get\` views and needs +the outcome with \`artifacts/${taskVerifierArtifact(task)}/verify.py\`, which reads the app's \`data/get\` views${task.readers === undefined ? "" : " and runs its readers in the computer"} and needs no model. It grades once the app's files have held for ${String(SETTLE_TIMING.windowMs / 1000)} s, so a save the app makes on its own within that time of the agent stopping counts; each result's \`settled\` is false when they were still changing ${String(SETTLE_TIMING.boundMs / 1000)} s after the first read. \`evals/${app}.toml\` names it, so a run of the bundle runs it. @@ -222,7 +310,7 @@ ${task.prompt Checks, each of which must pass: ${task.checks.map(({ id, ...check }) => `- \`${id}\`: \`${JSON.stringify(check)}\``).join("\n")} - +${readersReadme(task)} REA kept the task because its checks fail on the seed (${failingOnSeed.map((id) => `\`${id}\``).join(", ")}) and all pass on the captured end state. @@ -232,6 +320,138 @@ whose summary line says \`passed\`; a run that says \`failed\` did not finish an agent-env run --task ${task.name} --agent claude-code --model MODEL `; +/** Refuse a check or reader this app's computer cannot serve, naming it. */ +const refuseTargetProblems = ( + task: DesktopTask, + target: TaskTarget, + views: TaskViews, +): void => { + for (const check of task.checks) { + const problem = targetProblem(check, target, views); + if (problem !== undefined) + throw refuse( + `Check ${JSON.stringify(check.id)} ${problem}.`, + "task", + REWRITE, + ); + } + for (const reader of task.readers ?? []) { + const problem = readerProblem(reader, target); + if (problem !== undefined) + throw refuse( + `Reader ${JSON.stringify(reader.id)} ${problem}.`, + "task", + REWRITE, + ); + } +}; + +/** The readers' views at the end, as retained, and at the seed, as the bundle ships it: the app's own headless mode on those bytes. */ +const readersAtEnds = async ( + task: DesktopTask, + target: TaskTarget, +): Promise => { + const readers = task.readers ?? []; + if (readers.length === 0) return { seed: new Map(), end: new Map() }; + const end = await target.readerViews(readers, "end"); + return { seed: await target.readerViews(readers, "seed"), end }; +}; + +/** The README's section on a screen-judged task: its questions and how its judge is configured. */ +const screenReadme = (task: DesktopTask, app: string): string => ` +## Task \`${task.name}\` + +\`tasks/${task.name}.json\` deploys the app, has the agent work through the computer from the prompt below, and grades +the outcome with \`artifacts/${taskVerifierArtifact(task)}/verify.py\`, which takes a screenshot of the computer once the +agent stops and asks a model, the judge, each question below ${String(SCREEN_JUDGE_VOTES)} times; a check passes when most answers are yes. The +judge sees two labelled screenshots from the build first: the app at the seed, where the answer is no, and with the +task done, where it is yes. \`evals/${app}.toml\` names the task, so a run of the bundle runs it. + +${task.prompt + .split("\n") + .map((line) => `> ${line}`) + .join("\n")} + +Questions, each of which must be answered yes: + +${task.checks.map((check) => `- \`${check.id}\`: ${check.kind === "screen_judge" ? check.question : ""}`).join("\n")} + +The judge's model is \`[model.roles] judge\` in the runner's \`.agentenv/config.toml\`, else its \`[model] default\`, else +\`${DEFAULT_JUDGE_MODEL}\`, called through \`LITELLM_BASE_URL\` and \`LITELLM_API_KEY\` or \`[model] base_url\` and \`api_key\`. +\`agent-env run --judge-model\` does not reach it. Without an endpoint the verifier raises, so the run fails rather than +scoring the agent. The export judged no screen; REA's witness ran this judge on a fresh screenshot at each stage. + +To measure a solve rate, run the task k times (3 by default) and count the runs whose summary line says \`passed\`: + + agent-env run --task ${task.name} --agent claude-code --model MODEL +`; + +/** Admit a screen-judged task: no file shows the change, and the build's seed and end screenshots ship as the judge's examples. */ +const admitScreenTask = async ( + task: DesktopTask, + target: TaskTarget, + views: TaskViews, +): Promise => { + // Only a file the app owns could take a check instead; the export refuses the rest. + const owned = (all: ReadonlyMap) => + new Map([...all].filter(([file]) => isWithinAny(file, target.paths))); + const changed = readableChanges(owned(views.seed), owned(views.end)); + if (changed.length > 0) + throw refuse( + `A screen_judge judges state no file shows, but ${changed.join(", ")} changed between the seed and the end in a way a check can read.`, + "task", + `Check what changed in those files instead; ${DRAFT}.`, + ); + const [seed, end] = await Promise.all([ + target.screens("seed"), + target.screens("end"), + ]); + if (seed === undefined || end === undefined) + throw refuse( + `A screen_judge needs the screenshots a build computer takes at the seed and end checkpoints, its judge's examples, and the capture has no ${seed === undefined ? "seed" : "end"} screenshot.`, + "task", + "Capture the app in a build computer, marking seed and end there, and export from it.", + ); + if (seed === "too-large" || end === "too-large") + throw refuse( + `The ${seed === "too-large" ? "seed" : "end"} screenshot is larger than the ${String(SCREEN_EXAMPLE_MAX_BYTES / 1024 / 1024)} MiB a judge's example may take.`, + "task", + "Capture again with a simpler screen at that checkpoint.", + ); + const examples = { + fail: seed.toString("base64"), + pass: end.toString("base64"), + }; + return { + screenJudged: true, + summary: { + name: task.name, + checks: task.checks.length, + failing_on_seed: [], + }, + limitations: [ + ...(namesApp(task.prompt, target) + ? [] + : [ + `The task's prompt does not name the app (${target.title}), so start it with "In ${target.title}, ".`, + ]), + "The export judges no screen, so it did not check the screen_judge questions on the seed and the end: the witness does, running the bundle's own judge on a fresh screenshot at each stage, which needs a model endpoint.", + ], + readme: screenReadme(task, target.name), + files: [ + { + path: `tasks/${task.name}.json`, + content: renderTaskSteps(task, target.name), + }, + { + path: `artifacts/${taskVerifierArtifact(task)}/verify.py`, + content: renderTaskVerifier(task, target.name, examples), + }, + { path: `evals/${target.name}.toml`, content: renderTaskEval([task]) }, + ], + }; +}; + /** Read, validate and render the task a caller passed with the export, if any. */ /** Whether a prompt names the app by its title, environment or package, in any letter case. */ export const namesApp = ( @@ -275,28 +495,28 @@ export const admitDesktopTask = async ( REWRITE, ); const task = parsed.value; - const views: TaskViews = { - seed: viewsAt(target, "seed"), - end: viewsAt(target, "end"), + const captured: TaskViews = { + seed: capturedViewsAt(target, "seed"), + end: capturedViewsAt(target, "end"), }; - for (const check of task.checks) { - const problem = targetProblem(check, target, views); - if (problem !== undefined) - throw refuse( - `Check ${JSON.stringify(check.id)} ${problem}.`, - "task", - REWRITE, - ); - } - const end = evaluateTaskChecks(task.checks, views.end); + const views: TaskViews = { + seed: shippedSeedViews(target, captured.seed), + end: captured.end, + }; + refuseTargetProblems(task, target, views); + if (isScreenTask(task)) return admitScreenTask(task, target, captured); + const read = await readersAtEnds(task, target); + const end = evaluateTaskChecks(task.checks, views.end, read.end); const failing = end.find((outcome) => !outcome.passed); - if (failing !== undefined) + if (failing !== undefined) { + const check = task.checks.find(({ id }) => id === failing.id); throw refuse( - `Check ${JSON.stringify(failing.id)} fails on the captured end state: ${failing.description}.`, + `Check ${JSON.stringify(failing.id)} fails on the captured end state: ${failing.description}${check === undefined ? "" : readerRead(check, read.end)}.`, "task", `Write checks that the captured end state passes; ${DRAFT}.`, ); - const seed = evaluateTaskChecks(task.checks, views.seed).filter( + } + const seed = evaluateTaskChecks(task.checks, views.seed, read.seed).filter( (outcome) => !outcome.passed, ); if (seed.length === 0) diff --git a/src/application/desktop/DesktopTaskReaderCleanup.ts b/src/application/desktop/DesktopTaskReaderCleanup.ts new file mode 100644 index 0000000000..dac52fea8b --- /dev/null +++ b/src/application/desktop/DesktopTaskReaderCleanup.ts @@ -0,0 +1,133 @@ +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import type { TaskFileView } from "../../domain/desktop/appTaskChecks.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +import { RetainedCommandCleanup } from "../../process/RetainedCommandCleanup.js"; +import { + processWorkerCleanup, + type RetainedWorkerCleanup, +} from "../../process/OwnedWorkerRuntime.js"; +import { DESKTOP_PROVIDER } from "./DesktopCaptureRefusals.js"; + +const OPERATION = "export_desktop_application"; + +export interface ReaderFacts { + readonly views: ReadonlyMap; + readonly absent: readonly string[]; +} + +const primaryFailure = (cause: unknown): AnalysisError | undefined => { + if (cause instanceof AnalysisError) return cause; + if (cause === undefined) return undefined; + return cause instanceof OwnedCommandFailure && cause.reason === "cancelled" + ? new AnalysisCancelledError(OPERATION, { + cause: cause.cancellation?.reason, + }) + : new ProviderAdapterError(DESKTOP_PROVIDER.id, OPERATION, { + cause, + userMessage: `The task reader could not complete: ${cause instanceof Error ? cause.message : String(cause)}`, + }); +}; + +const readerDiagnostics = ({ views, absent }: ReaderFacts) => ({ + ...(views.size === 0 + ? {} + : { + completed_reader_views: Object.fromEntries( + [...views].map(([id, view]) => [ + id, + { + format: view.format, + view: view.view, + ...(view.compression === undefined + ? {} + : { compression: view.compression }), + ...(view.error === undefined ? {} : { error: view.error }), + }, + ]), + ), + }), + ...(absent.length === 0 ? {} : { completed_reader_absent: [...absent] }), +}); + +/** Preserve completed readers when a later reader fails or is cancelled. */ +export const readerFailure = ( + cause: unknown, + facts: ReaderFacts, +): AnalysisError => + ( + primaryFailure(cause) ?? + new ProviderAdapterError(DESKTOP_PROVIDER.id, OPERATION, { cause }) + ).retainDiagnostics(readerDiagnostics(facts)); + +/** Own an unconfirmed reader process and its working folder across requests. */ +class ReaderCleanup implements RetainedWorkerCleanup { + #retry: Promise | undefined; + constructor( + readonly owner: RetainedCommandCleanup, + readonly failure: ProviderCleanupError, + ) { + failure.bindCleanupResourceOwner(owner); + } + + retryRetainedCleanup(): Promise { + if (this.#retry !== undefined) return this.#retry; + const attempt = this.#release(); + this.#retry = attempt; + void attempt + .finally(() => { + this.#retry = undefined; + }) + .catch(() => {}); + return attempt; + } + + async #release(): Promise { + try { + await this.owner.close(); + processWorkerCleanup.delete(this); + this.failure.resolveCleanup(); + } catch (cause: unknown) { + this.failure.retainDiagnostics({ + retry_failure: cause instanceof Error ? cause.message : String(cause), + }); + throw this.failure; + } + } +} + +/** Retain the physical owner before projecting an interrupted reader result. */ +export const retainReaderCleanup = ( + folder: string, + cause: unknown, + facts: ReaderFacts, + folderFailure?: { readonly cause: unknown }, +): ProviderCleanupError | undefined => { + const owner = new RetainedCommandCleanup(folder, cause); + if (!owner.hasProcess && folderFailure === undefined) return undefined; + const original = primaryFailure(cause); + const resources = owner.cleanupResources; + const problem = + folderFailure?.cause ?? + (cause instanceof OwnedCommandFailure ? cause.cleanupFailure : undefined) ?? + cause; + const failure = new ProviderCleanupError( + DESKTOP_PROVIDER.id, + resources, + { + reason: problem instanceof Error ? problem.message : String(problem), + ...readerDiagnostics(facts), + }, + { + operation: OPERATION, + cause, + ...(original === undefined ? {} : { cleanupPrimary: original }), + remediationAction: + "Retry cleanup in this REA process before starting another reader; if it exits first, inspect the reported local resources.", + }, + ); + processWorkerCleanup.add(new ReaderCleanup(owner, failure)); + return failure; +}; diff --git a/src/application/desktop/DesktopTaskReaders.test.ts b/src/application/desktop/DesktopTaskReaders.test.ts new file mode 100644 index 0000000000..3e6958e002 --- /dev/null +++ b/src/application/desktop/DesktopTaskReaders.test.ts @@ -0,0 +1,168 @@ +import { existsSync } from "node:fs"; +import { chmod } from "node:fs/promises"; +import { expect, it, vi } from "vitest"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +import { processWorkerCleanup } from "../../process/OwnedWorkerRuntime.js"; +import type { ProviderProcessSupervisor } from "../../process/ProviderProcess.js"; +import { readerViewsOfBytes } from "./DesktopTaskReaders.js"; +import { desktopOperationFailure } from "./DesktopCaptureRefusals.js"; + +it("retains a reader process and its files after cleanup fails and retries before another reader", async () => { + const reason = new Error("caller cancelled"); + const stop = vi + .fn() + .mockResolvedValueOnce({ status: "incomplete", reason: "still running" }) + .mockResolvedValueOnce({ status: "stopped" }); + const dispose = vi.fn(); + const owner = { stop, dispose } as unknown as ProviderProcessSupervisor; + const commandFailure = new OwnedCommandFailure( + "cancelled", + "cancelled", + null, + "still running", + { cancellation: { reason } }, + ["reader-run"], + owner, + ); + const readers = [ + { + id: "first", + file: "first.txt", + argv: ["reader", "{file}"], + output: "json" as const, + }, + { + id: "second", + file: "second.txt", + argv: ["reader", "{file}"], + output: "json" as const, + }, + ]; + let folder = ""; + const run = vi.fn(async (argv: readonly string[]) => { + const args = JSON.parse(argv.slice(4).join("")) as { + id: string; + work: string; + }; + if (args.id === "second") { + folder = args.work; + throw commandFailure; + } + return { + exit_code: 0, + stdout: JSON.stringify({ view: { format: "json", view: { value: 42 } } }), + stderr: "", + timed_out: false, + }; + }); + const read = () => + readerViewsOfBytes( + readers, + async () => ({ kind: "bytes", bytes: Buffer.from("input") }), + run, + ); + const failure: unknown = await read().catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProviderCleanupError); + const error = failure as ProviderCleanupError; + try { + expect(error.cleanupPrimary?.cause).toBe(reason); + const cancelled = new AbortController(); + cancelled.abort(reason); + expect( + desktopOperationFailure( + error, + cancelled.signal, + "export_desktop_application", + "Desktop export", + ), + ).toBe(error); + expect(existsSync(folder)).toBe(true); + expect(projectAnalysisError(error)).toMatchObject({ + details: { + resources: ["reader-run", folder], + diagnostics: { + completed_reader_views: { + first: { format: "json", view: { value: 42 } }, + }, + }, + }, + }); + await expect(read()).rejects.toBe(error); + expect(run).toHaveBeenCalledTimes(2); + expect(existsSync(folder)).toBe(true); + if (process.platform !== "win32" && process.getuid?.() !== 0) { + await chmod(folder, 0o500); + try { + await expect(processWorkerCleanup.retry()).rejects.toBe(error); + expect(error.cleanupResources).toEqual([folder]); + expect(error.cleanupIncomplete).toBe(true); + } finally { + await chmod(folder, 0o700); + } + } + await Promise.all([ + processWorkerCleanup.retry(), + processWorkerCleanup.retry(), + ]); + expect(existsSync(folder)).toBe(false); + expect(error.cleanupIncomplete).toBe(false); + expect(dispose).toHaveBeenCalledTimes(1); + expect(stop).toHaveBeenCalledTimes(2); + } finally { + await processWorkerCleanup.retry(); + } +}); + +it.each([ + [ + { view: { format: "json", view: { value: 7 } } }, + { + completed_reader_views: { item: { format: "json", view: { value: 7 } } }, + }, + ], + [{ absent: true }, { completed_reader_absent: ["item"] }], +])( + "preserves the final reader answer if cancellation arrives during cleanup", + async (answer, diagnostics) => { + const output = JSON.stringify(answer); + const snapshot = { + exitCode: 0, + stdout: { + text: output, + bytes: Buffer.byteLength(output), + observedBytes: Buffer.byteLength(output), + }, + stderr: { text: "", bytes: 0, observedBytes: 0 }, + signal: null, + }; + const reason = new Error("late cancellation"); + const failure = await readerViewsOfBytes( + [ + { + id: "item", + file: "item.txt", + argv: ["reader", "{file}"], + output: "json", + }, + ], + async () => ({ kind: "bytes", bytes: Buffer.from("input") }), + async () => { + throw new OwnedCommandFailure( + "cancelled", + "cancelled", + snapshot, + null, + { cancellation: { reason } }, + ); + }, + ).catch((cause: unknown) => cause); + expect( + projectAnalysisError( + failure as import("../../domain/analysisErrorBase.js").AnalysisError, + ), + ).toMatchObject({ code: "cancelled", details: { diagnostics } }); + expect((failure as Error).cause).toBe(reason); + }, +); diff --git a/src/application/desktop/DesktopTaskReaders.ts b/src/application/desktop/DesktopTaskReaders.ts new file mode 100644 index 0000000000..19f202b3be --- /dev/null +++ b/src/application/desktop/DesktopTaskReaders.ts @@ -0,0 +1,125 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { DesktopTaskReader } from "../../domain/desktop/appTask.js"; +import type { TaskFileView } from "../../domain/desktop/appTaskChecks.js"; +import { processWorkerCleanup } from "../../process/OwnedWorkerRuntime.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +import { + readerFailure, + retainReaderCleanup, +} from "./DesktopTaskReaderCleanup.js"; +import { + READER_RUNNER_SLACK_S, + readerAnswerView, + readerCommand, + readerTimeout, + type ReaderCommandResult, +} from "../../domain/desktop/appTaskReaders.js"; + +/** How a reader's runner command runs: in a computer through computer-exec, or on this host. */ +export type ReaderCommandRun = ( + argv: readonly string[], + timeoutS: number, +) => Promise; + +/** A reader's file at one end of a capture: absent, unreadable and why, or its bytes. */ +export type ReaderFileBytes = + | { readonly kind: "absent" } + | { readonly kind: "error"; readonly error: string } + | { readonly kind: "bytes"; readonly bytes: Buffer }; + +/** + * Each reader's view of its file by reader id, read by REA's reader runner; + * an absent file has none. `pathOf` names bytes to read in place of the + * home's file. A runner that does not answer throws. + */ +export const readTaskReaderViews = async ( + readers: readonly DesktopTaskReader[], + run: ReaderCommandRun, + pathOf: (reader: DesktopTaskReader) => string | undefined = () => undefined, + work?: string, +): Promise> => { + const views = new Map(); + for (const reader of readers) { + const result = await run( + readerCommand(reader, pathOf(reader), work), + readerTimeout(reader) + READER_RUNNER_SLACK_S, + ); + if (result.exit_code !== 0 || result.timed_out) + throw new Error( + `REA's reader runner for ${reader.id} ${result.timed_out ? "timed out" : `exited ${String(result.exit_code)}`}: ${result.stderr.slice(-1024) || result.stdout.slice(-1024)}`, + ); + const view = readerAnswerView(result.stdout); + if (view !== undefined) views.set(reader.id, view); + } + return views; +}; + +/** Recover only a complete, valid answer emitted before later finalization failed. */ +const completedReaderAnswer = ( + cause: unknown, +): { readonly view: TaskFileView | undefined } | undefined => { + if ( + !(cause instanceof OwnedCommandFailure) || + cause.collectionFailure || + cause.snapshot?.exitCode !== 0 || + cause.snapshot.outputIncomplete === true + ) + return undefined; + const stdout = cause.snapshot.stdout; + if (stdout.invalidUtf8 || stdout.bytes !== stdout.observedBytes) + return undefined; + try { + return { view: readerAnswerView(stdout.text) }; + } catch { + return undefined; + } +}; + +/** Each reader's view of the bytes `bytesOf` gives for its file, each written to a private folder the runner reads. */ +export const readerViewsOfBytes = async ( + readers: readonly DesktopTaskReader[], + bytesOf: (reader: DesktopTaskReader) => Promise, + run: ReaderCommandRun, +): Promise> => { + const views = new Map(); + if (readers.length > 0) await processWorkerCleanup.retry(); + const absent: string[] = []; + for (const reader of readers) { + const file = await bytesOf(reader); + if (file.kind === "absent") continue; + if (file.kind === "error") { + views.set(reader.id, { format: null, view: null, error: file.error }); + continue; + } + const folder = await mkdtemp(join(tmpdir(), "rea-reader-bytes-")); + let failure: { readonly cause: unknown } | undefined; + try { + const path = join(folder, reader.file.split("/").at(-1) ?? "file"); + await writeFile(path, file.bytes); + // The runner works inside this folder, so removing it cleans up after a runner that was killed. + const read = await readTaskReaderViews([reader], run, () => path, folder); + for (const [id, view] of read) views.set(id, view); + if (!read.has(reader.id)) absent.push(reader.id); + } catch (cause: unknown) { + failure = { cause }; + const completed = completedReaderAnswer(cause); + if (completed !== undefined) { + if (completed.view === undefined) absent.push(reader.id); + else views.set(reader.id, completed.view); + } + } + const facts = { views, absent }; + const retained = retainReaderCleanup(folder, failure?.cause, facts); + if (retained !== undefined) throw retained; + try { + await rm(folder, { recursive: true, force: true }); + } catch (cause: unknown) { + throw retainReaderCleanup(folder, failure?.cause, facts, { cause }); + } + if (failure !== undefined) throw readerFailure(failure.cause, facts); + } + return views; +}; diff --git a/src/cli/desktopCaptureCommands.ts b/src/cli/desktopCaptureCommands.ts index 25eb27ea2f..636431f9ce 100644 --- a/src/cli/desktopCaptureCommands.ts +++ b/src/cli/desktopCaptureCommands.ts @@ -6,6 +6,10 @@ import type { Logger } from "pino"; import { captureDesktopApplication } from "../application/desktop/DesktopApplicationCapture.js"; import type { DesktopStateReaders } from "../application/desktop/DesktopApplicationState.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { bindArtifactFileCleanup } from "../artifacts/ArtifactFileOperation.js"; import type { Evidence } from "../domain/evidence.js"; import { localPathStringSchema } from "../domain/localPath.js"; import { err, ok } from "../domain/result.js"; @@ -126,7 +130,54 @@ const withStateReaders = async ( }); return result instanceof AnalysisError ? err(result) : ok(result); }, - sqlite, + { + close: async () => { + const failures: unknown[] = []; + try { + await sqlite.close(); + } catch (cause: unknown) { + failures.push(cause); + } + if (operation === "export_desktop_application") { + try { + const { OwnedWorkerRuntime } = + await import("../process/OwnedWorkerRuntime.js"); + await OwnedWorkerRuntime.retryPending(); + } catch (cause: unknown) { + failures.push(cause); + } + } + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) { + const typed = failures.map((cause) => { + if (cause instanceof AnalysisError) return cause; + const observation = + cause instanceof ArtifactReaderFailure + ? ArtifactReaderFailure.cleanupObservation(cause, []) + : { + resources: [], + reason: + cause instanceof Error ? cause.message : String(cause), + }; + const failure = new ProviderCleanupError( + "rea-desktop", + observation.resources, + { reason: observation.reason }, + { operation, cause }, + ); + bindArtifactFileCleanup(cause, failure); + return failure; + }); + const combined = analysisErrorWithCleanupFailure( + typed[0]!, + typed[1]!, + operation, + ); + combined.bindCleanupDependencies(typed); + throw combined; + } + }, + }, "rea-desktop", operation, signal, @@ -187,6 +238,12 @@ const registerDesktopExportCommand = ( .describe( "The seed was made for the task, as in a build computer, not taken from the user's own data; the README says so instead of keeping the bundle local", ), + screens: z + .string() + .optional() + .describe( + "Folder holding a screenshot of each checkpoint as .jpg, which a build computer takes; a task's screen_judge checks ship the seed's and the end's as their judge's examples", + ), ...checkpointOptions, }), run: ({ args, options }) => @@ -219,6 +276,10 @@ const registerDesktopExportCommand = ( seedCheckpoint: options.seedCheckpoint, endCheckpoint: options.endCheckpoint, syntheticSeed: options.syntheticSeed, + screens: + options.screens === undefined + ? undefined + : absoluteCallerPath(options.screens), }, readers, signal, diff --git a/src/contracts/desktopBuildToolContracts.ts b/src/contracts/desktopBuildToolContracts.ts index 37fdab016a..0d20b433b0 100644 --- a/src/contracts/desktopBuildToolContracts.ts +++ b/src/contracts/desktopBuildToolContracts.ts @@ -272,7 +272,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ name: "mark_build_checkpoint", ...toolContractMetadata("mark_build_checkpoint"), kind: "application", - description: `Take a named checkpoint of the app's state in the capture under way and wait for the capture to acknowledge it, with its counts (entries walked, paths changed since the checkpoint before, files retained) and up to ${String(MAX_ACKNOWLEDGED_FILES)} of the files that changed, those a window title named first, then those under state_paths and those the app held open, caches last, so you can see which file a save wrote. Mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the change is; a name already taken is refused.`, + description: `Take a named checkpoint of the app's state in the capture under way and wait for the capture to acknowledge it, with its counts (entries walked, paths changed since the checkpoint before, files retained) and up to ${String(MAX_ACKNOWLEDGED_FILES)} of the files that changed, those a window title named first, then those under state_paths and those the app held open, caches last, so you can see which file a save wrote. Mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the change is; a name already taken is refused. REA also keeps a screenshot of the screen at each checkpoint, which a screen_judge task ships as its judge's examples, so leave the screen showing the state at ${SEED_CHECKPOINT} and at ${END_CHECKPOINT}.`, inputSchema: z.strictObject({ computer_id: computerId, name: buildCheckpointNameSchema, @@ -380,7 +380,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ name: "export_build_environment", ...toolContractMetadata("export_build_environment"), kind: "application", - description: `Export the stopped capture as an agentenv computer app bundle with the given task, inside the computer, seeded at ${SEED_CHECKPOINT} and checked at ${END_CHECKPOINT}, then copy the bundle to a new or empty folder on this host, every file checked against its digest. A task the export refuses, such as a check that already passes on the seed, is an error naming why. Witness the bundle next: start_build_computer from the folder and check_build_task.`, + description: `Export the stopped capture as an agentenv computer app bundle with the given task, inside the computer, seeded at ${SEED_CHECKPOINT} and checked at ${END_CHECKPOINT}, then copy the bundle to a new or empty folder on this host, every file checked against its digest. A task the export refuses, such as a check that already passes on the seed, is an error naming why. A task of screen_judge checks, for state no file shows, is refused when a file shows the change; the export judges no screen, and ships the ${SEED_CHECKPOINT} and ${END_CHECKPOINT} screenshots as the judge's examples. Witness the bundle next: start_build_computer from the folder and check_build_task.`, inputSchema: z.strictObject({ computer_id: computerId, task: desktopTaskSchema, @@ -423,7 +423,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [ name: "check_build_task", ...toolContractMetadata("check_build_task"), kind: "application", - description: `Grade a task's checks against the app's live state on a computer of an exported bundle: REA reads the app's data/get views and evaluates each check as the bundle's verifier does, one result per check, once the app's files settle: it reads data/get every ${String(SETTLE_TIMING.intervalMs / 1000)} s and grades once every file has held for ${String(SETTLE_TIMING.windowMs / 1000)} s, counted from the read that returned its bytes, so an app that saves a few seconds after the last key is graded on what it saved; no read begins ${String(SETTLE_TIMING.boundMs / 1000)} s or more after the first one began, one still in flight then is cut (read_cut), and files still changing are graded on the last completed read, with settled false. With stage, the check is part of the witness, in order: seed (give directory; REA begins recording) must fail a check, and on a computer started from a bundle folder it refuses a task whose steps and verifier in the bundle differ from what the export writes for it; after_edit, after acting on the app from the task's prompt alone, must pass every check; near_miss, after a deliberately almost-right change, must fail one; after_reset, after act_on_build_computer action reset, must give each check the result it had on the seed. Only a check that fails on what the app's view shows counts as failing: one that fails because the view could not be read or is cut short is undecided, and a stage only undecided checks would settle files nothing and is refused, to be checked again. A stage that does not hold, or after_reset, ends the witness: REA ends the recording and writes witness.json, the record as Evidence, with the recording's segments into directory, and returns the record.`, + description: `Grade a task's checks against the app's live state on a computer of an exported bundle: REA reads the app's data/get views and evaluates each check as the bundle's verifier does, one result per check, once the app's files settle (a screen_judge task instead takes a screenshot and runs the bundle's own judge on it with the agentenv runner's Python and model endpoint, three votes a question): it reads data/get every ${String(SETTLE_TIMING.intervalMs / 1000)} s and grades once every file has held for ${String(SETTLE_TIMING.windowMs / 1000)} s, counted from the read that returned its bytes, so an app that saves a few seconds after the last key is graded on what it saved; no read begins ${String(SETTLE_TIMING.boundMs / 1000)} s or more after the first one began, one still in flight then is cut (read_cut), and files still changing are graded on the last completed read, with settled false. With stage, the check is part of the witness, in order: seed (give directory; REA begins recording) must fail a check, and on a computer started from a bundle folder it refuses a task whose steps and verifier in the bundle differ from what the export writes for it; after_edit, after acting on the app from the task's prompt alone, must pass every check; near_miss, after a deliberately almost-right change, must fail one; after_reset, after act_on_build_computer action reset, must give each check the result it had on the seed. Only a check that fails on what the app's view shows counts as failing: one that fails because the view could not be read or is cut short is undecided, and a stage only undecided checks would settle files nothing and is refused, to be checked again. A stage that does not hold, or after_reset, ends the witness: REA ends the recording and writes witness.json, the record as Evidence, with the recording's segments into directory, and returns the record.`, inputSchema: z.strictObject({ computer_id: computerId, task: desktopTaskSchema, diff --git a/src/contracts/toolEffects.test.ts b/src/contracts/toolEffects.test.ts index 334af4ec9f..58f8a3a0cb 100644 --- a/src/contracts/toolEffects.test.ts +++ b/src/contracts/toolEffects.test.ts @@ -38,11 +38,13 @@ describe("tool effects", () => { "capture_electron_scenario", "capture_native_ui_scenario", "capture_process_scenario", + "check_build_task", "close_frida_session", "close_gdb_session", "close_rizin_debug_session", "cutter_command", "execute_rizin_command", + "export_build_environment", "gdb_console", "install_adb_package", "instrument_with_frida", diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 26131095fd..9d12d5ebde 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -781,18 +781,22 @@ const TOOL_EFFECT_AUDIT: Readonly> = { changesUiState: true, idempotent: false, }), + // Runs the task's readers, commands the task names, on the captured files. export_build_environment: effects({ mutatesSession: true, writesFilesystem: true, launchesProcess: true, accessesNetwork: true, + executesForeignCode: true, idempotent: false, }), - // A witness stage records the display and writes its record on this host. + // A witness stage records the display, runs the task's readers or the bundle's judge, and writes its record here. check_build_task: effects({ mutatesSession: true, writesFilesystem: true, + launchesProcess: true, accessesNetwork: true, + executesForeignCode: true, idempotent: false, }), close_build_computer: effects({ diff --git a/src/desktop/AgentEnvComputer.ts b/src/desktop/AgentEnvComputer.ts index 698af453ea..0a6a621c3f 100644 --- a/src/desktop/AgentEnvComputer.ts +++ b/src/desktop/AgentEnvComputer.ts @@ -13,6 +13,11 @@ import type { ComputerHelper, ComputerRelease, } from "./AgentEnvComputerHelper.js"; +import { + dataGetSchema, + readHomeFiles, + writeHomeFiles, +} from "./ComputerHomeFiles.js"; import { drawnScreenshot } from "./DrawnScreenshot.js"; import { awaitHttpReply, type HttpReply } from "./HttpReply.js"; @@ -158,21 +163,6 @@ const recordingManifestSchema = z.looseObject({ }); /** An ended recording's manifest, as `computer-recording/v1` returns it. */ export type RecordingManifest = z.infer; -const dataGetSchema = z.union([ - z.object({ - result: z.object({ - parts: z.array( - z.looseObject({ - file: z.looseObject({ bytes: z.string().optional() }).optional(), - metadata: z - .record(z.string(), z.looseObject({ path: z.string().optional() })) - .optional(), - }), - ), - }), - }), - z.object({ error: z.looseObject({ message: z.string() }) }), -]); const rpcSchema = z.union([ z.object({ result: z.looseObject({}) }), z.object({ error: z.looseObject({ message: z.string() }) }), @@ -525,6 +515,18 @@ export class AgentEnvComputer { return bytes; } + /** Write files under the computer user's home, by home path, through the driver's `data/add`. */ + async addHomeFiles( + files: ReadonlyMap, + signal?: AbortSignal, + ): Promise { + await writeHomeFiles( + (body) => this.#request("/agentenv", body, signal), + (text) => this.redacted(text), + files, + ); + } + /** * The files under paths of the computer user's home, by home path, through * the driver's `data/get`; its answer is read within `bytes` of files. @@ -537,38 +539,19 @@ export class AgentEnvComputer { const ceiling = guestFileAnswerCeiling(); const refusal = guestFileAnswerRefusal(bytes, paths.length, ceiling); if (refusal !== undefined) throw refusal; - const answer = dataGetSchema.parse( - await this.#request( - "/agentenv", - { jsonrpc: "2.0", id: 1, method: "data/get", params: { paths } }, - signal, - { - limit: { - bytes: Math.min( - guestFileAnswerBytes(bytes, paths.length), - guestFileAnswerWorkingBytes(ceiling), - ), - what: `The computer's answer for ${paths.join(", ")}`, - }, - }, + const limit = { + bytes: Math.min( + guestFileAnswerBytes(bytes, paths.length), + guestFileAnswerWorkingBytes(ceiling), ), - ); - if ("error" in answer) - throw new Error( - `data/get failed: ${this.redacted(answer.error.message)}`, - ); + what: `The computer's answer for ${paths.join(", ")}`, + }; const retained = retainGuestFiles( - answer.result.parts.flatMap((part) => { - const encoded = part.file?.bytes; - return encoded === undefined - ? [] - : [ - { - path: Object.values(part.metadata ?? {})[0]?.path, - bytes: Buffer.from(encoded, "base64"), - }, - ]; - }), + await readHomeFiles( + (body) => this.#request("/agentenv", body, signal, { limit }), + (text) => this.redacted(text), + paths, + ), bytes, paths.length, ceiling, @@ -582,11 +565,12 @@ export class AgentEnvComputer { argv: readonly string[], timeoutS: number, signal?: AbortSignal, + requestId: string = randomUUID(), ): Promise { const result = execSchema.parse( await this.#request( "/agentenv/ext/computer-exec/run", - { argv, request_id: randomUUID(), timeout_s: timeoutS }, + { argv, request_id: requestId, timeout_s: timeoutS }, signal, { limit: { diff --git a/src/desktop/BuildComputerController.ts b/src/desktop/BuildComputerController.ts index 7d31cb419f..7ab7f9ae00 100644 --- a/src/desktop/BuildComputerController.ts +++ b/src/desktop/BuildComputerController.ts @@ -19,6 +19,8 @@ const CONTROLLER = fileURLToPath( ); /** The most file bytes one `data/get` brings back, so a large bundle arrives in several answers. */ const FETCH_BATCH_BYTES = 8 * 1024 * 1024; +/** Default time allowed for one REA command in the build computer. */ +export const BUILD_REA_TIMEOUT_S = 1800; /** * Linux `MAX_ARG_STRLEN` on a 4 KiB page, excluding the NUL the kernel counts. * One argument past this fails with E2BIG before the process starts. @@ -75,7 +77,15 @@ export type ControlledComputer = Pick< */ export const runBuildController = async ( computer: ControlledComputer, - command: "start" | "mark" | "stop" | "run" | "list", + command: + | "start" + | "mark" + | "stop" + | "run" + | "list" + | "run-begin" + | "run-poll" + | "run-cancel", argumentsJson: Readonly>, options: { readonly timeoutS: number; readonly signal?: AbortSignal }, ): Promise>> => { @@ -141,6 +151,10 @@ export const answerSchema = z.looseObject({ /** Why the controller refused a request before acting on it, such as a state path outside the home. */ refused: z.string().optional(), absent: z.boolean().optional(), + started: z.boolean().optional(), + pending: z.boolean().optional(), + finished: z.boolean().optional(), + cancelled: z.boolean().optional(), line: z.string().optional(), exited: z.int().optional(), timed_out: z.boolean().optional(), @@ -196,9 +210,12 @@ export const runRea = async ( argv: [...argv], stdout: options.stdout, files: { ...options.files }, - timeout_s: options.timeoutS ?? 1800, + timeout_s: options.timeoutS ?? BUILD_REA_TIMEOUT_S, + }, + { + timeoutS: options.timeoutS ?? BUILD_REA_TIMEOUT_S, + signal: options.signal, }, - { timeoutS: options.timeoutS ?? 1800, signal: options.signal }, ), ); if (answer.exit_code === undefined || answer.stdout_bytes === undefined) diff --git a/src/desktop/BuildComputerLongRuns.test.ts b/src/desktop/BuildComputerLongRuns.test.ts new file mode 100644 index 0000000000..ce52f27c3b --- /dev/null +++ b/src/desktop/BuildComputerLongRuns.test.ts @@ -0,0 +1,326 @@ +import { expect, it } from "vitest"; + +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { BuildComputerLongRuns } from "./BuildComputerLongRuns.js"; +import type { ControlledComputer } from "./BuildComputerController.js"; + +const response = (value: unknown) => ({ + exit_code: 0, + stdout: JSON.stringify(value), + stderr: "", + timed_out: false, + truncated: false, +}); +const options = (signal: AbortSignal) => ({ + work: ".rea-build", + stdout: ".rea-build/output.json", + files: {}, + timeoutS: 7080, + signal, +}); + +it("waits through several bounded calls for a command with a longer budget", async () => { + const timeouts: number[] = []; + const calls: string[] = []; + let polls = 0; + let budget: unknown; + const computer = { + exec: async (argv: readonly string[], timeout: number) => { + timeouts.push(timeout); + const command = argv[4] ?? ""; + calls.push(command); + const arguments_ = JSON.parse(argv.slice(5).join("")) as Record< + string, + unknown + >; + if (command === "run-begin") { + budget = arguments_.timeout_s; + return response({ started: true }); + } + if (command === "run-poll") + return response( + ++polls < 3 + ? { pending: true } + : { finished: true, exit_code: 0, stdout_bytes: 17, stderr: "" }, + ); + throw new Error(`unexpected ${command}`); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + expect( + await owner.run( + ["export-desktop-application"], + options(new AbortController().signal), + ), + ).toEqual({ stdoutBytes: 17, exitCode: 0, stderr: "" }); + expect(budget).toBe(7080); + expect(calls).toEqual(["run-begin", "run-poll", "run-poll", "run-poll"]); + expect(Math.max(...timeouts)).toBeLessThanOrEqual(3600); + expect(owner.cleanupResources).toEqual([]); +}); + +it("retains an interrupted command until a later stop confirms it ended", async () => { + const controller = new AbortController(); + let stops = 0; + const computer = { + exec: async (argv: readonly string[]) => { + const command = argv[4]; + if (command === "run-begin") return response({ started: true }); + if (command === "run-poll") { + controller.abort(new Error("cancelled")); + throw controller.signal.reason; + } + if (command === "run-cancel") + return response( + ++stops === 1 + ? { pending: true } + : { + finished: true, + exit_code: 130, + stdout_bytes: 12, + stderr: "stopped", + }, + ); + throw new Error(`unexpected ${String(command)}`); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + const failure = await owner + .run(["export-desktop-application"], options(controller.signal)) + .catch((error: unknown) => error); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected retained cleanup"); + expect(failure.cleanupPrimary).toBeInstanceOf(AnalysisCancelledError); + expect(owner.cleanupResources).toHaveLength(1); + expect(await owner.release()).toMatchObject({ verified: true }); + expect(failure.diagnostics).toMatchObject({ + completed_command: { stdout_bytes: 12, exit_code: 130 }, + }); + expect(failure.cleanupIncomplete).toBe(false); +}); + +it("retains cancellation arriving during failed final cleanup alongside the original error", async () => { + const controller = new AbortController(); + let stops = 0; + const computer = { + exec: async (argv: readonly string[]) => { + if (argv[4] === "run-begin") return response({ started: true }); + if (argv[4] === "run-poll") throw new Error("connection failed"); + if (argv[4] === "run-cancel") { + if (++stops === 1) { + controller.abort(new Error("cancelled")); + return response({ pending: true }); + } + return response({ finished: true, exit_code: 130, stdout_bytes: 3 }); + } + throw new Error("unexpected command"); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + const failure = await owner + .run(["export-desktop-application"], options(controller.signal)) + .catch((error: unknown) => error); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected retained cleanup"); + expect(failure.cleanupPrimary).toBeInstanceOf(AnalysisCancelledError); + expect(failure.cleanupPrimary?.secondaryFailures[0]?.userMessage).toContain( + "connection failed", + ); + expect(await owner.release()).toMatchObject({ verified: true }); +}); + +it("returns a timeout while retaining a command that is still stopping", async () => { + let stops = 0; + const computer = { + exec: async (argv: readonly string[]) => { + if (argv[4] === "run-begin") return response({ started: true }); + if (argv[4] === "run-poll") + return response({ pending: true, timed_out: true }); + if (argv[4] === "run-cancel") + return response( + ++stops === 1 + ? { pending: true, timed_out: true } + : { + finished: true, + timed_out: true, + exit_code: 130, + stdout_bytes: 0, + }, + ); + throw new Error("unexpected command"); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + const failure = await owner + .run(["export-desktop-application"], options(new AbortController().signal)) + .catch((error: unknown) => error); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected retained cleanup"); + expect(failure.cleanupPrimary?.userMessage).toContain( + "exceeded its time budget", + ); + expect(owner.cleanupResources).toHaveLength(1); + expect(await owner.release()).toMatchObject({ verified: true }); +}); + +it("enforces its deadline even if the supervisor never publishes a timeout", async () => { + const commands: string[] = []; + const computer = { + exec: async (argv: readonly string[]) => { + const command = argv[4] ?? ""; + commands.push(command); + if (command === "run-begin") return response({ started: true }); + if (command === "run-cancel") return response({ pending: true }); + throw new Error("the expired job must not be polled"); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + const failure = await owner + .run(["export-desktop-application"], { + ...options(new AbortController().signal), + timeoutS: 0, + }) + .catch((error: unknown) => error); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected retained cleanup"); + expect(failure.cleanupPrimary?.userMessage).toContain( + "exceeded its time budget", + ); + expect(commands).toEqual(["run-begin", "run-cancel"]); + expect(owner.cleanupResources).toHaveLength(1); +}); + +it("claims a not-yet-started job when cancellation interrupts its start", async () => { + const controller = new AbortController(); + const commands: string[] = []; + const computer = { + exec: async (argv: readonly string[]) => { + const command = argv[4] ?? ""; + commands.push(command); + if (command === "run-begin") { + controller.abort(new Error("cancelled")); + throw controller.signal.reason; + } + if (command === "run-cancel") return response({ absent: true }); + throw new Error("unexpected command"); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + await expect( + owner.run(["export-desktop-application"], options(controller.signal)), + ).rejects.toBeInstanceOf(AnalysisCancelledError); + expect(commands).toEqual(["run-begin", "run-cancel"]); + expect(owner.cleanupResources).toEqual([]); +}); + +it("retains a job whose cancellation cannot confirm ownership", async () => { + const computer = { + exec: async (argv: readonly string[]) => { + if (argv[4] === "run-begin") throw new Error("response lost"); + if (argv[4] === "run-cancel") + return response({ + error: "the command job ownership is not confirmed", + }); + throw new Error("unexpected command"); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + await expect( + owner.run( + ["export-desktop-application"], + options(new AbortController().signal), + ), + ).rejects.toBeInstanceOf(ProviderCleanupError); + expect(await owner.release()).toMatchObject({ verified: false }); + expect(owner.cleanupResources).toHaveLength(1); +}); + +it("preserves a completed command's receipt if the supervisor reports a later failure", async () => { + const computer = { + exec: async (argv: readonly string[]) => { + if (argv[4] === "run-begin") return response({ started: true }); + if (argv[4] === "run-poll") + return response({ + finished: true, + exit_code: 0, + stdout_bytes: 12, + error: "stderr close failed", + }); + throw new Error("unexpected command"); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + const failure = await owner + .run(["export-desktop-application"], options(new AbortController().signal)) + .catch((error: unknown) => error); + if (!(failure instanceof Error) || !("diagnostics" in failure)) + throw new Error("expected command failure"); + expect(failure.diagnostics).toMatchObject({ + completed_command: { + stdout_bytes: 12, + exit_code: 0, + error: "stderr close failed", + }, + }); + expect(owner.cleanupResources).toEqual([]); +}); + +it("preserves a new cancellation while retrying a previous job's cleanup", async () => { + const first = new AbortController(); + const second = new AbortController(); + let starts = 0; + let stops = 0; + const computer = { + exec: async (argv: readonly string[]) => { + if (argv[4] === "run-begin") { + starts += 1; + return response({ started: true }); + } + if (argv[4] === "run-poll") { + first.abort(new Error("first cancelled")); + throw first.signal.reason; + } + if (argv[4] === "run-cancel") { + stops += 1; + if (stops === 2) second.abort(new Error("second cancelled")); + return response( + stops < 3 + ? { pending: true } + : { finished: true, exit_code: 130, stdout_bytes: 0 }, + ); + } + throw new Error("unexpected command"); + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + const owner = new BuildComputerLongRuns(computer); + await expect( + owner.run(["export-desktop-application"], options(first.signal)), + ).rejects.toBeInstanceOf(ProviderCleanupError); + const failure = await owner + .run(["export-desktop-application"], options(second.signal)) + .catch((error: unknown) => error); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("expected retained cleanup"); + expect(failure.cleanupPrimary).toBeInstanceOf(AnalysisCancelledError); + expect(starts).toBe(1); + expect(await owner.release()).toMatchObject({ verified: true }); +}); diff --git a/src/desktop/BuildComputerLongRuns.ts b/src/desktop/BuildComputerLongRuns.ts new file mode 100644 index 0000000000..573e8dbf10 --- /dev/null +++ b/src/desktop/BuildComputerLongRuns.ts @@ -0,0 +1,293 @@ +import { randomUUID } from "node:crypto"; + +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { + AGENTENV_PROVIDER, + type ComputerRelease, +} from "./AgentEnvComputerHelper.js"; +import { + answerSchema, + runBuildController, + type Answer, + type ControlledComputer, +} from "./BuildComputerController.js"; + +const OPERATION = "export_build_environment"; +const POLL_S = 30; +const CANCEL_S = 10; + +interface Job { + readonly path: string; + readonly token: string; + readonly stdout: string; + readonly failures: Set>; +} + +/** Owns REA commands that need longer than one computer-exec call permits. */ +export class BuildComputerLongRuns { + readonly #jobs = new Set(); + + constructor(private readonly computer: ControlledComputer) {} + + get cleanupResources(): readonly string[] { + return [...this.#jobs].map(({ path }) => path); + } + + async run( + argv: readonly string[], + options: { + readonly work: string; + readonly stdout: string; + readonly files?: Readonly>; + readonly timeoutS: number; + readonly signal: AbortSignal; + }, + ): Promise<{ + readonly stdoutBytes: number; + readonly exitCode: number | null; + readonly stderr: string; + }> { + const { signal } = options; + if (signal.aborted) + throw new AnalysisCancelledError(OPERATION, { cause: signal.reason }); + const previous = await this.release(); + if (!previous.verified) + throw this.#cleanupFailure( + previous.reason, + signal.aborted + ? new AnalysisCancelledError(OPERATION, { cause: signal.reason }) + : undefined, + ); + if (signal.aborted) + throw new AnalysisCancelledError(OPERATION, { cause: signal.reason }); + const token = randomUUID(); + const job: Job = { + path: `${options.work}/jobs/${token}`, + token, + stdout: options.stdout, + failures: new Set(), + }; + this.#jobs.add(job); + const deadline = performance.now() + options.timeoutS * 1000; + try { + const started = answerSchema.parse( + await runBuildController( + this.computer, + "run-begin", + { + job: job.path, + token, + work: options.work, + stdout: options.stdout, + files: { ...options.files }, + argv: [...argv], + timeout_s: options.timeoutS, + }, + { timeoutS: POLL_S, signal }, + ), + ); + if (typeof started.refused === "string") { + this.#jobs.delete(job); + throw new Error(started.refused); + } + if (started.started !== true) + throw new Error( + typeof started.error === "string" + ? started.error + : "The REA command did not start.", + ); + for (;;) { + if (performance.now() >= deadline) + throw new Error( + "The REA command exceeded its time budget and is still stopping.", + ); + const answer = answerSchema.parse( + await runBuildController( + this.computer, + "run-poll", + { + job: job.path, + token, + timeout_s: POLL_S, + }, + { timeoutS: POLL_S, signal }, + ), + ); + if (answer.pending === true && answer.timed_out === true) + throw new Error( + "The REA command exceeded its time budget and is still stopping.", + ); + if (answer.pending === true) continue; + if (answer.finished !== true) + throw new Error( + typeof answer.error === "string" + ? answer.error + : "The REA command did not report its outcome.", + ); + this.#jobs.delete(job); + if (answer.timed_out === true || typeof answer.error === "string") { + const failure = new ProviderAdapterError( + AGENTENV_PROVIDER, + OPERATION, + { + userMessage: + answer.error ?? "The REA command exceeded its time budget.", + }, + ); + throw failure.retainDiagnostics({ + completed_command: this.#receipt(job, answer), + }); + } + if ( + typeof answer.exit_code !== "number" || + typeof answer.stdout_bytes !== "number" + ) + throw new Error( + "The REA command did not report its output and exit status.", + ); + return { + stdoutBytes: answer.stdout_bytes, + exitCode: answer.exit_code, + stderr: typeof answer.stderr === "string" ? answer.stderr : "", + }; + } + } catch (cause: unknown) { + let primary = signal.aborted + ? new AnalysisCancelledError(OPERATION, { cause: signal.reason }) + : cause instanceof AnalysisError + ? cause + : new ProviderAdapterError(AGENTENV_PROVIDER, OPERATION, { + userMessage: + cause instanceof Error ? cause.message : String(cause), + cause, + }); + if (signal.aborted && cause instanceof AnalysisError && cause !== primary) + primary.retainSecondaryFailure(cause); + if (!this.#jobs.has(job)) throw primary; + const released = await this.#cancel(job); + if (signal.aborted && !(primary instanceof AnalysisCancelledError)) { + const cancelled = new AnalysisCancelledError(OPERATION, { + cause: signal.reason, + }); + cancelled.retainSecondaryFailure(primary); + primary = cancelled; + } + if (released.verified) { + if (released.completed !== undefined) + primary.retainDiagnostics({ completed_command: released.completed }); + throw primary; + } + throw this.#cleanupFailure(released.reason, primary, job); + } + } + + /** Stop each command and retain any job whose completion cannot be confirmed. */ + async release(): Promise { + const failures: string[] = []; + for (const job of this.#jobs) { + const result = await this.#cancel(job); + if (!result.verified) failures.push(result.reason); + } + return failures.length === 0 + ? { verified: true, teardown: "agentenv" } + : { + verified: false, + reason: failures.join("; "), + resources: this.cleanupResources, + }; + } + + async #cancel( + job: Job, + ): Promise< + | { readonly verified: true; readonly completed?: JsonValue } + | { readonly verified: false; readonly reason: string } + > { + try { + const answer = answerSchema.parse( + await runBuildController( + this.computer, + "run-cancel", + { + job: job.path, + token: job.token, + timeout_s: CANCEL_S, + }, + { timeoutS: CANCEL_S }, + ), + ); + if (answer.absent !== true && answer.finished !== true) + return { + verified: false, + reason: + typeof answer.error === "string" + ? answer.error + : `The remote REA command has not confirmed that it stopped.${answer.progress === undefined ? "" : ` Supervisor output: ${answer.progress}`}`, + }; + const completed = + answer.finished === true ? this.#receipt(job, answer) : undefined; + this.#jobs.delete(job); + for (const reference of job.failures) { + const failure = reference.deref(); + if (completed !== undefined) + failure?.retainDiagnostics({ completed_command: completed }); + failure?.resolveCleanup(); + } + return { + verified: true, + ...(completed === undefined ? {} : { completed }), + }; + } catch (cause: unknown) { + return { + verified: false, + reason: cause instanceof Error ? cause.message : String(cause), + }; + } + } + + #receipt(job: Job, answer: Answer): JsonValue { + return { + stdout: job.stdout, + ...(typeof answer.stdout_bytes === "number" + ? { stdout_bytes: answer.stdout_bytes } + : {}), + ...(typeof answer.exit_code === "number" + ? { exit_code: answer.exit_code } + : {}), + ...(typeof answer.stderr === "string" ? { stderr: answer.stderr } : {}), + ...(typeof answer.error === "string" ? { error: answer.error } : {}), + ...(answer.timed_out === undefined + ? {} + : { timed_out: answer.timed_out }), + ...(answer.cancelled === undefined + ? {} + : { cancelled: answer.cancelled }), + }; + } + + #cleanupFailure( + reason: string, + primary?: AnalysisError, + job?: Job, + ): ProviderCleanupError { + const failure = new ProviderCleanupError( + AGENTENV_PROVIDER, + this.cleanupResources, + { reason }, + { + operation: OPERATION, + ...(primary === undefined ? {} : { cleanupPrimary: primary }), + remediationAction: + "Retry the export or close the build computer to retry stopping the owned command.", + }, + ); + failure.bindCleanupResourceOwner(this); + for (const pending of job === undefined ? this.#jobs : [job]) + pending.failures.add(new WeakRef(failure)); + return failure; + } +} diff --git a/src/desktop/ComputerHomeFiles.ts b/src/desktop/ComputerHomeFiles.ts new file mode 100644 index 0000000000..ebe9db541b --- /dev/null +++ b/src/desktop/ComputerHomeFiles.ts @@ -0,0 +1,82 @@ +import { z } from "zod"; + +/** Where a data-plane file part names its path in the home. */ +const DATA_OBJECTS = "urn:agentenv:data-objects/v1"; +/** A `data/get` answer: file parts, each naming its home path, or an error. */ +export const dataGetSchema = z.union([ + z.object({ + result: z.object({ + parts: z.array( + z.looseObject({ + file: z.looseObject({ bytes: z.string().optional() }).optional(), + metadata: z + .record(z.string(), z.looseObject({ path: z.string().optional() })) + .optional(), + }), + ), + }), + }), + z.object({ error: z.looseObject({ message: z.string() }) }), +]); +const answerSchema = z.union([ + z.object({ result: z.looseObject({}) }), + z.object({ error: z.looseObject({ message: z.string() }) }), +]); + +/** + * Write files under a computer user's home, by home path, through the + * driver's `data/add`, sent by `post` to the computer's own RPC route; + * `redacted` strips the control token from an error the computer returns. + */ +export const writeHomeFiles = async ( + post: (body: unknown) => Promise, + redacted: (text: string) => string, + files: ReadonlyMap, +): Promise => { + const parts = [...files].map(([path, bytes]) => ({ + kind: "file", + file: { bytes: bytes.toString("base64") }, + metadata: { [DATA_OBJECTS]: { path } }, + })); + const answer = answerSchema.parse( + await post({ + jsonrpc: "2.0", + id: 1, + method: "data/add", + params: { parts }, + }), + ); + if ("error" in answer) + throw new Error(`data/add failed: ${redacted(answer.error.message)}`); +}; + +/** The decoded file parts under paths of a computer user's home, each with the home path it names, through the driver's `data/get` sent by `post`. */ +export const readHomeFiles = async ( + post: (body: unknown) => Promise, + redacted: (text: string) => string, + paths: readonly string[], +): Promise< + readonly { readonly path: string | undefined; readonly bytes: Buffer }[] +> => { + const answer = dataGetSchema.parse( + await post({ + jsonrpc: "2.0", + id: 1, + method: "data/get", + params: { paths }, + }), + ); + if ("error" in answer) + throw new Error(`data/get failed: ${redacted(answer.error.message)}`); + return answer.result.parts.flatMap((part) => { + const encoded = part.file?.bytes; + return encoded === undefined + ? [] + : [ + { + path: Object.values(part.metadata ?? {})[0]?.path, + bytes: Buffer.from(encoded, "base64"), + }, + ]; + }); +}; diff --git a/src/desktop/ScreenJudgeProcess.integration.test.ts b/src/desktop/ScreenJudgeProcess.integration.test.ts new file mode 100644 index 0000000000..9659f25a3a --- /dev/null +++ b/src/desktop/ScreenJudgeProcess.integration.test.ts @@ -0,0 +1,89 @@ +import { execFileSync } from "node:child_process"; +import { chmod, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { ScreenJudgeProcess } from "./ScreenJudgeProcess.js"; + +const check = { id: "x", kind: "screen_judge", question: "ready?" } as const; + +it.skipIf(process.platform === "win32")( + "returns the judge's answer for its screenshot", + async () => { + const folder = await createTestTempDirectory("screen-answer-"); + const python = execFileSync("which", ["python3"], { + encoding: "utf8", + }).trim(); + const runner = join(folder, "agent-env"); + const verifier = join(folder, "verify.py"); + await writeFile(runner, `#!${python}\n`); + await chmod(runner, 0o700); + await writeFile( + verifier, + 'import json, sys\nprint(json.dumps({"id": "x", "result": open(sys.argv[3], "rb").read() == b"image", "description": "checked"}))\n', + ); + const judge = new ScreenJudgeProcess({ + ...process.env, + REA_AGENT_ENV_COMMAND: runner, + }); + expect( + await judge.judge( + verifier, + check, + Buffer.from("image"), + new AbortController().signal, + ), + ).toEqual({ id: "x", passed: true, description: "checked" }); + expect(await judge.close()).toMatchObject({ verified: true }); + }, +); + +it.skipIf(process.platform === "win32")( + "stops the judge and a separate-session child before cancellation settles", + async () => { + const folder = await createTestTempDirectory("screen-owner-"); + const python = execFileSync("which", ["python3"], { + encoding: "utf8", + }).trim(); + const runner = join(folder, "agent-env"); + const verifier = join(folder, "verify.py"); + const marker = join(folder, "child.pid"); + await writeFile(runner, `#!${python}\n`); + await chmod(runner, 0o700); + await writeFile( + verifier, + `import subprocess, sys, time\np = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True)\nopen(${JSON.stringify(marker)}, "w").write(str(p.pid))\ntime.sleep(60)\n`, + ); + const judge = new ScreenJudgeProcess({ + ...process.env, + REA_AGENT_ENV_COMMAND: runner, + }); + const controller = new AbortController(); + const running = judge.judge( + verifier, + check, + new Uint8Array([1]), + controller.signal, + ); + let pid = 0; + await vi.waitFor( + async () => { + pid = Number(await readFile(marker, "utf8")); + expect(pid).toBeGreaterThan(0); + }, + { timeout: 10000 }, + ); + controller.abort(new Error("cancelled by caller")); + await expect(running).rejects.toThrow(); + let state = ""; + try { + state = execFileSync("ps", ["-p", String(pid), "-o", "stat="], { + encoding: "utf8", + }).trim(); + } catch { + /* Process no longer exists. */ + } + expect(state === "" || state.startsWith("Z")).toBe(true); + expect(await judge.close()).toMatchObject({ verified: true }); + }, +); diff --git a/src/desktop/ScreenJudgeProcess.test.ts b/src/desktop/ScreenJudgeProcess.test.ts new file mode 100644 index 0000000000..0098e60ae9 --- /dev/null +++ b/src/desktop/ScreenJudgeProcess.test.ts @@ -0,0 +1,325 @@ +import { existsSync } from "node:fs"; +import { chmod } from "node:fs/promises"; +import { dirname } from "node:path"; +import { expect, it, vi } from "vitest"; + +import { + OwnedCommandFailure, + runOwnedCommand, +} from "../process/OwnedCommand.js"; +import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { ScreenJudgeProcess } from "./ScreenJudgeProcess.js"; + +vi.mock("./AgentEnvComputerHelper.js", async (importOriginal) => ({ + ...(await importOriginal()), + agentEnvInterpreter: vi.fn(async () => "/python"), +})); +vi.mock("../process/OwnedCommand.js", async (importOriginal) => ({ + ...(await importOriginal()), + runOwnedCommand: vi.fn(), +})); + +it("retains the screenshot and process owner after cancelled cleanup until retry succeeds", async () => { + const stop = vi + .fn() + .mockResolvedValueOnce({ status: "incomplete", reason: "still running" }) + .mockResolvedValueOnce({ status: "stopped" }); + const dispose = vi.fn(); + const owner = { stop, dispose } as unknown as ProviderProcessSupervisor; + const cancelled = new OwnedCommandFailure( + "cancelled", + "cancelled", + null, + "still running", + undefined, + ["run"], + owner, + ); + vi.mocked(runOwnedCommand).mockRejectedValueOnce(cancelled); + const judge = new ScreenJudgeProcess({}); + const controller = new AbortController(); + const failure: unknown = await judge + .judge( + "/verifier", + { id: "x", kind: "screen_judge", question: "ready?" }, + new Uint8Array([1]), + controller.signal, + ) + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProviderCleanupError); + expect((failure as ProviderCleanupError).cause).toBe(cancelled); + expect((failure as ProviderCleanupError).cleanupPrimary?._tag).toBe( + "AnalysisCancelledError", + ); + const call = vi.mocked(runOwnedCommand).mock.calls.at(-1); + if (call === undefined) throw new Error("judge was not launched"); + const folder = dirname(call[0].arguments[3]!); + expect(existsSync(folder)).toBe(true); + expect(await judge.close()).toMatchObject({ + verified: false, + resources: ["run", folder], + }); + expect(existsSync(folder)).toBe(true); + if (process.platform !== "win32" && process.getuid?.() !== 0) { + await chmod(folder, 0o500); + try { + expect(await judge.close()).toMatchObject({ + verified: false, + resources: [folder], + }); + expect((failure as ProviderCleanupError).cleanupResources).toEqual([ + folder, + ]); + } finally { + await chmod(folder, 0o700); + } + } + expect(await judge.close()).toMatchObject({ verified: true }); + expect(existsSync(folder)).toBe(false); + expect(stop).toHaveBeenCalledTimes(2); + expect(dispose).toHaveBeenCalledTimes(1); + expect((failure as ProviderCleanupError).cleanupIncomplete).toBe(false); +}); + +it("retries retained cleanup before launching another judge", async () => { + const stop = vi + .fn() + .mockResolvedValueOnce({ status: "incomplete", reason: "still running" }) + .mockResolvedValueOnce({ status: "stopped" }); + const owner = { + stop, + dispose: vi.fn(), + } as unknown as ProviderProcessSupervisor; + vi.mocked(runOwnedCommand).mockRejectedValueOnce( + new OwnedCommandFailure( + "cancelled", + "cancelled", + null, + "still running", + undefined, + ["run"], + owner, + ), + ); + const judge = new ScreenJudgeProcess({}); + const check = { id: "x", kind: "screen_judge", question: "ready?" } as const; + const signal = new AbortController().signal; + const first = await judge + .judge("/verifier", check, new Uint8Array([1]), signal) + .catch((cause: unknown) => cause); + const launches = vi.mocked(runOwnedCommand).mock.calls.length; + try { + await expect( + judge.judge("/verifier", check, new Uint8Array([1]), signal), + ).rejects.toMatchObject({ cleanupIncomplete: true }); + expect(vi.mocked(runOwnedCommand).mock.calls).toHaveLength(launches); + expect((first as ProviderCleanupError).cleanupIncomplete).toBe(true); + } finally { + const released = await judge.close(); + if (!released.verified) await judge.close(); + } + expect((first as ProviderCleanupError).cleanupIncomplete).toBe(false); +}); + +it("starts a later judge once retained cleanup succeeds", async () => { + const owner = { + stop: vi.fn().mockResolvedValue({ status: "stopped" }), + dispose: vi.fn(), + } as unknown as ProviderProcessSupervisor; + const text = JSON.stringify({ id: "x", result: true, description: "ready" }); + vi.mocked(runOwnedCommand) + .mockRejectedValueOnce( + new OwnedCommandFailure( + "process", + "failed", + null, + "running", + undefined, + ["run"], + owner, + ), + ) + .mockResolvedValueOnce({ + exitCode: 0, + stdout: { + text, + bytes: Buffer.byteLength(text), + observedBytes: Buffer.byteLength(text), + }, + stderr: { text: "", bytes: 0, observedBytes: 0 }, + signal: null, + }); + const judge = new ScreenJudgeProcess({}); + const check = { id: "x", kind: "screen_judge", question: "ready?" } as const; + const signal = new AbortController().signal; + const first: unknown = await judge + .judge("/verifier", check, new Uint8Array([1]), signal) + .catch((cause: unknown) => cause); + expect(first).toBeInstanceOf(ProviderCleanupError); + expect( + await judge.judge("/verifier", check, new Uint8Array([1]), signal), + ).toEqual({ id: "x", passed: true, description: "ready" }); + expect((first as ProviderCleanupError).cleanupIncomplete).toBe(false); + expect(await judge.close()).toMatchObject({ verified: true }); +}); + +it("reports a completed answer when process cleanup subsequently fails", async () => { + const text = JSON.stringify({ + id: "x", + result: true, + description: "observed", + }); + const snapshot = { + exitCode: 0, + stdout: { + text, + bytes: Buffer.byteLength(text), + observedBytes: Buffer.byteLength(text), + }, + stderr: { text: "", bytes: 0, observedBytes: 0 }, + signal: null, + }; + const owner = { + stop: vi.fn().mockResolvedValue({ status: "stopped" }), + dispose: vi.fn(), + } as unknown as ProviderProcessSupervisor; + vi.mocked(runOwnedCommand).mockRejectedValueOnce( + new OwnedCommandFailure( + "process", + "cleanup failed", + snapshot, + "still running", + undefined, + ["run"], + owner, + ), + ); + const judge = new ScreenJudgeProcess({}); + const failure: unknown = await judge + .judge( + "/verifier", + { id: "x", kind: "screen_judge", question: "ready?" }, + new Uint8Array([1]), + new AbortController().signal, + ) + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProviderCleanupError); + expect(projectAnalysisError(failure as ProviderCleanupError)).toMatchObject({ + details: { + diagnostics: { + completed_judge_result: { + id: "x", + passed: true, + description: "observed", + }, + }, + }, + }); + expect(await judge.close()).toMatchObject({ verified: true }); +}); + +it("waits for final process cleanup after cancellation and keeps its owner", async () => { + const controller = new AbortController(); + const reason = new Error("cancel during cleanup"); + const owner = { + stop: vi.fn().mockResolvedValue({ status: "stopped" }), + dispose: vi.fn(), + } as unknown as ProviderProcessSupervisor; + let rejectCommand: (cause: unknown) => void = () => {}; + vi.mocked(runOwnedCommand).mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectCommand = reject; + }), + ); + const judge = new ScreenJudgeProcess({}); + let settled = false; + const running = judge + .judge( + "/verifier", + { id: "x", kind: "screen_judge", question: "ready?" }, + new Uint8Array([1]), + controller.signal, + ) + .catch((cause: unknown) => cause) + .finally(() => { + settled = true; + }); + // Wait for the mocked command to receive control, not just for the judge to start. + await vi.waitFor(() => + expect(vi.mocked(runOwnedCommand).mock.calls.at(-1)?.[2]?.signal).toBe( + controller.signal, + ), + ); + controller.abort(reason); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + rejectCommand( + new OwnedCommandFailure( + "cancelled", + "cancelled", + null, + "still running", + { cancellation: { reason } }, + ["run"], + owner, + ), + ); + const failure = await running; + expect(failure).toBeInstanceOf(ProviderCleanupError); + expect((failure as ProviderCleanupError).cleanupPrimary?.cause).toBe(reason); + expect(await judge.close()).toMatchObject({ verified: true }); +}); + +it("keeps a completed answer when cancellation arrives during successful cleanup", async () => { + const text = JSON.stringify({ + id: "x", + result: true, + description: "completed", + }); + const snapshot = { + exitCode: 0, + stdout: { + text, + bytes: Buffer.byteLength(text), + observedBytes: Buffer.byteLength(text), + }, + stderr: { text: "", bytes: 0, observedBytes: 0 }, + signal: null, + }; + const reason = new Error("cancel during cleanup"); + vi.mocked(runOwnedCommand).mockRejectedValueOnce( + new OwnedCommandFailure("cancelled", "cancelled", snapshot, null, { + cancellation: { reason }, + }), + ); + const judge = new ScreenJudgeProcess({}); + const failure: unknown = await judge + .judge( + "/verifier", + { id: "x", kind: "screen_judge", question: "ready?" }, + new Uint8Array([1]), + new AbortController().signal, + ) + .catch((cause: unknown) => cause); + expect( + projectAnalysisError( + failure as import("../domain/analysisErrorBase.js").AnalysisError, + ), + ).toMatchObject({ + code: "cancelled", + details: { + diagnostics: { + completed_judge_result: { + id: "x", + passed: true, + description: "completed", + }, + }, + }, + }); + expect((failure as Error).cause).toBe(reason); + expect(await judge.close()).toMatchObject({ verified: true }); +}); diff --git a/src/desktop/ScreenJudgeProcess.ts b/src/desktop/ScreenJudgeProcess.ts new file mode 100644 index 0000000000..909095d304 --- /dev/null +++ b/src/desktop/ScreenJudgeProcess.ts @@ -0,0 +1,248 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { z } from "zod"; + +import type { ScreenJudge } from "../domain/desktop/appTaskScreen.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { RetainedCommandCleanup } from "../process/RetainedCommandCleanup.js"; +import { + OwnedCommandFailure, + runOwnedCommand, +} from "../process/OwnedCommand.js"; +import { + AGENTENV_PROVIDER, + agentEnvInterpreter, + type ComputerRelease, +} from "./AgentEnvComputerHelper.js"; + +const JUDGE_TIMEOUT_MS = 10 * 60 * 1000; +const answerSchema = z.object({ + id: z.string(), + result: z.boolean(), + description: z.string(), +}); + +interface Retained { + readonly owner: RetainedCommandCleanup; + readonly failure: ProviderCleanupError; +} + +/** A complete answer remains evidence even when subsequent process cleanup fails. */ +const completedAnswer = ( + cause: unknown, +): Awaited> | undefined => { + if ( + !(cause instanceof OwnedCommandFailure) || + cause.collectionFailure || + cause.snapshot?.exitCode !== 0 || + cause.snapshot.outputIncomplete === true + ) + return undefined; + const stdout = cause.snapshot.stdout; + if (stdout.invalidUtf8 || stdout.bytes !== stdout.observedBytes) + return undefined; + try { + const answer = answerSchema.safeParse( + JSON.parse(stdout.text.trim().split("\n").at(-1) ?? ""), + ); + return answer.success + ? { + id: answer.data.id, + passed: answer.data.result, + description: answer.data.description, + } + : undefined; + } catch { + return undefined; + } +}; + +/** Own the judge, its descendants and the screenshot until cleanup is confirmed. */ +export class ScreenJudgeProcess { + readonly #retained = new Set(); + + constructor(readonly environment: Readonly) {} + + readonly judge: ScreenJudge = async (verifier, check, image, signal) => { + const cleanup = await this.close(); + if (!cleanup.verified) { + const pending = [...this.#retained].map((item) => item.failure); + if (pending.length === 1) throw pending[0]; + throw new ProviderCleanupError( + AGENTENV_PROVIDER, + cleanup.resources, + { reason: cleanup.reason }, + { + operation: "check_build_task", + remediationAction: "Close the build computer to retry judge cleanup.", + }, + ).bindCleanupDependencies(pending); + } + if (signal.aborted) + throw new AnalysisCancelledError("check_build_task", { + cause: signal.reason, + }); + const python = await agentEnvInterpreter( + this.environment, + "check_build_task", + ); + const folder = await mkdtemp(join(tmpdir(), "rea-screen-judge-")); + let retained = false; + let primary: { readonly cause: unknown } | undefined; + let outcome: Awaited> | undefined; + try { + const screen = join(folder, "screen.jpg"); + await writeFile(screen, image); + const snapshot = await runOwnedCommand( + { + command: python, + arguments: [verifier, "judge", JSON.stringify(check), screen], + runId: randomUUID(), + expectedCommand: null, + hostEnvironment: { ...this.environment }, + env: { PYTHONDONTWRITEBYTECODE: "1" }, + sweepTokenOwnedProcesses: true, + }, + { timeoutMs: JUDGE_TIMEOUT_MS, diagnosticBytes: 1024 * 1024 }, + { signal }, + ); + const answer = answerSchema.parse( + JSON.parse(snapshot.stdout.text.trim().split("\n").at(-1) ?? ""), + ); + outcome = { + id: answer.id, + passed: answer.result, + description: answer.description, + }; + } catch (cause: unknown) { + primary = { cause }; + const cleanup = this.#retainProcessFailure(folder, cause); + if (cleanup !== undefined) { + retained = true; + primary = { cause: cleanup }; + } else if (cause instanceof OwnedCommandFailure) { + primary = { + cause: + cause.reason === "cancelled" + ? new AnalysisCancelledError("check_build_task", { + cause: cause.cancellation?.reason, + }) + : new ProviderAdapterError( + AGENTENV_PROVIDER, + "check_build_task", + { + cause, + userMessage: `The bundle's screen judge failed: ${cause.snapshot?.stderr.text.trim().split("\n").slice(-3).join(" ") || cause.message}`, + }, + ), + }; + const completed = completedAnswer(cause); + if (completed !== undefined && primary.cause instanceof AnalysisError) + primary.cause.retainDiagnostics({ + completed_judge_result: completed, + }); + } + } + if (!retained) await this.#removeScreen(folder, outcome, primary?.cause); + if (primary !== undefined) throw primary.cause; + if (outcome === undefined) throw new Error("Judge produced no outcome."); + return outcome; + }; + + async #removeScreen( + folder: string, + outcome: Awaited> | undefined, + primary: unknown, + ): Promise { + try { + await rm(folder, { recursive: true, force: true }); + } catch (cause: unknown) { + const failure = new ProviderCleanupError( + AGENTENV_PROVIDER, + [folder], + { + reason: cause instanceof Error ? cause.message : String(cause), + ...(outcome === undefined ? {} : { completed_judge_result: outcome }), + }, + { + operation: "check_build_task", + cause: primary ?? cause, + ...(primary instanceof AnalysisError + ? { cleanupPrimary: primary } + : {}), + remediationAction: + "Close the build computer to retry screenshot cleanup.", + }, + ); + const owner = new RetainedCommandCleanup(folder, undefined); + failure.bindCleanupResourceOwner(owner); + this.#retained.add({ owner, failure }); + throw failure; + } + } + + #retainProcessFailure( + folder: string, + cause: unknown, + ): ProviderCleanupError | undefined { + const owner = new RetainedCommandCleanup(folder, cause); + if (!owner.hasProcess) return undefined; + const completed = completedAnswer(cause); + const original = + cause instanceof OwnedCommandFailure && cause.reason === "cancelled" + ? new AnalysisCancelledError("check_build_task", { + cause: cause.cancellation?.reason, + }) + : new ProviderAdapterError(AGENTENV_PROVIDER, "check_build_task", { + cause, + }); + const message = cause instanceof Error ? cause.message : String(cause); + const failure = new ProviderCleanupError( + AGENTENV_PROVIDER, + owner.cleanupResources, + { + reason: + (cause instanceof OwnedCommandFailure + ? cause.cleanupFailure + : null) ?? message, + primary_failure: message, + ...(completed === undefined + ? {} + : { completed_judge_result: completed }), + }, + { + operation: "check_build_task", + cause, + ...(completed === undefined ? { cleanupPrimary: original } : {}), + remediationAction: "Close the build computer to retry judge cleanup.", + }, + ); + failure.bindCleanupResourceOwner(owner); + this.#retained.add({ owner, failure }); + return failure; + } + + /** Retry unconfirmed process and screenshot cleanup before releasing the computer. */ + async close(): Promise { + const failures: string[] = []; + const resources: string[] = []; + for (const item of this.#retained) { + try { + await item.owner.close(); + this.#retained.delete(item); + item.failure.resolveCleanup(); + } catch (cause: unknown) { + failures.push(cause instanceof Error ? cause.message : String(cause)); + resources.push(...item.owner.cleanupResources); + } + } + return failures.length === 0 + ? { verified: true, teardown: "agentenv" } + : { verified: false, reason: failures.join("; "), resources }; + } +} diff --git a/src/desktop/TaskReaderProcess.ts b/src/desktop/TaskReaderProcess.ts new file mode 100644 index 0000000000..622e9587a2 --- /dev/null +++ b/src/desktop/TaskReaderProcess.ts @@ -0,0 +1,42 @@ +import { randomUUID } from "node:crypto"; + +import type { ReaderCommandResult } from "../domain/desktop/appTaskReaders.js"; +import { runOwnedCommand } from "../process/OwnedCommand.js"; + +/** The most a reader runner prints, its one answer line, plus room for what Python says when it fails. */ +const RUNNER_OUTPUT_BYTES = 2 * 1024 * 1024; + +/** + * Run a reader's runner command on this host, as the export does inside the + * build computer, as an owned process: a timeout or a cancellation stops the + * runner and every process carrying its run token, the reader's session + * included, before it settles. + */ +export const runReaderHere = async ( + argv: readonly string[], + timeoutS: number, + signal?: AbortSignal, +): Promise => { + const [command = "", ...rest] = argv; + const snapshot = await runOwnedCommand( + { + command, + arguments: rest, + runId: randomUUID(), + expectedCommand: null, + sweepTokenOwnedProcesses: true, + }, + { + timeoutMs: timeoutS * 1000, + diagnosticBytes: RUNNER_OUTPUT_BYTES, + acceptNonZeroExit: true, + }, + signal === undefined ? {} : { signal }, + ); + return { + exit_code: snapshot.exitCode ?? null, + stdout: snapshot.stdout.text, + stderr: snapshot.stderr.text, + timed_out: false, + }; +}; diff --git a/src/domain/desktop/appExport.test.ts b/src/domain/desktop/appExport.test.ts index c2f328cea0..0209b577ba 100644 --- a/src/domain/desktop/appExport.test.ts +++ b/src/domain/desktop/appExport.test.ts @@ -215,6 +215,17 @@ describe("renderers", () => { expect(renderAppClass({ ...plan, paths: [] })).not.toContain("@get_data"); }); + it("gives an app without owned paths a reset that only relaunches it when its task judges the screen", () => { + const relaunched = renderAppClass({ + ...plan, + paths: [], + relaunchOnReset: true, + }); + expect(relaunched).toContain("@reset_data\n async def relaunch(self)"); + expect(relaunched).not.toContain("@get_data"); + expect(renderAppClass({ ...plan, paths: [] })).not.toContain("@reset_data"); + }); + it("loads captured dconf settings even without owned paths", () => { const module = renderAppClass({ ...plan, diff --git a/src/domain/desktop/appExport.ts b/src/domain/desktop/appExport.ts index 45f671170c..8d4bc685ad 100644 --- a/src/domain/desktop/appExport.ts +++ b/src/domain/desktop/appExport.ts @@ -17,6 +17,7 @@ import { PYTHON_IMPORTS, PYTHON_MODULE_NAMES, PYTHON_READY_CHECK, + ownlessBody, } from "./appExportPython.js"; import { leftOutFiles, @@ -67,6 +68,8 @@ export interface DesktopAppExportPlan { readonly windowClass: string; /** Home-relative paths the app owns for data add, get and reset. */ readonly paths: readonly string[]; + /** Relaunch an ownless app for a screen-judged task. */ + readonly relaunchOnReset?: boolean; readonly capturedHome: string; /** The checkpoint whose state the seed holds. */ readonly seedCheckpoint: string; @@ -517,14 +520,11 @@ const classBody = (plan: DesktopAppExportPlan): string => { _ensure_state_aliases(Path.home()) keep_initial(self.paths) ${PYTHON_DATA_HOOKS}${PYTHON_READY_CHECK}`; - if (plan.dconf.length > 0) - return ` """${title}, launched at boot with its captured settings. It owns no files, so it serves no data operations.""" - - def __init__(self) -> None: - super().__init__() - _load_dconf(Path.home()) -${PYTHON_READY_CHECK}`; - return ` """${title}, launched at boot. No state was observed, so it serves no data operations."""\n${PYTHON_READY_CHECK}`; + return ownlessBody( + title, + plan.dconf.length > 0, + plan.relaunchOnReset === true, + ); }; /** The app's package, or a package file's dependencies, then the extra packages, its program's package, its library packages and dconf-cli when settings are loaded. */ diff --git a/src/domain/desktop/appExportInstallLimitations.ts b/src/domain/desktop/appExportInstallLimitations.ts index fc909a0a3b..e1bbb6752b 100644 --- a/src/domain/desktop/appExportInstallLimitations.ts +++ b/src/domain/desktop/appExportInstallLimitations.ts @@ -78,6 +78,11 @@ export const installLimitations = ( : [ `The computer installs ${install.extraPackages.join(", ")} beside the application because the export was asked for them with --extra-package, as a build computer asks for the packages its request named.`, ]), + ...(install.extraPackages === undefined + ? [] + : [ + `The computer installs ${install.extraPackages.join(", ")} beside the application because the export was asked for them with --extra-package, as a build computer asks for the packages its request named.`, + ]), ...install.skipped.map( (repository) => `The apt repository ${repository} was left out.`, ), diff --git a/src/domain/desktop/appExportPython.ts b/src/domain/desktop/appExportPython.ts index 232f10bcd2..c075fd22c7 100644 --- a/src/domain/desktop/appExportPython.ts +++ b/src/domain/desktop/appExportPython.ts @@ -432,6 +432,36 @@ export const PYTHON_MODULE_NAMES: ReadonlySet = new Set([ ...PYTHON_CONSTANTS, ]); +/** + * The class body of an app that owns no files: launched at boot, its captured + * dconf settings loaded if any, serving no data operations, or, when its task + * judges its screen, a reset that only relaunches it. + */ +export const ownlessBody = ( + title: string, + dconf: boolean, + relaunch: boolean, +): string => { + const reset = relaunch + ? ` + @reset_data + async def relaunch(self) -> None: + """Nothing to restore: the app is stopped before this and relaunched after it, back to its first screen.""" +` + : ""; + const serves = relaunch + ? "and keeps its task's state on screen, so its reset only relaunches it" + : "so it serves no data operations"; + if (dconf) + return ` """${title}, launched at boot with its captured settings. It owns no files, ${serves}.""" + + def __init__(self) -> None: + super().__init__() + _load_dconf(Path.home()) +${reset}${PYTHON_READY_CHECK}`; + return ` """${title}, launched at boot. No state was observed, ${serves}."""\n${reset}${PYTHON_READY_CHECK}`; +}; + /** The data hooks of an app that owns paths; `_views` reads files, so it runs in a thread. */ export const PYTHON_DATA_HOOKS = ` @add_data diff --git a/src/domain/desktop/appExportPythonJson.ts b/src/domain/desktop/appExportPythonJson.ts index 9e55591e5f..ffa37079c1 100644 --- a/src/domain/desktop/appExportPythonJson.ts +++ b/src/domain/desktop/appExportPythonJson.ts @@ -1,3 +1,5 @@ +import { PYTHON_EXACT_JSON_NUMBER } from "./pythonExactJsonNumber.js"; + /** JSON parsing and bounded retained-view checks for exported desktop environments. */ export const PYTHON_JSON_READER = String.raw` #: A JSON token after any whitespace, by group: an opening, a closing, a comma, a colon, a string's quote, or a number @@ -7,40 +9,7 @@ JSON_TOKEN = re.compile(r'[ \t\n\r]*(?:([\[{])|([\]}])|(,)|(:)|(")|(-?(?:0|[1-9] JSON_SPACE = re.compile(r"[ \t\n\r]*") -class _Inexact(Exception): - """A JSON number that REA's state view refuses, since reading it would change it.""" - - -def _digits(text: str) -> str: - return re.split(r"[eE]", text.lstrip("-"))[0].replace(".", "").strip("0") or "0" - - -def _js_string(value: float) -> str: - """How JavaScript's String() writes a finite double: an integer below 1e21 in full, any other in shortest form.""" - return str(int(value)) if value.is_integer() and abs(value) < 1e21 else repr(value) - - -def _safe_number(text: str) -> bool: - """lossless-json's isSafeNumber: an integer spelling within 2^53, or a float spelling that keeps its digits.""" - if re.fullmatch(r"-?[0-9]+", text): - return len(text.lstrip("-")) <= 16 and abs(int(text)) <= MAX_SAFE_INTEGER - value = float(text) - if not math.isfinite(value): - return False - written = _js_string(value) - return written == text or _digits(written) == _digits(text) - - -def _exact_number(text: str): - """A JSON number read only when reading it changes nothing, as REA's state view rules: its spelling and its - written-back form both safe, and not -0.""" - value = float(text) - if (not _safe_number(text) or not math.isfinite(value) or not _safe_number(_js_string(value)) - or (value == 0 and math.copysign(1, value) < 0)): - raise _Inexact(text) - return int(text) if re.fullmatch(r"-?[0-9]+", text) else value - - +${PYTHON_EXACT_JSON_NUMBER} _INEXACT = object() _DEEP = object() _DEEP_INEXACT = object() diff --git a/src/domain/desktop/appTask.ts b/src/domain/desktop/appTask.ts index 66d3d0b61e..7932ca7e68 100644 --- a/src/domain/desktop/appTask.ts +++ b/src/domain/desktop/appTask.ts @@ -55,22 +55,37 @@ const kind = ( ...shape, }); +/** What a reader's argv or stdin names the file by: the path of a private copy of it. */ +export const READER_FILE_PLACEHOLDER = "{file}"; + +const readerField = { + reader: z + .string() + .min(1) + .optional() + .describe( + "The id of one of the task's readers: the check reads that reader's view of its file instead of the file's own. Give the reader's file as file.", + ), +}; + const checkSchema = z.discriminatedUnion("kind", [ kind("file_present", {}), kind("file_absent", {}), - kind("lines_present", { lines }), - kind("lines_absent", { lines }), + kind("lines_present", { lines, ...readerField }), + kind("lines_absent", { lines, ...readerField }), kind("line_count", { exact: line.optional(), contains: line.optional(), ...comparison, + ...readerField, }), kind("value_equals", { pointer: z.string(), value: jsonValueSchema, tolerance: z.number().min(0).optional(), + ...readerField, }), - kind("value_absent", { pointer: z.string() }), + kind("value_absent", { pointer: z.string(), ...readerField }), kind("text_equals", wholeText), kind("text_contains", { ...wholeText, value: z.string().min(1) }), kind("row_present", { table, where }), @@ -85,28 +100,112 @@ const checkSchema = z.discriminatedUnion("kind", [ "A selector of one or more [KEY=VALUE], [@KEY=VALUE] or [>PATH=VALUE] conditions that a member must meet to count, where PATH is up to three /-separated steps inside the member; every member counts without it, text nodes of an XML element's children included.", ), ...comparison, + ...readerField, + }), + z.strictObject({ + id: z.string().min(1).max(64).optional(), + kind: z.literal("screen_judge"), + question: z + .string() + .trim() + .min(1) + .max(2000) + .describe( + "A yes-or-no question about the app's screen that the task done makes true and the seed makes false, such as Does the display show 8.568?; a model judges it on a screenshot, for state no file shows.", + ), }), ]); -/** A task file as an agent writes it: a portable name, a prompt and declarative checks. */ +/** How long a reader may run when it does not say, and the most it may ask for. */ +export const READER_TIMEOUT_S = { default: 60, max: 300 } as const; + +const readerSchema = z.strictObject({ + id: z + .string() + .regex(/^[a-z0-9][a-z0-9-]{0,63}$/u) + .describe( + "Lowercase letters, digits and hyphens; a check names the reader by it.", + ), + file: z + .string() + .min(1) + .describe( + "The file the reader reads, a path under the home such as Documents/poster.xcf.", + ), + argv: z + .array(z.string().min(1)) + .min(1) + .max(64) + .describe( + `The app's own headless command, run in the computer on a private copy of the file that ${READER_FILE_PLACEHOLDER} names, in a fresh home without the display; each line it prints starting with "REA " is a line of its view.`, + ), + stdin: z + .string() + .max(64 * 1024) + .optional() + .describe( + `Text sent to the command's stdin, where ${READER_FILE_PLACEHOLDER} also names the copy.`, + ), + output: z + .enum(["json", "lines"]) + .describe( + "json: the marked lines, joined, are one JSON value that value_equals, value_absent and member_count point into; lines: they are the lines lines_present, lines_absent and line_count read.", + ), + timeout_s: z + .int() + .min(1) + .max(READER_TIMEOUT_S.max) + .optional() + .describe( + `Seconds the command may run, ${String(READER_TIMEOUT_S.default)} when omitted.`, + ), + display: z + .boolean() + .optional() + .describe( + "Run the command with the desktop's display, for an app without a headless mode; off when omitted.", + ), +}); + +/** A task file as an agent writes it: a portable name, a prompt, declarative checks and the readers some read through. */ export const desktopTaskSchema = z.strictObject({ name: z.string().min(1), prompt: z.string().trim().min(1), checks: z.array(checkSchema).min(1).max(32), + readers: z + .array(readerSchema) + .min(1) + .max(8) + .optional() + .describe( + "Commands that read a file REA cannot read itself, such as a binary document, with the app's own headless mode; a check reads one's view by naming it in reader.", + ), }); /** One check as written, its id optional. */ export type DesktopTaskCheckInput = z.infer; /** One check with its id settled. */ export type DesktopTaskCheck = DesktopTaskCheckInput & { readonly id: string }; +/** A command that reads one file with the app's own headless mode, for checks to read its view. */ +export type DesktopTaskReader = z.infer; -/** A task whose checks are well formed and have unique ids. */ +/** A task whose checks are well formed and have unique ids, with the readers its checks read through, if any. */ export interface DesktopTask { readonly name: string; readonly prompt: string; readonly checks: readonly DesktopTaskCheck[]; + readonly readers?: readonly DesktopTaskReader[]; } +/** The check kinds each reader output serves. */ +const READER_KINDS = { + json: ["value_equals", "value_absent", "member_count"], + lines: ["lines_present", "lines_absent", "line_count"], +} as const satisfies Record< + DesktopTaskReader["output"], + readonly DesktopTaskCheck["kind"][] +>; + /** Checks that count lines, rows or the members of an array. */ export type CountCheck = Extract< DesktopTaskCheck, @@ -201,7 +300,9 @@ const pointerProblem = (check: PointerCheck): string | undefined => { }; /** What is wrong with one check beyond its shape, or undefined. */ -const checkProblem = (check: DesktopTaskCheck): string | undefined => { +const checkProblem = ( + check: Exclude, +): string | undefined => { if (!isHomePath(check.file)) return `file ${JSON.stringify(check.file)} must be a path under the home, such as Documents/todo.txt, without a leading slash, . or .. segments`; if ( @@ -241,6 +342,64 @@ const checkProblem = (check: DesktopTaskCheck): string | undefined => { : `${JSON.stringify(stamped)} looks like a time the app stamps; leave volatile values out of checks`; }; +/** What is wrong with one reader on its own, or undefined. */ +const readerProblem = (reader: DesktopTaskReader): string | undefined => { + if (!isHomePath(reader.file)) + return `file ${JSON.stringify(reader.file)} must be a path under the home, such as Documents/poster.xcf, without a leading slash, . or .. segments`; + return [...reader.argv, reader.stdin ?? ""].some((part) => + part.includes(READER_FILE_PLACEHOLDER), + ) + ? undefined + : `its argv and stdin name no ${READER_FILE_PLACEHOLDER}, the copy of ${reader.file} it reads`; +}; + +/** What is wrong with the readers and the checks that read through them, or undefined. */ +const readersProblem = ( + readers: readonly DesktopTaskReader[], + checks: readonly DesktopTaskCheck[], +): string | undefined => { + const ids = readers.map((reader) => reader.id); + const repeated = ids.find((id, index) => ids.indexOf(id) !== index); + if (repeated !== undefined) + return `two readers have the id ${JSON.stringify(repeated)}.`; + for (const reader of readers) { + const problem = readerProblem(reader); + if (problem !== undefined) + return `reader ${JSON.stringify(reader.id)}: ${problem}.`; + } + for (const check of checks) { + if (!("reader" in check) || check.reader === undefined) continue; + const reader = readers.find(({ id }) => id === check.reader); + if (reader === undefined) + return `check ${JSON.stringify(check.id)} names reader ${JSON.stringify(check.reader)}, which the task does not define in readers.`; + if (check.file !== reader.file) + return `check ${JSON.stringify(check.id)} reads ${reader.id}, so its file is ${reader.file}, not ${check.file}.`; + const kinds: readonly string[] = READER_KINDS[reader.output]; + if (!kinds.includes(check.kind)) + return `check ${JSON.stringify(check.id)} is ${check.kind}, but reader ${reader.id} reads ${reader.output}, which ${kinds.join(", ").replace(/, (?=[^,]*$)/u, " and ")} check.`; + } + const unread = readers.find( + (reader) => + !checks.some((check) => "reader" in check && check.reader === reader.id), + ); + return unread === undefined + ? undefined + : `reader ${JSON.stringify(unread.id)} is read by no check; remove it or name it in a check's reader.`; +}; + +/** What is wrong with a task's screen judges beside its other checks and readers, or undefined. */ +const screenProblem = ( + checks: readonly DesktopTaskCheck[], + readers: readonly DesktopTaskReader[] | undefined, +): string | undefined => { + if (!checks.some(({ kind }) => kind === "screen_judge")) return undefined; + if (checks.some(({ kind }) => kind !== "screen_judge")) + return "a task with screen_judge checks holds only screen_judge checks: they judge state no file shows, so a change a file shows is checked in that file instead."; + return readers === undefined + ? undefined + : "a task with screen_judge checks takes no readers."; +}; + /** Parse a task file and settle its check ids, or say what is wrong with it. */ export const parseDesktopTask = ( value: unknown, @@ -248,7 +407,7 @@ export const parseDesktopTask = ( ): Result => { const parsed = desktopTaskSchema.safeParse(value); if (!parsed.success) return err(describeValidationFailure(parsed.error)); - const { name, prompt, checks } = parsed.data; + const { name, prompt, checks, readers } = parsed.data; if (!isPortableName(name) || name === "smoke") return err( `name ${JSON.stringify(name)} must be a portable bundle name other than smoke: lowercase letters, digits and hyphens.`, @@ -266,9 +425,18 @@ export const parseDesktopTask = ( if (repeated !== undefined) return err(`two checks have the id ${JSON.stringify(repeated)}.`); for (const check of settled) { - const problem = checkProblem(check); + const problem = + check.kind === "screen_judge" ? undefined : checkProblem(check); if (problem !== undefined) return err(`check ${JSON.stringify(check.id)}: ${problem}.`); } - return ok({ name, prompt: prompt.trim(), checks: settled }); + const problem = + screenProblem(settled, readers) ?? readersProblem(readers ?? [], settled); + if (problem !== undefined) return err(problem); + return ok({ + name, + prompt: prompt.trim(), + checks: settled, + ...(readers === undefined ? {} : { readers }), + }); }; diff --git a/src/domain/desktop/appTaskChecks.ts b/src/domain/desktop/appTaskChecks.ts index 68cf5483f7..ec79e26941 100644 --- a/src/domain/desktop/appTaskChecks.ts +++ b/src/domain/desktop/appTaskChecks.ts @@ -57,6 +57,8 @@ export type TaskCheckOutcome = { }; type Verdict = Omit; +/** A check that reads a file's view, every kind but the screen judge. */ +type FileCheck = Exclude; type Check = Extract< DesktopTaskCheck, { kind: Kind } @@ -151,7 +153,7 @@ const truncatedView = (format: string | null, view: JsonValue | null) => { /** A file's view of one format, or why the check cannot read it; an absent file reads as empty. */ const viewOf = ( - check: DesktopTaskCheck, + check: FileCheck, entry: TaskFileView | undefined, format: string | undefined, ): { view: JsonValue | null; truncated: boolean } | Verdict => { @@ -566,7 +568,7 @@ const rowVerdict = ( }; const verdict = ( - check: DesktopTaskCheck, + check: FileCheck, entry: TaskFileView | undefined, ): Verdict => { switch (check.kind) { @@ -597,12 +599,22 @@ const verdict = ( } }; -/** Evaluate each check against views by home path, as the exported verifier does. */ +/** Evaluate each check against views by home path, or its reader's view of its file by reader id, as the exported verifier does. */ export const evaluateTaskChecks = ( checks: readonly DesktopTaskCheck[], views: ReadonlyMap, + readers: ReadonlyMap = new Map(), ): readonly TaskCheckOutcome[] => checks.map((check) => ({ id: check.id, - ...verdict(check, views.get(check.file)), + ...(check.kind === "screen_judge" + ? unread( + "a screen judge's question is answered from the computer's screen by the bundle's judge, which REA's evaluator does not run", + ) + : verdict( + check, + "reader" in check && check.reader !== undefined + ? readers.get(check.reader) + : views.get(check.file), + )), })); diff --git a/src/domain/desktop/appTaskDraftFileNotes.test.ts b/src/domain/desktop/appTaskDraftFileNotes.test.ts index 01323443a4..40b0f7202b 100644 --- a/src/domain/desktop/appTaskDraftFileNotes.test.ts +++ b/src/domain/desktop/appTaskDraftFileNotes.test.ts @@ -65,4 +65,14 @@ describe("draftTaskFiles notes on thin files", () => { ["Documents/b.xml", ["cut_view"]], ]); }); + + it("points a binary file at a reader, the app's own headless mode, as the way to check inside it", () => { + const [binary] = draftOf({ + "Documents/poster.xcf": statePath("document", "binary", {}, {}), + }); + + const message = binary?.notes[0]?.message ?? ""; + expect(message).toContain("readers"); + expect(message).toContain("headless"); + }); }); diff --git a/src/domain/desktop/appTaskDraftFileNotes.ts b/src/domain/desktop/appTaskDraftFileNotes.ts index e17858575b..cdabcbeb63 100644 --- a/src/domain/desktop/appTaskDraftFileNotes.ts +++ b/src/domain/desktop/appTaskDraftFileNotes.ts @@ -65,7 +65,7 @@ const viewNotes = (file: string, change: ChangeShape): FileNote[] => { file, message: change.format === "binary" - ? `REA reads ${home} as binary, so no check can look inside it: save it in a format the app offers that REA reads, such as XML, JSON or text, plain or gzip-compressed, or a zip document, and capture again.` + ? `REA reads ${home} as binary, so no check can look inside it on its own: give the task one of its readers, the app's own headless command printing what the file holds, and check that reader's view; or save it in a format the app offers that REA reads, such as XML, JSON or text, plain or gzip-compressed, or a zip document, and capture again.` : `REA could not read ${home} at one end the same way as at the other, so no check can look inside it; its versions' view_error says why.`, }, ]; diff --git a/src/domain/desktop/appTaskPython.ts b/src/domain/desktop/appTaskPython.ts index afb4709549..d06086e564 100644 --- a/src/domain/desktop/appTaskPython.ts +++ b/src/domain/desktop/appTaskPython.ts @@ -2,6 +2,15 @@ import type { JsonValue } from "../jsonValue.js"; import { SETTLE_TIMING } from "./appSettle.js"; import type { DesktopTask } from "./appTask.js"; import { PYTHON_POINTER } from "./appTaskPythonPointer.js"; +import { + PYTHON_READER_VIEWS, + renderReaderConstants, +} from "./appTaskReaders.js"; +import { + isScreenTask, + renderScreenVerifier, + type ScreenExamples, +} from "./appTaskScreen.js"; /** The verifier's evaluator: the checks' semantics, kept equal to `evaluateTaskChecks`. */ export const PYTHON_TASK_EVALUATOR = String.raw` @@ -624,8 +633,22 @@ export const sameTaskVerifier = (hosted: string, bundled: string): boolean => { ); }; -/** The verifier artifact: read the app's state through its data/get once its files settle, then evaluate the task's checks. */ -export const renderTaskVerifier = (task: DesktopTask, app: string): string => +/** The verifier artifact: a screen-judged task's judge, with the build's examples, or else reads of the app's state through its data/get once its files settle and the task's checks on them. */ +export const renderTaskVerifier = ( + task: DesktopTask, + app: string, + examples?: ScreenExamples, +): string => { + if (!isScreenTask(task)) return renderFileVerifier(task, app); + if (examples === undefined) + throw new TypeError( + "A screen-judged task's verifier needs the build's examples.", + ); + return renderScreenVerifier(task, app, examples); +}; + +/** The verifier of a task whose checks read files: the app's data/get once its files settle, then the checks. */ +const renderFileVerifier = (task: DesktopTask, app: string): string => `"""Checks of the task ${task.name} against the ${app} app's state, exported by REA from a desktop application capture. It reads the app's data/get through the computer, takes the typed views of its files and evaluates each check: no @@ -640,7 +663,7 @@ import json import math import re import time - +${task.readers === undefined ? "" : "import sys\nimport types\nimport uuid\n"} from agentenv_protocol import client as protocol from agentenv_protocol.types import RPC_PATH @@ -657,7 +680,7 @@ SETTLE_WINDOW_S = ${pythonSeconds(SETTLE_TIMING.windowMs)} SETTLE_BOUND_S = ${pythonSeconds(SETTLE_TIMING.boundMs)} #: Where a data/get file part names its path in the home. DATA_OBJECTS = "urn:agentenv:data-objects/v1" -${PYTHON_TASK_EVALUATOR}${PYTHON_SETTLE} +${task.readers === undefined ? "" : renderReaderConstants(task.readers)}${PYTHON_TASK_EVALUATOR}${PYTHON_SETTLE}${task.readers === undefined ? "" : PYTHON_READER_VIEWS} async def verify(mcp_url, env_url): card = await protocol.get_card(env_url) @@ -665,7 +688,7 @@ async def verify(mcp_url, env_url): if child is None: raise RuntimeError(f"the computer at {env_url} lists no app named {APP}") views, settled, read_cut = await _settled_views(env_url + (child.get("url") or RPC_PATH).removesuffix(RPC_PATH)) - return [{**result, "settled": settled, "read_cut": read_cut} for result in evaluate(CHECKS, views)] +${task.readers === undefined ? "" : " readers = await _reader_views(env_url, card)\n"} return [{**result, "settled": settled, "read_cut": read_cut} for result in ${task.readers === undefined ? "evaluate(CHECKS, views)" : "evaluate_with_readers(CHECKS, views, readers)"}] `; /** The artifact folder of a task's verifier. */ diff --git a/src/domain/desktop/appTaskReaders.test.ts b/src/domain/desktop/appTaskReaders.test.ts new file mode 100644 index 0000000000..c8fbddd8db --- /dev/null +++ b/src/domain/desktop/appTaskReaders.test.ts @@ -0,0 +1,331 @@ +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { parseDesktopTask } from "./appTask.js"; +import { evaluateTaskChecks, type TaskFileView } from "./appTaskChecks.js"; +import { renderTaskVerifier } from "./appTaskPython.js"; +import { + PYTHON_READER_RUNNER, + readerAnswerView, + readerCommand, +} from "./appTaskReaders.js"; + +const portable = (name: string) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(name); +const POSTER = "Documents/poster.xcf"; + +const LAYERS = { + id: "layers", + file: POSTER, + argv: ["gimp-console-2.10", "-i", "-b", "-"], + stdin: '(gimp-file-load RUN-NONINTERACTIVE "{file}" "{file}")', + output: "lines", +}; +const LAYER_CHECK = { + id: "renamed", + kind: "lines_present", + file: POSTER, + reader: "layers", + lines: ["layer Summer Sale"], +}; + +const parse = (change: Record) => + parseDesktopTask( + { + name: "rename-layer", + prompt: "In GIMP, rename the Headline layer to Summer Sale and save it.", + checks: [LAYER_CHECK], + readers: [LAYERS], + ...change, + }, + portable, + ); + +/** Task changes the parser refuses for its readers, each with part of its message. */ +const REFUSALS: readonly (readonly [Record, string])[] = [ + [ + { checks: [{ ...LAYER_CHECK, reader: "objects" }] }, + 'reader "objects", which the task does not define', + ], + [ + { checks: [{ ...LAYER_CHECK, file: "Documents/other.xcf" }] }, + "reads layers, so its file is Documents/poster.xcf", + ], + [{ readers: [{ ...LAYERS, stdin: "(gimp-quit 0)" }] }, "name no {file}"], + [{ readers: [LAYERS, LAYERS] }, 'two readers have the id "layers"'], + [ + { + readers: [LAYERS, { ...LAYERS, id: "unused" }], + }, + 'reader "unused" is read by no check', + ], + [ + { readers: [{ ...LAYERS, file: "/home/ada/x.xcf" }] }, + "without a leading slash", + ], + [ + { + checks: [ + { + kind: "value_equals", + file: POSTER, + reader: "layers", + pointer: "/0", + value: 1, + }, + ], + }, + "reads lines, which lines_present, lines_absent and line_count check", + ], + [ + { + checks: [ + { + kind: "text_contains", + file: POSTER, + reader: "layers", + pointer: "/0", + value: "x", + }, + ], + }, + "at checks.0", + ], + [{ readers: [{ ...LAYERS, id: "Layers" }] }, "at readers.0.id"], + [{ readers: [{ ...LAYERS, timeout_s: 301 }] }, "at readers.0.timeout_s"], +]; + +describe("task readers", () => { + it("keeps readers that checks read through", () => { + const parsed = parse({}); + expect(parsed).toMatchObject({ + ok: true, + value: { readers: [LAYERS], checks: [LAYER_CHECK] }, + }); + }); + + it("leaves readers out of a task that has none", () => { + const parsed = parse({ + readers: undefined, + checks: [{ kind: "file_present", file: POSTER }], + }); + expect(parsed.ok && "readers" in parsed.value).toBe(false); + }); + + it.each(REFUSALS)("refuses %j", (change, message) => { + const parsed = parse(change); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.error).toContain(message); + }); + + it("evaluates a check on its reader's view rather than the file's own", () => { + const parsed = parse({ + checks: [ + LAYER_CHECK, + { + id: "moved", + kind: "value_equals", + file: "Documents/scene.blend", + reader: "objects", + pointer: "/Crate/location/0", + value: 2, + tolerance: 0.001, + }, + { + id: "count", + kind: "member_count", + file: "Documents/scene.blend", + reader: "objects", + pointer: "/Crate/location", + equals: 3, + }, + ], + readers: [ + LAYERS, + { + id: "objects", + file: "Documents/scene.blend", + argv: ["blender", "-b", "{file}"], + output: "json", + }, + ], + }); + if (!parsed.ok) throw new Error(parsed.error); + const files = new Map([ + [POSTER, { format: "binary", view: null }], + ["Documents/scene.blend", { format: "binary", view: null }], + ]); + const readers = new Map([ + [ + "layers", + { + format: "text", + view: { + lines: ["layer Summer Sale", "layer Background"], + line_count: 2, + truncated: false, + }, + }, + ], + [ + "objects", + { format: "json", view: { Crate: { location: [2.0004, 0, 0] } } }, + ], + ]); + + const outcomes = evaluateTaskChecks(parsed.value.checks, files, readers); + expect(outcomes.map(({ id, passed }) => [id, passed])).toEqual([ + ["renamed", true], + ["moved", true], + ["count", true], + ]); + // The file's own view is binary, which the same check without its reader cannot read. + expect( + evaluateTaskChecks( + [{ ...LAYER_CHECK, reader: undefined }] as never, + files, + readers, + )[0]?.description, + ).toContain("is binary"); + }); + + it("fails a check whose reader failed as undecided with the reader's reason, and one whose file is absent as decided", () => { + const parsed = parse({}); + if (!parsed.ok) throw new Error(parsed.error); + const failed = new Map([ + [ + "layers", + { + format: null, + view: null, + error: "the reader exited 1: GIMP could not open the file", + }, + ], + ]); + expect(evaluateTaskChecks(parsed.value.checks, new Map(), failed)).toEqual([ + { + id: "renamed", + passed: false, + undecided: true, + description: `${POSTER} has no view: the reader exited 1: GIMP could not open the file`, + }, + ]); + const [absent] = evaluateTaskChecks( + parsed.value.checks, + new Map(), + new Map(), + ); + expect(absent?.passed).toBe(false); + expect(absent?.undecided).toBeUndefined(); + }); +}); + +describe("the reader runner's command and answer", () => { + it("runs the runner's source in the computer's Python with the reader's arguments", () => { + const [python, isolated, flag, source, argument] = readerCommand( + { + id: "objects", + file: "Documents/scene.blend", + argv: ["blender", "-b", "{file}"], + output: "json", + timeout_s: 20, + }, + "/tmp/end/scene.blend", + ); + expect([python, isolated, flag, source]).toEqual([ + "python3", + "-I", + "-c", + PYTHON_READER_RUNNER, + ]); + expect(JSON.parse(argument ?? "")).toEqual({ + id: "objects", + path: "/tmp/end/scene.blend", + name: "scene.blend", + argv: ["blender", "-b", "{file}"], + stdin: null, + output: "json", + timeout_s: 20, + display: false, + }); + }); + + it("names the file by its home path when no other path is given, with the default timeout", () => { + const argument = readerCommand( + { id: "layers", file: POSTER, argv: ["gimp", "{file}"], output: "lines" }, + undefined, + ).at(-1); + expect(JSON.parse(argument ?? "")).toMatchObject({ + path: POSTER, + name: "poster.xcf", + timeout_s: 60, + }); + }); + + it.each([ + [ + '{"view": {"format": "json", "view": {"a": 1}}}', + { format: "json", view: { a: 1 } }, + ], + ['{"absent": true}', undefined], + [ + '{"error": "the reader exited 2; its stderr ends: boom"}', + { + format: null, + view: null, + error: "the reader exited 2; its stderr ends: boom", + }, + ], + ])("reads the answer %s", (line, view) => { + expect(readerAnswerView(`noise\n${line}\n`)).toEqual(view); + }); + + it("refuses an answer that is not the runner's", () => { + expect(() => readerAnswerView("Traceback (most recent call last)")).toThrow( + "REA's reader runner", + ); + }); +}); + +describe("a verifier without readers", () => { + it("renders byte for byte as the selector parent before readers", () => { + const rendered = renderTaskVerifier( + { + name: "add-milk", + prompt: "In Mousepad, add buy milk to my to-do list and save it.", + checks: [ + { + id: "milk", + kind: "lines_present", + file: "Documents/todo.txt", + lines: ["buy milk"], + }, + { + id: "price", + kind: "value_equals", + file: ".config/notes/settings.json", + pointer: "/size", + value: 12.5, + tolerance: 0.001, + }, + { + id: "rows", + kind: "member_count", + file: "Documents/todo.xml", + pointer: "/root/children", + matching: "[name=item]", + at_least: 2, + }, + ], + }, + "mousepad", + ); + + expect(rendered).not.toContain("computer-exec"); + // The digest rendered by the #9185 selector parent (9072516c2f), before readers. + // Older evaluator bodies remain subject to sameTaskVerifier's equivalence rules. + expect(createHash("sha256").update(rendered).digest("hex")).toBe( + "f23f2f374a0fd774d36b76c1cc382d29a748da53647252c9a831493bfa220142", + ); + }); +}); diff --git a/src/domain/desktop/appTaskReaders.ts b/src/domain/desktop/appTaskReaders.ts new file mode 100644 index 0000000000..e53cb09b83 --- /dev/null +++ b/src/domain/desktop/appTaskReaders.ts @@ -0,0 +1,475 @@ +import { z } from "zod"; + +import { jsonValueSchema } from "../jsonValue.js"; +import { READER_TIMEOUT_S, type DesktopTaskReader } from "./appTask.js"; +import { PYTHON_EXACT_JSON_NUMBER } from "./pythonExactJsonNumber.js"; +import type { TaskFileView } from "./appTaskChecks.js"; + +/** A runner command's outcome, as computer-exec answers it or a run on this host gives it. */ +export interface ReaderCommandResult { + readonly exit_code: number | null; + readonly stdout: string; + readonly stderr: string; + readonly timed_out: boolean; +} + +/** How long past a reader's own timeout its runner and the exec route may take to answer. */ +export const READER_RUNNER_SLACK_S = 30; + +/** + * REA's reader runner, one source for the witness, the export and the + * exported verifier: it runs a reader on a private copy of its file in a + * fresh home, without the display or the computer's control token, and + * answers with one JSON line. + */ +export const PYTHON_READER_RUNNER = String.raw`"""REA's reader runner: one task reader, an app's own headless command, run on a private copy of a file. + +It joins its JSON argument chunks and prints one JSON line: {"view": ...}, {"absent": true} when the file is not there, or +{"error": ...}. The command runs in a temporary folder with a fresh home, without the desktop's display (unless the +reader asks for it), the session bus or the computer's AGENTENV_ variables, so it reads none of the user's settings, +cannot hand the file to the running app and holds no control token; it can change only its copy, which goes with the +folder. Lines it prints starting with MARKER make its view; everything else it prints is left out. +""" +import json +import math +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time + +MARKER = "REA " +PLACEHOLDER = "{file}" +OUTPUT_BYTES = 1 << 20 +ANSWER_BYTES = OUTPUT_BYTES - (64 << 10) +STDERR_BYTES = 1024 +#: How often the runner looks whether the reader exited, and how long it waits for its streams to close after. +POLL_S = 0.05 +DRAIN_S = 2.0 +MAX_LINES = 5000 +MAX_SAFE_INTEGER = 2**53 - 1 +LINE_BREAK = re.compile(r"\r?\n") +SESSION = ("DISPLAY", "WAYLAND_DISPLAY", "XAUTHORITY") +NEVER = ("DBUS_SESSION_BUS_ADDRESS", "SESSION_MANAGER", "XDG_RUNTIME_DIR") +FRESH = (("XDG_CONFIG_HOME", ".config"), ("XDG_DATA_HOME", ".local/share"), ("XDG_CACHE_HOME", ".cache"), + ("XDG_STATE_HOME", ".local/state")) + + +${PYTHON_EXACT_JSON_NUMBER} + +def _say(answer: dict) -> int: + line = json.dumps(answer) + if len(line) > ANSWER_BYTES: + line = json.dumps({"error": f"the reader's view is larger than the {ANSWER_BYTES >> 10} KiB an answer holds"}) + print(line, flush=True) + return 0 + + +def _environment(home: str, display: bool) -> dict: + """The session's environment without what ties a command to the session or the computer, in a fresh home.""" + kept = {name: value for name, value in os.environ.items() + if not name.startswith("AGENTENV_") and name not in NEVER and (display or name not in SESSION)} + if display and "XAUTHORITY" not in kept and os.path.isfile(os.path.expanduser("~/.Xauthority")): + kept["XAUTHORITY"] = os.path.expanduser("~/.Xauthority") + kept["HOME"] = home + for name, folder in FRESH: + kept[name] = os.path.join(home, folder) + return kept + + +def _not_json(text: str): + raise ValueError(f"{text} is not JSON") + + +def _view(output: str, lines: list) -> dict: + if output == "lines": + return {"view": {"format": "text", "view": {"lines": lines[:MAX_LINES], "line_count": len(lines), + "truncated": len(lines) > MAX_LINES}}} + try: + value = json.loads("\n".join(lines), parse_int=_exact_number, parse_float=_exact_number, parse_constant=_not_json) + except _Inexact: + return {"error": "a number in the reader's view cannot be read exactly, such as an integer above 2^53"} + except (ValueError, RecursionError) as error: + return {"error": f"the reader's marked lines are not one JSON value: {error}"} + return {"view": {"format": "json", "view": value}} + + +def _tail(stderr: bytearray) -> str: + text = bytes(stderr).decode("utf-8", "replace").strip() + return f"; its stderr ends: {text}" if text else "" + + +def _drain(stream, kept: bytearray, limit: int, tail: bool, full: threading.Event) -> None: + """Read a stream to its end holding at most limit bytes: the first, setting full past them, or the last.""" + while chunk := stream.read1(1 << 16): + kept += chunk + if tail: + del kept[:-limit] + elif len(kept) > limit: + del kept[limit + 1:] + full.set() + + +def _feed(stream, text) -> None: + try: + if text is not None: + stream.write(text.encode()) + stream.close() + except OSError: + pass + + +def _run(arguments: dict, source: str, work: str) -> dict: + # The copy has a folder of its own, so no file name meets the runner's home. + copy = os.path.join(work, "file", arguments["name"]) + os.makedirs(os.path.dirname(copy)) + shutil.copyfile(source, copy) + home = os.path.join(work, "home") + os.makedirs(home) + argv = [part.replace(PLACEHOLDER, copy) for part in arguments["argv"]] + stdin = arguments.get("stdin") + timeout = arguments["timeout_s"] + try: + process = subprocess.Popen(argv, cwd=os.path.dirname(copy), env=_environment(home, arguments.get("display") is True), + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + except OSError as error: + return {"error": f"could not start {argv[0]}: {error.strerror or error}"} + printed, stderr, full = bytearray(), bytearray(), threading.Event() + threads = [threading.Thread(target=_drain, args=(process.stdout, printed, OUTPUT_BYTES, False, full), daemon=True), + threading.Thread(target=_drain, args=(process.stderr, stderr, STDERR_BYTES, True, full), daemon=True), + threading.Thread(target=_feed, args=(process.stdin, None if stdin is None else stdin.replace(PLACEHOLDER, copy)), + daemon=True)] + for thread in threads: + thread.start() + deadline = time.monotonic() + timeout + try: + while process.poll() is None and not full.is_set() and time.monotonic() < deadline: + time.sleep(POLL_S) + timed_out = process.poll() is None and not full.is_set() + finally: + # The app may leave helpers running, such as a document server: end its whole session. + try: + os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + process.wait() + for thread in threads: + thread.join(DRAIN_S) + if full.is_set(): + return {"error": "the reader printed more than 1 MiB"} + if timed_out: + return {"error": f"the reader ran past its {timeout} s{_tail(stderr)}"} + if process.returncode != 0: + return {"error": f"the reader exited {process.returncode}{_tail(stderr)}"} + lines = [line[len(MARKER):] for line in LINE_BREAK.split(printed.decode("utf-8", "replace")) if line.startswith(MARKER)] + if not lines: + return {"error": f"the reader printed no line starting with {json.dumps(MARKER)}{_tail(stderr)}"} + return _view(arguments["output"], lines) + + +def main(arguments: dict) -> int: + home = os.path.expanduser("~") + source = arguments["path"] if os.path.isabs(arguments["path"]) else os.path.join(home, arguments["path"]) + if not os.path.isfile(source): + return _say({"absent": True}) + # A remote caller owns this exact folder before starting the command, and + # removes it only after the command has settled. + lease = arguments.get("lease") + if lease is not None: + work = lease["path"] + os.mkdir(work, 0o700) + with open(work + ".owner", "x", encoding="ascii") as owner: + owner.write(lease["token"]) + return _say(_run(arguments, source, work)) + # A caller that names a work folder removes it itself, even when this runner is killed. + work = os.path.realpath(tempfile.mkdtemp(prefix="rea-reader-", dir=arguments.get("work"))) + try: + return _say(_run(arguments, source, work)) + finally: + shutil.rmtree(work, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main(json.loads("".join(sys.argv[1:])))) +`; + +/** Remove only the remote folder whose ownership marker matches the caller's lease. */ +export const PYTHON_READER_CLEANUP = String.raw` +import json, os, shutil, sys +lease = json.loads(sys.argv[1]) +path = lease["path"] +marker = path + ".owner" +if os.path.lexists(path) or os.path.lexists(marker): + with open(marker, encoding="ascii") as owner: + if owner.read() != lease["token"]: + raise RuntimeError("reader folder ownership does not match") + if os.path.lexists(path): + shutil.rmtree(path) + os.unlink(marker) +`; + +export interface ReaderLease { + readonly path: string; + readonly token: string; +} + +/** + * The command that runs a reader through the runner: on `path`, or on its + * file in the home when none is given, in a folder under `work` when given, + * which the caller then removes. + */ +export const readerCommand = ( + reader: DesktopTaskReader, + path: string | undefined, + work?: string, + lease?: ReaderLease, +): string[] => [ + "python3", + "-I", + "-c", + PYTHON_READER_RUNNER, + ...readerArgumentChunks(reader, path, work, lease), +]; + +/** The runner's one argument for a reader, as JSON. */ +export const readerArguments = ( + reader: DesktopTaskReader, + path: string | undefined, + work?: string, + lease?: ReaderLease, +): string => + JSON.stringify({ + id: reader.id, + path: path ?? reader.file, + name: reader.file.split("/").at(-1) ?? reader.file, + argv: reader.argv, + stdin: reader.stdin ?? null, + output: reader.output, + timeout_s: readerTimeout(reader), + display: reader.display ?? false, + ...(work === undefined ? {} : { work }), + ...(lease === undefined ? {} : { lease }), + }); + +/** Keep every argument below Linux's 128 KiB per-string limit, including four-byte UTF-8 characters. */ +const readerArgumentChunks = ( + reader: DesktopTaskReader, + path: string | undefined, + work?: string, + lease?: ReaderLease, +): string[] => { + const characters = Array.from(readerArguments(reader, path, work, lease)); + const chunks: string[] = []; + for (let index = 0; index < characters.length; index += 8192) + chunks.push(characters.slice(index, index + 8192).join("")); + return chunks; +}; + +/** The seconds a reader may run. */ +export const readerTimeout = (reader: DesktopTaskReader): number => + reader.timeout_s ?? READER_TIMEOUT_S.default; + +const answerSchema = z.union([ + z.strictObject({ absent: z.literal(true) }), + z.strictObject({ error: z.string() }), + z.strictObject({ + view: z.union([ + z.strictObject({ format: z.literal("json"), view: jsonValueSchema }), + z.strictObject({ + format: z.literal("text"), + view: z.strictObject({ + lines: z.array(z.string()), + line_count: z.int().nonnegative(), + truncated: z.boolean(), + }), + }), + ]), + }), +]); + +/** + * The view a runner's answer gives, its last line on stdout: undefined when + * the file is absent, a view with the reader's error when it failed. Throws + * when the output holds no answer of the runner's. + */ +export const readerAnswerView = (stdout: string): TaskFileView | undefined => { + const last = stdout.trim().split("\n").at(-1) ?? ""; + let parsed: unknown; + try { + parsed = JSON.parse(last); + } catch { + parsed = undefined; + } + const answer = answerSchema.safeParse(parsed); + if (!answer.success) + throw new Error( + `REA's reader runner did not answer; it printed: ${stdout.slice(-512)}`, + ); + const { data } = answer; + if ("absent" in data) return undefined; + if ("error" in data) return { format: null, view: null, error: data.error }; + return data.view; +}; + +/** The verifier's runs of the task's readers through the computer's computer-exec, as the witness runs them. */ +export const PYTHON_READER_VIEWS = String.raw` + +# agentenv loads a fresh verifier module for every execution. Keep abandoned +# leases in the hosting process, so a later module can retry their cleanup. +_reader_registry = sys.modules.setdefault("_rea_desktop_reader_leases_v1", types.ModuleType("_rea_desktop_reader_leases_v1")) +_PENDING_READERS = _reader_registry.__dict__.setdefault("pending", {}) + + +def _owned_task(awaitable): + task = asyncio.create_task(awaitable) + task.add_done_callback(lambda done: None if done.cancelled() else done.exception()) + return task + + +async def _finish_reader(env_url, card, pending): + if not pending["settled"]: + if pending["task"].cancelled(): + raise RuntimeError(f"Reader {pending['request_id']} has an unknown outcome; retained {pending['lease']['path']}") + try: + result = await asyncio.shield(pending["task"]) + except Exception as error: + raise RuntimeError(f"Reader {pending['request_id']} may still own {pending['lease']['path']}: {error}") from error + if not isinstance(result, dict) or result.get("timed_out"): + raise RuntimeError(f"Reader {pending['request_id']} may still be running; retained {pending['lease']['path']}") + pending["settled"] = True + if pending.get("cleanup_uncertain"): + raise RuntimeError(f"Cleanup may still be running; retained {pending['lease']['path']}") + pending.setdefault("cleanup_id", f"rea-reader-cleanup-{uuid.uuid4().hex}") + cleanup = {"argv": ["python3", "-I", "-c", READER_CLEANUP, json.dumps(pending["lease"])], + "request_id": pending["cleanup_id"], "timeout_s": RUNNER_SLACK_S} + if "cleanup_task" not in pending: + pending["cleanup_task"] = _owned_task( + protocol.invoke_extension(env_url, card, EXEC, cleanup, method="run", timeout=2 * RUNNER_SLACK_S)) + if pending["cleanup_task"].cancelled(): + raise RuntimeError(f"Cleanup has an unknown outcome; retained {pending['lease']['path']}") + try: + answer = await asyncio.shield(pending["cleanup_task"]) + except Exception as error: + pending.pop("cleanup_task", None) + raise RuntimeError(f"Cleanup of owned reader folder {pending['lease']['path']} is unconfirmed: {error}") from error + if not isinstance(answer, dict) or answer.get("timed_out"): + pending["cleanup_uncertain"] = True + raise RuntimeError(f"Cleanup may still be running; retained {pending['lease']['path']}") + if answer.get("exit_code") != 0: + pending.pop("cleanup_id", None) + pending.pop("cleanup_task", None) + raise RuntimeError(f"Could not remove owned reader folder {pending['lease']['path']}: {answer}") + + +async def _cleanup_readers(env_url, card, current=None): + for token, pending in list(_PENDING_READERS.items()): + if pending["env_url"] != env_url or (not pending["abandoned"] if current is None else token != current): + continue + if "finishing" not in pending: + pending["finishing"] = _owned_task(_finish_reader(env_url, card, pending)) + if pending["finishing"].cancelled(): + raise RuntimeError(f"Reader cleanup has an unknown outcome; retained {pending['lease']['path']}") + try: + await asyncio.shield(pending["finishing"]) + except Exception: + pending.pop("finishing", None) + raise + _PENDING_READERS.pop(token, None) + + +async def _reader_views(env_url, card): + """Each reader's view of its file, by its id, read by REA's reader runner in the computer; a file the home lacks + has none. A runner that does not answer raises, so the run fails rather than scoring the agent.""" + if not READERS: + return {} + if protocol.find_extension(card, EXEC) is None: + raise RuntimeError(f"the computer at {env_url} offers no {EXEC}, which runs the task's readers") + await _cleanup_readers(env_url, card) + views = {} + for reader in READERS: + token = uuid.uuid4().hex + lease = {"path": f"/tmp/rea-reader-{token}", "token": token} + arguments = json.loads("".join(reader["arguments"])) + arguments["lease"] = lease + encoded = json.dumps(arguments) + params = {"argv": ["python3", "-I", "-c", READER_RUNNER, + *(encoded[i:i + 8192] for i in range(0, len(encoded), 8192))], + "request_id": f"rea-reader-{token}", + "timeout_s": reader["timeout_s"] + RUNNER_SLACK_S} + task = _owned_task(protocol.invoke_extension( + env_url, card, EXEC, params, method="run", timeout=reader["timeout_s"] + 2 * RUNNER_SLACK_S)) + _PENDING_READERS[token] = {"env_url": env_url, "lease": lease, "task": task, + "request_id": params["request_id"], "settled": False, "abandoned": False} + try: + answer = await asyncio.shield(task) + except BaseException as error: + _PENDING_READERS[token]["abandoned"] = True + error.add_note(f"Reader command {params['request_id']} and folder {lease['path']} remain owned for retry.") + raise + if not isinstance(answer, dict) or answer.get("timed_out"): + _PENDING_READERS[token]["abandoned"] = True + raise RuntimeError(f"Reader {params['request_id']} may still be running; retained {lease['path']}") + _PENDING_READERS[token]["settled"] = True + try: + view = _reader_answer(reader["id"], answer) + if view is not None: + views[reader["id"]] = view + finally: + try: + await _cleanup_readers(env_url, card, token) + except asyncio.CancelledError as error: + _PENDING_READERS[token]["abandoned"] = True + error.add_note(f"Reader cleanup remains owned for {lease['path']}; completed reader views: {json.dumps(views)}") + raise + except Exception as error: + _PENDING_READERS[token]["abandoned"] = True + raise RuntimeError(f"{error}; completed reader views: {json.dumps(views)}") from error + return views + + +def evaluate_with_readers(checks: list, views: dict, readers: dict) -> list: + """One result per check as evaluate gives it, a check with reader graded on that reader's view of its file.""" + results = [] + for check in checks: + entry = views.get(check["file"]) if check.get("reader") is None else readers.get(check["reader"]) + passed, description = _verdict(check, entry) + results.append({"id": check["id"], "result": passed, "description": description}) + return results + + +def _reader_answer(name, answer): + """The view the runner's last line gives: None for an absent file, one with an error for a failed reader.""" + stdout = answer.get("stdout") if isinstance(answer, dict) and isinstance(answer.get("stdout"), str) else "" + try: + said = json.loads(stdout.strip().split("\n")[-1]) + except ValueError: + said = None + if isinstance(answer, dict) and answer.get("exit_code") == 0 and isinstance(said, dict): + if said == {"absent": True}: + return None + if set(said) == {"error"} and isinstance(said["error"], str): + return {"format": None, "view": None, "error": said["error"]} + view = said.get("view") if set(said) == {"view"} else None + if isinstance(view, dict) and view.get("format") in ("json", "text") and "view" in view: + return view + raise RuntimeError(f"REA's reader runner did not answer for {name}; it printed: {stdout[-512:]}") +`; + +/** The verifier's reader constants: the route, the runner's source once, then each reader's argument and timeout. */ +export const renderReaderConstants = ( + readers: readonly DesktopTaskReader[], +): string => `#: The computer route the readers run through, and how long past a reader's own timeout its runner may take. +EXEC = "urn:agentenv:computer-exec/v1" +RUNNER_SLACK_S = ${String(READER_RUNNER_SLACK_S)} +#: REA's reader runner, which runs each reader on a private copy of its file in a fresh home. +READER_RUNNER = ${JSON.stringify(PYTHON_READER_RUNNER)} +READER_CLEANUP = ${JSON.stringify(PYTHON_READER_CLEANUP)} +READERS = [ +${readers.map((reader) => ` {"id": ${JSON.stringify(reader.id)}, "arguments": ${JSON.stringify(readerArgumentChunks(reader, undefined))}, "timeout_s": ${String(readerTimeout(reader))}},`).join("\n")} +] +`; diff --git a/src/domain/desktop/appTaskScreen.test.ts b/src/domain/desktop/appTaskScreen.test.ts new file mode 100644 index 0000000000..5dac2660f7 --- /dev/null +++ b/src/domain/desktop/appTaskScreen.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from "vitest"; + +import { parseDesktopTask } from "./appTask.js"; +import { evaluateTaskChecks, type TaskFileView } from "./appTaskChecks.js"; +import { renderTaskVerifier } from "./appTaskPython.js"; +import { + DEFAULT_JUDGE_MODEL, + isScreenTask, + readableChanges, + screenExamplesOf, +} from "./appTaskScreen.js"; + +const portable = (name: string) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(name); +const SHOWN = { + id: "shown", + kind: "screen_judge", + question: "Does the calculator's display show the result 8.568?", +}; +const parse = (change: Record) => + parseDesktopTask( + { + name: "tip-total", + prompt: + "In Calculator, work out 18% of 47.60 and leave the result showing.", + checks: [SHOWN], + ...change, + }, + portable, + ); +const EXAMPLES = { fail: "c2VlZA==", pass: "ZW5k" }; + +describe("screen_judge checks", () => { + it("keeps a task whose checks are screen judges", () => { + const parsed = parse({}); + expect(parsed).toMatchObject({ ok: true, value: { checks: [SHOWN] } }); + expect(parsed.ok && isScreenTask(parsed.value)).toBe(true); + }); + + it.each([ + [ + { + checks: [ + SHOWN, + { kind: "file_present", file: "Documents/receipt.txt" }, + ], + }, + "a task with screen_judge checks holds only screen_judge checks", + ], + [ + { + readers: [ + { + id: "x", + file: "Documents/a.bin", + argv: ["cat", "{file}"], + output: "lines", + }, + ], + }, + "takes no readers", + ], + [{ checks: [{ ...SHOWN, question: " " }] }, "at checks.0.question"], + [{ checks: [{ ...SHOWN, file: "Documents/a.txt" }] }, "at checks.0"], + ])("refuses %j", (change, message) => { + const parsed = parse(change); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.error).toContain(message); + }); + + it("leaves a screen judge undecided in REA's evaluator, which reads no screen", () => { + const parsed = parse({}); + if (!parsed.ok) throw new Error(parsed.error); + const [outcome] = evaluateTaskChecks(parsed.value.checks, new Map()); + expect(outcome).toMatchObject({ + id: "shown", + passed: false, + undecided: true, + }); + expect(outcome?.description).toContain("screen"); + }); + + it("renders a verifier that screenshots the computer and asks the judge with the build's two examples", () => { + const parsed = parse({}); + if (!parsed.ok) throw new Error(parsed.error); + const verifier = renderTaskVerifier(parsed.value, "calculator", EXAMPLES); + + expect(verifier).toContain("computer_screenshot"); + expect(verifier).toContain( + `DEFAULT_JUDGE_MODEL = "${DEFAULT_JUDGE_MODEL}"`, + ); + expect(verifier).toContain('get_model_for_role("judge")'); + expect(verifier).toContain("VOTES = 3"); + expect(verifier).toContain("--judge-model"); + expect(screenExamplesOf(verifier)).toEqual(EXAMPLES); + expect(screenExamplesOf("CHECKS = []\n")).toBeUndefined(); + }); + + it("compares JSON objects by value and still detects a newly created null file", () => { + const seed = new Map([ + ["settings.json", { format: "json", view: { theme: "light", zoom: 1 } }], + ]); + const end = new Map([ + ["settings.json", { format: "json", view: { zoom: 1, theme: "light" } }], + ["new.json", { format: "json", view: null }], + ]); + expect(readableChanges(seed, end)).toEqual(["new.json"]); + }); + + it("finds the files whose change a check could read, ignoring unreadable and unchanged ones", () => { + const text = (...lines: string[]): TaskFileView => ({ + format: "text", + view: { lines, line_count: lines.length, truncated: false }, + }); + const seed = new Map([ + ["Documents/same.txt", text("a")], + ["Documents/edited.txt", text("a")], + ["Documents/gone.txt", text("a")], + ["Documents/blob.bin", { format: "binary", view: null }], + [".cache/broken.db", { format: null, view: null, error: "locked" }], + ]); + const end = new Map([ + ["Documents/same.txt", text("a")], + ["Documents/edited.txt", text("b")], + ["Documents/new.txt", text("c")], + ["Documents/blob.bin", { format: "binary", view: null }], + [".cache/broken.db", { format: null, view: null, error: "locked again" }], + ]); + expect(readableChanges(seed, end)).toEqual([ + "Documents/edited.txt", + "Documents/gone.txt", + "Documents/new.txt", + ]); + }); +}); diff --git a/src/domain/desktop/appTaskScreen.ts b/src/domain/desktop/appTaskScreen.ts new file mode 100644 index 0000000000..0de0f836c3 --- /dev/null +++ b/src/domain/desktop/appTaskScreen.ts @@ -0,0 +1,207 @@ +import type { DesktopTask, DesktopTaskCheck } from "./appTask.js"; +import { + sameJson, + type TaskCheckOutcome, + type TaskFileView, +} from "./appTaskChecks.js"; + +/** The model a bundle's screen judge asks when the runner's config names none for judges or by default. */ +export const DEFAULT_JUDGE_MODEL = "anthropic/claude-sonnet-5-5"; +/** How long the display is left to draw the agent's last action before the judge's screenshot. */ +export const SCREEN_SETTLE_MS = 1000; +/** The largest checkpoint screenshot the export ships as a judge's example. */ +export const SCREEN_EXAMPLE_MAX_BYTES = 2 * 1024 * 1024; +/** How many times the judge is asked each question; the check passes on most of them saying yes. */ +export const SCREEN_JUDGE_VOTES = 3; + +/** A screen judge check: a yes-or-no question about the app's screen. */ +export type ScreenJudgeCheck = Extract< + DesktopTaskCheck, + { kind: "screen_judge" } +>; + +/** The build's screenshots a screen-judged verifier ships, base64 JPEG: the seed's, where the answer is no, and the end's, where it is yes. */ +export interface ScreenExamples { + readonly fail: string; + readonly pass: string; +} + +/** Runs a bundle's own judge on one screenshot file, as the witness does, and gives the check's outcome. */ +export type ScreenJudge = ( + verifier: string, + check: ScreenJudgeCheck, + image: Uint8Array, + signal: AbortSignal, +) => Promise; + +/** Whether a task's checks are screen judges, which the parser lets stand only alone. */ +export const isScreenTask = (task: DesktopTask): boolean => + task.checks.some(({ kind }) => kind === "screen_judge"); + +const readable = (view: TaskFileView | undefined): boolean => + view === undefined || + (view.format !== null && + view.format !== "binary" && + typeof view.error !== "string"); + +/** The files whose change between two sets of views a check could read: readable or absent at both ends, and not the same. */ +export const readableChanges = ( + seed: ReadonlyMap, + end: ReadonlyMap, +): string[] => + [...new Set([...seed.keys(), ...end.keys()])] + .filter((file) => { + const [before, after] = [seed.get(file), end.get(file)]; + return ( + readable(before) && + readable(after) && + ((before === undefined) !== (after === undefined) || + !sameJson(before?.view ?? null, after?.view ?? null)) + ); + }) + .sort(); + +const EXAMPLES_LINE = /^EXAMPLES = (\{.*\})$/mu; + +/** The examples a screen-judged verifier holds, or undefined when it holds none. */ +export const screenExamplesOf = ( + verifier: string, +): ScreenExamples | undefined => { + const line = EXAMPLES_LINE.exec(verifier)?.[1]; + if (line === undefined) return undefined; + try { + const parsed: unknown = JSON.parse(line); + if ( + typeof parsed === "object" && + parsed !== null && + "fail" in parsed && + "pass" in parsed && + typeof parsed.fail === "string" && + typeof parsed.pass === "string" + ) + return { fail: parsed.fail, pass: parsed.pass }; + } catch { + // A line that is no JSON names no examples. + } + return undefined; +}; + +/** The judge's code: a screenshot through the computer's MCP tools, then votes through litellm on agentenv's model config. */ +const PYTHON_SCREEN_JUDGE = String.raw` +#: Calls a vote makes before the judge counts as not answering. +ATTEMPTS = 3 +VERDICT = {"type": "object", "additionalProperties": False, "required": ["verdict", "reason"], + "properties": {"verdict": {"type": "boolean"}, "reason": {"type": "string"}}} +INSTRUCTIONS = ("You check a desktop app's screen against one yes-or-no question. Answer from the screenshot alone: when " + "it does not show the answer, the verdict is false. Two labelled screenshots of the same app come first.") + + +def _image(data: str) -> dict: + return {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64," + data}} + + +def _endpoint(): + """The judge's model and the call's settings; no endpoint raises, so the run fails rather than scoring.""" + config = get_config() + model = config.get_model_for_role("judge") or DEFAULT_JUDGE_MODEL + try: + config.get_litellm_base_url() + settings = config.resolve_model_call(model).client_kwargs() + except Exception as error: + raise RuntimeError(f"the screen judge has no model endpoint: {error}") from error + return model, settings + + +async def _vote(model: str, settings: dict, question: str, screenshot: str): + content = [{"type": "text", "text": INSTRUCTIONS}, + {"type": "text", "text": f"The app before the task; for the question {json.dumps(question)} the verdict is false:"}, + _image(EXAMPLES["fail"]), + {"type": "text", "text": "The app with the task done; the verdict is true:"}, + _image(EXAMPLES["pass"]), + {"type": "text", "text": f"Question: {question}\nThe screenshot to judge:"}, + _image(screenshot)] + failure = None + for _ in range(ATTEMPTS): + try: + response = await litellm.acompletion( + model=model, messages=[{"role": "user", "content": content}], + response_format={"type": "json_schema", "json_schema": {"name": "screen_judge", "schema": VERDICT, + "strict": True}}, **settings) + answer = json.loads(response.choices[0].message.content or "") + if isinstance(answer, dict) and isinstance(answer.get("verdict"), bool): + return answer["verdict"], str(answer.get("reason", "")) + failure = ValueError(f"the judge answered {answer!r}") + except Exception as error: + failure = error + raise RuntimeError(f"the screen judge did not answer: {failure}") + + +async def judge_screen(check, screenshot: str) -> dict: + """One check's result on a screenshot, base64 JPEG: it passes when most of VOTES votes say yes.""" + model, settings = _endpoint() + votes = [await _vote(model, settings, check["question"], screenshot) for _ in range(VOTES)] + yes = sum(1 for verdict, _ in votes if verdict) + passed = 2 * yes > VOTES + reason = next(reason for verdict, reason in votes if verdict == passed) + said = yes if passed else VOTES - yes + return {"id": check["id"], "result": passed, + "description": f"the judge ({model}) said {'yes' if passed else 'no'} in {said} of {VOTES} votes: {reason}"} + + +async def _screenshot(mcp_url: str) -> str: + async with ToolSession(mcp_url=mcp_url) as session: + shot = await session.call_tool("computer_screenshot", {"format": "jpeg", "quality": 70}) + images = [block for block in shot.content if block.get("type") == "image"] + if shot.isError or not images: + raise RuntimeError("the computer took no screenshot for the screen judge") + return images[0]["data"] + + +async def verify(mcp_url, env_url): + await asyncio.sleep(SCREEN_SETTLE_S) + screenshot = await _screenshot(mcp_url) + return [await judge_screen(check, screenshot) for check in CHECKS] + + +if __name__ == "__main__": + # REA's witness runs the judge on a screenshot it took: judge . + with open(sys.argv[3], "rb") as handle: + shot = base64.b64encode(handle.read()).decode() + print(json.dumps(asyncio.run(judge_screen(json.loads(sys.argv[2]), shot)))) +`; + +/** The verifier artifact of a screen-judged task: the judge, its two examples from the build, and the checks' questions. */ +export const renderScreenVerifier = ( + task: DesktopTask, + app: string, + examples: ScreenExamples, +): string => + `"""Checks of the task ${task.name} against the ${app} app's screen, exported by REA from a desktop application capture. + +It takes a screenshot of the computer's display through its MCP tools and asks a model, the judge, each check's +yes-or-no question about it VOTES times, after two labelled examples from the build: the screen at the seed, where the +answer is no, and at the end, where it is yes. A check passes when most votes say yes. The judge's model is the runner +config's [model.roles] judge, else its [model] default, else DEFAULT_JUDGE_MODEL, called through LITELLM_BASE_URL and +LITELLM_API_KEY or [model] base_url and api_key; agent-env run's --judge-model does not reach it. Without an endpoint it +raises, so the run fails rather than scoring the agent. +""" +import asyncio +import base64 +import json +import sys + +import litellm +from agent_env.config import get_config +from agentenv_protocol.tool_client import ToolSession + +APP = ${JSON.stringify(app)} +CHECKS = [ +${task.checks.map((check) => ` ${JSON.stringify(check)},`).join("\n")} +] +VOTES = ${String(SCREEN_JUDGE_VOTES)} +#: Seconds the display is left to draw the agent's last action before the screenshot. +SCREEN_SETTLE_S = ${(SCREEN_SETTLE_MS / 1000).toFixed(1)} +DEFAULT_JUDGE_MODEL = ${JSON.stringify(DEFAULT_JUDGE_MODEL)} +#: The build's screenshots, base64 JPEG: the seed's, where the answer is no, and the end's, where it is yes. +EXAMPLES = ${JSON.stringify({ fail: examples.fail, pass: examples.pass })} +${PYTHON_SCREEN_JUDGE}`; diff --git a/src/domain/desktop/pythonExactJsonNumber.ts b/src/domain/desktop/pythonExactJsonNumber.ts new file mode 100644 index 0000000000..d3040028d6 --- /dev/null +++ b/src/domain/desktop/pythonExactJsonNumber.ts @@ -0,0 +1,37 @@ +/** Number parsing shared by exported state views and task readers. */ +export const PYTHON_EXACT_JSON_NUMBER = String.raw` +class _Inexact(Exception): + """A JSON number that REA's state view refuses, since reading it would change it.""" + + +def _digits(text: str) -> str: + return re.split(r"[eE]", text.lstrip("-"))[0].replace(".", "").strip("0") or "0" + + +def _js_string(value: float) -> str: + """How JavaScript's String() writes a finite double: an integer below 1e21 in full, any other in shortest form.""" + return str(int(value)) if value.is_integer() and abs(value) < 1e21 else repr(value) + + +def _safe_number(text: str) -> bool: + """lossless-json's isSafeNumber: an integer spelling within 2^53, or a float spelling that keeps its digits.""" + if re.fullmatch(r"-?[0-9]+", text): + return len(text.lstrip("-")) <= 16 and abs(int(text)) <= MAX_SAFE_INTEGER + value = float(text) + if not math.isfinite(value): + return False + written = _js_string(value) + return written == text or _digits(written) == _digits(text) + + +def _exact_number(text: str): + """A JSON number read only when reading it changes nothing, as REA's state view rules: its spelling and its + written-back form both safe, and not -0.""" + value = float(text) + if (not _safe_number(text) or not math.isfinite(value) or not _safe_number(_js_string(value)) + or (value == 0 and math.copysign(1, value) < 0)): + raise _Inexact(text) + return int(text) if re.fullmatch(r"-?[0-9]+", text) else value + + +`; diff --git a/src/process/OwnedWorkerRuntime.ts b/src/process/OwnedWorkerRuntime.ts index 0b9f66c7ac..4980e3ceb0 100644 --- a/src/process/OwnedWorkerRuntime.ts +++ b/src/process/OwnedWorkerRuntime.ts @@ -23,23 +23,28 @@ export interface OwnedWorkerRuntimeOptions { readonly cleanupRegistry?: WorkerCleanupRegistry; } -/** Retained worker runtimes one host retries before admitting another child. */ -export class WorkerCleanupRegistry { - readonly #pending = new Set(); +/** A resource owner that can retry its own retained cleanup. */ +export interface RetainedWorkerCleanup { + retryRetainedCleanup(): Promise; +} - has(runtime: OwnedWorkerRuntime): boolean { +/** Retained worker resources one host retries before admitting another child. */ +export class WorkerCleanupRegistry { + readonly #pending = new Set(); + + has(runtime: RetainedWorkerCleanup): boolean { return this.#pending.has(runtime); } - add(runtime: OwnedWorkerRuntime): void { + add(runtime: RetainedWorkerCleanup): void { this.#pending.add(runtime); } - delete(runtime: OwnedWorkerRuntime): void { + delete(runtime: RetainedWorkerCleanup): void { this.#pending.delete(runtime); } - async retry(except?: OwnedWorkerRuntime): Promise { + async retry(except?: RetainedWorkerCleanup): Promise { for (const owner of [...this.#pending]) if (owner !== except) await owner.retryRetainedCleanup(); } diff --git a/src/process/RetainedCommandCleanup.ts b/src/process/RetainedCommandCleanup.ts new file mode 100644 index 0000000000..af7af1e2da --- /dev/null +++ b/src/process/RetainedCommandCleanup.ts @@ -0,0 +1,64 @@ +import { rm } from "node:fs/promises"; +import { DarwinProcessOwnershipInspectionError } from "./DarwinProcessRunTokenReader.js"; +import { OwnedCommandFailure } from "./OwnedCommand.js"; + +/** Exact command and scratch-folder ownership across partial cleanup retries. */ +export class RetainedCommandCleanup { + readonly #owner; + readonly #preparation; + readonly #processResources: readonly string[]; + #stopped: boolean; + #prepared: boolean; + #folderPresent = true; + + constructor( + readonly folder: string, + cause: unknown, + ) { + this.#owner = + cause instanceof OwnedCommandFailure ? cause.cleanupOwner : undefined; + this.#preparation = + cause instanceof OwnedCommandFailure + ? cause.preparationFailure + : cause instanceof DarwinProcessOwnershipInspectionError + ? cause + : undefined; + this.#processResources = + cause instanceof OwnedCommandFailure ? cause.resources : []; + this.#stopped = this.#owner === undefined; + this.#prepared = this.#preparation?.cleanupOwner === undefined; + } + + get hasProcess(): boolean { + return !this.#stopped || !this.#prepared; + } + + get cleanupResources(): readonly string[] { + return [ + ...new Set([ + ...(!this.#stopped ? this.#processResources : []), + ...(!this.#prepared + ? (this.#preparation?.cleanupFailure?.cleanupResources ?? []) + : []), + ...(this.#folderPresent ? [this.folder] : []), + ]), + ]; + } + + async close(): Promise { + if (!this.#stopped && this.#owner !== undefined) { + const result = await this.#owner.stop(); + if (result.status === "incomplete") throw new Error(result.reason); + this.#owner.dispose(); + this.#stopped = true; + } + if (!this.#prepared) { + await this.#preparation?.cleanupOwner?.close(); + this.#prepared = true; + } + if (this.#folderPresent) { + await rm(this.folder, { recursive: true, force: true }); + this.#folderPresent = false; + } + } +} diff --git a/tests/boundary/mcp/buildCaptureMcp.test.ts b/tests/boundary/mcp/buildCaptureMcp.test.ts index 7f73dfbc77..4338ecee5e 100644 --- a/tests/boundary/mcp/buildCaptureMcp.test.ts +++ b/tests/boundary/mcp/buildCaptureMcp.test.ts @@ -200,6 +200,40 @@ it("captures the app inside the computer, acknowledges checkpoints, drafts the t expectNoToken(computer, [started, seed, stopped, exported]); }); +it("keeps a screenshot of each checkpoint in the computer, which the export takes as a screen judge's examples", async () => { + const { rea, client } = await fakes(); + const id = await startBuildComputer(client, REQUEST); + await callOk(client, "start_build_capture", { computer_id: id }); + for (const name of ["seed", "end"]) + await callOk(client, "mark_build_checkpoint", { computer_id: id, name }); + await callOk(client, "stop_build_capture", { computer_id: id }); + await callOk(client, "export_build_environment", { + computer_id: id, + task: TASK, + directory: join(await createTestTempDirectory("build-screens-"), "out"), + }); + + const shots = (await files(join(rea.home, ".rea-build"))).filter((path) => + path.endsWith(".jpg"), + ); + expect(shots.map((path) => path.split("/").slice(-2).join("/"))).toEqual([ + "screens/end.jpg", + "screens/seed.jpg", + ]); + const exported = ( + (await rea.notes("calls.jsonl")) as { argv: string[] }[] + ).findLast(({ argv }) => argv[0] === "export-desktop-application"); + const at = exported?.argv.indexOf("--screens") ?? -1; + expect(at).toBeGreaterThan(0); + expect(join(rea.home, exported?.argv[at + 1] ?? "")).toBe( + join( + rea.home, + ".rea-build", + shots[0]?.split("/").slice(0, -1).join("/") ?? "", + ), + ); +}); + it("takes checkpoints from a REA in the computer that predates their changed files, saying which changed is unknown, also from its log", async () => { const { client } = await fakes({ env: { FAKE_REA_UNLISTED_CHANGES: "1" }, diff --git a/tests/boundary/mcp/buildWitnessMcp.test.ts b/tests/boundary/mcp/buildWitnessMcp.test.ts index 6b92479bcd..6c4d07d6f6 100644 --- a/tests/boundary/mcp/buildWitnessMcp.test.ts +++ b/tests/boundary/mcp/buildWitnessMcp.test.ts @@ -43,6 +43,11 @@ const TASK = { ], }; const WITNESS_ORDER = ["seed", "after_edit", "near_miss", "after_reset"]; +/** The build's screenshots a screen-judged bundle ships as its judge's examples, large enough that its verifier passes 1 MiB. */ +const JUDGE_EXAMPLES = { + fail: Buffer.alloc(600 * 1024, 1).toString("base64"), + pass: Buffer.alloc(600 * 1024, 2).toString("base64"), +}; const lines = (...text: string[]) => ({ "Documents/today.txt": { format: "text", @@ -63,7 +68,10 @@ const taskBundle = async (task: unknown, app: string) => { "verify.py", ); await mkdir(join(verifier, ".."), { recursive: true }); - await writeFile(verifier, renderTaskVerifier(parsed.value, app)); + await writeFile( + verifier, + renderTaskVerifier(parsed.value, app, JUDGE_EXAMPLES), + ); await writeFile( join(bundle, "tasks", `${parsed.value.name}.json`), renderTaskSteps(parsed.value, app), @@ -75,8 +83,13 @@ const witnessed = async ( slowReset = 0, options: FakeComputerOptions = {}, task: unknown = TASK, - settle?: BuildSettle, + server: { + readonly settle?: BuildSettle; + /** Find agent-env on PATH instead of REA_AGENT_ENV_COMMAND naming it. */ + readonly onPath?: boolean; + } = {}, ) => { + const { settle, onPath = false } = server; const app: FakeApp = { name: "notes", state: lines("milk"), @@ -85,7 +98,13 @@ const witnessed = async ( unreadyAfterReset: slowReset, }; const { computer, runner } = await buildComputerFakes({ ...options, app }); - const client = await connectBuildServer(runner.environment, settle); + const { REA_AGENT_ENV_COMMAND: _named, ...unnamed } = runner.environment; + const client = onPath + ? await connectBuildServer(unnamed, settle, { + searchPath: runner.state, + archiveDirectory: runner.state, + }) + : await connectBuildServer(runner.environment, settle); const bundle = await taskBundle(task, app.name); const id = await startBuildComputer(client, { kind: "bundle", path: bundle }); const directory = join(await createTestTempDirectory("witness-"), "add-eggs"); @@ -100,7 +119,7 @@ const witnessed = async ( action, ...fields, }); - return { app, bundle, computer, client, id, directory, check, act }; + return { app, bundle, computer, runner, client, id, directory, check, act }; }; /** Act on the app and set its state as that act would leave it. */ @@ -263,6 +282,165 @@ it("witnesses a task: the seed fails, the edit passes, a near miss fails and so ).toContain("already witnessed"); }); +it("witnesses a task whose check reads a binary file through a reader run in the computer", async () => { + const poster = "Documents/poster.xcf"; + const task = { + name: "rename-layer", + prompt: + "In Pictures, rename the Headline layer to Summer Sale and save it.", + checks: [ + { + id: "renamed", + kind: "lines_present", + file: poster, + reader: "layers", + lines: ["layer Summer Sale"], + }, + ], + readers: [ + { + id: "layers", + file: poster, + argv: [ + "python3", + "-c", + "import sys; print('banner'); print('REA layer ' + open(sys.argv[1], 'rb').read().split(b'\\0')[-1].decode())", + "{file}", + ], + output: "lines", + }, + ], + }; + const home = await createTestTempDirectory("witness-home-"); + await mkdir(join(home, "Documents")); + const save = (layer: string) => + writeFile(join(home, poster), `gimp xcf\0${layer}`); + await save("Headline"); + const { act, check, computer, directory } = await witnessed( + 0, + { exec: { home, bin: home } }, + task, + ); + + await check({ task, stage: "seed", directory }); + await act("type", { text: "Summer Sale\n" }); + await save("Summer Sale"); + await check({ task, stage: "after_edit" }); + await act("type", { text: "Summer Sail\n" }); + await save("Summer Sail"); + const near = parsed(await check({ task, stage: "near_miss" })); + expect(near.results[0]?.description).toContain( + 'has no line "layer Summer Sale"', + ); + await act("reset"); + await save("Headline"); + const last = await check({ task, stage: "after_reset" }); + expectNoToken(computer, last); + const record = parsed(last).witness?.record; + expect(record?.verdict).toBe("held"); + expect(record?.stages.map((stage) => stage.results[0]?.passed)).toEqual([ + false, + true, + false, + false, + ]); + expect(await readFile(join(home, poster), "utf8")).toBe("gimp xcf\0Headline"); + const evidence = parseEvidence( + JSON.parse(await readFile(join(directory, "witness.json"), "utf8")), + ); + expect(evidence.limitations.join(" ")).toContain("computer-exec/v1"); +}); + +/** Stand-ins for the runner's model client, agentenv config and MCP client, whose judge answers what `judge-verdict` holds. */ +const JUDGE_STUBS: Readonly> = { + "litellm/__init__.py": String.raw` +import json, os, types +async def acompletion(model, messages, **_): + state = os.environ["FAKE_STATE"] + with open(os.path.join(state, "judge-calls"), "a") as log: + log.write(json.dumps({"model": model, "images": sum(1 for part in messages[0]["content"] if part["type"] == "image_url")}) + "\n") + verdict = open(os.path.join(state, "judge-verdict")).read().strip() == "yes" + answer = {"verdict": verdict, "reason": "the window shows the result" if verdict else "the window shows no result"} + return types.SimpleNamespace(choices=[types.SimpleNamespace(message=types.SimpleNamespace(content=json.dumps(answer)))]) +`, + "agent_env/config.py": String.raw` +import types +def get_config(): + call = types.SimpleNamespace(client_kwargs=lambda: {"api_key": "sk-test"}) + return types.SimpleNamespace(get_model_for_role=lambda role: None, get_litellm_base_url=lambda: "https://models.example", + resolve_model_call=lambda model: call) +`, + "agentenv_protocol/__init__.py": "", + "agentenv_protocol/tool_client.py": "class ToolSession:\n pass\n", +}; + +it.each([ + ["REA_AGENT_ENV_COMMAND names the runner", false], + ["the runner is found on PATH", true], +])( + "witnesses a screen-judged task with the bundle's own judge on a fresh screenshot at each stage when %s", + async (_, onPath) => { + const task = { + name: "result-shown", + prompt: "In Notes, show the total and leave it showing.", + checks: [ + { + id: "shown", + kind: "screen_judge", + question: "Does the window show the total?", + }, + ], + }; + const { act, check, computer, directory, runner } = await witnessed( + 0, + {}, + task, + { onPath }, + ); + for (const [path, source] of Object.entries(JUDGE_STUBS)) { + await mkdir(join(runner.state, "site", path, ".."), { recursive: true }); + await writeFile(join(runner.state, "site", path), source); + } + const shows = (verdict: boolean) => + writeFile(join(runner.state, "judge-verdict"), verdict ? "yes" : "no"); + + await shows(false); + const seed = parsed(await check({ task, stage: "seed", directory })); + expect(seed.results[0]).toMatchObject({ id: "shown", passed: false }); + expect(seed.results[0]?.description).toContain("3 of 3 votes"); + await act("type", { text: "=" }); + await shows(true); + await check({ task, stage: "after_edit" }); + await act("type", { text: "x" }); + await shows(false); + await check({ task, stage: "near_miss" }); + await act("reset"); + const last = await check({ task, stage: "after_reset" }); + expectNoToken(computer, last); + const record = parsed(last).witness?.record; + expect(record?.verdict).toBe("held"); + expect(record?.stages.map((stage) => stage.results[0]?.passed)).toEqual([ + false, + true, + false, + false, + ]); + // Three votes a stage, each shown the two examples and the stage's own screenshot. + const calls = (await readFile(join(runner.state, "judge-calls"), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { images: number }); + expect(calls).toHaveLength(12); + expect(calls.every(({ images }) => images === 3)).toBe(true); + // The record says how its results were reached: the bundle's judge, not data/get. + const evidence = parseEvidence( + JSON.parse(await readFile(join(directory, "witness.json"), "utf8")), + ); + expect(evidence.limitations.join(" ")).toContain("judge"); + expect(evidence.limitations.join(" ")).not.toContain("data/get"); + }, +); + it("fails the witness at after_reset when the reset leaves the near miss instead of the seed", async () => { const task = { ...TASK, @@ -724,7 +902,7 @@ const clocked = () => { }; it("grades a stage on what the app saves after the check starts, with progress while it waits", async () => { - const witness = await witnessed(0, {}, TASK, clocked().settle); + const witness = await witnessed(0, {}, TASK, { settle: clocked().settle }); const { app, check, directory } = witness; expect( parsed(await check({ stage: "seed", directory })).witness, @@ -764,7 +942,9 @@ it("grades files still changing at the bound on the last read, and records that ], }; const { settle, now } = clocked(); - const { app, check, directory } = await witnessed(0, {}, passing, settle); + const { app, check, directory } = await witnessed(0, {}, passing, { + settle: settle, + }); const reads: number[] = []; // Every read finds the file changed, with eggs on even reads only. app.onGet = () => { @@ -798,7 +978,9 @@ it("grades files still changing at the bound on the last read, and records that }); it("grades an app that saved before the check within the window plus one interval", async () => { - const { app, check } = await witnessed(0, {}, TASK, clocked().settle); + const { app, check } = await witnessed(0, {}, TASK, { + settle: clocked().settle, + }); app.state = lines("milk", "eggs"); const plain = parsed(await check({})); expect(plain).toMatchObject({ passed: 1, settled: true }); @@ -808,7 +990,7 @@ it("grades an app that saved before the check within the window plus one interva it("counts a slow read's changed files as quiet only from when it returned them", async () => { const { settle, pass } = clocked(); - const { app, check } = await witnessed(0, {}, TASK, settle); + const { app, check } = await witnessed(0, {}, TASK, { settle: settle }); let gets = 0; app.onGet = () => { gets += 1; @@ -838,7 +1020,7 @@ it("counts a slow read's changed files as quiet only from when it returned them" it("ends reads slower than the bound at the bound, grading the last completed read", async () => { const { settle, now, pass, fired } = clocked(); - const { app, check } = await witnessed(0, {}, TASK, settle); + const { app, check } = await witnessed(0, {}, TASK, { settle: settle }); const slow = (SETTLE.boundMs * 2) / 3; const reads: number[] = []; // Each read takes two thirds of the bound; only the first returns eggs. diff --git a/tests/boundary/providers/desktopLongBuildController.test.ts b/tests/boundary/providers/desktopLongBuildController.test.ts new file mode 100644 index 0000000000..f8b5758162 --- /dev/null +++ b/tests/boundary/providers/desktopLongBuildController.test.ts @@ -0,0 +1,108 @@ +import { execFile } from "node:child_process"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; +import { BuildComputerLongRuns } from "../../../src/desktop/BuildComputerLongRuns.js"; +import type { ControlledComputer } from "../../../src/desktop/BuildComputerController.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); + +const computerWithRea = async () => { + const home = await createTestTempDirectory("desktop-long-controller-"); + const bin = join(home, "bin"); + await mkdir(bin); + await writeFile( + join(bin, "rea"), + `#!/usr/bin/env python3 +import os, signal, sys, time +signal.signal(signal.SIGINT, lambda *_: sys.exit(130)) +if sys.argv[1] == "wait": + open(sys.argv[2], "w").close() + time.sleep(30) +print(os.environ.get("AGENTENV_CONTROL_TOKEN", "no token")) +`, + { mode: 0o755 }, + ); + const timeouts: number[] = []; + const computer = { + exec: async ( + argv: readonly string[], + timeout: number, + signal?: AbortSignal, + ) => { + timeouts.push(timeout); + const [command = "", ...args] = argv; + const { stdout, stderr } = await execute(command, args, { + cwd: home, + env: { + ...process.env, + HOME: home, + PATH: `${bin}:${process.env.PATH ?? ""}`, + AGENTENV_CONTROL_TOKEN: "test-token", + }, + ...(signal === undefined ? {} : { signal }), + }); + return { + exit_code: 0, + stdout, + stderr, + timed_out: false, + truncated: false, + }; + }, + homeFiles: async () => new Map(), + redacted: (text: string) => text, + } as ControlledComputer; + return { computer, home, timeouts }; +}; + +it("runs a long-budget export through bounded controller calls and publishes its result", async () => { + const { computer, home, timeouts } = await computerWithRea(); + const owner = new BuildComputerLongRuns(computer); + const result = await owner.run(["done"], { + work: ".rea-build", + stdout: ".rea-build/out.json", + timeoutS: 7080, + signal: new AbortController().signal, + }); + expect(result).toMatchObject({ exitCode: 0, stdoutBytes: 9 }); + expect(await readFile(join(home, ".rea-build/out.json"), "utf8")).toBe( + "no token\n", + ); + expect(Math.max(...timeouts)).toBeLessThanOrEqual(3600); + expect(owner.cleanupResources).toEqual([]); +}); + +it("stops and joins the real supervised command when the caller cancels", async () => { + const { computer, home } = await computerWithRea(); + const ready = join(home, "ready"); + const controller = new AbortController(); + const owner = new BuildComputerLongRuns(computer); + const run = owner + .run(["wait", ready], { + work: ".rea-build", + stdout: ".rea-build/out.json", + timeoutS: 7080, + signal: controller.signal, + }) + .catch((error: unknown) => error); + const until = Date.now() + 5000; + while ( + Date.now() < until && + !(await access(ready).then( + () => true, + () => false, + )) + ) + await new Promise((resolve) => setTimeout(resolve, 20)); + await expect(access(ready)).resolves.toBeUndefined(); + controller.abort(new Error("cancelled")); + const failure = await run; + expect(failure).toBeInstanceOf(AnalysisCancelledError); + expect(owner.cleanupResources).toEqual([]); +}); diff --git a/tests/boundary/providers/desktopScreenJudgePython.test.ts b/tests/boundary/providers/desktopScreenJudgePython.test.ts new file mode 100644 index 0000000000..69be681020 --- /dev/null +++ b/tests/boundary/providers/desktopScreenJudgePython.test.ts @@ -0,0 +1,225 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { renderTaskVerifier } from "../../../src/domain/desktop/appTaskPython.js"; +import { DEFAULT_JUDGE_MODEL } from "../../../src/domain/desktop/appTaskScreen.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const EXAMPLES = { fail: "c2VlZA==", pass: "ZW5k" }; +const SCREENSHOT = "c2NyZWVu"; +const CHECK = { + id: "shown", + kind: "screen_judge" as const, + question: "Does the display show 8.568?", +}; + +/** Stand-ins for the runner's modules: the computer's MCP tools, litellm and agentenv's config, driven by a scenario file. */ +const STUBS: Readonly> = { + "agentenv_protocol/__init__.py": "", + "agentenv_protocol/tool_client.py": String.raw` +import json, os, types +class ToolSession: + def __init__(self, *, mcp_url, **_): + self.mcp_url = mcp_url + async def __aenter__(self): + return self + async def __aexit__(self, *_): + return None + async def call_tool(self, name, arguments=None): + scenario = json.load(open(os.environ["SCENARIO"])) + with open(os.environ["CALLS"], "a") as log: + log.write(json.dumps({"tool": name, "arguments": arguments, "url": self.mcp_url}) + "\n") + return types.SimpleNamespace(isError=False, content=[{"type": "image", "mimeType": "image/jpeg", "data": scenario["image"]}]) +`, + "litellm/__init__.py": String.raw` +import json, os, types +async def acompletion(model, messages, response_format=None, **settings): + scenario = json.load(open(os.environ["SCENARIO"])) + with open(os.environ["CALLS"]) as log: + made = sum(1 for line in log if '"model"' in line) + content = messages[0]["content"] + with open(os.environ["CALLS"], "a") as log: + log.write(json.dumps({"model": model, "settings": settings, "format": response_format["type"], + "images": [part["image_url"]["url"] for part in content if part["type"] == "image_url"], + "texts": [part["text"] for part in content if part["type"] == "text"]}) + "\n") + answer = scenario["answers"][made] + if answer == "raise": + raise ConnectionError("the endpoint dropped the call") + return types.SimpleNamespace(choices=[types.SimpleNamespace(message=types.SimpleNamespace(content=json.dumps(answer)))]) +`, + "agent_env/__init__.py": "", + "agent_env/config/__init__.py": String.raw` +import json, os, types +def get_config(): + scenario = json.load(open(os.environ["SCENARIO"])) + def base_url(): + if scenario.get("no_endpoint"): + raise RuntimeError("No model endpoint configured: set [model] base_url or LITELLM_BASE_URL.") + return "https://models.example" + call = types.SimpleNamespace(client_kwargs=lambda: {"api_key": "sk-test", "api_base": "https://models.example"}) + return types.SimpleNamespace(get_model_for_role=lambda role: scenario.get("roles", {}).get(role), + get_litellm_base_url=base_url, resolve_model_call=lambda model: call) +`, +}; + +const DRIVER = String.raw` +import asyncio, importlib.util, json, sys +spec = importlib.util.spec_from_file_location("verifier", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +module.SCREEN_SETTLE_S = 0 +print(json.dumps(asyncio.run(module.verify("http://computer:8080/mcp", "http://computer:8080")))) +`; + +type Call = { + tool?: string; + model?: string; + settings?: Record; + format?: string; + images?: string[]; + texts?: string[]; +}; + +/** A folder with the stubs, the scenario and the rendered verifier of a task with `checks`. */ +const verifierFolder = async ( + scenario: Record, + checks: readonly (typeof CHECK)[] = [CHECK], +) => { + const folder = await createTestTempDirectory("screen-judge-"); + for (const [path, source] of Object.entries(STUBS)) { + await mkdir(join(folder, "site", path, ".."), { recursive: true }); + await writeFile(join(folder, "site", path), source); + } + await writeFile( + join(folder, "scenario.json"), + JSON.stringify({ image: SCREENSHOT, ...scenario }), + ); + await writeFile(join(folder, "calls.jsonl"), ""); + const verifier = join(folder, "verify.py"); + await writeFile( + verifier, + renderTaskVerifier( + { + name: "tip-total", + prompt: "In Calculator, work it out.", + checks: [...checks], + }, + "calculator", + EXAMPLES, + ), + ); + const env = { + PATH: process.env.PATH ?? "", + PYTHONPATH: join(folder, "site"), + SCENARIO: join(folder, "scenario.json"), + CALLS: join(folder, "calls.jsonl"), + }; + const calls = async () => + (await readFile(join(folder, "calls.jsonl"), "utf8")) + .trim() + .split("\n") + .filter((line) => line !== "") + .map((line) => JSON.parse(line) as Call); + return { folder, verifier, env, calls }; +}; + +const verify = async (scenario: Record) => { + const run = await verifierFolder(scenario); + const { stdout } = await execute("python3", ["-c", DRIVER, run.verifier], { + env: run.env, + }); + return { + results: JSON.parse(stdout) as { + id: string; + result: boolean; + description: string; + }[], + calls: await run.calls(), + }; +}; + +const yes = { verdict: true, reason: "the display shows 8.568" }; +const no = { verdict: false, reason: "the display shows 0" }; + +it("grades a check by the judge's majority of three votes on one fresh screenshot, shown the seed's and the end's examples", async () => { + const { results, calls } = await verify({ answers: [yes, no, yes] }); + + expect(results).toEqual([ + { + id: "shown", + result: true, + description: expect.stringContaining("2 of 3 votes") as string, + }, + ]); + const [shot, ...votes] = calls; + expect(shot).toMatchObject({ + tool: "computer_screenshot", + url: "http://computer:8080/mcp", + }); + expect(votes).toHaveLength(3); + for (const vote of votes) { + expect(vote).toMatchObject({ + model: DEFAULT_JUDGE_MODEL, + settings: { api_key: "sk-test", api_base: "https://models.example" }, + format: "json_schema", + images: [ + `data:image/jpeg;base64,${EXAMPLES.fail}`, + `data:image/jpeg;base64,${EXAMPLES.pass}`, + `data:image/jpeg;base64,${SCREENSHOT}`, + ], + }); + expect(vote.texts?.join(" ")).toContain(CHECK.question); + } +}); + +it("fails a check most votes say no to, naming the votes and a reason", async () => { + const { results } = await verify({ answers: [no, no, yes] }); + expect(results[0]).toMatchObject({ result: false }); + expect(results[0]?.description).toContain("2 of 3 votes"); + expect(results[0]?.description).toContain("the display shows 0"); +}); + +it("asks the model [model.roles] judge names, before the bundle's default", async () => { + const { calls } = await verify({ + answers: [yes, yes, yes], + roles: { judge: "openai/judge-model" }, + }); + expect( + calls.filter(({ model }) => model !== undefined).map(({ model }) => model), + ).toEqual(["openai/judge-model", "openai/judge-model", "openai/judge-model"]); +}); + +it("asks a vote again when its call fails, and raises when it never answers", async () => { + const retried = await verify({ answers: ["raise", yes, yes, yes] }); + expect(retried.results[0]?.result).toBe(true); + await expect( + verify({ answers: ["raise", "raise", "raise"] }), + ).rejects.toThrow("the screen judge did not answer"); +}); + +it("raises without a model endpoint, so the run fails rather than scoring the agent", async () => { + await expect( + verify({ answers: [yes, yes, yes], no_endpoint: true }), + ).rejects.toThrow("the screen judge has no model endpoint"); +}); + +it("judges a screenshot it is given when run as a script, as REA's witness runs it", async () => { + const run = await verifierFolder({ answers: [yes, yes, no] }); + const image = join(run.folder, "shot.jpg"); + await writeFile(image, Buffer.from("screen")); + const { stdout } = await execute( + "python3", + [run.verifier, "judge", JSON.stringify(CHECK), image], + { env: run.env }, + ); + expect(JSON.parse(stdout)).toMatchObject({ id: "shown", result: true }); + const votes = (await run.calls()).filter(({ model }) => model !== undefined); + expect(votes[0]?.images?.at(-1)).toBe( + `data:image/jpeg;base64,${Buffer.from("screen").toString("base64")}`, + ); +}); diff --git a/tests/boundary/providers/desktopTaskArrayIndex.test.ts b/tests/boundary/providers/desktopTaskArrayIndex.test.ts index 688070d32a..bacc7ce32c 100644 --- a/tests/boundary/providers/desktopTaskArrayIndex.test.ts +++ b/tests/boundary/providers/desktopTaskArrayIndex.test.ts @@ -50,3 +50,21 @@ print(json.dumps(results)) }), ); }); + +it("evaluates root-array member counts in the exported Python verifier", async () => { + const folder = await createTestTempDirectory("task-root-count-"); + const script = join(folder, "fixture.py"); + await writeFile( + script, + `import json, re, math +${PYTHON_TASK_EVALUATOR} +check = {"id": "count", "kind": "member_count", "file": "state.json", "pointer": "", "equals": 3} +print(json.dumps([evaluate([check], {"state.json": {"format": "json", "view": values, "truncated": False}})[0]["result"] for values in [[1, 2], [1, 2, 3]]])) +`, + ); + expect( + JSON.parse( + execFileSync("python3", ["-I", "-B", script], { encoding: "utf8" }), + ), + ).toEqual([false, true]); +}); diff --git a/tests/boundary/providers/desktopTaskPython.test.ts b/tests/boundary/providers/desktopTaskPython.test.ts index 833e05b31d..d536f74627 100644 --- a/tests/boundary/providers/desktopTaskPython.test.ts +++ b/tests/boundary/providers/desktopTaskPython.test.ts @@ -4,20 +4,34 @@ import { join } from "node:path"; import { expect, it } from "vitest"; -import type { DesktopTaskCheck } from "../../../src/domain/desktop/appTask.js"; -import { evaluateTaskChecks } from "../../../src/domain/desktop/appTaskChecks.js"; +import type { + DesktopTaskCheck, + DesktopTaskReader, +} from "../../../src/domain/desktop/appTask.js"; +import { + evaluateTaskChecks, + type TaskFileView, +} from "../../../src/domain/desktop/appTaskChecks.js"; import { CHECK_CASES, CHECK_VIEWS, } from "../../../src/domain/desktop/appTaskChecks.fixture.js"; import { SETTLE_TIMING } from "../../../src/domain/desktop/appSettle.js"; import { renderTaskVerifier } from "../../../src/domain/desktop/appTaskPython.js"; +import { + readerAnswerView, + readerCommand, +} from "../../../src/domain/desktop/appTaskReaders.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; /** The protocol client calls the verifier makes, replaced so it reads the given views. */ const STUB_CLIENT = ` def find_child(card, name): return next((child for child in card.get("children_environments") or [] if child.get("name") == name), None) + + +def find_extension(card, uri): + return next((ext for ext in (card.get("capabilities") or {}).get("extensions") or [] if ext.get("uri") == uri), None) `; /** @@ -33,6 +47,12 @@ from agentenv_protocol import client scenario = json.load(sys.stdin) saves, durations, called, reads, clock = scenario["saves"], scenario["durations"], [], [], [0.0] +answers, execs = scenario.get("answers") or {}, [] +cleanup_failures = [scenario.get("cleanup_failures", 0)] +cancel_cleanup = [scenario.get("cancel_cleanup", False)] +verify_task = None +reader_started, release_reader = asyncio.Event(), asyncio.Event() +reader_calls = [0] #: Virtual seconds each data/get takes, a binary fraction so the clock stays exact. READ_S = 0.125 @@ -43,7 +63,34 @@ def views_at(now): async def get_card(url, **_): called.append(url) - return {"name": "computer", "children_environments": [{"name": "notes", "url": "/svc/notes/agentenv"}]} + extensions = [{"uri": "urn:agentenv:computer-exec/v1"}] if scenario.get("exec", True) else [] + return {"name": "computer", "children_environments": [{"name": "notes", "url": "/svc/notes/agentenv"}], + "capabilities": {"extensions": extensions}} + + +async def invoke_extension(url, card, uri, params=None, timeout=30, verify=True, *, method=None, headers=None): + execs.append({"url": url, "uri": uri, "method": method, "timeout": timeout, **params}) + if params["request_id"].startswith("rea-reader-cleanup-"): + if cancel_cleanup[0]: + cancel_cleanup[0] = False + verify_task.cancel() + await asyncio.sleep(0) + if cleanup_failures[0] > 0: + cleanup_failures[0] -= 1 + return {"exit_code": 1, "stdout": "", "stderr": "permission denied", "timed_out": False} + if scenario.get("parallel_retry"): + await asyncio.sleep(0) + return {"exit_code": 0, "stdout": "", "stderr": "", "timed_out": False} + if scenario.get("concurrent"): + reader_calls[0] += 1 + if reader_calls[0] == 1: + reader_started.set() + await release_reader.wait() + else: + release_reader.set() + answer = answers[json.loads("".join(params["argv"][4:]))["id"]] + stdout = answer if isinstance(answer, str) else "Python noise\n" + json.dumps(answer) + "\n" + return {"exit_code": 0, "stdout": stdout, "stderr": "", "truncated": False, "timed_out": False} async def get_data(url, **_): @@ -74,20 +121,55 @@ async def wait_for(awaitable, timeout): return result -client.get_card, client.get_data = get_card, get_data -spec = importlib.util.spec_from_file_location("verifier", verifier) -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) -module.time = types.SimpleNamespace(monotonic=lambda: clock[0]) -module.asyncio = types.SimpleNamespace(sleep=sleep, wait_for=wait_for, TimeoutError=Cut) -results = asyncio.run(module.verify("http://mcp", "http://computer:8080")) -print(json.dumps({"called": called, "reads": reads, "ended": clock[0], "results": results})) +client.get_card, client.get_data, client.invoke_extension = get_card, get_data, invoke_extension +def load_verifier(): + spec = importlib.util.spec_from_file_location("verifier", verifier) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.time = types.SimpleNamespace(monotonic=lambda: clock[0]) + module.asyncio = types.SimpleNamespace(sleep=sleep, wait_for=wait_for, TimeoutError=Cut, + create_task=asyncio.create_task, shield=asyncio.shield, + CancelledError=asyncio.CancelledError) + return module + +module = load_verifier() +async def run(): + global verify_task, module + first_error = None + if scenario.get("concurrent"): + first = asyncio.create_task(module.verify("http://mcp", "http://computer:8080")) + await asyncio.wait_for(reader_started.wait(), timeout=2) + other = load_verifier() + second = asyncio.create_task(other.verify("http://mcp", "http://computer:8080")) + results, _ = await asyncio.wait_for(asyncio.gather(first, second), timeout=5) + return results, first_error + if scenario.get("retry"): + try: + verify_task = asyncio.create_task(module.verify("http://mcp", "http://computer:8080")) + await verify_task + except (RuntimeError, asyncio.CancelledError) as error: + first_error = str(error) + " ".join(getattr(error, "__notes__", ())) + if scenario.get("reload_verifier"): + module = load_verifier() + if scenario.get("parallel_retry"): + other = load_verifier() + first = asyncio.create_task(module.verify("http://mcp", "http://computer:8080")) + second = asyncio.create_task(other.verify("http://mcp", "http://computer:8080")) + results, _ = await asyncio.gather(first, second) + return results, first_error + results = await module.verify("http://mcp", "http://computer:8080") + return results, first_error + +results, first_error = asyncio.run(run()) +print(json.dumps({"called": called, "reads": reads, "ended": clock[0], "results": results, "execs": execs, "first_error": first_error})) `; type VerifierOutput = { called: string[]; + execs: Record[]; reads: number[]; ended: number; + first_error: string | null; results: { id: string; result: boolean; @@ -102,6 +184,17 @@ const runVerifier = async ( checks: readonly DesktopTaskCheck[], saves: readonly (readonly [number, Record])[], durations: readonly number[] = [], + computer: { + readonly readers?: readonly DesktopTaskReader[]; + readonly answers?: Readonly>; + readonly exec?: boolean; + readonly cleanupFailures?: number; + readonly retry?: boolean; + readonly cancelCleanup?: boolean; + readonly reloadVerifier?: boolean; + readonly concurrent?: boolean; + readonly parallelRetry?: boolean; + } = {}, ): Promise => { const folder = await createTestTempDirectory("desktop-task-python-"); await mkdir(join(folder, "agentenv_protocol")); @@ -115,13 +208,31 @@ const runVerifier = async ( await writeFile( verifier, renderTaskVerifier( - { name: "edit-notes", prompt: "Edit.", checks: [...checks] }, + { + name: "edit-notes", + prompt: "Edit.", + checks: [...checks], + ...(computer.readers === undefined + ? {} + : { readers: computer.readers }), + }, "notes", ), ); return JSON.parse( execFileSync("python3", ["-I", "-B", "-c", DRIVER, folder, verifier], { - input: JSON.stringify({ saves, durations }), + input: JSON.stringify({ + saves, + durations, + answers: computer.answers, + exec: computer.exec ?? true, + cleanup_failures: computer.cleanupFailures, + retry: computer.retry, + cancel_cleanup: computer.cancelCleanup, + reload_verifier: computer.reloadVerifier, + concurrent: computer.concurrent, + parallel_retry: computer.parallelRetry, + }), encoding: "utf8", }), ) as VerifierOutput; @@ -240,3 +351,285 @@ it("grades an app that saved before the check within the window plus one interva expect(graded(output)).toEqual([[true, true, false]]); expect(output.reads.at(-1)).toBeLessThanOrEqual(WINDOW_S + INTERVAL_S); }); + +const READERS: readonly DesktopTaskReader[] = [ + { + id: "layers", + file: "Documents/poster.xcf", + argv: ["gimp-console-2.10", "-i", "-b", "-"], + stdin: '(gimp-file-load RUN-NONINTERACTIVE "{file}" "{file}")', + output: "lines", + }, + { + id: "objects", + file: "Documents/scene.blend", + argv: ["blender", "-b", "{file}", "--python-expr", "print('REA {}')"], + output: "json", + timeout_s: 20, + }, + { + id: "broken", + file: "Documents/broken.xcf", + argv: ["gimp-console-2.10", "{file}"], + output: "lines", + }, + { + id: "gone", + file: "Documents/gone.xcf", + argv: ["gimp-console-2.10", "{file}"], + output: "lines", + }, +]; +/** What REA's reader runner answers for each reader, as its last line. */ +const ANSWERS: Readonly> = { + layers: { + view: { + format: "text", + view: { + lines: ["layer Summer Sale", "layer Background"], + line_count: 2, + truncated: false, + }, + }, + }, + objects: { + view: { + format: "json", + view: { + Crate: { location: [2.0004, 0, 0] }, + Camera: { location: [7, -6, 5] }, + }, + }, + }, + broken: { + error: "the reader exited 1; its stderr ends: cannot open the file", + }, + gone: { absent: true }, +}; +const READER_CHECKS: DesktopTaskCheck[] = [ + { + id: "renamed", + kind: "lines_present", + file: "Documents/poster.xcf", + reader: "layers", + lines: ["layer Summer Sale"], + }, + { + id: "kept", + kind: "lines_absent", + file: "Documents/poster.xcf", + reader: "layers", + lines: ["layer Headline"], + }, + { + id: "moved", + kind: "value_equals", + file: "Documents/scene.blend", + reader: "objects", + pointer: "/Crate/location/0", + value: 2, + tolerance: 0.001, + }, + { + id: "camera", + kind: "value_equals", + file: "Documents/scene.blend", + reader: "objects", + pointer: "/Camera/location/0", + value: 8, + }, + { + id: "objects", + kind: "member_count", + file: "Documents/scene.blend", + reader: "objects", + pointer: "/Crate/location", + equals: 3, + }, + { + id: "broken", + kind: "line_count", + file: "Documents/broken.xcf", + reader: "broken", + contains: "layer", + at_least: 1, + }, + { + id: "gone", + kind: "lines_absent", + file: "Documents/gone.xcf", + reader: "gone", + lines: ["layer"], + }, +]; + +it("sends large reader input in the same bounded arguments as the witness", async () => { + const reader = { ...READERS[0]!, stdin: "é".repeat(65536) }; + const output = await runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], { + readers: [reader], + answers: ANSWERS, + }); + const argv = output.execs[0]?.argv as string[]; + const arguments_ = JSON.parse(argv.slice(4).join("")) as Record< + string, + unknown + >; + expect(arguments_).toMatchObject({ + stdin: reader.stdin, + lease: { path: expect.stringMatching(/^\/tmp\/rea-reader-/u) }, + }); + expect( + argv.slice(4).every((argument) => Buffer.byteLength(argument) < 131072), + ).toBe(true); +}); + +it("reads each reader's view through the computer's computer-exec and grades it as REA does", async () => { + const output = await runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], { + readers: READERS, + answers: ANSWERS, + }); + + expect( + output.execs + .filter( + ({ request_id }) => + !String(request_id).startsWith("rea-reader-cleanup-"), + ) + .map(({ url, uri, method, argv, timeout_s }) => ({ + url, + uri, + method, + argv: (argv as string[]).slice(0, 4), + timeout_s, + })), + ).toEqual( + READERS.map((reader) => ({ + url: "http://computer:8080", + uri: "urn:agentenv:computer-exec/v1", + method: "run", + argv: readerCommand(reader, undefined).slice(0, 4), + timeout_s: (reader.timeout_s ?? 60) + 30, + })), + ); + // computer-exec answers a repeated request id with its first result, so each reader's is new. + expect(new Set(output.execs.map(({ request_id }) => request_id)).size).toBe( + READERS.length * 2, + ); + const readers = new Map(); + for (const [id, answer] of Object.entries(ANSWERS)) { + const view = readerAnswerView(JSON.stringify(answer)); + if (view !== undefined) readers.set(id, view); + } + const typescript = evaluateTaskChecks( + READER_CHECKS, + new Map(Object.entries(CHECK_VIEWS)), + readers, + ); + expect( + output.results.map(({ id, result, description }) => [ + id, + result, + description, + ]), + ).toEqual( + typescript.map(({ id, passed, description }) => [id, passed, description]), + ); + expect(output.results.map(({ result }) => result)).toEqual([ + true, + true, + true, + false, + true, + false, + true, + ]); +}); + +it("reports a cleanup failure with the completed view and retries the same folder on the next verification", async () => { + const output = await runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], { + readers: [READERS[0]!], + answers: ANSWERS, + cleanupFailures: 1, + retry: true, + reloadVerifier: true, + }); + expect(output.first_error).toContain("Could not remove owned reader folder"); + expect(output.first_error).toContain("completed reader views"); + const cleanups = output.execs.filter(({ request_id }) => + String(request_id).startsWith("rea-reader-cleanup-"), + ); + expect(cleanups).toHaveLength(3); + const lease = (entry: Record) => + JSON.parse((entry.argv as string[])[4]!); + expect(lease(cleanups[1]!)).toEqual(lease(cleanups[0]!)); + expect(cleanups[1]?.request_id).not.toBe(cleanups[0]?.request_id); +}); + +it("keeps cleanup owned when verification is cancelled during final cleanup", async () => { + const output = await runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], { + readers: [READERS[0]!], + answers: ANSWERS, + cancelCleanup: true, + retry: true, + reloadVerifier: true, + }); + expect(output.first_error).toContain("Reader cleanup remains owned"); + expect(output.first_error).toContain("completed reader views"); + const cleanups = output.execs.filter(({ request_id }) => + String(request_id).startsWith("rea-reader-cleanup-"), + ); + expect(cleanups).toHaveLength(2); +}); + +it("does not adopt another active verifier's reader lease", async () => { + const output = await runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], { + readers: [READERS[0]!], + answers: ANSWERS, + concurrent: true, + }); + expect( + output.execs + .slice(0, 2) + .every( + ({ request_id }) => + !String(request_id).startsWith("rea-reader-cleanup-"), + ), + ).toBe(true); +}); + +it("joins concurrent retries of one abandoned reader cleanup", async () => { + const output = await runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], { + readers: [READERS[0]!], + answers: ANSWERS, + cleanupFailures: 1, + retry: true, + parallelRetry: true, + }); + expect(output.first_error).toContain("Could not remove owned reader folder"); + const cleanups = output.execs.filter(({ request_id }) => + String(request_id).startsWith("rea-reader-cleanup-"), + ); + expect(cleanups).toHaveLength(4); + const lease = (entry: Record) => + JSON.parse((entry.argv as string[])[4]!); + expect(lease(cleanups[1]!)).toEqual(lease(cleanups[0]!)); +}); + +it.each([ + [{ exec: false }, "offers no urn:agentenv:computer-exec/v1"], + [ + { answers: { ...ANSWERS, layers: "Traceback (most recent call last)\n" } }, + "REA's reader runner", + ], +])( + "raises rather than grading when the readers cannot be run (%j)", + async (computer, message) => { + await expect( + runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], { + readers: READERS, + answers: ANSWERS, + ...computer, + }), + ).rejects.toThrow(message); + }, +); diff --git a/tests/boundary/providers/desktopTaskReaderRunner.test.ts b/tests/boundary/providers/desktopTaskReaderRunner.test.ts new file mode 100644 index 0000000000..3c7fedea74 --- /dev/null +++ b/tests/boundary/providers/desktopTaskReaderRunner.test.ts @@ -0,0 +1,367 @@ +import { execFile } from "node:child_process"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { readerViewsOfBytes } from "../../../src/application/desktop/DesktopTaskReaders.js"; +import type { DesktopTaskReader } from "../../../src/domain/desktop/appTask.js"; +import { + readerAnswerView, + readerCommand, + PYTHON_READER_CLEANUP, +} from "../../../src/domain/desktop/appTaskReaders.js"; +import { runReaderHere } from "../../../src/desktop/TaskReaderProcess.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const NOTE = "Documents/today.txt"; +const SECRET = "control-token-secret"; + +/** A fake app's headless mode: it reports what it was given, then changes the file it read. */ +const REPORTING_APP = String.raw` +import json, os, sys +path = sys.argv[1] +data = open(path, "rb").read().decode() +open(path, "ab").write(b" tampered") +print("App 1.0 starting") +print("REA " + json.dumps({ + "home": os.environ.get("HOME"), "display": os.environ.get("DISPLAY"), "bus": os.environ.get("DBUS_SESSION_BUS_ADDRESS"), + "config": os.environ.get("XDG_CONFIG_HOME"), "control": sorted(k for k in os.environ if k.startswith("AGENTENV_")), + "path": path, "cwd": os.getcwd(), "content": data, "stdin": sys.stdin.read()})) +print("App quit") +`; + +/** A computer's home with the note in it and the fake app beside it. */ +const computer = async () => { + const root = await createTestTempDirectory("reader-runner-"); + const home = join(root, "home"); + await mkdir(join(home, "Documents"), { recursive: true }); + await writeFile(join(home, NOTE), "milk\neggs\n"); + const app = join(root, "app.py"); + await writeFile(app, REPORTING_APP); + return { home, app }; +}; + +/** Run the runner as computer-exec runs it in the session: the session's environment, its control token included. */ +const run = async (home: string, reader: DesktopTaskReader, path?: string) => { + const [python = "", ...rest] = readerCommand(reader, path); + const { stdout } = await execute(python, rest, { + cwd: home, + env: { + PATH: process.env.PATH ?? "", + HOME: home, + DISPLAY: ":99", + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/1000/bus", + AGENTENV_CONTROL_TOKEN: SECRET, + }, + maxBuffer: 4 * 1024 * 1024, + }); + return stdout; +}; + +const reporting = (app: string, fields: Partial = {}) => + ({ + id: "report", + file: NOTE, + argv: ["python3", app, "{file}"], + output: "json", + ...fields, + }) as DesktopTaskReader; + +it("runs the reader on a private copy of the file in a fresh home, without the display, the session bus or the control token", async () => { + const { home, app } = await computer(); + const stdout = await run(home, reporting(app, { stdin: "open {file}" })); + + expect(stdout).not.toContain(SECRET); + const view = readerAnswerView(stdout); + const report = view?.view as Record; + expect(view?.format).toBe("json"); + expect(report).toMatchObject({ + display: null, + bus: null, + control: [], + content: "milk\neggs\n", + }); + const copy = String(report.path); + expect(copy.endsWith("/today.txt")).toBe(true); + expect(copy.startsWith(home)).toBe(false); + expect(report.stdin).toBe(`open ${copy}`); + // The copy and the fresh home are siblings in the runner's own folder. + expect(String(report.home).startsWith(dirname(dirname(copy)))).toBe(true); + expect(String(report.config).startsWith(String(report.home))).toBe(true); + expect(report.cwd).toBe(dirname(copy)); + // The reader changed only its copy, which is gone with its folder. + expect(await readFile(join(home, NOTE), "utf8")).toBe("milk\neggs\n"); + await expect(access(dirname(dirname(copy)))).rejects.toThrow(); +}); + +it("retains a remotely owned copy until the matching lease removes it", async () => { + const { home, app } = await computer(); + const root = await createTestTempDirectory("reader-lease-"); + const lease = { path: join(root, "owned"), token: "owner-token" }; + const [python = "", ...args] = readerCommand( + reporting(app), + undefined, + undefined, + lease, + ); + const { stdout } = await execute(python, args, { + env: { ...process.env, HOME: home }, + }); + expect(readerAnswerView(stdout)?.format).toBe("json"); + await expect( + access(join(lease.path, "file", "today.txt")), + ).resolves.toBeUndefined(); + await expect( + execute(python, [ + "-I", + "-c", + PYTHON_READER_CLEANUP, + JSON.stringify({ ...lease, token: "other" }), + ]), + ).rejects.toThrow(); + await expect(access(lease.path)).resolves.toBeUndefined(); + const refused = `import shutil\ndef fail(path): raise PermissionError("denied")\nshutil.rmtree = fail\nexec(${JSON.stringify(PYTHON_READER_CLEANUP)})`; + await expect( + execute(python, ["-I", "-c", refused, JSON.stringify(lease)]), + ).rejects.toThrow(); + expect(await readFile(`${lease.path}.owner`, "utf8")).toBe(lease.token); + await execute(python, [ + "-I", + "-c", + PYTHON_READER_CLEANUP, + JSON.stringify(lease), + ]); + await expect(access(lease.path)).rejects.toThrow(); + await expect(access(`${lease.path}.owner`)).rejects.toThrow(); +}); + +it("passes maximum-length Unicode stdin through bounded arguments to the real reader", async () => { + const { home, app } = await computer(); + const input = "é".repeat(8190) + "😀" + "é".repeat(57344); + const reader = reporting(app, { stdin: input }); + const command = readerCommand(reader, undefined); + expect(Buffer.byteLength(command.slice(4).join(""))).toBeGreaterThan(131072); + expect( + Math.max(...command.slice(4).map((part) => Buffer.byteLength(part))), + ).toBeLessThan(131072); + const report = readerAnswerView(await run(home, reader))?.view as Record< + string, + unknown + >; + expect(report.stdin).toBe(input); +}); + +it("keeps the display for a reader that asks for it, still without the control token", async () => { + const { home, app } = await computer(); + const report = readerAnswerView( + await run(home, reporting(app, { display: true })), + )?.view as Record; + expect(report).toMatchObject({ display: ":99", control: [] }); +}); + +it("reads the file a path names instead of the home's, under the home file's name", async () => { + const { home, app } = await computer(); + const blob = join(home, "..", "blob-0123"); + await writeFile(blob, "seed bytes"); + const report = readerAnswerView(await run(home, reporting(app), blob)) + ?.view as Record; + expect(report.content).toBe("seed bytes"); + expect(String(report.path).endsWith("/today.txt")).toBe(true); +}); + +it("makes the marked lines a text view and drops the rest", async () => { + const { home } = await computer(); + const view = readerAnswerView( + await run(home, { + id: "lines", + file: NOTE, + argv: [ + "python3", + "-c", + "import sys\nprint('ts> banner')\nfor line in open(sys.argv[1]): print('REA item ' + line.strip())\nprint('REA')\nprint('REAL news')", + "{file}", + ], + output: "lines", + }), + ); + expect(view).toEqual({ + format: "text", + view: { + lines: ["item milk", "item eggs"], + line_count: 2, + truncated: false, + }, + }); +}); + +it("answers absent for a file the home does not hold, without running the reader", async () => { + const { home } = await computer(); + const stdout = await run(home, { + id: "gone", + file: "Documents/gone.txt", + argv: ["sh", "-c", `touch '${home}/ran'; echo 'REA x' {file}`], + output: "lines", + }); + expect(readerAnswerView(stdout)).toBeUndefined(); + await expect(access(join(home, "ran"))).rejects.toThrow(); +}); + +it.each([ + [ + "exits non-zero", + ["sh", "-c", "echo 'REA x'; echo 'cannot open {file}' >&2; exit 3"], + "json", + ["the reader exited 3", "cannot open"], + ], + [ + "prints no marked line", + ["sh", "-c", "echo hello {file}"], + "lines", + ['printed no line starting with "REA "'], + ], + [ + "prints a view that is not JSON", + ["sh", "-c", "echo 'REA {not json' {file}"], + "json", + ["are not one JSON value"], + ], + [ + "prints a number JSON cannot hold exactly", + ["sh", "-c", "echo 'REA {\"id\": 9007199254740993}'; : {file}"], + "json", + ["cannot be read exactly"], + ], + ...["9007199254740993.0", "1e-400", "-0.0"].map((number) => [ + `prints an inexact decimal ${number}`, + ["sh", "-c", `echo 'REA {"id": ${number}}'; : {file}`], + "json", + ["cannot be read exactly"], + ]), + [ + "prints more than 1 MiB", + ["python3", "-c", "print('REA ' + 'x' * (1 << 20))", "{file}"], + "lines", + ["printed more than 1 MiB"], + ], + [ + "cannot start", + ["no-such-app-binary", "{file}"], + "lines", + ["could not start no-such-app-binary"], + ], +])( + "reports a reader that %s as an error view", + async (_, argv, output, facts) => { + const { home } = await computer(); + const view = readerAnswerView( + await run(home, { + id: "broken", + file: NOTE, + argv: argv as string[], + output: output as "json" | "lines", + }), + ); + expect(view?.format).toBeNull(); + for (const fact of facts) expect(view?.error).toContain(fact); + }, +); + +it("stops a reader past its timeout, with the processes it started", async () => { + const { home } = await computer(); + const started = Date.now(); + const view = readerAnswerView( + await run(home, { + id: "slow", + file: NOTE, + argv: [ + "sh", + "-c", + `(sleep 3; touch "${home}/late") & echo 'REA x'; sleep 30; : {file}`, + ], + output: "lines", + timeout_s: 1, + }), + ); + expect(view?.error).toContain("ran past its 1 s"); + expect(Date.now() - started).toBeLessThan(10_000); + await new Promise((resolve) => setTimeout(resolve, 3500)); + await expect(access(join(home, "late"))).rejects.toThrow(); +}); + +it("runs a reader whose file shares a name with the runner's own folders and files", async () => { + const { home } = await computer(); + for (const name of ["home", "stdout", "stderr"]) { + await writeFile(join(home, "Documents", name), `${name} bytes`); + const view = readerAnswerView( + await run(home, { + id: "named", + file: `Documents/${name}`, + argv: ["sh", "-c", 'echo REA $(cat "$1")', "sh", "{file}"], + output: "lines", + }), + ); + expect(view?.view, name).toEqual({ + lines: [`${name} bytes`], + line_count: 1, + truncated: false, + }); + } +}); + +it("stops a reader once it has printed more than 1 MiB, rather than at its timeout", async () => { + const { home } = await computer(); + const started = Date.now(); + const view = readerAnswerView( + await run(home, { + id: "flood", + file: NOTE, + argv: ["sh", "-c", "yes 'REA x'; : {file}"], + output: "lines", + timeout_s: 30, + }), + ); + expect(view?.error).toContain("printed more than 1 MiB"); + expect(Date.now() - started).toBeLessThan(10_000); +}); + +it("stops the reader, the processes it started and its folder when the export is cancelled", async () => { + const { home } = await computer(); + const out = join(home, "..", "out"); + await mkdir(out); + const controller = new AbortController(); + const read = readerViewsOfBytes( + [ + { + id: "stuck", + file: NOTE, + // Python, not the system's sleep, whose environment macOS withholds from the run token sweep. + argv: [ + "python3", + "-c", + `import os, time; open("${out}/cwd", "w").write(os.getcwd()); open("${out}/pid", "w").write(str(os.getpid())); time.sleep(60)`, + "{file}", + ], + output: "lines", + }, + ], + () => Promise.resolve({ kind: "bytes", bytes: Buffer.from("milk\n") }), + (argv, timeoutS) => runReaderHere(argv, timeoutS, controller.signal), + ); + const pid = async (): Promise => { + for (let tries = 0; tries < 200; tries += 1) { + const text = await readFile(join(out, "pid"), "utf8").catch(() => ""); + if (text.trim() !== "") return Number(text.trim()); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error("the reader never started"); + }; + const reader = await pid(); + controller.abort(new Error("cancelled")); + await expect(read).rejects.toThrow(); + expect(() => process.kill(reader, 0)).toThrow(); + const folder = (await readFile(join(out, "cwd"), "utf8")).trim(); + await expect(access(folder)).rejects.toThrow(); +}); diff --git a/tests/fixtures/buildComputerMcp.ts b/tests/fixtures/buildComputerMcp.ts index f8d0229b8f..ebd3edd153 100644 --- a/tests/fixtures/buildComputerMcp.ts +++ b/tests/fixtures/buildComputerMcp.ts @@ -1,3 +1,4 @@ +import type { BuildComputerDefaults } from "../../src/desktop/BuildComputerEnvironment.js"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -43,6 +44,7 @@ export const FIRST_READ: SettleTiming = { export const connectBuildServer = async ( environment: Readonly>, settle: BuildSettle = FIRST_READ, + defaults?: BuildComputerDefaults, ): Promise => { const session = createTestBinarySession(() => { throw new Error("build computers must not start a deep provider"); @@ -56,7 +58,7 @@ export const connectBuildServer = async ( ...environment, }, desktopBuildSettle: settle, - desktopBuildDefaults: await isolatedBuildDefaults(), + desktopBuildDefaults: defaults ?? (await isolatedBuildDefaults()), }, ); const client = new Client({ name: "build-computer-mcp", version: "1" }); diff --git a/tests/fixtures/fakeAgentEnvComputer.ts b/tests/fixtures/fakeAgentEnvComputer.ts index dada1986b0..588150fe92 100644 --- a/tests/fixtures/fakeAgentEnvComputer.ts +++ b/tests/fixtures/fakeAgentEnvComputer.ts @@ -20,7 +20,7 @@ import { type ServerResponse, } from "node:http"; import type { AddressInfo } from "node:net"; -import { join, relative } from "node:path"; +import { dirname, join, relative } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { @@ -421,7 +421,13 @@ const buildRoute = async ( id?: number; method?: string; argv?: string[]; - params?: { paths?: string[] }; + params?: { + paths?: string[]; + parts?: { + file: { bytes: string }; + metadata: Record; + }[]; + }; }; if (path === "/.well-known/agent-env.json") return { @@ -437,6 +443,19 @@ const buildRoute = async ( if (exec === undefined) return undefined; if (path === "/agentenv/ext/computer-exec/run") return execRoute(exec, token, request.argv ?? [], sessions); + // data/add writes each file part under the home, at the path its marker names. + if (path === "/agentenv" && request.method === "data/add") { + for (const part of request.params?.parts ?? []) { + const [marker] = Object.values(part.metadata); + if (marker === undefined) continue; + await mkdir(dirname(join(exec.home, marker.path)), { recursive: true }); + await writeFile( + join(exec.home, marker.path), + Buffer.from(part.file.bytes, "base64"), + ); + } + return { jsonrpc: "2.0", id: request.id, result: {} }; + } const paths = request.params?.paths ?? []; // Recordings keep the canned segment; other paths are the home's files. if (