diff --git a/.agents/skills/reverse-engineer-anything/references/desktop-build.md b/.agents/skills/reverse-engineer-anything/references/desktop-build.md
index efad507a94..233b49af47 100644
--- a/.agents/skills/reverse-engineer-anything/references/desktop-build.md
+++ b/.agents/skills/reverse-engineer-anything/references/desktop-build.md
@@ -72,6 +72,13 @@ the first source that has the app:
`/usr/share/applications` without `.desktop`, such as
`org.inkscape.Inkscape`. Add `window_class` when you know it; Inkscape's
is `Inkscape`. AbiWord's entry is `abiword` and its class `Abiword`.
+- To find the entry, read the package's file list: its page on
+ packages.ubuntu.com, or `apt-get download PKG && dpkg -c PKG_*.deb` where
+ apt runs. The entry may ship in a companion `-data` or `-common` package,
+ as Planner's in `planner-data`: read that package's list too. The entry's `StartupWMClass` may not be the window's real
+ class, so do not take `window_class` from it: MuseScore 3's says
+ `mscore3`, its window's is `MuseScore3`. The export uses the class the
+ capture saw.
## Drive the app
@@ -97,16 +104,25 @@ the first source that has the app:
environment's name.
- Prefer shortcuts and typing to long pointer paths: `key` takes `keys`, an
array holding one chord's keys, such as
- `{"action": "key", "keys": ["ctrl", "s"]}`, and `type` turns `\n` into
+ `{"action": "key", "keys": ["Control", "s"]}`, and `type` turns `\n` into
Enter.
- Before `type`, check in a screenshot that a text cursor or a focused
field is there. In apps with single-key shortcuts, such as Inkscape, text
typed without one runs as commands.
-- Save As opens in the home: click Documents in its sidebar and type only
- the file name. Answer the app's save questions, such as keeping the
- format, then check the saved path in the title or status bar, or the
- dialog's folder when the app shows only the name, and that it shows no
- unsaved mark, before you mark a checkpoint or check.
+- Save As differs by app. Check the dialog's current folder first: most
+ open in the home, so click Documents in the sidebar, but Scribus opens in
+ `~/Documents` and its sidebar has no Documents entry. Type only the file
+ name. A dialog taller than the 1280×800 screen hides its buttons:
+ maximize it from its title bar, or confirm it with `key` Return, as when
+ HomeBank's Save button is off-screen.
+- Answer the app's save questions, such as keeping the format, then check
+ the saved path in the title or status bar, or the dialog's folder when
+ the app shows only the name, and that it shows no unsaved mark, before
+ you mark a checkpoint or check. Where the title never shows one, as in
+ HomeBank, a greyed Save or Revert, or the file reopened from recent
+ files, confirms the save.
+- Qt spin boxes change their value on scroll: move the pointer off them
+ before you scroll.
- What the app saved under the home before `seed`, such as a document in
`~/Documents` or its settings, is part of the seed. A file you move to the
Trash is left out, and the export lists it in `limitations`. Trash a stray
@@ -126,12 +142,30 @@ the first source that has the app:
1. `start_build_computer` with
`{"source": {"kind": "request", "desktop_entry": ..., "install": ...}}`.
- The first start of an app builds its image and can take minutes. A
- failed start returns the build's error: fix the request, or stop when the
+ The first start of an app builds its image, and starts at the same time
+ on this host wait for each other; either can take minutes. A failed
+ start returns the build's error: fix the request, or stop when the
install cannot work, and close any `computer_id` the error names.
2. `start_build_capture`. It returns once the app's window is open. A
`package_lists_error` may stop the capture finding the package's
- sources; the export says when that matters.
+ sources; the export says when that matters. When the app exits before
+ its window opens, the error gives its exit code and the end of its
+ stderr, which may name what it lacks, such as a Java runtime: try
+ another install source, or stop as for an install this flow cannot do.
+ - REA counts a file as the app's document only when the app's window
+ title shows its file name or `~/` path. When the title shows something
+ else, such as MuseScore's score title, pass the path you will save the
+ document at in `state_paths`, relative to the home:
+ `{"computer_id": ..., "state_paths": ["Documents/MuseScore3/Scores/Morning_Song.mscz"]}`.
+ It need not exist yet. Add the app's settings folder when its name
+ differs from the app's, such as `.config/MuseScore`, so the seed keeps
+ the dismissed first-run dialogs. Do not pass the document as a launch
+ argument to make REA see it: `state_paths` is the way.
+ - An app whose document is a folder of pages, such as a Zim notebook
+ (`Notebooks/Notes`, a `.txt` file per page), never shows a file in its
+ title: pass the folder in `state_paths`. Every page in it is then a
+ document, so the bundle opens none at launch and the app starts where
+ it keeps its place; check the page the task changes.
3. Make the seed through the app and save it from the app under
`~/Documents`: build the document in the app, or write its source where
the app edits its format as text, then save it again from the app. Make
@@ -141,14 +175,57 @@ the first source that has the app:
otherwise starts on its own screen. For a starting file of the requester's,
read it here and rebuild it in the app with synthetic data in place of
personal data; REA cannot copy a file into the computer.
+ - An app that keeps its own data and saves after every change, such as
+ Osmo's contacts in `~/.local/share/osmo/contacts_records.xml`, has no
+ document: that state is the seed. Build it in the app and mark `seed`
+ once the app has written. To see whether it has, mark a throwaway
+ checkpoint, such as `probe-1`, and read its `changed_files`: up to ten
+ files that changed since the checkpoint before, those under
+ `state_paths` first, then those the app held open, caches last, with
+ `more_changed_files` counting the rest.
+ - The app's own compressed format works when it is gzip, such as
+ Gnumeric's `.gnumeric`, Dia's `.dia` or Inkscape's `.svgz`: REA reads
+ the document inside, and checks point into it as into the same
+ document uncompressed. A format REA reads as binary, such as one
+ compressed with xz, has no view: save in a format the app offers that
+ REA reads, then reopen the file once and check that a plain Ctrl+S
+ keeps that format.
4. Only once the seed is saved, `mark_build_checkpoint` with name `seed`.
5. Make the change the task asks for, as a user would, and save it.
6. Only once the change is saved, `mark_build_checkpoint` with name `end`.
7. `stop_build_capture`. Read `draft.files` (each file that changed, with
- its net change), `draft.task.checks` (suggested checks) and
- `limitations`. When `draft` is null, read `draft_refusal` first: a
- missing `seed` or `end` checkpoint means capturing again on a new build
- computer, while a failed draft command names its own cause to fix.
+ its net change and its `values`), `draft.task.checks` (suggested
+ checks), `draft.guards`, `draft.notes` and `limitations`. When `draft`
+ is null, read `draft_refusal` first: a missing `seed` or `end`
+ checkpoint means capturing again on a new build computer, while a failed
+ draft command names its own cause to fix. When `draft.files` is empty,
+ `draft.notes` says why:
+ - `no_change`: nothing was saved between `seed` and `end`. Capture again
+ on a new build computer, and save before marking `end`.
+ - `unowned_changes` with `no_document`: the app saved your document
+ where REA did not count it as the app's. Its `files` list what changed,
+ with their roles; the document is among them. Close the computer and
+ capture again on a new one with that path in `state_paths`; the export
+ refuses a check on a file outside the owned paths.
+ - `uncheckable_changes`: the app saved the change in dconf, as gedit does
+ its preferences, and no check reads dconf. Saving again does not help:
+ choose a change the app saves in a file, and capture again on a new
+ build computer.
+ - `unreadable_changes`: a dconf database changed but REA could not read
+ it; `error` says why. This is not evidence the app saved there: report
+ the error rather than changing the task.
+
+ A note with a `file` says why that file's suggestions are thin:
+ - `by_position`: only position names what changed; its `elements` lack
+ a naming attribute. Check an ancestor's text, or give the element a
+ name in the app, such as an id, and capture again.
+ - `view_state_only`: every change is view state, a renumbered id or a
+ time the app stamps, and its `pointers` are the checks kept. The edit
+ was likely not saved: capture again and save it before `end`.
+ - `long_text`: the element's text is too long to check whole; check a
+ phrase in it with `text_contains`.
+ - `cut_view` or `unreadable_view`: REA's view holds only part of the
+ file, or none a check can read, such as a binary format (step 3).
## Write the task
@@ -158,18 +235,37 @@ Write `{"name", "prompt", "checks"}` from the draft and what you did.
- `prompt`: what the requester wants, in their words and the app's terms,
naming the document as a user would and ending with saving it: "Change the
poster's headline in Documents/poster.svg to Summer Bake Sale and save
- it." Never say how the file stores it, and never give the checks.
+ it." Leave saving out for an app that saves on its own, after every change
+ or a few seconds after the last one, such as Osmo or Gnote: grading waits
+ until the app's files have held for 3 s, so the solving agent need not
+ wait for the save. A save more than 3 s after the agent stops is still
+ missed; name such an app in the report. Never say how the file stores it,
+ and never give the checks.
- Checks are file-based: content checks that prove the edit, never only that
a file exists or changed. Start from the suggestions; keep those that say
what the user did and drop what follows from how the app saves.
- Compare exact values, so a typo or a wrong number fails.
+- Read each file's `values` before writing checks: every value the change
+ added, changed and removed, with `before` and `after` (a scalar, or an
+ element as compact XML, cut at 600 characters) at the pointer a check
+ names it by, so copy the pointer and the `after` value into a check.
+ Only XML files, archives, JSON and key files have values: a text file's
+ or SQLite database's `change` already lists its lines and rows.
+ `refers_to` says what a number stands for, such as `category="44"`
+ naming ``; use it for the prompt's words.
+ Skip entries whose `note` is `by_position` (no check can use that
+ pointer) or noise (`view_state`, `renumbered`, `volatile`,
+ `timestamp`, `upkeep`). `values_cut` means a limit left some out.
- Select elements by what they hold, never by position: SVG text by id,
`/root/children/[name=g][@inkscape:label=Layer 1]/children/[name=text][@id=headline]/children/[name=tspan]/children/0`,
or an ODF paragraph by style,
`/entries/[name=content.xml]/view/root/children/[name=office:body]/children/[name=office:text]/children/[name=text:p][@text:style-name=Title]/children/0`.
A selector must match exactly one element, so give the target a unique id
- or style in the seed. The draft may suggest positional pointers; rewrite
- them. Only an element's text is read by position, as `children/0`.
+ or style in the seed. The draft's pointers already do this: a name-like
+ attribute such as Scribus's `ANNAME` before a coordinate, and two
+ attributes together where one is not enough, as a spreadsheet cell's
+ `[@Col=3][@Row=2]`. Only an element's text is read by position, as
+ `children/0`.
- When an app may split an element's text into runs, check it whole:
`text_equals` (or `text_contains` for a value inside a sentence) on the
element compares all its text, however it is split. Add
@@ -178,15 +274,43 @@ Write `{"name", "prompt", "checks"}` from the draft and what you did.
`{"kind": "text_contains", "file": "Documents/memo.abw", "pointer": "/root/children/[name=section]/children/[name=p][@xid=6]", "value": "at 3:30 PM"}`.
Point a text check at the paragraph or another element the edit keeps,
never at a run such as AbiWord's `[name=c]`: a reopened document splits
- the text into several runs. The draft lists only what changed; to name an
- unchanged element, such as another paragraph's `xid`, use the app's usual
- numbering and let the export confirm it, or read it in the exported
- bundle's `envs//seed/`.
+ the text into several runs. The draft lists only what changed; an
+ unchanged element is in `draft.guards` when it is a like sibling of a
+ changed one, and otherwise use the app's usual numbering, such as another
+ paragraph's `xid`, and let the export confirm it.
+- In an ODS spreadsheet no attribute names a row or a cell. Name the sheet
+ in the seed, and check its whole text: `text_equals` on
+ `/entries/[name=content.xml]/view/root/children/[name=office:body]/children/[name=office:spreadsheet]/children/[name=table:table][@table:name=Q1 Sales]`,
+ or a stretch of it with `text_contains`. Cell texts join with nothing
+ between, so a row reads `February23100`.
+- For a line added to free text, such as an item on a note's list, the
+ draft compares the whole text with `text_equals`. This rejects longer or
+ negated items, but can reject a correct edit with different ordering or
+ spacing. Report that limitation when the task permits such variations.
+- Some apps keep text in an attribute, such as Scribus's ``.
+ `text_equals` cannot join it, so check the attribute with `value_equals`:
+ `/root/children/[name=DOCUMENT]/children/[name=PAGEOBJECT][@ANNAME=Headline]/children/[name=StoryText]/children/[name=ITEXT]/attributes/CH`.
+ A partial edit the app restyles splits the text into several runs and
+ fails that check; list it in the report's limitations.
+- When the draft shows no values or no checks, a scratch export can expose
+ `envs//seed/` only after the task has at least one valid check: it must
+ pass on the captured end state and fail on the seed. Before the first
+ successful export, use `act_on_build_computer` to open the saved file in an
+ available GUI text editor or viewer and inspect its actual representation
+ without changing it. Use the visible saved content and the task's edit to
+ write a check; an export refusal can also name the end state's value, such
+ as `is "44", not "1"`. If the file cannot be inspected or expressed by a
+ supported check, report the limitation instead of guessing a check.
- Where easy, add one check that the rest is untouched, such as another
- text's value the edit must leave alone. It is the one unchanged value
- worth checking; it passes on the seed too, which is fine, since the export
- needs only one check to fail there.
-- Give `value_equals` a `tolerance` for a number the app writes as text.
+ text's value the edit must leave alone. `draft.guards` offers them at
+ their seed values, which the end holds too: a field on the nearest like
+ siblings, one the user sets, such as Planner's `work`, before a default
+ they all share; the text an addition kept; a text file's lines beside its
+ new one. Copy one or two into `checks`; they pass on the seed, which is
+ fine, since the export needs only one check to fail there.
+- The draft suggests a decimal the app writes as text, such as `146.83`,
+ as a number with `tolerance: 0.001`; keep the tolerance for any number
+ you check that way.
- When the draft shows an XML document's changes as `lines`, REA read it
as text, for example because its DOCTYPE declares entities. Selectors are
impossible there, and `line_count` with `contains` only approximates a
@@ -215,6 +339,17 @@ is no longer empty. Keep
`seed_files`, `scrubbed_files`, `unscrubbed_files` and `limitations` for the
report.
+The limitations name the folders the app wrote that the bundle neither owns
+nor seeds, such as `~/.config/MuseScore`, so the app starts without them:
+expect its first-run wizard, a tour or a "restore session?" question at every
+boot. If the witness shows one, capture again on a new build computer with
+those folders in `state_paths`, and dismiss the dialogs before `seed`. Lock
+files the app holds while it runs, such as LibreOffice's `.lock` and
+`.~lock.#`, are left out of the seed and listed after "Left out of the
+seed", so you need not close the document or delete them before `seed`. The
+limitations also name seeded WebKit or Chromium storage and caches; when the
+witness's boot is clean, list them in the report.
+
Keep the build computer open until the witness holds: a task fix exports
again from it.
@@ -231,12 +366,19 @@ checks. Give every `check_build_task` the task exactly as exported.
seed holds exactly one and the app takes a file. When the export's
`limitations` say it opens no document, open it from the prompt's path,
as a solving agent would. Take a `wait` screenshot first, since the
- desktop may not have drawn yet.
+ desktop may not have drawn yet. When the start fails because the
+ computer never got ready, and its error names lock files in the bundle's
+ seed, delete them from the bundle and start again.
2. `check_build_task` with `stage: "seed"` and `directory`, the absolute
path of a new or empty folder for the record. A check must fail.
Recording starts. Later stages take no `directory`.
3. Do the task from the prompt with `act_on_build_computer` and save.
`check_build_task` with `stage: "after_edit"`: every check must pass.
+ Each check waits until the app's files have held for 3 s, as the
+ bundle's verifier does, so an app that saves on its own needs no wait of
+ yours. A result with `settled: false` means the files were still
+ changing at the 15 s bound, and `read_cut: true` that a slow read was
+ still running there.
4. Make a near miss on top of it: a plausible wrong edit, such as a typo or
a wrong number, and save. `check_build_task` with `stage: "near_miss"`:
a check must fail.
@@ -286,7 +428,9 @@ Report:
`witness.json` and the recording;
- the seed's contents: `seed_files` and what each holds;
- `scrubbed_files` and `unscrubbed_files`; an unscrubbed file still names
- the host or user, so it needs review before sharing;
+ the host or user, so it needs review before sharing. `recompressed_files`
+ are gzip seeds whose headers were cleared, their content unchanged: give
+ their count;
- limitations: the capture's, the export's and anything you could not do;
- solver runs, if any: the model, the runs and the passes;
- the build time and the questions you asked.
diff --git a/.agents/skills/reverse-engineer-anything/references/environments.md b/.agents/skills/reverse-engineer-anything/references/environments.md
index 63dd4e78b6..521357983a 100644
--- a/.agents/skills/reverse-engineer-anything/references/environments.md
+++ b/.agents/skills/reverse-engineer-anything/references/environments.md
@@ -308,9 +308,20 @@ Run `rea describe-desktop-application-task CAPTURE --format json`. Read:
and last version, with its `role`, the `baseline` and `final` versions and
the net `change`. Text changes list each added and removed line once, and
`line_counts` gives lines found at both ends whose count changed.
+- `values` in each of those files: what the change added, changed and
+ removed, `before` and `after`, at the pointer a check names it by, with
+ `refers_to` naming the element a key such as `category="44"` stands for.
+ Skip entries noted `by_position` or as noise. Only XML files, archives,
+ JSON and key files have values: a text file's or database's change
+ already lists its lines and rows.
- `task`: a draft whose `checks` are suggestions from the net change of the
documents the user worked on, or of the app's own state when no document
changed.
+- `guards`: optional checks that the rest kept its seed values, such as a
+ like sibling's field or a text file's neighbouring lines; add one or two
+ to show the rest is untouched.
+- `notes`: why `files` is empty, or why a file's suggestions are thin, such
+ as `by_position` when no attribute names the changed element.
### Write the prompt
@@ -346,7 +357,9 @@ document saved as a zip archive, such as FreeCAD's `.FCStd`, reads as its
entries: point into an entry's view, and name array members by selector
rather than position, as
`/entries/[name=Document.xml]/view/root/children/[name=Object][@name=Hole]`,
-so the check survives an app version that writes them in another order. Give
+so the check survives an app version that writes them in another order. A
+gzip-compressed document, such as Gnumeric's `.gnumeric`, reads as the
+document inside: point into it as into the same document uncompressed. Give
`value_equals` a `tolerance` to compare a number the app writes as text, such
as `"5.0000000000000000"`, with a number. Check what the user wrote in a
document with `text_equals` on its paragraph element, not `value_equals` on a
diff --git a/bridge/agentenv/capture.py b/bridge/agentenv/capture.py
index 0162918f17..599bd2e7f0 100644
--- a/bridge/agentenv/capture.py
+++ b/bridge/agentenv/capture.py
@@ -1,7 +1,9 @@
"""Run REA inside an agentenv build computer for REA on the host: capture the app, draft its task and export it.
REA sends this file's source through the computer's ``computer-exec/v1`` route as
-``python3 -I -c ``; each command prints one JSON line and exits. ``start``
+``python3 -I -c ``. JSON that would exceed the bytes Linux
+passes as one string travels as several arguments, which this file joins before parsing. Each command prints
+one JSON line and exits. ``start``
leaves the capture running on its own: its stdin is a FIFO it holds open for reading and writing, so it never
sees the end of its input, and ``mark`` and ``stop`` write their requests there and wait for the capture's own
acknowledgement on its progress log. A small shell around the capture writes its exit status to a file, the
@@ -91,6 +93,13 @@ def _exit_code(capture: str):
return None
+def _ended(capture: str, code: int) -> dict:
+ """A capture that has exited: its status, the end of its progress log and its output's size, which REA reads
+ whole for the Evidence or failure it ended with, whichever step was waiting on it."""
+ return {"exited": code, "progress": _tail(os.path.join(capture, "progress.log")),
+ "output_bytes": os.path.getsize(os.path.join(capture, "capture.out"))}
+
+
def _event_line(path: str, offset: int, kinds: set, name):
"""The first progress line after ``offset`` whose event is of one of ``kinds``, for ``name`` when given."""
try:
@@ -119,17 +128,49 @@ def _wait(capture: str, offset: int, kinds: set, name, timeout: float) -> dict:
return {"line": line}
code = _exit_code(capture)
if code is not None:
- return {"exited": code, "progress": _tail(log), "output": _tail(os.path.join(capture, "capture.out"))}
+ return _ended(capture, code)
if time.monotonic() > deadline:
return {"timed_out": True, "progress": _tail(log)}
time.sleep(POLL_S)
+def _within(path: str, folder: str) -> bool:
+ return path == folder or path.startswith(folder + os.sep)
+
+
+def _state_paths(values: list, work: str):
+ """Each state path as an absolute path strictly under the home and outside REA's work folder, both as named
+ and with its links resolved, or the refusal of the first that is not. The capture resolves the links again
+ and retains what they lead to, so a link out of the home is refused here. Each passes on as named, so the
+ capture keeps what its spelling means, such as a hidden settings folder."""
+ home = os.path.normpath(os.path.expanduser("~"))
+ real_home, real_work = os.path.realpath(home), os.path.realpath(work)
+ spellings = {home, real_home}
+ paths = []
+ for value in values:
+ relative = value[2:] if value.startswith("~/") else value
+ path = os.path.normpath(os.path.join(home, relative))
+ if ".." in relative.split("/") or not any(path.startswith(spelling + os.sep) for spelling in spellings):
+ return None, f"The state path {value} is not under the computer's home, {home}."
+ # Every link on the way is followed, a dangling one too; a path that does not exist yet resolves
+ # through its nearest existing folder.
+ resolved = os.path.realpath(path)
+ if not resolved.startswith(real_home + os.sep):
+ return None, f"The state path {value} leads through a link to {resolved}, which is not under the computer's home, {home}."
+ if _within(path, work) or _within(resolved, real_work):
+ return None, f"The state path {value} is in REA's work folder {work}, which every capture leaves out."
+ paths.append(path)
+ return paths, None
+
+
def start(arguments: dict) -> int:
work, capture = os.path.abspath(arguments["work"]), os.path.abspath(arguments["capture"])
rea = shutil.which("rea")
if rea is None:
return _say({"error": "REA is not installed in this computer; start the computer from a request, which installs it."})
+ state_paths, refusal = _state_paths(arguments.get("state_paths", []), work)
+ if refusal is not None:
+ return _say({"refused": refusal})
if os.path.exists(capture):
return _say({"error": f"the capture folder {capture} is already used"})
apt = _apt(work, arguments.get("apt_timeout_s", 600))
@@ -143,6 +184,8 @@ def start(arguments: dict) -> int:
"--output", os.path.join(capture, "capture"), "--exclude", work, "--json"]
for value in arguments.get("arguments", []):
argv += ["--argument", value]
+ for path in state_paths:
+ argv += ["--state-path", path]
if arguments.get("package_file"):
argv += ["--package-file", arguments["package_file"]]
# Reading and writing, so the capture never reads the end of its input between requests.
@@ -175,11 +218,13 @@ def _request(capture: str, line: str) -> bool:
def mark(arguments: dict) -> int:
+ """Request a checkpoint and answer with the capture's acknowledgement line as it wrote it: the checkpoint's
+ counts and up to ten of the files that changed, home-relative, with a count of the rest."""
capture, name = os.path.abspath(arguments["capture"]), arguments["name"]
code = _exit_code(capture)
log = os.path.join(capture, "progress.log")
if code is not None:
- return _say({"exited": code, "progress": _tail(log)})
+ return _say(_ended(capture, code))
# A checkpoint already taken, whose answer was lost, is answered from the capture's own log.
taken = _event_line(log, 0, {"checkpoint"}, name)
if taken is not None:
@@ -270,4 +315,4 @@ def listing(arguments: dict) -> int:
COMMANDS = {"start": start, "mark": mark, "stop": stop, "run": run, "list": listing}
if __name__ == "__main__":
- sys.exit(COMMANDS[sys.argv[1]](json.loads(sys.argv[2])))
+ sys.exit(COMMANDS[sys.argv[1]](json.loads("".join(sys.argv[2:]))))
diff --git a/docs/desktop-application-capture.md b/docs/desktop-application-capture.md
index 6a72342ea6..7b8f8ef14c 100644
--- a/docs/desktop-application-capture.md
+++ b/docs/desktop-application-capture.md
@@ -94,12 +94,20 @@ Progress lines on stderr are JSON, `{"rea_progress": {"phase":
"desktop_capture", "message": ...}}`. Those a driver waits on also carry an
`event`: `{"kind": "window_open"}` once the window opens; `{"kind":
"checkpoint", "name", "trigger", "at_ms", "entries", "changes",
-"retained_files", "retained_bytes"}` once a checkpoint has been taken, the
-baseline, `stop` and `exit` included, with the paths walked, the paths
-changed since the checkpoint before and the files retained; and `{"kind":
-"checkpoint_refused", "name", "reason"}` for a requested name it refused. A
-driver that sends `checkpoint ` waits for the event naming it rather
-than for a fixed time.
+"retained_files", "retained_bytes", "changed_files", "more_changed_files"}`
+once a checkpoint has been taken, the baseline, `stop` and `exit` included,
+with the paths walked, the paths changed since the checkpoint before, the
+files retained, and up to ten of the files (not folders) that changed,
+home-relative: those under a `--state-path` or absolute `--argument` first,
+then those the application held open, then those in its state directories,
+then the rest, and last caches (a folder whose name ends in `cache`, such as
+`~/.cache`) and WebKit or Chromium storage, each by path, with a count of the
+rest. An application can rewrite dozens of cache files on a save, and KDE's
+holds its service cache open, so the saved document would otherwise fall past
+the ten;
+and `{"kind": "checkpoint_refused", "name", "reason"}` for a requested name it
+refused. A driver that sends `checkpoint ` waits for the event naming it
+rather than for a fixed time.
When the session ends the capture sends SIGTERM to the launched process, its
process group and every process that carries the run token, waits up to five
@@ -310,7 +318,9 @@ the whole name is compared, so `~/.config/GIMPshop` does not. Dotfiles in the
home named after the application (`.rc`, or
`.` followed by `-`, `.` or `_`, for a name of three or more characters)
that exist at the start join them, since an application may only read such a
-file during the capture. A `flatpak run` launch is named after its application
+file during the capture. So does KDE's settings file in the config folder,
+`rc` such as Tellico's `~/.config/tellicorc`, for each such name of three
+or more characters, whether or not it exists yet. A `flatpak run` launch is named after its application
id rather than the `flatpak` launcher, and adds `~/.var/app/`.
**Session.** When it started, how long it ran and why it ended; the launched
@@ -318,8 +328,9 @@ process's exit status; every process seen in the application's tree; every
client window with its class pair, titles, owner pid and type, marked when it
belongs to the application; when the first application window appeared; the
most common class pair of the application's windows, for matching its window
-later; files the application's processes held open under the walked roots; the first
-8 KiB of the launched process's stderr; and whether cleanup was verified. Each
+later; files the application's processes held open under the walked roots; the last
+8 KiB of the launched process's stderr, where a failing app's last words are,
+with `stderr_truncated` when earlier output was dropped; and whether cleanup was verified. Each
process records whether the capture started it: the launched process, a
process carrying the capture's run token, or a descendant of one, even when
its environment cannot be read, as for a zombie or a sandboxed renderer.
@@ -345,7 +356,9 @@ directory, which are named in the limitations instead; a selected path that is
not a directory brings the `-wal` and `-journal` paths beside it, so a database
selected on its own is captured with its logs), the state directories
as walked (their physical spelling, which differs from the inventory's when one
-is a symlink), the paths every walk left out (the output folder and
+is a symlink), the selected paths (each `--state-path` and content-root
+`--argument` as named, absolute, beside the physical path walked, so a
+symlinked settings folder keeps what its name means), the paths every walk left out (the output folder and
`--exclude` values), and the checkpoints in order: `baseline` before
launch, each requested checkpoint, `stop` or `time-limit`, and `exit` after the
application has gone. Each checkpoint records how many paths its walk saw,
@@ -374,7 +387,8 @@ unreadable SQLite sidecar from an absent one. Captures made before this field
was added must be made again before state inspection, task drafting or export.
Directories the application held files open in, but whose starting content was
not retained, are listed so a second capture can pass them with
-`--state-path`.
+`--state-path`, or `state_paths` on a build computer's `start_build_capture`,
+and a limitation names them, spelled from `~/` under the home.
**Limitations.** What the capture could not establish, such as a truncated
walk or a missing window.
diff --git a/docs/desktop-application-export.md b/docs/desktop-application-export.md
index c3b15bc6b8..c3487dfa43 100644
--- a/docs/desktop-application-export.md
+++ b/docs/desktop-application-export.md
@@ -33,8 +33,10 @@ The command returns Evidence with operation `export_desktop_application` and
predicate type `rea.desktop-application-export`, listing the environment name,
the seed and end `checkpoints`, owned paths, seed files, the seeds it
rewrote to remove the capture host's name and user (`scrubbed_files`), the
-bundle files that still hold them (`unscrubbed_files`), and written files,
-the last three by their path in the bundle. A failed export, cancellation
+gzip seeds it compressed again only to clear their headers
+(`recompressed_files`), the bundle files that still hold them
+(`unscrubbed_files`), and written files, the last four by their path in the
+bundle. A failed export, cancellation
included, removes the bundle it had not yet published, so no partial copy of the
user's data remains. If the bundle was published but its output handles could
not be closed, the command reports a cleanup error that carries the export
@@ -154,9 +156,11 @@ The class declares:
Files the capture recorded under a symlinked state directory's physical
spelling are mapped back to the innermost such directory's home spelling,
unless that spelling is outside the captured home, as when the home itself is
- a symbolic link. When two files map to one home path, the export uses the one
- present before the capture, else the first by path, and a limitation names
- the other.
+ a symbolic link. When a selected link names an application state directory,
+ the bundle seeds the application's state path and restores the selected
+ alias at boot. Selections nested under that alias are seeded under the same
+ state path. An existing incompatible alias stops setup without replacing it. When two files map to one home path, the export uses the one present
+ before the capture, else the first by path, and a limitation names the other.
- Data hooks, because declaring any hook switches off agentenv's default data
plane: the class seeds the home once per boot and keeps it as the initial
state with `keep_initial`, `data/add` and `data/reset` call agentenv's `add`
@@ -169,10 +173,43 @@ retain is not seeded: one in the state view without retained content at that
checkpoint, or one the capture omitted (for example too large or past the
content budget). The result lists them in `unseeded_files` and the README
names them. When the application held files open in folders whose starting
-content the capture did not retain, a limitation names those folders. The
-desktop Trash, `~/.local/share/Trash`, is never the application's, so a file
-the user moved there is neither owned nor seeded, even when a window title
-names it; a limitation lists the files the Trash held at the seed.
+content the capture did not retain, a limitation names those folders, spelled
+from `~/` under the home: a file there is seeded only when the capture
+retained it, as it does one written during the capture, and the limitation
+names the seed files there or says there are none. The desktop Trash,
+`~/.local/share/Trash`, is never the application's, so a file the user moved
+there is neither owned nor seeded, even when a window title names it; a
+limitation lists the files the Trash held at the seed.
+
+Files that changed during the capture outside the owned paths are neither
+owned nor seeded, so the app starts without what it saved there: MuseScore
+keeps its settings in `~/.config/MuseScore`, which its name rules miss, and
+showed its first-run wizard on every boot. A limitation and the README name
+those folders, leaving out desktop services' files, each collapsed to its
+folder in `~/.config`, `~/.local/share`, `~/.local/state` or `~/.local`, the
+hidden folder of the home it lies in, or else its own folder, up to eight and
+a count of the rest. Capture again naming them with `--state-path`, or
+`state_paths` on a build computer, to own them.
+
+Lock files the app holds while it runs are left out of the seed: a copied one
+makes the app find its profile or document in use, and a LibreOffice Calc
+bundle whose seed held its profile's `.lock` showed no window at boot. They
+are `lockfile`, `lock`, `.lock`, `.parentlock`, `parent.lock` and
+`SingletonLock` anywhere, LibreOffice's document lock `.~lock.#`, and
+`*.lock` or `*.lck` directly in one of the application's state directories.
+Elsewhere such a file may be a project's own, such as `Cargo.lock` or
+`yarn.lock`, which pin dependencies, and is seeded, even under a folder whose
+name starts with a dot. A file the state view makes a document, or one named
+itself by `--state-path` or an absolute `--argument`, is never left out,
+whatever its name. A limitation and the README list them, up to
+eight and a count, after "Left out of the seed". They stay under the owned
+paths, so `data/get` returns the lock the running app holds, and `data/reset`
+restores the seed without them. Other seed files the app keeps for its own
+running are seeded as captured and named, collapsed to the folder that holds
+them: WebKit or Chromium storage (such as `hsts-storage.sqlite`, `storage/`,
+`mediakeys/`, `Local Storage/`, `IndexedDB/`, `Cookies`), which may hold the
+app's own data, and caches (a folder whose name ends in `cache`); a stale one
+can change how the app starts.
The application's dconf keys at the seed, if any, are written once when the
app first starts, also for an application that owns no files, and the class adds
@@ -182,7 +219,11 @@ existing user database through a `file-db:` profile, then runs `dconf compile`
on that dump and the captured keys, in that name order so the captured keys
win, and replaces `~/.config/dconf/user`. If that fails, the server prints a
one-line warning and the app starts without the captured keys. `data/reset` does
-not restore them.
+not restore them. No file of the bundle holds them, so `data/get` does not
+return them and no task check reads them; a draft whose only change between
+its checkpoints is to the application's dconf keys says so with
+`uncheckable_changes`, and one whose dconf database changed but could not be
+read says so with `unreadable_changes`.
## The seed checkpoint
@@ -252,6 +293,26 @@ export removes them from what the bundle ships:
boot, as it did the captured home before. The replacements read as the
computer's own: its user is `user`, with its home in `/home/user`, and
`localhost` is the name every host has for itself.
+- A gzip-compressed seed, such as a Gnumeric workbook, is decompressed, up to
+ 256 MiB, scrubbed as the format the state view read its document as, and
+ compressed again whether or not its document changed, so no file name,
+ comment or time its header held ships: with no file name, comment or extra
+ field, modification time 0 and the operating system unknown, so the same
+ document gives the same bytes on every host. It is deflated at the fastest
+ level, or stored when deflating would make it expand more than the 100 times
+ a view reads, so a seed that read still reads. It is listed in
+ `scrubbed_files` when its document changed, and otherwise in
+ `recompressed_files`, which the README counts rather than lists, so the
+ example and template files an app copies on first run do not bury the
+ document scrubbed; one already in that form ships as captured. The class copies it at
+ boot as it is, so the home it holds stays `/home/user`. One that does not
+ decompress ships as captured, its bytes searched, and one the state view did
+ not read is only searched. One whose document needs no edit but a member of
+ which has an extra field, where BGZF (`.vcf.gz`, `.bam`) and dictzip
+ (`.dict.dz`) files index their blocks, also ships as captured, its bytes
+ searched, so its blocks stay where its index says; one whose document
+ changed is still compressed again as one member, losing them, as the
+ export's limitation says.
- Binary files, SQLite databases among them, and text files over 256 MiB are
copied as captured. Then every file the bundle ships, the class module with
its dconf keys and the README included, is searched by a looser rule than
@@ -318,6 +379,39 @@ declared vendor repository installs the package already; when the URL is not
written as `http://` or `https://` or holds a user name or password; and when
the file is built for an architecture other than `amd64`, `arm64` or `all`.
+## Grading once the app has saved
+
+A task's verifier, `artifacts/-check/verify.py`, grades the app's files
+only once they have settled, because an app can save on its own timer after the
+last change. Gnote saves a note 4 s after the last key, so a read taken as
+soon as the agent stops finds the text on screen but not yet in the file. The
+verifier reads the app's `data/get` every 0.5 s. It compares each file's bytes
+between reads by their SHA-256, as `data/get` returns them, and grades once
+every file has held for 3 s. The 3 s count from when a read returned the
+current bytes, not when it began, so a slow read that brings a change late
+does not count its own run as quiet. 15 s after the first read began, the
+bound, the verifier grades the last completed read: no read starts at or
+after the bound, and a read still running then is cut. Each result records
+whether the files settled in `settled`, and whether the bound cut a read in
+`read_cut`.
+
+The window has a cost and a limit:
+
+- An app whose files do not change during the check, such as one that saved
+ at once, is graded 3 s after the first read returns.
+- A save that lands more than 3 s after the first read returns is still
+ missed. An app that saves 4 s after the last key is caught only when the
+ agent stops at least a second after that key.
+- Files the app rewrites more often than every 3 s, such as a log, never
+ settle, so every check waits until the bound.
+- The bound counts the reads too, so a slow `data/get` ends the check at
+ 15 s. Only the first read, which every grade needs, runs to its own
+ timeout.
+
+The bound and window are `SETTLE_BOUND_S` and `SETTLE_WINDOW_S` in the
+verifier. `check_build_task` waits the same way before it grades each
+witness stage; see [witnessing](desktop-build-witness.md#witness).
+
## Typed views
The views follow the [state view](desktop-application-state.md)'s formats and
@@ -332,7 +426,10 @@ as their entries, read with Python's `zipfile` from the file in place, with
REA's entry, size, compression ratio and archive limits and its wording for an
entry it does not read. The module reads the central directory itself, as REA
does, and shows `zipfile` only the records it lists, reading each entry in
-bounded chunks no further than a byte past its declared size. A JSON file
+bounded chunks no further than a byte past its declared size. A gzip file is
+the view of the document inside it, with `"compression": "gzip"` beside its
+format, decompressed with Python's `zlib` within the same entry size and ratio
+limits, its members joined as REA joins them. A JSON file
holding a number JavaScript cannot read exactly, such as an integer above 2^53,
or whose retained value nests more than 500 levels deep, gets an `error` instead of a view, as in
REA. Duplicate keys keep their last value before the depth check. Text that is not one JSON
diff --git a/docs/desktop-application-state.md b/docs/desktop-application-state.md
index 5bc2cd6fec..c586838fc1 100644
--- a/docs/desktop-application-state.md
+++ b/docs/desktop-application-state.md
@@ -40,7 +40,16 @@ The role follows from the reasons, in this order: `shared_desktop` for a file
in the desktop Trash, which the user deleted, whatever else names it;
`application_state` for application keys in dconf, or a state directory outside shared desktop
locations; `document` for a file named in a window title or selected by the
-caller, even if the application holds it open; `application_state` for another
+caller, even if the application holds it open, except that a selected file
+under a hidden folder of the home, such as `~/.config/MuseScore`, a dotfile
+there, or a lock an application holds, such as LibreOffice's `.~lock.#`
+beside a document, is `application_state`, judged by the path as the caller
+named it, so a file reached through `~/.config/MuseScore` stays the application's
+when that folder is a link to `~/settings/MuseScore`; the innermost selected
+path holding a file decides, so a score selected itself inside the link's
+target keeps its own name and is a document (a capture written before it
+recorded selected paths is judged by their physical spelling);
+`application_state` for another
open file outside shared locations; `shared_desktop` for other files in shared
locations; `unrelated` otherwise. A window title names a file only when no
other file name character adjoins the name, so `day.txt` does not match
@@ -75,6 +84,9 @@ whether the file was present, the SHA-256 of its retained bytes, and a view:
| `text` | Other UTF-8 text | Lines, up to 5,000 |
| `binary` | NUL bytes or invalid UTF-8 | None |
+A gzip file is read as the document inside it; see
+[compressed documents](#compressed-documents).
+
A JSON value holding a number JavaScript would round, such as an integer above
2^53, is not read rather than shown rounded.
@@ -193,6 +205,37 @@ so an archive of hundreds of thousands of entries costs no more to read
than one of a thousand. An archive whose central directory cannot be read has a
`view_error` of its own.
+## Compressed documents
+
+Gnumeric's `.gnumeric`, Dia's `.dia`, Xournal++'s `.xopp`, GnuCash's default
+book and Inkscape's `.svgz` are XML compressed with gzip. A file that starts
+with gzip's magic bytes, `1f 8b 08`, whatever its name, is read as the
+document inside it: its members joined, then given a format and view by the
+rules above under the file's own name, so a pointer or check into it is
+written exactly as for the same document uncompressed, and the task draft is
+the same. The version records the compression beside the format:
+
+```json
+{
+ "format": "xml",
+ "compression": "gzip",
+ "view": {
+ "root": { "name": "gnm:Workbook", "attributes": {}, "children": [] },
+ "truncated": false
+ },
+ "view_error": null
+}
+```
+
+The document is read only when it is at most 16 MiB and at most 100 times the
+file's size, an archive entry's limits, so decompressing stops there; past
+them, or when the stream is damaged or cut short, the version has `format`
+null and a `view_error`, and a check into it is undecided. A SQLite or dconf
+database inside keeps its format and has a `view_error`, an archive inside is
+read as an archive, and another gzip stream inside is binary. A gzip file
+inside an archive reads as binary. Other compression, such as xz, Zstandard
+or bzip2, is not read: such a file is binary.
+
## Kinds of change
Between consecutive versions a change is `created`, `deleted`, or a
diff --git a/docs/desktop-application-tasks.md b/docs/desktop-application-tasks.md
index bc01fa5118..7e644ebf04 100644
--- a/docs/desktop-application-tasks.md
+++ b/docs/desktop-application-tasks.md
@@ -42,9 +42,31 @@ operation `describe_desktop_application_task` and predicate type
database is compared by its rows, since its hash leaves out the `-wal`
file. Text changes list each line
once and add `line_counts`, the lines found at both ends whose number of
- occurrences changed. Views appear only for the files the suggestions cover,
- with arrays past 200 items and strings past 2,000 characters cut
+ occurrences changed. An XML file's change names its members by the
+ [selectors](#pointers-and-selectors) checks use, as an archive's already
+ does, and by position only where nothing names a member or where only the
+ members' order changed. Views appear only for the files the suggestions
+ cover, with arrays past 200 items and strings past 2,000 characters cut
(`view_cut`).
+- `values`, in each of those files: the values its change added, changed and
+ removed, each with the `pointer` a check names it by, its `kind` (`added`,
+ `changed` or `removed`), and `before` and `after`: the scalar, an element as
+ compact XML with the text the XML reader kept, spaces between runs included,
+ or a JSON value as compact JSON, cut at 600 characters, and null at the end
+ that lacks it. A file created or deleted lists what its one end holds,
+ `added` or `removed`: an archive's entries, an XML file's root element, a
+ JSON file's members or a key file's keys, each whole once. A value only
+ position reaches has `note: "by_position"`, and its pointer cannot go into a
+ check; noise an app writes as it saves has `view_state`, `renumbered`,
+ `volatile`, `timestamp` or `upkeep` (see `task`). `refers_to` names up to
+ three other elements a value equals the `key` or `id` of, with the
+ `attribute` holding it, as HomeBank's `category="44"` names
+ ``; where several elements share that key,
+ the one whose tag shares the attribute's stem, as `payee` and ``. What a check can
+ name comes first, then what only position reaches, then noise. A file lists
+ at most 20 values and 8 KB, the draft 24 KB, and `values_cut` says when a
+ limit left some out. Values cover XML, archives, JSON and key files; a text
+ or SQLite file's change and suggestions already carry its lines and rows.
- `task`: a draft with the name `change`, a placeholder prompt the export
refuses, and suggested checks derived from the net change of the files with
role `document`, or of the app's own state files when no document changed.
@@ -56,21 +78,151 @@ operation `describe_desktop_application_task` and predicate type
and the JSON, key file and XML views of a zip archive's entries, gain checks
by [selectors](#pointers-and-selectors) and never through an element by
position: in XML, first [`text_equals`](#whole-text) for each element whose
- text changed, then an element added whole that it exists, one removed whole
- that it is absent, then values changed, added and removed, an entry or a
- JSON value added whole through its members, and an empty array or object as
- it is. The text check names the paragraph rather than a run inside it: it
- goes up from an element sitting in its parent's text, and from one named by
- its tag or position alone to the nearest ancestor named by an attribute, as
- AbiWord's `p` by its `xid`; the text nodes and runs inside it get no other
- check, but an element holding no text that is added or removed there, such
- as an image or a line break, keeps its own, since the text check never
- reads it. Whitespace an app lays out between elements is not checked, and
- values that look like a time the app stamps, in ISO 8601 or as C's
- `asctime` writes it (`Thu Nov 5 09:20:31 2026`), are left out.
+ text changed, then elements added or removed whole, then values changed,
+ added and removed, an entry or a JSON value added whole through its
+ members, and an empty array or object as it is.
+ - A selector names a member by its tag, then by its identifiers (`name`,
+ `id`, `xid`, `xml:id`), then by one other attribute, a name-like key such
+ as Scribus's `ANNAME` or `inkscape:label`, a worded value or an integer of
+ up to five digits first, then by two such attributes together, as
+ Gnumeric's cells by `[@Col=3][@Row=2]`, and last by a coordinate, a long
+ number or a token, such as `XPOS` or `ItemID`, or an attribute naming
+ view state, such as which layer is `selected`. A pair counts every
+ member holding both its values, whichever attributes that member ranks
+ first. An element with identifiers is named by those alone. The selector
+ must name the member alone at both ends, with the same values. An
+ element whose one identifier changed, at its place among its tag's
+ members, with every other attribute and its children alike, is that
+ element renamed, as a MyPaint layer renamed from `Untitled` to `Sun`: it
+ is named by its first other attribute naming it alone at both ends, as
+ `[@src=data/layer-00.png]`, so the rename reads as its `name` changed,
+ and the old name gets a check that it names nothing, unless its change
+ is noise, as an id the app renumbered.
+ An identifier the app numbers by position names nothing: where the
+ members of a tag hold it as 1..N (or 0..N-1) in document order at both
+ ends, members were inserted or removed, and every member another
+ attribute names alone at both ends kept its number or moved it the way
+ the count did, one at least, as Planner's task `id` after a task is
+ inserted, members are paired by their other attributes, so an insertion
+ reads as one task added. A value two members swap, or an id appended at
+ the end, as Tellico's next ``, leaves the ids identity.
+ - The text check names the paragraph rather than a run inside it: it goes up
+ from an element sitting in its parent's text, and from one named by its
+ tag or position alone to the nearest ancestor named by an attribute, as
+ AbiWord's `p` by its `xid`; where no attribute names one, to the nearest
+ ancestor within three levels its tag names with no step by position, as
+ MuseScore's `VBox` around untitled `` elements; and where only
+ position reaches the element, to the ancestor above the outermost step by
+ position, by an attribute of its own that names it alone where one does,
+ as an ODS sheet's `table:table[@table:name=Q1 Sales]` above its rows. The
+ text nodes and runs inside it get no other check, but an element holding
+ no text that is added or removed there, such as an image or a line break,
+ keeps its own, since the text check never reads it. A decimal that is an
+ element's only text, such as a spreadsheet cell's, is checked by its
+ number instead. Free text that gains a line, such as a shopping list,
+ uses `text_equals` on the whole text: the current XML checks cannot
+ compare an unstructured line as a whole item. This can reject a correct
+ edit that puts the item elsewhere or changes spacing. A record added
+ or removed whole, an element holding only elements, its fields, such as
+ Tellico's `` or a record of repeated `` elements, never
+ makes the list holding records of its kind a check, whose text would hold
+ the records the edit left alone, except a record removed whole that only
+ position names, whose absence only the list's text shows. Text made of
+ runs is no record: an element whose tag names a paragraph or a piece of
+ text (`p`, Writer's `text:p` and `text:h`, `h1` to `h6`, `li`, `para`,
+ SVG's `text` and `tspan`, a `span` or an `a`), or whose runs split a
+ sentence at a space, as `Meet ` and a linked `Monday`. Nor does an
+ element holding a date the app stamps in parts take a text check.
+ - An element added to a list, where another element of its tag is at either
+ end, gets a `value_equals` for each of its attributes, best ranked first,
+ at most six, leaving out its selector's attributes, identifiers, times,
+ volatile values and view state, as HomeBank's new `` gets its
+ `amount`, `category` and `payee`. An element whose other attributes, two
+ or more, a like sibling holds every one of, as a new Planner task copies
+ its neighbour's defaults, and any other element added whole, get a check
+ that it exists. A record added to a list that a selector names, as
+ Tellico's ``, gets a `text_equals` for each field, at most
+ five, each by its selector among the record's fields, as `[name=title]`,
+ leaving out a field repeating the record's identifier, a date the app
+ stamps in parts, and blank text or times. A record only position names,
+ such as Osmo's `` of fields, also gets a `text_contains` for each
+ new field of three or more characters, at most five, at the nearest
+ ancestor a check can name, each absent from that ancestor's text at the
+ seed.
+ - A decimal written as text, such as `146.83`, is suggested as a number
+ with `tolerance: 0.001`, unless the seed's value is that close too.
+ - Noise leaves the suggestions when the file's change holds anything
+ else, a change only position reaches included: view state, which a tag
+ or attribute on the way names (the cursor, a selection, scrolling, zoom,
+ the viewport, windows, splitters, Inkscape's `namedview`, view and
+ configuration settings, the printer, a save's history, edit time,
+ editing duration and cycles, last-saved time, the generator and
+ document statistics), or the value of a name-like attribute or `key` on
+ the way, when it is one token of several words, as LibreOffice's
+ `config:config-item[@config:name=CursorPositionX]` inside
+ `[@config:name=ooo:view-settings]`; an attribute ending in `ID` or `Id`,
+ or an `rsid`, that the app renumbered, with everything else in its
+ element, as Gnumeric empties a formula cell whose shared expression it
+ renumbers, and an identifier the app numbers by position; volatile
+ values, UUIDs and epoch times, and durations and counts under a
+ document's metadata; times the app stamps, also as a date written in
+ parts whose tag, or one it sits in, names when the app made, changed or
+ saved something, as Tellico's `` of ``, `` and
+ `` (`cdate`, `mdate`, `created`, `modified`, `saved`, `updated`),
+ which no check ever reads, while a date a user enters in parts, as a
+ birthday, is an edit like any other; and `upkeep`, the
+ `loext:tab-stop-distance` attributes LibreOffice fills in `styles.xml`
+ while another archive entry holds an edit. However much noise
+ comes first, it never takes a check's place. When the change holds
+ nothing else, the checks stay and a note flags them: up to three in a
+ document, and as many as fit in the app's own state, where a view
+ setting may be the task.
+ - Whitespace an app lays out between elements is not checked, and values
+ that look like a time the app stamps, in ISO 8601, as C's `asctime`
+ writes it (`Thu Nov 5 09:20:31 2026`) or as a time of day to the
+ second (`05:16:21.675302540`), are left out; noise never makes its
+ ancestor's whole text a check.
+- `guards`: checks an author may add to show the edit left the rest alone,
+ kept out of `task.checks`, with ids `guard-1` and on: for each suggested
+ `value_equals` or `text_equals` on a member of a list, a field on up to
+ two like siblings, nearest first, at its seed value, which the end holds
+ too: the check's own field where like siblings hold different values in
+ it, and otherwise one that varies among them, as Planner's `work` rather
+ than the `type="normal"` every task shares; never through a sibling only
+ position or a renumbered id pairs. A text file gets a `lines_present` of the
+ unchanged list items beside its first new line with the same marker, as
+ Zim's other `[ ]` action items, or else its nearest lines with words in
+ them. At most four a file. They pass on the seed, which the export allows,
+ since one check failing there is enough.
+- `notes`: why `files` is empty, or why a file's suggestions are thin. Each
+ note has a `code` and a `message`. For a draft with no files: `no_change`
+ when no file under the home changed between the two checkpoints and nor
+ did the application's dconf keys; `unowned_changes` when files changed but
+ none under the owned paths, with up to five of them in `files`, each with
+ its home path and role (desktop services' files last), and how many more
+ in `more`; `uncheckable_changes` when the application's keys in a dconf
+ database changed, with the database in `files`: no check reads dconf, and
+ the export only writes the keys as they were at the seed checkpoint, so
+ make the task's change one the app saves in a file; `unreadable_changes`
+ when a dconf database changed but REA could not read it, with the read
+ failure in `error` (install `dconf-cli` or set `REA_DCONF_COMMAND` when it
+ is missing), since whose keys changed is then unknown; and `no_document`
+ when no file has the document role, as when no window title shows the
+ file's name. A capture again with `--state-path` naming the file, or
+ `state_paths` on a build computer's `start_build_capture`, makes it the
+ app's. For a drafted file, named in `file`: `by_position` when
+ changes reach their element only by position, the tags no attribute names
+ alone in `elements`, or when only the elements' order changed;
+ `view_state_only` when every change is noise, the checks kept in
+ `pointers`; `long_text` when an element's text changed past the 2,000
+ characters a text check compares, the elements in `pointers`; `cut_view`
+ when the views hold only part of the file; and `unreadable_view` when no
+ check can read it, as a document REA reads as binary.
The suggestions are a starting point. Keep the ones that say what the user
-did; drop the ones that only follow from how the app saves its file. A LibreCAD
+did; drop the ones that only follow from how the app saves its file. Read
+`values` for what changed and its pointers, and `refers_to` for what a number
+stands for, before writing the prompt and any check of your own. A LibreCAD
capture that added four holes to a drawing rewrote it from DXF R12 to DXF 2007,
so most of its lines changed; the draft suggests counts of `SECTION`, `ENDSEC`
and `CIRCLE`, and only `CIRCLE` going from 5 to 9 is the user's change.
@@ -129,7 +281,9 @@ The rules both evaluators follow:
`{"groups": {"GROUP": {"KEY": "VALUE"}}}`, so a setting is
`/groups/GROUP/KEY`; XML as `{"root": {"name", "attributes", "children"}}`;
SQLite as `{"tables": {"TABLE": {"columns", "rows"}}}`; a zip archive as
- `{"entries": [{"name", "format", "view", …}]}`.
+ `{"entries": [{"name", "format", "view", …}]}`; a gzip file, such as a
+ Gnumeric `.gnumeric`, as the document inside it, so its pointers are those
+ of the same document uncompressed.
- With `tolerance`, a non-negative number, `value` must be a number, and the
value found passes when it is a number, or a string that is a decimal
numeral such as `"5.0000000000000000"`, at most `tolerance` from `value`.
@@ -322,7 +476,10 @@ The bundle then holds, besides the app and `tasks/smoke.json`:
The verifier finds the app among the computer's child environments, calls its
`data/get` under the run's grant of the computer's control token, and reads
-the `state` data part. It raises when the app cannot be reached, so the run
+the `state` data part. It reads again until the app's files settle, as
+[grading once the app has saved](desktop-application-export.md#grading-once-the-app-has-saved)
+describes, and each result's `settled` says whether they did, and `read_cut`
+whether the bound cut a read still running. It raises when the app cannot be reached, so the run
fails rather than scoring the agent. The README lists the task, its prompt, its
checks and the recipe below.
diff --git a/docs/desktop-build-computer.md b/docs/desktop-build-computer.md
index 979b92651f..ebb94b3ec4 100644
--- a/docs/desktop-build-computer.md
+++ b/docs/desktop-build-computer.md
@@ -45,9 +45,11 @@ From a request, REA renders a bundle in a private temporary folder: a computer
app on the `ubuntu-24.04` base with the `reference` driver that installs the
app, opens it on demand with `gtk-launch ` from the computer
user's home, so its open and save dialogs start there rather than in the
-framework's folder for the app, and keeps no seed and no data hooks. The app is not open when the computer starts: agentenv lists it
-in the desktop's Applications menu under Other by the environment's name, such
-as `mousepad`, and that entry launches it through the app's server.
+framework's folder for the app, and keeps no seed and no data hooks. The app is not open when the computer starts:
+`start_build_capture` opens it under REA's capture. When no capture is under
+way, open it from the desktop's Applications menu, where agentenv lists it
+under Other by the environment's name, such as `mousepad`; that entry launches
+it through the app's server.
When the app is Electron, set `electron` to true. The install rewrites that
desktop entry's command with the same policy an export uses: `--no-sandbox`
@@ -96,6 +98,10 @@ in the folder and its files stay as they were shared.
REA runs the deploy with `run_bundle(keep=True, emulate=False)` on the local
sandbox provider, so only the host's own architecture is built. The first start
of a bundle builds its image and can take minutes; later starts reuse it.
+agentenv makes a start wait while another agentenv run on this host writes its
+bundle's ids, so starts at the same time wait for each other, several minutes
+when a few run together; the progress line then says it is waiting for another
+agent-env run to finish writing this bundle's ids.
While it waits, REA sends an MCP progress notification every five seconds with
agentenv's latest line, so a client that resets its request timeout on
progress keeps waiting. REA then waits until the computer's readiness checks
diff --git a/docs/desktop-build-witness.md b/docs/desktop-build-witness.md
index 0d064b7429..7f7f159714 100644
--- a/docs/desktop-build-witness.md
+++ b/docs/desktop-build-witness.md
@@ -10,7 +10,7 @@ grades the witness with the same checks the exported verifier runs.
| Tool | What it does |
| -------------------------- | ------------------------------------------------------------------------------------------- |
| `start_build_capture` | Downloads the package lists as the computer's user and launches the app under REA's capture |
-| `mark_build_checkpoint` | Takes a named checkpoint and returns the capture's acknowledgement with its counts |
+| `mark_build_checkpoint` | Takes a named checkpoint and returns the capture's counts and the files that changed |
| `stop_build_capture` | Stops the capture and returns its checkpoints and the task draft between `seed` and `end` |
| `export_build_environment` | Exports the bundle with the agent's task inside the computer and copies it to a folder here |
| `check_build_task` | Grades a task against the app's live state; with a `stage`, files the check in the witness |
@@ -41,6 +41,34 @@ See [desktop builds](https://github.com/morluto/rea/blob/main/.agents/skills/rev
from a `.deb` file, the build layer keeps that file at
`/opt/rea/package.deb` and the capture records it with `--package-file`. Do not open the app from
the menu first: the capture refuses an app whose window is already open.
+ When the capture ends before the window opens, REA reads its output. An app
+ that exits at launch, such as one missing a runtime, is reported with its
+ program, exit code, run time and the last lines of its stderr, at most 8
+ lines and 800 characters of the last 8 KiB the capture keeps, and
+ `details.diagnostics` keeps the capture's record of the session. A capture
+ that failed, such as one refusing to start, gives its own message and
+ remediation, with the whole failure in `details.diagnostics.command_failure`
+ and, when its cleanup is incomplete, the run tokens and paths it left in
+ `details.resources`. A checkpoint or stop that finds the capture ended
+ reads its output the same way.
+
+ `state_paths` names files or folders in the computer's home that hold the
+ app's state, relative to the home, after `~/` or absolute under it, at
+ most 16; each becomes a `--state-path` of the capture, so it is retained
+ from the start and its files count as the app's even before they exist.
+ REA counts a file as a document only when an app window's title shows its
+ name or `~/` path, so name the file the app will save when its title shows
+ something else, as MuseScore's shows a score's title: a file outside
+ hidden folders, other than a lock such as LibreOffice's `.~lock.#`,
+ becomes a document, which the bundle owns, seeds and opens
+ at launch, and one under a hidden folder such as `~/.config/MuseScore` the
+ app's own state, also when that folder is a link elsewhere in the home. A
+ path with `..`, or naming the home itself, is refused before the computer
+ is asked; the computer refuses an absolute path outside its home, one in
+ `~/.rea-build`, and one a link leads out of the home or into
+ `~/.rea-build`, resolving a path that does not exist yet through its
+ nearest existing folder, and starts nothing.
+
3. Make the seed through the app with `act_on_build_computer`, saving it
where the user keeps such files, such as `~/Documents`, then
`mark_build_checkpoint` `seed`.
@@ -48,22 +76,56 @@ See [desktop builds](https://github.com/morluto/rea/blob/main/.agents/skills/rev
`mark_build_checkpoint` `end`.
5. `stop_build_capture`. The app is closed and the capture finishes. The
result lists the checkpoints with their counts, the package and the
- sources the capture found, its limitations and the draft: each file that
- changed between `seed` and `end` with its net change, and suggested
- checks with a placeholder prompt.
+ sources the capture found, its limitations and the draft: each file under
+ the paths the export owns that changed between `seed` and `end`, with its
+ net change and, for XML files, archives, JSON and key files, its `values`,
+ the values the change added, changed and removed with what each was and
+ became, at the pointer a check names it by; suggested checks with a
+ placeholder prompt; and `guards`, optional checks that the rest kept its
+ seed values. The draft's
+ [task section](desktop-application-tasks.md#describe-the-change) describes
+ each. `notes`, each with a `code` and a `message`, say why a draft has no
+ files: `no_change` when no file under the home changed between them and
+ nor did the app's dconf keys; `unowned_changes` when files changed but
+ none under the owned paths, listing up to five in `files` with their
+ roles, desktop files last, and counting the rest in `more`;
+ `uncheckable_changes` when the app's keys in dconf were read and changed,
+ listing the database in `files`, since no check reads dconf (a version
+ the capture kept no copy of, as it keeps none of the database at the
+ start, counts as holding none of the app's keys);
+ `unreadable_changes` when a dconf database changed but REA could not read
+ it, such as without `dconf-cli`, with the read `error`, so whose keys
+ changed is unknown; and `no_document` when no file has the document role.
+ `unowned_changes` and `no_document` point at `state_paths`. They also say,
+ for a drafted `file`, why its suggestions are thin: `by_position`,
+ `view_state_only`, `long_text`, `cut_view` or `unreadable_view`.
6. Write the task: its name, the prompt in the user's own words, and checks
- that hold after the change and not on the seed.
+ that hold after the change and not on the seed, from the suggestions,
+ `values` and any `guards` worth keeping.
7. `export_build_environment` with the task and a new or empty `directory`.
The export runs inside the computer with `--seed-checkpoint seed
--end-checkpoint end`, so it seeds what was saved at `seed`, scrubs the
computer's host name and user from it, and refuses a task whose checks
- all pass on the seed. A task larger than 96 KiB as REA passes it to the
- computer is refused before anything runs. It also passes `--synthetic-seed`,
- since the agent made the seed for the task, so the README does not call it
- the user's real data. When the seed holds exactly one document and the
- app's desktop entry takes files, the bundle's app opens that document at
- launch. A file moved to the desktop Trash is not seeded, and the export's
- limitations name what the Trash held at `seed`. For a
+ all pass on the seed. It also passes `--synthetic-seed`, since the agent
+ made the seed for the task, so the README does not call it the user's real
+ data. When the seed holds exactly one document and the app's desktop entry
+ takes files, the bundle's app opens that document at launch. A file moved
+ to the desktop Trash is not seeded, and the export's limitations name what
+ the Trash held at `seed`. The limitations and the README also name the
+ folders where files changed during the capture that the bundle neither
+ owns nor seeds, such as settings that keep a first-run wizard away, up to
+ eight collapsed to their top folder, such as `~/.config/MuseScore`, and a
+ count of the rest; capture again with them in `state_paths` to own them.
+ Lock files the app holds while it runs, such as LibreOffice's profile
+ `.lock` and its document lock `.~lock.#`, are left out of the seed,
+ since a copied one can keep the app from starting, and the limitations
+ and the README list them after "Left out of the seed"; a project's
+ `Cargo.lock` or `yarn.lock`, a document or a path in `state_paths` is
+ seeded. They name seeded
+ WebKit or Chromium storage and caches too, which the app keeps for its
+ own running. When the application held files open in a folder whose
+ starting content the capture did not retain, a limitation names the
+ folder and the seed files in it. For a
`.deb` app that no apt source the capture found offers, the export takes
the request's URL with `--package-url`; one apt offers installs from apt. REA lists the bundle's files with their sizes
and SHA-256 in the computer, into a file beside the bundle that it reads
@@ -78,7 +140,15 @@ A checkpoint name is lowercase letters, digits, dot, dash and underscore; the
capture's own `baseline`, `stop`, `time-limit` and `exit` are refused, and so
is a name already taken. The acknowledgement comes from the capture after it
has walked the home and retained what changed: `entries` walked, `changes`
-since the checkpoint before, `retained_files` and `retained_bytes`.
+since the checkpoint before, `retained_files` and `retained_bytes`, and up to
+ten `changed_files`, home-relative, with `more_changed_files` counting the
+rest. Folders are left out; files under `state_paths` come first, then
+those the app held open, then those in its state directories, then the rest,
+and caches and WebKit or Chromium storage last, each by path, so a save's
+document leads the list. When the computer's REA is older than these lists,
+as when `REA_BUILD_COMPUTER_REA_ARCHIVE` still names an earlier build, the
+checkpoint is still taken and both are `null`: which files changed is
+unknown. Pack the current REA into the archive to get them back.
Each step can be asked again after its answer was lost:
@@ -102,7 +172,9 @@ computer's control token. REA sends each command through agentenv's
`computer-exec/v1` route with that token, which stays in REA, and reads larger
outputs, such as the draft, the export's Evidence and the bundle's file
list, from files under `~/.rea-build` through `data/get`, since a command's
-output is capped at 1 MiB.
+output is capped at 1 MiB. In a command's Evidence or failure read this way,
+as in its own output, REA replaces the token in every string, after checking
+the Evidence's digest; the bundle's files arrive as they are.
Calls on one computer run one at a time, in the order REA receives them:
an action, a capture step, a check or a close waits until the call before
@@ -116,7 +188,11 @@ cancels the call in progress on it and refuses the calls still waiting.
Start a fresh computer from the exported folder with `start_build_computer`
and `{"kind": "bundle", "path": ...}`; agentenv boots the app with the seed,
-on the seed's document when the export opens one.
+on the seed's document when the export opens one. When the computer never
+gets ready and the bundle's seed holds lock files, the start's error names
+them: delete them from the bundle, or export again, which leaves them out.
+A bundle records no state directories, so the error names only files by a
+profile lock's name and LibreOffice's document locks.
Then, with the task exactly as exported:
| `stage` | Before it | Its checks must |
@@ -129,7 +205,18 @@ Then, with the task exactly as exported:
`check_build_task` reads the app's `data/get`, takes the views it returns
beside the files and evaluates each check with REA's evaluator, the one the
export validates tasks with and the exported verifier mirrors, one result per
-check. Without `stage` it only grades. The `seed` stage begins a recording
+check. Like the verifier, it grades only once the app's files settle, since an
+app such as Gnote saves a few seconds after the last key. It reads `data/get`
+every 0.5 s and grades once every file's bytes have held for 3 s, counted
+from the read that returned them. 15 s after the first read began, it grades
+the last completed read: no read starts then, and one still running is cut.
+It sends progress notifications while it waits. The result's `settled` says
+whether the files settled, and `read_cut` whether the bound cut a read; each
+stage in the record keeps both. Every check therefore costs at least 3 s,
+and a save that lands more than 3 s after the first read returns is still
+missed; see
+[grading once the app has saved](desktop-application-export.md#grading-once-the-app-has-saved).
+Without `stage` it only grades. The `seed` stage begins a recording
of the display under REA's own request id, one per computer; a seed check
that fails after it, such as on a failed read of the app's state or a lost
reply to the recording's start, can be asked again, and asks for the
@@ -171,13 +258,14 @@ A stage whose checks do not show what it must ends the witness, and so does
- `recording/NNNN.mp4`, the recording's segments;
- `witness.json`, REA Evidence (`rea.desktop-build-witness`) holding the
verdict, the stage that failed if one did, the task's name and SHA-256,
- each stage's results with how many logged actions came before it, the
+ each stage's results with how many logged actions came before it and
+ whether the app's files had settled (`settled`), the
computer's [action log](desktop-build-computer.md#act) as it keeps it, each
action with its arguments, timing and outcome, with `actions_dropped`
counting the earliest actions it no longer keeps, and the recording's
segments with their sizes and SHA-256. The record's limitations say when
the log dropped actions or cut arguments, which each entry names in
- `truncated_arguments`.
+ `truncated_arguments`, and which stages graded files that had not settled.
The stages' rules read the log's count of actions and the last applied
action of each kind, which it keeps for the whole session, so they hold
diff --git a/docs/specs/desktop-app-environments/build-witness/spec.md b/docs/specs/desktop-app-environments/build-witness/spec.md
index 4cded391d1..8acb70e8a5 100644
--- a/docs/specs/desktop-app-environments/build-witness/spec.md
+++ b/docs/specs/desktop-app-environments/build-witness/spec.md
@@ -28,23 +28,49 @@ and the recording and writes the record as Evidence.
out of every walk. A start returns once the capture reports the window
open; a refused or early-ended start stops the capture and reports its
output.
+- `start_build_capture` takes `state_paths`, at most 16 files or folders in
+ the home, relative to it, after `~/` or absolute, without `..` and not the
+ home itself; the controller resolves each under the computer's home, both
+ as named and with its links followed (a path not there yet through its
+ nearest existing folder), refuses one outside it or in `~/.rea-build`
+ either way before launching anything, and passes each as named to the
+ capture as `--state-path`. A selected file is a document, or the app's own
+ state under a hidden folder of the home as named, before links, or when it
+ is a lock the app holds, such as LibreOffice's `.~lock.#`.
- Acknowledgement: `capture-desktop-application` reports, beside each progress
message, an `event`: `window_open`; `checkpoint` with its name, trigger,
- time, entries walked, changes and retained files and bytes, once it is
- taken, the baseline, `stop` and `exit` included; and `checkpoint_refused`
- with the reason. `mark_build_checkpoint` writes `checkpoint ` and
- waits for the event naming it, returning its counts. Names follow a narrow
- pattern, without the capture's own names; a name already taken is refused.
+ time, entries walked, changes and retained files and bytes, and up to ten
+ changed files, home-relative, those under a selected path first, then
+ those the app held open, caches and browser-engine storage last, with a count
+ of the rest, once it is taken, the baseline, `stop` and `exit` included;
+ and `checkpoint_refused` with the reason. `mark_build_checkpoint` writes
+ `checkpoint ` and waits for the event naming it, returning its counts
+ and changed files, both `null` when the REA in the computer predates them
+ and its event leaves them out. Names follow a narrow pattern, without the capture's own
+ names; a name already taken is refused.
- `stop_build_capture` sends `stop`, waits for the exit status, reads the
capture's Evidence and, when `seed` and `end` were acknowledged, runs
`describe-desktop-application-task --seed-checkpoint seed --end-checkpoint
end` and returns the changed files, their net change and suggested checks.
+ A draft with no files carries `notes`, coded `no_change`,
+ `unowned_changes` (up to five changed home paths with their roles, and a
+ count of the rest), `uncheckable_changes` (dconf databases whose app keys
+ were read and changed, which no check reads, a version kept without its
+ bytes counting as holding none), `unreadable_changes` (dconf
+ databases that changed but could not be read, with the read `error`) and
+ `no_document`; a drafted file whose suggestions are thin carries one
+ naming its `file`, coded `by_position`, `view_state_only`, `long_text`,
+ `cut_view` or `unreadable_view`.
- `export_build_environment` writes the agent's task under `~/.rea-build`,
runs `export-desktop-application` with it and the same checkpoints into a
new folder there, reports a refused task with the export's own reason,
lists the bundle with sizes and SHA-256, fetches it through the driver's
`data/get` in answers of at most 8 MiB, checks each file and writes the
- caller's new or empty folder through the safe output tree.
+ caller's new or empty folder through the safe output tree. The export
+ leaves lock files the app holds while it runs out of the seed, but never a
+ document or selected path, listing them in its limitations and README; a
+ start from a bundle whose computer never gets ready names the files its
+ seed holds by a profile lock's name or as a document lock.
- `check_build_task` reads the one app's `data/get` through the card's
child URL, takes the state views beside its files and evaluates the task's
checks with `evaluateTaskChecks`, one result per check. With `stage`, the
diff --git a/docs/specs/desktop-app-environments/build-witness/verification.md b/docs/specs/desktop-app-environments/build-witness/verification.md
index f0d53e8048..2db8bbd157 100644
--- a/docs/specs/desktop-app-environments/build-witness/verification.md
+++ b/docs/specs/desktop-app-environments/build-witness/verification.md
@@ -280,6 +280,123 @@ taken 18, 23 and 21 turns for $0.70, $1.55 and $0.51 ($0.92). Cache reads
fell from 585-714K tokens to 181-225K; the cost is mostly cache writes,
which varied from 17K to 189K tokens whatever the turns.
+## Naming the app's state
+
+Run on macOS with source tests and gates; no build computer was started.
+Each test below failed on the head before this change, with only the tests
+and their fixtures added, and passes now.
+
+- `buildCaptureStatePathsMcp.test.ts`: `start_build_capture` with
+ `state_paths` passes the note's path to the capture as `--state-path`, the
+ draft covers it as a document with no notes, and an export with a check on
+ it is accepted; without it the draft is empty with `unowned_changes`
+ (the note, `unrelated`) and `no_document`, and the export's refusal names
+ `state_paths`. A path through `..`, the home itself and 17 paths are
+ refused by the schema, `/etc/hosts` by the computer, and none starts a
+ capture. Before, `state_paths` was an unknown argument.
+- `DesktopTaskDraftNotes.test.ts`: a draft with files has no notes; one with
+ no change between seed and end has only `no_change`; one whose changes lie
+ outside the owned paths lists five with their roles, desktop files last,
+ and counts two more, beside `no_document`.
+- `stateRoles.test.ts`: a selected file under a hidden folder of the home is
+ application state, elsewhere a document.
+- `DesktopApplicationExport.test.ts`, `appExportUnowned.test.ts`: the export's
+ limitations and README name folders the capture saw change outside the
+ owned paths, collapsed to their top folder and without desktop caches, up
+ to eight and a count, and the seeded lock file, WebKit storage and cache
+ folders.
+
+## Lock files, changed files and held-open folders
+
+Run on macOS with source tests and gates; no build computer was started. A
+hold-out LibreOffice Calc bundle seeded `~/.config/libreoffice/4/.lock` and
+showed no window at boot; its export also said nothing in `~/Documents` was
+seeded while `seed_files` held `Documents/q1-sales.ods`. Each test below
+failed on the head before this change, with only the tests and their
+fixtures added, and passes now.
+
+- `DesktopApplicationExport.test.ts`: a capture whose seed holds a profile
+ `.lock` and a document lock `.~lock.today.txt#` exports without either in
+ `seed_files` or the bundle, and lists both after "Left out of the seed" in
+ one limitation and the README; with `~/Documents` held open, its
+ limitation names `~/Documents/today.txt` as seeded from there.
+- `appExportUnowned.test.ts`: profile, browser and document locks are lock
+ files, and `*.lock` or `*.lck` only in a hidden folder, so `Cargo.lock`
+ and `yarn.lock` in a project are not; the capture's limitation names the
+ held-open folders from `~/`, and the export's names the seed files there
+ or says there are none.
+- `desktopCapture.test.ts`: a checkpoint's acknowledgement names its changed
+ files home-relative, the held-open document first, then state, then the
+ rest, without folders, and names ten of thirteen with three more.
+- `buildCaptureMcp.test.ts`: `mark_build_checkpoint` returns the changed
+ files the capture acknowledged.
+- `buildComputerMcp.test.ts`: a bundle's start that never got ready names
+ the lock files its seed holds; one without locks, or failing otherwise,
+ names none.
+
+The Linux-only capture test's acknowledgement now expects the changed state
+file; it is skipped on macOS and was not run.
+
+## Linked state paths, dconf changes, project lock files and older checkpoint replies
+
+Run on macOS with source tests and gates; no build computer was started.
+Each test below failed on the head before this change, with only the tests
+and their fixtures added, and passes now.
+
+- `buildCaptureStatePathsMcp.test.ts`: the computer refuses state paths that
+ a link leads out of the home, existing or not yet (through the nearest
+ existing folder, or a dangling link), or into `~/.rea-build`, and starts
+ nothing; a link that stays in the home is followed and the path passed to
+ the capture as named. Before, the first such path started a capture.
+- `DesktopCaptureScope.test.ts`: the capture scope records each selected
+ path as named beside the physical path it walks.
+- `DesktopApplicationExport.test.ts`: with `~/.config/MuseScore` selected as
+ a link to `~/settings/MuseScore` beside a selected note, the bundle's
+ launch opens the note. With only the capture's record of selected paths
+ added, the settings file was still a document and the launch opened none.
+- `DesktopTaskDraftNotes.test.ts`: a draft whose only change between `seed`
+ and `end` is to the app's keys in dconf carries `uncheckable_changes`
+ naming `~/.config/dconf/user`, not `no_change`; one where only another
+ group's keys changed still has `no_change`.
+- `appExportUnowned.test.ts`: `*.lock` and `*.lck` are locks only directly
+ in a state directory, so `Projects/.repo/Cargo.lock` is not; and 20,000
+ separate unowned folders collapse in tens of milliseconds, where comparing
+ every pair took 12.9 seconds.
+- `DesktopApplicationExport.test.ts`: `Projects/.repo/Cargo.lock` and
+ `.dotfiles/web/yarn.lock`, which the capture recognized as documents, and
+ a lock-named file the caller selected are seeded, a `*.lock` directly in a
+ state directory is left out, and the profile and document locks still are.
+- `buildCaptureMcp.test.ts`: checkpoints acknowledged without
+ `changed_files` and `more_changed_files`, as the previous REA in a build
+ computer does, are taken with both `null`, also when a lost reply is read
+ again from the capture's log. Before, the host refused the acknowledgement
+ although the checkpoint was taken.
+- `desktopCapture.test.ts`: a checkpoint's changed files name a document
+ under a selected path first, though the app did not hold it open, and
+ caches and browser-engine storage last, even one the app held open or in a
+ state directory, so ten KDE service-cache files no longer push the saved
+ document past the cut, as they did in a held-out Tellico build.
+- `DesktopApplicationExport.test.ts`: with `~/.config/MuseScore` selected as
+ a link to `~/Documents` and the note inside it selected itself, the
+ launch opens the note while the settings beside it stay the app's; before,
+ the link's hidden name made the note application state and the launch
+ opened none.
+- `DesktopTaskDraftNotes.test.ts`: a dconf database that changed but could
+ not be read gives `unreadable_changes` with the reader's error, not
+ `uncheckable_changes`.
+- `buildCaptureMcp.test.ts`: `stop_build_capture` on a capture whose only
+ change between `seed` and `end` is a dconf database the draft cannot read
+ (no `dconf` on its `PATH`) returns the `unreadable_changes` note with its
+ `error`; before, the build tool dropped the field.
+- `LinuxApplicationInventory.test.ts`: the inventory lists KDE's
+ `~/.config/rc` before it exists, so a Tellico capture owns
+ `~/.config/tellicorc`, whose absence brought back Tip of the Day at boot;
+ `DesktopApplicationExport.test.ts` shows such a file, written during the
+ capture, owned and seeded (that test passed before, given the inventory).
+
+The capture's recording of selected paths runs only on Linux; its scope is
+tested on macOS, and the capture itself was not run.
+
## Limits of this verification
- The witness acts were scripted by this agent from the task's prompt; no
diff --git a/docs/specs/desktop-app-environments/capture-for-building/spec.md b/docs/specs/desktop-app-environments/capture-for-building/spec.md
index acf5dcaa71..6ea8b9a8cc 100644
--- a/docs/specs/desktop-app-environments/capture-for-building/spec.md
+++ b/docs/specs/desktop-app-environments/capture-for-building/spec.md
@@ -55,7 +55,9 @@ and the bundle holds neither the capture host's name nor its user's.
path and content, escapes decoded, for the host name and `user@host`
anywhere and the captured home as a whole path, with the replacements set
aside. `scrubbed_files` lists the rewritten seeds and `unscrubbed_files`
- every file found, by bundle path; the README lists both.
+ every file found, by bundle path; the README lists both. gzip seeds
+ compressed again only to clear their headers, their documents unchanged,
+ are in `recompressed_files` instead, which the README counts.
- Tasks: checks whose values name `user@host` or the host name by that rule
are refused, like those that name the captured home, but not for their
field names, kind or id; and the checks that must fail on the seed are
diff --git a/docs/specs/desktop-app-environments/capture/spec.md b/docs/specs/desktop-app-environments/capture/spec.md
index 637516e6c1..b8dc38fd42 100644
--- a/docs/specs/desktop-app-environments/capture/spec.md
+++ b/docs/specs/desktop-app-environments/capture/spec.md
@@ -68,7 +68,8 @@ folder to build the environment; this step builds nothing.
lists candidate state directories under the XDG folders and dot-folders
named after the package, executable, desktop entry's full id and last part,
and name, and every folder present there whose name is one of those in
- another letter case, such as GIMP's `~/.config/GIMP`; a `flatpak run`
+ another letter case, such as GIMP's `~/.config/GIMP`, and KDE's
+ `rc` in the config folder, such as `~/.config/tellicorc`; a `flatpak run`
launch uses the application id (its first operand
after options and their values) instead of the launcher's name and adds
`~/.var/app/`. An `--executable` runs from the caller's
@@ -78,7 +79,9 @@ folder to build the environment; this step builds nothing.
`--state-path` values and absolute arguments other than `/` or a directory
containing the home directory, which the limitations name instead, plus the
`-wal` and `-journal` paths beside each of those selected paths that is not
- a directory, so a database selected on its own is captured with its logs. The
+ a directory, so a database selected on its own is captured with its logs.
+ Each selected path is also recorded as named, beside its physical
+ spelling, so a role can follow from the name the caller gave it. The
baseline walk runs before launch and retains every file under the content
roots; each later checkpoint reports created, modified, deleted and retyped
paths against the previous walk and retains every changed file. A creation
diff --git a/docs/specs/desktop-app-environments/state-view/spec.md b/docs/specs/desktop-app-environments/state-view/spec.md
index d63791fbe7..0e56b7f86e 100644
--- a/docs/specs/desktop-app-environments/state-view/spec.md
+++ b/docs/specs/desktop-app-environments/state-view/spec.md
@@ -41,7 +41,11 @@ generation reads the kinds of change.
([XML text checks](../xml-text-checks/spec.md)); key files up to 4 MiB
whose every line parses; zip archives by
their local file header, as their entries with each entry's own format and
- view ([document views](../document-views/spec.md)); text; binary.
+ view ([document views](../document-views/spec.md)); text; binary. A gzip
+ file, told by its magic bytes, is read as the document inside, its members
+ joined, within a zip entry's 16 MiB and 100-times limits, and its version
+ records `compression: "gzip"` beside that document's format; past the
+ limits, or damaged, it has a `view_error`.
- Changes between versions are created, deleted, or lines, values, keys or
rows with added, removed and changed lists (100 each, with counts); content
for binary files and unreadable when a view is missing. Values are JSON
@@ -57,5 +61,6 @@ generation reads the kinds of change.
- No export, seed selection or task generation.
- No parsing of other binary formats (GVDB other than dconf, LevelDB, Realm,
- protobuf); they are listed as binary.
+ protobuf), or of compression other than gzip (xz, Zstandard, bzip2); they
+ are listed as binary.
- No redaction: views contain the user's real data, as the capture does.
diff --git a/docs/specs/desktop-app-environments/tasks/spec.md b/docs/specs/desktop-app-environments/tasks/spec.md
index b8715b73ea..7cbb35c63a 100644
--- a/docs/specs/desktop-app-environments/tasks/spec.md
+++ b/docs/specs/desktop-app-environments/tasks/spec.md
@@ -21,7 +21,14 @@ never runs code the agent wrote.
both ends whose count changed); and a draft task with a placeholder prompt
and checks suggested from the net change of documents, or of application
state when no document changed. Views are shown only for those files, cut at
- 200 items and 2,000 characters.
+ 200 items and 2,000 characters. A draft with no files says why in `notes`:
+ `no_change`, `unowned_changes` with up to five changed home paths and their
+ roles, `uncheckable_changes` with the dconf databases whose application
+ keys were read and changed, `unreadable_changes` with the dconf databases
+ that changed but could not be read and the read `error`, and
+ `no_document`; a drafted file whose suggestions are thin gets one naming
+ its `file`: `by_position`, `view_state_only`, `long_text`, `cut_view` or
+ `unreadable_view`.
- The task file is JSON with a portable `name` other than `smoke`, a `prompt`
and 1 to 32 `checks`, parsed by a zod schema in `src/domain/desktop/`. Check
kinds: `file_present`, `file_absent`, `lines_present`, `lines_absent`,
diff --git a/scripts/pack-build-computer-runtime.mjs b/scripts/pack-build-computer-runtime.mjs
index 3acafbb0c8..f3b10f3876 100644
--- a/scripts/pack-build-computer-runtime.mjs
+++ b/scripts/pack-build-computer-runtime.mjs
@@ -9,8 +9,9 @@ import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
+import { npmInvocation } from "./lib/npm-invocation.mjs";
+
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
-const npm = process.platform === "win32" ? "npm.cmd" : "npm";
const run = (command, args, cwd) => {
const result = spawnSync(command, args, { cwd, stdio: "inherit" });
@@ -31,7 +32,8 @@ if (destination === undefined || !destination.endsWith(".tar.xz")) {
const archive = resolve(destination);
const work = await mkdtemp(join(tmpdir(), "rea-build-runtime-"));
try {
- run(npm, ["pack", "--pack-destination", work], root);
+ const pack = npmInvocation(["pack", "--pack-destination", work], process.env);
+ run(pack.command, pack.arguments, root);
const [packed] = (await readdir(work)).filter((name) =>
name.endsWith(".tgz"),
);
@@ -43,7 +45,11 @@ try {
join(root, "package-lock.json"),
join(runtime, "package-lock.json"),
);
- run(npm, ["ci", "--omit=dev", "--omit=optional"], runtime);
+ const install = npmInvocation(
+ ["ci", "--omit=dev", "--omit=optional"],
+ process.env,
+ );
+ run(install.command, install.arguments, runtime);
run("tar", ["-cJf", archive, "-C", work, "rea-prod"], work);
const { size } = await stat(archive);
process.stdout.write(
diff --git a/src/application/desktop/DesktopApplicationCapture.test.ts b/src/application/desktop/DesktopApplicationCapture.test.ts
index 4dc0d54ee9..6ac6d439bf 100644
--- a/src/application/desktop/DesktopApplicationCapture.test.ts
+++ b/src/application/desktop/DesktopApplicationCapture.test.ts
@@ -475,7 +475,7 @@ describe.skipIf(process.platform !== "linux")(
evidence.normalized_result,
);
const seed = capture.state.checkpoints[1];
- // The acknowledgement comes after the checkpoint is taken and carries its counts.
+ // The acknowledgement comes after the checkpoint is taken and carries its counts and the file the app wrote.
expect(parsed[2]).toEqual({
kind: "checkpoint",
name: "seed",
@@ -485,6 +485,8 @@ describe.skipIf(process.platform !== "linux")(
changes: seed?.changes.length,
retained_files: 1,
retained_bytes: 8,
+ changed_files: [".config/fakeapp/state.ini"],
+ more_changed_files: 0,
});
expect(parsed[3]).toEqual({
kind: "checkpoint_refused",
diff --git a/src/application/desktop/DesktopApplicationCapture.ts b/src/application/desktop/DesktopApplicationCapture.ts
index b825dd358d..c825121bf6 100644
--- a/src/application/desktop/DesktopApplicationCapture.ts
+++ b/src/application/desktop/DesktopApplicationCapture.ts
@@ -6,6 +6,7 @@ import {
projectAnalysisError,
type AnalysisErrorProjection,
} from "../../domain/analysisErrorProjection.js";
+import { unretainedOpenedLimitation } from "../../domain/desktop/appExportUnowned.js";
import {
checkpointTaken,
type DesktopApplicationCapture,
@@ -110,6 +111,7 @@ const limitationsFor = (
recorder: CheckpointRecorder,
facts: {
readonly unretained: readonly string[];
+ readonly home: string;
readonly broadArguments: readonly string[];
readonly unreadableWindowSamples: number;
},
@@ -156,10 +158,8 @@ const limitationsFor = (
session.processes.some((process) => !process.started_by_capture),
"An application window belonged to a process the capture did not start; that process was left running.",
);
- add(
- facts.unretained.length > 0,
- "The application held files open in directories whose starting content was not retained; pass them with --state-path to retain it.",
- );
+ const unretained = unretainedOpenedLimitation(facts.unretained, facts.home);
+ if (unretained !== undefined) limitations.push(unretained);
return limitations;
};
@@ -264,7 +264,11 @@ const recordCapture = async (
const acknowledge = async () => {
const taken = recorder.checkpoints.at(-1);
if (taken === undefined) return;
- const event = checkpointTaken(taken);
+ const event = checkpointTaken(
+ taken,
+ home,
+ context.scope.selectedPaths.map(({ physical }) => physical),
+ );
await host.report(
`Recorded checkpoint ${taken.name}: ${String(event.changes)} change(s), ${String(event.retained_files)} file(s) retained.`,
event,
@@ -314,6 +318,7 @@ const recordCapture = async (
content_roots: [...scope.contentRoots],
baseline_non_files: recorder.baselineNonFiles,
state_directories: [...scope.stateDirectories],
+ selected_paths: [...context.scope.selectedPaths],
excluded: [...scope.excluded],
checkpoints: recorder.checkpoints,
content_files: store.files,
@@ -325,6 +330,7 @@ const recordCapture = async (
...admitted.limitations,
...limitationsFor(session, recorder, {
unretained,
+ home,
broadArguments: context.scope.broadArguments,
unreadableWindowSamples: outcome.unreadableWindowSamples,
}),
diff --git a/src/application/desktop/DesktopApplicationExport.test.ts b/src/application/desktop/DesktopApplicationExport.test.ts
index ebac7538ba..b0f2e65438 100644
--- a/src/application/desktop/DesktopApplicationExport.test.ts
+++ b/src/application/desktop/DesktopApplicationExport.test.ts
@@ -3,6 +3,7 @@ import { access, readFile } from "node:fs/promises";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
+import { z } from "zod";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
@@ -22,10 +23,12 @@ import {
import {
exportDesktopApplication,
seedLimitation,
+ trashedFilesLimitation,
} from "./DesktopApplicationExport.js";
import {
EDITED_NOTE,
HOME,
+ NOTE,
capture,
change,
checkpoint,
@@ -248,6 +251,165 @@ describe("exportDesktopApplication", () => {
});
});
+it("seeds the app's state at its own path and retains a selected alias", async () => {
+ const local = capture();
+ const physical = `${HOME}/.config/notes`;
+ const { result, directory } = await exportCapture({
+ ...local,
+ state: {
+ ...local.state,
+ content_roots: [physical],
+ state_directories: [physical, `${HOME}/.local/share/notes`],
+ selected_paths: [
+ { path: `${HOME}/settings`, physical, kind: "directory" },
+ {
+ path: `${HOME}/settings/profiles/work`,
+ physical: `${physical}/profiles/work`,
+ kind: "directory",
+ },
+ ],
+ checkpoints: [
+ checkpoint(
+ "baseline",
+ "baseline",
+ [],
+ [
+ retained(`${physical}/settings.ini`, "[ui]\nwrap=false\n"),
+ retained(`${physical}/profiles/work/profile.ini`, "milk\n"),
+ ],
+ ),
+ ],
+ },
+ });
+ if (result instanceof Error) throw result;
+ expect(result.normalized_result).toMatchObject({
+ paths: [".config/notes"],
+ seed_files: [
+ ".config/notes/profiles/work/profile.ini",
+ ".config/notes/settings.ini",
+ ],
+ });
+ expect(
+ await readFile(
+ join(directory, "envs/notes/seed/.config/notes/settings.ini"),
+ "utf8",
+ ),
+ ).toBe("[ui]\nwrap=false\n");
+ expect(
+ await readFile(
+ join(
+ directory,
+ "envs/notes/seed/.config/notes/profiles/work/profile.ini",
+ ),
+ "utf8",
+ ),
+ ).toBe("milk\n");
+ const module = await readFile(
+ join(directory, "envs/notes/notes_app.py"),
+ "utf8",
+ );
+ expect(module).toContain('("settings", ".config/notes")');
+});
+
+it.each(["settings/notes", "chosen/notes"])(
+ "restores the selected %s spelling alongside the app's walked state path",
+ async (selected) => {
+ const local = capture();
+ const physical = `${HOME}/settings/notes`;
+ const { result, directory } = await exportCapture({
+ ...local,
+ state: {
+ ...local.state,
+ roots: [HOME],
+ content_roots: [physical],
+ state_directories: [physical, `${HOME}/.local/share/notes`],
+ selected_paths: [
+ { path: `${HOME}/${selected}`, physical, kind: "directory" },
+ ],
+ checkpoints: [
+ checkpoint(
+ "baseline",
+ "baseline",
+ [],
+ [retained(`${physical}/settings.ini`, "[ui]\nwrap=false\n")],
+ ),
+ ],
+ },
+ });
+ if (result instanceof Error) throw result;
+ expect(result.normalized_result).toMatchObject({
+ seed_files: expect.arrayContaining([".config/notes/settings.ini"]),
+ });
+ expect(
+ await readFile(
+ join(directory, "envs/notes/seed/.config/notes/settings.ini"),
+ "utf8",
+ ),
+ ).toBe("[ui]\nwrap=false\n");
+ const module = await readFile(
+ join(directory, "envs/notes/notes_app.py"),
+ "utf8",
+ );
+ expect(module).toContain(`("${selected}", ".config/notes")`);
+ },
+);
+
+it.each([true, false])(
+ "omits only a child alias supplied by its parent (%s)",
+ async (redundant) => {
+ const local = capture();
+ const parent = `${HOME}/.config/notes`;
+ const child = `${parent}/state/notes`;
+ const { result, directory } = await exportCapture({
+ ...local,
+ application: {
+ ...local.application,
+ state_directories: [
+ { path: parent, exists_at_start: true },
+ {
+ path: redundant ? child : `${HOME}/.local/share/separate`,
+ exists_at_start: true,
+ },
+ ],
+ },
+ state: {
+ ...local.state,
+ content_roots: [parent],
+ state_directories: [parent, child],
+ selected_paths: [
+ { path: `${HOME}/settings`, physical: parent, kind: "directory" },
+ {
+ path: `${HOME}/settings/state/notes`,
+ physical: child,
+ kind: "directory",
+ },
+ ],
+ checkpoints: [
+ checkpoint(
+ "baseline",
+ "baseline",
+ [],
+ [retained(`${child}/settings.ini`, "[ui]\nwrap=false\n")],
+ ),
+ ],
+ },
+ });
+ if (result instanceof Error) throw result;
+ const module = await readFile(
+ join(directory, "envs/notes/notes_app.py"),
+ "utf8",
+ );
+ expect(module).toContain('("settings", ".config/notes")');
+ expect(module).not.toContain(
+ '("settings/state/notes", ".config/notes/state/notes")',
+ );
+ if (!redundant)
+ expect(module).toContain(
+ '("settings/state/notes", ".local/share/separate")',
+ );
+ },
+);
+
describe("exportDesktopApplication and the desktop Trash", () => {
it("leaves a file the user moved to the Trash out of the seed and the owned paths, and says so", async () => {
const local = capture();
@@ -306,12 +468,297 @@ describe("exportDesktopApplication and the desktop Trash", () => {
"Documents/today.txt",
],
});
- const warning = result.limitations.find((line) =>
- line.includes("Trash/files/today.txt"),
+ const trashPaths = [
+ ".local/share/Trash/files/today.txt",
+ ".local/share/Trash/info/today.txt.trashinfo",
+ ];
+ const notice = trashedFilesLimitation(trashPaths);
+ expect(result.limitations).toContain(notice);
+ for (const path of trashPaths) expect(notice).toContain(`~/${path}`);
+ });
+});
+
+describe("exportDesktopApplication of state it does not own or keeps for itself", () => {
+ it("names the folders the app wrote that the bundle neither owns nor seeds, and the web storage and caches it seeds", async () => {
+ const local = capture();
+ const [baseline, edited] = local.state.checkpoints;
+ if (baseline === undefined || edited === undefined)
+ throw new Error("the fixture has two checkpoints");
+ const text = "x\n";
+ // In the state directories from the start, so seeded.
+ const kept = [
+ `${HOME}/.local/share/notes/storage/salt`,
+ `${HOME}/.local/share/notes/hsts-storage.sqlite`,
+ `${HOME}/.config/notes/GPUCache/data_0`,
+ ];
+ // Written during the capture outside them; the font cache is the desktop's.
+ const written = [
+ `${HOME}/.config/MuseScore/MuseScore3.ini`,
+ `${HOME}/.local/share/MuseScore/MuseScore3/session`,
+ `${HOME}/.cache/fontconfig/cache-1`,
+ ];
+ const folder = await writeCaptureFolder(
+ {
+ ...local,
+ state: {
+ ...local.state,
+ checkpoints: [
+ {
+ ...baseline,
+ retained: [
+ ...baseline.retained,
+ ...kept.map((path) => retained(path, text)),
+ ],
+ },
+ {
+ ...edited,
+ changes: [
+ ...edited.changes,
+ ...written.map((path) => change(path, { after: text })),
+ ],
+ retained: [
+ ...edited.retained,
+ ...written.map((path) => retained(path, text)),
+ ],
+ },
+ ],
+ },
+ },
+ undefined,
+ [text],
);
- expect(warning).toContain("~/.local/share/Trash/info/today.txt.trashinfo");
- expect(warning).toMatch(/Trash.*2 file.*not seeded/iu);
- expect(warning).toMatch(/restore.*capture again/iu);
+ const directory = join(
+ await createTestTempDirectory("desktop-export-"),
+ "bundle",
+ );
+
+ const result = await exportDesktopApplication(
+ { capture: folder, directory, name: undefined, tasks: [] },
+ readers,
+ );
+
+ if (result instanceof Error) throw result;
+ expect(result.normalized_result).toMatchObject({
+ seed_files: expect.arrayContaining(
+ kept.map((path) => path.slice(HOME.length + 1)),
+ ),
+ });
+ const unowned = result.limitations.find((limitation) =>
+ limitation.includes("~/.config/MuseScore"),
+ );
+ expect(unowned).toContain("~/.local/share/MuseScore");
+ expect(unowned).toContain("state_paths");
+ const runtime = result.limitations.find((limitation) =>
+ limitation.includes("~/.local/share/notes/storage"),
+ );
+ for (const path of [
+ "~/.local/share/notes/hsts-storage.sqlite",
+ "~/.config/notes/GPUCache",
+ ])
+ expect(runtime).toContain(path);
+ // Desktop caches are left out, and paths collapse to the folder that holds them.
+ const listed = result.limitations.join("\n");
+ for (const path of ["fontconfig", "MuseScore3/session", "storage/salt"])
+ expect(listed).not.toContain(path);
+ const readme = await readFile(join(directory, "README.md"), "utf8");
+ for (const path of [
+ "~/.config/MuseScore",
+ "~/.local/share/MuseScore",
+ "~/.local/share/notes/storage",
+ "~/.config/notes/GPUCache",
+ ])
+ expect(readme).toContain(`\`${path}\``);
+ expect(readme).toContain("`state_paths`");
+ });
+});
+
+/** The fixture with a profile lock and a document lock written by `edited`, and its notes folder held open. */
+const lockedCapture = (): DesktopApplicationCapture => {
+ const local = capture();
+ const [baseline, edited] = local.state.checkpoints;
+ if (baseline === undefined || edited === undefined)
+ throw new Error("the fixture has two checkpoints");
+ const locks = [
+ [PROFILE_LOCK, PROFILE_LOCK_TEXT],
+ [DOCUMENT_LOCK, DOCUMENT_LOCK_TEXT],
+ ] as const;
+ return {
+ ...local,
+ state: {
+ ...local.state,
+ checkpoints: [
+ baseline,
+ {
+ ...edited,
+ changes: [
+ ...edited.changes,
+ ...locks.map(([path, text]) => change(path, { after: text })),
+ ],
+ retained: [
+ ...edited.retained,
+ ...locks.map(([path, text]) => retained(path, text)),
+ ],
+ },
+ ],
+ unretained_opened_directories: [`${HOME}/Documents`],
+ },
+ };
+};
+const PROFILE_LOCK = `${HOME}/.config/notes/.lock`;
+const PROFILE_LOCK_TEXT = "[Lockdata]\nUser=ada\nHost=localhost\n";
+const DOCUMENT_LOCK = `${HOME}/Documents/.~lock.today.txt#`;
+const DOCUMENT_LOCK_TEXT =
+ ",ada,localhost,04.11.2026 00:00,file:///home/ada/.config/notes;\n";
+
+const exportLocked = async (
+ recorded = lockedCapture(),
+ extraContent: readonly string[] = [],
+) => {
+ const folder = await writeCaptureFolder(recorded, undefined, [
+ PROFILE_LOCK_TEXT,
+ DOCUMENT_LOCK_TEXT,
+ ...extraContent,
+ ]);
+ const directory = join(
+ await createTestTempDirectory("desktop-export-"),
+ "bundle",
+ );
+ const result = await exportDesktopApplication(
+ {
+ capture: folder,
+ directory,
+ name: undefined,
+ tasks: [],
+ seedCheckpoint: "edited",
+ },
+ readers,
+ );
+ if (result instanceof Error) throw result;
+ return { result, directory };
+};
+
+describe("exportDesktopApplication of lock files and folders the app held files open in", () => {
+ it("leaves the profile lock and the document lock out of the seed and lists them as left out", async () => {
+ const { result, directory } = await exportLocked();
+
+ const seedFiles = z
+ .object({ seed_files: z.array(z.string()) })
+ .parse(result.normalized_result).seed_files;
+ expect(seedFiles).toContain("Documents/today.txt");
+ expect(seedFiles).toContain(".config/notes/settings.ini");
+ expect(seedFiles).not.toContain(".config/notes/.lock");
+ expect(seedFiles).not.toContain("Documents/.~lock.today.txt#");
+ expect(
+ await exists(join(directory, "envs/notes/seed/.config/notes/.lock")),
+ ).toBe(false);
+ expect(
+ await exists(
+ join(directory, "envs/notes/seed/Documents/.~lock.today.txt#"),
+ ),
+ ).toBe(false);
+ const leftOut = result.limitations.filter((limitation) =>
+ limitation.startsWith("Left out of the seed:"),
+ );
+ expect(leftOut).toHaveLength(1);
+ expect(leftOut[0]).toContain(
+ "~/.config/notes/.lock, ~/Documents/.~lock.today.txt#.",
+ );
+ // Nothing else calls them seeded.
+ expect(
+ result.limitations.filter((limitation) => limitation.includes(".lock")),
+ ).toEqual(leftOut);
+ const readme = await readFile(join(directory, "README.md"), "utf8");
+ expect(readme).toContain("- Left out of the seed:");
+ expect(readme).toContain(
+ "`~/.config/notes/.lock`, `~/Documents/.~lock.today.txt#`",
+ );
+ });
+
+ it("seeds project lock files the capture recognized as documents and a lock-named file the caller selected, and leaves out a *.lock file directly in a state directory", async () => {
+ const locked = lockedCapture();
+ const [baseline, edited] = locked.state.checkpoints;
+ if (baseline === undefined || edited === undefined)
+ throw new Error("the fixture has two checkpoints");
+ const text = "x\n";
+ // Under folders whose names start with a dot, which made them look like an app's locks.
+ const projects = [
+ `${HOME}/Projects/.repo/Cargo.lock`,
+ `${HOME}/.dotfiles/web/yarn.lock`,
+ ];
+ const selected = `${HOME}/.config/notes/notes.lock`;
+ const stateLock = `${HOME}/.config/notes/db.lock`;
+ const written = [...projects, selected, stateLock];
+ const recorded: DesktopApplicationCapture = {
+ ...locked,
+ session: {
+ ...locked.session,
+ windows: locked.session.windows.map((window) => ({
+ ...window,
+ titles: [...window.titles, "Cargo.lock - Notes", "yarn.lock - Notes"],
+ })),
+ },
+ state: {
+ ...locked.state,
+ content_roots: [...locked.state.content_roots, selected],
+ checkpoints: [
+ baseline,
+ {
+ ...edited,
+ changes: [
+ ...edited.changes,
+ ...written.map((path) => change(path, { after: text })),
+ ],
+ retained: [
+ ...edited.retained,
+ ...written.map((path) => retained(path, text)),
+ ],
+ },
+ ],
+ },
+ };
+
+ const { result, directory } = await exportLocked(recorded, [text]);
+
+ const seedFiles = z
+ .object({ seed_files: z.array(z.string()) })
+ .parse(result.normalized_result).seed_files;
+ expect(seedFiles).toEqual(
+ expect.arrayContaining([
+ "Projects/.repo/Cargo.lock",
+ ".dotfiles/web/yarn.lock",
+ ".config/notes/notes.lock",
+ ]),
+ );
+ for (const project of [
+ "Projects/.repo/Cargo.lock",
+ ".dotfiles/web/yarn.lock",
+ ])
+ expect(
+ await exists(join(directory, "envs/notes/seed", project)),
+ project,
+ ).toBe(true);
+ expect(seedFiles).not.toContain(".config/notes/db.lock");
+ expect(seedFiles).not.toContain(".config/notes/.lock");
+ const leftOut = result.limitations.filter((limitation) =>
+ limitation.startsWith("Left out of the seed:"),
+ );
+ expect(leftOut).toHaveLength(1);
+ expect(leftOut[0]).toContain(
+ "~/.config/notes/.lock, ~/.config/notes/db.lock, ~/Documents/.~lock.today.txt#.",
+ );
+ });
+
+ it("names the seed files under a folder the app held files open in, rather than saying nothing there is seeded", async () => {
+ const { result } = await exportLocked();
+
+ const heldOpen = result.limitations.find((limitation) =>
+ limitation.startsWith("The application held files open in"),
+ );
+ expect(heldOpen).toContain("held files open in ~/Documents,");
+ expect(heldOpen).toContain(
+ "the seed holds ~/Documents/today.txt from there",
+ );
+ expect(heldOpen).not.toContain("nothing there is seeded");
});
});
@@ -592,6 +1039,68 @@ describe("exportDesktopApplication home paths", () => {
const seed = (directory: string, path: string) =>
readFile(join(directory, "envs/notes/seed", path), "utf8");
+ it("owns and seeds KDE's ~/.config/rc, which the app first wrote during the capture", async () => {
+ const local = capture();
+ const [baseline, edited] = local.state.checkpoints;
+ if (baseline === undefined || edited === undefined)
+ throw new Error("the fixture has two checkpoints");
+ const rc = `${HOME}/.config/notesrc`;
+ const text = "[General]\nShowTipOfDay=false\n";
+ // The inventory names it before it exists, as it names the app's folders.
+ const folder = await writeCaptureFolder(
+ {
+ ...local,
+ application: {
+ ...local.application,
+ state_directories: [
+ ...local.application.state_directories,
+ { path: rc, exists_at_start: false },
+ ],
+ },
+ state: {
+ ...local.state,
+ content_roots: [...local.state.content_roots, rc],
+ state_directories: [...(local.state.state_directories ?? []), rc],
+ checkpoints: [
+ baseline,
+ {
+ ...edited,
+ changes: [
+ ...edited.changes,
+ change(rc, { after: text }, { state: true }),
+ ],
+ retained: [...edited.retained, retained(rc, text)],
+ },
+ ],
+ },
+ },
+ undefined,
+ [text],
+ );
+ const directory = join(
+ await createTestTempDirectory("desktop-export-"),
+ "bundle",
+ );
+
+ const result = await exportDesktopApplication(
+ {
+ capture: folder,
+ directory,
+ name: undefined,
+ tasks: [],
+ seedCheckpoint: "edited",
+ },
+ readers,
+ );
+
+ if (result instanceof Error) throw result;
+ expect(result.normalized_result).toMatchObject({
+ paths: expect.arrayContaining([".config/notesrc"]),
+ seed_files: expect.arrayContaining([".config/notesrc"]),
+ });
+ expect(await seed(directory, ".config/notesrc")).toBe(text);
+ });
+
it("keeps a file under a symlinked home at its path under the walked home", async () => {
const home = "/data/ada";
const { result, directory } = await exportWalked({
@@ -670,12 +1179,20 @@ describe("exportDesktopApplication home paths", () => {
});
});
-/** Export the seed-then-end capture, launched from a desktop entry with `exec`, seeded at `seed`. */
+/** Export the seed-then-end capture, or `recorded` with `extraContent`, launched from a desktop entry with `exec`, seeded at `seed`. */
const exportSeeded = async (
exec: string | null,
- options: { readonly syntheticSeed?: boolean } = {},
+ {
+ recorded,
+ extraContent = [],
+ ...options
+ }: {
+ readonly syntheticSeed?: boolean;
+ readonly recorded?: DesktopApplicationCapture;
+ readonly extraContent?: readonly string[];
+ } = {},
) => {
- const local = seedEndCapture();
+ const local = recorded ?? seedEndCapture();
const folder = await writeCaptureFolder(
{
...local,
@@ -698,7 +1215,7 @@ const exportSeeded = async (
},
},
undefined,
- [EDITED_NOTE],
+ [EDITED_NOTE, ...extraContent],
);
const directory = join(
await createTestTempDirectory("desktop-export-"),
@@ -729,6 +1246,45 @@ const exportSeeded = async (
const launchLimitations = (limitations: readonly string[]) =>
limitations.filter((line) => line.includes(NO_LAUNCH_DOCUMENT));
+/**
+ * The seed-then-end capture of an app whose titles name no file, given the
+ * note and `~/.config/MuseScore`, a link to `target`, as state paths, with
+ * MuseScore's settings there from the start.
+ */
+const linkedSettingsCapture = (target: string): DesktopApplicationCapture => {
+ const base = seedEndCapture();
+ const [baseline, ...later] = base.state.checkpoints;
+ if (baseline === undefined) throw new Error("the fixture has a baseline");
+ return {
+ ...base,
+ session: {
+ ...base.session,
+ windows: base.session.windows.map((window) => ({
+ ...window,
+ titles: ["Untitled - Notes"],
+ })),
+ },
+ state: {
+ ...base.state,
+ content_roots: [...base.state.content_roots, target, NOTE],
+ selected_paths: [
+ { path: `${HOME}/.config/MuseScore`, physical: target },
+ { path: NOTE, physical: NOTE },
+ ],
+ checkpoints: [
+ {
+ ...baseline,
+ retained: [
+ ...baseline.retained,
+ retained(`${target}/MuseScore3.ini`, "[ui]\nwrap=false\n"),
+ ],
+ },
+ ...later,
+ ],
+ },
+ };
+};
+
describe("exportDesktopApplication launch document", () => {
it("opens the seed's one document when the desktop entry takes files", async () => {
const { result, module, readme } = await exportSeeded("notes %U");
@@ -745,6 +1301,37 @@ describe("exportDesktopApplication launch document", () => {
expect(launchLimitations(result.limitations)).toEqual([]);
});
+ it.each([
+ ["elsewhere in the home", "settings/MuseScore"],
+ [
+ "to the note's own folder, where the note's deeper selection decides",
+ "Documents",
+ ],
+ ])(
+ "opens the selected note when a selected settings link leads %s",
+ async (_, target) => {
+ const { result, module } = await exportSeeded("notes %U", {
+ recorded: linkedSettingsCapture(`${HOME}/${target}`),
+ });
+
+ // The selected link is ~/.config/MuseScore, so the settings keep that
+ // spelling and the note stays the one document.
+ expect(result.normalized_result).toMatchObject({
+ seed_files: expect.arrayContaining([
+ "Documents/today.txt",
+ ".config/MuseScore/MuseScore3.ini",
+ ]),
+ });
+ expect(result.normalized_result).not.toMatchObject({
+ seed_files: expect.arrayContaining([`${target}/MuseScore3.ini`]),
+ });
+ expect(module).toContain(
+ `exec 'notes' \\"$HOME\\"/'Documents/today.txt'"]`,
+ );
+ expect(launchLimitations(result.limitations)).toEqual([]);
+ },
+ );
+
it.each(["notes", null])(
"launches as captured, and says why in the README and limitations, when Exec is %j",
async (exec) => {
@@ -817,6 +1404,66 @@ describe("exportDesktopApplication launch document", () => {
});
});
+describe("exportDesktopApplication launch document beside its lock", () => {
+ it("leaves LibreOffice's lock out of a selected folder's seed, so the document beside it is the one the launch opens", async () => {
+ const base = seedEndCapture();
+ const [baseline, seed, ...later] = base.state.checkpoints;
+ if (baseline === undefined || seed === undefined)
+ throw new Error("the fixture has a baseline and a seed");
+ const documents = `${HOME}/Documents`;
+ const recorded: DesktopApplicationCapture = {
+ ...base,
+ session: {
+ ...base.session,
+ windows: base.session.windows.map((window) => ({
+ ...window,
+ titles: ["Untitled - Notes"],
+ })),
+ },
+ state: {
+ ...base.state,
+ content_roots: [...base.state.content_roots, documents],
+ selected_paths: [{ path: documents, physical: documents }],
+ checkpoints: [
+ baseline,
+ {
+ ...seed,
+ changes: [
+ ...seed.changes,
+ change(DOCUMENT_LOCK, { after: DOCUMENT_LOCK_TEXT }),
+ ],
+ retained: [
+ ...seed.retained,
+ retained(DOCUMENT_LOCK, DOCUMENT_LOCK_TEXT),
+ ],
+ },
+ ...later,
+ ],
+ },
+ };
+
+ const { result, module } = await exportSeeded("notes %U", {
+ recorded,
+ extraContent: [DOCUMENT_LOCK_TEXT],
+ });
+
+ const seedFiles = z
+ .object({ seed_files: z.array(z.string()) })
+ .parse(result.normalized_result).seed_files;
+ expect(seedFiles).toContain("Documents/today.txt");
+ expect(seedFiles).not.toContain("Documents/.~lock.today.txt#");
+ expect(
+ result.limitations.filter((line) =>
+ line.startsWith("Left out of the seed:"),
+ ),
+ ).toEqual([expect.stringContaining("~/Documents/.~lock.today.txt#")]);
+ expect(module).toContain(
+ `exec 'notes' \\"$HOME\\"/'Documents/today.txt'"]`,
+ );
+ expect(launchLimitations(result.limitations)).toEqual([]);
+ });
+});
+
describe("exportDesktopApplication bundle writes", () => {
it.each(["cancelled", "out of space"])(
"removes the unpublished bundle when its write fails (%s)",
diff --git a/src/application/desktop/DesktopApplicationExport.ts b/src/application/desktop/DesktopApplicationExport.ts
index eba8d18622..4c6b017805 100644
--- a/src/application/desktop/DesktopApplicationExport.ts
+++ b/src/application/desktop/DesktopApplicationExport.ts
@@ -16,6 +16,14 @@ import {
type SeedScrubReport,
} from "../../domain/desktop/appExport.js";
import { INSTALL_SCRIPT } from "../../domain/desktop/appExportInstall.js";
+import {
+ heldOpenLimitation,
+ leftOutLocksLimitation,
+ runtimeSeeds,
+ runtimeSeedsLimitation,
+ unownedChangedFolders,
+ unownedFoldersLimitation,
+} from "../../domain/desktop/appExportUnowned.js";
import { installLimitations } from "../../domain/desktop/appExportInstallLimitations.js";
import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js";
import {
@@ -88,6 +96,8 @@ export interface DesktopExportResult {
readonly unseeded_files: readonly string[];
/** Seed files rewritten to remove the capture host's name and user. */
readonly scrubbed_files: readonly string[];
+ /** gzip seed files compressed again only to clear their headers, their documents as captured. */
+ readonly recompressed_files: readonly string[];
/** Seed files that still hold them, in bytes the export does not rewrite or in their name. */
readonly unscrubbed_files: readonly string[];
readonly dconf_groups: number;
@@ -187,16 +197,23 @@ const planExport = (
request.packageUrl,
);
const electron = application.executable.electron_markers.length > 0;
- const { homePath, files, collisions, paths } = exportedStateFiles(
- capture,
- state,
- );
+ const { homePath, stateAliases, files, collisions, paths, isLeftOutLock } =
+ exportedStateFiles(capture, state);
const { text: dconf, ...dconfScrub } = seedDconf(
state,
context.seed,
context.identity,
);
- const { seeds, textSeeds } = seedVersions(files, paths, context.seed);
+ const { seeds, textSeeds, locks } = seedVersions(
+ files,
+ paths,
+ context.seed,
+ isLeftOutLock,
+ );
+ const unseeded = unseededFiles(capture, homePath, files, {
+ paths,
+ seed: context.seed,
+ });
const launch = planLaunch(capture, electron, seeds, files);
const plan: DesktopAppExportPlan = {
name: source,
@@ -222,10 +239,13 @@ const planExport = (
syntheticSeed: request.syntheticSeed === true,
seedFiles: [...seeds.keys()].sort(),
textSeeds,
- unseededFiles: unseededFiles(capture, homePath, files, {
- paths,
- seed: context.seed,
- }),
+ stateAliases,
+ unseededFiles: unseeded.filter((file) => !isLeftOutLock(file)),
+ leftOutLocks: [
+ ...new Set([...locks, ...unseeded.filter(isLeftOutLock)]),
+ ].sort(),
+ unownedFolders: unownedChangedFolders(files, paths),
+ runtimeSeeds: runtimeSeeds([...seeds.keys()]),
dconf,
captureEvidenceId: context.evidenceId,
};
@@ -245,6 +265,14 @@ const planExport = (
};
};
+/** The limitation naming files the desktop Trash held at the seed; none when it held none. */
+export const trashedFilesLimitation = (
+ files: readonly string[],
+): string | undefined =>
+ files.length === 0
+ ? undefined
+ : `The desktop Trash held ${String(files.length)} file(s) at the seed checkpoint, which are not seeded: ${files.map((home) => `~/${home}`).join(", ")}. Restore one the task needs and capture again.`;
+
/** The limitation on where the seed came from: made for the task, or the user's real state, kept local. */
export const seedLimitation = (syntheticSeed: boolean): string =>
syntheticSeed
@@ -252,7 +280,7 @@ export const seedLimitation = (syntheticSeed: boolean): string =>
: "The seed is the user's real state at the seed checkpoint; keep the bundle local.";
const SCRUB_LIMITATION =
- "Binary state files are seeded as captured; only text files, UTF-8 or UTF-16 with a byte-order mark, have the capture host's name, user@host and home path replaced.";
+ "Binary state files are seeded as captured; only text files, UTF-8 or UTF-16 with a byte-order mark, and gzip documents, scrubbed inside, have the capture host's name, user@host and home path replaced. A gzip document so rewritten is compressed again as one member with no extra field, so a BGZF or dictzip file loses the blocks its index names.";
/** The bundle's generated files, the class module flagged when its dconf keys hold the identity, and its README, made once the seeds' scrub is known. */
const bundleText = (
@@ -299,10 +327,22 @@ const exportLimitations = (
readonly collisions: readonly HomePathCollision[];
readonly identity: HostIdentity;
readonly trashed: readonly string[];
+ readonly files: ReadonlyMap;
},
): string[] => {
- const opened = capture.state.unretained_opened_directories;
+ const opened = heldOpenLimitation(
+ capture.state.unretained_opened_directories,
+ capture.host.home,
+ plan.seedFiles.flatMap((seed) => {
+ const source = facts.files.get(seed)?.path;
+ return source === undefined ? [] : [{ seed, source }];
+ }),
+ );
const opens = launchDocumentLimitation(plan.opens);
+ const unowned = unownedFoldersLimitation(plan.unownedFolders);
+ const runtime = runtimeSeedsLimitation(plan.runtimeSeeds);
+ const locks = leftOutLocksLimitation(plan.leftOutLocks);
+ const trashed = trashedFilesLimitation(facts.trashed);
return [
seedLimitation(plan.syntheticSeed),
SCRUB_LIMITATION,
@@ -313,21 +353,16 @@ const exportLimitations = (
: [
`${String(plan.unseededFiles.length)} file(s) under the owned paths existed at the seed but were not retained, so they are not seeded.`,
]),
- ...(facts.trashed.length === 0
- ? []
- : [
- `The desktop Trash held ${String(facts.trashed.length)} file(s) at the seed checkpoint, which are not seeded: ${facts.trashed.map((home) => `~/${home}`).join(", ")}. Restore one the task needs and capture again.`,
- ]),
+ ...(locks === undefined ? [] : [locks]),
+ ...(unowned === undefined ? [] : [unowned]),
+ ...(runtime === undefined ? [] : [runtime]),
+ ...(trashed === undefined ? [] : [trashed]),
...(facts.identity.hostname === null || facts.identity.user === null
? [
"The capture does not record the host's name and user, so only the captured home is removed from the seed; capture again to remove them too.",
]
: []),
- ...(opened.length === 0
- ? []
- : [
- `The application held files open in ${opened.join(", ")}, whose starting content the capture did not retain, so nothing there is seeded.`,
- ]),
+ ...(opened === undefined ? [] : [opened]),
...facts.collisions.map(
({ home, kept, dropped }) =>
`${dropped} and ${kept} both map to ~/${home}; the export uses ${kept} and leaves out ${dropped}.`,
@@ -379,11 +414,22 @@ const runExport = async (
);
const admitted = await admitDesktopTask(
request.tasks,
- { ...plan, files, identity, ends, seedViews },
+ {
+ ...plan,
+ files,
+ identity,
+ ends,
+ seedViews,
+ seeded: new Set(seeds.keys()),
+ },
signal,
);
const { generated, readme } = bundleText(plan, admitted, dconf.holdsIdentity);
- let scrub: SeedScrubReport = { scrubbed: [], unscrubbed: [] };
+ let scrub: SeedScrubReport = {
+ scrubbed: [],
+ recompressed: [],
+ unscrubbed: [],
+ };
const unclosed = await withRetainedContent(
request.capture,
(content) =>
@@ -403,6 +449,7 @@ const runExport = async (
collisions,
identity,
trashed,
+ files,
});
const result: DesktopExportResult = {
directory: request.directory,
@@ -416,6 +463,7 @@ const runExport = async (
seed_files: plan.seedFiles,
unseeded_files: plan.unseededFiles,
scrubbed_files: [...scrub.scrubbed].sort(),
+ recompressed_files: [...scrub.recompressed].sort(),
unscrubbed_files: [...scrub.unscrubbed].sort(),
dconf_groups: dconf.groups,
files: [
@@ -460,6 +508,7 @@ export const exportDesktopApplication = async (
seed_files: [...result.seed_files],
unseeded_files: [...result.unseeded_files],
scrubbed_files: [...result.scrubbed_files],
+ recompressed_files: [...result.recompressed_files],
unscrubbed_files: [...result.unscrubbed_files],
files: [...result.files],
limitations: [...result.limitations],
diff --git a/src/application/desktop/DesktopApplicationState.test.ts b/src/application/desktop/DesktopApplicationState.test.ts
index b6a1086a8b..17e847f6fd 100644
--- a/src/application/desktop/DesktopApplicationState.test.ts
+++ b/src/application/desktop/DesktopApplicationState.test.ts
@@ -60,6 +60,44 @@ const statePathsOf = async (folder: string, using = readers) => {
return new Map(state.paths.map((entry) => [entry.home_path, entry]));
};
+it("treats both explicitly selected files as documents when one ends in -journal", async () => {
+ const report = `${HOME}/Documents/report`;
+ const journal = `${report}-journal`;
+ const base = extended({
+ baseline: [
+ retained(report, "old report"),
+ retained(journal, "old journal"),
+ ],
+ changes: [
+ change(report, { before: "old report", after: "new report" }),
+ change(journal, { before: "old journal", after: "new journal" }),
+ ],
+ later: [retained(report, "new report"), retained(journal, "new journal")],
+ });
+ const folder = await writeCaptureFolder(
+ {
+ ...base,
+ state: {
+ ...base.state,
+ content_roots: [...base.state.content_roots, report, journal],
+ selected_paths: [report, journal].map((path) => ({
+ path,
+ physical: path,
+ kind: "file" as const,
+ })),
+ },
+ },
+ undefined,
+ ["old report", "old journal", "new report", "new journal"],
+ );
+ const byPath = await statePathsOf(folder);
+ for (const file of ["Documents/report", "Documents/report-journal"])
+ expect(byPath.get(file)).toMatchObject({
+ role: "document",
+ reasons: ["selected_path"],
+ });
+});
+
describe("inspectDesktopApplicationState", () => {
it("assigns roles and reads each version into typed views and changes", async () => {
const folder = await writeCaptureFolder(capture());
diff --git a/src/application/desktop/DesktopApplicationState.ts b/src/application/desktop/DesktopApplicationState.ts
index ff2510a067..15a3326cc1 100644
--- a/src/application/desktop/DesktopApplicationState.ts
+++ b/src/application/desktop/DesktopApplicationState.ts
@@ -12,9 +12,9 @@ import type {
DesktopApplicationState,
DesktopStatePath,
} from "../../domain/desktop/desktopState.js";
+import { openStateDocument } from "../../domain/desktop/gzipDocument.js";
import { diffStateViews } from "../../domain/desktop/stateChanges.js";
import {
- detectStateFormat,
keyfileView,
plainRecord,
SQLITE_SIDECARS,
@@ -24,6 +24,7 @@ import {
applicationNames,
isApplicationDconfGroup,
roleFacts,
+ selectedHomePath,
stateReasons,
stateRole,
} from "../../domain/desktop/stateRoles.js";
@@ -146,7 +147,7 @@ const dconfGroups = (text: string, names: ReadonlySet) => {
* removal keeps the directory's owner and is reported with the read's outcome.
*/
const withStaged = async (
- files: readonly { readonly name: string; readonly bytes: Buffer }[],
+ files: readonly { readonly name: string; readonly bytes: Uint8Array }[],
read: (paths: readonly string[]) => Promise,
): Promise => {
const root = await PrivateRuntimeRoot.create({
@@ -184,10 +185,10 @@ type ViewOutcome =
const failed = (error: string): ViewOutcome => ({ view: null, error });
-/** One retained version's verified bytes, with its format already detected. */
+/** One retained version's verified document, a gzip file's decompressed, with its format already detected. */
interface ReadInput {
readonly path: string;
- readonly bytes: Buffer;
+ readonly bytes: Uint8Array;
readonly version: Version;
}
@@ -340,8 +341,14 @@ const readVersion = async (
version.view_error = `retained content ${bytes.error}`;
return;
}
- version.format = detectStateFormat(path, bytes.value);
- const outcome = await viewOf({ path, bytes: bytes.value, version }, context);
+ const document = openStateDocument(path, bytes.value);
+ version.format = document.format;
+ if (document.compression !== undefined)
+ version.compression = document.compression;
+ const outcome =
+ "error" in document
+ ? failed(document.error)
+ : await viewOf({ path, bytes: document.bytes, version }, context);
version.view = outcome.view;
version.view_error = outcome.error;
};
@@ -377,10 +384,14 @@ const describePath = async (
),
];
const home = capture.host.home;
+ const role = stateRole(reasons, selectedHomePath(facts, path));
return {
path,
home_path: path.startsWith(`${home}/`) ? path.slice(home.length + 1) : null,
- role: stateRole(reasons),
+ role:
+ facts.sqliteSidecars?.has(path) === true && role === "unrelated"
+ ? "application_state"
+ : role,
reasons,
format:
versions.findLast((version) => version.format !== null)?.format ?? null,
@@ -400,6 +411,7 @@ const STATE_LIMITATIONS = [
"JSON and key file views read files of at most 4 MiB, JSON nested at most 500 levels and only numbers JavaScript holds exactly; other files of those formats have a view_error.",
"dconf views keep only groups whose path names the application.",
"Zip views list at most 1,000 entries and read an entry only when it is unencrypted, stored or deflated, at most 16 MiB and at most 100 times its compressed size, up to 64 MiB per archive; databases and archives inside an archive are not read.",
+ "A gzip file's view is its document's, read only when it is at most 16 MiB and at most 100 times the file's size; a database inside one is not read, nor a gzip file inside an archive.",
];
/** Read a capture folder into the application's state view. */
diff --git a/src/application/desktop/DesktopArchiveTask.test.ts b/src/application/desktop/DesktopArchiveTask.test.ts
index 28a0824664..7588c8c3eb 100644
--- a/src/application/desktop/DesktopArchiveTask.test.ts
+++ b/src/application/desktop/DesktopArchiveTask.test.ts
@@ -1,5 +1,6 @@
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
+import { gzipSync } from "node:zlib";
import { describe, expect, it } from "vitest";
@@ -144,6 +145,55 @@ const exportTask = async (folder: string, checks: unknown[]) => {
return { result, directory };
};
+it.each([
+ { seedTitle: "done", accepted: false },
+ { seedTitle: "before", accepted: true },
+])(
+ "checks the recompressed gzip archive's real seed view (accepted: $accepted)",
+ async ({ seedTitle, accepted }) => {
+ const archive = (title: string) =>
+ zipArchive([
+ { name: "state.json", data: JSON.stringify({ title }), method: 0 },
+ { name: "padding.txt", data: "x".repeat(60_000), method: 0 },
+ ]);
+ const seed = gzipSync(archive(seedTitle), { level: 9 });
+ const end = gzipSync(archive("done"), { level: 0 });
+ const folder = await archiveCapture([seed, end]);
+ const state = await inspectDesktopApplicationState(folder, readers);
+ if (!("normalized_result" in state)) throw state;
+ const versions = desktopApplicationStateSchema
+ .parse(state.normalized_result)
+ .paths.find((entry) => entry.home_path === LIST)?.versions;
+ expect(versions?.[0]).toMatchObject({
+ format: null,
+ compression: "gzip",
+ });
+ expect(versions?.at(-1)).toMatchObject({
+ format: "zip",
+ compression: "gzip",
+ });
+ const { result } = await exportTask(folder, [
+ {
+ kind: "value_equals",
+ file: LIST,
+ pointer: "/entries/[name=state.json]/view/title",
+ value: "done",
+ },
+ ]);
+ if (accepted) {
+ if (result instanceof Error) throw result;
+ expect(result.normalized_result).toMatchObject({
+ task: { failing_on_seed: ["check-1"] },
+ });
+ } else {
+ expect(result).toBeInstanceOf(Error);
+ expect((result as Error & { userMessage: string }).userMessage).toContain(
+ "already passes on the seed",
+ );
+ }
+ },
+);
+
describe("documents saved as zip archives", () => {
it("reads each version into its entries and lists the change by entry and element name", async () => {
const evidence = await inspectDesktopApplicationState(
@@ -272,6 +322,57 @@ describe("documents saved as zip archives", () => {
});
});
+describe("task draft values and guards", () => {
+ /** The draft's files, checks and guards, as describe-desktop-application-task returns them. */
+ const draftOf = async (folder: string) => {
+ const drafted = await draftDesktopApplicationTask(folder, readers);
+ if (drafted instanceof Error) throw drafted;
+ return drafted.normalized_result as {
+ files: {
+ file: string;
+ values: { pointer: string; kind: string }[];
+ values_cut: boolean;
+ }[];
+ task: { checks: { id: string }[] };
+ guards: { id: string }[];
+ };
+ };
+
+ it("lists each archive's values by the pointers checks use, and guards the export accepts beside the checks", async () => {
+ const folder = await archiveCapture();
+
+ const { files, task, guards } = await draftOf(folder);
+
+ const plate = files.find(({ file }) => file === PLATE);
+ expect(plate?.values_cut).toBe(false);
+ expect(
+ plate?.values.map(({ kind, pointer }) => [kind, pointer]),
+ ).toContainEqual([
+ "added",
+ "/entries/[name=Document.xml]/view/root/children/[name=Objects]/children/[name=Object][@name=Hole]",
+ ]);
+ expect(guards.map(({ id }) => id)).toEqual(
+ guards.map((_, index) => `guard-${String(index + 1)}`),
+ );
+ expect(guards.length).toBeGreaterThan(0);
+ // Guards hold at the seed and the end, so the task still fails on the seed only through its checks.
+ const { result } = await exportTask(folder, [...task.checks, ...guards]);
+ if (result instanceof Error) throw result;
+ });
+
+ it("points a refused check at the draft's values", async () => {
+ const folder = await archiveCapture();
+
+ const { result } = await exportTask(folder, [{ ...HOLE, value: 7 }]);
+
+ expect(result).toBeInstanceOf(Error);
+ const remediation = (result as Error & { remediationAction?: string })
+ .remediationAction;
+ for (const name of ["values", "stop_build_capture"])
+ expect(remediation).toContain(name);
+ });
+});
+
describe("paragraphs split into runs", () => {
it("drafts and exports a paragraph's whole text once an edit split it into runs, which its first run does not hold", async () => {
const run = (text: string) =>
diff --git a/src/application/desktop/DesktopBuildBundle.ts b/src/application/desktop/DesktopBuildBundle.ts
index 014ad26e06..08495ade24 100644
--- a/src/application/desktop/DesktopBuildBundle.ts
+++ b/src/application/desktop/DesktopBuildBundle.ts
@@ -1,5 +1,12 @@
-import { copyFile, mkdir, readFile, stat, writeFile } from "node:fs/promises";
-import { dirname, isAbsolute, join } from "node:path";
+import {
+ copyFile,
+ mkdir,
+ readdir,
+ readFile,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import { dirname, isAbsolute, join, relative, sep } from "node:path";
import {
renderBuildBundle,
@@ -10,6 +17,7 @@ import {
AnalysisCapabilityUnavailableError,
AnalysisInputError,
} from "../../domain/analysisErrorCore.js";
+import { isLockFile } from "../../domain/desktop/appExportUnowned.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js";
@@ -108,3 +116,31 @@ export const writeRequestBundle = async (
request: BuildRequest,
environment: Readonly,
) => writeBuildBundle(request, await reaPayload(environment));
+
+/**
+ * The lock files in a bundle folder's seeds, as `envs//seed/`; none when it has no seed. A bundle records no state directories, so
+ * only a profile lock's name or LibreOffice's document lock counts here.
+ */
+export const seededLockFiles = async (
+ bundle: string,
+): Promise => {
+ const environments = join(bundle, "envs");
+ const names = await readdir(environments).catch(() => []);
+ const locks: string[] = [];
+ for (const name of names) {
+ const seed = join(environments, name, "seed");
+ const entries = await readdir(seed, {
+ recursive: true,
+ withFileTypes: true,
+ }).catch(() => []);
+ for (const entry of entries) {
+ const home = relative(seed, join(entry.parentPath, entry.name))
+ .split(sep)
+ .join("/");
+ if (entry.isFile() && isLockFile(home, []))
+ locks.push(`envs/${name}/seed/${home}`);
+ }
+ }
+ return locks.sort();
+};
diff --git a/src/application/desktop/DesktopBuildCapture.ts b/src/application/desktop/DesktopBuildCapture.ts
index c5997d61c5..816d5650bd 100644
--- a/src/application/desktop/DesktopBuildCapture.ts
+++ b/src/application/desktop/DesktopBuildCapture.ts
@@ -16,6 +16,11 @@ import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { err, ok, type Result } from "../../domain/result.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import type { ControlledComputer } from "../../desktop/BuildComputerController.js";
+import {
+ endedCapture,
+ notAnswered,
+ type CaptureStep,
+} from "./DesktopBuildCaptureFailure.js";
import {
adapterError,
controller,
@@ -39,11 +44,31 @@ const START = "start_build_capture";
const MARK = "mark_build_checkpoint";
const STOP = "stop_build_capture";
const EXPORT = "export_build_environment";
+/** The start's step, for a capture that ended before the app's window opened. */
+const STARTING: CaptureStep = {
+ operation: START,
+ when: "before the app's window opened",
+ afterAppExit:
+ "The app does not run as installed: check what it needs, such as a runtime its stderr names, or install it from another source on a new build computer, and close this one.",
+};
+/** A checkpoint's step, for a capture that ended before acknowledging it. */
+const marking = (name: string): CaptureStep => ({
+ operation: MARK,
+ when: `before checkpoint ${name}`,
+ afterAppExit:
+ "The capture ended with the app: stop_build_capture reads what it recorded, then start_build_capture captures again.",
+});
const NOT_RUNNING = [
"No capture is running on this computer.",
"Start one with start_build_capture.",
] as const;
+/** What the agent asks of a capture's launch: arguments for the app, and files or folders in the home that are its state. */
+export interface CaptureInput {
+ readonly arguments?: readonly string[] | undefined;
+ readonly state_paths?: readonly string[] | undefined;
+}
+
const exportOptionsSchema = z.object({
options: z.object({ properties: z.record(z.string(), z.unknown()) }),
});
@@ -68,7 +93,9 @@ const checkExportRuntime = async (
{ stdout: path, timeoutS: 30, signal },
);
if (command.exitCode !== 0 && command.stderr !== "")
- return err(schemaCommandError(command.exitCode, command.stderr));
+ return err(
+ schemaCommandError(command.exitCode, computer.redacted(command.stderr)),
+ );
if (command.stdoutBytes > 1024 * 1024)
return err(
adapterError(
@@ -90,12 +117,23 @@ const checkExportRuntime = async (
return err(
schemaCommandError(
command.exitCode,
- content.toString("utf8").slice(-2048),
+ computer.redacted(content.toString("utf8")).slice(-2048),
),
);
- const schema = exportOptionsSchema.safeParse(
- JSON.parse(content.toString("utf8")),
- );
+ let rawSchema: unknown;
+ try {
+ rawSchema = JSON.parse(content.toString("utf8"));
+ } catch {
+ // Parser errors can quote only a fragment of a credential, which cannot
+ // be removed reliably by exact-token redaction.
+ return err(
+ adapterError(
+ START,
+ "The build computer's REA export schema is invalid JSON.",
+ ),
+ );
+ }
+ const schema = exportOptionsSchema.safeParse(rawSchema);
if (!schema.success)
return err(
adapterError(
@@ -113,37 +151,33 @@ const checkExportRuntime = async (
);
return ok(undefined);
} catch (cause: unknown) {
+ if (signal.aborted)
+ return err(new AnalysisCancelledError(START, { cause: signal.reason }));
+ const message = errorText(cause);
+ const redacted = computer.redacted(message);
return err(
- signal.aborted
- ? new AnalysisCancelledError(START, { cause: signal.reason })
- : adapterError(
- START,
- `Could not read the build computer's REA export schema: ${errorText(cause)}`,
- cause,
- ),
+ adapterError(
+ START,
+ `Could not read the build computer's REA export schema: ${redacted}`,
+ redacted === message ? cause : undefined,
+ ),
);
}
};
+/** A taken checkpoint as the tool reports it: changed files null when the REA in the computer predates them. */
type CheckpointTaken = Omit<
Extract,
- "kind"
->;
+ "kind" | "changed_files" | "more_changed_files"
+> & {
+ readonly changed_files: string[] | null;
+ readonly more_changed_files: number | null;
+};
/** The capture's progress event in a line the controller waited for. */
const eventOf = (line: string): DesktopCaptureEvent | undefined =>
captureProgressEvent(line);
-/** Why the capture is not answering, from what the controller saw instead of the event. */
-const notAnswered = (operation: string, answer: Answer): AnalysisError =>
- adapterError(
- operation,
- answer.error ??
- (answer.exited === undefined
- ? `The capture did not answer in time. Its last progress: ${answer.progress ?? ""}`
- : `The capture ended with exit status ${String(answer.exited)}. ${answer.output ?? ""} ${answer.progress ?? ""}`.trim()),
- );
-
/**
* One capture in the computer. `unreleased` is a capture that may be
* running although its start failed, kept until a stop is confirmed;
@@ -204,7 +238,7 @@ export class DesktopBuildCapture {
}
async start(
- input: { readonly arguments?: readonly string[] },
+ input: CaptureInput,
signal: AbortSignal,
): Promise<
Result<
@@ -261,10 +295,20 @@ export class DesktopBuildCapture {
return err(started.error.failure);
}
+ /**
+ * The capture launched, then the caller cancelled before the start was
+ * reported. The next start stops it; until then it is not a second capture.
+ */
+ abandon(): void {
+ const run = this.#run;
+ if (run?.status === "running" && run.stopped === undefined)
+ run.status = "unreleased";
+ }
+
async #launch(
run: CaptureRun,
entry: string,
- input: { readonly arguments?: readonly string[] },
+ input: CaptureInput,
signal: AbortSignal,
): Promise<
Result<
@@ -286,6 +330,7 @@ export class DesktopBuildCapture {
capture: run.folder,
desktop_entry: entry,
arguments: [...(input.arguments ?? [])],
+ state_paths: [...(input.state_paths ?? [])],
...(this.#packageFile === undefined
? {}
: { package_file: BUILD_PACKAGE_FILE }),
@@ -295,11 +340,33 @@ export class DesktopBuildCapture {
{ timeoutS: 720, signal },
);
} catch (cause: unknown) {
- return err({ failure: adapterError(START, errorText(cause), cause) });
+ return err({
+ failure:
+ cause instanceof AnalysisError
+ ? cause
+ : adapterError(START, errorText(cause), cause),
+ });
}
+ if (answer.refused !== undefined)
+ return err({
+ failure: refused(
+ START,
+ answer.refused,
+ "Name files or folders under the computer's home in state_paths, relative to it or absolute, with no link on the way leading out of it.",
+ ),
+ answer,
+ });
const event = answer.line === undefined ? undefined : eventOf(answer.line);
if (event?.kind !== "window_open")
- return err({ failure: notAnswered(START, answer), answer });
+ return err({
+ failure: await endedCapture(
+ this.computer,
+ { ...STARTING, folder: run.folder },
+ answer,
+ signal,
+ ),
+ answer,
+ });
return ok({
desktop_entry: entry,
package_lists: answer.apt?.lists ?? 0,
@@ -352,12 +419,33 @@ export class DesktopBuildCapture {
answer.line === undefined ? undefined : eventOf(answer.line);
if (event?.kind === "checkpoint_refused")
return err(refused(MARK, event.reason, "Use another checkpoint name."));
- if (event?.kind !== "checkpoint") return err(notAnswered(MARK, answer));
+ if (event?.kind !== "checkpoint")
+ return err(
+ await endedCapture(
+ this.computer,
+ { ...marking(name), folder: run.folder },
+ answer,
+ signal,
+ ),
+ );
run.checkpoints.add(name);
- const { kind: _kind, ...taken } = event;
- return ok(taken);
+ const {
+ kind: _kind,
+ changed_files: changed,
+ more_changed_files: more,
+ ...taken
+ } = event;
+ return ok({
+ ...taken,
+ changed_files: changed ?? null,
+ more_changed_files: more ?? null,
+ });
} catch (cause: unknown) {
- return err(adapterError(MARK, errorText(cause), cause));
+ return err(
+ cause instanceof AnalysisError
+ ? cause
+ : adapterError(MARK, errorText(cause), cause),
+ );
}
}
diff --git a/src/application/desktop/DesktopBuildCaptureFailure.ts b/src/application/desktop/DesktopBuildCaptureFailure.ts
new file mode 100644
index 0000000000..c7a92c9d5c
--- /dev/null
+++ b/src/application/desktop/DesktopBuildCaptureFailure.ts
@@ -0,0 +1,176 @@
+import { AnalysisError } from "../../domain/analysisErrorBase.js";
+import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
+import {
+ desktopApplicationCaptureSchema,
+ desktopCaptureProgressSchema,
+ type DesktopApplicationCapture,
+} from "../../domain/desktop/desktopCapture.js";
+import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
+import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
+import type { ControlledComputer } from "../../desktop/BuildComputerController.js";
+import {
+ adapterError,
+ readOutput,
+ reportedFailure,
+ type Answer,
+} from "./DesktopBuildCommands.js";
+
+/** The most lines, and characters, a message quotes from the end of the app's stderr. */
+const STDERR_QUOTE = { lines: 8, characters: 800 } as const;
+/** The most characters a message quotes of a program or a progress line. */
+const LINE_QUOTE = 300;
+
+/** The end of `text`, at most `limit` characters, marked when cut. */
+const ending = (text: string, limit: number): string => {
+ const characters = [...text.trim()];
+ return characters.length <= limit
+ ? characters.join("")
+ : `…${characters.slice(-limit).join("")}`;
+};
+
+/** The last whole lines of the app's stderr within the quote's bounds. */
+const stderrEnding = (stderr: string): string => {
+ const lines = stderr.trim().split("\n");
+ const kept: string[] = [];
+ let size = 0;
+ for (const line of lines.toReversed()) {
+ const grown = size + line.length + 1;
+ if (
+ kept.length === STDERR_QUOTE.lines ||
+ (kept.length > 0 && grown > STDERR_QUOTE.characters)
+ )
+ break;
+ kept.unshift(line);
+ size = grown;
+ }
+ const quoted = ending(kept.join("\n"), STDERR_QUOTE.characters);
+ return kept.length < lines.length ? `…\n${quoted}` : quoted;
+};
+
+/** The capture's last progress message in a tail of its progress log, or its last line. */
+const lastProgress = (progress: string | undefined): string | undefined => {
+ const lines = (progress ?? "")
+ .split("\n")
+ .map((line) => line.trim())
+ .filter((line) => line.length > 0);
+ for (let index = lines.length - 1; index >= 0; index -= 1) {
+ try {
+ const parsed = desktopCaptureProgressSchema.safeParse(
+ JSON.parse(lines[index] ?? ""),
+ );
+ if (parsed.success) return parsed.data.rea_progress.message;
+ } catch {
+ // A line cut by the tail, or one the capture's runtime wrote.
+ }
+ }
+ return lines.at(-1);
+};
+
+/** Why the capture is not answering, from what the controller saw instead of the event; its output stays out of the message. */
+export const notAnswered = (
+ operation: string,
+ answer: Answer,
+): AnalysisError => {
+ if (answer.error !== undefined) return adapterError(operation, answer.error);
+ const last = lastProgress(answer.progress);
+ const progress =
+ last === undefined ? "" : ` Its last progress: ${ending(last, LINE_QUOTE)}`;
+ return adapterError(
+ operation,
+ answer.exited === undefined
+ ? `The capture did not answer in time.${progress}`
+ : `The capture ended with exit status ${String(answer.exited)} before it answered.${progress}`,
+ );
+};
+
+/** The step a capture that ended did not answer, as its errors name it. */
+export interface CaptureStep {
+ readonly operation: string;
+ /** When the capture failed, after "The capture failed", such as "before checkpoint seed". */
+ readonly when: string;
+ /** What to do when the app itself exited and so ended the capture. */
+ readonly afterAppExit: string;
+}
+
+/** The app exited and so ended the capture: how it exited and what it said last, with the capture's record kept as details. */
+const appExited = (
+ step: CaptureStep,
+ recorded: DesktopApplicationCapture,
+ exitStatus: number,
+): AnalysisError | undefined => {
+ const { application, session } = recorded;
+ if (session.end_reason !== "application_exited") return undefined;
+ const { exit } = session;
+ const ended =
+ exit === null
+ ? "ended"
+ : exit.signal === null
+ ? `exited with code ${String(exit.code)}`
+ : `was ended by ${exit.signal}`;
+ const window =
+ session.first_window_ms === null ? " without opening a window" : "";
+ const stderr =
+ session.stderr.trim() === ""
+ ? "it wrote nothing to stderr"
+ : `its stderr ends: ${stderrEnding(session.stderr)}`;
+ return new ProviderAdapterError(AGENTENV_PROVIDER, step.operation, {
+ userMessage: `${ending(application.argv[0] ?? "The app", LINE_QUOTE)} ${ended} after ${String(session.duration_ms)} ms${window}; ${stderr}`,
+ remediationAction: step.afterAppExit,
+ diagnostics: {
+ capture_exit_status: exitStatus,
+ application: { argv: [...application.argv] },
+ session: {
+ end_reason: session.end_reason,
+ duration_ms: session.duration_ms,
+ exit: session.exit,
+ first_window_ms: session.first_window_ms,
+ stderr: session.stderr,
+ stderr_truncated: session.stderr_truncated,
+ },
+ limitations: [...recorded.limitations],
+ },
+ });
+};
+
+/**
+ * Why a capture step got no answer. When the capture has ended, its own
+ * output says how, read the same way for every step that waits on it: a
+ * failure it reported, with its details, or the app exiting; otherwise, or
+ * when that output cannot be read, what the controller saw.
+ */
+export const endedCapture = async (
+ computer: ControlledComputer,
+ step: CaptureStep & { readonly folder: string },
+ answer: Answer,
+ signal: AbortSignal,
+): Promise => {
+ if (answer.exited === undefined || answer.output_bytes === undefined)
+ return notAnswered(step.operation, answer);
+ try {
+ const output = await readOutput(
+ computer,
+ `${step.folder}/capture.out`,
+ answer.output_bytes,
+ signal,
+ );
+ if (output.kind === "failure")
+ return reportedFailure(
+ step.operation,
+ output,
+ `The capture failed ${step.when}: `,
+ );
+ const recorded = desktopApplicationCaptureSchema.safeParse(output.result);
+ return (
+ (recorded.success
+ ? appExited(step, recorded.data, answer.exited)
+ : undefined) ?? notAnswered(step.operation, answer)
+ );
+ } catch (cause: unknown) {
+ if (cause instanceof AnalysisError) return cause;
+ if (signal.aborted)
+ return new AnalysisCancelledError(step.operation, {
+ cause: signal.reason,
+ });
+ return notAnswered(step.operation, answer);
+ }
+};
diff --git a/src/application/desktop/DesktopBuildCommands.ts b/src/application/desktop/DesktopBuildCommands.ts
index 356817e6a0..ac4fb4f4e2 100644
--- a/src/application/desktop/DesktopBuildCommands.ts
+++ b/src/application/desktop/DesktopBuildCommands.ts
@@ -6,7 +6,16 @@ import {
SEED_CHECKPOINT,
} from "../../domain/desktop/buildCapture.js";
import { parseEvidence } from "../../domain/evidence.js";
+import {
+ jsonObjectSchema,
+ jsonValueSchema,
+ type JsonValue,
+} from "../../domain/jsonValue.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
+import {
+ guestFileAnswerCeiling,
+ guestFileAnswerRefusal,
+} from "../../desktop/AgentEnvComputer.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import {
runRea as runControllerRea,
@@ -32,15 +41,19 @@ const cliFailureSchema = z.looseObject({
code: z.string(),
message: z.string(),
remediation: z.looseObject({ action: z.string() }).optional(),
+ details: z.record(z.string(), jsonValueSchema).optional(),
});
+/** A failure a REA command in the computer reported, and the whole projection it reported it as. */
+export interface CommandFailure {
+ readonly failure: z.infer;
+ readonly reported: Readonly>;
+}
+
/** What REA reads from a command it ran in the computer: its Evidence result, or the failure it reported. */
export type CommandOutput =
| { readonly kind: "result"; readonly result: unknown }
- | {
- readonly kind: "failure";
- readonly failure: z.infer;
- };
+ | ({ readonly kind: "failure" } & CommandFailure);
export const errorText = (cause: unknown): string =>
cause instanceof Error ? cause.message : String(cause);
@@ -55,6 +68,44 @@ export const adapterError = (
...(cause === undefined ? {} : { cause }),
});
+const cleanupTargetsSchema = z.object({
+ cleanup: z.literal("incomplete"),
+ resources: z.array(z.string()),
+ cleanup_reason: z.string().optional(),
+});
+
+/**
+ * A failure a REA command in the computer reported, as this step's error:
+ * its message after `lead`, its remediation, the whole failure under
+ * `details.diagnostics.command_failure`, and an incomplete cleanup's targets,
+ * such as run tokens and paths, kept as this error's own, since its
+ * remediation names them as listed.
+ */
+export const reportedFailure = (
+ operation: string,
+ { failure, reported }: CommandFailure,
+ lead = "",
+): ProviderAdapterError => {
+ const cleanup = cleanupTargetsSchema.safeParse(failure.details);
+ return new ProviderAdapterError(AGENTENV_PROVIDER, operation, {
+ userMessage: `${lead}${failure.message}`,
+ ...(failure.remediation === undefined
+ ? {}
+ : { remediationAction: failure.remediation.action }),
+ diagnostics: { command_failure: reported },
+ ...(cleanup.success
+ ? {
+ cleanup: {
+ reason:
+ cleanup.data.cleanup_reason ??
+ "the command's cleanup is incomplete",
+ resources: cleanup.data.resources,
+ },
+ }
+ : {}),
+ });
+};
+
/** A valid request the build's current state cannot take, with what to do instead. */
export const refused = (
operation: string,
@@ -80,18 +131,53 @@ export const runRea = (
): ReturnType =>
runControllerRea(computer, argv, { ...options, work: WORK });
-/** The Evidence result, or the reported failure, a REA command wrote to a file of `bytes` in the computer. */
+/** `value` with `redact` applied to each of its strings and keys. */
+const redactedJson = (
+ value: JsonValue,
+ redact: (text: string) => string,
+): JsonValue => {
+ if (typeof value === "string") return redact(value);
+ if (Array.isArray(value))
+ return value.map((item) => redactedJson(item, redact));
+ if (value === null || typeof value !== "object") return value;
+ return Object.fromEntries(
+ Object.entries(value).map(([key, item]) => [
+ redact(key),
+ redactedJson(item, redact),
+ ]),
+ );
+};
+
+/**
+ * The Evidence result, or the reported failure, a REA command wrote to a file
+ * of `bytes` in the computer, with the control token replaced in every
+ * string, as in a command's own output; the Evidence is checked first, since
+ * its digest covers what the command wrote.
+ */
export const readOutput = async (
computer: ControlledComputer,
path: string,
bytes: number,
signal: AbortSignal,
): Promise => {
+ const refusal = guestFileAnswerRefusal(bytes, 1, guestFileAnswerCeiling());
+ if (refusal !== undefined) throw refusal;
const content = (await computer.homeFiles([path], bytes, signal)).get(path);
if (content === undefined) throw new Error(`The computer has no ${path}.`);
const value: unknown = JSON.parse(content.toString("utf8"));
- const failure = cliFailureSchema.safeParse(value);
- return failure.success
- ? { kind: "failure", failure: failure.data }
- : { kind: "result", result: parseEvidence(value).normalized_result };
+ const redact = (text: string) => computer.redacted(text);
+ if (cliFailureSchema.safeParse(value).success) {
+ const reported = jsonObjectSchema.parse(
+ redactedJson(jsonObjectSchema.parse(value), redact),
+ );
+ return {
+ kind: "failure",
+ failure: cliFailureSchema.parse(reported),
+ reported,
+ };
+ }
+ return {
+ kind: "result",
+ result: redactedJson(parseEvidence(value).normalized_result, redact),
+ };
};
diff --git a/src/application/desktop/DesktopBuildComputers.ts b/src/application/desktop/DesktopBuildComputers.ts
index 9277cbe9c2..aa2e93a7df 100644
--- a/src/application/desktop/DesktopBuildComputers.ts
+++ b/src/application/desktop/DesktopBuildComputers.ts
@@ -1,6 +1,8 @@
import { randomUUID } from "node:crypto";
import type { BuildSource } from "../../domain/desktop/appBuild.js";
+import { counted } from "../../domain/desktop/appExportUnowned.js";
+import { SETTLE_TIMING } from "../../domain/desktop/appSettle.js";
import {
driverCall,
type ComputerAction,
@@ -23,6 +25,7 @@ import {
AgentEnvComputer,
AppDataRefusal,
ComputerStartFailure,
+ computerFailedOnReadiness,
type DriverResult,
} from "../../desktop/AgentEnvComputer.js";
import {
@@ -36,8 +39,9 @@ import {
BuildComputerActionLog,
type ReadonlyActionLog,
} from "./DesktopBuildActionLog.js";
-import { writeRequestBundle } from "./DesktopBuildBundle.js";
+import { seededLockFiles, writeRequestBundle } from "./DesktopBuildBundle.js";
import { DesktopBuildCapture } from "./DesktopBuildCapture.js";
+import type { BuildSettle } from "./DesktopBuildSettle.js";
import { DesktopBuildTurns } from "./DesktopBuildTurns.js";
import { DesktopBuildWitness } from "./DesktopBuildWitness.js";
@@ -195,6 +199,7 @@ const liveComputer = (
computer: AgentEnvComputer,
workspace: PrivateRuntimeRoot | undefined,
source: BuildSource,
+ settle: BuildSettle,
): HeldComputer => {
const actions = new BuildComputerActionLog();
const turns = new DesktopBuildTurns();
@@ -212,6 +217,7 @@ const liveComputer = (
witness: new DesktopBuildWitness(
computer,
actions,
+ settle,
source.kind === "bundle" ? source.path : undefined,
),
turns,
@@ -221,6 +227,32 @@ const liveComputer = (
};
};
+/**
+ * A bundle's start that failed on readiness, naming the lock files its seed
+ * holds, which an app that finds its profile or document in use may never
+ * start past; any other failure as it is. Readiness is `reason`, not the
+ * message.
+ */
+/** The start failure's notice that the bundle seed still holds lock files. */
+export const seededLockStartNotice = (locks: readonly string[]): string =>
+ `The bundle's seed holds lock files, which can keep the app from starting when it finds its profile or document in use: ${counted(locks)}. Delete them from the bundle, or export it again, which leaves them out, and start again.`;
+
+const withSeededLocks = async (
+ source: BuildSource,
+ cause: unknown,
+): Promise => {
+ if (source.kind !== "bundle" || !computerFailedOnReadiness(cause))
+ return cause;
+ const locks = await seededLockFiles(source.path);
+ return locks.length === 0
+ ? cause
+ : new ComputerStartFailure(
+ `${cause.message} ${seededLockStartNotice(locks)}`,
+ cause.remediation,
+ { cause: cause.cause, reason: cause.reason },
+ );
+};
+
/**
* Why a start failed, with the helper's output, and, when its teardown is
* unconfirmed, the id REA keeps that teardown under for close to retry.
@@ -274,7 +306,10 @@ export class DesktopBuildComputers {
readonly #starts = new Set();
#closed = false;
- constructor(private readonly environment: Readonly) {}
+ constructor(
+ private readonly environment: Readonly,
+ private readonly settle: BuildSettle = SETTLE_TIMING,
+ ) {}
async start(
input: {
@@ -367,7 +402,10 @@ export class DesktopBuildComputers {
shot.value.arguments,
signal,
);
- this.#computers.set(id, liveComputer(computer, target.workspace, source));
+ this.#computers.set(
+ id,
+ liveComputer(computer, target.workspace, source, this.settle),
+ );
return ok({
value: {
computer_id: id,
@@ -395,7 +433,12 @@ export class DesktopBuildComputers {
closing: true,
});
const release = await this.#retire(id);
- return err(startFailure(cause, signal, owner?.output(), { id, release }));
+ const failed = signal.aborted
+ ? cause
+ : await withSeededLocks(source, cause);
+ return err(
+ startFailure(failed, signal, owner?.output(), { id, release }),
+ );
}
}
diff --git a/src/application/desktop/DesktopBuildExport.ts b/src/application/desktop/DesktopBuildExport.ts
index becee0b379..98203190e2 100644
--- a/src/application/desktop/DesktopBuildExport.ts
+++ b/src/application/desktop/DesktopBuildExport.ts
@@ -17,6 +17,7 @@ import {
errorText,
readOutput,
refused,
+ reportedFailure,
runRea,
} from "./DesktopBuildCommands.js";
import {
@@ -25,13 +26,12 @@ import {
} from "./DesktopOutputTree.js";
const EXPORT = "export_build_environment";
-/** The most bytes the task takes in the controller's one JSON argument, which Linux caps at 128 KiB. */
-const TASK_ARGUMENT_BYTES = 96 * 1024;
-
const exportResultSchema = z.looseObject({
environment: z.string(),
seed_files: z.array(z.string()),
scrubbed_files: z.array(z.string()),
+ // An export from before gzip seeds were listed apart has none.
+ recompressed_files: z.array(z.string()).default([]),
unscrubbed_files: z.array(z.string()),
limitations: z.array(z.string()),
task: jsonValueSchema,
@@ -53,6 +53,7 @@ export type ExportedBuild = {
readonly task: JsonValue;
readonly seed_files: string[];
readonly scrubbed_files: string[];
+ readonly recompressed_files: string[];
readonly unscrubbed_files: string[];
readonly limitations: string[];
};
@@ -127,15 +128,6 @@ export const exportBuild = async (
const task = `${folder}/task-${attempt}.json`;
const stdout = `${folder}/export-${attempt}.json`;
const taskFile = `${JSON.stringify(request.task, null, 2)}\n`;
- const taskBytes = Buffer.byteLength(JSON.stringify(taskFile));
- if (taskBytes > TASK_ARGUMENT_BYTES)
- return err(
- refused(
- EXPORT,
- `The task takes ${String(taskBytes)} bytes in the command REA runs in the computer, more than the ${String(TASK_ARGUMENT_BYTES)} it allows a task.`,
- "Shorten the task's prompt or checks and export again.",
- ),
- );
try {
const { stdoutBytes: bytes } = await runRea(
computer,
@@ -170,7 +162,7 @@ export const exportBuild = async (
output.failure.remediation?.action ??
"Rewrite the task and export again.",
)
- : adapterError(EXPORT, output.failure.message),
+ : reportedFailure(EXPORT, output),
);
const result = exportResultSchema.parse(output.result);
const written = await writeBundle(
@@ -188,6 +180,7 @@ export const exportBuild = async (
task: result.task,
seed_files: result.seed_files,
scrubbed_files: result.scrubbed_files,
+ recompressed_files: result.recompressed_files,
unscrubbed_files: result.unscrubbed_files,
limitations: result.limitations,
});
diff --git a/src/application/desktop/DesktopBuildOutputRead.test.ts b/src/application/desktop/DesktopBuildOutputRead.test.ts
new file mode 100644
index 0000000000..43d779fc04
--- /dev/null
+++ b/src/application/desktop/DesktopBuildOutputRead.test.ts
@@ -0,0 +1,123 @@
+import { describe, expect, it } from "vitest";
+
+import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
+import {
+ GUEST_FILE_ANSWER_SOURCE,
+ guestFileAnswerCeiling,
+} from "../../desktop/AgentEnvComputer.js";
+import { readOutput } from "./DesktopBuildCommands.js";
+import { endedCapture } from "./DesktopBuildCaptureFailure.js";
+
+describe("endedCapture", () => {
+ it("returns the size refusal instead of a generic exit when the capture output is too large", async () => {
+ const error = await endedCapture(
+ {
+ homeFiles: () => Promise.resolve(new Map()),
+ redacted: (text) => text,
+ exec: () => Promise.reject(new Error("unused")),
+ },
+ {
+ operation: "desktop.capture",
+ when: "before checkpoint seed",
+ afterAppExit: "unused",
+ folder: "/tmp/capture",
+ },
+ { exited: 0, output_bytes: Number.MAX_SAFE_INTEGER },
+ new AbortController().signal,
+ );
+
+ expect(error).toBeInstanceOf(AnalysisResourceConstraintError);
+ expect(error.message).toContain("max-old-space-size");
+ });
+});
+
+describe("readOutput", () => {
+ it("does not fetch a command output whose reported size exceeds the host answer ceiling", async () => {
+ let fetched = false;
+ const failure = await readOutput(
+ {
+ homeFiles: () => {
+ fetched = true;
+ return Promise.resolve(new Map());
+ },
+ redacted: (text) => text,
+ exec: () => Promise.reject(new Error("unused")),
+ },
+ "out.json",
+ Number.MAX_SAFE_INTEGER,
+ new AbortController().signal,
+ ).catch((cause: unknown) => cause);
+ expect(fetched).toBe(false);
+ expect(failure).toBeInstanceOf(AnalysisResourceConstraintError);
+ if (!(failure instanceof AnalysisResourceConstraintError)) return;
+ expect(failure.reportedLimits).toMatchObject({
+ guest_file_bytes: Number.MAX_SAFE_INTEGER,
+ file_count: 1,
+ answer_ceiling_bytes: guestFileAnswerCeiling(),
+ ceiling_source: GUEST_FILE_ANSWER_SOURCE,
+ });
+ });
+
+ it("reads a reported failure that fits the ceiling", async () => {
+ const body = Buffer.from(
+ JSON.stringify({ code: "busy", message: "the app is busy" }),
+ );
+ let reported = 0;
+ const output = await readOutput(
+ {
+ homeFiles: (_paths, bytes) => {
+ reported = bytes;
+ return Promise.resolve(new Map([["out.json", body]]));
+ },
+ redacted: (text) => text,
+ exec: () => Promise.reject(new Error("unused")),
+ },
+ "out.json",
+ body.length,
+ new AbortController().signal,
+ );
+ expect(reported).toBe(body.length);
+ expect(output.kind).toBe("failure");
+ if (output.kind !== "failure") return;
+ expect(output.failure).toMatchObject({
+ code: "busy",
+ message: "the app is busy",
+ });
+ });
+});
+
+describe("endedCapture", () => {
+ it("keeps a capacity error from the capture output, including its recovery", async () => {
+ let fetched = false;
+ const failure = await endedCapture(
+ {
+ homeFiles: () => {
+ fetched = true;
+ return Promise.resolve(new Map());
+ },
+ redacted: (text) => text,
+ exec: () => Promise.reject(new Error("unused")),
+ },
+ {
+ operation: "desktop_build_capture_start",
+ when: "before checkpoint seed",
+ afterAppExit: "Start the capture again.",
+ folder: "capture",
+ },
+ { exited: 0, output_bytes: Number.MAX_SAFE_INTEGER },
+ new AbortController().signal,
+ );
+ expect(fetched).toBe(false);
+ expect(failure).toBeInstanceOf(AnalysisResourceConstraintError);
+ if (!(failure instanceof AnalysisResourceConstraintError)) return;
+ expect(failure.operation).toBe("data/get");
+ expect(failure.retryable).toBe(false);
+ expect(failure.remediationAction).toContain("--max-old-space-size");
+ expect(failure.reportedLimits).toMatchObject({
+ guest_file_bytes: Number.MAX_SAFE_INTEGER,
+ file_count: 1,
+ answer_ceiling_bytes: guestFileAnswerCeiling(),
+ ceiling_source: GUEST_FILE_ANSWER_SOURCE,
+ });
+ });
+});
diff --git a/src/application/desktop/DesktopBuildSettle.ts b/src/application/desktop/DesktopBuildSettle.ts
new file mode 100644
index 0000000000..8d523635c6
--- /dev/null
+++ b/src/application/desktop/DesktopBuildSettle.ts
@@ -0,0 +1,97 @@
+import { setTimeout as delay } from "node:timers/promises";
+
+import {
+ appFileDigests,
+ type SettleTiming,
+} from "../../domain/desktop/appSettle.js";
+
+/** The time a settle keeps; tests substitute a virtual clock. */
+export interface SettleClock {
+ now(): number;
+ delay(ms: number, signal: AbortSignal): Promise;
+ /** A signal that aborts once `ms` have passed. */
+ timeout(ms: number): AbortSignal;
+}
+
+const monotonicClock: SettleClock = {
+ now: () => performance.now(),
+ delay: (ms, signal) => delay(ms, undefined, { signal }),
+ // A timeout takes whole milliseconds.
+ timeout: (ms) => AbortSignal.timeout(Math.max(1, Math.floor(ms))),
+};
+
+/** How a build computer reads an app until its files settle, on real time unless a clock is given. */
+export interface BuildSettle extends SettleTiming {
+ readonly clock?: SettleClock;
+}
+
+/**
+ * The last completed data/get answer of a settled read, whether the app's
+ * files had held for the window by then, and whether the bound cut a read
+ * still in flight.
+ */
+export interface SettledRead {
+ readonly parts: readonly unknown[];
+ readonly settled: boolean;
+ readonly readCut: boolean;
+}
+
+const seconds = (ms: number) => `${(ms / 1000).toFixed(1)} s`;
+
+/**
+ * Read an app's data/get every interval until each file's bytes have held
+ * for the window, counted from the read that returned them, or until the
+ * bound, as the exported verifier does; `waiting` hears why it reads again.
+ * No read starts at or past the bound, and one in flight there is cut.
+ */
+export const settledAppData = async (
+ read: (signal: AbortSignal) => Promise,
+ timing: BuildSettle,
+ signal: AbortSignal,
+ waiting: (message: string) => Promise,
+): Promise => {
+ const clock = timing.clock ?? monotonicClock;
+ const start = clock.now();
+ const elapsed = () => clock.now() - start;
+ // The first read is the one a grade needs, so only its own timeout bounds it.
+ let parts = await read(signal);
+ let files = appFileDigests(parts);
+ // Offsets from the first read's start: when the last completed read began,
+ // and when a read first returned the bytes it holds.
+ let began = 0;
+ let since = elapsed();
+ const unsettled = () => began - since < timing.windowMs;
+ while (unsettled()) {
+ // Reads come an interval apart, and the one that can settle the files as soon as the window allows.
+ const next = Math.max(
+ Math.min(began + timing.intervalMs, since + timing.windowMs),
+ elapsed(),
+ );
+ if (next >= timing.boundMs)
+ return { parts, settled: false, readCut: false };
+ await clock.delay(Math.max(0, next - elapsed()), signal);
+ const at = Math.max(next, elapsed());
+ if (at >= timing.boundMs) return { parts, settled: false, readCut: false };
+ const bound = clock.timeout(timing.boundMs - at);
+ let latest: readonly unknown[];
+ try {
+ latest = await read(AbortSignal.any([signal, bound]));
+ } catch (cause: unknown) {
+ if (bound.aborted && !signal.aborted)
+ return { parts, settled: false, readCut: true };
+ throw cause;
+ }
+ began = at;
+ parts = latest;
+ const digests = appFileDigests(latest);
+ if (digests !== files) {
+ since = elapsed();
+ files = digests;
+ }
+ if (unsettled())
+ await waiting(
+ `waiting for the app's files to hold for ${seconds(timing.windowMs)}: unchanged for ${seconds(Math.max(0, began - since))}, ${seconds(began)} of at most ${seconds(timing.boundMs)}`,
+ );
+ }
+ return { parts, settled: true, readCut: false };
+};
diff --git a/src/application/desktop/DesktopBuildSteps.ts b/src/application/desktop/DesktopBuildSteps.ts
index bfd82865d8..ceff338b5e 100644
--- a/src/application/desktop/DesktopBuildSteps.ts
+++ b/src/application/desktop/DesktopBuildSteps.ts
@@ -5,6 +5,7 @@ import {
} from "../../domain/analysisErrorCore.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
+import type { ProgressReporter } from "../ProgressReporter.js";
import type { StoppedCapture } from "./DesktopBuildSummary.js";
import type {
BuildComputerSteps,
@@ -41,6 +42,7 @@ export class DesktopBuildSteps {
input: {
readonly computer_id: string;
readonly arguments?: readonly string[] | undefined;
+ readonly state_paths?: readonly string[] | undefined;
},
signal: AbortSignal,
): Promise<
@@ -60,21 +62,21 @@ export class DesktopBuildSteps {
if (steps === undefined) return err(unknownComputer("start_build_capture"));
// The screenshot of the opened app takes the same turn as the start.
return steps.turns.run("start_build_capture", signal, async (turn) => {
- const started = await steps.capture.start(
- input.arguments === undefined ? {} : { arguments: input.arguments },
- turn,
- );
+ const { computer_id: _id, ...launch } = input;
+ const started = await steps.capture.start(launch, turn);
if (!started.ok) return started;
// A failed screenshot only leaves it out, but a close or abort meanwhile cancels the start: it saved nothing to report.
const shot = await steps.computer
.callTool("computer_screenshot", {}, turn)
.catch(() => undefined);
- if (turn.aborted)
+ if (turn.aborted) {
+ steps.capture.abandon();
return err(
new AnalysisCancelledError("start_build_capture", {
cause: turn.reason,
}),
);
+ }
return ok({
value: {
computer_id: input.computer_id,
@@ -131,10 +133,11 @@ export class DesktopBuildSteps {
readonly computer_id: string;
},
signal: AbortSignal,
+ progress?: ProgressReporter,
): Promise, AnalysisError>> {
const { computer_id: id, ...request } = input;
return this.#on(id, "check_build_task", signal, (steps, turn) =>
- steps.witness.check(request, turn),
+ steps.witness.check(request, turn, progress),
);
}
diff --git a/src/application/desktop/DesktopBuildSummary.ts b/src/application/desktop/DesktopBuildSummary.ts
index a7c63a08c3..e0b83a202d 100644
--- a/src/application/desktop/DesktopBuildSummary.ts
+++ b/src/application/desktop/DesktopBuildSummary.ts
@@ -6,6 +6,8 @@ import {
SEED_CHECKPOINT,
} from "../../domain/desktop/buildCapture.js";
import { installsFromPackageFile } from "../../domain/desktop/appExportInstall.js";
+import { draftNoteSchema } from "../../domain/desktop/appTaskDraftNotes.js";
+import { draftValueSchema } from "../../domain/desktop/appTaskDraftValues.js";
import { desktopApplicationCaptureSchema } from "../../domain/desktop/desktopCapture.js";
import {
jsonObjectSchema,
@@ -15,9 +17,9 @@ import {
import { err, ok, type Result } from "../../domain/result.js";
import type { ControlledComputer } from "../../desktop/BuildComputerController.js";
import {
- adapterError,
ENDS,
readOutput,
+ reportedFailure,
runRea,
} from "./DesktopBuildCommands.js";
@@ -29,6 +31,8 @@ const draftResultSchema = z.looseObject({
file: z.string(),
role: z.string(),
change: jsonValueSchema,
+ values: z.array(draftValueSchema).default([]),
+ values_cut: z.boolean().default(false),
}),
),
task: z.looseObject({
@@ -36,6 +40,82 @@ const draftResultSchema = z.looseObject({
prompt: z.string(),
checks: z.array(jsonObjectSchema),
}),
+ guards: z.array(jsonObjectSchema).default([]),
+ notes: z.array(draftNoteSchema).default([]),
+});
+
+/** A drafted file's value as the stop returns it. */
+type DraftValue = {
+ readonly pointer: string;
+ readonly kind: string;
+ readonly before: JsonValue;
+ readonly after: JsonValue;
+ readonly note?: string;
+ readonly refers_to?: {
+ readonly attribute?: string;
+ readonly value: string;
+ readonly element: string;
+ }[];
+};
+
+/** A draft's note as the stop returns it. */
+type DraftNote = {
+ readonly code: string;
+ readonly message: string;
+ readonly file?: string;
+ readonly files?: { readonly file: string; readonly role: string }[];
+ readonly more?: number;
+ readonly elements?: string[];
+ readonly pointers?: string[];
+ readonly error?: string;
+};
+
+/** A value without the optional fields the draft left out. */
+const draftValue = ({
+ pointer,
+ kind,
+ before,
+ after,
+ note,
+ refers_to,
+}: z.infer): DraftValue => ({
+ pointer,
+ kind,
+ before,
+ after,
+ ...(note === undefined ? {} : { note }),
+ ...(refers_to === undefined
+ ? {}
+ : {
+ refers_to: refers_to.map(({ attribute, value, element }) => ({
+ ...(attribute === undefined ? {} : { attribute }),
+ value,
+ element,
+ })),
+ }),
+});
+
+/** A note without the optional fields the draft left out. */
+const draftNote = ({
+ code,
+ message,
+ file,
+ files,
+ more,
+ elements,
+ pointers,
+ error,
+}: z.infer): DraftNote => ({
+ code,
+ message,
+ ...(file === undefined ? {} : { file }),
+ ...(files === undefined
+ ? {}
+ : { files: files.map(({ file: home, role }) => ({ file: home, role })) }),
+ ...(more === undefined ? {} : { more }),
+ ...(elements === undefined ? {} : { elements }),
+ ...(pointers === undefined ? {} : { pointers }),
+ ...(error === undefined ? {} : { error }),
});
/** A stopped capture: its checkpoints, what it found of the app and the draft between seed and end. */
@@ -61,12 +141,19 @@ export type StoppedCapture = {
readonly file: string;
readonly role: string;
readonly change: JsonValue;
+ /** The values the change added, changed and removed, at the pointers a check names them by. */
+ readonly values: DraftValue[];
+ readonly values_cut: boolean;
}[];
readonly task: {
readonly name: string;
readonly prompt: string;
readonly checks: Readonly>[];
};
+ /** Checks that the rest stays as it was, outside the task until an author adds them. */
+ readonly guards: Readonly>[];
+ /** Why the draft has no files, or why a file's suggestions are thin. */
+ readonly notes: DraftNote[];
} | null;
readonly draft_refusal?: string;
};
@@ -105,16 +192,20 @@ const draftTask = async (
draft: {
seed_checkpoint: SEED_CHECKPOINT,
end_checkpoint: END_CHECKPOINT,
- files: draft.files.map(({ file, role, change }) => ({
+ files: draft.files.map(({ file, role, change, values, values_cut }) => ({
file,
role,
change,
+ values: values.map(draftValue),
+ values_cut,
})),
task: {
name: draft.task.name,
prompt: draft.task.prompt,
checks: draft.task.checks,
},
+ guards: draft.guards,
+ notes: draft.notes.map(draftNote),
},
};
};
@@ -147,9 +238,7 @@ export const summarizeCapture = async (
signal,
);
if (output.kind === "failure")
- return err(
- adapterError(STOP, `The capture failed: ${output.failure.message}`),
- );
+ return err(reportedFailure(STOP, output, "The capture failed: "));
const recorded = desktopApplicationCaptureSchema.parse(output.result);
const checkpoints = new Set(
recorded.state.checkpoints.map((checkpoint) => checkpoint.name),
diff --git a/src/application/desktop/DesktopBuildWitness.test.ts b/src/application/desktop/DesktopBuildWitness.test.ts
index 5e836e4ebf..911d7cfad1 100644
--- a/src/application/desktop/DesktopBuildWitness.test.ts
+++ b/src/application/desktop/DesktopBuildWitness.test.ts
@@ -7,6 +7,7 @@ import {
AnalysisCancelledError,
AnalysisInputError,
} from "../../domain/analysisErrorCore.js";
+import { SETTLE_TIMING } from "../../domain/desktop/appSettle.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { BuildComputerActionLog } from "./DesktopBuildActionLog.js";
import {
@@ -34,7 +35,12 @@ const computer = {
} as WitnessedComputer;
const witnessAt = async (bundle: string) =>
- new DesktopBuildWitness(computer, new BuildComputerActionLog(), bundle);
+ new DesktopBuildWitness(
+ computer,
+ new BuildComputerActionLog(),
+ SETTLE_TIMING,
+ bundle,
+ );
describe("bundle task reads", () => {
it("returns a cancelled result when reading the bundle's task is aborted", async () => {
diff --git a/src/application/desktop/DesktopBuildWitness.ts b/src/application/desktop/DesktopBuildWitness.ts
index b77bfd0951..bd3aaabddd 100644
--- a/src/application/desktop/DesktopBuildWitness.ts
+++ b/src/application/desktop/DesktopBuildWitness.ts
@@ -12,6 +12,7 @@ import {
type desktopTaskSchema,
type DesktopTask,
} from "../../domain/desktop/appTask.js";
+import { SETTLE_TIMING } from "../../domain/desktop/appSettle.js";
import {
evaluateTaskChecks,
type TaskCheckOutcome,
@@ -19,6 +20,7 @@ import {
import {
renderTaskSteps,
renderTaskVerifier,
+ sameTaskVerifier,
taskVerifierArtifact,
} from "../../domain/desktop/appTaskPython.js";
import {
@@ -42,9 +44,14 @@ import type {
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import { readHostFile } from "../../desktop/HostFileRead.js";
import { isPortableBundleName } from "../environment/environmentBundle.js";
+import {
+ silentProgressReporter,
+ type ProgressReporter,
+} from "../ProgressReporter.js";
import type { ReadonlyActionLog } from "./DesktopBuildActionLog.js";
import { errorText } from "./DesktopBuildCommands.js";
import { desktopOperationFailure } from "./DesktopCaptureRefusals.js";
+import { settledAppData, type BuildSettle } from "./DesktopBuildSettle.js";
import {
writeWitnessRecord,
type EndedWitness,
@@ -70,12 +77,23 @@ type WitnessSummary = {
readonly record: WitnessRecord | null;
};
+/** A task's results on the app's views, whether its files had settled, and whether the bound cut a read. */
+type Graded = {
+ readonly results: TaskCheckOutcome[];
+ readonly settled: boolean;
+ readonly read_cut: boolean;
+};
+
/** One check of the task against the app's live state, and the witness stage it filed, if any. */
export type CheckedTask = {
readonly task: string;
readonly passed: number;
readonly failed: number;
readonly results: TaskCheckOutcome[];
+ /** Whether the app's files had held for the settle window when they were graded, rather than at its bound. */
+ readonly settled: boolean;
+ /** Whether a read still in flight at the settle bound was cut, so the last completed read was graded. */
+ readonly read_cut: boolean;
readonly witness: WitnessSummary | null;
};
@@ -128,7 +146,10 @@ const bundleTaskDifference = async (
!isDeepStrictEqual(listed, JSON.parse(renderTaskSteps(task, app)))
)
return `the bundle's ${stepsPath} differs from this task's steps.`;
- return verifier.toString("utf8") === renderTaskVerifier(task, app)
+ return sameTaskVerifier(
+ renderTaskVerifier(task, app),
+ verifier.toString("utf8"),
+ )
? undefined
: `the bundle's ${verifierPath} differs from this task's checks.`;
};
@@ -144,6 +165,11 @@ interface OpenWitness extends EndedWitness {
record?: WitnessRecord;
/** A manifest whose segments still need to be fetched. */
pendingManifest?: RecordingManifest | undefined;
+ /**
+ * The action log as it stood when the record was first written. A retry
+ * uses this copy, so later activity cannot drop the witnessed actions.
+ */
+ logged?: Pick;
/**
* A published record whose handles are still open. The error already carries
* the record and the cleanup lease; every later close failure returns it.
@@ -171,9 +197,16 @@ const summaryOf = (
});
/** The pass and fail counts of one set of results. */
-const counted = (task: DesktopTask, results: TaskCheckOutcome[]) => {
+const counted = (task: DesktopTask, { results, settled, read_cut }: Graded) => {
const passed = results.filter((result) => result.passed).length;
- return { task: task.name, passed, failed: results.length - passed, results };
+ return {
+ task: task.name,
+ passed,
+ failed: results.length - passed,
+ results,
+ settled,
+ read_cut,
+ };
};
/**
@@ -210,6 +243,7 @@ export class DesktopBuildWitness {
constructor(
private readonly computer: WitnessedComputer,
private readonly actions: ReadonlyActionLog,
+ private readonly settle: BuildSettle = SETTLE_TIMING,
private readonly bundle?: string,
) {}
@@ -220,6 +254,7 @@ export class DesktopBuildWitness {
readonly directory?: string | undefined;
},
signal: AbortSignal,
+ progress: ProgressReporter = silentProgressReporter,
): Promise> {
const parsed = parseDesktopTask(input.task, isPortableBundleName);
if (!parsed.ok)
@@ -243,27 +278,38 @@ export class DesktopBuildWitness {
? undefined
: await this.#admit(task, stage, input.directory, signal);
if (admitted?.ok === false) return admitted;
- const graded = await this.#grade(task, signal);
+ const graded = await this.#grade(task, signal, progress);
if (!graded.ok) return graded;
- const results = graded.value;
- const checked = counted(task, results);
+ const checked = counted(task, graded.value);
if (stage === undefined || admitted === undefined)
return ok({ ...checked, witness: null });
- const filed = await this.#file(stage, admitted.value, results, signal);
+ const filed = await this.#file(stage, admitted.value, graded.value, signal);
return filed.ok ? ok({ ...checked, witness: filed.value }) : filed;
}
- /** The task's checks against the app's live views, one result per check. */
+ /** The task's checks against the app's views once its files settle, one result per check. */
async #grade(
task: DesktopTask,
signal: AbortSignal,
- ): Promise> {
+ progress: ProgressReporter,
+ ): Promise> {
try {
- const views = appStateViews(
- await this.computer.appData("data/get", signal),
+ const read = await settledAppData(
+ (turn) => this.computer.appData("data/get", turn),
+ this.settle,
+ signal,
+ (message) =>
+ progress
+ .report({ phase: CHECK, completed: 0, total: null, message })
+ .catch(() => undefined),
);
+ const views = appStateViews(read.parts);
return views.ok
- ? ok([...evaluateTaskChecks(task.checks, views.value)])
+ ? ok({
+ results: [...evaluateTaskChecks(task.checks, views.value)],
+ settled: read.settled,
+ read_cut: read.readCut,
+ })
: err(
refused(
views.error,
@@ -290,7 +336,7 @@ export class DesktopBuildWitness {
async #file(
stage: WitnessStage,
actionsBefore: number,
- results: TaskCheckOutcome[],
+ { results, settled, read_cut }: Graded,
signal: AbortSignal,
): Promise> {
const witness = this.#witness;
@@ -328,6 +374,8 @@ export class DesktopBuildWitness {
checked_at: new Date().toISOString(),
expectation: stageExpectation(stage),
held,
+ settled,
+ read_cut,
results,
...(unrestored.length === 0
? {}
@@ -361,10 +409,14 @@ export class DesktopBuildWitness {
witness.state = "saved";
return ok(summaryOf(filed, null, witness.record));
}
+ witness.logged ??= {
+ entries: [...this.actions.entries],
+ dropped: this.actions.dropped,
+ };
const written = await writeWitnessRecord(
this.computer,
witness,
- this.actions,
+ witness.logged,
signal,
witness.pendingManifest,
);
@@ -410,7 +462,7 @@ export class DesktopBuildWitness {
),
);
return ok({
- ...counted(task, filed.results),
+ ...counted(task, filed),
witness: summaryOf(filed, null, witness.record),
});
}
@@ -436,7 +488,10 @@ export class DesktopBuildWitness {
);
const saved = await this.#save(witness, filed, signal);
return saved.ok
- ? ok({ ...counted(task, filed.results), witness: saved.value })
+ ? ok({
+ ...counted(task, filed),
+ witness: saved.value,
+ })
: saved;
}
diff --git a/src/application/desktop/DesktopBuildWitnessRecord.ts b/src/application/desktop/DesktopBuildWitnessRecord.ts
index 5f9c2e9ba3..7095c15756 100644
--- a/src/application/desktop/DesktopBuildWitnessRecord.ts
+++ b/src/application/desktop/DesktopBuildWitnessRecord.ts
@@ -213,6 +213,29 @@ const logLimitations = (
: []),
];
+/** Which stages graded files that had not settled by the bound, and which of them had a read cut there. */
+const unsettledLimitation = (
+ stages: readonly WitnessStageRecord[],
+): string[] => {
+ const named = (cut: boolean) =>
+ stages.flatMap(({ stage, settled, read_cut }) =>
+ settled || (cut && !read_cut) ? [] : [stage],
+ );
+ const [unsettled, cut] = [named(false), named(true)];
+ return [
+ ...(unsettled.length === 0
+ ? []
+ : [
+ `The app's files had not held for the settle window by its bound when these stages were checked, so each graded its last completed read, with settled false: ${unsettled.join(", ")}.`,
+ ]),
+ ...(cut.length === 0
+ ? []
+ : [
+ `A read still running at the settle bound was cut, with read_cut true, in: ${cut.join(", ")}.`,
+ ]),
+ ];
+};
+
/** How a witness write ended: saved, retryable, or published with cleanup still open. */
export type WitnessWrite =
| { readonly kind: "saved"; readonly record: WitnessRecord }
@@ -286,6 +309,7 @@ export const writeWitnessRecord = async (
? [`The recording is missing: ${recording.error}`]
: []),
...logLimitations(actions, dropped),
+ ...unsettledLimitation(witness.stages),
],
});
const files = [
diff --git a/src/application/desktop/DesktopCaptureScope.test.ts b/src/application/desktop/DesktopCaptureScope.test.ts
index 7218deb6da..9f71ab792a 100644
--- a/src/application/desktop/DesktopCaptureScope.test.ts
+++ b/src/application/desktop/DesktopCaptureScope.test.ts
@@ -1,4 +1,4 @@
-import { mkdir, realpath, writeFile } from "node:fs/promises";
+import { mkdir, realpath, symlink, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { describe, expect, it } from "vitest";
@@ -107,6 +107,40 @@ describe("captureScope", () => {
expect.objectContaining({ path: `${database}-journal`, kind: "created" }),
]);
});
+
+ it("records each selected path as named beside the physical path it walks, so a link keeps what its name means", async () => {
+ const root = await realpath(
+ await createTestTempDirectory("desktop-scope-"),
+ );
+ const home = join(root, "home/ada");
+ const settings = join(home, "settings/MuseScore");
+ const linked = join(home, ".config/MuseScore");
+ const score = join(home, "Documents/song.mscz");
+ await mkdir(settings, { recursive: true });
+ await mkdir(dirname(linked));
+ await symlink(settings, linked);
+
+ const { scope, selectedPaths } = await captureScope(
+ { ...request([`${linked}/MuseScore3.ini`]), statePaths: [linked, score] },
+ inventory(),
+ home,
+ join(root, "out"),
+ );
+
+ expect(selectedPaths).toEqual([
+ { path: linked, physical: settings, kind: "directory" },
+ { path: score, physical: score, kind: "file" },
+ {
+ path: `${linked}/MuseScore3.ini`,
+ physical: `${settings}/MuseScore3.ini`,
+ kind: "file",
+ },
+ ]);
+ expect(scope.contentRoots).toEqual(
+ expect.arrayContaining([settings, score]),
+ );
+ expect(scope.contentRoots).not.toContain(linked);
+ });
});
describe("applicationWindowMatching", () => {
diff --git a/src/application/desktop/DesktopCaptureScope.ts b/src/application/desktop/DesktopCaptureScope.ts
index 9b441a44c6..e06b22f65a 100644
--- a/src/application/desktop/DesktopCaptureScope.ts
+++ b/src/application/desktop/DesktopCaptureScope.ts
@@ -1,5 +1,5 @@
import { lstat } from "node:fs/promises";
-import { basename, isAbsolute } from "node:path";
+import { basename, isAbsolute, resolve } from "node:path";
import type { DesktopApplicationInventory } from "../../domain/desktop/desktopCapture.js";
import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js";
@@ -10,33 +10,35 @@ import type { CheckpointScope } from "./DesktopCheckpointRecorder.js";
const unique = (paths: readonly string[]) => [...new Set(paths)];
-/**
- * The `-wal` and `-journal` paths beside each selected path that is not a
- * directory: a database selected on its own is unreadable without them, and
- * only a content root records whether they exist.
- */
-const sidecarsOf = async (paths: readonly string[]) => {
- const sidecars: string[] = [];
- for (const path of paths) {
- const directory = await lstat(path).then(
- (stats) => stats.isDirectory(),
- () => false,
- );
- if (!directory)
- sidecars.push(...SQLITE_SIDECARS.map(({ suffix }) => `${path}${suffix}`));
- }
- return sidecars;
-};
+/** Whether the walked path is a directory. A missing path is not one, matching the sidecar decision. */
+const pathKind = async (path: string): Promise<"file" | "directory"> =>
+ (await lstat(path).then(
+ (stats) => stats.isDirectory(),
+ () => false,
+ ))
+ ? "directory"
+ : "file";
-/** The checkpoint scope, and absolute arguments too broad to be part of it. */
+/** A path the caller selected, absolute as named, and the physical path walked for it. */
+export interface SelectedPath {
+ readonly path: string;
+ readonly physical: string;
+ /** The walked path's kind, so a later export does not infer a directory from changes. */
+ readonly kind: "file" | "directory";
+}
+
+/** The checkpoint scope, the selected paths in it, and absolute arguments too broad to be part of it. */
export interface CaptureScope {
readonly scope: CheckpointScope;
+ /** `--state-path` values, then absolute arguments, in the scope. */
+ readonly selectedPaths: readonly SelectedPath[];
/** `/` or a directory containing the home directory, named by `--argument`. */
readonly broadArguments: readonly string[];
}
/**
- * Paths in their physical spelling, so walks and exclusions agree. An
+ * Paths in their physical spelling, so walks and exclusions agree; a selected
+ * path keeps the spelling it was named by too, which a link would lose. An
* absolute argument becomes a content root unless it is `/` or contains the
* home directory, which would retain and walk whole trees. A selected path
* that is not a directory brings its SQLite sidecars as content roots too.
@@ -54,23 +56,50 @@ export const captureScope = async (
): Promise => {
const canonical = (paths: readonly string[]) =>
Promise.all(paths.map((path) => canonicalPath(path)));
+ const named = (paths: readonly string[]) =>
+ Promise.all(
+ paths.map(async (path) => ({
+ path: resolve(path),
+ physical: await canonicalPath(path),
+ })),
+ );
const stateDirectories = await canonical(
inventory.state_directories.map((entry) => entry.path),
);
- const argumentPaths = await canonical(
+ const argumentPaths = await named(
request.arguments.filter((value) => isAbsolute(value)),
);
const broadArguments = unique(
- argumentPaths.filter((path) => isWithinAny(home, [path])),
+ argumentPaths
+ .map(({ physical }) => physical)
+ .filter((path) => isWithinAny(home, [path])),
);
- const selected = [
- ...(await canonical(request.statePaths)),
- ...argumentPaths.filter((path) => !broadArguments.includes(path)),
- ];
+ const selectedPaths = await Promise.all(
+ [
+ ...new Map(
+ [
+ ...(await named(request.statePaths)),
+ ...argumentPaths.filter(
+ ({ physical }) => !broadArguments.includes(physical),
+ ),
+ ].map((entry) => [entry.path, entry]),
+ ).values(),
+ ].map(async (entry) => ({
+ ...entry,
+ kind: await pathKind(entry.physical),
+ })),
+ );
+ const selected = unique(selectedPaths.map(({ physical }) => physical));
+ // A database selected on its own is unreadable without its logs. A directory
+ // is not a database, so a neighbor named like a log stays a normal file.
const contentRoots = unique([
...stateDirectories,
...selected,
- ...(await sidecarsOf(selected)),
+ ...selectedPaths.flatMap(({ physical, kind }) =>
+ kind === "directory"
+ ? []
+ : SQLITE_SIDECARS.map(({ suffix }) => `${physical}${suffix}`),
+ ),
]);
return {
scope: {
@@ -83,6 +112,7 @@ export const captureScope = async (
excluded: unique([output, ...(await canonical(request.excludes))]),
maxEntries: request.maxEntries,
},
+ selectedPaths,
broadArguments,
};
};
diff --git a/src/application/desktop/DesktopExportFiles.ts b/src/application/desktop/DesktopExportFiles.ts
index 6479d34908..bd77a580de 100644
--- a/src/application/desktop/DesktopExportFiles.ts
+++ b/src/application/desktop/DesktopExportFiles.ts
@@ -7,18 +7,23 @@ import type {
SeedScrubReport,
} from "../../domain/desktop/appExport.js";
import type { TaskFileView } from "../../domain/desktop/appTaskChecks.js";
+import {
+ compressionOf,
+ openStateDocument,
+} from "../../domain/desktop/gzipDocument.js";
import {
identityLeftIn,
type HostIdentity,
} from "../../domain/desktop/hostIdentity.js";
import { scrubSeed } from "../../domain/desktop/seedScrub.js";
import {
- detectStateFormat,
isTextualFormat,
textualView,
} from "../../domain/desktop/stateFormats.js";
+import { err } from "../../domain/result.js";
import { refuse } from "./DesktopExportInstall.js";
import type { RetainedContent } from "./DesktopRetainedContent.js";
+import { readZipView } from "./DesktopZipView.js";
/** A file the export writes from bytes it made. */
export interface GeneratedFile {
@@ -90,8 +95,8 @@ const retainedBytes = async (
};
/**
- * The view of each text seed the scrub rewrites, as the bundle ships it, for
- * a task's checks on the seed: the computer starts from these bytes.
+ * The view of each rewritten seed, including a formerly unreadable gzip
+ * seed, as the bundle ships it: the computer starts from these bytes.
*/
export const shippedSeedViews = async (
content: RetainedContent,
@@ -102,20 +107,31 @@ export const shippedSeedViews = async (
const views = new Map();
for (const copy of copies) {
const seed = copy.seed;
- if (seed === undefined || !isTextualFormat(seed.format)) continue;
+ if (
+ seed === undefined ||
+ (seed.format !== null && !isTextualFormat(seed.format))
+ )
+ continue;
const bytes = await retainedBytes(content, copy, signal);
const shipped = scrubSeed(bytes, seed, identity).bytes;
if (shipped === undefined) continue;
- const format = detectStateFormat(seed.home, shipped);
- const view = isTextualFormat(format)
- ? textualView(format, shipped)
- : undefined;
- views.set(
- seed.home,
- view === undefined || view.ok
- ? { format, view: view?.value ?? null }
- : { format, view: null, error: view.error },
- );
+ const document = openStateDocument(seed.home, shipped);
+ const { format } = document;
+ const view =
+ "error" in document
+ ? err(document.error)
+ : format === "zip"
+ ? await readZipView(document.bytes, signal)
+ : isTextualFormat(format)
+ ? textualView(format, document.bytes)
+ : undefined;
+ views.set(seed.home, {
+ format,
+ ...compressionOf(document),
+ ...(view === undefined || view.ok
+ ? { view: view?.value ?? null }
+ : { view: null, error: view.error }),
+ });
}
return views;
};
@@ -123,8 +139,10 @@ export const shippedSeedViews = async (
/**
* Write the generated files, then each retained file as it is read, so one at
* a time is in memory: a seed whose text names the capture host becomes a
- * generated file with its own digest. The README follows, listing them. Every
- * file is listed as unscrubbed whose path or content still holds the identity.
+ * generated file with its own digest, and a gzip seed whose document did not
+ * change is listed apart as recompressed. The README follows, listing them.
+ * Every file is listed as unscrubbed whose path or content still holds the
+ * identity.
*/
export const writeFiles = async (
tree: SafeOutputTree,
@@ -133,6 +151,7 @@ export const writeFiles = async (
signal: AbortSignal | undefined,
): Promise => {
const scrubbed: string[] = [];
+ const recompressed: string[] = [];
const unscrubbed: string[] = [];
const write = async (
path: string,
@@ -173,10 +192,12 @@ export const writeFiles = async (
if (scrub?.bytes === undefined)
await write(copy.path, bytes, { sha256: copy.digest, holdsIdentity });
else {
- scrubbed.push(copy.path);
+ (scrub.recompressed === true ? recompressed : scrubbed).push(copy.path);
await write(copy.path, scrub.bytes, { holdsIdentity });
}
}
- await generated(bundle.readme({ scrubbed, unscrubbed: [...unscrubbed] }));
- return { scrubbed, unscrubbed };
+ await generated(
+ bundle.readme({ scrubbed, recompressed, unscrubbed: [...unscrubbed] }),
+ );
+ return { scrubbed, recompressed, unscrubbed };
};
diff --git a/src/application/desktop/DesktopExportLaunch.test.ts b/src/application/desktop/DesktopExportLaunch.test.ts
new file mode 100644
index 0000000000..b332ae35d8
--- /dev/null
+++ b/src/application/desktop/DesktopExportLaunch.test.ts
@@ -0,0 +1,213 @@
+import { describe, expect, it } from "vitest";
+
+import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js";
+import type { DesktopApplicationState } from "../../domain/desktop/desktopState.js";
+import { planLaunch } from "./DesktopExportLaunch.js";
+
+const HOME = "/home/ada";
+type StatePath = DesktopApplicationState["paths"][number];
+
+const file = (physical: string, role: StatePath["role"]): StatePath =>
+ ({ path: physical, role }) as StatePath;
+
+describe("planLaunch captured paths", () => {
+ it.each([
+ ["/mnt/data/books.db", "/mnt/data/books.db"],
+ ["file:///mnt/data/books.db?mode=edit#page", "/mnt/data/books.db"],
+ ])("retargets selected outside-home argument %s", (argument, physical) => {
+ const capture = {
+ host: { home: HOME },
+ application: {
+ argv: ["editor", argument],
+ working_directory: HOME,
+ desktop_entry: { exec: "editor %U" },
+ state_directories: [],
+ },
+ state: {
+ state_directories: [],
+ selected_paths: [
+ { path: `${HOME}/Documents/books.db`, physical, kind: "file" },
+ {
+ path: `${HOME}/.config/vendor`,
+ physical: `${HOME}/Documents`,
+ kind: "directory",
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture;
+ const files = new Map([["Documents/books.db", file(physical, "document")]]);
+ const seeds = new Map([
+ ["Documents/books.db", { sha256: "a".repeat(64), format: "text" }],
+ ]);
+ const launch = planLaunch(capture, false, seeds, files);
+ expect(launch.opens).toEqual({ kind: "captured" });
+ expect(launch.command.join(" ")).toContain("Documents/books.db");
+ expect(launch.command.join(" ")).not.toContain("/mnt/data/books.db");
+ expect(launch.command.join(" ")).not.toContain(".config/vendor/books.db");
+ if (argument.startsWith("file:"))
+ expect(launch.command.join(" ")).toContain("?mode=edit#page");
+ });
+
+ it("leaves an unrelated outside-home argument untouched", () => {
+ const capture = {
+ host: { home: HOME },
+ application: {
+ argv: ["editor", "/mnt/other/unselected.db"],
+ working_directory: HOME,
+ desktop_entry: { exec: "editor %U" },
+ state_directories: [],
+ },
+ state: {
+ state_directories: [],
+ selected_paths: [
+ {
+ path: `${HOME}/Documents/books.db`,
+ physical: "/mnt/data/books.db",
+ kind: "file",
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture;
+ const files = new Map([
+ ["Documents/books.db", file("/mnt/data/books.db", "document")],
+ ]);
+ const command = planLaunch(capture, false, new Map(), files).command.join(
+ " ",
+ );
+ expect(command).toContain("/mnt/other/unselected.db");
+ });
+});
+
+describe("planLaunch URL and argument spelling", () => {
+ it("keeps an encoded filename when a file URL has repeated separators", () => {
+ const physical = `${HOME}/data/A budget.svg`;
+ const capture = {
+ host: { home: HOME },
+ application: {
+ argv: [
+ "editor",
+ `file://localhost${HOME}//data//%41%20budget.svg?mode=edit#page`,
+ ],
+ working_directory: HOME,
+ desktop_entry: { exec: "editor %U" },
+ state_directories: [],
+ },
+ state: {
+ state_directories: [],
+ selected_paths: [{ path: `${HOME}/Documents/A budget.svg`, physical }],
+ },
+ } as unknown as DesktopApplicationCapture;
+ const files = new Map([
+ ["Documents/A budget.svg", file(physical, "document")],
+ ]);
+ const command = planLaunch(capture, false, new Map(), files).command.join(
+ " ",
+ );
+ expect(command).toContain("/Documents/%41%20budget.svg?mode=edit#page");
+ });
+
+ it("opens the selected document spelling when its captured file URL names the physical path", () => {
+ const physical = `${HOME}/data/A budget.svg`;
+ const capture = {
+ host: { home: HOME },
+ application: {
+ argv: [
+ "editor",
+ `file://localhost${HOME}/data/%41%20budget.svg?mode=edit#page`,
+ ],
+ working_directory: HOME,
+ desktop_entry: { exec: "editor %U" },
+ state_directories: [],
+ },
+ state: {
+ state_directories: [],
+ selected_paths: [{ path: `${HOME}/Documents/A budget.svg`, physical }],
+ },
+ } as unknown as DesktopApplicationCapture;
+ const files = new Map([
+ ["Documents/A budget.svg", file(physical, "document")],
+ ]);
+ const command = planLaunch(capture, false, new Map(), files).command.join(
+ " ",
+ );
+ expect(command).toContain("file://localhost");
+ expect(command).toContain("/Documents/%41%20budget.svg?mode=edit#page");
+ expect(command).not.toContain("/data/%41%20budget.svg");
+ });
+
+ it("rewrites a nested settings folder and leaves a numeric option", () => {
+ const profile = `${HOME}/settings/geany/profiles/work/geany.conf`;
+ const files = new Map([
+ [
+ ".config/geany/profiles/work/geany.conf",
+ file(profile, "application_state"),
+ ],
+ [
+ "Documents/poster.svg",
+ file(`${HOME}/Documents/poster.svg`, "document"),
+ ],
+ ]);
+ const capture = {
+ host: { home: HOME },
+ application: {
+ argv: ["geany", "--config=settings/geany/profiles/work", "--line=10"],
+ working_directory: HOME,
+ desktop_entry: { exec: "geany" },
+ state_directories: [{ path: `${HOME}/.config/geany` }],
+ },
+ state: {
+ state_directories: [`${HOME}/settings/geany`],
+ selected_paths: [
+ {
+ path: `${HOME}/.config/geany`,
+ physical: `${HOME}/settings/geany`,
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture;
+
+ const command = planLaunch(capture, false, new Map(), files).command.join(
+ " ",
+ );
+
+ expect(command).toContain(`"$HOME"/'.config/geany/profiles/work'`);
+ expect(command).toContain("'--line=10'");
+ expect(command).not.toContain("settings/geany");
+ });
+
+ it("keeps the program an env prefix runs when a captured file sits in a folder of that name", () => {
+ const config = `${HOME}/work/geany/config.ini`;
+ const files = new Map([
+ [".config/vendor/geany/config.ini", file(config, "application_state")],
+ ]);
+ const capture = {
+ host: { home: HOME },
+ application: {
+ argv: ["env", "GDK_BACKEND=x11", "geany", "note.txt"],
+ working_directory: `${HOME}/work`,
+ desktop_entry: { exec: "geany %f" },
+ state_directories: [{ path: `${HOME}/.config/vendor` }],
+ },
+ state: {
+ state_directories: [`${HOME}/work`],
+ selected_paths: [
+ { path: `${HOME}/.config/vendor`, physical: `${HOME}/work` },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture;
+ const note = file(`${HOME}/work/note.txt`, "document");
+ files.set("work/note.txt", note);
+
+ const command = planLaunch(
+ capture,
+ false,
+ new Map([["work/note.txt", { sha256: "a".repeat(64), format: "text" }]]),
+ files,
+ ).command.join(" ");
+
+ expect(command).toContain("'GDK_BACKEND=x11'");
+ expect(command).toContain("'geany'");
+ expect(command).toContain(`"$HOME"/'.config/vendor/note.txt'`);
+ expect(command).not.toContain(".config/vendor/geany");
+ });
+});
diff --git a/src/application/desktop/DesktopExportLaunch.ts b/src/application/desktop/DesktopExportLaunch.ts
index 84b40ec2f9..92f069a139 100644
--- a/src/application/desktop/DesktopExportLaunch.ts
+++ b/src/application/desktop/DesktopExportLaunch.ts
@@ -1,4 +1,5 @@
import {
+ capturedHomePath,
launchCommand,
type DesktopAppExportPlan,
} from "../../domain/desktop/appExport.js";
@@ -15,51 +16,105 @@ import type { SeedVersion } from "./DesktopExportFiles.js";
type StatePath = DesktopApplicationState["paths"][number];
-/** The owned files present at the seed with their digests, and those read as text. */
+/**
+ * The owned files present at the seed with their digests, and those read as
+ * uncompressed text, which boot respells as UTF-8, but for the lock files
+ * `isLeftOutLock` names, which the app holds while it runs: a copied one makes
+ * the app find its profile or document in use, so those are left out.
+ */
export const seedVersions = (
files: ReadonlyMap,
paths: readonly string[],
seed: number,
+ isLeftOutLock: (home: string) => boolean,
): {
readonly seeds: ReadonlyMap;
readonly textSeeds: readonly string[];
+ readonly locks: readonly string[];
} => {
const seeds = new Map();
const textSeeds: string[] = [];
+ const locks: string[] = [];
for (const [home, path] of files) {
const version = versionAt(path, seed);
if (!isWithinAny(home, paths) || version?.present !== true) continue;
- if (version.sha256 === null) continue;
- seeds.set(home, { sha256: version.sha256, format: version.format });
- if (isTextualFormat(version.format)) textSeeds.push(home);
+ if (isLeftOutLock(home)) locks.push(home);
+ else if (version.sha256 !== null) {
+ seeds.set(home, { sha256: version.sha256, format: version.format });
+ if (isTextualFormat(version.format) && version.compression === undefined)
+ textSeeds.push(home);
+ }
}
- return { seeds, textSeeds: textSeeds.sort() };
+ return { seeds, textSeeds: textSeeds.sort(), locks: locks.sort() };
+};
+
+/** Home paths of captured files and of settings directories, in either spelling. */
+const capturedLaunchPaths = (
+ capture: DesktopApplicationCapture,
+ files: ReadonlyMap,
+ homeOf: (path: string) => string | null,
+): ((path: string) => boolean) => {
+ const { home } = capture.host;
+ const captured = new Set(files.keys());
+ const remember = (absolute: string) => {
+ if (!absolute.startsWith(`${home}/`)) return;
+ const relative = absolute.slice(home.length + 1);
+ captured.add(relative);
+ const spelled = homeOf(absolute);
+ if (spelled !== null) captured.add(spelled);
+ };
+ for (const entry of capture.application.state_directories)
+ remember(entry.path);
+ for (const directory of capture.state.state_directories ?? [])
+ remember(directory);
+ for (const selected of capture.state.selected_paths ?? []) {
+ remember(selected.path);
+ remember(selected.physical);
+ }
+ for (const file of files.values()) remember(file.path);
+ const known = [...captured];
+ // A settings folder between a selected root and a captured file is a path
+ // too. An option value that merely resolves under the home is not.
+ return (path) =>
+ captured.has(path) || known.some((file) => isWithinAny(file, [path]));
};
/** The launch command, opening the seed's one document when the app takes files. */
export const planLaunch = (
- { application, host }: DesktopApplicationCapture,
+ capture: DesktopApplicationCapture,
electron: boolean,
seeds: ReadonlyMap,
files: ReadonlyMap,
): Pick => {
+ const { application, host } = capture;
const isDocument = (home: string) => files.get(home)?.role === "document";
+ const homeOf = capturedHomePath(capture);
+ const capturedDirectory = application.working_directory;
+ const workingDirectory = (() => {
+ if (!capturedDirectory.startsWith(`${host.home}/`))
+ return capturedDirectory;
+ const relative = capturedDirectory.slice(host.home.length + 1);
+ const seed = homeOf(capturedDirectory) ?? relative;
+ return seed === relative ? capturedDirectory : `${host.home}/${seed}`;
+ })();
const launch = openSeedDocument(
electronLaunchArguments(application.argv, electron),
{
home: host.home,
- workingDirectory: application.working_directory,
+ // Resolve captured relative arguments against the directory the app used,
+ // then place the process in the seeded spelling of that directory.
+ workingDirectory: capturedDirectory,
+ launchDirectory: workingDirectory,
exec: application.desktop_entry?.exec ?? null,
documents: [...seeds.keys()].filter(isDocument).sort(),
isDocument,
+ isCaptured: capturedLaunchPaths(capture, files, homeOf),
+ spell: (path) => homeOf(`${host.home}/${path}`) ?? path,
+ mapOutside: homeOf,
},
);
return {
- command: launchCommand(
- launch.argv,
- host.home,
- application.working_directory,
- ),
+ command: launchCommand(launch.argv, host.home, workingDirectory),
opens: launch.opens,
};
};
diff --git a/src/application/desktop/DesktopGzipDocument.test.ts b/src/application/desktop/DesktopGzipDocument.test.ts
new file mode 100644
index 0000000000..d8f9a432fd
--- /dev/null
+++ b/src/application/desktop/DesktopGzipDocument.test.ts
@@ -0,0 +1,291 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { gunzipSync, gzipSync } from "node:zlib";
+
+import { describe, expect, it } from "vitest";
+
+import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
+import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js";
+import { desktopApplicationStateSchema } from "../../domain/desktop/desktopState.js";
+import { openStateDocument } from "../../domain/desktop/gzipDocument.js";
+import { exportDesktopApplication } from "./DesktopApplicationExport.js";
+import { inspectDesktopApplicationState } from "./DesktopApplicationState.js";
+import { draftDesktopApplicationTask } from "./DesktopTaskDraft.js";
+import {
+ HOME,
+ capture,
+ change,
+ readers,
+ retained,
+ writeCaptureFolder,
+} from "./desktopCapture.fixture.js";
+
+const BUDGET = "Documents/Budget.gnumeric";
+const HOSTNAME = "e81232be457d";
+const SEED = "envs/notes/seed/Documents/Budget.gnumeric";
+const CELLS =
+ "/root/children/[name=gnm:Sheets]/children/[name=gnm:Sheet]/children/[name=gnm:Cells]/children";
+
+/** A Gnumeric workbook's XML with its rent, author and any comment as given. */
+const workbook = (
+ rent: string,
+ author: string,
+ comment = "",
+) => `
+
+ author${author}
+ ${comment}
+ Budget
+ Rent
+ ${rent}
+
+
+`;
+
+/**
+ * The notes capture of a host with the workbook, named in a window title,
+ * saved before and after the user changed the rent and set the author to the
+ * value the scrub writes, each save made by `save`, with any other files the
+ * app kept unchanged by their home path.
+ */
+const budgetCapture = (
+ save: (text: string) => Buffer,
+ comment = "",
+ kept: Readonly> = {},
+) => {
+ const before = save(workbook("1200", `ada@${HOSTNAME}`, comment));
+ const after = save(workbook("1350", "user@localhost", comment));
+ const base = capture();
+ const [first, second] = base.state.checkpoints;
+ if (first === undefined || second === undefined) throw new Error("fixture");
+ const path = `${HOME}/${BUDGET}`;
+ const captured: DesktopApplicationCapture = {
+ ...base,
+ host: { ...base.host, hostname: HOSTNAME, user: "ada" },
+ session: {
+ ...base.session,
+ windows: base.session.windows.map((window) => ({
+ ...window,
+ titles: [...window.titles, "Budget.gnumeric - Notes"],
+ })),
+ },
+ state: {
+ ...base.state,
+ checkpoints: [
+ {
+ ...first,
+ retained: [
+ ...first.retained,
+ retained(path, before),
+ ...Object.entries(kept).map(([home, bytes]) =>
+ retained(`${HOME}/${home}`, bytes),
+ ),
+ ],
+ },
+ {
+ ...second,
+ changes: [...second.changes, change(path, { before, after })],
+ retained: [...second.retained, retained(path, after)],
+ },
+ ],
+ },
+ };
+ return writeCaptureFolder(captured, undefined, [
+ before,
+ after,
+ ...Object.values(kept),
+ ]);
+};
+
+const gzipped = (text: string) => gzipSync(text);
+const plain = (text: string) => Buffer.from(text);
+
+/** The workbook's path in the capture's state view. */
+const budgetState = async (folder: string) => {
+ const evidence = await inspectDesktopApplicationState(folder, readers);
+ if (!("normalized_result" in evidence)) throw evidence;
+ return desktopApplicationStateSchema
+ .parse(evidence.normalized_result)
+ .paths.find((path) => path.home_path === BUDGET);
+};
+
+const draftOf = async (folder: string) => {
+ const drafted = await draftDesktopApplicationTask(folder, readers);
+ if (drafted instanceof Error) throw drafted;
+ return drafted.normalized_result;
+};
+
+/** Export the capture with a task that checks the rent and the author the scrub writes. */
+const exportBudget = async (folder: string) => {
+ const scratch = await createTestTempDirectory("desktop-gzip-task-");
+ const task = join(scratch, "task.json");
+ const directory = join(scratch, "bundle");
+ await writeFile(
+ task,
+ JSON.stringify({
+ name: "rent",
+ prompt: "Set the rent to 1350 in my budget and save it.",
+ checks: [
+ {
+ id: "rent",
+ kind: "value_equals",
+ file: BUDGET,
+ pointer: `${CELLS}/[name=gnm:Cell][@Row=0][@Col=1]/children/0`,
+ value: "1350",
+ },
+ {
+ id: "author",
+ kind: "text_equals",
+ file: BUDGET,
+ pointer:
+ "/root/children/[name=gnm:Summary]/children/[name=gnm:Item]/children/[name=gnm:val-string]",
+ value: "user@localhost",
+ },
+ ],
+ }),
+ );
+
+ const result = await exportDesktopApplication(
+ { capture: folder, directory, name: undefined, tasks: [task] },
+ readers,
+ );
+ return { result, directory };
+};
+
+describe("documents saved gzip-compressed", () => {
+ it("reads each version as the XML inside, the view the same workbook has uncompressed, and records the compression", async () => {
+ const compressed = await budgetState(await budgetCapture(gzipped));
+ const uncompressed = await budgetState(await budgetCapture(plain));
+
+ expect(compressed).toMatchObject({
+ role: "document",
+ format: "xml",
+ versions: [
+ { format: "xml", compression: "gzip", view_error: null },
+ { format: "xml", compression: "gzip", view_error: null },
+ ],
+ changes: [{ kind: "values", changed_count: 2 }],
+ });
+ expect(uncompressed?.versions[0]?.compression).toBeUndefined();
+ expect(compressed?.versions.map(({ view }) => view)).toEqual(
+ uncompressed?.versions.map(({ view }) => view),
+ );
+ expect(compressed?.changes).toEqual(uncompressed?.changes);
+ });
+
+ it("drafts the same files, values, checks, guards and notes as the same workbook uncompressed", async () => {
+ const compressed = await draftOf(await budgetCapture(gzipped));
+
+ expect(compressed).toEqual(await draftOf(await budgetCapture(plain)));
+ expect(
+ (compressed as { task: { checks: unknown[] } }).task.checks,
+ ).toContainEqual(
+ expect.objectContaining({
+ file: BUDGET,
+ pointer: `${CELLS}/[name=gnm:Cell][@Col=1]`,
+ value: "1350",
+ }),
+ );
+ });
+
+ it("ships the seed scrubbed inside and still gzip, copied as it is at boot, and checks the seed as shipped", async () => {
+ const { result, directory } = await exportBudget(
+ await budgetCapture(gzipped),
+ );
+
+ if (result instanceof Error) throw result;
+ expect(result.normalized_result).toMatchObject({
+ scrubbed_files: [SEED],
+ unscrubbed_files: [],
+ task: { failing_on_seed: ["rent"] },
+ });
+ const shipped = await readFile(join(directory, SEED));
+ expect([...shipped.subarray(0, 4)]).toEqual([0x1f, 0x8b, 8, 0]);
+ expect(gunzipSync(shipped).toString("utf8")).toBe(
+ workbook("1200", "user@localhost"),
+ );
+ const module = await readFile(
+ join(directory, "envs/notes/notes_app.py"),
+ "utf8",
+ );
+ // Boot respells text seeds as UTF-8, so the gzip workbook is not one of them.
+ expect(module).toContain(
+ 'TEXT_SEEDS = frozenset([".config/notes/settings.ini"])',
+ );
+ });
+
+ it("lists gzip seeds recompressed only to clear their headers apart from the document whose content was scrubbed", async () => {
+ const templates = ["blank", "invoice", "loan"].map(
+ (name) => `.local/share/notes/templates/${name}.xml.gz`,
+ );
+ const kept = Object.fromEntries(
+ templates.map((home) => [home, gzipSync(``)]),
+ );
+
+ const { result, directory } = await exportBudget(
+ await budgetCapture(gzipped, "", kept),
+ );
+
+ if (result instanceof Error) throw result;
+ const recompressed = templates.map((home) => `envs/notes/seed/${home}`);
+ expect(result.normalized_result).toMatchObject({
+ scrubbed_files: [SEED],
+ recompressed_files: recompressed,
+ unscrubbed_files: [],
+ });
+ const readme = await readFile(join(directory, "README.md"), "utf8");
+ expect(readme).toContain(`\`${SEED}\``);
+ expect(recompressed.filter((path) => readme.includes(path))).toEqual([]);
+ expect(readme).toMatch(/\b3 gzip file\(s\)/u);
+ for (const path of recompressed)
+ expect(
+ gunzipSync(await readFile(join(directory, path))).toString("utf8"),
+ ).toBe(``);
+ });
+
+ it("checks a recompressed formerly unreadable seed as shipped and scrubs its document", async () => {
+ const save = (text: string) =>
+ gzipSync(text, { level: text.includes(">1200<") ? 9 : 0 });
+ const folder = await budgetCapture(save, "x".repeat(60_000));
+ const state = await budgetState(folder);
+ expect(state?.versions[0]).toMatchObject({
+ format: null,
+ compression: "gzip",
+ });
+ expect(state?.versions[0]?.view_error).toContain("expands more than");
+ const { result, directory } = await exportBudget(folder);
+ if (result instanceof Error) throw result;
+ expect(result.normalized_result).toMatchObject({
+ task: { failing_on_seed: ["rent"] },
+ scrubbed_files: [SEED],
+ });
+ const shipped = await readFile(join(directory, SEED));
+ expect(openStateDocument(BUDGET, shipped)).toMatchObject({
+ format: "xml",
+ compression: "gzip",
+ });
+ expect(gunzipSync(shipped).toString("utf8")).toBe(
+ workbook("1200", "user@localhost", "x".repeat(60_000)),
+ );
+ });
+
+ it("keeps a seed the app stored uncompressed readable once scrubbed, where deflating it would pass the 100 times a view reads", async () => {
+ const stored = (text: string) => gzipSync(text, { level: 0 });
+ const comment = "x".repeat(60_000);
+
+ const { result, directory } = await exportBudget(
+ await budgetCapture(stored, comment),
+ );
+
+ if (result instanceof Error) throw result;
+ expect(result.normalized_result).toMatchObject({
+ task: { failing_on_seed: ["rent"] },
+ });
+ const shipped = await readFile(join(directory, SEED));
+ expect(openStateDocument(BUDGET, shipped)).toMatchObject({
+ format: "xml",
+ compression: "gzip",
+ });
+ expect(openStateDocument(BUDGET, shipped)).not.toHaveProperty("error");
+ });
+});
diff --git a/src/application/desktop/DesktopTaskDraft.ts b/src/application/desktop/DesktopTaskDraft.ts
index 349b4cd258..ae9aa81ca5 100644
--- a/src/application/desktop/DesktopTaskDraft.ts
+++ b/src/application/desktop/DesktopTaskDraft.ts
@@ -2,6 +2,7 @@ import { AnalysisError } from "../../domain/analysisErrorBase.js";
import { exportedStateFiles } from "../../domain/desktop/appExport.js";
import { DRAFT_PROMPT } from "../../domain/desktop/appTask.js";
import { draftTaskFiles } from "../../domain/desktop/appTaskDraft.js";
+import { draftNotes } from "../../domain/desktop/appTaskDraftNotes.js";
import { desktopApplicationStateSchema } from "../../domain/desktop/desktopState.js";
import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
@@ -25,10 +26,21 @@ const OPERATION = "describe_desktop_application_task";
const DRAFT_LIMITATIONS = [
"Files are those under the paths the export owns, whose state differs between the seed and end checkpoints, by default the capture's first and last; data/get serves nothing else.",
- "Suggested checks and views cover documents, or the app's own state when no document changed; each check is derived from the net change and is a starting point, not a task.",
+ "Suggested checks, values, guards and views cover documents, or the app's own state when no document changed; each check is derived from the net change and is a starting point, not a task.",
+ "Values cover XML, archives, JSON and key files: at most 20 entries and 8 KB a file and 24 KB a draft, each value cut at 600 characters (values_cut says when any is left out).",
"Views longer than 200 items or strings longer than 2,000 characters are cut here (view_cut); inspect-desktop-application-state shows every view whole.",
];
+/** Checks with ids `PREFIX-1`, `PREFIX-2` and on, in order. */
+const numbered = (
+ prefix: string,
+ checks: readonly Check[],
+) =>
+ checks.map((check, index) => ({
+ id: `${prefix}-${String(index + 1)}`,
+ ...check,
+ }));
+
const draft = async (
folder: string,
readers: DesktopStateReaders,
@@ -47,12 +59,33 @@ const draft = async (
);
const { checkpoints } = state.capture;
const ends = selectStateEnds(checkpoints, selection, OPERATION);
- const { files, paths } = exportedStateFiles(capture, state);
+ const { files, dconf, paths } = exportedStateFiles(capture, state);
const drafted = draftTaskFiles(
files,
(home) => isWithinAny(home, paths),
ends,
);
+ const covered = (index: number) => {
+ const point = capture.state.checkpoints[index];
+ return (
+ point !== undefined &&
+ !point.truncated &&
+ point.unreadable_directories === 0 &&
+ point.unchecked_paths === 0 &&
+ point.unconfirmed_changes === 0
+ );
+ };
+ const notes = draftNotes(files, {
+ drafted,
+ owned: paths,
+ dconf,
+ ends,
+ checkpoints,
+ observed: Array.from(
+ { length: ends.end - ends.seed + 1 },
+ (_, offset) => ends.seed + offset,
+ ).every(covered),
+ });
const environment = exportEnvironmentName(capture, undefined);
const result = {
application: {
@@ -65,23 +98,30 @@ const draft = async (
baseline: checkpoints[ends.seed],
final: checkpoints[ends.end],
},
- files: drafted.map(({ file, role, baseline, final, change }) => ({
- file,
- role,
- baseline,
- final,
- change,
- })),
+ files: drafted.map(
+ ({ file, role, baseline, final, change, values, values_cut }) => ({
+ file,
+ role,
+ baseline,
+ final,
+ change,
+ values,
+ values_cut,
+ }),
+ ),
task: {
name: "change",
prompt: DRAFT_PROMPT,
- checks: drafted
- .flatMap((file) => file.suggested)
- .map((check, index) => ({
- id: `check-${String(index + 1)}`,
- ...check,
- })),
+ checks: numbered(
+ "check",
+ drafted.flatMap((file) => file.suggested),
+ ),
},
+ guards: numbered(
+ "guard",
+ drafted.flatMap((file) => file.guards),
+ ),
+ notes,
} as unknown as JsonValue;
signal?.throwIfAborted();
return createEvidence(undefined, DESKTOP_PROVIDER, {
diff --git a/src/application/desktop/DesktopTaskDraftNotes.test.ts b/src/application/desktop/DesktopTaskDraftNotes.test.ts
new file mode 100644
index 0000000000..247e6e2584
--- /dev/null
+++ b/src/application/desktop/DesktopTaskDraftNotes.test.ts
@@ -0,0 +1,393 @@
+import { readFile } from "node:fs/promises";
+
+import { describe, expect, it } from "vitest";
+import { z } from "zod";
+
+import type {
+ DesktopApplicationCapture,
+ DesktopStateChange,
+} from "../../domain/desktop/desktopCapture.js";
+import type { DesktopStateReaders } from "./DesktopApplicationState.js";
+import { draftDesktopApplicationTask } from "./DesktopTaskDraft.js";
+import {
+ EDITED_NOTE,
+ HOME,
+ NOTE,
+ SAVED_NOTE,
+ change,
+ readers,
+ retained,
+ seedEndCapture,
+ writeCaptureFolder,
+} from "./desktopCapture.fixture.js";
+
+const ENDS = { seed: "seed", end: "end" };
+const draftSchema = z.looseObject({
+ files: z.array(z.unknown()),
+ notes: z.array(
+ z.looseObject({
+ code: z.string(),
+ files: z
+ .array(z.object({ file: z.string(), role: z.string() }))
+ .optional(),
+ more: z.int().optional(),
+ error: z.string().optional(),
+ }),
+ ),
+});
+
+/** The draft's notes for a capture, between seed and end. */
+const notesOf = async (
+ recorded: DesktopApplicationCapture,
+ extraContent: readonly (string | Buffer)[] = [],
+ stateReaders: DesktopStateReaders = readers,
+) => {
+ const result = await draftDesktopApplicationTask(
+ await writeCaptureFolder(recorded, undefined, [
+ EDITED_NOTE,
+ ...extraContent,
+ ]),
+ stateReaders,
+ undefined,
+ ENDS,
+ );
+ if (result instanceof Error) throw result;
+ return draftSchema.parse(result.normalized_result);
+};
+
+/** The seed-and-end capture with the `end` checkpoint replaced. */
+const withEnd = (
+ base: DesktopApplicationCapture,
+ end: {
+ changes: DesktopStateChange[];
+ kept: ReturnType[];
+ },
+): DesktopApplicationCapture => ({
+ ...base,
+ state: {
+ ...base.state,
+ checkpoints: base.state.checkpoints.map((point) =>
+ point.name === "end"
+ ? { ...point, changes: end.changes, retained: end.kept }
+ : point,
+ ),
+ },
+});
+
+const database = `${HOME}/.config/dconf/user`;
+// dconf databases by the keys they hold; the reader dumps each as dconf does.
+const dumps: Readonly> = {
+ "GVariant wrap=false": "[org/example/notes]\nwrap=false\n",
+ "GVariant wrap=true": "[org/example/notes]\nwrap=true\n",
+ "GVariant wallpaper=blue":
+ "[org/example/notes]\nwrap=false\n\n[org/gnome/desktop/background]\ncolor='blue'\n",
+ "GVariant desktop only": "[org/gnome/desktop/background]\ncolor='blue'\n",
+};
+const dconfReaders: DesktopStateReaders = {
+ ...readers,
+ dconf: async (path) => {
+ const dump = dumps[await readFile(path, "latin1")];
+ return dump === undefined
+ ? { ok: false, error: "not a fixture database" }
+ : { ok: true, text: dump };
+ },
+};
+
+describe("draftDesktopApplicationTask notes", () => {
+ it("has none when the draft has files", async () => {
+ const draft = await notesOf(seedEndCapture());
+
+ expect(draft.files).toHaveLength(1);
+ expect(draft.notes).toEqual([]);
+ });
+
+ it("passes on a drafted file's own notes, such as a document REA reads as binary", async () => {
+ const [before, after] = [1, 2].map((version) =>
+ Buffer.from([0x1f, 0x8b, 0x08, 0x00, version]),
+ );
+ if (before === undefined || after === undefined) throw new Error("fixture");
+ const base = seedEndCapture();
+ const compressed: DesktopApplicationCapture = {
+ ...base,
+ state: {
+ ...base.state,
+ checkpoints: base.state.checkpoints.map((point) =>
+ point.name === "seed"
+ ? {
+ ...point,
+ changes: [change(NOTE, { after: before })],
+ retained: [retained(NOTE, before)],
+ }
+ : point.name === "end"
+ ? {
+ ...point,
+ changes: [change(NOTE, { before, after })],
+ retained: [retained(NOTE, after)],
+ }
+ : point,
+ ),
+ },
+ };
+
+ const draft = await notesOf(compressed, [before, after]);
+
+ expect(draft.files).toHaveLength(1);
+ expect(draft.notes).toEqual([
+ expect.objectContaining({
+ code: "unreadable_view",
+ file: "Documents/today.txt",
+ }),
+ ]);
+ });
+
+ it("says no file changed between seed and end when none did", async () => {
+ const draft = await notesOf(
+ withEnd(seedEndCapture(), { changes: [], kept: [] }),
+ );
+
+ expect(draft.files).toEqual([]);
+ // The window title named the note, so the capture recognized a document.
+ expect(draft.notes.map(({ code }) => code)).toEqual(["no_change"]);
+ });
+
+ it("does not call reordered saved lines unchanged when the value diff is empty", async () => {
+ const base = seedEndCapture();
+ const note = `${HOME}/Documents/today.txt`;
+ const before = "milk\neggs\n";
+ const after = "eggs\nmilk\n";
+ const capture = {
+ ...base,
+ state: {
+ ...base.state,
+ checkpoints: base.state.checkpoints.map((point) =>
+ point.name === "seed"
+ ? {
+ ...point,
+ changes: [change(note, { after: before })],
+ retained: [retained(note, before)],
+ }
+ : point.name === "end"
+ ? {
+ ...point,
+ changes: [change(note, { before, after })],
+ retained: [retained(note, after)],
+ }
+ : point,
+ ),
+ },
+ };
+ const draft = await notesOf(capture, [before, after]);
+ expect(draft.files).toEqual([]);
+ expect(draft.notes.map(({ code }) => code)).not.toContain("no_change");
+ expect(draft.notes.map(({ code }) => code)).toContain(
+ "uncheckable_changes",
+ );
+ });
+
+ it("does not call an empty change list unchanged when a checkpoint was cut short", async () => {
+ const base = withEnd(seedEndCapture(), { changes: [], kept: [] });
+ const cut: DesktopApplicationCapture = {
+ ...base,
+ state: {
+ ...base.state,
+ checkpoints: base.state.checkpoints.map((point) =>
+ point.name === "end" ? { ...point, truncated: true } : point,
+ ),
+ },
+ };
+ const draft = await notesOf(cut);
+ expect(draft.notes.map(({ code }) => code)).toContain("incomplete_capture");
+ expect(draft.notes.map(({ code }) => code)).not.toContain("no_change");
+ });
+
+ it("does not call the ends unchanged when a checkpoint between them left a deletion unconfirmed", async () => {
+ const base = withEnd(seedEndCapture(), { changes: [], kept: [] });
+ const seedAt = base.state.checkpoints.findIndex(
+ (point) => point.name === "seed",
+ );
+ const seed = base.state.checkpoints[seedAt];
+ if (seed === undefined) throw new Error("the fixture has a seed");
+ const partial: DesktopApplicationCapture = {
+ ...base,
+ state: {
+ ...base.state,
+ checkpoints: [
+ ...base.state.checkpoints.slice(0, seedAt + 1),
+ {
+ ...seed,
+ name: "during",
+ trigger: "request",
+ unconfirmed_changes: 1,
+ changes: [],
+ },
+ ...base.state.checkpoints.slice(seedAt + 1),
+ ],
+ },
+ };
+ const draft = await notesOf(partial);
+
+ expect(draft.notes.map(({ code }) => code)).toContain("incomplete_capture");
+ expect(draft.notes.map(({ code }) => code)).not.toContain("no_change");
+ });
+});
+
+describe("draftDesktopApplicationTask notes for unowned files", () => {
+ it("lists the first five changed files outside the owned paths, desktop files last, when no document was recognized", async () => {
+ const base = seedEndCapture();
+ const untitled: DesktopApplicationCapture = {
+ ...base,
+ session: {
+ ...base.session,
+ windows: base.session.windows.map((window) => ({
+ ...window,
+ titles: ["Shopping - Notes"],
+ })),
+ },
+ };
+ const note = `${HOME}/Documents/today.txt`;
+ const others = ["a", "b", "c", "d", "e"].map(
+ (name) => `${HOME}/Music/${name}.txt`,
+ );
+ const fonts = `${HOME}/.cache/fontconfig/cache-1`;
+ const draft = await notesOf(
+ withEnd(untitled, {
+ changes: [
+ change(note, { before: SAVED_NOTE, after: EDITED_NOTE }),
+ ...[fonts, ...others].map((path) => change(path, { after: "x\n" })),
+ ],
+ kept: [
+ retained(note, EDITED_NOTE),
+ ...[fonts, ...others].map((path) => retained(path, "x\n")),
+ ],
+ }),
+ ["x\n"],
+ );
+
+ expect(draft.files).toEqual([]);
+ expect(draft.notes.map(({ code }) => code)).toEqual([
+ "unowned_changes",
+ "no_document",
+ ]);
+ const [unowned] = draft.notes;
+ expect(unowned?.files).toEqual([
+ { file: "Documents/today.txt", role: "unrelated" },
+ ...["a", "b", "c", "d"].map((name) => ({
+ file: `Music/${name}.txt`,
+ role: "unrelated",
+ })),
+ ]);
+ // Music/e.txt and the shared font cache are counted, not listed.
+ expect(unowned?.more).toBe(2);
+ });
+});
+
+describe("draftDesktopApplicationTask notes for retained dconf state", () => {
+ it("says a change the app saved only in dconf is one no check reads, rather than that nothing changed", async () => {
+ /** The seed-and-end capture whose only change from seed to end is the dconf database's, to `after`. */
+ const savedInDconf = (after: string): DesktopApplicationCapture => {
+ const ended = withEnd(seedEndCapture(), {
+ changes: [change(database, { before: "GVariant wrap=false", after })],
+ kept: [retained(database, after)],
+ });
+ const [baseline, ...later] = ended.state.checkpoints;
+ if (baseline === undefined) throw new Error("the fixture has a baseline");
+ return {
+ ...ended,
+ state: {
+ ...ended.state,
+ checkpoints: [
+ {
+ ...baseline,
+ retained: [
+ ...baseline.retained,
+ retained(database, "GVariant wrap=false"),
+ ],
+ },
+ ...later,
+ ],
+ },
+ };
+ };
+
+ const saved = await notesOf(
+ savedInDconf("GVariant wrap=true"),
+ Object.keys(dumps),
+ dconfReaders,
+ );
+ const desktop = await notesOf(
+ savedInDconf("GVariant wallpaper=blue"),
+ Object.keys(dumps),
+ dconfReaders,
+ );
+ // The reader fails on this database, as it does when dconf-cli is missing.
+ const unread = await notesOf(
+ savedInDconf("GVariant unknown"),
+ [...Object.keys(dumps), "GVariant unknown"],
+ dconfReaders,
+ );
+
+ expect(saved.files).toEqual([]);
+ expect(saved.notes.map(({ code }) => code)).toEqual([
+ "uncheckable_changes",
+ ]);
+ expect(saved.notes[0]?.files).toEqual([
+ { file: ".config/dconf/user", role: "application_state" },
+ ]);
+ // Desktop-only changes do not establish that no file changed.
+ expect(desktop.notes.map(({ code }) => code)).toEqual([
+ "uncheckable_changes",
+ ]);
+ // A database REA could not read changed: whose keys is unknown, and the read error is kept.
+ expect(unread.notes).toMatchObject([
+ {
+ code: "unreadable_changes",
+ files: [{ file: ".config/dconf/user", role: "application_state" }],
+ error: "not a fixture database",
+ },
+ ]);
+ });
+});
+
+describe("draftDesktopApplicationTask notes on a dconf database the capture kept no copy of", () => {
+ it("keeps a dconf database unknown when the capture kept no copy of its starting bytes", async () => {
+ /** As the recorder writes it: the first walk keeps no copy of the database, so its change has no starting digest. */
+ const unkept = (after: string): DesktopApplicationCapture => {
+ const written = change(database, {
+ before: "GVariant wrap=false",
+ after,
+ });
+ return withEnd(seedEndCapture(), {
+ changes: [
+ {
+ ...written,
+ before:
+ written.before === null
+ ? null
+ : { ...written.before, sha256: null },
+ },
+ ],
+ kept: [retained(database, after)],
+ });
+ };
+
+ const saved = await notesOf(
+ unkept("GVariant wrap=true"),
+ Object.keys(dumps),
+ dconfReaders,
+ );
+ const desktop = await notesOf(
+ unkept("GVariant desktop only"),
+ Object.keys(dumps),
+ dconfReaders,
+ );
+
+ expect(saved.notes.map(({ code }) => code)).toEqual(["unreadable_changes"]);
+ expect(saved.notes[0]?.error).toContain("content not retained");
+ expect(saved.notes[0]?.files).toEqual([
+ { file: ".config/dconf/user", role: "application_state" },
+ ]);
+ expect(desktop.notes.map(({ code }) => code)).toEqual([
+ "unreadable_changes",
+ ]);
+ });
+});
diff --git a/src/application/desktop/DesktopTaskExport.test.ts b/src/application/desktop/DesktopTaskExport.test.ts
index 2e7c80f1bd..21659f3fef 100644
--- a/src/application/desktop/DesktopTaskExport.test.ts
+++ b/src/application/desktop/DesktopTaskExport.test.ts
@@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import { exportDesktopApplication } from "./DesktopApplicationExport.js";
+import { shippedSeedCheckView } from "./DesktopTaskExport.js";
import { draftDesktopApplicationTask } from "./DesktopTaskDraft.js";
import {
NOTE as NOTE_PATH,
@@ -388,3 +389,14 @@ describe("draftDesktopApplicationTask", () => {
expect(exported).not.toBeInstanceOf(Error);
});
});
+
+describe("shippedSeedCheckView", () => {
+ const captured = { format: "binary" as const, view: null };
+ const shipped = { format: "text" as const, view: { text: "shipped" } };
+
+ it("treats a file the bundle omits as absent and keeps a shipped non-text file", () => {
+ expect(shippedSeedCheckView(false, undefined, captured)).toBeUndefined();
+ expect(shippedSeedCheckView(true, undefined, captured)).toBe(captured);
+ expect(shippedSeedCheckView(true, shipped, captured)).toBe(shipped);
+ });
+});
diff --git a/src/application/desktop/DesktopTaskExport.ts b/src/application/desktop/DesktopTaskExport.ts
index 8a48d8fa6c..dc754c1e57 100644
--- a/src/application/desktop/DesktopTaskExport.ts
+++ b/src/application/desktop/DesktopTaskExport.ts
@@ -10,6 +10,7 @@ import {
versionView,
type TaskFileView,
} from "../../domain/desktop/appTaskChecks.js";
+import { SETTLE_TIMING } from "../../domain/desktop/appSettle.js";
import {
renderTaskEval,
renderTaskSteps,
@@ -17,6 +18,7 @@ import {
taskVerifierArtifact,
} from "../../domain/desktop/appTaskPython.js";
import type { DesktopAppExportPlan } from "../../domain/desktop/appExport.js";
+import { STATE_PATH_INPUTS } from "../../domain/desktop/buildCapture.js";
import {
versionAt,
type DesktopStatePath,
@@ -41,6 +43,9 @@ import { refuse } from "./DesktopExportInstall.js";
const OPERATION = "export_desktop_application";
const REWRITE = "Fix the task file and export again.";
+/** Where the change shows on either path: a capture inside a build computer cannot be described with the CLI, so its draft comes from stop_build_capture. */
+const DRAFT =
+ "each changed file's values in the task draft (describe-desktop-application-task, or stop_build_capture on a build computer) show what it held at the seed and at the end, at the pointer a check names it by";
/** What the export knows of the app: its name, owned paths, captured home, host identity, state files by home path and the checkpoints it reads. */
type TaskTarget = Pick<
@@ -50,8 +55,10 @@ type TaskTarget = Pick<
readonly files: ReadonlyMap;
readonly identity: HostIdentity;
readonly ends: StateEnds;
- /** Views of the seeds the export rewrites, as it ships them. */
+ /** Views of the text seeds the export rewrites, as it ships them. */
readonly seedViews: ReadonlyMap;
+ /** Home paths the bundle actually ships, including files that are not text. */
+ readonly seeded: ReadonlySet;
};
/** A task the capture validated: the files it adds to the bundle, its README section and its summary. */
@@ -138,7 +145,7 @@ const targetProblem = (
views: TaskViews,
): string | undefined => {
if (!isWithinAny(check.file, target.paths))
- return `names ${check.file}, which is not under the app's owned paths (${target.paths.join(", ") || "none"}); data/get serves only those`;
+ return `names ${check.file}, which is not under the app's owned paths (${target.paths.join(", ") || "none"}); data/get serves only those. For the bundle to own it, capture again naming it with ${STATE_PATH_INPUTS}`;
// What the check compares and names; its field names, kind and id label it.
const { id: _id, kind: _kind, ...fields } = check;
for (const value of Object.values(fields).flatMap(valueStrings)) {
@@ -156,6 +163,17 @@ const targetProblem = (
: undefined;
};
+/**
+ * A seed check's view of one file. A file the bundle does not ship is absent,
+ * including a lock the capture saw. A shipped text file uses the scrubbed
+ * view; any other shipped file uses the captured view.
+ */
+export const shippedSeedCheckView = (
+ seeded: boolean,
+ shipped: TaskFileView | undefined,
+ captured: TaskFileView | undefined,
+): TaskFileView | undefined => (seeded ? (shipped ?? captured) : undefined);
+
/** Views of every file at the end checkpoint, or at the seed as the bundle ships it. */
const viewsAt = (
target: TaskTarget,
@@ -167,9 +185,15 @@ const viewsAt = (
// A file not retained at the seed is not seeded, so the computer starts without it.
if (end === "seed" && version?.present === true && version.sha256 === null)
continue;
- const shipped = end === "seed" ? target.seedViews.get(home) : undefined;
+ const captured = version === undefined ? undefined : versionView(version);
const view =
- shipped ?? (version === undefined ? undefined : versionView(version));
+ end === "seed"
+ ? shippedSeedCheckView(
+ target.seeded.has(home),
+ target.seedViews.get(home),
+ captured,
+ )
+ : captured;
if (view !== undefined) views.set(home, view);
}
return views;
@@ -184,7 +208,9 @@ const taskReadme = (
\`tasks/${task.name}.json\` deploys the app, has the agent work through the computer from the prompt below, and grades
the outcome with \`artifacts/${taskVerifierArtifact(task)}/verify.py\`, which reads the app's \`data/get\` views and needs
-no model. \`evals/${app}.toml\` names it, so a run of the bundle runs it.
+no model. It grades once the app's files have held for ${String(SETTLE_TIMING.windowMs / 1000)} s, so a save the app makes on its own within that
+time of the agent stopping counts; each result's \`settled\` is false when they were still changing ${String(SETTLE_TIMING.boundMs / 1000)} s after the
+first read. \`evals/${app}.toml\` names it, so a run of the bundle runs it.
${task.prompt
.split("\n")
@@ -248,7 +274,7 @@ export const admitDesktopTask = async (
throw refuse(
`Check ${JSON.stringify(failing.id)} fails on the captured end state: ${failing.description}.`,
"task",
- "Write checks that the captured end state passes; describe-desktop-application-task shows its views.",
+ `Write checks that the captured end state passes; ${DRAFT}.`,
);
const seed = evaluateTaskChecks(task.checks, views.seed).filter(
(outcome) => !outcome.passed,
@@ -269,7 +295,7 @@ export const admitDesktopTask = async (
.map(({ id, description }) => `${JSON.stringify(id)}: ${description}`)
.join("; ")}).`,
"task",
- "Add a check that fails on what the seed's views hold, such as a line or row of a file read whole; describe-desktop-application-task shows its views.",
+ `Add a check that fails on what the seed's views hold, such as a line or row of a file read whole; ${DRAFT}.`,
);
return {
summary: {
diff --git a/src/composition/capabilityServices.ts b/src/composition/capabilityServices.ts
index 505f509b29..fad6ecc4d0 100644
--- a/src/composition/capabilityServices.ts
+++ b/src/composition/capabilityServices.ts
@@ -118,6 +118,7 @@ import type { PcapAnalysisPort } from "../application/pcap/PcapAnalysisPort.js";
import type { FlutterBuildAnalysisPort } from "../application/flutter/FlutterBuildAnalysisPort.js";
import type { JebAnalysisPort } from "../application/jeb/JebAnalysisPort.js";
import type { AdbDeviceAnalysisPort } from "../application/adb/AdbDeviceAnalysisPort.js";
+import type { BuildSettle } from "../application/desktop/DesktopBuildSettle.js";
/** Typed injection inputs shared by a capability host, independent of transport. */
export interface CapabilityServiceOptions {
@@ -156,6 +157,8 @@ export interface CapabilityServiceOptions {
readonly electronActiveObservation?: ElectronActiveObservationPort;
readonly javascriptRuntimeObservation?: JavaScriptRuntimeObservationPort;
readonly fridaInstrumentation?: FridaInstrumentationPort;
+ /** How build computers read an app until its files settle; the shipped timing on real time unless a test shortens or clocks it. */
+ readonly desktopBuildSettle?: BuildSettle;
readonly reverseEngineering?: ReverseEngineeringService;
}
@@ -329,7 +332,11 @@ const modules = {
),
desktopApplication: service(() => createDesktopApplicationService()),
desktopBuildComputers: capability(
- ({ providerEnvironment }) => new DesktopBuildComputers(providerEnvironment),
+ ({ providerEnvironment, options }) =>
+ new DesktopBuildComputers(
+ providerEnvironment,
+ options.desktopBuildSettle,
+ ),
(value) => value.closeAll(),
),
goBinary: service(
diff --git a/src/contracts/desktopBuildToolContracts.ts b/src/contracts/desktopBuildToolContracts.ts
index bd70671268..cfc60a73ae 100644
--- a/src/contracts/desktopBuildToolContracts.ts
+++ b/src/contracts/desktopBuildToolContracts.ts
@@ -1,19 +1,27 @@
import { z } from "zod";
import { buildSourceSchema } from "../domain/desktop/appBuild.js";
+import { SETTLE_TIMING } from "../domain/desktop/appSettle.js";
import { desktopTaskSchema } from "../domain/desktop/appTask.js";
import {
buildCheckpointNameSchema,
+ buildStatePathSchema,
END_CHECKPOINT,
+ MAX_BUILD_STATE_PATHS,
SEED_CHECKPOINT,
} from "../domain/desktop/buildCapture.js";
+import { draftNoteSchema } from "../domain/desktop/appTaskDraftNotes.js";
+import { draftValueSchema } from "../domain/desktop/appTaskDraftValues.js";
import { WITNESS_STAGES } from "../domain/desktop/buildWitness.js";
import {
COMPUTER_ACTIONS,
computerActionInputSchema,
screenshotOptionsSchema,
} from "../domain/desktop/computerActions.js";
-import { desktopCheckpointSchema } from "../domain/desktop/desktopCapture.js";
+import {
+ desktopCheckpointSchema,
+ MAX_ACKNOWLEDGED_FILES,
+} from "../domain/desktop/desktopCapture.js";
import {
jsonObjectSchema,
jsonValueSchema,
@@ -48,6 +56,16 @@ const checkResult = z.strictObject({
"Present on a failure for want of a readable or whole view, such as a file the app could not read just then or a view cut short; it shows nothing about the task, so a witness stage counts no such failure.",
),
});
+const settledField = z
+ .boolean()
+ .describe(
+ `Whether the app's files had held for ${String(SETTLE_TIMING.windowMs / 1000)} s, counted from the read that returned them, when they were graded; false when they were still changing ${String(SETTLE_TIMING.boundMs / 1000)} s after the first read began, and the last completed read was graded.`,
+ );
+const readCutField = z
+ .boolean()
+ .describe(
+ `Whether a read still in flight ${String(SETTLE_TIMING.boundMs / 1000)} s after the first read began was cut there, so the last completed read was graded.`,
+ );
const strings = z.array(z.string());
const stageRecord = z.strictObject({
stage: z.enum(WITNESS_STAGES),
@@ -58,6 +76,8 @@ const stageRecord = z.strictObject({
checked_at: z.string(),
expectation: z.enum(["some_fail", "all_pass"]),
held: z.boolean(),
+ settled: settledField,
+ read_cut: readCutField,
results: z.array(checkResult),
remediation: z
.string()
@@ -121,7 +141,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
...toolContractMetadata("start_build_computer"),
kind: "application",
description:
- "Start a local agentenv computer to build a desktop app environment on, and return its id with a first screenshot as an image. From a request, REA renders a bundle that installs the app from the Ubuntu archive, an apt repository or a .deb URL, opens it on demand from its desktop entry (it is not open at first: open it from the desktop's Applications menu, under Other, by the environment's name), and installs REA itself in a build-only layer from REA_BUILD_COMPUTER_REA_ARCHIVE. Set source.electron for an Electron app: the install rewrites that desktop entry's command with the same launch policy an export uses, adding --no-sandbox right after the program unless the command already has it, because the container does not grant the user namespaces Chromium needs and Electron exits without the flag. From a bundle folder, such as an exported one, it starts that bundle's smoke task as it is. Needs Docker and the agentenv runner named by REA_AGENT_ENV_COMMAND; the first start of a bundle builds its image, which downloads packages and can take minutes. screenshot sets the first screenshot's region, format and quality. The computer stays up, reachable on this host, until close_build_computer or the server closes; when a start fails before its teardown is confirmed, the error's computer_id names what close_build_computer retries.",
+ "Start a local agentenv computer to build a desktop app environment on, and return its id with a first screenshot as an image. From a request, REA renders a bundle that installs the app from the Ubuntu archive, an apt repository or a .deb URL, opens it on demand from its desktop entry (it is not open at first: start_build_capture opens it under capture, and only when no capture is under way is it opened from the desktop's Applications menu, under Other, by the environment's name), and installs REA itself in a build-only layer from REA_BUILD_COMPUTER_REA_ARCHIVE. Set source.electron for an Electron app: the install rewrites that desktop entry's command with the same launch policy an export uses, adding --no-sandbox right after the program unless the command already has it, because the container does not grant the user namespaces Chromium needs and Electron exits without the flag. From a bundle folder, such as an exported one, it starts that bundle's smoke task as it is. Needs Docker and the agentenv runner named by REA_AGENT_ENV_COMMAND; the first start of a bundle builds its image, which downloads packages and can take minutes, and agentenv makes a start wait while another agentenv run on this host writes its bundle's ids, so starts at the same time can wait minutes for each other. screenshot sets the first screenshot's region, format and quality. The computer stays up, reachable on this host, until close_build_computer or the server closes; when a start fails before its teardown is confirmed, the error's computer_id names what close_build_computer retries.",
inputSchema: z.strictObject({
source: buildSourceSchema,
screenshot: screenshotOptionsSchema.optional(),
@@ -183,7 +203,11 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
},
{
title: "Save",
- input: { computer_id: EXAMPLE_ID, action: "key", keys: ["ctrl+s"] },
+ input: {
+ computer_id: EXAMPLE_ID,
+ action: "key",
+ keys: ["Control", "s"],
+ },
},
],
},
@@ -191,7 +215,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
name: "start_build_capture",
...toolContractMetadata("start_build_capture"),
kind: "application",
- description: `Begin REA's capture of the app on a build computer started from a request, inside the computer: REA downloads the package lists as the computer's user so the capture finds the app's sources, launches the app from its desktop entry under capture (do not open it from the menu first) and returns once its window is open, with a screenshot. Then make the seed through the app and save it, mark_build_checkpoint ${SEED_CHECKPOINT}, make the change the task asks for and save it, mark_build_checkpoint ${END_CHECKPOINT}, and stop_build_capture.`,
+ description: `Begin REA's capture of the app on a build computer started from a request, inside the computer: REA downloads the package lists as the computer's user so the capture finds the app's sources, launches the app from its desktop entry under capture (do not open it from the menu first) and returns once its window is open, with a screenshot. When the app exits before its window opens, the error gives its exit code and the end of its stderr, with the capture's record of the session in its details. REA counts a file as the app's document when an app window's title shows its name or ~/ path; name the file the app will save in state_paths when its title shows something else, such as a score's title. Then make the seed through the app and save it, mark_build_checkpoint ${SEED_CHECKPOINT}, make the change the task asks for and save it, mark_build_checkpoint ${END_CHECKPOINT}, and stop_build_capture.`,
inputSchema: z.strictObject({
computer_id: computerId,
arguments: z
@@ -201,6 +225,13 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
.describe(
"Arguments appended to the app's launch command, such as a file to open.",
),
+ state_paths: z
+ .array(buildStatePathSchema)
+ .max(MAX_BUILD_STATE_PATHS)
+ .optional()
+ .describe(
+ "Files or folders in the computer's home that hold the app's state, relative to the home (Documents/Scores/song.mscz), after ~/ or absolute under it; they need not exist yet. The capture retains them from its start and counts their files as the app's: a document, which the bundle owns, seeds and opens at launch when it is the seed's only one, or the app's own state under a hidden folder such as .config/MuseScore. An absolute path outside the home is refused.",
+ ),
}),
outputSchema: z.strictObject({
computer_id: z.uuid(),
@@ -222,13 +253,20 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
title: "Capture the requested app",
input: { computer_id: EXAMPLE_ID },
},
+ {
+ title: "Capture an app whose title shows a score's title, not its file",
+ input: {
+ computer_id: EXAMPLE_ID,
+ state_paths: ["Documents/MuseScore3/Scores/Morning_Song.mscz"],
+ },
+ },
],
},
{
name: "mark_build_checkpoint",
...toolContractMetadata("mark_build_checkpoint"),
kind: "application",
- description: `Take a named checkpoint of the app's state in the capture under way and wait for the capture to acknowledge it, with its counts: entries walked, paths changed since the checkpoint before, and files retained. Mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the change is; a name already taken is refused.`,
+ description: `Take a named checkpoint of the app's state in the capture under way and wait for the capture to acknowledge it, with its counts (entries walked, paths changed since the checkpoint before, files retained) and up to ${String(MAX_ACKNOWLEDGED_FILES)} of the files that changed, those under state_paths first, then those the app held open, caches last, so you can see which file a save wrote. Mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the change is; a name already taken is refused.`,
inputSchema: z.strictObject({
computer_id: computerId,
name: buildCheckpointNameSchema,
@@ -243,6 +281,20 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
changes: z.int().nonnegative(),
retained_files: z.int().nonnegative(),
retained_bytes: z.int().nonnegative(),
+ changed_files: z
+ .array(z.string())
+ .max(MAX_ACKNOWLEDGED_FILES)
+ .nullable()
+ .describe(
+ "Files, not folders, changed since the checkpoint before, relative to the home: those under state_paths, those the app held open, those in its state directories, the rest, then caches and browser-engine storage, each by path. null when the REA in the computer is older and does not report them, so which files changed is unknown; the checkpoint was still taken.",
+ ),
+ more_changed_files: z
+ .int()
+ .nonnegative()
+ .nullable()
+ .describe(
+ "How many more files changed than changed_files names; null when changed_files is.",
+ ),
}),
}),
examples: [
@@ -256,7 +308,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
name: "stop_build_capture",
...toolContractMetadata("stop_build_capture"),
kind: "application",
- description: `Stop the capture: the app is closed and the capture finishes inside the computer. Returns its checkpoints, the package and sources it found, its limitations and, when ${SEED_CHECKPOINT} and ${END_CHECKPOINT} were marked, the task draft between them: the files that changed, with their net change, and suggested checks. Write the task from the draft, its prompt in the user's words, and give it to export_build_environment.`,
+ description: `Stop the capture: the app is closed and the capture finishes inside the computer. Returns its checkpoints, the package and sources it found, its limitations and, when ${SEED_CHECKPOINT} and ${END_CHECKPOINT} were marked, the task draft between them: the files that changed under the paths the bundle owns, each with its net change and, for XML files, archives, JSON and key files, the values it added, changed and removed (before and after, at the pointers a check names them by); suggested checks; and guards, optional checks that the rest stays as it was. Notes say why a draft is empty (nothing changed; the changes lie outside the owned paths, listed with their roles; the app saved its change only in dconf, which no check reads; a dconf database changed that REA could not read; or REA recognized no document: capture again with state_paths naming the file, or make a change the app saves in a file) or why a file's suggestions are thin (only position names the change, it is all view state, its text is too long, or its view is cut or unreadable). Write the task from the draft, its prompt in the user's words, and give it to export_build_environment.`,
inputSchema: z.strictObject({ computer_id: computerId }),
outputSchema: z.strictObject({
computer_id: z.uuid(),
@@ -285,6 +337,14 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
file: z.string(),
role: z.string(),
change: jsonValueSchema,
+ values: z
+ .array(draftValueSchema)
+ .describe(
+ "The values the change added, changed and removed, ready to copy into a check: what a check can name first, then what only position reaches, then noise; at most 20 entries and 8 KB a file and 24 KB a draft. Only XML files, archives, JSON and key files have values: a text file's or SQLite database's change already lists its lines and rows.",
+ ),
+ values_cut: z
+ .boolean()
+ .describe("Whether a limit left values out."),
}),
),
task: z.strictObject({
@@ -292,6 +352,16 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
prompt: z.string(),
checks: z.array(jsonObjectSchema),
}),
+ guards: z
+ .array(jsonObjectSchema)
+ .describe(
+ "Optional checks that the rest keeps its seed value, which the end holds too: a field on like siblings, one that varies among them before a default they all share; the text an addition kept; a text file's lines beside its new one. Add those worth keeping to the task's checks.",
+ ),
+ notes: z
+ .array(draftNoteSchema)
+ .describe(
+ "Why the draft has no files, or why a file's suggested checks are thin; empty when neither.",
+ ),
})
.nullable(),
draft_refusal: z.string().optional(),
@@ -327,6 +397,8 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
task: jsonValueSchema,
seed_files: strings,
scrubbed_files: strings,
+ // gzip seeds compressed again only to clear their headers, their documents as captured.
+ recompressed_files: strings,
unscrubbed_files: strings,
limitations: strings,
}),
@@ -345,7 +417,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
name: "check_build_task",
...toolContractMetadata("check_build_task"),
kind: "application",
- description: `Grade a task's checks against the app's live state on a computer of an exported bundle: REA reads the app's data/get views and evaluates each check as the bundle's verifier does, one result per check. With stage, the check is part of the witness, in order: seed (give directory; REA begins recording) must fail a check, and on a computer started from a bundle folder it refuses a task whose steps and verifier in the bundle differ from what the export writes for it; after_edit, after acting on the app from the task's prompt alone, must pass every check; near_miss, after a deliberately almost-right change, must fail one; after_reset, after act_on_build_computer action reset, must give each check the result it had on the seed. Only a check that fails on what the app's view shows counts as failing: one that fails because the view could not be read or is cut short is undecided, and a stage only undecided checks would settle files nothing and is refused, to be checked again. A stage that does not hold, or after_reset, ends the witness: REA ends the recording and writes witness.json, the record as Evidence, with the recording's segments into directory, and returns the record.`,
+ description: `Grade a task's checks against the app's live state on a computer of an exported bundle: REA reads the app's data/get views and evaluates each check as the bundle's verifier does, one result per check, once the app's files settle: it reads data/get every ${String(SETTLE_TIMING.intervalMs / 1000)} s and grades once every file has held for ${String(SETTLE_TIMING.windowMs / 1000)} s, counted from the read that returned its bytes, so an app that saves a few seconds after the last key is graded on what it saved; no read begins ${String(SETTLE_TIMING.boundMs / 1000)} s or more after the first one began, one still in flight then is cut (read_cut), and files still changing are graded on the last completed read, with settled false. With stage, the check is part of the witness, in order: seed (give directory; REA begins recording) must fail a check, and on a computer started from a bundle folder it refuses a task whose steps and verifier in the bundle differ from what the export writes for it; after_edit, after acting on the app from the task's prompt alone, must pass every check; near_miss, after a deliberately almost-right change, must fail one; after_reset, after act_on_build_computer action reset, must give each check the result it had on the seed. Only a check that fails on what the app's view shows counts as failing: one that fails because the view could not be read or is cut short is undecided, and a stage only undecided checks would settle files nothing and is refused, to be checked again. A stage that does not hold, or after_reset, ends the witness: REA ends the recording and writes witness.json, the record as Evidence, with the recording's segments into directory, and returns the record.`,
inputSchema: z.strictObject({
computer_id: computerId,
task: desktopTaskSchema,
@@ -367,6 +439,8 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
passed: z.int().nonnegative(),
failed: z.int().nonnegative(),
results: z.array(checkResult),
+ settled: settledField,
+ read_cut: readCutField,
witness: z
.strictObject({
stage: z.enum(WITNESS_STAGES),
diff --git a/src/desktop/AgentEnvComputer.ts b/src/desktop/AgentEnvComputer.ts
index 094f5abbaf..698af453ea 100644
--- a/src/desktop/AgentEnvComputer.ts
+++ b/src/desktop/AgentEnvComputer.ts
@@ -5,6 +5,7 @@ import { getHeapStatistics } from "node:v8";
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
import { z } from "zod";
+import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js";
import { StructuredContentMcpClient } from "../providers/StructuredContentMcpClient.js";
import { validateMcpHttpResponse } from "../providers/Utf8McpResponse.js";
@@ -26,10 +27,105 @@ const ENVELOPE_BYTES = 64 * 1024;
/** An exec answer: two streams of at most 1 MiB each, which JSON escaping can grow sixfold. */
const EXEC_RESPONSE_BYTES = 16 * 1024 * 1024;
/** The base64 answer for `bytes` of files, with each file's envelope. */
-const dataAnswerBytes = (bytes: number, files: number) =>
+export const guestFileAnswerBytes = (bytes: number, files: number): number =>
Math.ceil(bytes / 3) * 4 + ENVELOPE_BYTES * Math.max(files, 1);
/** The most REA reads into memory for one answer of unknown size: a quarter of the V8 heap, as for recordings. */
const heapQuarter = () => Math.floor(getHeapStatistics().heap_size_limit / 4);
+/** Where the data/get answer ceiling comes from. The guest's reported size does not change it. */
+export const GUEST_FILE_ANSWER_SOURCE = "v8.heap_size_limit/4";
+/**
+ * Copies of one answer held together while it is parsed: the raw chunks, the
+ * concatenated buffer, and the text. The decoded files replace the text
+ * afterward and are smaller than the base64 body.
+ */
+export const GUEST_FILE_ANSWER_COPIES = 3;
+
+/** The host ceiling for one data/get answer, including base64 expansion and each file's envelope. */
+export const guestFileAnswerCeiling = (): number => heapQuarter();
+
+/** Wire bytes that fit in the ceiling once the parse's copies are counted. */
+export const guestFileAnswerWorkingBytes = (answerCeiling: number): number =>
+ Math.floor(answerCeiling / GUEST_FILE_ANSWER_COPIES);
+
+const guestFileAnswerError = (
+ fileBytes: number,
+ files: number,
+ answerCeiling: number,
+ remediation: string,
+ limits: Readonly>,
+): AnalysisResourceConstraintError =>
+ new AnalysisResourceConstraintError(
+ "data/get",
+ "memory",
+ remediation,
+ {
+ guest_file_bytes: fileBytes,
+ file_count: files,
+ answer_ceiling_bytes: answerCeiling,
+ answer_working_bytes: guestFileAnswerWorkingBytes(answerCeiling),
+ answer_copies: GUEST_FILE_ANSWER_COPIES,
+ ceiling_source: GUEST_FILE_ANSWER_SOURCE,
+ ...limits,
+ },
+ { remediationAction: remediation, retryable: false },
+ );
+
+/**
+ * Refuse a data/get whose base64 answer, counted once for each copy the parse
+ * holds, would exceed the host ceiling. A larger report is refused before the
+ * answer is read. The reported size does not raise the ceiling.
+ */
+export const guestFileAnswerRefusal = (
+ fileBytes: number,
+ files: number,
+ answerCeiling: number,
+): AnalysisResourceConstraintError | undefined => {
+ const answerBytes = guestFileAnswerBytes(fileBytes, files);
+ const workingBytes = guestFileAnswerWorkingBytes(answerCeiling);
+ if (answerBytes <= workingBytes) return undefined;
+ const remediation = `This data/get answer is limited to ${String(workingBytes)} wire bytes, one quarter of this process's V8 heap divided by ${String(GUEST_FILE_ANSWER_COPIES)} copies held while the answer is parsed (source ${GUEST_FILE_ANSWER_SOURCE}; the heap quarter is ${String(answerCeiling)} bytes). The computer reported ${String(fileBytes)} file bytes across ${String(files)} paths, which expand to ${String(answerBytes)} bytes once each file is base64 and its envelope is counted. The reported size does not raise the ceiling. Restart REA with a larger Node --max-old-space-size so this heap quarter covers the answer.`;
+ return guestFileAnswerError(fileBytes, files, answerCeiling, remediation, {
+ answer_bytes: answerBytes,
+ });
+};
+
+/** One decoded part of a data/get answer. */
+export interface GuestFilePart {
+ readonly path: string | undefined;
+ readonly bytes: Buffer;
+}
+
+/**
+ * The decoded files, or a refusal when they contain more bytes than the
+ * computer reported. An underreported size must not admit the larger body.
+ */
+export const retainGuestFiles = (
+ parts: readonly GuestFilePart[],
+ reportedBytes: number,
+ files: number,
+ answerCeiling: number,
+):
+ | { readonly ok: true; readonly files: ReadonlyMap }
+ | { readonly ok: false; readonly error: AnalysisResourceConstraintError } => {
+ let decodedBytes = 0;
+ const retained = new Map();
+ for (const part of parts) {
+ decodedBytes += part.bytes.length;
+ if (part.path !== undefined) retained.set(part.path, part.bytes);
+ }
+ if (decodedBytes <= reportedBytes) return { ok: true, files: retained };
+ const remediation = `The computer reported ${String(reportedBytes)} file bytes, but the data/get answer decoded to ${String(decodedBytes)} bytes. The extra bytes were not kept. A reported size is only a ceiling when the answer is no larger.`;
+ return {
+ ok: false,
+ error: guestFileAnswerError(
+ reportedBytes,
+ files,
+ answerCeiling,
+ remediation,
+ { decoded_file_bytes: decodedBytes },
+ ),
+ };
+};
const readinessSchema = z.object({
ready: z.boolean(),
@@ -128,18 +224,37 @@ const readBounded = async (
return Buffer.concat(chunks, bytes).toString("utf8");
};
+/** Why a computer did not come up. Readiness is a fact, not a phrase in the message. */
+export type ComputerStartReason = "handshake" | "not_ready" | "unusable";
+
/** Why a computer did not come up, with what the caller can do about it. */
export class ComputerStartFailure extends Error {
+ readonly reason: ComputerStartReason;
+
constructor(
message: string,
readonly remediation: string | undefined,
- options?: ErrorOptions,
+ options?: ErrorOptions & { readonly reason?: ComputerStartReason },
) {
super(message, options);
this.name = "ComputerStartFailure";
+ this.reason = options?.reason ?? "unusable";
}
}
+/**
+ * Whether `cause` is a start that failed because the computer never became
+ * ready. The helper's own sentence is classified once, where it is read.
+ */
+export const computerFailedOnReadiness = (
+ cause: unknown,
+): cause is ComputerStartFailure =>
+ cause instanceof ComputerStartFailure && cause.reason === "not_ready";
+
+/** agentenv's probe says the computer never became ready. That sentence is outside REA. */
+const helperWasNotReady = (error: string | undefined) =>
+ error !== undefined && /\bnot ready\b|wasn't ready|didn't boot/u.test(error);
+
/** Every few seconds while a computer comes up, report the phase and agentenv's latest line. */
const progressHeartbeat = (
helper: ComputerHelper,
@@ -257,9 +372,15 @@ export class AgentEnvComputer {
throw new ComputerStartFailure(
`agentenv did not bring the computer up: ${handshake?.error ?? "its helper ended without a handshake"}.`,
"Read the helper's output; `agent-env run --task smoke --no-emulate` reproduces the deploy.",
+ {
+ reason: helperWasNotReady(handshake?.error)
+ ? "not_ready"
+ : "handshake",
+ },
);
phase.name = "waiting for the computer's readiness checks";
let client: StructuredContentMcpClient | undefined;
+ let stage: "readiness" | undefined;
try {
client = await connectDriver(handshake.mcp_url, signal);
const computer = new AgentEnvComputer({
@@ -269,7 +390,9 @@ export class AgentEnvComputer {
helper,
client,
});
+ stage = "readiness";
await computer.#waitReady(signal);
+ stage = undefined;
phase.name = "waiting for the desktop to draw";
const frame = await drawnScreenshot(
(retry) => computer.callTool("computer_screenshot", {}, retry),
@@ -279,10 +402,11 @@ export class AgentEnvComputer {
} catch (cause: unknown) {
await client?.close().catch(() => undefined);
if (signal.aborted) throw signal.reason;
+ if (cause instanceof ComputerStartFailure) throw cause;
throw new ComputerStartFailure(
`The computer came up but REA could not use it: ${cause instanceof Error ? cause.message : String(cause)}`,
undefined,
- { cause },
+ { cause, reason: stage === "readiness" ? "not_ready" : "unusable" },
);
}
}
@@ -410,6 +534,9 @@ export class AgentEnvComputer {
bytes: number,
signal?: AbortSignal,
): Promise> {
+ const ceiling = guestFileAnswerCeiling();
+ const refusal = guestFileAnswerRefusal(bytes, paths.length, ceiling);
+ if (refusal !== undefined) throw refusal;
const answer = dataGetSchema.parse(
await this.#request(
"/agentenv",
@@ -417,7 +544,10 @@ export class AgentEnvComputer {
signal,
{
limit: {
- bytes: dataAnswerBytes(bytes, paths.length),
+ bytes: Math.min(
+ guestFileAnswerBytes(bytes, paths.length),
+ guestFileAnswerWorkingBytes(ceiling),
+ ),
what: `The computer's answer for ${paths.join(", ")}`,
},
},
@@ -425,17 +555,26 @@ export class AgentEnvComputer {
);
if ("error" in answer)
throw new Error(
- `data/get failed: ${this.#redacted(answer.error.message)}`,
+ `data/get failed: ${this.redacted(answer.error.message)}`,
);
- return new Map(
+ const retained = retainGuestFiles(
answer.result.parts.flatMap((part) => {
- const path = Object.values(part.metadata ?? {})[0]?.path;
const encoded = part.file?.bytes;
- return path === undefined || encoded === undefined
+ return encoded === undefined
? []
- : [[path, Buffer.from(encoded, "base64")] as const];
+ : [
+ {
+ path: Object.values(part.metadata ?? {})[0]?.path,
+ bytes: Buffer.from(encoded, "base64"),
+ },
+ ];
}),
+ bytes,
+ paths.length,
+ ceiling,
);
+ if (!retained.ok) throw retained.error;
+ return retained.files;
}
/** Run a command in the computer's desktop session as its user; output is cut at the computer's cap. */
@@ -460,8 +599,8 @@ export class AgentEnvComputer {
);
return {
...result,
- stdout: this.#redacted(result.stdout),
- stderr: this.#redacted(result.stderr),
+ stdout: this.redacted(result.stdout),
+ stderr: this.redacted(result.stderr),
};
}
@@ -498,7 +637,7 @@ export class AgentEnvComputer {
),
);
if ("error" in answer)
- throw new AppDataRefusal(method, this.#redacted(answer.error.message));
+ throw new AppDataRefusal(method, this.redacted(answer.error.message));
return method === "data/get"
? z.object({ parts: z.array(z.unknown()) }).parse(answer.result).parts
: [];
@@ -529,16 +668,19 @@ export class AgentEnvComputer {
const waiting = status.checks.filter((check) => !check.ok);
const failed = waiting.find((check) => check.final === true);
if (failed !== undefined || Date.now() > deadline)
- throw new Error(
+ throw new ComputerStartFailure(
failed === undefined
? `The computer was not ready within ${String(READY_TIMEOUT_MS / 1000)} s; waiting on ${waiting.map((check) => check.name).join(", ")}.`
: `The computer's ${failed.name} check failed: ${failed.detail ?? "no detail"}.`,
+ "Read the helper's output; `agent-env run --task smoke --no-emulate` reproduces the deploy.",
+ { reason: "not_ready" },
);
await delay(500, undefined, { signal });
}
}
- #redacted(text: string): string {
+ /** `text` with the control token replaced, as every answer REA reads from the computer's routes and commands. */
+ redacted(text: string): string {
return this.#token === null
? text
: text.replaceAll(this.#token, "[control token]");
@@ -576,7 +718,7 @@ export class AgentEnvComputer {
const text = await readBounded(response, limit.bytes, limit.what);
if (response.status < 200 || response.status > 299)
throw new Error(
- `The computer answered ${path} with HTTP ${String(response.status)}: ${this.#redacted(text.slice(0, 512))}`,
+ `The computer answered ${path} with HTTP ${String(response.status)}: ${this.redacted(text.slice(0, 512))}`,
);
return JSON.parse(text) as unknown;
}
diff --git a/src/desktop/ApplicationStateDirectories.ts b/src/desktop/ApplicationStateDirectories.ts
index 8c81e887ed..301e6b3599 100644
--- a/src/desktop/ApplicationStateDirectories.ts
+++ b/src/desktop/ApplicationStateDirectories.ts
@@ -54,8 +54,9 @@ const recasedEntries = async (
/**
* Folder names under XDG and dot-directories where the application may keep
- * state, and the dotfiles in the home named after it, which an application
- * may only read during a capture.
+ * state, KDE's `rc` settings file in the config folder, and the
+ * dotfiles in the home named after it, which an application may only read
+ * during a capture.
*/
export const stateDirectories = async (
names: readonly (string | null)[],
@@ -88,10 +89,13 @@ export const stateDirectories = async (
...bases.map((base) => recasedEntries(base, "", lowered)),
recasedEntries(home, ".", lowered),
]);
+ const [config = join(home, ".config")] = bases;
const paths = new Set([
...[...variants].flatMap((name) => [
...bases.map((base) => join(base, name)),
join(home, `.${name}`),
+ // KDE's settings file, such as Tellico's `~/.config/tellicorc`, which the app may first write during the capture.
+ ...(name.length < 3 ? [] : [join(config, `${name}rc`)]),
]),
...recased.flat(),
...dotfiles,
diff --git a/src/desktop/BuildComputerController.test.ts b/src/desktop/BuildComputerController.test.ts
index 366b45afd7..017a297e83 100644
--- a/src/desktop/BuildComputerController.test.ts
+++ b/src/desktop/BuildComputerController.test.ts
@@ -1,8 +1,22 @@
+import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
-import { fetchHomeFolder } from "./BuildComputerController.js";
+import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
+import { guestFileAnswerBytes } from "./AgentEnvComputer.js";
+import {
+ batchListedFiles,
+ controllerArgumentPieces,
+ fetchHomeFolder,
+ LINUX_ARGUMENT_VECTOR_BYTES,
+ LINUX_SINGLE_ARGUMENT_BYTES,
+ runBuildController,
+} from "./BuildComputerController.js";
const MIB = 1024 * 1024;
const sha256 = (bytes: Buffer) =>
@@ -29,6 +43,7 @@ const computer = (files: ReadonlyMap, steps: string[]) => {
truncated: false,
timed_out: false,
}),
+ redacted: (text: string) => text,
homeFiles: (paths: readonly string[], bytes: number) => {
if (paths[0]?.endsWith(".listing.json") === true)
return Promise.resolve(new Map([[paths[0], listing]]));
@@ -87,3 +102,158 @@ describe("fetchHomeFolder", () => {
expect(steps).toEqual([]);
});
});
+
+describe("batchListedFiles", () => {
+ it("starts another batch before each file's envelope pushes the answer over the budget", () => {
+ const files = [1, 2, 3].map((index) => ({
+ path: `f${String(index)}`,
+ size: 10,
+ }));
+ const budget = guestFileAnswerBytes(20, 2);
+ expect(
+ batchListedFiles(files, {
+ rawBytes: 8 * 1024 * 1024,
+ answerBytes: budget,
+ }).map((batch) => batch.files.length),
+ ).toEqual([2, 1]);
+ });
+});
+
+describe("controller arguments", () => {
+ it("splits a full start request so each argument fits Linux and the pieces join back", async () => {
+ const launchArguments = Array.from({ length: 32 }, () => "a".repeat(4096));
+ const payload = {
+ work: ".rea-build",
+ capture: ".rea-build/capture",
+ desktop_entry: "notes.desktop",
+ arguments: launchArguments,
+ state_paths: [] as string[],
+ timeout_s: 120,
+ apt_timeout_s: 600,
+ };
+ let argv: readonly string[] = [];
+ await runBuildController(
+ {
+ exec: (next) => {
+ argv = next;
+ return Promise.resolve({
+ exit_code: 0,
+ stdout: "{}\n",
+ stderr: "",
+ truncated: false,
+ timed_out: false,
+ });
+ },
+ homeFiles: () => Promise.resolve(new Map()),
+ redacted: (text) => text,
+ },
+ "start",
+ payload,
+ { timeoutS: 1 },
+ );
+ expect(argv.slice(0, 5)).toEqual([
+ "python3",
+ "-I",
+ "-c",
+ expect.any(String),
+ "start",
+ ]);
+ for (const argument of argv)
+ expect(Buffer.byteLength(argument)).toBeLessThanOrEqual(
+ LINUX_SINGLE_ARGUMENT_BYTES,
+ );
+ expect(Buffer.byteLength(JSON.stringify(payload))).toBeGreaterThan(
+ LINUX_SINGLE_ARGUMENT_BYTES,
+ );
+ expect(JSON.parse(argv.slice(5).join(""))).toEqual(payload);
+ });
+
+ it("refuses a request whose arguments together exceed Linux's command limit", async () => {
+ let called = false;
+ const payload = { prompt: "a".repeat(LINUX_ARGUMENT_VECTOR_BYTES) };
+ await expect(
+ runBuildController(
+ {
+ exec: () => {
+ called = true;
+ return Promise.resolve({
+ exit_code: 0,
+ stdout: "{}\n",
+ stderr: "",
+ truncated: false,
+ timed_out: false,
+ });
+ },
+ homeFiles: () => Promise.resolve(new Map()),
+ redacted: (text) => text,
+ },
+ "run",
+ payload,
+ { timeoutS: 1 },
+ ),
+ ).rejects.toMatchObject({
+ _tag: "AnalysisResourceConstraintError",
+ resource: "transport",
+ reportedLimits: {
+ actual_bytes: expect.any(Number),
+ max_bytes: LINUX_ARGUMENT_VECTOR_BYTES,
+ },
+ remediationAction: expect.any(String),
+ } satisfies Partial);
+ expect(called).toBe(false);
+ });
+
+ it("keeps a character split on a UTF-16 pair whole and joins the controller's real entry point", async () => {
+ const text = `${"a".repeat(LINUX_SINGLE_ARGUMENT_BYTES - 1)}\u{1F600}rest`;
+ const pieces = controllerArgumentPieces(text);
+ expect(pieces.join("")).toBe(text);
+ for (const piece of pieces)
+ expect(Buffer.byteLength(piece)).toBeLessThanOrEqual(
+ LINUX_SINGLE_ARGUMENT_BYTES,
+ );
+ const root = await mkdtemp(join(tmpdir(), "rea-controller-args-"));
+ try {
+ const output = join(root, "listing.json");
+ const payload = JSON.stringify({ root, output });
+ const mid = Math.floor(payload.length / 2);
+ const script = fileURLToPath(
+ new URL("../../bridge/agentenv/capture.py", import.meta.url),
+ );
+ const stdout = execFileSync(
+ "python3",
+ ["-I", "-B", script, "list", payload.slice(0, mid), payload.slice(mid)],
+ { encoding: "utf8" },
+ );
+ const answer: unknown = JSON.parse(
+ stdout.trim().split("\n").at(-1) ?? "",
+ );
+ expect(answer).toMatchObject({
+ listing_bytes: expect.any(Number),
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+});
+
+describe("batchListedFiles", () => {
+ it("splits many tiny files before their envelopes pass the answer budget", () => {
+ const files = Array.from({ length: 3000 }, (_, index) => ({
+ path: `home/file-${String(index)}`,
+ size: 1,
+ }));
+ const answerBytes = guestFileAnswerBytes(1, 1);
+ const batches = batchListedFiles(files, {
+ rawBytes: 8 * MIB,
+ answerBytes,
+ });
+
+ expect(batches.length).toBe(files.length);
+ expect(
+ batches.every(
+ (batch) =>
+ guestFileAnswerBytes(batch.bytes, batch.files.length) <= answerBytes,
+ ),
+ ).toBe(true);
+ });
+});
diff --git a/src/desktop/BuildComputerController.ts b/src/desktop/BuildComputerController.ts
index 2bb829d4ba..7d31cb419f 100644
--- a/src/desktop/BuildComputerController.ts
+++ b/src/desktop/BuildComputerController.ts
@@ -1,22 +1,72 @@
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
+import { Buffer } from "node:buffer";
import { z } from "zod";
+import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
import type { JsonValue } from "../domain/jsonValue.js";
-import type { AgentEnvComputer } from "./AgentEnvComputer.js";
+import {
+ guestFileAnswerBytes,
+ guestFileAnswerCeiling,
+ guestFileAnswerWorkingBytes,
+ type AgentEnvComputer,
+} from "./AgentEnvComputer.js";
const CONTROLLER = fileURLToPath(
new URL("../../bridge/agentenv/capture.py", import.meta.url),
);
/** The most file bytes one `data/get` brings back, so a large bundle arrives in several answers. */
const FETCH_BATCH_BYTES = 8 * 1024 * 1024;
+/**
+ * Linux `MAX_ARG_STRLEN` on a 4 KiB page, excluding the NUL the kernel counts.
+ * One argument past this fails with E2BIG before the process starts.
+ */
+export const LINUX_SINGLE_ARGUMENT_BYTES = 32 * 4096 - 1;
+/**
+ * A usual Linux `ARG_MAX`, leaving room for the environment the kernel counts
+ * with the arguments. A larger vector fails with E2BIG before the process starts.
+ */
+export const LINUX_ARGUMENT_VECTOR_BYTES = 2 * 1024 * 1024 - 256 * 1024;
+
+const argumentVectorBytes = (argv: readonly string[]): number =>
+ argv.reduce((sum, argument) => sum + Buffer.byteLength(argument) + 1, 0);
let source: Promise | undefined;
-/** The computer routes REA's build controller uses. */
-export type ControlledComputer = Pick;
+const isLowSurrogate = (text: string, index: number): boolean => {
+ const code = text.charCodeAt(index);
+ return code >= 0xdc00 && code <= 0xdfff;
+};
+
+/** Pieces of `text` that each fit in one Linux argument and join back to it. */
+export const controllerArgumentPieces = (text: string): readonly string[] => {
+ const pieces: string[] = [];
+ let start = 0;
+ while (start < text.length) {
+ let low = start + 1;
+ let high = text.length;
+ while (low < high) {
+ const mid = Math.ceil((low + high) / 2);
+ const bytes = Buffer.byteLength(text.slice(start, mid));
+ if (bytes <= LINUX_SINGLE_ARGUMENT_BYTES) low = mid;
+ else high = mid - 1;
+ }
+ let end = low;
+ if (end < text.length && isLowSurrogate(text, end)) end -= 1;
+ if (end <= start) end = Math.min(text.length, start + 1);
+ pieces.push(text.slice(start, end));
+ start = end;
+ }
+ return pieces;
+};
+
+/** The computer routes REA's build controller uses, and the control token's redaction for what it reads. */
+export type ControlledComputer = Pick<
+ AgentEnvComputer,
+ "exec" | "homeFiles" | "redacted"
+>;
/**
* Run one command of REA's build controller in the computer, as its user and
@@ -31,15 +81,42 @@ export const runBuildController = async (
): Promise>> => {
const { timeoutS, signal } = options;
source ??= readFile(CONTROLLER, "utf8");
+ const script = await source;
+ const scriptBytes = Buffer.byteLength(script);
+ if (scriptBytes > LINUX_SINGLE_ARGUMENT_BYTES)
+ throw new AnalysisResourceConstraintError(
+ "build controller",
+ "transport",
+ "The controller source exceeds Linux's per-argument limit.",
+ { actual_bytes: scriptBytes, max_bytes: LINUX_SINGLE_ARGUMENT_BYTES },
+ {
+ remediationAction:
+ "Use a REA runtime whose controller fits the Linux argument limit.",
+ retryable: false,
+ },
+ );
+ const argv = [
+ "python3",
+ "-I",
+ "-c",
+ script,
+ command,
+ ...controllerArgumentPieces(JSON.stringify(argumentsJson)),
+ ];
+ const vectorBytes = argumentVectorBytes(argv);
+ if (vectorBytes > LINUX_ARGUMENT_VECTOR_BYTES)
+ throw new AnalysisResourceConstraintError(
+ "build controller",
+ "transport",
+ "The command arguments exceed Linux's transport limit (E2BIG).",
+ { actual_bytes: vectorBytes, max_bytes: LINUX_ARGUMENT_VECTOR_BYTES },
+ {
+ remediationAction: "Reduce the size of the request, then retry.",
+ retryable: false,
+ },
+ );
const result = await computer.exec(
- [
- "python3",
- "-I",
- "-c",
- await source,
- command,
- JSON.stringify(argumentsJson),
- ],
+ argv,
// The controller's own waits end first, so it can answer.
timeoutS + 60,
signal,
@@ -61,6 +138,8 @@ export const runBuildController = async (
/** What REA's build controller answers a command with. The wire format stays here. */
export const answerSchema = z.looseObject({
error: z.string().optional(),
+ /** Why the controller refused a request before acting on it, such as a state path outside the home. */
+ refused: z.string().optional(),
absent: z.boolean().optional(),
line: z.string().optional(),
exited: z.int().optional(),
@@ -180,21 +259,47 @@ const listFolder = async (
return listingSchema.parse(JSON.parse(content.toString("utf8")));
};
-/** The answers for one home folder: files listed, then fetched in batches of about `FETCH_BATCH_BYTES`. */
-const batched = (files: z.infer["files"]) => {
- const batches: { files: typeof files; bytes: number }[] = [];
+/** One listed file and the answer budget a batch must fit. */
+export interface ListedBatchFile {
+ readonly path: string;
+ readonly size: number;
+}
+
+/**
+ * Files fetched together. A batch stops before its raw bytes pass
+ * `rawBytes` or its base64 answer, including each file's envelope, passes
+ * `answerBytes`.
+ */
+export const batchListedFiles = (
+ files: readonly File[],
+ limits: { readonly rawBytes: number; readonly answerBytes: number },
+): { files: File[]; bytes: number }[] => {
+ const batches: { files: File[]; bytes: number }[] = [];
for (const file of files) {
const last = batches.at(-1);
- if (last === undefined || last.bytes + file.size > FETCH_BATCH_BYTES)
+ const nextBytes = (last?.bytes ?? 0) + file.size;
+ const nextCount = (last?.files.length ?? 0) + 1;
+ const fits =
+ last !== undefined &&
+ nextBytes <= limits.rawBytes &&
+ guestFileAnswerBytes(nextBytes, nextCount) <= limits.answerBytes;
+ if (last === undefined || !fits)
batches.push({ files: [file], bytes: file.size });
else {
last.files.push(file);
- last.bytes += file.size;
+ last.bytes = nextBytes;
}
}
return batches;
};
+/** The answers for one home folder: files listed, then fetched in batches of about `FETCH_BATCH_BYTES` that also fit the host answer budget. */
+const batched = (files: z.infer["files"]) =>
+ batchListedFiles(files, {
+ rawBytes: FETCH_BATCH_BYTES,
+ answerBytes: guestFileAnswerWorkingBytes(guestFileAnswerCeiling()),
+ });
+
/**
* Bring every regular file under a home folder of the computer back to REA:
* listed with sizes and digests in the computer, fetched through `data/get`
diff --git a/src/desktop/ComputerStartFailure.test.ts b/src/desktop/ComputerStartFailure.test.ts
new file mode 100644
index 0000000000..09fbc608ef
--- /dev/null
+++ b/src/desktop/ComputerStartFailure.test.ts
@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ ComputerStartFailure,
+ computerFailedOnReadiness,
+} from "./AgentEnvComputer.js";
+
+describe("computerFailedOnReadiness", () => {
+ it("reads the readiness fact, including a final check whose message never says not ready", () => {
+ const failed = new ComputerStartFailure(
+ "The computer's app check failed: no detail.",
+ "Read the helper's output.",
+ { reason: "not_ready" },
+ );
+ const wording = new ComputerStartFailure(
+ "it didn't boot: the computer wasn't ready",
+ undefined,
+ { reason: "handshake" },
+ );
+
+ expect(computerFailedOnReadiness(failed)).toBe(true);
+ expect(computerFailedOnReadiness(wording)).toBe(false);
+ expect(computerFailedOnReadiness(new Error("not ready"))).toBe(false);
+ });
+});
diff --git a/src/desktop/DesktopApplicationSession.test.ts b/src/desktop/DesktopApplicationSession.test.ts
index c330c4b306..4ed042113d 100644
--- a/src/desktop/DesktopApplicationSession.test.ts
+++ b/src/desktop/DesktopApplicationSession.test.ts
@@ -139,3 +139,27 @@ describe.skipIf(process.platform !== "linux")(
}, 30_000);
},
);
+
+describe.skipIf(process.platform === "win32")(
+ "runDesktopSession stderr",
+ () => {
+ it("keeps the end of an app's stderr when it writes more than the session keeps", async () => {
+ const root = await createTestTempDirectory("desktop-session-stderr-");
+ const cause =
+ "[error] notes: Unable to find an appropriate java runtime.";
+ // About 16 KiB of warnings, then the cause, as an app that fails at launch writes.
+ const outcome = await runDesktopSession(
+ shellPlan(
+ root,
+ `i=0; while [ $i -lt 400 ]; do echo "[warning] notes: probing runtime $i of 400" >&2; i=$((i+1)); done; echo '${cause}' >&2; exit 1`,
+ ),
+ );
+
+ const { stderr, stderr_truncated } = outcome.session;
+ expect(stderr.trimEnd().endsWith(cause)).toBe(true);
+ expect(stderr).not.toContain("runtime 0 of 400");
+ expect(Buffer.byteLength(stderr)).toBeLessThanOrEqual(8 * 1024);
+ expect(stderr_truncated).toBe(true);
+ }, 30_000);
+ },
+);
diff --git a/src/desktop/DesktopApplicationSession.ts b/src/desktop/DesktopApplicationSession.ts
index f06918152b..45ad0a1f17 100644
--- a/src/desktop/DesktopApplicationSession.ts
+++ b/src/desktop/DesktopApplicationSession.ts
@@ -28,6 +28,7 @@ import {
} from "./DesktopSessionObservations.js";
import type { WindowSample, X11WindowObserver } from "./X11WindowObserver.js";
+/** The most of the launched process's stderr a session keeps: its latest bytes, where a failing app's last words are. */
const STDERR_BYTES = 8 * 1024;
const TERMINATION_GRACE_MS = 5_000;
@@ -312,7 +313,10 @@ const launch = async (
cleanup:
spawned.cleanup ?? (() => cleanupOwnedProcessGroup(spawned.ownership)),
},
- { maxDiagnosticBytes: { stdout: 0, stderr: STDERR_BYTES } },
+ {
+ maxDiagnosticBytes: { stdout: 0, stderr: STDERR_BYTES },
+ diagnosticRetention: "tail",
+ },
);
return { spawned, supervisor };
};
diff --git a/src/desktop/GuestFileAnswer.test.ts b/src/desktop/GuestFileAnswer.test.ts
new file mode 100644
index 0000000000..b9aafdf3a9
--- /dev/null
+++ b/src/desktop/GuestFileAnswer.test.ts
@@ -0,0 +1,77 @@
+import { describe, expect, it } from "vitest";
+
+import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
+import {
+ AgentEnvComputer,
+ GUEST_FILE_ANSWER_COPIES,
+ GUEST_FILE_ANSWER_SOURCE,
+ guestFileAnswerBytes,
+ guestFileAnswerCeiling,
+ guestFileAnswerRefusal,
+ guestFileAnswerWorkingBytes,
+ retainGuestFiles,
+} from "./AgentEnvComputer.js";
+
+describe("guest file answers", () => {
+ it("counts base64, the envelope, and the copies held while the answer is parsed", () => {
+ expect(guestFileAnswerBytes(3, 1)).toBeGreaterThan(3);
+ expect(guestFileAnswerBytes(3, 2)).toBeGreaterThan(
+ guestFileAnswerBytes(3, 1),
+ );
+ const wire = guestFileAnswerBytes(3, 1);
+ expect(guestFileAnswerRefusal(3, 1, wire)).toBeInstanceOf(
+ AnalysisResourceConstraintError,
+ );
+ expect(
+ guestFileAnswerRefusal(3, 1, wire * GUEST_FILE_ANSWER_COPIES),
+ ).toBeUndefined();
+ expect(guestFileAnswerWorkingBytes(wire * GUEST_FILE_ANSWER_COPIES)).toBe(
+ wire,
+ );
+ });
+
+ it("refuses a reported size that expands past the working budget before naming a fetch", () => {
+ const ceiling = 1000;
+ const reported = 3000;
+ const refusal = guestFileAnswerRefusal(reported, 2, ceiling);
+ expect(refusal).toBeInstanceOf(AnalysisResourceConstraintError);
+ if (refusal === undefined) return;
+ expect(refusal.reportedLimits).toEqual({
+ guest_file_bytes: reported,
+ file_count: 2,
+ answer_bytes: guestFileAnswerBytes(reported, 2),
+ answer_ceiling_bytes: ceiling,
+ answer_working_bytes: guestFileAnswerWorkingBytes(ceiling),
+ answer_copies: GUEST_FILE_ANSWER_COPIES,
+ ceiling_source: GUEST_FILE_ANSWER_SOURCE,
+ });
+ expect(refusal.remediationAction).toContain(GUEST_FILE_ANSWER_SOURCE);
+ expect(refusal.remediationAction).toContain("--max-old-space-size");
+ expect(refusal.remediationAction).toContain("Restart REA");
+ expect(guestFileAnswerBytes(reported, 2)).toBeGreaterThan(
+ guestFileAnswerWorkingBytes(ceiling),
+ );
+ });
+
+ it("does not keep file bytes the computer reported as smaller", () => {
+ const retained = retainGuestFiles(
+ [{ path: "out.json", bytes: Buffer.from("larger") }],
+ 1,
+ 1,
+ 1_000_000,
+ );
+ expect(retained.ok).toBe(false);
+ if (retained.ok) return;
+ expect(retained.error.reportedLimits).toMatchObject({
+ guest_file_bytes: 1,
+ decoded_file_bytes: Buffer.from("larger").length,
+ });
+ });
+
+ it("does not ask the computer for a home file past that same ceiling", async () => {
+ const failure = await AgentEnvComputer.prototype.homeFiles
+ .call({}, ["out.json"], Number.MAX_SAFE_INTEGER)
+ .catch((cause: unknown) => cause);
+ expect(failure).toBeInstanceOf(AnalysisResourceConstraintError);
+ });
+});
diff --git a/src/desktop/LinuxApplicationInventory.test.ts b/src/desktop/LinuxApplicationInventory.test.ts
index e530e79eb5..974b772430 100644
--- a/src/desktop/LinuxApplicationInventory.test.ts
+++ b/src/desktop/LinuxApplicationInventory.test.ts
@@ -352,5 +352,24 @@ describe.skipIf(process.platform === "win32")(
expect(paths).not.toContain(join(home, ".notesy"));
expect(paths).not.toContain(join(home, ".other.ini"));
});
+
+ it("adds KDE's ~/.config/rc, which the application may first write during the capture", async () => {
+ const installed = await fixture();
+ const { home } = installed;
+ await mkdir(join(home, ".config"), { recursive: true });
+ await writeFile(join(home, ".config/org.example.notesrc"), "x\n");
+
+ const inventory = await inventoryOf("org.example.notes", installed);
+
+ expect(inventory.state_directories).toEqual(
+ expect.arrayContaining([
+ { path: join(home, ".config/notesrc"), exists_at_start: false },
+ {
+ path: join(home, ".config/org.example.notesrc"),
+ exists_at_start: true,
+ },
+ ]),
+ );
+ });
},
);
diff --git a/src/desktop/NpmPackInvocation.test.ts b/src/desktop/NpmPackInvocation.test.ts
new file mode 100644
index 0000000000..874d4c405a
--- /dev/null
+++ b/src/desktop/NpmPackInvocation.test.ts
@@ -0,0 +1,23 @@
+import { describe, expect, it } from "vitest";
+
+import { npmInvocation } from "../../scripts/lib/npm-invocation.mjs";
+
+describe("npmInvocation for the build-computer pack", () => {
+ it("keeps a temporary directory that contains spaces as one argument", () => {
+ const destination =
+ "C:\\Users\\Ada Lovelace\\AppData\\Local\\Temp\\rea-build-runtime-1";
+ const entry =
+ "C:\\Program Files\\nodejs\\node_modules\\npm\\bin\\npm-cli.js";
+ const invoked = npmInvocation(["pack", "--pack-destination", destination], {
+ npm_execpath: entry,
+ });
+
+ expect(invoked.arguments).toEqual([
+ entry,
+ "pack",
+ "--pack-destination",
+ destination,
+ ]);
+ expect(invoked.command).not.toMatch(/npm\.cmd$/u);
+ });
+});
diff --git a/src/domain/desktop/appExport.test.ts b/src/domain/desktop/appExport.test.ts
index 748023955f..d3203f583d 100644
--- a/src/domain/desktop/appExport.test.ts
+++ b/src/domain/desktop/appExport.test.ts
@@ -1,6 +1,9 @@
import { describe, expect, it } from "vitest";
+import type { DesktopApplicationCapture } from "./desktopCapture.js";
+
import {
+ capturedHomePath,
appModuleFile,
className,
environmentName,
@@ -39,6 +42,9 @@ const plan: DesktopAppExportPlan = {
seedFiles: ["Documents/todo.txt"],
textSeeds: ["Documents/todo.txt"],
unseededFiles: [],
+ leftOutLocks: [],
+ unownedFolders: [],
+ runtimeSeeds: [],
dconf: "",
captureEvidenceId: "ev_1",
};
@@ -384,3 +390,273 @@ describe("README and class wording", () => {
expect(renderReadme(own)).not.toContain("`envs/mousepad/seed/`");
});
});
+
+it("keeps a nested selection under the state path of a selected alias", () => {
+ const home = "/home/ada";
+ const state = `${home}/.config/geany`;
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [{ path: state }] },
+ state: {
+ state_directories: [state],
+ selected_paths: [
+ { path: `${home}/settings`, physical: state, kind: "directory" },
+ {
+ path: `${home}/settings/profiles/work`,
+ physical: `${state}/profiles/work`,
+ kind: "directory",
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+ expect(path(`${state}/profiles/work/geany.conf`)).toBe(
+ ".config/geany/profiles/work/geany.conf",
+ );
+});
+
+describe("capturedHomePath", () => {
+ it("keeps the application's state directory when a selected link points to it", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [{ path: `${home}/.config/geany` }] },
+ state: {
+ state_directories: [`${home}/.config/geany`],
+ selected_paths: [
+ {
+ path: `${home}/settings`,
+ physical: `${home}/.config/geany`,
+ kind: "directory",
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+ expect(path(`${home}/.config/geany/geany.conf`)).toBe(
+ ".config/geany/geany.conf",
+ );
+ });
+
+ it("uses the selected spelling of a linked settings folder, and a database's logs", () => {
+ const home = "/home/ada";
+ const physical = `${home}/settings/vendor`;
+ const database = `${home}/data/books.db`;
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [] },
+ state: {
+ selected_paths: [
+ { path: `${home}/settings`, physical: `${home}/settings` },
+ { path: `${home}/.config/vendor`, physical },
+ { path: `${home}/Documents/linked.db`, physical: database },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path(`${physical}/app.conf`)).toBe(".config/vendor/app.conf");
+ expect(path(`${home}/Documents/note.txt`)).toBe("Documents/note.txt");
+ expect(path(database)).toBe("Documents/linked.db");
+ expect(path(`${database}-wal`)).toBe("Documents/linked.db-wal");
+ expect(path(`${database}-journal`)).toBe("Documents/linked.db-journal");
+ });
+
+ it("maps a database selected through a link together with its write-ahead log", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [] },
+ state: {
+ selected_paths: [
+ { path: `${home}/.config/app.db`, physical: `${home}/data/app.db` },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+ expect(path(`${home}/data/app.db`)).toBe(".config/app.db");
+ expect(path(`${home}/data/app.db-wal`)).toBe(".config/app.db-wal");
+ expect(path(`${home}/data/app.db-journal`)).toBe(".config/app.db-journal");
+ });
+
+ it("keeps a walked settings folder when a broader directory was also selected", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: {
+ state_directories: [{ path: `${home}/.config/notes` }],
+ },
+ state: {
+ state_directories: [`${home}/settings/notes`],
+ selected_paths: [
+ { path: `${home}/settings`, physical: `${home}/settings` },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path(`${home}/settings/notes/settings.ini`)).toBe(
+ ".config/notes/settings.ini",
+ );
+ expect(path(`${home}/settings/other.txt`)).toBe("settings/other.txt");
+ });
+
+ it("does not treat a database log name as a directory of other files", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [] },
+ state: {
+ selected_paths: [
+ { path: `${home}/settings`, physical: `${home}/settings` },
+ {
+ path: `${home}/.config/notes`,
+ physical: `${home}/settings/notes`,
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path(`${home}/settings/notes/settings.ini`)).toBe(
+ ".config/notes/settings.ini",
+ );
+ expect(path(`${home}/settings/notes-journal/today.txt`)).toBe(
+ "settings/notes-journal/today.txt",
+ );
+ });
+
+ it("moves a walked state database's logs with the database", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: {
+ state_directories: [{ path: `${home}/.notes-state.db` }],
+ },
+ state: {
+ state_directories: [`${home}/settings/notes.db`],
+ selected_paths: [
+ { path: `${home}/settings`, physical: `${home}/settings` },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path(`${home}/settings/notes.db`)).toBe(".notes-state.db");
+ expect(path(`${home}/settings/notes.db-wal`)).toBe(".notes-state.db-wal");
+ expect(path(`${home}/settings/notes.db-journal`)).toBe(
+ ".notes-state.db-journal",
+ );
+ });
+
+ it("keeps a selected file whose name ends in -journal", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [] },
+ state: {
+ selected_paths: [
+ {
+ path: `${home}/.config/vendor/state`,
+ physical: `${home}/data/vendor-journal`,
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path(`${home}/data/vendor-journal`)).toBe(".config/vendor/state");
+ });
+});
+
+describe("capturedHomePath outside the home", () => {
+ it("keeps a database log when the database link leaves the home", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [] },
+ state: {
+ selected_paths: [
+ {
+ path: `${home}/Documents/books.db`,
+ physical: "/mnt/data/books.db",
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path("/mnt/data/books.db")).toBe("Documents/books.db");
+ expect(path("/mnt/data/books.db-wal")).toBe("Documents/books.db-wal");
+ expect(path("/mnt/data/books.db-journal")).toBe(
+ "Documents/books.db-journal",
+ );
+ });
+
+ it("keeps a database's own name for its log when a parent folder moves", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [] },
+ state: {
+ selected_paths: [
+ { path: `${home}/.config/app`, physical: `${home}/settings` },
+ {
+ path: `${home}/settings/book.db`,
+ physical: `${home}/settings/book.db`,
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path(`${home}/settings/book.db`)).toBe("settings/book.db");
+ expect(path(`${home}/settings/book.db-wal`)).toBe("settings/book.db-wal");
+ expect(path(`${home}/settings/book.db-journal`)).toBe(
+ "settings/book.db-journal",
+ );
+ expect(path(`${home}/settings/other.txt`)).toBe(".config/app/other.txt");
+ });
+
+ it("keeps an ordinary file beside a selected directory in the parent folder", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [] },
+ state: {
+ content_roots: [
+ `${home}/settings/work`,
+ `${home}/settings/work/notes.txt`,
+ `${home}/settings/work-journal`,
+ ],
+ selected_paths: [
+ { path: `${home}/.config/app`, physical: `${home}/settings` },
+ { path: `${home}/settings/work`, physical: `${home}/settings/work` },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path(`${home}/settings/work-journal`)).toBe(
+ ".config/app/work-journal",
+ );
+ });
+
+ it("keeps a neighbor of a selected directory recorded as a directory", () => {
+ const home = "/home/ada";
+ const path = capturedHomePath({
+ host: { home },
+ application: { state_directories: [] },
+ state: {
+ selected_paths: [
+ {
+ path: `${home}/.config/app`,
+ physical: `${home}/settings`,
+ kind: "directory",
+ },
+ {
+ path: `${home}/settings/work`,
+ physical: `${home}/settings/work`,
+ kind: "directory",
+ },
+ ],
+ },
+ } as unknown as DesktopApplicationCapture);
+
+ expect(path(`${home}/settings/work-journal`)).toBe(
+ ".config/app/work-journal",
+ );
+ expect(path(`${home}/settings/work/notes.txt`)).toBe(
+ "settings/work/notes.txt",
+ );
+ });
+});
diff --git a/src/domain/desktop/appExport.ts b/src/domain/desktop/appExport.ts
index c9062d9e71..9c836e9ce9 100644
--- a/src/domain/desktop/appExport.ts
+++ b/src/domain/desktop/appExport.ts
@@ -10,13 +10,19 @@ import {
PYTHON_MODULE_NAMES,
PYTHON_READY_CHECK,
} from "./appExportPython.js";
+import {
+ isLockFile,
+ unownedReadmeNotes,
+ type RuntimeSeeds,
+} from "./appExportUnowned.js";
import type { DesktopApplicationCapture } from "./desktopCapture.js";
import type { DesktopApplicationState } from "./desktopState.js";
import { isWithinAny } from "./desktopStateDiff.js";
+import { SQLITE_SIDECARS } from "./stateFormats.js";
import { SEED_HOME } from "./hostIdentity.js";
import { capturedFileUrl, type LaunchDocument } from "./seedDocument.js";
import { posix } from "node:path";
-import { stateDirectoryOf } from "./stateRoles.js";
+import { roleFacts, stateDirectoryOf } from "./stateRoles.js";
/** agentenv-framework release the exported class was checked against. */
export const DESKTOP_EXPORT_FRAMEWORK_VERSION = "0.9.1431";
@@ -58,8 +64,19 @@ export interface DesktopAppExportPlan {
readonly seedFiles: readonly string[];
/** Seed files read as text, which get the seed's home path respelled. */
readonly textSeeds: readonly string[];
+ /** Home-relative selected aliases that resolve to an app state path. */
+ readonly stateAliases?: readonly {
+ readonly alias: string;
+ readonly target: string;
+ }[];
/** Files under the owned paths that existed before the capture but whose bytes were not retained. */
readonly unseededFiles: readonly string[];
+ /** Lock files under the owned paths at the seed, left out of it so a stale one cannot keep the app from starting. */
+ readonly leftOutLocks: readonly string[];
+ /** Folders under the home where files changed during the capture that the bundle neither owns nor seeds. */
+ readonly unownedFolders: readonly string[];
+ /** Seed files the app keeps for its own running: web engine storage and caches. */
+ readonly runtimeSeeds: readonly RuntimeSeeds[];
/** `dconf dump` key file of the application's groups at the seed, or empty. */
readonly dconf: string;
readonly captureEvidenceId: string;
@@ -140,9 +157,7 @@ export const ownedPaths = (
};
/** A captured path's home-relative path, in the innermost walked state directory's own spelling unless that leaves the home. */
-export const capturedHomePath = (
- capture: DesktopApplicationCapture,
-): ((path: string) => string | null) => {
+const capturedHomePlan = (capture: DesktopApplicationCapture) => {
const { home } = capture.host;
const relative = (path: string) =>
path.startsWith(`${home}/`) ? path.slice(home.length + 1) : null;
@@ -152,23 +167,134 @@ export const capturedHomePath = (
const renamed = (walked.length === logical.length ? walked : [])
.flatMap((physical, index) => {
const path = logical[index]?.path;
- return path === undefined || path === physical
- ? []
- : [{ physical, logical: path }];
+ return path === undefined ? [] : [{ physical, logical: path }];
})
.sort((left, right) => right.physical.length - left.physical.length);
- return (path) => {
- const directory = renamed.find(({ physical }) =>
- isWithinAny(path, [physical]),
- );
- const respelled =
- directory === undefined
- ? null
- : relative(
- `${directory.logical}${path.slice(directory.physical.length)}`,
- );
- return respelled ?? relative(path);
+ const selections = [...(capture.state.selected_paths ?? [])];
+ const stateAliases = capturedStateAliases(capture);
+ const aliasPaths = [...stateAliases].sort(
+ (left, right) => right.alias.length - left.alias.length,
+ );
+ const selectedSpelling = (path: string) => {
+ for (const { alias, target } of aliasPaths) {
+ const prefix = `${home}/${alias}`;
+ if (isWithinAny(path, [prefix]))
+ return `${home}/${target}${path.slice(prefix.length)}`;
+ }
+ return path;
};
+ const recorded = [
+ ...(capture.state.content_roots ?? []),
+ ...(capture.state.state_directories ?? []),
+ ...logical.map((entry) => entry.path),
+ ...selections.flatMap(({ path, physical }) => [path, physical]),
+ ];
+ const directories = new Set();
+ for (const selected of selections)
+ if (selected.kind === "directory") {
+ directories.add(selected.path);
+ directories.add(selected.physical);
+ }
+ for (const checkpoint of capture.state.checkpoints ?? [])
+ for (const change of checkpoint.changes) {
+ const facts = change.after ?? change.before;
+ if (facts?.type === "directory") directories.add(change.path);
+ }
+ const known = new Set(recorded);
+ for (const path of known) {
+ let parent = path;
+ for (;;) {
+ const slash = parent.lastIndexOf("/");
+ if (slash <= 0) break;
+ parent = parent.slice(0, slash);
+ if (known.has(parent)) directories.add(parent);
+ }
+ }
+ const isDirectory = (path: string) => directories.has(path);
+ // The longest physical path wins, whether it is a selected link or a walked
+ // settings folder. A broad selection of ~/settings must not hide
+ // ~/.config/notes when that folder was walked at ~/settings/notes.
+ const spellings = [
+ ...renamed,
+ ...selections.map(({ physical, path }) => ({
+ physical,
+ logical: selectedSpelling(path),
+ })),
+ ].sort((left, right) => right.physical.length - left.physical.length);
+ const spell = (path: string) => {
+ for (const { physical, logical } of spellings) {
+ if (!isWithinAny(path, [physical])) continue;
+ const spelled = relative(`${logical}${path.slice(physical.length)}`);
+ if (spelled !== null) return { spelled, specificity: physical.length };
+ }
+ const plain = relative(path);
+ return plain === null ? undefined : { spelled: plain, specificity: 0 };
+ };
+ const homePath = (path: string) => {
+ const own = spell(path);
+ // A database log follows the database only when that database's mapping is
+ // more specific than the log file's own. A selected file whose name ends
+ // in -wal or -journal keeps its own spelling.
+ for (const { suffix } of SQLITE_SIDECARS) {
+ if (!path.endsWith(suffix)) continue;
+ const database = path.slice(0, -suffix.length);
+ const base = spell(database);
+ // A directory selected under a renamed folder is not a database. A file
+ // with an explicit mapping is, even when that mapping keeps its name or
+ // the file sits outside the home.
+ if (
+ isDirectory(database) ||
+ base === undefined ||
+ base.specificity === 0 ||
+ (own !== undefined && base.specificity <= own.specificity)
+ )
+ continue;
+ return `${base.spelled}${suffix}`;
+ }
+ return own?.spelled ?? null;
+ };
+ return { homePath, stateAliases };
+};
+
+/** A captured path's home-relative spelling in the shared export plan. */
+export const capturedHomePath = (capture: DesktopApplicationCapture) =>
+ capturedHomePlan(capture).homePath;
+
+/** Selected links to an app state root must keep both the app's name and the selected name. */
+const capturedStateAliases = (capture: DesktopApplicationCapture) => {
+ const { home } = capture.host;
+ const walked = capture.state.state_directories ?? [];
+ const roots = capture.application.state_directories;
+ if (walked.length !== roots.length) return [];
+ const relative = (path: string) =>
+ path.startsWith(`${home}/`) ? path.slice(home.length + 1) : null;
+ const aliases = (capture.state.selected_paths ?? []).flatMap(
+ ({ path, physical }) => {
+ const index = walked.indexOf(physical);
+ const state = index < 0 ? undefined : roots[index]?.path;
+ const alias = relative(path);
+ const target = state === undefined ? null : relative(state);
+ return alias === null || target === null || alias === target
+ ? []
+ : [{ alias, target }];
+ },
+ );
+ const retained: typeof aliases = [];
+ for (const entry of aliases.toSorted(
+ (left, right) => left.alias.length - right.alias.length,
+ )) {
+ const parent = retained.findLast(
+ ({ alias }) => alias !== entry.alias && isWithinAny(entry.alias, [alias]),
+ );
+ if (
+ parent !== undefined &&
+ `${parent.target}${entry.alias.slice(parent.alias.length)}` ===
+ entry.target
+ )
+ continue;
+ retained.push(entry);
+ }
+ return retained;
};
type StatePath = DesktopApplicationState["paths"][number];
@@ -180,50 +306,83 @@ export interface HomePathCollision {
readonly dropped: string;
}
-/** The state view's files by home path, dconf databases aside; of files sharing one, the first present before the capture is kept. */
+/**
+ * The state view's files by home path, and its dconf databases apart, which
+ * no file of the bundle holds; of files sharing one, the first present before
+ * the capture is kept.
+ */
export const stateFilesByHomePath = (
paths: readonly StatePath[],
homePath: (path: string) => string | null,
): {
readonly files: ReadonlyMap;
+ readonly dconf: ReadonlyMap;
readonly collisions: readonly HomePathCollision[];
} => {
const absent = (path: StatePath) =>
path.versions[0]?.present === true ? 0 : 1;
const files = new Map();
+ const dconf = new Map();
const collisions: HomePathCollision[] = [];
for (const path of [...paths].sort(
(left, right) => absent(left) - absent(right),
)) {
const home = homePath(path.path);
- if (home === null || path.format === "dconf") continue;
+ if (home === null) continue;
+ if (path.format === "dconf") {
+ dconf.set(home, path);
+ continue;
+ }
const kept = files.get(home);
if (kept === undefined) files.set(home, path);
else collisions.push({ home, kept: kept.path, dropped: path.path });
}
- return { files, collisions };
+ return { files, dconf, collisions };
};
-/** The state view's files by home path and the paths the app owns, as the export decides them. */
+/**
+ * The state view's files and dconf databases by home path, the paths the app
+ * owns, and which home files the seed leaves out as locks the app holds, as
+ * the export decides them. A document or a path the caller selected is never
+ * left out, whatever its name.
+ */
export const exportedStateFiles = (
capture: DesktopApplicationCapture,
state: DesktopApplicationState,
) => {
- const homePath = capturedHomePath(capture);
- const { files, collisions } = stateFilesByHomePath(state.paths, homePath);
+ const { homePath, stateAliases } = capturedHomePlan(capture);
+ const homePaths = (paths: readonly string[]) =>
+ paths.map(homePath).filter((path): path is string => path !== null);
+ const { files, dconf, collisions } = stateFilesByHomePath(
+ state.paths,
+ homePath,
+ );
const withRole = (role: string) =>
[...files].filter(([, path]) => path.role === role).map(([home]) => home);
+ const stateDirectories = homePaths([
+ ...capture.application.state_directories.map((entry) => entry.path),
+ ...(capture.state.state_directories ?? []),
+ ]);
const paths = ownedPaths(
withRole("application_state"),
withRole("document"),
- [
- ...capture.application.state_directories.map((entry) => entry.path),
- ...(capture.state.state_directories ?? []),
- ]
- .map(homePath)
- .filter((path): path is string => path !== null),
+ stateDirectories,
);
- return { homePath, files, collisions, paths };
+ const usersOwn = new Set([
+ ...withRole("document"),
+ ...homePaths(roleFacts(capture).selectedRoots),
+ ]);
+ const isLeftOutLock = (home: string) =>
+ !usersOwn.has(home) && isLockFile(home, stateDirectories);
+ return {
+ homePath,
+ stateAliases,
+ files,
+ dconf,
+ collisions,
+ paths,
+ isLeftOutLock,
+ };
};
/** Text safe inside a Python triple-quoted docstring. */
@@ -329,6 +488,7 @@ const classBody = (plan: DesktopAppExportPlan): string => {
if not INITIAL.exists():
_seed_home(Path.home())
_load_dconf(Path.home())
+ _ensure_state_aliases(Path.home())
keep_initial(self.paths)
${PYTHON_DATA_HOOKS}${PYTHON_READY_CHECK}`;
if (plan.dconf.length > 0)
@@ -420,6 +580,7 @@ SEED_HOME = ${py(SEED_HOME)}
PATHS = ${pyList(plan.paths)}
DCONF = ${py(plan.dconf)}
TEXT_SEEDS = frozenset(${pyList(plan.textSeeds)})
+STATE_ALIASES = [${(plan.stateAliases ?? []).map(({ alias, target }) => `(${py(alias)}, ${py(target)})`).join(", ")}]
WINDOW_CLASS = ${py(plan.windowClass)}
${PYTHON_HELPERS}
@@ -522,9 +683,14 @@ export const launchDocumentNote = ({
? `\n- ${launchDocumentLimitation(opens)}`
: "";
-/** Bundle files: seeds the export rewrote to remove the capture host's identity, and files that still hold it. */
+/**
+ * Bundle files: seeds the export rewrote to remove the capture host's
+ * identity, gzip seeds it compressed again only to clear their headers, and
+ * files that still hold the identity.
+ */
export interface SeedScrubReport {
readonly scrubbed: readonly string[];
+ readonly recompressed: readonly string[];
readonly unscrubbed: readonly string[];
}
@@ -534,11 +700,20 @@ const homeList = (paths: readonly string[]) =>
const bundleList = (paths: readonly string[]) =>
paths.map((path) => `\`${path}\``).join(", ");
-const scrubNotes = ({ scrubbed, unscrubbed }: SeedScrubReport): string =>
+const scrubNotes = ({
+ scrubbed,
+ recompressed,
+ unscrubbed,
+}: SeedScrubReport): string =>
[
...(scrubbed.length === 0
? []
: [`\n Rewritten: ${bundleList(scrubbed)}.`]),
+ ...(recompressed.length === 0
+ ? []
+ : [
+ `\n ${String(recompressed.length)} gzip file(s) were compressed again only to clear their headers, their content unchanged.`,
+ ]),
...(unscrubbed.length === 0
? []
: [
@@ -551,7 +726,7 @@ const scrubNotes = ({ scrubbed, unscrubbed }: SeedScrubReport): string =>
export const renderReadme = (
plan: DesktopAppExportPlan,
taskSection = "",
- scrub: SeedScrubReport = { scrubbed: [], unscrubbed: [] },
+ scrub: SeedScrubReport = { scrubbed: [], recompressed: [], unscrubbed: [] },
): string =>
`# ${plan.title} computer app
@@ -572,7 +747,7 @@ ${DESKTOP_EXPORT_FRAMEWORK_VERSION}.
: `
- Not seeded, because the capture did not retain their bytes at the seed: ${homeList(plan.unseededFiles)}.
Capture again with \`--state-path\` naming them, or a larger \`--max-file-bytes\`, to include them.`
- }
+ }${unownedReadmeNotes(plan.unownedFolders, plan.runtimeSeeds, plan.leftOutLocks)}
- \`tasks/smoke.json\` deploys the app.
${taskSection.length === 0 ? "Run it with `agent-env run `." : "Run the task with `agent-env run `, or only the deploy with `--task smoke`."}
diff --git a/src/domain/desktop/appExportPython.ts b/src/domain/desktop/appExportPython.ts
index 1237265b55..a6872e8fd2 100644
--- a/src/domain/desktop/appExportPython.ts
+++ b/src/domain/desktop/appExportPython.ts
@@ -46,6 +46,7 @@ LINE_BREAK = re.compile(r"\r?\n")
SQLITE_HEADER = b"SQLite format 3\x00"
GVDB_HEADER = b"GVariant"
ZIP_HEADER = b"PK\x03\x04"
+GZIP_HEADER = b"\x1f\x8b\x08"
XML_SPACE = "xml:space"
MAX_VIEW_BYTES = 64 * 1024 * 1024
MAX_SAFE_INTEGER = 2**53 - 1
@@ -115,6 +116,25 @@ def _seed_home(home: Path) -> None:
target.write_bytes(data)
+def _ensure_state_aliases(home: Path) -> None:
+ """Restore selected aliases without replacing any existing user data."""
+ aliases = sorted(STATE_ALIASES, key=lambda pair: len(Path(pair[0]).parts))
+ for alias, target in aliases:
+ link = home / alias
+ destination = home / target
+ has_children = any(Path(alias) in Path(child).parents for child, _ in aliases)
+ if link.is_symlink() and link.resolve() == destination.resolve():
+ if has_children:
+ destination.mkdir(parents=True, exist_ok=True)
+ continue
+ if os.path.lexists(link):
+ raise FileExistsError(f"Cannot restore selected state alias; path already exists: {link}")
+ link.parent.mkdir(parents=True, exist_ok=True)
+ if has_children:
+ destination.mkdir(parents=True, exist_ok=True)
+ link.symlink_to(destination)
+
+
def _dconf(arguments: list, environment: dict) -> str:
return subprocess.run(["dconf", *arguments], env={**os.environ, **environment}, capture_output=True, text=True,
check=True).stdout
@@ -272,6 +292,17 @@ def _text_view(name: str, text: str, size: int) -> dict:
return _text(text)
+def _bytes_view(name: str, data: bytes) -> dict:
+ """The view of a document that is no database or archive: its text's, or none when it is binary."""
+ try:
+ text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None
+ except UnicodeDecodeError:
+ text = None
+ if text is None:
+ return {"format": "binary", "view": None, "truncated": False}
+ return _text_view(name, text, len(data))
+
+
def _view(path: Path) -> dict:
"""A typed JSON view of one file in the shapes of REA's state view; truncated says the view is partial."""
too_large = {"format": None, "view": None, "error": "larger than 64 MiB, the most the view reads"}
@@ -279,21 +310,18 @@ def _view(path: Path) -> dict:
return too_large
with path.open("rb") as file:
data = file.read(8192)
- whole = not data.startswith(SQLITE_HEADER) and b"\x00" not in data
+ whole = data.startswith(GZIP_HEADER) or (not data.startswith(SQLITE_HEADER) and b"\x00" not in data)
data += file.read(MAX_VIEW_BYTES + 1 - len(data)) if whole else b""
if data.startswith(SQLITE_HEADER):
return _sqlite(path)
if data.startswith(ZIP_HEADER):
- return _zip(path)
+ with path.open("rb") as file:
+ return _zip(file, os.fstat(file.fileno()).st_size)
if len(data) > MAX_VIEW_BYTES:
return too_large
- try:
- text = data.decode("utf-8-sig") if whole else None
- except UnicodeDecodeError:
- text = None
- if text is None:
- return {"format": "binary", "view": None, "truncated": False}
- return _text_view(path.name, text, len(data))
+ if data.startswith(GZIP_HEADER):
+ return _gzip(path.name, data)
+ return _bytes_view(path.name, data)
def _views(home: Path, paths: list) -> dict:
@@ -319,6 +347,7 @@ const PYTHON_CONSTANTS = [
"PATHS",
"DCONF",
"TEXT_SEEDS",
+ "STATE_ALIASES",
"WINDOW_CLASS",
] as const;
diff --git a/src/domain/desktop/appExportPythonZip.ts b/src/domain/desktop/appExportPythonZip.ts
index ea54b4b077..9da8204e33 100644
--- a/src/domain/desktop/appExportPythonZip.ts
+++ b/src/domain/desktop/appExportPythonZip.ts
@@ -1,7 +1,8 @@
/**
- * The exported module's zip reader: the central directory read a record at a
- * time, entries read in bounded chunks, and each entry's view by the rules a
- * file's follow, with REA's limits and wording. Part of the module's helpers.
+ * The exported module's zip and gzip readers: the central directory read a
+ * record at a time, entries read in bounded chunks, a gzip file's document
+ * within an entry's limits, and each one's view by the rules a file's follow,
+ * with REA's limits and wording. Part of the module's helpers.
*/
export const PYTHON_ZIP_HELPERS = String.raw`
MAX_ZIP_ENTRIES = 1000
@@ -9,6 +10,8 @@ MAX_ZIP_ENTRY_BYTES = 16 * 1024 * 1024
MAX_ZIP_BYTES = 64 * 1024 * 1024
MAX_ZIP_RATIO = 100
ZIP_READ_CHUNK = 64 * 1024
+#: Small, since the end of each gzip member copies what is left of the piece it was read from.
+GZIP_READ_CHUNK = 4 * 1024
ZIP_END = b"PK\x05\x06"
ZIP_CENTRAL = b"PK\x01\x02"
ZIP64_LOCATOR = b"PK\x06\x07"
@@ -24,13 +27,7 @@ def _entry_view(name: str, data: bytes) -> dict:
for header, form, inside in NOT_READ_INSIDE:
if data.startswith(header):
return {"format": form, "view": None, "error": f"{inside} inside an archive is not read"}
- try:
- text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None
- except UnicodeDecodeError:
- text = None
- if text is None:
- return {"format": "binary", "view": None}
- return _text_view(name, text, len(data))
+ return _bytes_view(name, data)
def _zip_refusal(info, spent: int):
@@ -210,31 +207,74 @@ class _BoundedArchive(io.RawIOBase):
return count
-def _zip(path: Path) -> dict:
+def _zip(file, size: int) -> dict:
"""A zip archive's file entries in archive order, up to REA's entry, size, ratio and archive limits; nothing is
extracted to disk, no archive inside is opened, and zipfile is shown only the directory records listed."""
entries, spent = [], 0
try:
- with path.open("rb") as file:
- directory = _zip_directory(file, os.fstat(file.fileno()).st_size, MAX_ZIP_ENTRIES)
- if directory is None:
- raise zipfile.BadZipFile("no readable central directory")
- start, declared, records, count = directory
- kept = b"".join(records)
- offset = declared if declared <= 0xFFFFFFFF else start
- # zipfile follows a Zip64 locator signature wherever one sits before the end record, and the kept records
- # may end in those bytes, so a real Zip64 end record and locator for them come first.
- tail = (kept
- + struct.pack("<4sQ2H2I4Q", ZIP64_END, 44, 45, 45, 0, 0, len(records), len(records), len(kept), offset)
- + struct.pack("<4sIQI", ZIP64_LOCATOR, 0, offset + len(kept), 1)
- + struct.pack("<4s4H2IH", ZIP_END, 0, 0, len(records), len(records), len(kept), offset, 0))
- with zipfile.ZipFile(_BoundedArchive(file, start, tail)) as archive:
- for info in archive.infolist():
- refusal = _zip_refusal(info, spent)
- spent += info.file_size if refusal is None else 0
- entries.append(_zip_entry(archive, info, refusal))
+ directory = _zip_directory(file, size, MAX_ZIP_ENTRIES)
+ if directory is None:
+ raise zipfile.BadZipFile("no readable central directory")
+ start, declared, records, count = directory
+ kept = b"".join(records)
+ offset = declared if declared <= 0xFFFFFFFF else start
+ # zipfile follows a Zip64 locator signature wherever one sits before the end record, and the kept records
+ # may end in those bytes, so a real Zip64 end record and locator for them come first.
+ tail = (kept
+ + struct.pack("<4sQ2H2I4Q", ZIP64_END, 44, 45, 45, 0, 0, len(records), len(records), len(kept), offset)
+ + struct.pack("<4sIQI", ZIP64_LOCATOR, 0, offset + len(kept), 1)
+ + struct.pack("<4s4H2IH", ZIP_END, 0, 0, len(records), len(records), len(kept), offset, 0))
+ with zipfile.ZipFile(_BoundedArchive(file, start, tail)) as archive:
+ for info in archive.infolist():
+ refusal = _zip_refusal(info, spent)
+ spent += info.file_size if refusal is None else 0
+ entries.append(_zip_entry(archive, info, refusal))
except (zipfile.BadZipFile, zlib.error, EOFError, NotImplementedError, OSError, ValueError, struct.error):
return {"format": "zip", "view": None, "error": "not a readable zip archive"}
view = {"entries": entries, "entry_count": count, "truncated": count > len(entries)}
return {"format": "zip", "view": view, "truncated": view["truncated"]}
+
+
+def _gunzip(data: bytes, limit: int):
+ """A gzip stream's document, its members joined as REA joins them, or None once it passes limit bytes, where
+ decompressing stops: data after a member that starts with a zero byte is padding and ends the stream. It is read
+ a piece at a time, so a member's end carries over only the rest of its piece. Raises zlib.error or EOFError when
+ the stream is damaged or cut short."""
+ document, member, rest, at = bytearray(), zlib.decompressobj(16 + zlib.MAX_WBITS), b"", 0
+ while True:
+ if not rest:
+ if at >= len(data):
+ raise EOFError("the gzip stream is cut short")
+ rest, at = data[at:at + GZIP_READ_CHUNK], at + GZIP_READ_CHUNK
+ document += member.decompress(rest, limit + 1 - len(document))
+ if len(document) > limit:
+ return None
+ if not member.eof:
+ rest = member.unconsumed_tail
+ continue
+ rest = member.unused_data
+ if not rest and at < len(data):
+ rest, at = data[at:at + GZIP_READ_CHUNK], at + GZIP_READ_CHUNK
+ if not rest or rest[0] == 0:
+ return bytes(document)
+ member = zlib.decompressobj(16 + zlib.MAX_WBITS)
+
+
+def _gzip(name: str, data: bytes) -> dict:
+ """A gzip file's view: its document's, decompressed within an archive entry's size and ratio limits and read by
+ the rules a file's follow; a database inside is not read."""
+ compressed = {"format": None, "compression": "gzip", "view": None}
+ try:
+ document = _gunzip(data, min(MAX_ZIP_ENTRY_BYTES, MAX_ZIP_RATIO * len(data)))
+ except (zlib.error, EOFError):
+ return {**compressed, "error": "not a readable gzip stream"}
+ if document is None:
+ return {**compressed, "error": f"expands more than {MAX_ZIP_RATIO} times, so it is not read"
+ if MAX_ZIP_RATIO * len(data) < MAX_ZIP_ENTRY_BYTES
+ else "larger than the 16 MiB a compressed document's view reads"}
+ for header, form, inside in NOT_READ_INSIDE:
+ if form != "zip" and document.startswith(header):
+ return {**compressed, "format": form, "error": f"{inside} inside a gzip file is not read"}
+ view = _zip(io.BytesIO(document), len(document)) if document.startswith(ZIP_HEADER) else _bytes_view(name, document)
+ return {**view, "compression": "gzip"}
`;
diff --git a/src/domain/desktop/appExportUnowned.test.ts b/src/domain/desktop/appExportUnowned.test.ts
new file mode 100644
index 0000000000..3009f2dc74
--- /dev/null
+++ b/src/domain/desktop/appExportUnowned.test.ts
@@ -0,0 +1,244 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ heldOpenLimitation,
+ isLockFile,
+ runtimeSeeds,
+ unownedChangedFolders,
+ unownedFoldersLimitation,
+ unretainedOpenedLimitation,
+} from "./appExportUnowned.js";
+import type { DesktopStatePath } from "./desktopState.js";
+
+const version = (index: number) => ({
+ checkpoint_index: index,
+ checkpoint: index === 0 ? "baseline" : "end",
+ present: true,
+ sha256: null,
+ format: null,
+ view: null,
+ view_error: null,
+});
+
+/** A state file with its role, changed during the capture unless `unchanged`. */
+const file = (
+ role: DesktopStatePath["role"],
+ unchanged = false,
+): DesktopStatePath => ({
+ path: "/home/ada/x",
+ home_path: "x",
+ role,
+ reasons: [],
+ format: null,
+ versions: unchanged ? [version(0)] : [version(0), version(1)],
+ changes: unchanged
+ ? []
+ : [
+ {
+ from_index: 0,
+ to_index: 1,
+ from: "baseline",
+ to: "end",
+ kind: "content",
+ added: [],
+ removed: [],
+ changed: [],
+ added_count: 0,
+ removed_count: 0,
+ changed_count: 0,
+ complete: true,
+ },
+ ],
+});
+
+describe("unownedChangedFolders", () => {
+ it("collapses changed files outside the owned paths to their top state folder, leaving out desktop and unchanged files", () => {
+ const files = new Map([
+ [".config/MuseScore/MuseScore3.ini", file("unrelated")],
+ [".config/MuseScore/plugins/a.qml", file("unrelated")],
+ [".local/share/MuseScore/MuseScore3/session", file("unrelated")],
+ [".musescore/cache/x", file("unrelated")],
+ [".config/mimeapps.list", file("unrelated")],
+ ["Music/Scores/sketch.mscz", file("unrelated")],
+ ["notes.txt", file("unrelated")],
+ // The owned folder's parent holds it, so this file is named alone.
+ ["Documents/MuseScore3/stray.txt", file("unrelated")],
+ ["Documents/MuseScore3/Scores/song.mscz", file("document")],
+ [".cache/fontconfig/cache-1", file("shared_desktop")],
+ [".config/other/unchanged.ini", file("unrelated", true)],
+ ]);
+
+ expect(
+ unownedChangedFolders(files, ["Documents/MuseScore3/Scores"]),
+ ).toEqual([
+ ".config/MuseScore",
+ ".config/mimeapps.list",
+ ".local/share/MuseScore",
+ ".musescore",
+ "Documents/MuseScore3/stray.txt",
+ "Music/Scores",
+ "notes.txt",
+ ]);
+ });
+
+ it("names eight folders in its limitation and counts the rest", () => {
+ const folders = Array.from({ length: 10 }, (_, n) => `.config/app${n}`);
+
+ const limitation = unownedFoldersLimitation(folders) ?? "";
+
+ expect(limitation).toContain("10 folder(s)");
+ expect(limitation).toContain("~/.config/app7,");
+ expect(limitation).not.toContain("~/.config/app8");
+ expect(limitation).toContain("2 more");
+ expect(unownedFoldersLimitation([])).toBeUndefined();
+ });
+
+ it("collapses tens of thousands of separate folders in time that grows with their number, not its square", () => {
+ const count = 20_000;
+ const files = new Map();
+ for (let n = 0; n < count; n += 1)
+ files.set(`Projects/p${String(n)}/notes.txt`, file("unrelated"));
+ // One folder nested in another collapses into it; one beside it whose name it starts does not.
+ files.set("Projects/p1/sub/deep.txt", file("unrelated"));
+ files.set("Projects/p1-old/notes.txt", file("unrelated"));
+ files.set("Owned/a.txt", file("document"));
+
+ const started = performance.now();
+ const folders = unownedChangedFolders(files, ["Owned/b"]);
+ const elapsedMs = performance.now() - started;
+
+ expect(folders).toHaveLength(count + 2);
+ expect(folders).toEqual(
+ expect.arrayContaining(["Projects/p1", "Projects/p1-old", "Owned/a.txt"]),
+ );
+ expect(folders).not.toContain("Projects/p1/sub");
+ // Comparing each folder with every other took tens of seconds at this count.
+ expect(elapsedMs).toBeLessThan(2_000);
+ });
+});
+
+describe("isLockFile", () => {
+ it("knows profile and document locks anywhere, and a *.lock or *.lck file only directly in the app's state directory", () => {
+ const stateDirectories = [".config/app", ".local/share/app"];
+ const locks = [
+ ".config/libreoffice/4/.lock",
+ "Documents/.~lock.q1-sales.ods#",
+ ".config/osmo/lockfile",
+ ".config/app/SingletonLock",
+ ".mozilla/firefox/ab12.default/.parentlock",
+ ".mozilla/firefox/ab12.default/parent.lock",
+ ".config/app/db.lock",
+ ".local/share/app/prefs.lck",
+ ];
+ const others = [
+ "Projects/tool/Cargo.lock",
+ "Projects/web/yarn.lock",
+ // A folder whose name starts with a dot does not make a project's lock file an app's.
+ "Projects/.repo/Cargo.lock",
+ ".dotfiles/web/yarn.lock",
+ ".cargo/registry/Cargo.lock",
+ "Documents/.notes.lock",
+ ".config/app/plugins/db.lock",
+ ".config/other/db.lock",
+ "Documents/locksmith.txt",
+ "Documents/~lock.ods",
+ ".config/app/lock.json",
+ ];
+ const isLock = (file: string) => isLockFile(file, stateDirectories);
+
+ expect(locks.filter(isLock)).toEqual(locks);
+ expect(others.filter(isLock)).toEqual([]);
+ });
+
+ it("knows a *.lock file directly in a state directory nested in another", () => {
+ // A custom XDG_STATE_HOME can put one app's state folder inside another's.
+ const stateDirectories = [".config/app", ".config/app/state/app"];
+
+ expect(isLockFile(".config/app/state/app/db.lock", stateDirectories)).toBe(
+ true,
+ );
+ expect(isLockFile(".config/app/state/db.lock", stateDirectories)).toBe(
+ false,
+ );
+ });
+});
+
+describe("the limitations on folders the app held files open in", () => {
+ it("names the folders, under the home from it, in the capture's", () => {
+ expect(
+ unretainedOpenedLimitation(
+ ["/home/user/Documents", "/home/user", "/media/usb"],
+ "/home/user",
+ ),
+ ).toContain("held files open in ~/Documents, ~, /media/usb, whose");
+ expect(unretainedOpenedLimitation([], "/home/user")).toBeUndefined();
+ });
+
+ it("names the seed files there in the export's, and says when there are none", () => {
+ const home = "/home/user";
+ const seeded = heldOpenLimitation([`${home}/Documents`], home, [
+ {
+ seed: ".config/libreoffice/4/user/registrymodifications.xcu",
+ source: `${home}/.config/libreoffice/4/user/registrymodifications.xcu`,
+ },
+ {
+ seed: "Documents/q1-sales.ods",
+ source: `${home}/Documents/q1-sales.ods`,
+ },
+ ]);
+ const none = heldOpenLimitation([`${home}/Documents`], home, [
+ {
+ seed: ".config/libreoffice/4/user/registrymodifications.xcu",
+ source: `${home}/.config/libreoffice/4/user/registrymodifications.xcu`,
+ },
+ ]);
+
+ expect(seeded).toContain("held files open in ~/Documents,");
+ expect(seeded).toContain(
+ "the seed holds ~/Documents/q1-sales.ods from there",
+ );
+ expect(seeded).not.toContain("registrymodifications");
+ expect(none).toContain("the seed holds none from there");
+ expect(
+ heldOpenLimitation([`${home}/data`], home, [
+ { seed: "Documents/note.txt", source: `${home}/data/note.txt` },
+ ]),
+ ).toContain("the seed holds ~/Documents/note.txt from there");
+ expect(heldOpenLimitation([], home, [])).toBeUndefined();
+ });
+});
+
+describe("runtimeSeeds", () => {
+ it("finds WebKit and Chromium storage and caches, each by the folder that holds it, and nothing else", () => {
+ expect(
+ runtimeSeeds([
+ ".local/share/osmo/hsts-storage.sqlite",
+ ".local/share/osmo/mediakeys/v1/salt",
+ ".local/share/osmo/storage/HbE7/HbE7/origin",
+ ".local/share/osmo/storage/salt",
+ ".config/app/Local Storage/leveldb/000003.log",
+ ".config/app/Cookies",
+ ".config/app/Code Cache/js/index",
+ ".config/app/Cache/Cache_Data/data_0",
+ ".config/app/settings.json",
+ ".local/share/notes/storage/notes.json",
+ "Documents/locksmith.txt",
+ ]),
+ ).toEqual([
+ {
+ kind: "web_storage",
+ paths: [
+ ".config/app/Cookies",
+ ".config/app/Local Storage",
+ ".local/share/osmo/hsts-storage.sqlite",
+ ".local/share/osmo/mediakeys",
+ ".local/share/osmo/storage",
+ ],
+ },
+ {
+ kind: "cache",
+ paths: [".config/app/Cache", ".config/app/Code Cache"],
+ },
+ ]);
+ });
+});
diff --git a/src/domain/desktop/appExportUnowned.ts b/src/domain/desktop/appExportUnowned.ts
new file mode 100644
index 0000000000..4927b1b938
--- /dev/null
+++ b/src/domain/desktop/appExportUnowned.ts
@@ -0,0 +1,259 @@
+import {
+ STATE_PATH_INPUTS,
+ STATE_PATH_INPUTS_MARKDOWN,
+} from "./buildCapture.js";
+import type { DesktopStatePath } from "./desktopState.js";
+import { isWithinAny } from "./desktopStateDiff.js";
+import {
+ isLockName,
+ runtimeFile,
+ type RuntimeFileKind,
+} from "./runtimeFiles.js";
+import { stateDirectoryOf } from "./stateRoles.js";
+
+/** The most paths a limitation or README line names before counting the rest. */
+const MAX_LISTED = 8;
+
+/** Home folders other apps keep files in too, most specific first: a path in one collapses to its child there. */
+const SHARED_PARENTS = [
+ ".local/share",
+ ".local/state",
+ ".config",
+ ".cache",
+ ".local",
+];
+
+const folderOf = (path: string) => path.split("/").slice(0, -1).join("/");
+
+/** The folders a home path lies in, outermost first: `a` and `a/b` for `a/b/c`. */
+const ancestorsOf = (path: string): readonly string[] => {
+ const ancestors: string[] = [];
+ for (let at = path.indexOf("/"); at >= 0; at = path.indexOf("/", at + 1))
+ ancestors.push(path.slice(0, at));
+ return ancestors;
+};
+
+/**
+ * A home file's top state folder: its folder in a shared folder, such as
+ * `.config/MuseScore`, the hidden folder of the home it lies in, or else its
+ * own folder; a file directly in the home or a shared folder is itself.
+ */
+const topFolder = (file: string): string => {
+ const shared = SHARED_PARENTS.find((parent) => file.startsWith(`${parent}/`));
+ const rest = (
+ shared === undefined ? file : file.slice(shared.length + 1)
+ ).split("/");
+ const [first = file] = rest;
+ if (rest.length === 1) return file;
+ if (shared !== undefined) return `${shared}/${first}`;
+ return first.startsWith(".") ? first : folderOf(file);
+};
+
+/**
+ * The folders under the home where files changed during the capture that the
+ * bundle neither owns nor seeds, each collapsed to its top state folder, or
+ * to the file's own folder or the file when that holds an owned path, nested
+ * ones into the outermost. Desktop services' files are left out. A path is
+ * looked up by the folders it lies in, so the work grows with the number of
+ * files and their depth rather than with the folders compared pairwise.
+ */
+export const unownedChangedFolders = (
+ files: ReadonlyMap,
+ owned: readonly string[],
+): readonly string[] => {
+ const ownedPaths = new Set(owned);
+ // The owned paths and every folder holding one.
+ const holdingOwned = new Set(
+ owned.flatMap((path) => [path, ...ancestorsOf(path)]),
+ );
+ const isOwned = (home: string) =>
+ ownedPaths.has(home) ||
+ ancestorsOf(home).some((folder) => ownedPaths.has(folder));
+ const folders = new Set();
+ for (const [home, path] of files) {
+ if (
+ path.changes.length === 0 ||
+ path.role === "shared_desktop" ||
+ isOwned(home)
+ )
+ continue;
+ folders.add(
+ [topFolder(home), folderOf(home)].find(
+ (folder) => folder.length > 0 && !holdingOwned.has(folder),
+ ) ?? home,
+ );
+ }
+ return [...folders]
+ .filter(
+ (folder) => !ancestorsOf(folder).some((other) => folders.has(other)),
+ )
+ .sort();
+};
+
+/** What a seed file the app keeps for its own running, seeded as captured, is. */
+export type RuntimeSeedKind = RuntimeFileKind;
+
+/** Seed files of one kind, each collapsed to the folder that holds that kind, or itself. */
+export interface RuntimeSeeds {
+ readonly kind: RuntimeSeedKind;
+ readonly paths: readonly string[];
+}
+
+/**
+ * Whether a home file is a lock an app holds while it runs: a profile lock's
+ * name anywhere, LibreOffice's document lock, or a `*.lock` or `*.lck` file
+ * directly in one of the app's state directories, home-relative. Elsewhere
+ * such a file may be a project's own, such as `Cargo.lock`, which pins
+ * dependencies rather than marking a profile in use.
+ */
+export const isLockFile = (
+ file: string,
+ stateDirectories: readonly string[],
+): boolean => {
+ if (isLockName(file)) return true;
+ const name = file.split("/").at(-1) ?? "";
+ const folder = folderOf(file);
+ // Each directory on its own, so one nested in another is still found.
+ return (
+ /\.(?:lock|lck)$/iu.test(name) &&
+ folder.length > 0 &&
+ stateDirectories.some(
+ (directory) => stateDirectoryOf(file, [directory]) === folder,
+ )
+ );
+};
+
+const KINDS: readonly RuntimeSeedKind[] = ["web_storage", "cache"];
+const KIND_LABELS: Readonly> = {
+ web_storage: "WebKit or Chromium storage",
+ cache: "caches",
+};
+
+/** The seed's web engine storage and caches, which the app keeps for its own running rather than as the user's data. */
+export const runtimeSeeds = (
+ seedFiles: readonly string[],
+): readonly RuntimeSeeds[] => {
+ const byKind = new Map>();
+ for (const file of seedFiles) {
+ const found = runtimeFile(file);
+ if (found === undefined) continue;
+ const paths = byKind.get(found.kind) ?? new Set();
+ paths.add(found.path);
+ byKind.set(found.kind, paths);
+ }
+ return KINDS.flatMap((kind) => {
+ const paths = byKind.get(kind);
+ return paths === undefined ? [] : [{ kind, paths: [...paths].sort() }];
+ });
+};
+
+/** Names as a list, the first eight named and the rest counted. */
+export const counted = (names: readonly string[]): string => {
+ const named = names.slice(0, MAX_LISTED).join(", ");
+ const rest = names.length - MAX_LISTED;
+ return rest > 0 ? `${named}, and ${String(rest)} more` : named;
+};
+
+/** Home paths as a list, the first eight named and the rest counted. */
+const listed = (paths: readonly string[], markdown: boolean): string =>
+ counted(paths.map((path) => (markdown ? `\`~/${path}\`` : `~/${path}`)));
+
+/** An absolute path relative to the home: empty for the home itself, undefined outside it. */
+const underHome = (path: string, home: string): string | undefined =>
+ path === home
+ ? ""
+ : path.startsWith(`${home}/`)
+ ? path.slice(home.length + 1)
+ : undefined;
+
+/** An absolute path as a limitation spells it: `~` and `~/` in the home, else as it is. */
+const fromHome = (path: string, home: string): string => {
+ const relative = underHome(path, home);
+ return relative === undefined
+ ? path
+ : relative === ""
+ ? "~"
+ : `~/${relative}`;
+};
+
+/** The capture's limitation on folders the app held files open in whose starting content it did not retain; none when there are none. */
+export const unretainedOpenedLimitation = (
+ directories: readonly string[],
+ home: string,
+): string | undefined =>
+ directories.length === 0
+ ? undefined
+ : `The application held files open in ${counted(directories.map((directory) => fromHome(directory, home)))}, whose starting content was not retained; name them with ${STATE_PATH_INPUTS} to retain it.`;
+
+/**
+ * The export's limitation on those folders: a file there is seeded only when
+ * the capture retained it, as it does one written during the capture, and
+ * the seed files there are named; none when there are no such folders.
+ */
+export const heldOpenLimitation = (
+ directories: readonly string[],
+ home: string,
+ seedFiles: readonly { readonly seed: string; readonly source: string }[],
+): string | undefined => {
+ if (directories.length === 0) return undefined;
+ const there = seedFiles
+ .filter((file) =>
+ directories.some((folder) => isWithinAny(file.source, [folder])),
+ )
+ .map((file) => file.seed);
+ return `The application held files open in ${counted(directories.map((directory) => fromHome(directory, home)))}, whose content from before the capture was not retained: a file there is seeded only when the capture retained it, such as one written during the capture; the seed holds ${there.length === 0 ? "none" : listed(there, false)} from there.`;
+};
+
+/** The limitation naming lock files left out of the seed; none when there were none. */
+export const leftOutLocksLimitation = (
+ locks: readonly string[],
+): string | undefined =>
+ locks.length === 0
+ ? undefined
+ : `Left out of the seed: lock files the app holds while it runs, since a copied one makes the app find its profile or document in use and can keep it from starting: ${listed(locks, false)}. data/reset restores the seed without them.`;
+
+/** The limitation naming unowned folders the capture saw change; none when there are none. */
+export const unownedFoldersLimitation = (
+ folders: readonly string[],
+): string | undefined =>
+ folders.length === 0
+ ? undefined
+ : `Files changed during the capture in ${String(folders.length)} folder(s) under the home that the bundle neither owns nor seeds, so the app starts without what it saved there, such as settings a first-run dialog writes: ${listed(folders, false)}. To own them, capture again naming them with ${STATE_PATH_INPUTS}.`;
+
+/** The limitation naming seeded web engine storage and caches; none when the seed holds none. */
+export const runtimeSeedsLimitation = (
+ seeds: readonly RuntimeSeeds[],
+): string | undefined =>
+ seeds.length === 0
+ ? undefined
+ : `The seed holds files the app keeps for its own running, seeded as captured: ${seeds.map(({ kind, paths }) => `${KIND_LABELS[kind]} ${listed(paths, false)}`).join("; ")}. A stale one can change how the app starts.`;
+
+/** The README lines on unowned folders, runtime seeds and left-out locks, each starting on a new line; empty when there are none. */
+export const unownedReadmeNotes = (
+ folders: readonly string[],
+ seeds: readonly RuntimeSeeds[],
+ locks: readonly string[],
+): string =>
+ [
+ ...(folders.length === 0
+ ? []
+ : [
+ `- Neither owned nor seeded, so the app starts without what it saved there: files changed during the capture in
+ ${listed(folders, true)}.
+ To own them, capture again naming them with ${STATE_PATH_INPUTS_MARKDOWN}.`,
+ ]),
+ ...(seeds.length === 0
+ ? []
+ : [
+ `- Seeded as captured, though the app keeps them for its own running, so a stale one can change how it starts:
+ ${seeds.map(({ kind, paths }) => `${KIND_LABELS[kind]} ${listed(paths, true)}`).join("; ")}.`,
+ ]),
+ ...(locks.length === 0
+ ? []
+ : [
+ `- Left out of the seed: lock files the app holds while it runs, since a copied one can make it find its profile or document
+ in use and keep it from starting: ${listed(locks, true)}.`,
+ ]),
+ ]
+ .map((line) => `\n${line}`)
+ .join("");
diff --git a/src/domain/desktop/appSettle.ts b/src/domain/desktop/appSettle.ts
new file mode 100644
index 0000000000..2d5b0616f1
--- /dev/null
+++ b/src/domain/desktop/appSettle.ts
@@ -0,0 +1,63 @@
+import { createHash } from "node:crypto";
+
+/** How an app's data/get is read until its files settle; the exported verifier renders the same values. */
+export interface SettleTiming {
+ /** Between one read's start and the next. */
+ readonly intervalMs: number;
+ /** How long the files must hold, from the read that returned them, before they are graded. */
+ readonly windowMs: number;
+ /** After the first read begins: no read starts then, one in flight is cut, and the last completed read is graded. */
+ readonly boundMs: number;
+}
+
+/**
+ * Grading waits for the app's own save: Gnote saves a note 4 s after the
+ * last key, so a read right after the agent stops sees the text unsaved. A
+ * save that lands more than the window after the first read is still
+ * missed, and an app whose files never change pays the window.
+ */
+export const SETTLE_TIMING: SettleTiming = {
+ intervalMs: 500,
+ windowMs: 3_000,
+ boundMs: 15_000,
+};
+
+/** Where a data/get file part names its path in the home. */
+const DATA_OBJECTS = "urn:agentenv:data-objects/v1";
+
+const textField = (value: unknown, key: string): string | undefined => {
+ const field =
+ typeof value === "object" && value !== null
+ ? (value as Record)[key]
+ : undefined;
+ return typeof field === "string" ? field : undefined;
+};
+
+/**
+ * The files an app's data/get answer holds, each by its home path, or its
+ * place among the parts, with the SHA-256 of its bytes as sent: what
+ * changes when the app saves, compared between reads.
+ */
+export const appFileDigests = (parts: readonly unknown[]): string =>
+ JSON.stringify(
+ parts
+ .flatMap((part, index) => {
+ if (textField(part, "kind") !== "file") return [];
+ const { file, metadata } = part as {
+ readonly file?: unknown;
+ readonly metadata?: Record | null;
+ };
+ const content = textField(file, "bytes") ?? textField(file, "uri");
+ return [
+ [
+ textField(metadata?.[DATA_OBJECTS], "path") ?? `#${String(index)}`,
+ createHash("sha256")
+ .update(content ?? "")
+ .digest("hex"),
+ ],
+ ];
+ })
+ .sort(([left = ""], [right = ""]) =>
+ left < right ? -1 : left > right ? 1 : 0,
+ ),
+ );
diff --git a/src/domain/desktop/appTaskChecks.fixture.ts b/src/domain/desktop/appTaskChecks.fixture.ts
index 777b6a2972..320150fccb 100644
--- a/src/domain/desktop/appTaskChecks.fixture.ts
+++ b/src/domain/desktop/appTaskChecks.fixture.ts
@@ -26,6 +26,7 @@ export const CHECK_VIEWS: Readonly> = {
tags: ["a", "b"],
"a/b": { "~k": null },
scale: "1.2500",
+ shaped: { name: "Cell", attributes: {}, children: ["437.8"] },
},
},
".config/notes/app.ini": {
@@ -192,12 +193,58 @@ export const CHECK_VIEWS: Readonly> = {
truncated: false,
},
},
+ "Documents/cut-number.xml": {
+ format: "xml",
+ view: {
+ root: element("Cell", {}, ["437.", "8"]),
+ truncated: true,
+ },
+ },
+ "Documents/budget.xml": {
+ format: "xml",
+ view: {
+ root: element("Cells", {}, [
+ element("Cell", { Row: "2", Col: "3" }, ["437.80000000000001"]),
+ element("Cell", { Row: "2", Col: "2" }, ["398.2"]),
+ element("Cell", { Row: "3", Col: "3" }, ["174.9"]),
+ element("Cell", { Row: "5", Col: "3" }, [
+ "437.",
+ element("span", {}, ["80"]),
+ ]),
+ element("Cell", { Row: "6", Col: "3" }, ["four hundred"]),
+ ]),
+ truncated: false,
+ },
+ },
"Documents/photo.png": { format: "binary", view: null },
"Documents/huge.txt": {
format: null,
view: null,
error: "larger than 64 MiB, the most the view reads",
},
+ "Documents/Budget.gnumeric": {
+ format: "xml",
+ compression: "gzip",
+ view: {
+ root: element("gnm:Workbook", {}, [
+ element("gnm:Name", {}, ["Budget"]),
+ element("gnm:Cell", { Row: "1", Col: "2" }, ["437.80000000000001"]),
+ ]),
+ truncated: false,
+ },
+ },
+ "Documents/bomb.gnumeric": {
+ format: null,
+ compression: "gzip",
+ view: null,
+ error: "expands more than 100 times, so it is not read",
+ },
+ "Documents/damaged.gnumeric": {
+ format: null,
+ compression: "gzip",
+ view: null,
+ error: "not a readable gzip stream",
+ },
};
/** An element of REA's XML view. */
@@ -511,6 +558,100 @@ const TOLERANCE_CASES: readonly (readonly [DesktopTaskCheckInput, boolean])[] =
},
false,
],
+ // A cell's text by two attributes together, in either order, as drafts name cells.
+ [
+ {
+ kind: "value_equals",
+ file: "Documents/budget.xml",
+ pointer: "/root/children/[name=Cell][@Col=3][@Row=2]/children/0",
+ value: 437.8,
+ tolerance: 0.001,
+ },
+ true,
+ ],
+ [
+ {
+ kind: "value_equals",
+ file: "Documents/budget.xml",
+ pointer: "/root/children/[name=Cell][@Row=2][@Col=3]/children/0",
+ value: 437.9,
+ tolerance: 0.001,
+ },
+ false,
+ ],
+ [
+ {
+ kind: "value_equals",
+ file: "Documents/budget.xml",
+ pointer: "/root/children/[name=Cell][@Col=3]/children/0",
+ value: 437.8,
+ tolerance: 0.001,
+ },
+ false,
+ ],
+ // The cell element, so a split run and an equivalent decimal are the same number.
+ [
+ {
+ kind: "value_equals",
+ file: "Documents/budget.xml",
+ pointer: "/root/children/[name=Cell][@Row=2][@Col=3]",
+ value: 437.8,
+ tolerance: 0.001,
+ },
+ true,
+ ],
+ [
+ {
+ kind: "value_equals",
+ file: "Documents/budget.xml",
+ pointer: "/root/children/[name=Cell][@Row=5][@Col=3]",
+ value: 437.8,
+ tolerance: 0.001,
+ },
+ true,
+ ],
+ [
+ {
+ kind: "value_equals",
+ file: "Documents/budget.xml",
+ pointer: "/root/children/[name=Cell][@Row=5][@Col=3]",
+ value: 437.89,
+ tolerance: 0.001,
+ },
+ false,
+ ],
+ [
+ {
+ kind: "value_equals",
+ file: "Documents/budget.xml",
+ pointer: "/root/children/[name=Cell][@Row=6][@Col=3]",
+ value: 400,
+ tolerance: 0.001,
+ },
+ false,
+ ],
+ // A cut view dropped the text between 437. and 8. The joined numeral is not the file's number.
+ [
+ {
+ kind: "value_equals",
+ file: "Documents/cut-number.xml",
+ pointer: "/root",
+ value: 437.8,
+ tolerance: 0.001,
+ },
+ false,
+ ],
+ // A JSON object shaped like an element is not a cell's text.
+ [
+ {
+ kind: "value_equals",
+ file: settings,
+ pointer: "/shaped",
+ value: 437.8,
+ tolerance: 0.001,
+ },
+ false,
+ ],
];
const meeting = "Documents/meeting.xml";
@@ -874,6 +1015,50 @@ const CUT_VIEW_CASES: readonly (readonly [DesktopTaskCheckInput, boolean])[] = [
],
];
+const budget = "Documents/Budget.gnumeric";
+
+/** A gzip file's document checked as the document, and one past the limits or damaged read by no check. */
+const GZIP_CASES: readonly (readonly [DesktopTaskCheckInput, boolean])[] = [
+ [
+ {
+ kind: "value_equals",
+ file: budget,
+ pointer: "/root/children/[name=gnm:Cell][@Row=1][@Col=2]/children/0",
+ value: 437.8,
+ tolerance: 0.001,
+ },
+ true,
+ ],
+ [
+ {
+ kind: "text_equals",
+ file: budget,
+ pointer: "/root/children/[name=gnm:Name]",
+ value: "Budget",
+ },
+ true,
+ ],
+ [{ kind: "lines_present", file: budget, lines: ["Budget"] }, false],
+ [
+ {
+ kind: "value_absent",
+ file: "Documents/bomb.gnumeric",
+ pointer: "/root/attributes/x",
+ },
+ false,
+ ],
+ [
+ {
+ kind: "text_contains",
+ file: "Documents/damaged.gnumeric",
+ pointer: "/root",
+ value: "x",
+ },
+ false,
+ ],
+ [{ kind: "file_present", file: "Documents/damaged.gnumeric" }, true],
+];
+
/** Every kind of check against those views, each with the outcome both evaluators must give. */
export const CHECK_CASES: readonly (readonly [
DesktopTaskCheckInput,
@@ -1124,4 +1309,5 @@ export const CHECK_CASES: readonly (readonly [
...TOLERANCE_CASES,
...TEXT_CASES,
...CUT_VIEW_CASES,
+ ...GZIP_CASES,
];
diff --git a/src/domain/desktop/appTaskChecks.test.ts b/src/domain/desktop/appTaskChecks.test.ts
index c71e687c38..a01920da3b 100644
--- a/src/domain/desktop/appTaskChecks.test.ts
+++ b/src/domain/desktop/appTaskChecks.test.ts
@@ -317,6 +317,12 @@ describe("evaluateTaskChecks", () => {
file: "Documents/photo.png",
lines: ["x"],
},
+ {
+ id: "gzip",
+ kind: "value_absent",
+ file: "Documents/bomb.gnumeric",
+ pointer: "/root/attributes/x",
+ },
] as never,
new Map(Object.entries(CHECK_VIEWS)),
);
@@ -328,6 +334,7 @@ describe("evaluateTaskChecks", () => {
["cut", false, true],
["shown", false, undefined],
["binary", false, undefined],
+ ["gzip", false, true],
]);
});
});
@@ -524,6 +531,77 @@ describe("evaluateTaskChecks text on a cut view", () => {
});
});
+describe("evaluateTaskChecks numeric element text", () => {
+ const grade = (file: string, pointer: string, value: number) =>
+ evaluateTaskChecks(
+ [
+ {
+ id: "c",
+ kind: "value_equals",
+ file,
+ pointer,
+ value,
+ tolerance: 0.001,
+ } as never,
+ ],
+ new Map(Object.entries(CHECK_VIEWS)),
+ )[0];
+
+ it("accepts an equivalent or split decimal on a whole cell, and refuses a cut cell or a JSON object", () => {
+ const cell = (row: string) =>
+ `/root/children/[name=Cell][@Row=${row}][@Col=3]`;
+ expect(grade("Documents/budget.xml", cell("2"), 437.8)?.passed).toBe(true);
+ expect(grade("Documents/budget.xml", cell("5"), 437.8)?.passed).toBe(true);
+ expect(grade("Documents/budget.xml", cell("5"), 437.89)?.passed).toBe(
+ false,
+ );
+ expect(
+ grade("Documents/budget.xml", cell("6"), 400)?.undecided,
+ ).toBeUndefined();
+ expect(grade("Documents/cut-number.xml", "/root", 437.8)).toMatchObject({
+ passed: false,
+ undecided: true,
+ });
+ const shaped = grade(".config/notes/settings.json", "/shaped", 437.8);
+ expect(shaped?.passed).toBe(false);
+ expect(shaped?.undecided).toBeUndefined();
+ expect(shaped?.description).toContain("not a number");
+ });
+
+ it("decides a numeral text node in a cut view, because that node was kept whole", () => {
+ const [outcome] = evaluateTaskChecks(
+ [
+ {
+ id: "c",
+ kind: "value_equals",
+ file: "n.xml",
+ pointer: "/root/children/0",
+ value: 437.8,
+ tolerance: 0.001,
+ } as never,
+ ],
+ new Map([
+ [
+ "n.xml",
+ {
+ format: "xml",
+ view: {
+ root: {
+ name: "Cell",
+ attributes: {},
+ children: ["437.80"],
+ },
+ truncated: true,
+ },
+ },
+ ],
+ ]),
+ );
+ expect(outcome?.passed).toBe(true);
+ expect(outcome?.undecided).toBeUndefined();
+ });
+});
+
describe("evaluateTaskChecks selectors and tolerance", () => {
const plate = "Documents/plate.FCStd";
const objects = "/entries/[name=Document.xml]/view/root/children";
diff --git a/src/domain/desktop/appTaskChecks.ts b/src/domain/desktop/appTaskChecks.ts
index 165adbe5a3..7a02b79d3b 100644
--- a/src/domain/desktop/appTaskChecks.ts
+++ b/src/domain/desktop/appTaskChecks.ts
@@ -17,6 +17,8 @@ import {
/** One file as the app's data/get reports it: its view, or why it has none. An absent file has no entry. */
export interface TaskFileView {
readonly format: string | null;
+ /** Present when the view is of the document inside a gzip file, which checks read as the document. */
+ readonly compression?: string;
readonly view: JsonValue | null;
readonly error?: string | null;
}
@@ -91,7 +93,7 @@ export const sameJson = (
const NUMERAL = /^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$/u;
/** A number, or a numeral string read as one; undefined for anything else or a numeral past the doubles. */
-const numberOf = (value: JsonValue): number | undefined => {
+export const numberOf = (value: JsonValue): number | undefined => {
if (typeof value === "number") return value;
if (typeof value !== "string" || !NUMERAL.test(value)) return undefined;
const number = Number(value);
@@ -199,14 +201,33 @@ const lineVerdict = (
);
};
-/** A value compared within a tolerance: numbers, and numeral strings, by their difference. */
+/**
+ * Whether `value` is an XML element whose joined text is the number: an XML
+ * file, or an XML entry inside a zip. A JSON object with the same shape is
+ * the value itself, not a cell.
+ */
+const joinsElementText = (
+ format: string | null | undefined,
+ view: JsonValue | null,
+ pointer: string,
+ value: JsonValue,
+): boolean => {
+ if (!isXmlElement(value)) return false;
+ if (format === "xml") return true;
+ if (format !== "zip") return false;
+ const entry = zipEntryAt(view, pointer);
+ return typeof entry === "object" && entry.format === "xml";
+};
+
+/** A value compared within a tolerance: numbers, and numeral strings, by their difference. An XML element's text is joined only when `join` says the view is XML. */
const nearVerdict = (
at: string,
found: JsonValue,
expected: number,
tolerance: number,
+ join: boolean,
): Verdict => {
- const number = numberOf(found);
+ const number = numberOf(join ? elementText(found) : found);
const within = `within ${shown(tolerance)} of ${shown(expected)}`;
if (number === undefined)
return fail(`${at} is ${shown(found)}, not a number`);
@@ -278,8 +299,17 @@ const valueVerdict = (
if (whole && truncated) return undecided(check.file);
if (check.kind === "value_equals") {
if (located.kind === "none") return noValue(at, located, truncated);
- if (check.tolerance !== undefined && typeof check.value === "number")
- return nearVerdict(at, located.value, check.value, check.tolerance);
+ if (check.tolerance !== undefined && typeof check.value === "number") {
+ // A cut element's kept runs can join into a number the file never held.
+ const join = joinsElementText(
+ entry?.format,
+ read.view,
+ check.pointer,
+ located.value,
+ );
+ if (join && truncated) return undecided(check.file);
+ return nearVerdict(at, located.value, check.value, check.tolerance, join);
+ }
return sameJson(located.value, check.value)
? pass(`${at} is ${shown(check.value)}`)
: fail(`${at} is ${shown(located.value)}, not ${shown(check.value)}`);
diff --git a/src/domain/desktop/appTaskDraft.fixture.ts b/src/domain/desktop/appTaskDraft.fixture.ts
new file mode 100644
index 0000000000..833c6b5c71
--- /dev/null
+++ b/src/domain/desktop/appTaskDraft.fixture.ts
@@ -0,0 +1,80 @@
+import type { JsonValue } from "../jsonValue.js";
+import type { DesktopTaskCheckInput } from "./appTask.js";
+import { evaluateTaskChecks } from "./appTaskChecks.js";
+import { draftTaskFiles, type TaskDraftFile } from "./appTaskDraft.js";
+import type { DesktopStatePath } from "./desktopState.js";
+import { xmlView } from "./stateFormats.js";
+
+type Format = DesktopStatePath["format"];
+type Role = DesktopStatePath["role"];
+
+/** A file's state at checkpoint 0 (the seed) and 1 (the end); a null view is a file absent there. */
+export const statePath = (
+ role: Role,
+ format: Format,
+ before: JsonValue | null,
+ after: JsonValue | null,
+): DesktopStatePath => ({
+ path: "/home/ada/x",
+ home_path: "x",
+ role,
+ reasons: [],
+ format,
+ versions: [before, after].map((view, index) => ({
+ checkpoint_index: index,
+ checkpoint: index === 0 ? "seed" : "end",
+ present: view !== null,
+ sha256: view === null ? null : String(index).repeat(64),
+ format: view === null ? null : format,
+ view,
+ view_error: null,
+ })),
+ changes: [],
+});
+
+/** The draft of files by home path, every one served, from the seed to the end. */
+export const draftOf = (
+ files: Record,
+): readonly TaskDraftFile[] =>
+ draftTaskFiles(new Map(Object.entries(files)), () => true, {
+ seed: 0,
+ end: 1,
+ });
+
+/** An XML text's view, which a test's own text always has. */
+export const xml = (text: string): JsonValue => {
+ const view = xmlView(text);
+ if (view === undefined) throw new Error(`not XML: ${text}`);
+ return view;
+};
+
+/** The one drafted file of an XML file's two saves. */
+export const xmlDraft = (
+ before: string,
+ after: string,
+ {
+ role = "document",
+ file = "Documents/doc.xml",
+ }: { role?: Role; file?: string } = {},
+): TaskDraftFile => {
+ const [drafted] = draftOf({
+ [file]: statePath(role, "xml", xml(before), xml(after)),
+ });
+ if (drafted === undefined) throw new Error("the file did not change");
+ return drafted;
+};
+
+/** Which checks pass on one XML view of a file, in order. */
+export const passing = (
+ file: string,
+ view: JsonValue,
+ checks: readonly DesktopTaskCheckInput[],
+): boolean[] =>
+ evaluateTaskChecks(
+ checks.map((check, index) => ({ ...check, id: `c${String(index)}` })),
+ new Map([[file, { format: "xml" as const, view }]]),
+ ).map(({ passed }) => passed);
+
+/** The pointer a check reads, when it reads one. */
+export const pointerOf = (check: DesktopTaskCheckInput): string | undefined =>
+ "pointer" in check ? check.pointer : undefined;
diff --git a/src/domain/desktop/appTaskDraft.test.ts b/src/domain/desktop/appTaskDraft.test.ts
index 5cdd203467..a28f2baa85 100644
--- a/src/domain/desktop/appTaskDraft.test.ts
+++ b/src/domain/desktop/appTaskDraft.test.ts
@@ -1370,6 +1370,33 @@ const listing = (entries: JsonValue[]): JsonValue => ({
});
describe("draftTaskFiles archives added to whole", () => {
+ it("checks JSON values that resemble XML elements using the entry format", () => {
+ const file = "Documents/a.zip";
+ const before = listing([zipEntry("a.json", "json", {})]);
+ const saved = (label: string) =>
+ listing([
+ zipEntry("a.json", "json", {
+ element: { name: "Cell", attributes: {}, children: [], label },
+ }),
+ ]);
+ const [result] = draft({
+ [file]: path("document", "zip", before, saved("keep")),
+ });
+ const checks = result?.suggested ?? [];
+ expect(checks).toContainEqual({
+ kind: "value_equals",
+ file,
+ pointer: "/entries/[name=a.json]/view/element/label",
+ value: "keep",
+ });
+ expect(passing(file, checks, saved("keep"), "zip").every(Boolean)).toBe(
+ true,
+ );
+ expect(passing(file, checks, saved("wrong"), "zip").every(Boolean)).toBe(
+ false,
+ );
+ });
+
it("suggests checks inside a JSON value or a readable entry added whole", () => {
const [file] = draft({
"Documents/a.zip": path(
@@ -1496,8 +1523,8 @@ describe("draftTaskFiles archive elements by identity", () => {
{
kind: "value_equals",
file: "Documents/plate.zip",
- pointer: `${objects}/[name=Object][@name=Hole001]/name`,
- value: "Object",
+ pointer: `${objects}/[name=Object][@name=Hole001]/attributes/type`,
+ value: "Part::Cylinder",
},
{
kind: "value_absent",
@@ -1733,8 +1760,8 @@ describe("draftTaskFiles archives", () => {
{
kind: "value_equals",
file: "Documents/plate.FCStd",
- pointer: `${objects}/[name=Object][@name=Hole]/name`,
- value: "Object",
+ pointer: `${objects}/[name=Object][@name=Hole]/attributes/type`,
+ value: "Part::Cylinder",
},
{
kind: "value_absent",
diff --git a/src/domain/desktop/appTaskDraft.ts b/src/domain/desktop/appTaskDraft.ts
index 06109b843e..6a42c3061e 100644
--- a/src/domain/desktop/appTaskDraft.ts
+++ b/src/domain/desktop/appTaskDraft.ts
@@ -1,20 +1,29 @@
import type { JsonValue } from "../jsonValue.js";
import { isTimestamp, type DesktopTaskCheckInput } from "./appTask.js";
+import { fileNotes, type FileNote } from "./appTaskDraftFileNotes.js";
+import { lineGuards } from "./appTaskDraftLineGuards.js";
import {
ARCHIVE,
- escapeToken,
innermost,
- namedSuggestions,
+ namedDraft,
+ presenceValues,
XML_FILE,
+ type NamedFacts,
} from "./appTaskDraftNamed.js";
+import { rowSuggestions } from "./appTaskDraftRows.js";
+import {
+ draftValues,
+ oneEnd,
+ withinDraftBudget,
+ type DraftValue,
+} from "./appTaskDraftValues.js";
import {
versionAt,
type DesktopStatePath,
type StateEnds,
} from "./desktopState.js";
-import { countTaskRows } from "./appTaskChecks.js";
-import { diffStateViews, sqliteRowIdentity } from "./stateChanges.js";
-import { resolvePointer } from "./viewPointer.js";
+import { diffNamedMembers, diffStateViews } from "./stateChanges.js";
+import { escapeToken, resolvePointer } from "./viewPointer.js";
type Version = DesktopStatePath["versions"][number];
type Suggestion = DesktopTaskCheckInput;
@@ -135,20 +144,95 @@ const keyPointers = (
),
);
+/** A file's draft: its suggested checks, the values its change holds, guards, and the facts its notes rest on. */
+interface FileDraft {
+ readonly suggested: readonly Suggestion[];
+ readonly values: readonly DraftValue[];
+ readonly valuesCut: boolean;
+ readonly guards: readonly Suggestion[];
+ readonly facts?: NamedFacts;
+}
+
+const NO_VALUES = { values: [], valuesCut: false, guards: [] } as const;
+
+/** A JSON or key file's change at pointers into its view, an absent end's view null; a key file's diff names `group/key`. */
+const plainChange = (
+ format: "json" | "keyfile",
+ baseline: Version,
+ final: Version,
+) => {
+ if (format === "json") return diffNamedMembers(baseline.view, final.view);
+ const change = diffStateViews(format, baseline.view, final.view);
+ const pointers = keyPointers([baseline.view, final.view]);
+ const at = (entries: readonly string[]) =>
+ entries.flatMap((entry) => pointers.get(entry) ?? []);
+ return {
+ ...change,
+ added: at(change.added),
+ changed: at(change.changed),
+ removed: at(change.removed),
+ };
+};
+
+/** The values a JSON or key file's change holds; every pointer is the document's, and none is noise. */
+const plainValues = (
+ change: ReturnType,
+ baseline: Version,
+ final: Version,
+) =>
+ draftValues(change, {
+ ends: versions(baseline, final),
+ listed: () => true,
+ noise: () => undefined,
+ format: () => "json",
+ referenceRoots: () => [],
+ });
+
+/**
+ * The values of a file created or deleted, apart from the check of its
+ * presence: what its one end holds, added or removed, in an XML file,
+ * archive, JSON or key file, within the limits a changed file's have.
+ */
+const presenceOnly = (
+ baseline: Version,
+ final: Version,
+): Pick => {
+ const created = !baseline.present;
+ const format = (created ? final : baseline).format;
+ const listed =
+ format === "zip" || format === "xml"
+ ? presenceValues(
+ versions(baseline, final),
+ format === "zip" ? ARCHIVE : XML_FILE,
+ created,
+ )
+ : format === "json" || format === "keyfile"
+ ? plainValues(
+ oneEnd(plainChange(format, baseline, final), created),
+ baseline,
+ final,
+ )
+ : { values: [], cut: false };
+ return { values: listed.values, valuesCut: listed.cut };
+};
+
const valueSuggestions = (
file: string,
baseline: Version,
final: Version,
-): Suggestion[] => {
+ document: boolean,
+): FileDraft => {
const format = final.format;
- if (format === "zip" || format === "xml")
- return namedSuggestions(
+ if (format === "zip" || format === "xml") {
+ return namedDraft(
file,
versions(baseline, final),
format === "zip" ? ARCHIVE : XML_FILE,
- LIMIT,
+ { limit: LIMIT, document },
);
- if (format !== "json" && format !== "keyfile") return [];
+ }
+ if (format !== "json" && format !== "keyfile")
+ return { suggested: [], ...NO_VALUES };
const change = diffStateViews(format, baseline.view, final.view);
const pointers = keyPointers([baseline.view, final.view]);
const pointer = (entry: string) =>
@@ -173,178 +257,48 @@ const valueSuggestions = (
if (at !== undefined && at !== "")
suggestions.push({ kind: "value_absent", file, pointer: at });
}
- return suggestions;
-};
-
-interface TableView {
- readonly columns: readonly string[];
- readonly key: readonly string[];
- readonly rows: readonly (readonly JsonValue[])[];
-}
-
-const tableViews = (version: Version): ReadonlyMap => {
- const tables = new Map();
- const view = version.present ? version.view : null;
- if (!isRecord(view) || !isRecord(view.tables)) return tables;
- for (const [name, table] of Object.entries(view.tables)) {
- if (!isRecord(table) || table.truncated !== false) continue;
- const strings = (value: JsonValue | undefined) =>
- Array.isArray(value)
- ? value.filter((item) => typeof item === "string")
- : [];
- tables.set(name, {
- columns: strings(table.columns),
- key: strings(table.primary_key),
- rows: Array.isArray(table.rows) ? table.rows.filter(Array.isArray) : [],
- });
- }
- return tables;
-};
-
-/** Column values of a row, without volatile values or the columns left out. */
-const rowWhere = (
- table: TableView,
- row: readonly JsonValue[],
- keep: (column: string) => boolean,
-): Record =>
- Object.fromEntries(
- table.columns.flatMap((column, index) => {
- const value = row[index] ?? null;
- return keep(column) && !isTimestamp(value) ? [[column, value]] : [];
- }),
+ const { values, cut } = plainValues(
+ plainChange(format, baseline, final),
+ baseline,
+ final,
);
-
-/** Match uniquely keyed rows separately from multiset row conditions. */
-const keyedRows = (table: TableView) =>
- new Map(
- table.rows.flatMap((row) => {
- const identity = sqliteRowIdentity(table.columns, table.key, row);
- return identity.keyed ? [[identity.value, row] as const] : [];
- }),
- );
-
-/** The predicates a task can express after volatile values are omitted. */
-const rowConditions = (table: TableView) => {
- const conditions = new Map>();
- for (const row of table.rows) {
- if (sqliteRowIdentity(table.columns, table.key, row).keyed) continue;
- const where = rowWhere(table, row, () => true);
- if (Object.keys(where).length === 0) continue;
- conditions.set(JSON.stringify(where), where);
- }
- return conditions;
+ return { suggested: suggestions, values, valuesCut: cut, guards: [] };
};
-const unkeyedRowSuggestions = (
- file: string,
- table: string,
- baseline: TableView,
- final: TableView,
-): Suggestion[] => {
- const conditions = new Map([
- ...rowConditions(final),
- ...rowConditions(baseline),
- ]);
- const suggestions: Suggestion[] = [];
- for (const where of conditions.values()) {
- const previous = countTaskRows(baseline.columns, baseline.rows, where);
- const count = countTaskRows(final.columns, final.rows, where);
- if (previous === count) continue;
- if (count === 0)
- suggestions.push({ kind: "row_absent", file, table, where });
- else if (previous === 0)
- suggestions.push({ kind: "row_present", file, table, where });
- else
- suggestions.push({
- kind: "row_count",
- file,
- table,
- where,
- equals: count,
- });
- }
- return suggestions;
-};
-
-const rowSuggestions = (
- file: string,
- baseline: Version,
- final: Version,
-): Suggestion[] => {
- const before = tableViews(baseline);
- const suggestions: Suggestion[] = [];
- const push = (
- kind: "row_present" | "row_absent",
- table: string,
- where: Record,
- ) => {
- if (Object.keys(where).length > 0)
- suggestions.push({ kind, file, table, where });
- };
- for (const [name, table] of tableViews(final)) {
- const old = before.get(name) ?? { ...table, rows: [] };
- const oldRows = keyedRows(old);
- const newRows = keyedRows(table);
- for (const [id, row] of newRows) {
- const previous = oldRows.get(id);
- if (previous === undefined)
- push(
- "row_present",
- name,
- rowWhere(table, row, (column) => !table.key.includes(column)),
- );
- else if (JSON.stringify(previous) !== JSON.stringify(row))
- push(
- "row_present",
- name,
- rowWhere(
- table,
- row,
- (column) =>
- table.key.includes(column) ||
- JSON.stringify(row[table.columns.indexOf(column)]) !==
- JSON.stringify(previous[old.columns.indexOf(column)]),
- ),
- );
- }
- for (const [id, row] of oldRows)
- if (!newRows.has(id))
- push(
- "row_absent",
- name,
- rowWhere(old, row, (column) => old.key.includes(column)),
- );
- suggestions.push(...unkeyedRowSuggestions(file, name, old, table));
- if (old.rows.length !== table.rows.length)
- suggestions.push({
- kind: "row_count",
- file,
- table: name,
- equals: table.rows.length,
- });
- }
- return suggestions;
-};
-
-/** Checks that express how a file changed from its baseline to its final version. */
+/** Checks that express how a file changed from its baseline to its final version, with the values, guards and facts beside them. */
const suggestedChecks = (
file: string,
baseline: Version,
final: Version,
-): readonly Suggestion[] => {
- if (!final.present) return [{ kind: "file_absent", file }];
+ document: boolean,
+): FileDraft => {
+ if (!final.present)
+ return {
+ suggested: [{ kind: "file_absent", file }],
+ ...NO_VALUES,
+ ...presenceOnly(baseline, final),
+ };
const created: Suggestion[] = baseline.present
? []
: [{ kind: "file_present", file }];
- const content =
+ const content: FileDraft =
final.format === "text"
- ? textSuggestions(file, baseline, final)
+ ? {
+ ...NO_VALUES,
+ suggested: textSuggestions(file, baseline, final),
+ guards: baseline.present
+ ? lineGuards(file, linesOf(baseline), linesOf(final))
+ : [],
+ }
: !baseline.present
- ? []
+ ? { suggested: [], ...NO_VALUES, ...presenceOnly(baseline, final) }
: final.format === "sqlite"
- ? rowSuggestions(file, baseline, final)
- : valueSuggestions(file, baseline, final);
- return [...created, ...content].slice(0, MAX_SUGGESTIONS);
+ ? { suggested: rowSuggestions(file, baseline, final), ...NO_VALUES }
+ : valueSuggestions(file, baseline, final, document);
+ return {
+ ...content,
+ suggested: [...created, ...content.suggested].slice(0, MAX_SUGGESTIONS),
+ };
};
/** How a file changed from its baseline to its final version, or undefined when it ends as it began. */
@@ -372,7 +326,10 @@ const netChange = (baseline: Version, final: Version) => {
return { kind: "unreadable" };
if (baseline.format !== final.format) return { kind: "format" };
if (final.format === "binary") return { kind: "content" };
- const difference = diffStateViews(final.format, baseline.view, final.view);
+ // XML members are listed by the selectors naming them, as checks name them.
+ const difference = diffStateViews(final.format, baseline.view, final.view, {
+ named: true,
+ });
if (
difference.added_count +
difference.removed_count +
@@ -399,20 +356,24 @@ export interface TaskDraftFile {
readonly final: ReturnType;
readonly change: NonNullable>;
readonly suggested: readonly Suggestion[];
+ /** The values the change added, changed and removed, at the pointers a check names them by. */
+ readonly values: readonly DraftValue[];
+ /** Whether a limit left values out. */
+ readonly values_cut: boolean;
+ /** Checks that the rest stays as it was, for an author to add by choice. */
+ readonly guards: readonly Suggestion[];
+ /** Why the file's suggestions are thin, when they are. */
+ readonly notes: readonly FileNote[];
}
-/**
- * Each file under the owned paths whose state differs between its versions at
- * the seed and end checkpoints, with suggested checks for the files the user
- * worked on: documents, or the app's own state when no document changed.
- */
-export const draftTaskFiles = (
+/** Each file `include` takes whose state differs between its versions at the seed and end checkpoints, by home path. */
+export const changedStateFiles = (
files: ReadonlyMap,
- served: (home: string) => boolean,
ends: StateEnds,
-): readonly TaskDraftFile[] => {
- const changed = [...files]
- .filter(([home]) => served(home))
+ include: (home: string) => boolean = () => true,
+) =>
+ [...files]
+ .filter(([home]) => include(home))
.flatMap(([home, path]) => {
const baseline = versionAt(path, ends.seed);
const final = versionAt(path, ends.end);
@@ -427,15 +388,50 @@ export const draftTaskFiles = (
: [{ home, path, baseline, final, change }];
})
.sort((left, right) => (left.home < right.home ? -1 : 1));
+
+/** The format a change's file has at its end, for its notes. */
+const changeShape = (change: TaskDraftFile["change"], final: Version) => ({
+ kind: "kind" in change ? change.kind : "values",
+ complete: "complete" in change ? change.complete : undefined,
+ format: final.format,
+});
+
+/**
+ * Each file under the owned paths whose state differs between its versions at
+ * the seed and end checkpoints, with suggested checks, values, guards and
+ * notes for the files the user worked on: documents, or the app's own state
+ * when no document changed. Values stay within the draft's byte budget, in
+ * file order.
+ */
+export const draftTaskFiles = (
+ files: ReadonlyMap,
+ served: (home: string) => boolean,
+ ends: StateEnds,
+): readonly TaskDraftFile[] => {
+ const changed = changedStateFiles(files, ends, served);
const role = changed.some(({ path }) => path.role === "document")
? "document"
: "application_state";
- return changed.map(({ home, path, baseline, final, change }) => ({
- file: home,
- role: path.role,
- baseline: draftVersion(baseline, path.role === role),
- final: draftVersion(final, path.role === role),
- change,
- suggested: path.role === role ? suggestedChecks(home, baseline, final) : [],
- }));
+ return withinDraftBudget(
+ changed.map(({ home, path, baseline, final, change }) => {
+ const drafted = path.role === role;
+ const draft: FileDraft = drafted
+ ? suggestedChecks(home, baseline, final, role === "document")
+ : { suggested: [], ...NO_VALUES };
+ return {
+ file: home,
+ role: path.role,
+ baseline: draftVersion(baseline, drafted),
+ final: draftVersion(final, drafted),
+ change,
+ suggested: draft.suggested,
+ values: draft.values,
+ values_cut: draft.valuesCut,
+ guards: draft.guards,
+ notes: drafted
+ ? fileNotes(home, changeShape(change, final), draft.facts)
+ : [],
+ };
+ }),
+ );
};
diff --git a/src/domain/desktop/appTaskDraftAdded.test.ts b/src/domain/desktop/appTaskDraftAdded.test.ts
new file mode 100644
index 0000000000..5691f1e331
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftAdded.test.ts
@@ -0,0 +1,898 @@
+import { describe, expect, it } from "vitest";
+
+import type { DesktopTaskCheckInput } from "./appTask.js";
+import { evaluateTaskChecks } from "./appTaskChecks.js";
+import { passing, xml, xmlDraft } from "./appTaskDraft.fixture.js";
+
+const FILE = "Documents/doc.xml";
+
+it("does not include a sibling generated UUID when a record field changes", () => {
+ const before =
+ 'Old text123e4567-e89b-12d3-a456-426614174000';
+ const after = before.replace("Old text", "New text");
+ const later = after.replace(
+ "123e4567-e89b-12d3-a456-426614174000",
+ "123e4567-e89b-12d3-a456-426614174001",
+ );
+ const checks = xmlDraft(before, after).suggested;
+ expect(checks.length).toBeGreaterThan(0);
+ expect(JSON.stringify(checks)).not.toContain(
+ "123e4567-e89b-12d3-a456-426614174000",
+ );
+ expect(passing(FILE, xml(later), checks).every(Boolean)).toBe(true);
+});
+
+it("keeps stable added field text while omitting nested save times and generated IDs", () => {
+ const before =
+ 'Old';
+ const after =
+ 'OldWater filter1793870600123e4567-e89b-12d3-a456-426614174000';
+ const later = after
+ .replace("1793870600", "1793870700")
+ .replace(
+ "123e4567-e89b-12d3-a456-426614174000",
+ "123e4567-e89b-12d3-a456-426614174001",
+ );
+ const checks = xmlDraft(before, after).suggested;
+ expect(checks.length).toBeGreaterThan(0);
+ expect(JSON.stringify(checks)).not.toMatch(
+ /1793870600|123e4567-e89b-12d3-a456-426614174000/u,
+ );
+ expect(passing(FILE, xml(later), checks).every(Boolean)).toBe(true);
+});
+
+it("checks clean and partially noisy fields of the same added record", () => {
+ const before = 'Old';
+ const after =
+ 'OldShoppingWater filter1793870600';
+ const checks = xmlDraft(before, after).suggested;
+ expect(checks).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ value: "Shopping" }),
+ expect.objectContaining({ kind: "text_contains", value: "Water filter" }),
+ ]),
+ );
+ expect(
+ passing(FILE, xml(after.replace("1793870600", "1793870700")), checks).every(
+ Boolean,
+ ),
+ ).toBe(true);
+ expect(
+ passing(
+ FILE,
+ xml(after.replace("Water filter", "Wrong item")),
+ checks,
+ ).every(Boolean),
+ ).toBe(false);
+});
+
+it("keeps a nested phone number that resembles an epoch timestamp", () => {
+ const before =
+ '+12125550100';
+ const after =
+ '+12125550100+12025550123';
+ const file = xmlDraft(before, after);
+ expect(file.suggested).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ value: "+12025550123" }),
+ ]),
+ );
+});
+
+it("keeps epoch-shaped phone fields and attributes while treating saved metadata as volatile", () => {
+ const before =
+ '2125550100';
+ const after =
+ '21255501002025550123';
+ const file = xmlDraft(before, after);
+ expect(
+ file.suggested.filter(
+ (check) => "value" in check && check.value === "2025550123",
+ ),
+ ).not.toHaveLength(0);
+ expect(file.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ note: "volatile",
+ pointer: expect.stringContaining("/attributes/saved"),
+ }),
+ ]),
+ );
+});
+
+it("treats a changed epoch-shaped phone field as an edit", () => {
+ const file = xmlDraft(
+ "2125550100",
+ "2025550123",
+ );
+ expect(file.suggested.length).toBeGreaterThan(0);
+ expect(file.values.every(({ note }) => note !== "volatile")).toBe(true);
+});
+
+it("omits oversized added attributes while retaining shorter checks and reporting the omitted pointer", () => {
+ const before = '';
+ const after = ``;
+ const file = xmlDraft(before, after);
+ expect(JSON.stringify(file.suggested)).not.toContain("x".repeat(2001));
+ expect(file.suggested).toEqual(
+ expect.arrayContaining([expect.objectContaining({ value: "edited" })]),
+ );
+ expect(file.notes).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ code: "long_text",
+ pointers: expect.arrayContaining([
+ expect.stringContaining("/attributes/note"),
+ ]),
+ }),
+ ]),
+ );
+});
+
+describe("draftTaskFiles for elements no attribute names", () => {
+ /** A score whose title frame holds untitled texts, as MuseScore 3 saves one. */
+ const score = (title: string, composer: string) =>
+ `10${title}${composer}C`;
+
+ it("checks the whole text of the nearest ancestor its tag names, within three levels", () => {
+ const before = score("Morning Song", "Theo Ashby");
+ const after = score("Evening Lullaby", "Clara Hollis");
+
+ const file = xmlDraft(before, after);
+
+ expect(file.suggested).toEqual([
+ {
+ kind: "text_equals",
+ file: FILE,
+ pointer:
+ "/root/children/[name=Score]/children/[name=Staff]/children/[name=VBox]",
+ value: "10TitleEvening LullabyComposerClara Hollis",
+ },
+ ]);
+ expect(passing(FILE, xml(before), file.suggested)).toEqual([false]);
+ expect(passing(FILE, xml(after), file.suggested)).toEqual([true]);
+ });
+
+ /** An address book of records with no attributes, as Osmo keeps its contacts. */
+ const contacts = (...records: string[]) =>
+ `${records
+ .map(
+ (fields) =>
+ `None${fields}0`,
+ )
+ .join("")}`;
+ const HANNAH =
+ "HannahOkaforhannah.okafor@example.com";
+ const DELPHINE =
+ "DelphineMoreau555-0163delphine.moreau@example.com";
+
+ it("checks each new field of a record added whole, each absent from the list's text at the seed, and never the list's whole text", () => {
+ const before = contacts(HANNAH);
+ const after = contacts(HANNAH, DELPHINE);
+
+ const file = xmlDraft(before, after, {
+ role: "application_state",
+ file: ".local/share/osmo/contacts.xml",
+ });
+
+ const list = "/root/children/[name=contacts_records]";
+ expect(
+ file.suggested.map((check) => [
+ check.kind,
+ "pointer" in check ? check.pointer : undefined,
+ ]),
+ ).toEqual([
+ ["text_contains", list],
+ ["text_contains", list],
+ ["text_contains", list],
+ ["text_contains", list],
+ ]);
+ // Not "None" or "0": the seed's text already holds them.
+ expect(
+ file.suggested.flatMap((check) =>
+ check.kind === "text_contains" ? [check.value] : [],
+ ),
+ ).toEqual([
+ "Delphine",
+ "Moreau",
+ "555-0163",
+ "delphine.moreau@example.com",
+ ]);
+ const file_ = ".local/share/osmo/contacts.xml";
+ expect(passing(file_, xml(before), file.suggested)).toEqual([
+ false,
+ false,
+ false,
+ false,
+ ]);
+ expect(passing(file_, xml(after), file.suggested).every(Boolean)).toBe(
+ true,
+ );
+ });
+
+ it("skips an added record's identifier and stamped date, and notes a field past the text limit", () => {
+ const home = ".local/share/osmo/contacts.xml";
+ const essay = "N".repeat(2001);
+ const book = (records: string) =>
+ `${records}`;
+ const kept =
+ "NoneHannah";
+ const before = book(kept);
+ const after = book(
+ `${kept}124Delphine2026117${essay}`,
+ );
+ const file = xmlDraft(before, after, {
+ role: "application_state",
+ file: home,
+ });
+ const values = file.suggested.flatMap((check) =>
+ check.kind === "text_contains" ? [check.value] : [],
+ );
+ expect(values).toContain("Delphine");
+ expect(values).not.toContain("124");
+ expect(values).not.toContain("2026");
+ expect(values.some((value) => value.includes(essay))).toBe(false);
+ expect(file.notes.map(({ code }) => code)).toContain("long_text");
+ });
+});
+
+describe("draftTaskFiles for a named record that also holds a save time", () => {
+ it("checks the edited field, not the record text that joins the timestamp", () => {
+ const box = (value: string, modified: string) =>
+ `Keep2026-11-01T10:12:00Z${value}${modified}`;
+ const before = box("Old", "2026-11-07T10:12:00Z");
+ const saved = box("New", "2026-11-07T10:12:00Z");
+ const restamped = box("New", "2026-11-08T09:00:00Z");
+ const file = xmlDraft(before, saved);
+ const values = file.suggested.flatMap((check) =>
+ "value" in check && typeof check.value === "string" ? [check.value] : [],
+ );
+ expect(values.some((value) => value.includes("2026-11-07T10:12:00Z"))).toBe(
+ false,
+ );
+ expect(values).toContain("New");
+ expect(passing(FILE, xml(restamped), file.suggested).every(Boolean)).toBe(
+ true,
+ );
+ });
+
+ it("checks the paragraph, which a later save keeps, when the new text sits in a run beside a save time", () => {
+ const box = (inner: string, modified: string) =>
+ `
${inner}
${modified}`;
+ const before = box("Old", "2026-11-07T10:12:00Z");
+ const saved = box("New", "2026-11-07T10:12:00Z");
+ const unwrapped = box("New", "2026-11-08T09:00:00Z");
+ const file = xmlDraft(before, saved);
+
+ expect(
+ file.suggested.flatMap((check) =>
+ "pointer" in check ? [check.pointer] : [],
+ ),
+ ).toEqual([
+ "/root/children/[name=box]/children/[name=body]/children/[name=p]",
+ ]);
+ expect(passing(FILE, xml(unwrapped), file.suggested).every(Boolean)).toBe(
+ true,
+ );
+ });
+});
+
+describe("draftTaskFiles for a record only position names, removed whole", () => {
+ const contacts = (...names: string[]) =>
+ `${names
+ .map(
+ (name) =>
+ `None${name}0`,
+ )
+ .join("")}`;
+
+ it("checks the list's text, the one check that can tell the record is gone", () => {
+ const before = contacts("Hannah", "Delphine");
+ const after = contacts("Hannah");
+
+ const file = xmlDraft(before, after, {
+ role: "application_state",
+ file: ".local/share/osmo/contacts.xml",
+ });
+
+ expect(file.suggested).toMatchObject([
+ {
+ kind: "text_equals",
+ pointer: "/root/children/[name=contacts_records]",
+ },
+ ]);
+ const home = ".local/share/osmo/contacts.xml";
+ expect(
+ [before, after, contacts("Delphine")].map((text) =>
+ passing(home, xml(text), file.suggested),
+ ),
+ ).toEqual([[false], [true], [false]]);
+ });
+});
+
+describe("draftTaskFiles for records whatever their fields are called", () => {
+ it("checks each new field of a record of repeated children only position names, in a list at the root", () => {
+ const records = (...people: [string, string][]) =>
+ `${people
+ .map(
+ ([first, last]) =>
+ `${first}${last}`,
+ )
+ .join("")}`;
+ const before = records(["Hannah", "Okafor"]);
+ const after = records(["Hannah", "Okafor"], ["Delphine", "Moreau"]);
+
+ const file = xmlDraft(before, after);
+
+ expect(file.suggested).toEqual(
+ ["Delphine", "Moreau"].map((value) => ({
+ kind: "text_contains",
+ file: FILE,
+ pointer: "/root",
+ value,
+ })),
+ );
+ expect(
+ [before, after].map((text) => passing(FILE, xml(text), file.suggested)),
+ ).toEqual([
+ [false, false],
+ [true, true],
+ ]);
+ });
+
+ it("checks the name inside a nested field and leaves its save time out", () => {
+ const records = (extra: string) =>
+ `Hannah${extra}`;
+ const added = `Delphine2026-11-07T10:12:00Z`;
+ const before = records("");
+ const after = records(added);
+ const restamped = records(
+ added.replace("2026-11-07T10:12:00Z", "2026-11-08T09:00:00Z"),
+ );
+ const file = xmlDraft(before, after);
+ const values = file.suggested.flatMap((check) =>
+ check.kind === "text_contains" ? [check.value] : [],
+ );
+
+ expect(values).toEqual(["Delphine"]);
+ expect(passing(FILE, xml(restamped), file.suggested).every(Boolean)).toBe(
+ true,
+ );
+ });
+
+ it("checks each name around a nested save time, not the two names joined", () => {
+ const records = (extra: string) =>
+ `Hannah${extra}`;
+ const added = `Delphine2026-11-07T10:12:00ZMoreau`;
+ const file = xmlDraft(records(""), records(added));
+ const values = file.suggested.flatMap((check) =>
+ check.kind === "text_contains" ? [check.value] : [],
+ );
+
+ expect(values).toEqual(["Delphine", "Moreau"]);
+ expect(
+ passing(FILE, xml(records(added)), file.suggested).every(Boolean),
+ ).toBe(true);
+ });
+
+ it("suggests no check a cut view leaves undecided for a record only position names, added before the cut", () => {
+ const book = (...people: string[]) =>
+ `${people
+ .map(
+ (name) =>
+ `${name}Okafor`,
+ )
+ .join(
+ "",
+ )}${"x".repeat(2600)}`;
+ const before = book("Hannah", "Bo");
+ const after = book("Hannah", "Bo", "Delphine");
+ const outcomes = (text: string, checks: readonly DesktopTaskCheckInput[]) =>
+ evaluateTaskChecks(
+ checks.map((check, index) => ({ ...check, id: `c${String(index)}` })),
+ new Map([[FILE, { format: "xml" as const, view: xml(text) }]]),
+ ).map(({ passed, undecided }) => [passed, undecided === true]);
+
+ const file = xmlDraft(before, after);
+
+ expect(xml(after)).toMatchObject({ truncated: true });
+ // The record is reported as only position reaching it, not dropped.
+ expect(file.notes.map(({ code }) => code)).toEqual([
+ "cut_view",
+ "by_position",
+ ]);
+ // Each suggestion decides at both ends: failing at the seed, passing at the end.
+ expect(outcomes(before, file.suggested)).toEqual(
+ file.suggested.map(() => [false, false]),
+ );
+ expect(outcomes(after, file.suggested)).toEqual(
+ file.suggested.map(() => [true, false]),
+ );
+ });
+
+ it("checks a field of a record a selector names, beside its other fields, though the record holds a save date", () => {
+ const stamp = `20261106`;
+ const bookmarks = (...entries: [string, string, string][]) =>
+ `${entries
+ .map(
+ ([id, title, link]) =>
+ `${title}${link}${stamp}`,
+ )
+ .join("")}`;
+ const docs: [string, string, string] = [
+ "1",
+ "Docs",
+ "https://example.com/docs",
+ ];
+ const before = bookmarks(docs);
+ const after = bookmarks(docs, [
+ "2",
+ "Release notes",
+ "https://example.com/notes",
+ ]);
+
+ const file = xmlDraft(before, after);
+
+ const bookmark = "/root/children/[name=bookmark][@id=2]";
+ expect(file.suggested).toEqual([
+ {
+ kind: "text_equals",
+ file: FILE,
+ pointer: `${bookmark}/children/[name=title]`,
+ value: "Release notes",
+ },
+ {
+ kind: "text_equals",
+ file: FILE,
+ pointer: `${bookmark}/children/[name=link]`,
+ value: "https://example.com/notes",
+ },
+ ]);
+ });
+});
+
+describe("draftTaskFiles for an identified record's own text", () => {
+ it("checks the text of repeated unnamed fields", () => {
+ const records = (people: readonly (readonly [string, string, string])[]) =>
+ `${people
+ .map(
+ ([id, first, last]) =>
+ `${first}${last}`,
+ )
+ .join("")}`;
+ const before = records([["one", "Ada", "Lovelace"]]);
+ const after = records([
+ ["one", "Ada", "Lovelace"],
+ ["two", "Delphine", "Moreau"],
+ ]);
+ const file = xmlDraft(before, after);
+ const shown = file.suggested.flatMap((check) =>
+ "value" in check && typeof check.value === "string" ? [check.value] : [],
+ );
+
+ expect(shown).toEqual(["Delphine", "Moreau"]);
+ expect(
+ file.suggested.every(
+ (check) => "pointer" in check && check.pointer.includes("[@id=two]"),
+ ),
+ ).toBe(true);
+ expect(
+ file.suggested.some(
+ (check) => "value" in check && check.value === "record",
+ ),
+ ).toBe(false);
+ expect(passing(FILE, xml(after), file.suggested).every(Boolean)).toBe(true);
+ expect(passing(FILE, xml(before), file.suggested).every(Boolean)).toBe(
+ false,
+ );
+ const stolen = records([
+ ["one", "Delphine", "Moreau"],
+ ["two", "Ada", "Lovelace"],
+ ]);
+ expect(passing(FILE, xml(stolen), file.suggested).every(Boolean)).toBe(
+ false,
+ );
+ });
+
+ it("checks a phone number and a clock time a person entered", () => {
+ const people = (extra: string) =>
+ `Ada${extra}`;
+ const before = people("");
+ const after = people(
+ `Delphine202555016309:30:00`,
+ );
+ const file = xmlDraft(before, after);
+ const shown = file.suggested.flatMap((check) =>
+ "value" in check && typeof check.value === "string" ? [check.value] : [],
+ );
+
+ expect(shown).toContain("2025550163");
+ expect(shown).toContain("09:30:00");
+ expect(passing(FILE, xml(before), file.suggested).some(Boolean)).toBe(
+ false,
+ );
+ });
+});
+
+describe("draftTaskFiles for an added record whose words the seed already has", () => {
+ const list = (people: readonly (readonly [string, string])[]) =>
+ `${people
+ .map(
+ ([first, last]) =>
+ `${first}${last}`,
+ )
+ .join("")}`;
+
+ it("checks the list text when every new field repeats a word the seed has", () => {
+ const before = list([["Ada", "Lin"]]);
+ const after = list([
+ ["Ada", "Lin"],
+ ["Lin", "Ada"],
+ ]);
+ const file = xmlDraft(before, after);
+
+ expect(file.suggested.map(({ kind }) => kind)).toEqual(["text_equals"]);
+ expect(passing(FILE, xml(before), file.suggested)).toEqual([false]);
+ expect(passing(FILE, xml(after), file.suggested)).toEqual([true]);
+ });
+
+ it("keeps a save time out of the list check and still requires the added record", () => {
+ const stamp = "2026-11-07T10:12:00Z";
+ const later = "2026-11-08T09:00:00Z";
+ const dated = (people: readonly (readonly [string, string, string])[]) =>
+ `${people
+ .map(
+ ([first, last, time]) =>
+ `${first}${last}${time}`,
+ )
+ .join("")}`;
+ const before = dated([["Ada", "Lin", stamp]]);
+ const after = dated([
+ ["Ada", "Lin", stamp],
+ ["Lin", "Ada", later],
+ ]);
+ const file = xmlDraft(before, after);
+ const values = file.suggested.flatMap((check) =>
+ "value" in check && typeof check.value === "string" ? [check.value] : [],
+ );
+
+ expect(
+ values.some((value) => value.includes(stamp) || value.includes(later)),
+ ).toBe(false);
+ expect(values).toEqual(["LinAda"]);
+ expect(passing(FILE, xml(after), file.suggested).every(Boolean)).toBe(true);
+ expect(
+ passing(
+ FILE,
+ xml(
+ dated([
+ ["Ada", "Lin", stamp],
+ ["Lin", "Ada", stamp],
+ ]),
+ ),
+ file.suggested,
+ ).every(Boolean),
+ ).toBe(true);
+ expect(
+ passing(FILE, xml(before), file.suggested).some((passed) => !passed),
+ ).toBe(true);
+ expect(
+ passing(FILE, xml(dated([["Lin", "Ada", later]])), file.suggested).every(
+ Boolean,
+ ),
+ ).toBe(true);
+ const duplicate = xmlDraft(
+ before,
+ dated([
+ ["Ada", "Lin", stamp],
+ ["Ada", "Lin", later],
+ ]),
+ );
+ const duplicateValues = duplicate.suggested.flatMap((check) =>
+ "value" in check && typeof check.value === "string" ? [check.value] : [],
+ );
+ expect(duplicateValues).not.toContain("AdaLin");
+ });
+
+ it("does not check a long list whole when the added record only repeats its words", () => {
+ const people = Array.from(
+ { length: 80 },
+ () => ["NorthwindHouse", "QuillPublisher"] as const,
+ );
+ const before = list(people);
+ const after = list([...people, ["QuillPublisher", "NorthwindHouse"]]);
+ const file = xmlDraft(before, after);
+
+ expect(file.suggested.some(({ kind }) => kind === "text_equals")).toBe(
+ false,
+ );
+ expect(
+ file.suggested.some(
+ (check) =>
+ "value" in check &&
+ (check.value === "NorthwindHouse" ||
+ check.value === "QuillPublisher"),
+ ),
+ ).toBe(false);
+ });
+});
+
+describe("draftTaskFiles for a paragraph made of runs", () => {
+ /** A Writer document's body: a paragraph, and any added after it. */
+ const body = (added: string) =>
+ `Plan for Monday${added}`;
+ const meeting = (day: string) =>
+ `Meet ${day}`;
+
+ it("checks a new paragraph of runs of different tags by its text, as text rather than a record of fields", () => {
+ const before = body("");
+ const after = body(meeting("Monday"));
+
+ const file = xmlDraft(before, after);
+
+ // The text the paragraph sits in, whole: no word of it by itself.
+ expect(file.suggested).toEqual([
+ {
+ kind: "text_equals",
+ file: FILE,
+ pointer:
+ "/root/children/[name=office:body]/children/[name=office:text]",
+ value: "Plan for MondayMeet Monday",
+ },
+ ]);
+ // The seed's text already holds Monday, so a paragraph naming another day must fail a check.
+ expect(
+ passing(FILE, xml(body(meeting("Tuesday"))), file.suggested),
+ ).toContain(false);
+ expect(passing(FILE, xml(after), file.suggested).every(Boolean)).toBe(true);
+ });
+});
+
+describe("draftTaskFiles for an element added to a list", () => {
+ const ledger = (operations: string) =>
+ `${operations}`;
+ const RENT =
+ '';
+
+ it("checks each attribute of the new member, best ranked first, leaving out its selector, identifiers, times and volatile values", () => {
+ const groceries =
+ '';
+
+ const file = xmlDraft(ledger(RENT), ledger(RENT + groceries));
+
+ const check = (
+ key: string,
+ value: string | number,
+ tolerance?: number,
+ ) => ({
+ kind: "value_equals",
+ file: FILE,
+ pointer: `/root/children/[name=ope][@name=groceries]/attributes/${key}`,
+ value,
+ ...(tolerance === undefined ? {} : { tolerance }),
+ });
+ expect(file.suggested).toEqual([
+ check("wording", "Weekly groceries"),
+ check("account", "1"),
+ check("category", "44"),
+ check("flags", "0"),
+ check("date", "739924"),
+ // A decimal the app writes as text is checked by its number.
+ check("amount", -64.18, 0.001),
+ ]);
+ });
+
+ it("checks at most six attributes, and an element with no like sibling by its presence", () => {
+ const wide = ` `${key}="value ${key}"`)
+ .join(" ")} wording="Weekly groceries"/>`;
+ const added = xmlDraft(ledger(RENT), ledger(RENT + wide));
+ expect(added.suggested).toHaveLength(6);
+
+ const alone = xmlDraft(ledger(""), ledger(''));
+ expect(alone.suggested).toEqual([
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: "/root/children/[name=budget]/name",
+ value: "budget",
+ },
+ ]);
+ });
+});
+
+describe("draftTaskFiles for decimals", () => {
+ it("checks a decimal the app writes as text by its number within 0.001, unless the seed's is that close too", () => {
+ const shape = (x: string, y: string, size: string) =>
+ ``;
+
+ const file = xmlDraft(
+ shape("146.83", "10.5", "12"),
+ shape("150.25", "10.5000001", "14"),
+ );
+
+ const logo = "/root/children/[name=shape]";
+ expect(file.suggested).toEqual([
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: `${logo}/attributes/x`,
+ value: 150.25,
+ tolerance: 0.001,
+ },
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: `${logo}/attributes/y`,
+ value: "10.5000001",
+ },
+ // An integer stays the string it is.
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: `${logo}/attributes/size`,
+ value: "14",
+ },
+ ]);
+ });
+
+ it("checks a decimal by the element's whole text, so a save that splits the same digits across a run passes", () => {
+ const cell = (inner: string) =>
+ `${inner}1`;
+ const before = cell("405.1");
+ const after = cell("437.8");
+ const file = xmlDraft(before, after);
+
+ const passes = (inner: string) =>
+ passing(FILE, xml(cell(inner)), file.suggested);
+ expect(file.suggested).toMatchObject([
+ { kind: "value_equals", value: 437.8, tolerance: 0.001 },
+ ]);
+ for (const same of [
+ "437.8",
+ "437.80",
+ "437.80000000000001",
+ "437.8",
+ ])
+ expect(passes(same)).toEqual([true]);
+ expect(passes("437.89")).toEqual([false]);
+ expect(passes("four hundred")).toEqual([false]);
+ });
+
+ it("checks a decimal so it fails at a seed whose integer is within the tolerance of it", () => {
+ const sheet = (rate: string) =>
+ `${rate}5`;
+ const before = sheet("0");
+ const after = sheet("0.0005");
+
+ const file = xmlDraft(before, after);
+
+ expect(file.suggested).toHaveLength(1);
+ expect(
+ [before, after].map((text) => passing(FILE, xml(text), file.suggested)),
+ ).toEqual([[false], [true]]);
+ });
+});
+
+describe("draftTaskFiles for records in a list, as Tellico keeps books", () => {
+ /** A book collection: field definitions, then entries by id, each stamped with the day it was made and changed, in parts. */
+ const books = (...entries: string[]) =>
+ `${entries.join("")}`;
+ const stamp = (tag: string, day: string) =>
+ `<${tag} calendar="gregorian">202611${day}`;
+ const book = (
+ id: string,
+ [title, author, publisher, year]: readonly string[],
+ [made, changed]: readonly [string, string] = ["06", "06"],
+ ) =>
+ `${title ?? ""}${author ?? ""}${publisher ?? ""}${year ?? ""}${id}${stamp("cdate", made)}${stamp("mdate", changed)}`;
+ const LANTERN = ["The Lantern Keeper", "Mira Okonkwo", "Quill Books", "2019"];
+ const SALT = ["Salt Roads", "Tomas Lindqvist", "Northwind House", "2015"];
+ const ORCHARD = [
+ "The Glass Orchard",
+ "Aurelio Benn",
+ "Northwind House",
+ "2022",
+ ];
+ const entry = (id: string) =>
+ `/root/children/[name=collection]/children/[name=entry][@id=${id}]`;
+
+ it("checks each field of a record appended to the list by the id it was given, never the list's whole text or the dates the app stamps", () => {
+ const before = books(book("1", LANTERN), book("2", SALT));
+ const after = books(
+ book("1", LANTERN),
+ book("2", SALT),
+ book("3", ORCHARD),
+ );
+
+ const file = xmlDraft(before, after);
+
+ expect(file.values.map(({ pointer }) => pointer)).toEqual([entry("3")]);
+ expect(file.suggested).toEqual(
+ ["title", "authors", "publisher", "pub_year"].map((field, index) => ({
+ kind: "text_equals",
+ file: FILE,
+ pointer: `${entry("3")}/children/[name=${field}]`,
+ value: ORCHARD[index],
+ })),
+ );
+ // The same edit saved on another day passes too.
+ const later = books(
+ book("1", LANTERN),
+ book("2", SALT),
+ book("3", ORCHARD, ["09", "09"]),
+ );
+ expect(
+ [before, after, later].map((text) =>
+ passing(FILE, xml(text), file.suggested),
+ ),
+ ).toEqual([
+ [false, false, false, false],
+ [true, true, true, true],
+ [true, true, true, true],
+ ]);
+ });
+
+ it("checks a date a user enters in parts, as a birthday, as any other edit, since no tag names it bookkeeping", () => {
+ const people = (day: string) =>
+ `Ada199005${day}Bo19850102`;
+ const before = people("12");
+ const after = people("14");
+
+ const file = xmlDraft(before, after);
+
+ expect(file.values).toEqual([
+ {
+ pointer:
+ "/root/children/[name=person][@id=1]/children/[name=birthday]/children/[name=day]/children/0",
+ kind: "changed",
+ before: "12",
+ after: "14",
+ },
+ ]);
+ expect(file.suggested).not.toEqual([]);
+ expect(
+ [before, after].map((text) => passing(FILE, xml(text), file.suggested)),
+ ).toEqual([
+ file.suggested.map(() => false),
+ file.suggested.map(() => true),
+ ]);
+ expect(file.notes).toEqual([]);
+ });
+
+ it("notes a date stamped in parts as a timestamp, never a check, and checks an edit beside it by the field it changed", () => {
+ const before = books(book("1", LANTERN), book("2", SALT));
+ const renamed = ["Salt Road", ...SALT.slice(1)];
+
+ const edited = xmlDraft(
+ before,
+ books(book("1", LANTERN), book("2", renamed, ["06", "07"])),
+ );
+ const restamped = xmlDraft(
+ before,
+ books(book("1", LANTERN), book("2", SALT, ["06", "07"])),
+ );
+
+ expect(edited.suggested).toEqual([
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: `${entry("2")}/children/[name=title]/children/0`,
+ value: "Salt Road",
+ },
+ ]);
+ expect(
+ edited.values.map(({ pointer, note }) => [pointer, note ?? null]),
+ ).toEqual([
+ [`${entry("2")}/children/[name=title]/children/0`, null],
+ [
+ `${entry("2")}/children/[name=mdate]/children/[name=day]/children/0`,
+ "timestamp",
+ ],
+ ]);
+ expect(restamped.suggested).toEqual([]);
+ expect(restamped.notes.map(({ code }) => code)).toEqual([
+ "view_state_only",
+ ]);
+ });
+});
diff --git a/src/domain/desktop/appTaskDraftAdded.ts b/src/domain/desktop/appTaskDraftAdded.ts
new file mode 100644
index 0000000000..a8ba1674fa
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftAdded.ts
@@ -0,0 +1,385 @@
+import type { JsonValue } from "../jsonValue.js";
+import type { DesktopTaskCheckInput } from "./appTask.js";
+import { elementText, isXmlElement } from "./appTaskChecks.js";
+import {
+ isRenumberedKey,
+ isBookkeeping,
+ isBookkeepingName,
+ inSplitDate,
+ isStamp,
+ isVolatileValue,
+} from "./appTaskDraftNoise.js";
+import { valueEqualsCheck } from "./appTaskDraftScalars.js";
+import {
+ holdsFields,
+ holdsNoisyText,
+ wholeTextCheck,
+} from "./appTaskDraftText.js";
+import {
+ isIdentifyingAttribute,
+ isViewStateName,
+ rankAttributes,
+} from "./attributeRank.js";
+import { escapeToken, pairMembers, resolvePointer } from "./viewPointer.js";
+
+type Suggestion = DesktopTaskCheckInput;
+
+/** Attributes of an added element checked one by one, and new fields of an added record. */
+const MAX_ATTRIBUTE_CHECKS = 6;
+const MAX_FIELD_CHECKS = 5;
+/** A field that repeats its record's identifier, as Tellico's ``, which the app numbers. */
+const ID_FIELD = /^(?:[^:]*:)?x?id$/iu;
+/** The fewest attributes a sibling must share for an added element to count as its copy: one alone may match by chance. */
+const MIN_COPIED = 2;
+/** The shortest field text checked: shorter text is too likely to be found elsewhere. */
+const MIN_FIELD_CHARS = 3;
+
+/** What checks of an element added whole read: the file, its two ends, whether a check is decided by name there, and the longest text a check compares. */
+export interface AddedContext {
+ readonly file: string;
+ readonly ends: {
+ readonly baseline: JsonValue | null;
+ readonly final: JsonValue | null;
+ };
+ readonly named: (view: JsonValue | null, pointer: string) => boolean;
+ readonly chars: number;
+ /** Fields left unchecked because their text is past `chars`. */
+ readonly long?: string[];
+}
+
+type Element = Extract> & {
+ readonly name: string;
+ readonly children: JsonValue[];
+};
+
+const isRecord = (
+ value: JsonValue | undefined,
+): value is Record =>
+ typeof value === "object" && value !== null && !Array.isArray(value);
+
+const attributesOf = (element: Element) =>
+ isRecord(element.attributes) ? element.attributes : {};
+
+const listOf = (pointer: string) => pointer.replace(/\/[^/]+$/u, "");
+const parentElementOf = (pointer: string): string | undefined =>
+ /^(?.+)\/children\/[^/]+$/u.exec(pointer)?.groups?.parent;
+
+/** How many elements with a tag the list a member sits in holds, in a view. */
+const likeCount = (view: JsonValue | null, pointer: string, tag: string) => {
+ const list = resolvePointer(view, listOf(pointer));
+ return Array.isArray(list)
+ ? list.filter((item) => isXmlElement(item) && item.name === tag).length
+ : 0;
+};
+
+/** Whether a like sibling at either end holds every one of the attributes as they are, so they are the app's defaults for a new member rather than what the user entered. */
+const copiesSibling = (
+ { ends }: AddedContext,
+ pointer: string,
+ element: Element,
+ attributes: readonly { readonly key: string; readonly value: string }[],
+) =>
+ attributes.length >= MIN_COPIED &&
+ [ends.baseline, ends.final].some((view) => {
+ const list = resolvePointer(view, listOf(pointer));
+ return (
+ Array.isArray(list) &&
+ list.some(
+ (item) =>
+ item !== element &&
+ isXmlElement(item) &&
+ item.name === element.name &&
+ attributes.every(
+ ({ key, value }) => attributesOf(item)[key] === value,
+ ),
+ )
+ );
+ });
+
+/** The attribute keys the last segment of a pointer selects by. */
+const selectorKeys = (pointer: string): ReadonlySet => {
+ const segment = (pointer.split("/").at(-1) ?? "")
+ .replaceAll("~1", "/")
+ .replaceAll("~0", "~");
+ return new Set(
+ [...segment.matchAll(/\[@(?[^=\]]+)=/gu)].map(
+ (match) => match.groups?.key ?? "",
+ ),
+ );
+};
+
+/**
+ * A `value_equals` per attribute of an element added whole to a list, as
+ * HomeBank adds an ``, ranked by `attributeTier`, at most six: what the
+ * user entered. Its selector's own attributes, identifiers, times, volatile
+ * values and view state are left out. Empty for an element with no like
+ * sibling at either end, or whose other attributes a like sibling holds
+ * every one of, as a new Planner task copies its neighbour's defaults; such
+ * an element keeps the check that it is there.
+ */
+export const attributeChecks = (
+ context: AddedContext,
+ pointer: string,
+ element: Element,
+): Suggestion[] => {
+ const { ends, file } = context;
+ if (
+ likeCount(ends.final, pointer, element.name) < 2 &&
+ likeCount(ends.baseline, pointer, element.name) === 0
+ )
+ return [];
+ const selected = selectorKeys(pointer);
+ const own = Object.entries(
+ isRecord(element.attributes) ? element.attributes : {},
+ ).flatMap(([key, value]) =>
+ typeof value === "string" &&
+ !selected.has(key) &&
+ !isIdentifyingAttribute(key)
+ ? [{ key, value }]
+ : [],
+ );
+ if (copiesSibling(context, pointer, element, own)) return [];
+ const eligible = own.filter(
+ ({ key, value }) =>
+ !isStamp(value) &&
+ !isVolatileValue(
+ value,
+ isBookkeeping(element) || isBookkeepingName(key),
+ ) &&
+ !isViewStateName(key) &&
+ !isRenumberedKey(key),
+ );
+ for (const { key, value } of eligible)
+ if (value.length > context.chars)
+ context.long?.push(`${pointer}/attributes/${escapeToken(key)}`);
+ return rankAttributes(
+ eligible.filter(({ value }) => value.length <= context.chars),
+ )
+ .slice(0, MAX_ATTRIBUTE_CHECKS)
+ .map(({ key, value }) =>
+ valueEqualsCheck(
+ file,
+ `${pointer}/attributes/${escapeToken(key)}`,
+ value,
+ ),
+ );
+};
+
+/**
+ * A field's text with save-time leaves removed, split where a timestamp sat
+ * between them. `Delphine` and `Moreau` around a save time stay two checks;
+ * joining them would look for text the file never contains.
+ */
+const keptFieldPieces = (node: JsonValue, bookkeeping = false): string[] => {
+ if (typeof node === "string")
+ return isStamp(node) || isVolatileValue(node.trim(), bookkeeping)
+ ? []
+ : [node];
+ if (!isXmlElement(node)) return [];
+ const inBookkeeping = bookkeeping || isBookkeeping(node);
+ const pieces: string[] = [];
+ let run = "";
+ const flush = () => {
+ if (run !== "") pieces.push(run);
+ run = "";
+ };
+ for (const child of node.children) {
+ if (typeof child === "string") {
+ if (isStamp(child) || isVolatileValue(child.trim(), inBookkeeping))
+ flush();
+ else run += child;
+ continue;
+ }
+ if (!isXmlElement(child)) continue;
+ if (isStamp(elementText(child))) {
+ flush();
+ continue;
+ }
+ const nested = keptFieldPieces(child, inBookkeeping);
+ if (nested.length !== 1) {
+ flush();
+ pieces.push(...nested);
+ continue;
+ }
+ run += nested[0] ?? "";
+ }
+ flush();
+ return pieces;
+};
+
+/** An element's text with save-time leaves removed, in document order. */
+const plainText = (node: JsonValue | null): string => {
+ if (typeof node === "string") return isStamp(node) ? "" : node;
+ if (!isXmlElement(node)) return "";
+ if (isStamp(elementText(node))) return "";
+ return node.children.map((child) => plainText(child)).join("");
+};
+
+/** The nearest ancestor both ends hold that a check names without a step by position. */
+const keptNamedAncestor = (context: AddedContext, pointer: string) => {
+ const { ends } = context;
+ for (
+ let at = parentElementOf(pointer);
+ at !== undefined;
+ at = parentElementOf(at)
+ )
+ if (
+ context.named(ends.final, at) &&
+ isXmlElement(resolvePointer(ends.baseline, at)) &&
+ isXmlElement(resolvePointer(ends.final, at))
+ )
+ return at;
+ return undefined;
+};
+
+/**
+ * A `text_contains` per new field of a record added whole that only position
+ * names, as Osmo adds a `` of `` and `` among
+ * others, at the nearest ancestor both ends hold that a check can name: up to
+ * five fields of three or more characters, each absent from that ancestor's
+ * text at the seed, so each fails there. A record a selector names is checked
+ * itself, so the words have to be in that record. The field's own path is
+ * kept, so an identifier, a date stamped in parts and text past the limit are
+ * left out, as `recordChecks` leaves them out; a field past the limit is
+ * named in `context.long`. Empty for an element that `holdsFields` says is
+ * no record, as a paragraph of runs, or that has no like sibling.
+ */
+export const fieldChecks = (
+ context: AddedContext,
+ pointer: string,
+ element: Element,
+): Suggestion[] => {
+ const { ends, file } = context;
+ if (!holdsFields(element) || likeCount(ends.final, pointer, element.name) < 2)
+ return [];
+ const at = keptNamedAncestor(context, pointer);
+ if (at === undefined) return [];
+ // The record's own selector. Words already in another record still have to be here.
+ const onRecord = context.named(ends.final, pointer);
+ const seen = onRecord
+ ? ""
+ : elementText(resolvePointer(ends.baseline, at) ?? null);
+ const { keys } = pairMembers(element.children, []).newer;
+ const texts: string[] = [];
+ let alreadySeen = false;
+ element.children.forEach((field, index) => {
+ if (!isXmlElement(field)) return;
+ const key = keys[index] ?? String(index);
+ const fieldAt = `${pointer}/children/${key.startsWith("[") ? escapeToken(key) : key}`;
+ if (ID_FIELD.test(field.name) || inSplitDate(ends.final, fieldAt)) return;
+ for (const value of keptFieldPieces(field)) {
+ const text = value.trim();
+ if (
+ text.length < MIN_FIELD_CHARS ||
+ !/[\p{L}\d]/u.test(text) ||
+ isStamp(text) ||
+ isVolatileValue(text, isBookkeeping(field))
+ )
+ continue;
+ if (seen.includes(text)) {
+ alreadySeen = true;
+ continue;
+ }
+ if (value.length > context.chars) {
+ context.long?.push(fieldAt);
+ continue;
+ }
+ texts.push(text);
+ }
+ });
+ if (texts.length > 0)
+ return [...new Set(texts)].slice(0, MAX_FIELD_CHECKS).map((value) => ({
+ kind: "text_contains" as const,
+ file,
+ pointer: onRecord ? pointer : at,
+ value,
+ }));
+ if (onRecord) return [];
+ // Every new field repeats text the seed already has. The list's text shows the record was added, when it fits and holds no save time. A save time stays out of the check: the new record's own text has to be there, and the seed's text has to remain.
+ const holder = resolvePointer(ends.final, at);
+ const before = resolvePointer(ends.baseline, at);
+ const now = elementText(holder ?? null);
+ const plain = plainText(holder ?? null);
+ const seedPlain = plainText(before ?? null);
+ if (
+ !alreadySeen ||
+ plain === seedPlain ||
+ plain.length === 0 ||
+ plain.length > context.chars
+ )
+ return [];
+ if (plain === now)
+ return [{ kind: "text_equals", file, pointer: at, value: now }];
+ const addedPlain = plainText(element);
+ const seed = elementText(before ?? null);
+ // The seed's own text already passes, so it is not a check. Only text the new record adds, which the seed does not hold, shows that the record is there.
+ return addedPlain.length >= MIN_FIELD_CHARS &&
+ now.includes(addedPlain) &&
+ !seed.includes(addedPlain)
+ ? [{ kind: "text_contains", file, pointer: at, value: addedPlain }]
+ : [];
+};
+
+/**
+ * A check per field of a record added whole to a list that a selector names,
+ * as Tellico adds an `` of ``, `<authors>`, `<publisher>`
+ * and `<pub_year>`: each child element a selector names among the record's
+ * children, by its whole text as `wholeTextCheck` checks one, at most five;
+ * leaving out a field repeating the record's identifier, a date the app
+ * stamps in parts, and text that is blank, a time, a volatile value or past
+ * the limit. Empty for an element that `holdsFields` says is no record, or
+ * that has no like sibling.
+ */
+export const recordChecks = (
+ context: AddedContext,
+ pointer: string,
+ element: Element,
+): Suggestion[] => {
+ const { ends, file } = context;
+ if (
+ !holdsFields(element) ||
+ (likeCount(ends.final, pointer, element.name) < 2 &&
+ likeCount(ends.baseline, pointer, element.name) === 0)
+ )
+ return [];
+ const { keys } = pairMembers(element.children, []).newer;
+ return element.children
+ .flatMap<Suggestion>((field, index) => {
+ const key = keys[index] ?? "";
+ const at = `${pointer}/children/${escapeToken(key)}`;
+ if (
+ !isXmlElement(field) ||
+ !key.startsWith("[") ||
+ ID_FIELD.test(field.name) ||
+ inSplitDate(ends.final, at) ||
+ !context.named(ends.final, at)
+ )
+ return [];
+ const value = elementText(field);
+ if (holdsNoisyText(field))
+ return keptFieldPieces(field).flatMap((piece) => {
+ const text = piece.trim();
+ return text.length < MIN_FIELD_CHARS ||
+ text.length > context.chars ||
+ !/[\p{L}\d]/u.test(text) ||
+ isStamp(text) ||
+ isVolatileValue(text, isBookkeeping(field))
+ ? []
+ : [
+ {
+ kind: "text_contains" as const,
+ file,
+ pointer: at,
+ value: text,
+ },
+ ];
+ });
+ return value.trim() === "" ||
+ value.length > context.chars ||
+ isStamp(value) ||
+ isVolatileValue(value, isBookkeeping(field))
+ ? []
+ : [wholeTextCheck(file, at, value, undefined)];
+ })
+ .slice(0, MAX_FIELD_CHECKS);
+};
diff --git a/src/domain/desktop/appTaskDraftFileNotes.test.ts b/src/domain/desktop/appTaskDraftFileNotes.test.ts
new file mode 100644
index 0000000000..6cc08ddeda
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftFileNotes.test.ts
@@ -0,0 +1,58 @@
+import { describe, expect, it } from "vitest";
+
+import type { JsonValue } from "../jsonValue.js";
+import { draftOf, statePath, xml, xmlDraft } from "./appTaskDraft.fixture.js";
+
+describe("draftTaskFiles notes on thin files", () => {
+ it("says which elements only position names when no check can name a change", () => {
+ const table = (last: string) =>
+ `<doc><row><cell>1</cell><cell>2</cell></row><row><cell>3</cell><cell>${last}</cell></row></doc>`;
+
+ const file = xmlDraft(table("4"), table("5"));
+
+ expect(file.suggested).toEqual([]);
+ expect(file.notes).toEqual([
+ expect.objectContaining({
+ code: "by_position",
+ file: "Documents/doc.xml",
+ elements: ["row", "cell"],
+ }),
+ ]);
+ });
+
+ it("says which elements' text changed past what a text check compares", () => {
+ const essay = (ending: string) =>
+ `<doc><section><p>${"Lorem ipsum ".repeat(200)}${ending}</p></section></doc>`;
+
+ const file = xmlDraft(essay("end."), essay("finale."));
+
+ expect(
+ file.suggested.filter(({ kind }) => kind.startsWith("text_")),
+ ).toEqual([]);
+ expect(file.notes).toEqual([
+ expect.objectContaining({
+ code: "long_text",
+ pointers: ["/root/children/[name=section]/children/[name=p]"],
+ }),
+ ]);
+ });
+
+ it("says when a view holds only part of the file, or none of it a check can read", () => {
+ const cut = (text: string): JsonValue => ({
+ ...(xml(`<doc><p>${text}</p></doc>`) as Record<string, JsonValue>),
+ truncated: true,
+ });
+
+ const drafted = draftOf({
+ "Documents/a.gnumeric": statePath("document", "binary", {}, {}),
+ "Documents/b.xml": statePath("document", "xml", cut("x"), cut("y")),
+ });
+
+ expect(
+ drafted.map(({ file, notes }) => [file, notes.map(({ code }) => code)]),
+ ).toEqual([
+ ["Documents/a.gnumeric", ["unreadable_view"]],
+ ["Documents/b.xml", ["cut_view"]],
+ ]);
+ });
+});
diff --git a/src/domain/desktop/appTaskDraftFileNotes.ts b/src/domain/desktop/appTaskDraftFileNotes.ts
new file mode 100644
index 0000000000..be0f3885d5
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftFileNotes.ts
@@ -0,0 +1,104 @@
+import type { NamedFacts } from "./appTaskDraftNamed.js";
+
+/** Why a file in a draft has few or no suggested checks, one code per reason. */
+export const FILE_NOTE_CODES = [
+ "by_position",
+ "view_state_only",
+ "long_text",
+ "cut_view",
+ "unreadable_view",
+] as const;
+
+/** One reason a drafted file's suggestions are thin, with the facts it rests on. */
+export interface FileNote {
+ readonly code: (typeof FILE_NOTE_CODES)[number];
+ readonly file: string;
+ readonly message: string;
+ /** `by_position`: the tags of elements no attribute names alone, up to five. */
+ readonly elements?: readonly string[];
+ /** `long_text`: the elements whose text is too long; `view_state_only`: the noise checks kept. Up to five. */
+ readonly pointers?: readonly string[];
+}
+
+const MAX_LISTED = 5;
+
+/** What a file's change says about whether its views can show it. */
+export interface ChangeShape {
+ readonly kind: string;
+ readonly complete: boolean | undefined;
+ readonly format: string | null;
+}
+
+/** Notes on what a file's views could not show: one cut short, or one no check can read. */
+const viewNotes = (file: string, change: ChangeShape): FileNote[] => {
+ const home = `~/${file}`;
+ if (
+ change.kind === "unreadable" ||
+ change.kind === "format" ||
+ (change.kind === "content" && change.format === "binary")
+ )
+ return [
+ {
+ code: "unreadable_view",
+ file,
+ message:
+ change.format === "binary"
+ ? `REA reads ${home} as binary, so no check can look inside it: save it in a format the app offers that REA reads, such as XML, JSON or text, plain or gzip-compressed, or a zip document, and capture again.`
+ : `REA could not read ${home} at one end the same way as at the other, so no check can look inside it; its versions' view_error says why.`,
+ },
+ ];
+ return change.kind === "content" || change.complete === false
+ ? [
+ {
+ code: "cut_view",
+ file,
+ message: `REA's view of ${home} holds only part of it, so the change may lie outside what a check can see; check what the views do show, or keep the document smaller.`,
+ },
+ ]
+ : [];
+};
+
+/** Notes on why a file's suggested checks are thin, from the facts its draft gathered. */
+const factNotes = (file: string, facts: NamedFacts): FileNote[] => {
+ const home = `~/${file}`;
+ const notes: FileNote[] = [];
+ if (facts.positional > 0 || facts.changes === 0) {
+ const elements = facts.positionalTags.slice(0, MAX_LISTED);
+ notes.push({
+ code: "by_position",
+ file,
+ message:
+ facts.changes === 0
+ ? `${home} changed only in where its elements sit, which no check names; check a value an attribute names instead.`
+ : `${String(facts.positional)} change(s) in ${home} reach their element only by position, so no check names them${elements.length === 0 ? "" : `: ${elements.join(", ")} has no attribute naming it alone among its siblings`}. Their values carry note by_position; check an ancestor's text or a value an attribute names, or give the element a name in the app.`,
+ ...(elements.length === 0 ? {} : { elements }),
+ });
+ }
+ if (facts.changes > 0 && facts.noise === facts.changes)
+ notes.push({
+ code: "view_state_only",
+ file,
+ message: `Every change in ${home} is view state, a renumbered id, a volatile value or a time the app stamps, not an edit: ${facts.flagged.length === 0 ? "no check is suggested" : "the suggested checks are those, kept so the draft is not empty"}. Save the edit the task is about, then capture again.`,
+ ...(facts.flagged.length === 0
+ ? {}
+ : { pointers: facts.flagged.slice(0, MAX_LISTED) }),
+ });
+ if (facts.long.length > 0)
+ notes.push({
+ code: "long_text",
+ file,
+ message: `${String(facts.long.length)} value(s) in ${home} exceed the 2,000-character check limit, so whole-value checks were omitted; use a smaller check where supported.`,
+ pointers: facts.long.slice(0, MAX_LISTED),
+ });
+ return notes;
+};
+
+/** Why a drafted file's suggestions are thin: its views could not show the change, or its draft's facts. */
+export const fileNotes = (
+ file: string,
+ change: ChangeShape,
+ facts: NamedFacts | undefined,
+): FileNote[] => [
+ ...viewNotes(file, change),
+ ...(facts === undefined ? [] : factNotes(file, facts)),
+];
diff --git a/src/domain/desktop/appTaskDraftGuards.test.ts b/src/domain/desktop/appTaskDraftGuards.test.ts
new file mode 100644
index 0000000000..bafeb3905a
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftGuards.test.ts
@@ -0,0 +1,290 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+ draftOf,
+ passing,
+ pointerOf,
+ statePath,
+ xml,
+ xmlDraft,
+} from "./appTaskDraft.fixture.js";
+import { draftGuards } from "./appTaskDraftGuards.js";
+import { lineGuards } from "./appTaskDraftLineGuards.js";
+
+const FILE = "Documents/doc.xml";
+
+it("does not guard whole sibling text that contains an unchanged timestamp", () => {
+ const document = (title: string) =>
+ `<doc><item id="a"><title>${title}Keep2026-11-01T10:12:00ZStable`;
+ const baseline = xml(document("Old"));
+ const final = xml(document("New"));
+ const guards = draftGuards(
+ {
+ file: FILE,
+ ends: { baseline, final },
+ named: () => true,
+ noisy: () => false,
+ chars: 2000,
+ },
+ [
+ {
+ kind: "text_equals",
+ file: FILE,
+ pointer: "/root/children/[name=item][@id=a]",
+ value: "New",
+ },
+ ],
+ );
+ expect(guards).toEqual([
+ expect.objectContaining({
+ pointer: "/root/children/[name=item][@id=c]",
+ value: "Stable",
+ }),
+ ]);
+});
+
+it("does not guard whole sibling text that contains an unchanged generated UUID", () => {
+ const document = (title: string) =>
+ `${title}Keep123e4567-e89b-12d3-a456-426614174000Stable`;
+ const baseline = xml(document("Old"));
+ const final = xml(document("New"));
+ const guards = draftGuards(
+ {
+ file: FILE,
+ ends: { baseline, final },
+ named: () => true,
+ noisy: () => false,
+ chars: 2000,
+ },
+ [
+ {
+ kind: "text_equals",
+ file: FILE,
+ pointer: "/root/children/[name=item][@id=a]",
+ value: "New",
+ },
+ ],
+ );
+ expect(guards).toEqual([
+ expect.objectContaining({
+ pointer: "/root/children/[name=item][@id=c]",
+ value: "Stable",
+ }),
+ ]);
+});
+
+describe("draftTaskFiles guards", () => {
+ /** A flyer's frames, each named, its text in an attribute, as Scribus saves them. */
+ const flyer = (headline: string) =>
+ `${[
+ ["10.5", "Headline", headline],
+ ["10.5", "Details", "Saturday at 7 PM"],
+ ["20", "Footer", "Free entry"],
+ ]
+ .map(
+ ([x, name, text]) =>
+ ``,
+ )
+ .join("")}`;
+ const frame = (name: string) =>
+ `/root/children/[name=DOCUMENT]/children/[name=PAGEOBJECT][@ANNAME=${name}]/children/[name=StoryText]/children/[name=ITEXT]/attributes/CH`;
+
+ it("offers the field a check reads on the nearest like siblings, at its seed value, outside the suggestions", () => {
+ const before = flyer("Riverside Jazz Night");
+ const after = flyer("Harbor Lights Jazz Festival");
+
+ const file = xmlDraft(before, after);
+
+ expect(file.suggested.map(pointerOf)).toEqual([frame("Headline")]);
+ // The logo has no text, so it guards nothing.
+ expect(file.guards).toEqual([
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: frame("Details"),
+ value: "Saturday at 7 PM",
+ },
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: frame("Footer"),
+ value: "Free entry",
+ },
+ ]);
+ expect(passing(FILE, xml(before), file.guards)).toEqual([true, true]);
+ expect(passing(FILE, xml(after), file.guards)).toEqual([true, true]);
+ });
+
+ it("guards no sibling whose field changed too, and a number by the cell's whole text", () => {
+ const sheet = (march: string, april: string) =>
+ `398.2${march}${april}174.9`;
+ // Each cell by the fewest attributes naming it alone.
+ const cell = (selector: string) =>
+ `/root/children/[name=Cells]/children/[name=Cell]${selector}`;
+
+ const file = xmlDraft(sheet("405.1", "421.75"), sheet("437.8", "430.1"));
+
+ expect(file.suggested.map(pointerOf)).toEqual([
+ cell("[@Col=3][@Row=2]"),
+ cell("[@Col=4]"),
+ ]);
+ expect(file.guards).toEqual([
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: cell("[@Col=2]"),
+ value: 398.2,
+ tolerance: 0.001,
+ },
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: cell("[@Row=3]"),
+ value: 174.9,
+ tolerance: 0.001,
+ },
+ ]);
+ });
+});
+
+describe("draftTaskFiles guards beyond a sibling's same field", () => {
+ it("guards a field a user sets on the siblings, not a default they all share", () => {
+ const plan = (designType: string) =>
+ ``;
+ const task = (name: string) =>
+ `/root/children/[name=tasks]/children/[name=task][@name=${name}]/attributes`;
+
+ const file = xmlDraft(plan("normal"), plan("milestone"));
+
+ expect(file.suggested.map(pointerOf)).toEqual([`${task("Design")}/type`]);
+ expect(file.guards).toEqual([
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: `${task("Requirements")}/work`,
+ value: "28800",
+ },
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: `${task("Launch")}/work`,
+ value: "14400",
+ },
+ ]);
+ });
+
+ it("guards no sibling paired by an id the app renumbers", () => {
+ const sheet = (march: string) =>
+ `${march}=sum(C2:C4)=sum(D2:D4)`;
+
+ const file = xmlDraft(sheet("405.1"), sheet("437.8"));
+
+ expect(file.suggested.map(pointerOf)).toEqual([
+ "/root/children/[name=Cell][@Row=2]",
+ ]);
+ expect(file.guards.map(pointerOf)).toEqual([
+ "/root/children/[name=Cell][@Col=4]",
+ ]);
+ });
+
+ it("guards a text file's list items beside the line an edit added", () => {
+ const page = (...items: string[]) => {
+ const lines = [
+ "====== Meeting Notes ======",
+ "Action items",
+ ...items,
+ "",
+ "Next sync: Thursday at 10:00",
+ ];
+ return { lines, line_count: lines.length, truncated: false };
+ };
+ const PRIYA = "[ ] Priya to send the revised budget";
+ const MARCO = "[ ] Marco to book a room for the offsite";
+ const file = "Notebooks/Notes/Meeting_Notes.txt";
+
+ const [drafted] = draftOf({
+ [file]: statePath(
+ "document",
+ "text",
+ page(PRIYA, MARCO),
+ page(PRIYA, MARCO, "[ ] Dana to update the release checklist"),
+ ),
+ });
+
+ expect(drafted?.guards).toEqual([
+ { kind: "lines_present", file, lines: [PRIYA, MARCO] },
+ ]);
+ });
+});
+
+describe("draftGuards on a long list", () => {
+ /** A ledger of 4,000 operations, each worded as `wording` says. */
+ const ledger = (wording: (index: number) => string) =>
+ xml(
+ `${Array.from(
+ { length: 4000 },
+ (_, index) =>
+ ``,
+ ).join("")}`,
+ );
+ const payment = (index: number) => `Payment ${String(index)}`;
+ /** The guards of a check of op2000's wording, and a spy called once per sibling a guard is sought on. */
+ const guardsOf = (
+ baseline: ReturnType,
+ final: ReturnType,
+ ) => {
+ const noisy = vi.fn(() => false);
+ const guards = draftGuards(
+ {
+ file: FILE,
+ ends: { baseline, final },
+ named: () => true,
+ noisy,
+ chars: 2000,
+ },
+ [
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: "/root/children/[name=ope][@name=op2000]/attributes/wording",
+ value: "October rent",
+ },
+ ],
+ );
+ return { guards, noisy };
+ };
+
+ it("reads siblings only until a check has its two guards", () => {
+ const { guards, noisy } = guardsOf(
+ ledger((index) => (index === 2000 ? "Rent" : payment(index))),
+ ledger((index) => (index === 2000 ? "October rent" : payment(index))),
+ );
+
+ expect(guards.map(pointerOf)).toEqual([
+ "/root/children/[name=ope][@name=op1999]/attributes/wording",
+ "/root/children/[name=ope][@name=op2001]/attributes/wording",
+ ]);
+ expect(noisy).toHaveBeenCalledTimes(2);
+ });
+
+ it("reads at most the 16 nearest siblings when none qualifies, as when every one changed", () => {
+ const { guards, noisy } = guardsOf(
+ ledger(payment),
+ ledger((index) => `${payment(index)} (paid)`),
+ );
+
+ expect(guards).toEqual([]);
+ expect(noisy.mock.calls.length).toBeLessThanOrEqual(16);
+ });
+});
+
+describe("lineGuards text bound", () => {
+ it("leaves a huge unchanged line out of the guard", () => {
+ const huge = "word ".repeat(500);
+ expect(
+ lineGuards("Documents/note.txt", [huge, "milk"], ["eggs", huge, "milk"]),
+ ).toEqual([
+ { kind: "lines_present", file: "Documents/note.txt", lines: ["milk"] },
+ ]);
+ });
+});
diff --git a/src/domain/desktop/appTaskDraftGuards.ts b/src/domain/desktop/appTaskDraftGuards.ts
new file mode 100644
index 0000000000..4b96485ace
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftGuards.ts
@@ -0,0 +1,282 @@
+import type { JsonValue } from "../jsonValue.js";
+import type { DesktopTaskCheckInput } from "./appTask.js";
+import { elementText, isXmlElement, sameJson } from "./appTaskChecks.js";
+import {
+ holdsSplitDate,
+ isRenumberedKey,
+ isStamp,
+} from "./appTaskDraftNoise.js";
+import { holdsNoisyText } from "./appTaskDraftText.js";
+import { decimalOf, valueEqualsCheck } from "./appTaskDraftScalars.js";
+import { isIdentifyingAttribute, rankAttributes } from "./attributeRank.js";
+import { escapeToken, pairMembers, resolvePointer } from "./viewPointer.js";
+
+type Suggestion = DesktopTaskCheckInput;
+
+/** Guards a file's draft offers, and how many one suggested check takes. */
+const MAX_GUARDS = 4;
+const MAX_GUARDS_PER_CHECK = 2;
+/** Like siblings a guard is sought on per list, nearest first, so a long list costs no more than a short one. */
+const MAX_SIBLINGS_READ = 16;
+
+/** What guards read: a file's two ends, whether a check there is decided by name, and noise. */
+export interface GuardContext {
+ readonly file: string;
+ readonly ends: {
+ readonly baseline: JsonValue | null;
+ readonly final: JsonValue | null;
+ };
+ readonly named: (view: JsonValue | null, pointer: string) => boolean;
+ readonly noisy: (pointer: string) => boolean;
+ readonly chars: number;
+}
+
+/** Where a pointer steps into a list by selector, innermost first: the list, the member's segment and the rest of the pointer after it. */
+const listSteps = (pointer: string) => {
+ const segments = pointer.split("/");
+ const steps: { list: string; member: string; rest: string[] }[] = [];
+ for (let index = segments.length - 1; index >= 2; index -= 1) {
+ const member = segments[index] ?? "";
+ if (segments[index - 1] === "children" && !/^\d+$/u.test(member))
+ steps.push({
+ list: segments.slice(0, index).join("/"),
+ member,
+ rest: segments.slice(index + 1),
+ });
+ }
+ return steps;
+};
+
+/** The guard at a sibling's pointer, when it holds the same value at both ends, by name, and is no noise. */
+const guardAt = (
+ context: GuardContext,
+ check: Suggestion,
+ pointer: string,
+): Suggestion | undefined => {
+ const { ends, file } = context;
+ if (
+ context.noisy(pointer) ||
+ !context.named(ends.baseline, pointer) ||
+ !context.named(ends.final, pointer)
+ )
+ return undefined;
+ const before = resolvePointer(ends.baseline, pointer);
+ const after = resolvePointer(ends.final, pointer);
+ if (before === undefined || after === undefined) return undefined;
+ if (
+ check.kind === "value_equals" &&
+ "tolerance" in check &&
+ isXmlElement(before) &&
+ isXmlElement(after)
+ ) {
+ const value = elementText(before);
+ if (
+ value.trim() === "" ||
+ value !== elementText(after) ||
+ value.length > context.chars ||
+ holdsNoisyText(before) ||
+ holdsNoisyText(after) ||
+ holdsSplitDate(ends.baseline, pointer) ||
+ holdsSplitDate(ends.final, pointer) ||
+ isStamp(value)
+ )
+ return undefined;
+ return decimalOf(value) === undefined
+ ? { kind: "text_equals", file, pointer, value }
+ : valueEqualsCheck(file, pointer, value);
+ }
+ if (check.kind === "text_equals") {
+ if (!isXmlElement(before) || !isXmlElement(after)) return undefined;
+ const value = elementText(before);
+ return value.trim() !== "" &&
+ value === elementText(after) &&
+ value.length <= context.chars &&
+ !holdsNoisyText(before) &&
+ !holdsNoisyText(after) &&
+ !holdsSplitDate(ends.baseline, pointer) &&
+ !holdsSplitDate(ends.final, pointer) &&
+ !isStamp(value)
+ ? { kind: "text_equals", file, pointer, value }
+ : undefined;
+ }
+ return typeof before !== "object" &&
+ sameJson(before, after) &&
+ !isStamp(before) &&
+ (typeof before !== "string" || before.length <= context.chars)
+ ? valueEqualsCheck(file, pointer, before)
+ : undefined;
+};
+
+/** Whether a selector key keys a member by a value the app renumbers, as Gnumeric's `[@ExprID=2]`. */
+const renumberedKey = (key: string) =>
+ [...key.matchAll(/\[@(?[^=\]]+)=/gu)].some(({ groups }) =>
+ isRenumberedKey(groups?.attribute ?? ""),
+ );
+
+/** A like sibling of a checked member: its pointer by the selector it pairs by at both ends, and its element at the seed. */
+interface Sibling {
+ readonly member: string;
+ readonly element: Record;
+}
+
+/** The like siblings of the member a step goes through, nearest first, each paired by a selector, not by position or a renumbered id, and every like member at the seed. */
+const siblingsOf = (
+ context: GuardContext,
+ step: ReturnType[number],
+): { readonly siblings: Sibling[]; readonly all: JsonValue[] } => {
+ const newer = resolvePointer(context.ends.final, step.list);
+ const older = resolvePointer(context.ends.baseline, step.list);
+ const member = resolvePointer(
+ context.ends.final,
+ `${step.list}/${step.member}`,
+ );
+ if (!Array.isArray(newer) || !Array.isArray(older) || !isXmlElement(member))
+ return { siblings: [], all: [] };
+ const at = newer.indexOf(member);
+ const likes = newer
+ .map((item, index) => ({ item, index }))
+ .filter(
+ ({ item, index }) =>
+ index !== at && isXmlElement(item) && item.name === member.name,
+ )
+ .toSorted(
+ (left, right) =>
+ Math.abs(left.index - at) - Math.abs(right.index - at) ||
+ left.index - right.index,
+ );
+ const all = older.filter(
+ (item) => isXmlElement(item) && item.name === member.name,
+ );
+ if (likes.length === 0) return { siblings: [], all };
+ const pairing = pairMembers(newer, older);
+ return {
+ siblings: likes.flatMap(({ index }) => {
+ const key = pairing.newer.keys[index];
+ const partner = pairing.newer.partners[index];
+ const element = partner === undefined ? undefined : older[partner];
+ return key === undefined ||
+ /^\d+$/u.test(key) ||
+ renumberedKey(key) ||
+ partner === undefined ||
+ pairing.older.keys[partner] !== key ||
+ !isXmlElement(element)
+ ? []
+ : [{ member: escapeToken(key), element }];
+ }),
+ all,
+ };
+};
+
+const attributesOf = ({ attributes }: Record) =>
+ typeof attributes === "object" &&
+ attributes !== null &&
+ !Array.isArray(attributes)
+ ? attributes
+ : {};
+
+/** Whether like members at the seed hold more than one value at a path into them, as Planner's `work` does and its shared `type="normal"` does not; each path read once. */
+const variesAmong = (all: readonly JsonValue[]) => {
+ const known = new Map();
+ return (rest: readonly string[]): boolean => {
+ const pointer = `/${rest.join("/")}`;
+ const varied =
+ known.get(pointer) ??
+ new Set(
+ all.map((item) =>
+ JSON.stringify(resolvePointer(item, pointer) ?? null),
+ ),
+ ).size > 1;
+ known.set(pointer, varied);
+ return varied;
+ };
+};
+
+/**
+ * The path into a sibling a guard reads: the check's own field where like
+ * members hold different values in it; otherwise, as for a tag or a default
+ * every sibling shares, the sibling's best ranked attribute that varies, one
+ * a user sets, as Planner's `work`; the check's own field when none does.
+ */
+const guardField = (
+ sibling: Sibling,
+ varies: (rest: readonly string[]) => boolean,
+ rest: readonly string[],
+) => {
+ if (rest.length > 0 && varies(rest)) return rest;
+ const selected = new Set(
+ [...sibling.member.matchAll(/\[@(?[^=\]]+)=/gu)].map(
+ ({ groups }) => groups?.key ?? "",
+ ),
+ );
+ const [best] = rankAttributes(
+ Object.entries(attributesOf(sibling.element)).flatMap(([key, value]) =>
+ typeof value === "string" &&
+ !selected.has(key) &&
+ !isIdentifyingAttribute(key) &&
+ !isRenumberedKey(key) &&
+ !isStamp(value) &&
+ varies(["attributes", escapeToken(key)])
+ ? [{ key, value }]
+ : [],
+ ),
+ );
+ return best === undefined ? rest : ["attributes", escapeToken(best.key)];
+};
+
+/** Up to two guards for one check: a field on the 16 nearest like siblings of the innermost list it goes through that has any, preferring one that varies among them. */
+const guardsFor = (context: GuardContext, check: Suggestion): Suggestion[] => {
+ if (check.kind !== "value_equals" && check.kind !== "text_equals") return [];
+ for (const step of listSteps(check.pointer)) {
+ const { siblings, all } = siblingsOf(context, step);
+ const varies = variesAmong(all);
+ const guards: Suggestion[] = [];
+ for (const sibling of siblings.slice(0, MAX_SIBLINGS_READ)) {
+ if (guards.length === MAX_GUARDS_PER_CHECK) break;
+ const field =
+ check.kind === "value_equals" &&
+ "tolerance" in check &&
+ step.rest.length === 0
+ ? []
+ : check.kind === "value_equals"
+ ? guardField(sibling, varies, step.rest)
+ : step.rest;
+ const pointer = [step.list, sibling.member, ...field].join("/");
+ const guard = guardAt(context, check, pointer);
+ if (guard !== undefined) guards.push(guard);
+ }
+ if (guards.length > 0) return guards;
+ }
+ return [];
+};
+
+/**
+ * Checks that the rest stays as it was, for an author to add by choice: a
+ * field on up to two like siblings of the innermost list each suggested check
+ * goes through, nearest first among the `MAX_SIBLINGS_READ` nearest, at its
+ * seed value, which the end holds too: the
+ * field the check reads where it varies among them, otherwise one that does;
+ * never through a sibling paired by position or a renumbered id; at most
+ * `MAX_GUARDS` a file, none at a suggested pointer.
+ */
+export const draftGuards = (
+ context: GuardContext,
+ suggested: readonly Suggestion[],
+): Suggestion[] => {
+ const taken = new Set(
+ suggested.flatMap((check) => ("pointer" in check ? [check.pointer] : [])),
+ );
+ const guards: Suggestion[] = [];
+ for (const check of suggested) {
+ if (guards.length === MAX_GUARDS) break;
+ for (const guard of guardsFor(context, check))
+ if (
+ guards.length < MAX_GUARDS &&
+ "pointer" in guard &&
+ !taken.has(guard.pointer)
+ ) {
+ taken.add(guard.pointer);
+ guards.push(guard);
+ }
+ }
+ return guards;
+};
diff --git a/src/domain/desktop/appTaskDraftLineGuards.ts b/src/domain/desktop/appTaskDraftLineGuards.ts
new file mode 100644
index 0000000000..8d34bfcf23
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftLineGuards.ts
@@ -0,0 +1,58 @@
+import type { DesktopTaskCheckInput } from "./appTask.js";
+
+/** Lines a text file's guard names, and how far from the added line it looks for them. */
+const MAX_GUARD_LINES = 2;
+const NEIGHBOURHOOD = 6;
+/** A guard line longer than this is not copied into the task. */
+const MAX_GUARD_TEXT = 2000;
+
+/** A list item's marker, as Zim's `[ ]` checkbox, a `*` bullet or `1.`; undefined for a line that is no list item. */
+const markerOf = (line: string): string | undefined =>
+ /^\s*(?\[[ xX*>]?\]|[-*+•]|\d+[.)])\s/u.exec(line)?.groups?.marker;
+
+/**
+ * A `lines_present` guard of a text file's unchanged lines beside the first
+ * line an edit added, for an author to add by choice: the list items next to
+ * it with the same marker, as the other `[ ]` action items around a new one,
+ * nearest first and kept in file order; or, when it is no list item or has no
+ * such sibling, the nearest lines with words in them. Each is a line the
+ * seed and the end both hold. None when nothing was added.
+ */
+export const lineGuards = (
+ file: string,
+ before: readonly string[],
+ after: readonly string[],
+): DesktopTaskCheckInput[] => {
+ const seed = new Set(before);
+ const at = after.findIndex((line) => !seed.has(line));
+ const added = after[at];
+ if (added === undefined) return [];
+ const near = after
+ .map((line, index) => ({ line, index }))
+ .filter(
+ ({ line, index }) =>
+ index !== at &&
+ Math.abs(index - at) <= NEIGHBOURHOOD &&
+ seed.has(line) &&
+ line.length <= MAX_GUARD_TEXT &&
+ /[\p{L}\d]/u.test(line),
+ )
+ .toSorted(
+ (left, right) =>
+ Math.abs(left.index - at) - Math.abs(right.index - at) ||
+ left.index - right.index,
+ );
+ const marker = markerOf(added);
+ const siblings = near.filter(
+ ({ line }) => marker !== undefined && markerOf(line) === marker,
+ );
+ const lines = [
+ ...new Set(
+ (siblings.length > 0 ? siblings : near)
+ .slice(0, MAX_GUARD_LINES)
+ .toSorted((left, right) => left.index - right.index)
+ .map(({ line }) => line),
+ ),
+ ];
+ return lines.length === 0 ? [] : [{ kind: "lines_present", file, lines }];
+};
diff --git a/src/domain/desktop/appTaskDraftNamed.ts b/src/domain/desktop/appTaskDraftNamed.ts
index 3497b10a2b..0f8fedfde0 100644
--- a/src/domain/desktop/appTaskDraftNamed.ts
+++ b/src/domain/desktop/appTaskDraftNamed.ts
@@ -1,14 +1,43 @@
import type { JsonValue } from "../jsonValue.js";
-import { isTimestamp, type DesktopTaskCheckInput } from "./appTask.js";
+import type { DesktopTaskCheckInput } from "./appTask.js";
import { isXmlElement } from "./appTaskChecks.js";
-import { wholeTextSuggestions } from "./appTaskDraftText.js";
-import { diffNamedMembers } from "./stateChanges.js";
-import { locatePointer, pairMembers, resolvePointer } from "./viewPointer.js";
+import {
+ attributeChecks,
+ fieldChecks,
+ recordChecks,
+ type AddedContext,
+} from "./appTaskDraftAdded.js";
+import { draftGuards } from "./appTaskDraftGuards.js";
+import {
+ attributeKeyOf,
+ changeNoise,
+ inSplitDate,
+ isStamp,
+ type NoiseReason,
+} from "./appTaskDraftNoise.js";
+import { valueEqualsCheck } from "./appTaskDraftScalars.js";
+import { isIdentifyingAttribute } from "./attributeRank.js";
+import {
+ wholeTextSuggestions,
+ type WholeTextSuggestions,
+} from "./appTaskDraftText.js";
+import { draftValues, oneEnd, type DraftValue } from "./appTaskDraftValues.js";
+import {
+ diffNamedMembers,
+ type DesktopStateDifference,
+} from "./stateChanges.js";
+import {
+ escapeToken,
+ locatePointer,
+ pairMembers,
+ resolvePointer,
+} from "./viewPointer.js";
import {
entryWithoutView,
zipArrayLeavesOut,
zipEntryAt,
zipLeavesOutAt,
+ zipView,
} from "./zipView.js";
type Suggestion = DesktopTaskCheckInput;
@@ -18,10 +47,6 @@ const isRecord = (
): value is Record =>
typeof value === "object" && value !== null && !Array.isArray(value);
-/** A pointer segment for a key: `~` and `/` escaped as RFC 6901 writes them. */
-export const escapeToken = (token: string) =>
- token.replaceAll("~", "~0").replaceAll("/", "~1");
-
/** Pointers without a listed descendant: a container listed with its members is suggested through them. */
export const innermost = (pointers: readonly string[]): readonly string[] =>
pointers.filter(
@@ -70,7 +95,9 @@ const checkRoot = (
* element. `root` is where checks for a pointer the diff lists start, undefined
* where the evaluators do not read alike; `xml` whether a pointer goes into an
* XML view; `leavesOut` whether the view may lack part of what a pointer
- * reaches, and `arrayLeavesOut` members of the array a pointer names.
+ * reaches, and `arrayLeavesOut` members of the array a pointer names;
+ * `listed` whether a pointer is a value of the document a values list shows;
+ * `empty` the view of a file that holds nothing.
*/
interface CheckTree {
readonly root: (
@@ -80,25 +107,64 @@ interface CheckTree {
readonly xml: (view: JsonValue | null, pointer: string) => boolean;
readonly leavesOut: (view: JsonValue | null, pointer: string) => boolean;
readonly arrayLeavesOut: (view: JsonValue | null, array: string) => boolean;
+ readonly listed: (view: JsonValue | null, pointer: string) => boolean;
+ readonly empty: JsonValue | null;
}
+const valueFormat = (
+ tree: CheckTree,
+ view: JsonValue | null,
+ pointer: string,
+) =>
+ tree === ARCHIVE && /^\/entries\/[^/]+$/u.test(pointer)
+ ? ("archive" as const)
+ : tree.xml(view, pointer)
+ ? ("xml" as const)
+ : ("json" as const);
+
+const referenceRoots = (
+ tree: CheckTree,
+ view: JsonValue | null,
+): JsonValue[] => {
+ if (tree !== ARCHIVE) return view === null ? [] : [view];
+ if (!isRecord(view) || !Array.isArray(view.entries)) return [];
+ return view.entries.flatMap((entry) =>
+ isRecord(entry) &&
+ entry.format === "xml" &&
+ entry.view !== undefined &&
+ entry.view !== null
+ ? [entry.view]
+ : [],
+ );
+};
+
export const ARCHIVE: CheckTree = {
root: checkRoot,
xml: (view, pointer) => readableEntry(view, pointer)?.format === "xml",
leavesOut: zipLeavesOutAt,
arrayLeavesOut: zipArrayLeavesOut,
+ // An entry added or removed whole, or a value inside a readable one; not an entry's size or digest.
+ listed: (view, pointer) =>
+ /^\/entries\/[^/]+$/u.test(pointer) ||
+ checkRoot(view, pointer) !== undefined,
+ // An archive with no entries, so each entry of one created or deleted is listed by its name.
+ empty: zipView([], 0),
};
/** An XML file's view leaves out what its `truncated` says, anywhere in the tree. */
const truncatedXml = (view: JsonValue | null) =>
isRecord(view) && view.truncated === true;
+const inXmlRoot = (pointer: string) =>
+ pointer === "/root" || pointer.startsWith("/root/");
+
export const XML_FILE: CheckTree = {
- root: (_view, pointer) =>
- pointer === "/root" || pointer.startsWith("/root/") ? pointer : undefined,
+ root: (_view, pointer) => (inXmlRoot(pointer) ? pointer : undefined),
xml: () => true,
leavesOut: truncatedXml,
arrayLeavesOut: truncatedXml,
+ listed: (_view, pointer) => inXmlRoot(pointer),
+ empty: null,
};
/** Whether a check at a pointer is decided by name: no object or array reached by position, no array searched that the view may cut. */
@@ -115,7 +181,11 @@ const byName = (view: JsonValue | null, pointer: string, tree: CheckTree) => {
const elementName = (value: JsonValue | undefined): string | undefined =>
isXmlElement(value) ? value.name : undefined;
-/** A text node of an XML view holding only whitespace, which an app writes as it lays out the file. */
+/**
+ * A text node of an XML view holding only whitespace: what the XML reader
+ * keeps of it is text, such as the space between two runs, but a check of it
+ * alone would tie the edit to where the runs fall, so it gets none.
+ */
const isLayoutText = (
tree: CheckTree,
view: JsonValue | null,
@@ -127,102 +197,452 @@ const isLayoutText = (
/\/children\/[^/]+$/u.test(pointer) &&
tree.xml(view, pointer);
+/** The tags of elements a pointer reaches by position, which no attribute names alone among their siblings. */
+const positionalTags = (view: JsonValue | null, pointer: string): string[] => {
+ const segments = pointer.split("/");
+ return segments.flatMap((segment, index) => {
+ if (segments[index - 1] !== "children" || !/^\d+$/u.test(segment))
+ return [];
+ const element = resolvePointer(
+ view,
+ segments.slice(0, index + 1).join("/"),
+ );
+ return isXmlElement(element) ? [element.name] : [];
+ });
+};
+
+/** Facts a file's notes rest on, gathered as the suggestions are filtered. */
+export interface NamedFacts {
+ /** Changes no check took because only position reaches them. */
+ readonly positional: number;
+ /** The tags of the elements position alone reaches on their way. */
+ readonly positionalTags: readonly string[];
+ /** Elements whose changed text no check took, because it was past the limit. */
+ readonly long: readonly string[];
+ /** The changes the diff lists, and how many of them are noise. */
+ readonly changes: number;
+ readonly noise: number;
+ /** Suggested checks that are noise, kept because nothing else was. */
+ readonly flagged: readonly string[];
+}
+
+/** A file's draft by named pointers: its suggested checks, values, guards and the facts its notes rest on. */
+export interface NamedDraft {
+ readonly suggested: Suggestion[];
+ readonly values: DraftValue[];
+ readonly valuesCut: boolean;
+ readonly guards: Suggestion[];
+ readonly facts: NamedFacts;
+}
+
+/** Noise checks a document keeps when its change is nothing else, so an author still sees one. */
+const MAX_FLAGGED = 3;
+/** Guards a file's draft offers. */
+const MAX_GUARDS = 4;
+
+/** The draft's options: its limits, and whether the file is a document, which keeps fewer noise checks than the app's own state. */
+export interface NamedOptions {
+ readonly limit: { readonly suggestions: number; readonly chars: number };
+ readonly document: boolean;
+}
+
+/** What one pass over the named diff reads: the file, its ends, where checks go, the limits, the text checks already placed and noise. */
+interface CollectContext {
+ readonly file: string;
+ readonly ends: AddedContext["ends"];
+ readonly tree: CheckTree;
+ readonly limit: NamedOptions["limit"];
+ readonly texts: WholeTextSuggestions;
+ readonly noise: (pointer: string) => NoiseReason | undefined;
+}
+
+/** Whether a check reads a pointer that is noise. */
+const isNoisy = (noise: CollectContext["noise"], check: Suggestion): boolean =>
+ "pointer" in check && noise(check.pointer) !== undefined;
+
+/**
+ * The checks a pass gathers, elements' apart from values', those that are
+ * noise apart from the rest, each list at most the limit; and the changes it
+ * left to position, `real` those that are no noise.
+ */
+interface Collected {
+ readonly context: CollectContext;
+ readonly elements: Suggestion[];
+ readonly values: Suggestion[];
+ readonly noise: {
+ readonly elements: Suggestion[];
+ readonly values: Suggestion[];
+ };
+ readonly positional: {
+ count: number;
+ real: number;
+ readonly tags: Set;
+ };
+ /** Fields of an added record left unchecked because their text is past the limit. */
+ readonly fieldLong: string[];
+}
+
+const named = (
+ { context }: Collected,
+ view: JsonValue | null,
+ pointer: string,
+) => byName(view, pointer, context.tree);
+
+/** Count a change no check names because position alone reaches it; one a cut view rejects is the view's to explain. */
+const unnamed = (
+ collected: Collected,
+ view: JsonValue | null,
+ pointer: string,
+) => {
+ if (
+ collected.context.texts.covers(pointer) ||
+ locatePointer(view, pointer).byPosition.length === 0
+ )
+ return;
+ collected.positional.count += 1;
+ if (collected.context.noise(pointer) === undefined)
+ collected.positional.real += 1;
+ for (const tag of positionalTags(view, pointer))
+ collected.positional.tags.add(tag);
+};
+
+const room = ({ context }: Collected, list: readonly Suggestion[]) =>
+ list.length < context.limit.suggestions;
+
+/** Add checks to the elements' or values' list while it has room, those that are noise to their own list, so noise never takes the place of a check that is not. */
+const take = (
+ collected: Collected,
+ kind: "elements" | "values",
+ checks: readonly Suggestion[],
+) => {
+ for (const check of checks) {
+ const list = isNoisy(collected.context.noise, check)
+ ? collected.noise[kind]
+ : collected[kind];
+ if (room(collected, list)) list.push(check);
+ }
+};
+
+/** A check that a scalar, or an empty container, holds its final value. */
+const equals = (
+ collected: Collected,
+ pointer: string,
+ value: JsonValue | undefined,
+) => {
+ const { file, ends, tree, texts, noise } = collected.context;
+ const list =
+ noise(pointer) === undefined ? collected.values : collected.noise.values;
+ if (
+ !room(collected, list) ||
+ texts.covers(pointer) ||
+ value === undefined ||
+ (isRecord(value) && Object.keys(value).length > 0) ||
+ isStamp(value) ||
+ (tree.xml(ends.final, pointer) && inSplitDate(ends.final, pointer)) ||
+ isLayoutText(tree, ends.final, pointer, value)
+ )
+ return;
+ if (named(collected, ends.final, pointer))
+ take(collected, "values", [
+ valueEqualsCheck(
+ file,
+ pointer,
+ value,
+ resolvePointer(ends.baseline, pointer),
+ ),
+ ]);
+ else unnamed(collected, ends.final, pointer);
+};
+
+/**
+ * Checks of an element added whole: a record only position names by its new
+ * fields, beside any whole text; a list member by its own attributes; any
+ * other by its presence.
+ */
+const elementChecks = (
+ collected: Collected,
+ pointer: string,
+ value: JsonValue,
+): void => {
+ const { file, ends, texts, tree } = collected.context;
+ // A date the app stamps in parts is never a check.
+ if (!isXmlElement(value) || inSplitDate(ends.final, pointer)) return;
+ const added = {
+ file,
+ ends,
+ // Its checks read an element's text, which a cut view may lack.
+ named: (view: JsonValue | null, at: string) =>
+ named(collected, view, at) && !tree.leavesOut(view, at),
+ chars: collected.context.limit.chars,
+ long: collected.fieldLong,
+ };
+ if (!named(collected, ends.final, `${pointer}/name`)) {
+ const fields = fieldChecks(added, pointer, value);
+ take(collected, "values", fields);
+ // Its new fields, checked at an ancestor, name it as well as text does.
+ if (fields.length === 0 && !texts.covers(pointer))
+ unnamed(collected, ends.final, pointer);
+ return;
+ }
+ if (texts.covers(pointer) || !room(collected, collected.elements)) return;
+ const records = recordChecks(added, pointer, value);
+ // Repeated unnamed fields, such as two `` children, have no selector. Their text is the content check; the tag name is not.
+ const own = [
+ ...attributeChecks(added, pointer, value),
+ ...(records.length > 0 ? records : fieldChecks(added, pointer, value)),
+ ];
+ take(
+ collected,
+ "elements",
+ own.length > 0
+ ? own
+ : [
+ {
+ kind: "value_equals",
+ file,
+ pointer: `${pointer}/name`,
+ value: value.name,
+ },
+ ],
+ );
+};
+
+/** Checks of a value added whole: an element as `elementChecks` says, a container through its members, a scalar as it is. */
+const addedChecks = (
+ collected: Collected,
+ pointer: string,
+ value: JsonValue | undefined,
+): void => {
+ if (
+ !room(collected, collected.elements) &&
+ !room(collected, collected.values)
+ )
+ return;
+ if (
+ collected.context.tree.xml(collected.context.ends.final, pointer) &&
+ isXmlElement(value)
+ )
+ elementChecks(collected, pointer, value);
+ else if (collected.context.texts.covers(pointer)) return;
+ else if (Array.isArray(value) && value.length > 0) {
+ const { keys } = pairMembers(value, []).newer;
+ value.forEach((item, index) =>
+ addedChecks(
+ collected,
+ `${pointer}/${escapeToken(keys[index] ?? String(index))}`,
+ item,
+ ),
+ );
+ } else if (isRecord(value) && Object.keys(value).length > 0)
+ for (const [key, item] of Object.entries(value))
+ addedChecks(collected, `${pointer}/${escapeToken(key)}`, item);
+ // A scalar, or an empty array or object, is checked as it is.
+ else equals(collected, pointer, value);
+};
+
+/** A check that a value is gone, only where the entry still has a view to lack it in. */
+const removedCheck = (collected: Collected, pointer: string) => {
+ const { file, ends, tree, texts } = collected.context;
+ const before = resolvePointer(ends.baseline, pointer);
+ if (
+ texts.covers(pointer) ||
+ isLayoutText(tree, ends.baseline, pointer, before) ||
+ tree.root(ends.baseline, pointer) !== pointer ||
+ tree.root(ends.final, pointer) !== pointer
+ )
+ return;
+ if (
+ named(collected, ends.baseline, pointer) &&
+ named(collected, ends.final, pointer)
+ )
+ take(collected, elementName(before) === undefined ? "values" : "elements", [
+ { kind: "value_absent", file, pointer },
+ ]);
+ else unnamed(collected, ends.baseline, pointer);
+};
+
+/**
+ * A check that the identifier an element was renamed from names nothing at
+ * the end, as a layer renamed from `Untitled` that its `src` names: its tag
+ * and old value, where that named the element alone at the seed. An element
+ * its tag alone names gets none; the check of its identifier says as much.
+ */
+const renamedFrom = (collected: Collected, pointer: string) => {
+ const { file, ends, tree } = collected.context;
+ const key = attributeKeyOf(pointer);
+ const owner = pointer.split("/").slice(0, -2).join("/");
+ const list = /^(?.+\/children)\/[^/]+$/u.exec(owner)?.groups?.list;
+ const before = resolvePointer(ends.baseline, owner);
+ const old = resolvePointer(ends.baseline, pointer);
+ if (
+ !tree.xml(ends.baseline, owner) ||
+ key === undefined ||
+ list === undefined ||
+ !(owner.split("/").at(-1) ?? "").includes("][@") ||
+ !isIdentifyingAttribute(key) ||
+ /[=[\]]/u.test(key) ||
+ !isXmlElement(before) ||
+ typeof old !== "string" ||
+ old.includes("][")
+ )
+ return;
+ const at = `${list}/${escapeToken(`[name=${before.name}][@${key}=${old}]`)}`;
+ const seed = locatePointer(ends.baseline, at);
+ if (
+ seed.kind === "value" &&
+ seed.value === before &&
+ locatePointer(ends.final, at).kind === "none" &&
+ tree.root(ends.final, at) === at &&
+ named(collected, ends.baseline, at) &&
+ named(collected, ends.final, at)
+ )
+ take(collected, "values", [{ kind: "value_absent", file, pointer: at }]);
+};
+
+/** The checks one pass over the named diff suggests, with what it left out by position. */
+const collectChecks = (
+ context: CollectContext,
+ change: DesktopStateDifference,
+): Collected => {
+ const collected: Collected = {
+ context,
+ elements: [],
+ values: [],
+ noise: { elements: [], values: [] },
+ positional: { count: 0, real: 0, tags: new Set() },
+ fieldLong: [],
+ };
+ const { ends, tree } = context;
+ for (const pointer of change.added) {
+ const root = tree.root(ends.final, pointer);
+ if (root !== undefined)
+ addedChecks(collected, root, resolvePointer(ends.final, root));
+ }
+ for (const pointer of innermost(change.changed))
+ if (tree.root(ends.final, pointer) === pointer) {
+ equals(collected, pointer, resolvePointer(ends.final, pointer));
+ // An id the app renumbered, or other noise, renamed nothing.
+ if (context.noise(pointer) === undefined) renamedFrom(collected, pointer);
+ }
+ for (const pointer of change.removed) removedCheck(collected, pointer);
+ return collected;
+};
+
/**
* Checks by the selectors a keyed diff names members with, in an archive's
* entries or an XML file: the whole text of each element whose text changed
* first, then elements added or removed whole, then values changed, added and
- * removed. A JSON value or an entry added whole is checked through its
- * members. Nothing goes through an object or array by position.
+ * removed. An element added to a list is checked by its own attributes, a
+ * record only position names by its new fields, a decimal by its number. A
+ * JSON value or an entry added whole is checked through its members. Nothing
+ * goes through an object or array by position. Noise (view state, renumbered
+ * ids, volatile values, times, upkeep) leaves the suggestions when anything
+ * else changed, a change only position reaches included; when noise is all
+ * there is, it stays, flagged: up to three in a document, as many as fit in
+ * the app's own state, where a view setting may be the task. Beside them: the
+ * values the change added, changed and removed, and guards that the rest
+ * stays as it was.
*/
-export const namedSuggestions = (
+export const namedDraft = (
file: string,
- ends: {
- readonly baseline: JsonValue | null;
- readonly final: JsonValue | null;
- },
+ ends: AddedContext["ends"],
tree: CheckTree,
- limit: { readonly suggestions: number; readonly chars: number },
-): Suggestion[] => {
+ options: NamedOptions,
+): NamedDraft => {
const change = diffNamedMembers(ends.baseline, ends.final);
+ const noise = changeNoise(change, ends, tree.xml);
+ const named = (view: JsonValue | null, pointer: string) =>
+ byName(view, pointer, tree) && !tree.leavesOut(view, pointer);
const texts = wholeTextSuggestions(file, change, ends, {
- limit,
+ limit: options.limit,
inTree: (pointer) =>
tree.xml(ends.final, pointer) &&
tree.root(ends.final, pointer) === pointer,
- name: (pointer) =>
- byName(ends.final, pointer, tree) && !tree.leavesOut(ends.final, pointer)
- ? pointer
- : undefined,
+ name: (pointer) => (named(ends.final, pointer) ? pointer : undefined),
+ noise: (pointer) => noise(pointer) !== undefined,
});
- const elements: Suggestion[] = [];
- const values: Suggestion[] = [];
- const equals = (pointer: string, value: JsonValue | undefined) => {
- if (
- values.length < limit.suggestions &&
- !texts.covers(pointer) &&
- value !== undefined &&
- !(isRecord(value) && Object.keys(value).length > 0) &&
- !isTimestamp(value) &&
- !isLayoutText(tree, ends.final, pointer, value) &&
- byName(ends.final, pointer, tree)
- )
- values.push({ kind: "value_equals", file, pointer, value });
- };
- const added = (pointer: string, value: JsonValue | undefined): void => {
- if (
- (elements.length >= limit.suggestions &&
- values.length >= limit.suggestions) ||
- texts.covers(pointer)
- )
- return;
- const name = elementName(value);
- if (name !== undefined) {
- if (
- elements.length < limit.suggestions &&
- byName(ends.final, `${pointer}/name`, tree)
- )
- elements.push({
- kind: "value_equals",
+ const checks = collectChecks(
+ { file, ends, tree, limit: options.limit, texts, noise },
+ change,
+ );
+ const noisy = (check: Suggestion) => isNoisy(noise, check);
+ const clean = [
+ ...texts.suggestions.filter((check) => !noisy(check)),
+ ...checks.elements,
+ ...checks.values,
+ ];
+ const flaggable = [
+ ...texts.suggestions.filter(noisy),
+ ...checks.noise.elements,
+ ...checks.noise.values,
+ ];
+ const edited = clean.length > 0 || checks.positional.real > 0;
+ const suggested = (
+ edited
+ ? clean
+ : options.document
+ ? flaggable.slice(0, MAX_FLAGGED)
+ : flaggable
+ ).slice(0, options.limit.suggestions);
+ const listed = [...change.added, ...change.changed, ...change.removed];
+ const { values, cut } = draftValues(change, {
+ ends,
+ listed: tree.listed,
+ noise,
+ format: (view, pointer) => valueFormat(tree, view, pointer),
+ referenceRoots: (view) => referenceRoots(tree, view),
+ });
+ const guards = edited
+ ? draftGuards(
+ {
file,
- pointer: `${pointer}/name`,
- value: name,
- });
- } else if (Array.isArray(value) && value.length > 0) {
- const { keys } = pairMembers(value, []).newer;
- value.forEach((item, index) =>
- added(`${pointer}/${escapeToken(keys[index] ?? String(index))}`, item),
- );
- } else if (isRecord(value) && Object.keys(value).length > 0)
- for (const [key, item] of Object.entries(value))
- added(`${pointer}/${escapeToken(key)}`, item);
- // A scalar, or an empty array or object, is checked as it is.
- else equals(pointer, value);
+ ends,
+ named,
+ noisy: (pointer) => noise(pointer) !== undefined,
+ chars: options.limit.chars,
+ },
+ suggested,
+ ).slice(0, MAX_GUARDS)
+ : [];
+ return {
+ suggested,
+ values,
+ valuesCut: cut,
+ guards,
+ facts: {
+ positional: checks.positional.count,
+ positionalTags: [...checks.positional.tags],
+ long: [...new Set([...texts.long, ...checks.fieldLong])],
+ changes: listed.length,
+ noise: listed.filter((pointer) => noise(pointer) !== undefined).length,
+ flagged: edited
+ ? []
+ : suggested.flatMap((check) =>
+ "pointer" in check ? [check.pointer] : [],
+ ),
+ },
};
- for (const pointer of change.added) {
- const root = tree.root(ends.final, pointer);
- if (root !== undefined) added(root, resolvePointer(ends.final, root));
- }
- for (const pointer of innermost(change.changed))
- if (tree.root(ends.final, pointer) === pointer)
- equals(pointer, resolvePointer(ends.final, pointer));
- // An absence is checked only where the entry still has a view to lack it in.
- for (const pointer of change.removed)
- if (
- !texts.covers(pointer) &&
- !isLayoutText(
- tree,
- ends.baseline,
- pointer,
- resolvePointer(ends.baseline, pointer),
- ) &&
- tree.root(ends.baseline, pointer) === pointer &&
- tree.root(ends.final, pointer) === pointer &&
- byName(ends.baseline, pointer, tree) &&
- byName(ends.final, pointer, tree)
- )
- (elementName(resolvePointer(ends.baseline, pointer)) === undefined
- ? values
- : elements
- ).push({ kind: "value_absent", file, pointer });
- return [...texts.suggestions, ...elements, ...values];
+};
+
+/**
+ * The values of an archive or XML file created or deleted: what its one end
+ * holds against `tree.empty`, each entry, or the root element, whole once, as
+ * one added to a file that changed is listed, within the same limits.
+ */
+export const presenceValues = (
+ ends: AddedContext["ends"],
+ tree: CheckTree,
+ created: boolean,
+): { readonly values: DraftValue[]; readonly cut: boolean } => {
+ const from = created
+ ? { baseline: tree.empty, final: ends.final }
+ : { baseline: ends.baseline, final: tree.empty };
+ const change = oneEnd(diffNamedMembers(from.baseline, from.final), created);
+ return draftValues(change, {
+ ends: from,
+ listed: tree.listed,
+ noise: changeNoise(change, from, tree.xml),
+ format: (view, pointer) => valueFormat(tree, view, pointer),
+ referenceRoots: (view) => referenceRoots(tree, view),
+ });
};
diff --git a/src/domain/desktop/appTaskDraftNoise.test.ts b/src/domain/desktop/appTaskDraftNoise.test.ts
new file mode 100644
index 0000000000..e54444ccca
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftNoise.test.ts
@@ -0,0 +1,357 @@
+import { describe, expect, it } from "vitest";
+
+import type { JsonValue } from "../jsonValue.js";
+import {
+ draftOf,
+ pointerOf,
+ statePath,
+ xml,
+ xmlDraft,
+} from "./appTaskDraft.fixture.js";
+
+const CELLS =
+ "/root/children/[name=Sheet]/children/[name=Cells]/children/[name=Cell]";
+const SELECTIONS = "/root/children/[name=Sheet]/children/[name=Selections]";
+
+/** A spreadsheet's save: where the cursor was, its cells, and what the app stamps on the file. */
+const workbook = (save: {
+ cursor: string;
+ march: string;
+ totals: readonly [string, string];
+ session: string;
+ saved: string;
+ date: string;
+}) =>
+ `Sheets 1.0${save.date}${save.march}17.5${save.totals[0] === "1" ? "" : "=sum(C2:C4)"}${save.totals[1] === "1" ? "" : "=sum(D2:D4)"}`;
+
+const SEED = {
+ cursor: "0",
+ march: "405.1",
+ totals: ["2", "3"],
+ session: "0b5c1f0e-3c1d-4f7a-9a3e-2d7f5b8c9e10",
+ saved: "1793870431",
+ date: "2026-11-05T23:45:25Z",
+} as const;
+/** The save after the edit: the app also moved the cursor, renumbered the shared formulas and stamped the file anew. */
+const EDITED = {
+ cursor: "3",
+ march: "437.8",
+ totals: ["1", "1"],
+ session: "7d1e2a90-5b4c-4e8f-8a21-6c3b9f0d4e57",
+ saved: "1793870500",
+ date: "2026-11-05T23:47:01Z",
+} as const;
+
+describe("draftTaskFiles noise", () => {
+ it("leaves view state, renumbered ids, volatile values and times out of a document's suggestions when an edit is there, noting each in values", () => {
+ const file = xmlDraft(workbook(SEED), workbook(EDITED));
+
+ expect(file.suggested.map(pointerOf)).toEqual([`${CELLS}[@Col=3][@Row=2]`]);
+ const noted = (note: string) =>
+ file.values
+ .filter((value) => value.note === note)
+ .map(({ pointer }) => pointer)
+ .toSorted();
+ expect(noted("view_state")).toEqual(
+ [
+ `${SELECTIONS}/attributes/CursorCol`,
+ `${SELECTIONS}/attributes/CursorRow`,
+ `${SELECTIONS}/children/[name=Selection]/attributes/endCol`,
+ `${SELECTIONS}/children/[name=Selection]/attributes/startCol`,
+ ].toSorted(),
+ );
+ // The emptied formula text goes with the renumbered id of its cell.
+ expect(noted("renumbered")).toEqual(
+ [
+ `${CELLS}[@Col=3][@Row=5]/attributes/ExprID`,
+ `${CELLS}[@Col=3][@Row=5]/children/0`,
+ `${CELLS}[@Col=4][@Row=5]/attributes/ExprID`,
+ `${CELLS}[@Col=4][@Row=5]/children/0`,
+ ].toSorted(),
+ );
+ expect(noted("volatile")).toEqual([
+ "/root/children/[name=meta]/attributes/saved",
+ "/root/children/[name=meta]/attributes/session",
+ ]);
+ expect(noted("timestamp")).toEqual([
+ "/root/children/[name=meta]/children/[name=date]/children/0",
+ ]);
+ // The edit leads, noise last.
+ expect(file.values[0]?.note).toBeUndefined();
+ expect(file.notes).toEqual([]);
+ });
+
+ it("keeps a formula edit when the cell's ExprID changes too", () => {
+ const sheet = (march: string, formula: string, id: string) =>
+ `${march}${formula}`;
+ const file = xmlDraft(
+ sheet("405.1", "=sum(C2:C4)", "2"),
+ sheet("437.8", "=sum(C2:C5)", "4"),
+ );
+ const shown = file.suggested.flatMap((check) =>
+ "value" in check && typeof check.value === "string" ? [check.value] : [],
+ );
+
+ expect(shown).toContain("=sum(C2:C5)");
+ expect(
+ file.values
+ .filter(({ note }) => note === "renumbered")
+ .map(({ pointer }) => pointer),
+ ).toEqual([
+ "/root/children/[name=Sheet]/children/[name=Cells]/children/[name=Cell][@Row=5]/attributes/ExprID",
+ ]);
+ });
+
+ it("keeps an edit that follows more noise than a file has room for checks", () => {
+ const cursors = (value: string) =>
+ Array.from(
+ { length: 12 },
+ (_, index) => `cursor${String(index)}="${value}"`,
+ ).join(" ");
+ const shape = (cursor: string, width: string) =>
+ ``;
+
+ const file = xmlDraft(shape("1", "10"), shape("2", "12"));
+
+ expect(file.suggested).toEqual([
+ {
+ kind: "value_equals",
+ file: "Documents/doc.xml",
+ pointer: "/root/children/[name=shape]/attributes/width",
+ value: "12",
+ },
+ ]);
+ expect(file.notes).toEqual([]);
+ });
+
+ it("keeps up to three noise checks, flagged in a note, when noise is all a document's change holds", () => {
+ const file = xmlDraft(workbook(SEED), workbook({ ...SEED, cursor: "3" }));
+
+ const kept = file.suggested.map(pointerOf);
+ expect(kept).toHaveLength(3);
+ for (const pointer of kept) expect(pointer).toContain(SELECTIONS);
+ expect(file.notes).toEqual([
+ expect.objectContaining({
+ code: "view_state_only",
+ file: "Documents/doc.xml",
+ pointers: kept,
+ }),
+ ]);
+ });
+
+ it("keeps every view-state check of the app's own state when it is all that changed, flagged, since a view setting may be the task", () => {
+ const file = xmlDraft(workbook(SEED), workbook({ ...SEED, cursor: "3" }), {
+ role: "application_state",
+ file: ".config/sheets/state.xml",
+ });
+
+ const kept = file.suggested.map(pointerOf);
+ expect(kept).toHaveLength(4);
+ for (const pointer of kept) expect(pointer).toContain(SELECTIONS);
+ expect(file.values.map(({ note }) => note)).toEqual(
+ Array.from({ length: 4 }, () => "view_state"),
+ );
+ expect(file.notes).toEqual([
+ expect.objectContaining({ code: "view_state_only", pointers: kept }),
+ ]);
+ });
+
+ it("keeps a document's geometry as content, since a CAD sketch's Geometry is what its user drew", () => {
+ const sketch = (radius: string) =>
+ ``;
+ const file = xmlDraft(sketch("10"), sketch("12.5"));
+
+ expect(file.suggested.map(pointerOf)).toEqual([
+ "/root/children/[name=Object]/children/[name=Geometry]/children/[name=Circle]/attributes/Radius",
+ ]);
+ expect(file.values.filter((value) => value.note === "view_state")).toEqual(
+ [],
+ );
+ });
+});
+
+describe("draftTaskFiles noise named in values and metadata", () => {
+ /** A spreadsheet saved as ODF keeps it: view and printer settings named by a `name` value, edit time and counts, a stamped header time. */
+ const saved = (save: {
+ duration: string;
+ cycles: string;
+ total: string;
+ saves: string;
+ cursor: string;
+ printer: string;
+ time: string;
+ february: string;
+ cleaning: string;
+ }) =>
+ `Sheets 24.2${save.duration}${save.cycles}${save.total}${save.saves}${save.cursor}2true${save.printer}${save.february}${save.cleaning}`;
+ const SEED_SAVE = {
+ duration: "PT1M42S",
+ cycles: "1",
+ total: "PT1M",
+ saves: "1",
+ cursor: "0",
+ printer: "",
+ time: "05:13:42.339510419",
+ february: "21300",
+ cleaning: "40",
+ };
+ const EDITED_SAVE = {
+ duration: "PT2M2S",
+ cycles: "2",
+ total: "PT2M",
+ saves: "2",
+ cursor: "1",
+ printer: "Generic Printer",
+ time: "05:16:21.675302540",
+ february: "23100",
+ cleaning: "45",
+ };
+
+ it("demotes what a name-like value names as view state, edit time and counts under the metadata, and a time of day, but not a phrase a user named", () => {
+ const file = xmlDraft(saved(SEED_SAVE), saved(EDITED_SAVE));
+
+ const body = "/root/children/[name=body]/children/[name=cell]";
+ expect(file.suggested.map(pointerOf)).toEqual([
+ `${body}[@name=February]`,
+ `${body}[@name=Window cleaning]`,
+ ]);
+ const noted = Object.fromEntries(
+ file.values.map(({ pointer, note }) => [pointer, note ?? "edit"]),
+ );
+ const meta = "/root/children/[name=meta]/children";
+ const settings = "/root/children/[name=settings]/children/[name=item-set]";
+ expect(noted).toEqual({
+ [`${body}[@name=February]/children/0`]: "edit",
+ [`${body}[@name=Window cleaning]/children/0`]: "edit",
+ [`${meta}/[name=editing-duration]/children/0`]: "view_state",
+ [`${meta}/[name=editing-cycles]/children/0`]: "view_state",
+ [`${meta}/[name=total-time]/children/0`]: "volatile",
+ [`${meta}/[name=save-count]/children/0`]: "volatile",
+ [`${settings}[@name=ooo:view-settings]/children/[name=item][@name=CursorPositionX]/children/0`]:
+ "view_state",
+ [`${settings}[@name=ooo:configuration-settings]/children/[name=item][@name=PrinterName]/children/0`]:
+ "view_state",
+ "/root/children/[name=header]/children/[name=time]/attributes/value":
+ "timestamp",
+ });
+ });
+});
+
+describe("draftTaskFiles noise in the app's own state", () => {
+ /** A note as Gnote keeps it: its text, the times it stamps and where the cursor was. */
+ const note = (text: string, changed: string, cursor: string) =>
+ `Shopping List${text}${changed}${cursor}`;
+
+ it("leaves the cursor and stamped times out of the suggestions when the note's text changed too, noting each in values", () => {
+ const file = xmlDraft(
+ note("Shopping List\n\neggs", "2026-11-06T04:52:36.702406Z", "19"),
+ note(
+ "Shopping List\n\neggs\nolive oil",
+ "2026-11-06T04:54:17.918949Z",
+ "78",
+ ),
+ { role: "application_state", file: ".local/share/gnote/list.note" },
+ );
+
+ expect(file.suggested.map(pointerOf)).toEqual([
+ "/root/children/[name=text]/children/[name=note-content]",
+ ]);
+ expect(file.values.map(({ pointer, note: why }) => [pointer, why])).toEqual(
+ [
+ [
+ "/root/children/[name=text]/children/[name=note-content]/children/0",
+ undefined,
+ ],
+ ["/root/children/[name=last-change-date]/children/0", "timestamp"],
+ ["/root/children/[name=cursor-position]/children/0", "view_state"],
+ ],
+ );
+ expect(file.notes).toEqual([]);
+ });
+});
+
+describe("draftTaskFiles upkeep in an archive", () => {
+ const entry = (name: string, view: string) => ({
+ name,
+ size: view.length,
+ sha256: view,
+ format: "xml",
+ view: xml(view),
+ view_error: null,
+ });
+ const archive = (content: string, text: string, heading = text) => ({
+ entries: [
+ entry(
+ "content.xml",
+ `
${content}
`,
+ ),
+ entry(
+ "styles.xml",
+ ``,
+ ),
+ ],
+ entry_count: 2,
+ truncated: false,
+ });
+ const drafted = (before: JsonValue, after: JsonValue) => {
+ const [file] = draftOf({
+ "Documents/flyer.odt": statePath("document", "zip", before, after),
+ });
+ if (file === undefined) throw new Error("the file did not change");
+ return file;
+ };
+ const filled =
+ "/entries/[name=styles.xml]/view/root/children/[name=style][@name=Text]/children/[name=props]/attributes/loext:tab-stop-distance";
+
+ it("demotes an attribute several elements gained while another entry holds the edit", () => {
+ const file = drafted(
+ archive("Spring Sale", ""),
+ archive("Summer Sale", ``),
+ );
+
+ expect(file.suggested.map(pointerOf)).toEqual([
+ "/entries/[name=content.xml]/view/root/children/[name=body]/children/[name=p]",
+ ]);
+ expect(file.values.find(({ pointer }) => pointer === filled)?.note).toBe(
+ "upkeep",
+ );
+ });
+
+ it("suggests an attribute an entry gained when nothing else changed, since that may be the edit", () => {
+ const file = drafted(
+ archive("Spring Sale", ""),
+ archive("Spring Sale", ``),
+ );
+
+ expect(file.suggested.map(pointerOf)).toEqual([
+ filled,
+ filled.replace("[@name=Text]", "[@name=Heading]"),
+ ]);
+ expect(file.values.map(({ note }) => note)).toEqual([undefined, undefined]);
+ });
+
+ it("keeps a font size added to one style when the document text changed too", () => {
+ const file = drafted(
+ archive("Spring Sale", ""),
+ archive("Summer Sale", ``, ""),
+ );
+ const size =
+ "/entries/[name=styles.xml]/view/root/children/[name=style][@name=Text]/children/[name=props]/attributes/font-size";
+
+ expect(file.suggested.map(pointerOf)).toEqual([
+ "/entries/[name=content.xml]/view/root/children/[name=body]/children/[name=p]",
+ size,
+ ]);
+ expect(file.values.every(({ note }) => note === undefined)).toBe(true);
+ });
+
+ it("keeps a formatting edit when another archive entry also changes", () => {
+ const file = drafted(
+ archive("Spring Sale", ""),
+ archive("Summer Sale", ``),
+ );
+ expect(file.suggested.map(pointerOf)).toContain(
+ "/entries/[name=styles.xml]/view/root/children/[name=style][@name=Text]/children/[name=props]/attributes/fo:font-weight",
+ );
+ });
+});
diff --git a/src/domain/desktop/appTaskDraftNoise.ts b/src/domain/desktop/appTaskDraftNoise.ts
new file mode 100644
index 0000000000..4ef71aa37a
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftNoise.ts
@@ -0,0 +1,380 @@
+import type { JsonValue } from "../jsonValue.js";
+import { isTimestamp } from "./appTask.js";
+import { elementText, isXmlElement } from "./appTaskChecks.js";
+import {
+ isIdentifyingAttribute,
+ isNameLikeKey,
+ isViewStateName,
+ VIEW_STATE,
+} from "./attributeRank.js";
+import { isPositionalKey, positionalKeys } from "./positionalKeys.js";
+import type { DesktopStateDifference } from "./stateChanges.js";
+import { resolvePointer } from "./viewPointer.js";
+
+/** Why a change is what an app writes as it saves or shows a document, rather than what its user made. */
+export const NOISE_REASONS = [
+ "view_state",
+ "renumbered",
+ "volatile",
+ "timestamp",
+ "upkeep",
+] as const;
+export type NoiseReason = (typeof NOISE_REASONS)[number];
+
+/**
+ * Whether a name-like attribute's value names view state: one token of two or
+ * more words, as LibreOffice's `CursorPositionX` or `ooo:view-settings`, never
+ * a phrase a user wrote, such as a task called `Window cleaning`.
+ */
+const isViewStateValue = (value: string): boolean =>
+ /^[\p{L}\d]+(?:[-_:.][\p{L}\d]+)*$/u.test(value) &&
+ (/[-_:.]/u.test(value) || /\p{Ll}\p{Lu}/u.test(value)) &&
+ VIEW_STATE.test(value);
+
+/** An attribute that says what its element is called, whose value may name view state: name-like, or a `key`. */
+const namesElement = (key: string) =>
+ isNameLikeKey(key) || /^(?:[^:]*:)?key$/iu.test(key);
+
+/** An attribute an app renumbers as it saves: a generated id such as Gnumeric's `ExprID` or Scribus's `ItemID`, or LibreOffice's `rsid`. */
+export const isRenumberedKey = (key: string): boolean => {
+ const local = key.split(":").at(-1) ?? key;
+ return (
+ !isIdentifyingAttribute(key) &&
+ /(?:[a-z]I[dD]|(?:^|[-_])[rR][sS][iI][dD])$/u.test(local)
+ );
+};
+
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu;
+/** Seconds since 1970 of 2000-01-01 and 2100-01-01: an epoch time a save stamps falls between. */
+const EPOCH_RANGE = [946_684_800, 4_102_444_800] as const;
+
+/** UUIDs and epoch-shaped values in named save metadata are likely generated. */
+export const isVolatileValue = (
+ value: string,
+ bookkeeping = false,
+): boolean => {
+ if (UUID.test(value)) return true;
+ if (!bookkeeping) return false;
+ if (!/^\d{10}(?:\d{3})?$/u.test(value)) return false;
+ const seconds = Number(value.slice(0, 10));
+ if (!(seconds > EPOCH_RANGE[0] && seconds < EPOCH_RANGE[1])) return false;
+ return true;
+};
+
+/**
+ * A machine time of day, as LibreOffice stamps a header's `text:time-value`
+ * with `05:16:21.675302540`. A clock a person writes, such as `09:30:00`,
+ * has no fraction and no zone, so it stays text.
+ */
+const TIME_OF_DAY =
+ /^\d{2}:\d{2}:\d{2}(?:[.,]\d+(?:Z|[+-]\d{2}:?\d{2})?|Z|[+-]\d{2}:?\d{2})$/u;
+
+/** Whether a value is a time an app stamps: a date and time `isTimestamp` reads, or a time of day to the second. */
+export const isStamp = (value: JsonValue): boolean =>
+ isTimestamp(value) ||
+ (typeof value === "string" && TIME_OF_DAY.test(value.trim()));
+
+/** A part of a date or time an app writes as an element of its own. */
+const DATE_PART = /^(?:year|month|day|hour|minute|second)$/u;
+
+/**
+ * A tag naming when an app made, changed or saved something, as Tellico's
+ * `cdate` and `mdate`, or `created`, `modified`, `saved` or `updated`.
+ */
+const BOOKKEEPING =
+ /^[cm]date$|creat(?:ed|ion)|modif(?:ied|ication)|updated|saved|changed|accessed|timestamp/u;
+
+export const isBookkeepingName = (name: string) =>
+ BOOKKEEPING.test((name.split(":").at(-1) ?? name).toLowerCase());
+
+export const isBookkeeping = (element: Element) =>
+ isBookkeepingName(element.name);
+
+/**
+ * Whether a value is a date written in parts, as Tellico writes an entry's
+ * `` with ``, `` and ``: an element whose children
+ * are such parts, year, month and day among them, each holding only digits.
+ */
+const isDateInParts = (value: JsonValue | undefined): boolean => {
+ if (!isXmlElement(value)) return false;
+ const parts = new Set();
+ for (const child of value.children) {
+ if (typeof child === "string" && /^[ \t\r\n]*$/u.test(child)) continue;
+ const part = isXmlElement(child)
+ ? (child.name.split(":").at(-1) ?? child.name).toLowerCase()
+ : "";
+ if (!DATE_PART.test(part) || !/^\s*\d+\s*$/u.test(elementText(child)))
+ return false;
+ parts.add(part);
+ }
+ return ["year", "month", "day"].every((part) => parts.has(part));
+};
+
+/** An ISO 8601 duration, as ODF's `meta:editing-duration` `PT2M2S`. */
+const DURATION =
+ /^P(?=\d|T\d)(?:\d+Y)?(?:\d+M)?(?:\d+W)?(?:\d+D)?(?:T(?=\d)(?:\d+H)?(?:\d+M)?(?:\d+(?:[.,]\d+)?S)?)?$/u;
+/** A document's metadata element, as ODF's `office:meta` or AbiWord's `metadata`. */
+const META = /^(?:[^:]*:)?meta(?:data)?$/iu;
+/** A name for a count the app keeps, as `meta:editing-cycles` or `meta:cell-count`. */
+const COUNTER = /count|cycles?|revision/iu;
+
+/** The key a pointer names an attribute by, unescaped; undefined when it names no attribute. */
+export const attributeKeyOf = (pointer: string): string | undefined => {
+ const segments = pointer.split("/");
+ return segments.at(-2) === "attributes"
+ ? segments.at(-1)?.replaceAll("~1", "/").replaceAll("~0", "~")
+ : undefined;
+};
+
+/** The element a pointer to one of its attributes names. */
+const ownerOfAttribute = (pointer: string) =>
+ pointer.split("/").slice(0, -2).join("/");
+
+type Element = Record & {
+ name: string;
+ children: JsonValue[];
+};
+
+const isRecord = (
+ value: JsonValue | undefined,
+): value is Record =>
+ typeof value === "object" && value !== null && !Array.isArray(value);
+
+/** The elements a pointer steps into through `children`, read from the view, outermost first. */
+const elementsAlong = (view: JsonValue | null, pointer: string): Element[] => {
+ const segments = pointer.split("/");
+ const elements: Element[] = [];
+ for (let index = 2; index < segments.length; index += 1) {
+ if (segments[index - 1] !== "children") continue;
+ const element = resolvePointer(
+ view,
+ segments.slice(0, index + 1).join("/"),
+ );
+ if (isXmlElement(element)) elements.push(element);
+ }
+ return elements;
+};
+
+/**
+ * Whether elements along a pointer, outermost first, reach a date the app
+ * stamps in parts: one written in parts that it, or an element it sits in,
+ * names bookkeeping. A date a user enters in parts, as a birthday, is none.
+ */
+const stampedAlong = (elements: readonly Element[]) => {
+ let named = false;
+ return elements.some((element) => {
+ named ||= isBookkeeping(element);
+ return named && isDateInParts(element);
+ });
+};
+
+/** Whether a pointer is a date the app stamps in parts, or inside one. */
+export const inSplitDate = (view: JsonValue | null, pointer: string): boolean =>
+ stampedAlong(elementsAlong(view, pointer));
+
+/** Whether an element holds a date the app stamps in parts, `named` when a tag it sits in names bookkeeping. */
+const holdsStamped = (value: JsonValue, named: boolean): boolean =>
+ isXmlElement(value) &&
+ value.children.some((child) => {
+ if (!isXmlElement(child)) return false;
+ const inside = named || isBookkeeping(child);
+ return (inside && isDateInParts(child)) || holdsStamped(child, inside);
+ });
+
+/** Whether the element at a pointer is, sits in or holds a date the app stamps in parts, whose text no check compares. */
+export const holdsSplitDate = (
+ view: JsonValue | null,
+ pointer: string,
+): boolean => {
+ const along = elementsAlong(view, pointer);
+ const value = resolvePointer(view, pointer);
+ return (
+ stampedAlong(along) ||
+ (value !== undefined && holdsStamped(value, along.some(isBookkeeping)))
+ );
+};
+
+/** Whether an element is view state by its tag, or by a name-like attribute naming view state, as `[@config:name=CursorPositionX]`. */
+const showsView = (element: Element) =>
+ isViewStateName(element.name) ||
+ Object.entries(isRecord(element.attributes) ? element.attributes : {}).some(
+ ([key, value]) =>
+ typeof value === "string" && namesElement(key) && isViewStateValue(value),
+ );
+
+/** A file's views at its two ends. */
+interface Ends {
+ readonly baseline: JsonValue | null;
+ readonly final: JsonValue | null;
+}
+
+/** Whether a duration or a count under a document's metadata, which the app keeps as it saves. */
+const metadataCount = (
+ elements: readonly Element[],
+ key: string | undefined,
+ value: JsonValue | undefined,
+) => {
+ if (
+ typeof value !== "string" ||
+ !elements.some(({ name }) => META.test(name))
+ )
+ return false;
+ if (DURATION.test(value.trim())) return true;
+ const own = key ?? elements.at(-1)?.name ?? "";
+ return /^\d+$/u.test(value.trim()) && COUNTER.test(own);
+};
+
+/** Whether an identifier change is one the app made by numbering a list by position, as Planner's task ids shift after an insertion. */
+const positionalRenumbering = (ends: Ends) => {
+ const lists = new Map>();
+ return (pointer: string, key: string) => {
+ const owner = ownerOfAttribute(pointer);
+ const list = owner.split("/").slice(0, -1).join("/");
+ const element =
+ resolvePointer(ends.final, owner) ?? resolvePointer(ends.baseline, owner);
+ if (!list.endsWith("/children") || !isXmlElement(element)) return false;
+ let keys = lists.get(list);
+ if (keys === undefined) {
+ const newer = resolvePointer(ends.final, list);
+ const older = resolvePointer(ends.baseline, list);
+ keys =
+ Array.isArray(newer) && Array.isArray(older)
+ ? positionalKeys(newer, older)
+ : new Set();
+ lists.set(list, keys);
+ }
+ return isPositionalKey(keys, element.name, key);
+ };
+};
+
+/** A pointer into an archive entry's view: the entry's part of it, as `/entries/[name=styles.xml]/view/`. */
+const ENTRY_VIEW = /^\/entries\/[^/]+\/view\//u;
+
+/**
+ * Attributes an archive entry's elements only gained, as LibreOffice fills in
+ * `loext:tab-stop-distance` across `styles.xml` on save, while another entry
+ * holds the edit: whether each pointer is one.
+ */
+const upkeepOf = (
+ change: Pick,
+ ends: Ends,
+ noise: (pointer: string) => NoiseReason | undefined,
+ xml: (view: JsonValue | null, pointer: string) => boolean,
+) => {
+ const added = new Set(change.added);
+ const gained = (pointer: string) =>
+ added.has(pointer) &&
+ xml(ends.final, pointer) &&
+ attributeKeyOf(pointer) === "loext:tab-stop-distance" &&
+ pointer.startsWith("/entries/[name=styles.xml]/view/") &&
+ [ends.baseline, ends.final].every((view) =>
+ isXmlElement(resolvePointer(view, ownerOfAttribute(pointer))),
+ );
+ const edited = new Set();
+ /** How many elements of an entry gained one attribute key. One gain is an edit; a repeated gain is the app filling a default in. */
+ const filled = new Map>();
+ for (const pointer of [
+ ...change.added,
+ ...change.changed,
+ ...change.removed,
+ ]) {
+ const [entry] = ENTRY_VIEW.exec(pointer) ?? [];
+ if (entry === undefined || noise(pointer) !== undefined) continue;
+ const key = attributeKeyOf(pointer);
+ if (!gained(pointer) || key === undefined) {
+ edited.add(entry);
+ continue;
+ }
+ const counts = filled.get(entry) ?? new Map();
+ filled.set(entry, counts);
+ counts.set(key, (counts.get(key) ?? 0) + 1);
+ }
+ return (pointer: string) => {
+ const [entry] = ENTRY_VIEW.exec(pointer) ?? [];
+ const key = attributeKeyOf(pointer);
+ const count =
+ entry === undefined || key === undefined
+ ? 0
+ : (filled.get(entry)?.get(key) ?? 0);
+ return (
+ edited.size > 0 &&
+ entry !== undefined &&
+ !edited.has(entry) &&
+ gained(pointer) &&
+ count >= 2
+ );
+ };
+};
+
+/**
+ * Why each pointer into a file is noise, or undefined when it is not: a tag,
+ * attribute or name-like attribute value on its way naming view state, as
+ * `[@config:name=CursorPositionX]`; an attribute an app renumbers, an
+ * identifier it numbers by position, or anything inside an element both ends
+ * hold whose renumbered attribute changed, as Gnumeric empties a formula cell
+ * whose shared expression it renumbers; a UUID or epoch time, or a duration or
+ * count under the document's metadata; a time the app stamps, whole or in
+ * parts (`inSplitDate`); the known style default an archive entry gained
+ * while another entry holds the edit.
+ */
+export const changeNoise = (
+ change: Pick,
+ ends: Ends,
+ xml: (view: JsonValue | null, pointer: string) => boolean,
+): ((pointer: string) => NoiseReason | undefined) => {
+ const renumbered = change.changed.flatMap((pointer) => {
+ const key = attributeKeyOf(pointer);
+ return xml(ends.final, pointer) && key !== undefined && isRenumberedKey(key)
+ ? [ownerOfAttribute(pointer)]
+ : [];
+ });
+ const byPosition = positionalRenumbering(ends);
+ const known = new Map();
+ const base = (pointer: string): NoiseReason | undefined => {
+ if (known.has(pointer)) return known.get(pointer);
+ const view =
+ resolvePointer(ends.final, pointer) === undefined
+ ? ends.baseline
+ : ends.final;
+ const isXml = xml(view, pointer);
+ const key = isXml ? attributeKeyOf(pointer) : undefined;
+ const elements = isXml ? elementsAlong(view, pointer) : [];
+ const value = resolvePointer(view, pointer);
+ const reason: NoiseReason | undefined =
+ elements.some(showsView) || (key !== undefined && isViewStateName(key))
+ ? "view_state"
+ : (key !== undefined &&
+ (isRenumberedKey(key) ||
+ (isIdentifyingAttribute(key) && byPosition(pointer, key)))) ||
+ renumbered.some((element) => {
+ if (pointer !== element && !pointer.startsWith(`${element}/`))
+ return false;
+ if (pointer === element) return false;
+ const next = resolvePointer(ends.final, pointer);
+ const text =
+ typeof next === "string"
+ ? next.trim()
+ : isXmlElement(next)
+ ? elementText(next).trim()
+ : undefined;
+ // Gnumeric clears a shared formula when it renumbers the cell. A formula that still has text was edited.
+ return text === undefined || text === "";
+ })
+ ? "renumbered"
+ : (typeof value === "string" &&
+ isVolatileValue(
+ value,
+ elements.some(isBookkeeping) ||
+ (key !== undefined && isBookkeepingName(key)),
+ )) ||
+ metadataCount(elements, key, value)
+ ? "volatile"
+ : isStamp(value ?? null) || stampedAlong(elements)
+ ? "timestamp"
+ : undefined;
+ known.set(pointer, reason);
+ return reason;
+ };
+ const upkeep = upkeepOf(change, ends, base, xml);
+ return (pointer) => base(pointer) ?? (upkeep(pointer) ? "upkeep" : undefined);
+};
diff --git a/src/domain/desktop/appTaskDraftNotes.ts b/src/domain/desktop/appTaskDraftNotes.ts
new file mode 100644
index 0000000000..e8cfc44004
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftNotes.ts
@@ -0,0 +1,259 @@
+import { z } from "zod";
+
+import { changedStateFiles, type TaskDraftFile } from "./appTaskDraft.js";
+import { FILE_NOTE_CODES, type FileNote } from "./appTaskDraftFileNotes.js";
+import { STATE_PATH_INPUTS } from "./buildCapture.js";
+import {
+ versionAt,
+ type DesktopStatePath,
+ type StateEnds,
+} from "./desktopState.js";
+import { diffStateViews } from "./stateChanges.js";
+
+/** Why a task draft holds no files, one code per reason; a new reason adds its code. */
+const EMPTY_DRAFT_CODES = [
+ "no_change",
+ "incomplete_capture",
+ "unowned_changes",
+ "uncheckable_changes",
+ "unreadable_changes",
+ "no_document",
+] as const;
+
+/** Why a draft is empty, or a file in it has few or no suggested checks: one code per reason. */
+export const DRAFT_NOTE_CODES = [
+ ...EMPTY_DRAFT_CODES,
+ ...FILE_NOTE_CODES,
+] as const;
+
+/** A draft's note as the draft writes it. */
+export const draftNoteSchema = z.strictObject({
+ code: z.enum(DRAFT_NOTE_CODES),
+ message: z.string(),
+ file: z
+ .string()
+ .optional()
+ .describe(
+ "by_position, view_state_only, long_text, cut_view, unreadable_view: the drafted file the note is about.",
+ ),
+ files: z
+ .array(z.strictObject({ file: z.string(), role: z.string() }))
+ .optional()
+ .describe(
+ "unowned_changes: changed home paths outside the owned paths, up to 5. uncheckable_changes: dconf databases whose app keys changed. unreadable_changes: dconf databases that changed but could not be read.",
+ ),
+ more: z
+ .int()
+ .nonnegative()
+ .optional()
+ .describe("unowned_changes: how many more changed."),
+ error: z
+ .string()
+ .optional()
+ .describe("unreadable_changes: why REA could not read the dconf database."),
+ elements: z
+ .array(z.string())
+ .optional()
+ .describe(
+ "by_position: tags of the elements no attribute names alone, up to 5.",
+ ),
+ pointers: z
+ .array(z.string())
+ .optional()
+ .describe(
+ "long_text: the elements whose text is too long to check whole; view_state_only: the noise checks kept. Up to 5.",
+ ),
+});
+
+/** One reason a task draft holds no files, with the facts it rests on. */
+export interface EmptyDraftNote {
+ readonly code: (typeof EMPTY_DRAFT_CODES)[number];
+ readonly message: string;
+ /**
+ * `unowned_changes`: the changed home paths outside the owned paths, with
+ * their roles, up to five. `uncheckable_changes`: the dconf databases whose
+ * application keys changed. `unreadable_changes`: the dconf databases that
+ * changed but could not be read.
+ */
+ readonly files?: readonly { readonly file: string; readonly role: string }[];
+ /** `unowned_changes`: how many more changed. */
+ readonly more?: number;
+ /** `unreadable_changes`: why REA could not read them. */
+ readonly error?: string;
+}
+
+/** Why a draft is empty, or why one of its files has thin suggestions. */
+export type DraftNote = EmptyDraftNote | FileNote;
+
+const MAX_NOTED_FILES = 5;
+/** Files a user may have made first, desktop services' last. */
+const ROLE_ORDER: Readonly> = {
+ document: 0,
+ application_state: 1,
+ unrelated: 2,
+ shared_desktop: 3,
+};
+
+const homeList = (paths: readonly string[]) =>
+ paths.map((path) => `~/${path}`).join(", ");
+
+type Version = DesktopStatePath["versions"][number];
+
+/**
+ * How a changed dconf database's application keys differ between two
+ * versions: why one could not be read, whether its application groups
+ * differ, or undefined when only other groups changed. A version kept without
+ * its bytes stays unread: the missing copy is not an empty key list.
+ */
+const dconfKeysChange = (
+ baseline: Version,
+ final: Version,
+): { readonly error: string } | "keys" | undefined => {
+ const unread = [baseline, final].find(
+ (version) =>
+ version.present &&
+ (version.sha256 === null ||
+ version.view_error !== null ||
+ version.format !== "dconf"),
+ );
+ if (unread !== undefined)
+ return {
+ error:
+ unread.sha256 === null
+ ? (unread.view_error ?? "content not retained")
+ : (unread.view_error ??
+ `read as ${unread.format ?? "an unknown format"}, not a dconf database`),
+ };
+ const keys = diffStateViews(
+ "dconf",
+ baseline.present ? baseline.view : null,
+ final.present ? final.view : null,
+ );
+ return keys.added_count + keys.removed_count + keys.changed_count > 0
+ ? "keys"
+ : undefined;
+};
+
+/**
+ * Why a draft of `drafted` files is thin: for a draft with files, each file's
+ * own notes; for one with none, that nothing changed between the seed and end
+ * checkpoints, that the files that changed lie outside the owned paths, that
+ * the app's keys in a dconf database changed, which no check reads, or that a
+ * dconf database changed that could not be read, so whether the app's keys
+ * did is unknown; and, whichever, when no file has the document role.
+ */
+export const draftNotes = (
+ files: ReadonlyMap,
+ facts: {
+ readonly drafted: readonly Pick[];
+ readonly owned: readonly string[];
+ /** The capture's dconf databases by home path, which `files` leaves out. */
+ readonly dconf: ReadonlyMap;
+ readonly ends: StateEnds;
+ readonly checkpoints: readonly string[];
+ /**
+ * False when the seed or end checkpoint stopped early or could not read
+ * part of the home, so an empty change list is not evidence of no change.
+ */
+ readonly observed: boolean;
+ },
+): readonly DraftNote[] => {
+ if (facts.drafted.length > 0)
+ return facts.drafted.flatMap(({ notes }) => notes);
+ const seed = facts.checkpoints[facts.ends.seed] ?? "the seed";
+ const end = facts.checkpoints[facts.ends.end] ?? "the end";
+ const changed = changedStateFiles(files, facts.ends)
+ .map(({ home, path }) => ({ file: home, role: path.role }))
+ .sort(
+ (left, right) =>
+ ROLE_ORDER[left.role] - ROLE_ORDER[right.role] ||
+ (left.file < right.file ? -1 : 1),
+ );
+ const settings: { file: string; role: string }[] = [];
+ const unread: { file: string; role: string; error: string }[] = [];
+ for (const { home, path, baseline, final } of changedStateFiles(
+ facts.dconf,
+ facts.ends,
+ )) {
+ const keys = dconfKeysChange(baseline, final);
+ if (keys === "keys") settings.push({ file: home, role: path.role });
+ else if (keys !== undefined)
+ unread.push({ file: home, role: path.role, error: keys.error });
+ }
+ const unresolvedFileChanges = [
+ ...files.values(),
+ ...facts.dconf.values(),
+ ].some((path) => {
+ const baseline = versionAt(path, facts.ends.seed);
+ const final = versionAt(path, facts.ends.end);
+ if (baseline === undefined || final === undefined) return true;
+ if (baseline.checkpoint_index === final.checkpoint_index) return false;
+ if (!baseline.present && !final.present) return false;
+ if (baseline.present !== final.present) return true;
+ return (
+ baseline.sha256 === null ||
+ final.sha256 === null ||
+ baseline.sha256 !== final.sha256 ||
+ final.format === "sqlite"
+ );
+ });
+ const notes: EmptyDraftNote[] = [];
+ if (!facts.observed)
+ notes.push({
+ code: "incomplete_capture",
+ message: `Checkpoint ${seed} or ${end} did not cover the whole home, so a change can be missing from the list. Walk it again before treating an empty list as no change.`,
+ });
+ if (
+ facts.observed &&
+ changed.length === 0 &&
+ settings.length === 0 &&
+ unread.length === 0 &&
+ !unresolvedFileChanges
+ )
+ notes.push({
+ code: "no_change",
+ message: `No file under the home changed between checkpoints ${seed} and ${end}: save the change in the app before taking ${end}.`,
+ });
+ if (
+ facts.observed &&
+ unresolvedFileChanges &&
+ changed.length === 0 &&
+ settings.length === 0 &&
+ unread.length === 0
+ )
+ notes.push({
+ code: "uncheckable_changes",
+ message: `File identities changed or could not be compared between ${seed} and ${end}, but the captured values do not yield a draftable change. Inspect the files and write an appropriate check.`,
+ });
+ if (changed.length > 0) {
+ const listed = changed.slice(0, MAX_NOTED_FILES);
+ const more = changed.length - listed.length;
+ notes.push({
+ code: "unowned_changes",
+ message: `${String(changed.length)} file(s) changed between ${seed} and ${end}, none under the paths the export owns (${homeList(facts.owned) || "none"}), so data/get serves none of them: ${listed.map(({ file, role }) => `~/${file} (${role})`).join(", ")}${more === 0 ? "" : `, and ${String(more)} more`}. If one is the app's document or data, capture again naming it with ${STATE_PATH_INPUTS}.`,
+ files: listed,
+ more,
+ });
+ }
+ if (settings.length > 0)
+ notes.push({
+ code: "uncheckable_changes",
+ message: `The app's keys in dconf (${homeList(settings.map(({ file }) => file))}) changed between ${seed} and ${end}, and no check can read them: checks read files under the owned paths, and the export only writes the app's keys as they were at ${seed} when the app first starts. Make the task's change one the app saves in a file.`,
+ files: settings,
+ });
+ if (unread.length > 0) {
+ const error = [...new Set(unread.map((entry) => entry.error))].join("; ");
+ notes.push({
+ code: "unreadable_changes",
+ message: `${homeList(unread.map(({ file }) => file))} changed between ${seed} and ${end}, but REA could not read it (${error}), so whether the app's keys there changed is unknown. When dconf-cli is missing, install it or set REA_DCONF_COMMAND, and draft again.`,
+ files: unread.map(({ file, role }) => ({ file, role })),
+ error,
+ });
+ }
+ if (![...files.values()].some((path) => path.role === "document"))
+ notes.push({
+ code: "no_document",
+ message: `REA recognized no file as the app's document: no app window's title showed a file's name or ~/ path, and no state path named one. When the app saves a document, name it with ${STATE_PATH_INPUTS}.`,
+ });
+ return notes;
+};
diff --git a/src/domain/desktop/appTaskDraftPositional.test.ts b/src/domain/desktop/appTaskDraftPositional.test.ts
new file mode 100644
index 0000000000..20bf6c0043
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftPositional.test.ts
@@ -0,0 +1,226 @@
+import { describe, expect, it } from "vitest";
+
+import type { JsonValue } from "../jsonValue.js";
+import { passing, xml, xmlDraft } from "./appTaskDraft.fixture.js";
+import { pairMembers, resolvePointer } from "./viewPointer.js";
+
+const FILE = "Documents/doc.xml";
+
+/** The members of the array at a pointer into an XML text's view. */
+const membersAt = (text: string, pointer: string): JsonValue[] => {
+ const members = resolvePointer(xml(text), pointer);
+ if (!Array.isArray(members)) throw new Error(`no array at ${pointer}`);
+ return members;
+};
+
+describe("draftTaskFiles for a change only position reaches", () => {
+ /** A sheet as an ODS spreadsheet keeps it: rows no attribute names, each cell's text in a paragraph, and a save counter. */
+ const sheet = (february: string, total: string, cycles: string) =>
+ `${cycles}
Month
Sales
February
${february}
Total
${total}
`;
+
+ it("checks the whole text of the ancestor above the step by position, by its name, and demotes the noise beside it rather than suggesting it", () => {
+ const before = sheet("21300", "59625", "1");
+ const after = sheet("23100", "61425", "2");
+
+ const file = xmlDraft(before, after);
+
+ expect(file.suggested).toEqual([
+ {
+ kind: "text_equals",
+ file: FILE,
+ pointer: "/root/children/[name=table][@name=Q1 Sales]",
+ value: "MonthSalesFebruary23100Total61425",
+ },
+ ]);
+ expect(passing(FILE, xml(before), file.suggested)).toEqual([false]);
+ expect(passing(FILE, xml(after), file.suggested)).toEqual([true]);
+ expect(file.notes.map(({ code }) => code)).not.toContain("view_state_only");
+ });
+
+ it("suggests no noise when position alone reaches the edit and no ancestor's text can take a check", () => {
+ // The table's text is past the limit a text check compares.
+ const long = "x".repeat(2100);
+ const wide = (cell: string, cursor: string) =>
+ `${cursor}
${long}${cell}
`;
+
+ const file = xmlDraft(wide("21300", "1"), wide("23100", "2"));
+
+ expect(file.suggested).toEqual([]);
+ expect(file.notes.map(({ code }) => code)).toEqual(
+ expect.arrayContaining(["by_position"]),
+ );
+ expect(file.notes.map(({ code }) => code)).not.toContain("view_state_only");
+ });
+});
+
+describe("identifiers an app numbers by position", () => {
+ /** A project plan as Planner saves it: tasks numbered 1..N in document order on every save. */
+ const task = (id: number, name: string, work: string) =>
+ ``;
+ const plan = (...tasks: [string, string][]) =>
+ `${tasks.map(([name, work], index) => task(index + 1, name, work)).join("")}`;
+ const SEED: [string, string][] = [
+ ["Requirements", "28800"],
+ ["Design", "86400"],
+ ["Launch", "28800"],
+ ];
+ const INSERTED: [string, string][] = [
+ ["Requirements", "28800"],
+ ["Design", "86400"],
+ ["User testing", "28800"],
+ ["Launch", "28800"],
+ ];
+ const tasksOf = (text: string) =>
+ membersAt(text, "/root/children/0/children");
+
+ it("pairs tasks by name once an insertion shifts the ids, so the new task is added rather than its neighbour renamed", () => {
+ const pairing = pairMembers(
+ tasksOf(plan(...INSERTED)),
+ tasksOf(plan(...SEED)),
+ );
+
+ expect(pairing.newer.keys).toEqual([
+ "[name=task][@name=Requirements]",
+ "[name=task][@name=Design]",
+ "[name=task][@name=User testing]",
+ "[name=task][@name=Launch]",
+ ]);
+ expect(pairing.newer.partners).toEqual([0, 1, undefined, 2]);
+ });
+
+ it("checks that the inserted task is there, by name, and notes its neighbour's shifted id as renumbered", () => {
+ const file = xmlDraft(plan(...SEED), plan(...INSERTED));
+
+ const tasks = "/root/children/[name=tasks]/children/[name=task]";
+ // The new task holds its neighbour's defaults, so only its presence says what the user did.
+ expect(file.suggested).toEqual([
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: `${tasks}[@name=User testing]/name`,
+ value: "task",
+ },
+ ]);
+ expect(file.values.map(({ pointer, note }) => [pointer, note])).toEqual([
+ [`${tasks}[@name=User testing]`, undefined],
+ [`${tasks}[@name=Launch]/attributes/id`, "renumbered"],
+ ]);
+ });
+
+ it("checks a deleted task is gone by its name, and never that the id its neighbour was renumbered from is", () => {
+ const file = xmlDraft(
+ plan(...SEED),
+ plan(...SEED.filter(([name]) => name !== "Design")),
+ );
+
+ const tasks = "/root/children/[name=tasks]/children/[name=task]";
+ expect(file.suggested).toEqual([
+ { kind: "value_absent", file: FILE, pointer: `${tasks}[@name=Design]` },
+ ]);
+ });
+
+ it("keeps ids as identity when two items only swap a colour, so the swap reads as two colour edits", () => {
+ const list = (first: string, second: string) =>
+ ``;
+ const [before, after] = [list("red", "blue"), list("blue", "red")];
+
+ expect(
+ pairMembers(
+ membersAt(after, "/root/children"),
+ membersAt(before, "/root/children"),
+ ),
+ ).toEqual({
+ newer: {
+ keys: ["[name=item][@id=1]", "[name=item][@id=2]"],
+ partners: [0, 1],
+ },
+ older: {
+ keys: ["[name=item][@id=1]", "[name=item][@id=2]"],
+ partners: [0, 1],
+ },
+ });
+ const file = xmlDraft(before, after);
+ const items = "/root/children/[name=item]";
+ expect(
+ file.values.map(({ pointer, note }) => [pointer, note ?? null]),
+ ).toEqual([
+ [`${items}[@id=1]/attributes/color`, null],
+ [`${items}[@id=2]/attributes/color`, null],
+ ]);
+ expect(file.suggested.map(({ kind }) => kind)).toEqual([
+ "value_equals",
+ "value_equals",
+ ]);
+ expect(file.notes).toEqual([]);
+ });
+
+ it("keeps ids that did not shift as identity, so a deletion leaves the rest paired by id", () => {
+ const stable = (...ids: number[]) =>
+ `${ids.map((id) => ``).join("")}`;
+ const items = (text: string) => membersAt(text, "/root/children");
+
+ expect(
+ pairMembers(items(stable(1, 3)), items(stable(1, 2, 3))).newer.keys,
+ ).toEqual(["[name=item][@id=1]", "[name=item][@id=3]"]);
+ });
+});
+
+describe("an element renamed", () => {
+ /** A painting's layer stack as MyPaint saves it: each layer by name, its pixels in `src`, the one selected flagged. */
+ const stack = (top: string) =>
+ `${[
+ [top, "00", "576", "true"],
+ ["Sky", "01", "0", "false"],
+ ["Background", "02", "0", "false"],
+ ]
+ .map(
+ ([name, layer, x, selected]) =>
+ ``,
+ )
+ .join("")}`;
+ const LAYER = "/root/children/[name=stack]/children/[name=layer]";
+
+ it("pairs a layer whose name alone changed, at its place among the layers, by another attribute naming it alone rather than the one selected", () => {
+ const layers = (top: string) =>
+ membersAt(stack(top), "/root/children/0/children");
+
+ const pairing = pairMembers(layers("Sun"), layers("Untitled"));
+
+ const keys = [
+ "[name=layer][@src=data/layer-00.png]",
+ "[name=layer][@name=Sky]",
+ "[name=layer][@name=Background]",
+ ];
+ expect(pairing).toEqual({
+ newer: { keys, partners: [0, 1, 2] },
+ older: { keys, partners: [0, 1, 2] },
+ });
+ });
+
+ it("checks the renamed layer's new name by the attribute that names it, and that the old name names nothing", () => {
+ const file = xmlDraft(stack("Untitled"), stack("Sun"));
+
+ expect(file.suggested).toEqual([
+ {
+ kind: "value_equals",
+ file: FILE,
+ pointer: `${LAYER}[@src=data~1layer-00.png]/attributes/name`,
+ value: "Sun",
+ },
+ {
+ kind: "value_absent",
+ file: FILE,
+ pointer: `${LAYER}[@name=Untitled]`,
+ },
+ ]);
+ expect(
+ [stack("Untitled"), stack("Sun"), stack("Moon")].map((text) =>
+ passing(FILE, xml(text), file.suggested),
+ ),
+ ).toEqual([
+ [false, false],
+ [true, true],
+ [false, true],
+ ]);
+ });
+});
diff --git a/src/domain/desktop/appTaskDraftRows.ts b/src/domain/desktop/appTaskDraftRows.ts
new file mode 100644
index 0000000000..99dece0d88
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftRows.ts
@@ -0,0 +1,164 @@
+import type { JsonValue } from "../jsonValue.js";
+import { isTimestamp, type DesktopTaskCheckInput } from "./appTask.js";
+import { countTaskRows } from "./appTaskChecks.js";
+import type { DesktopStatePath } from "./desktopState.js";
+import { sqliteRowIdentity } from "./stateChanges.js";
+
+type Version = DesktopStatePath["versions"][number];
+type Suggestion = DesktopTaskCheckInput;
+
+const isRecord = (
+ value: JsonValue | null | undefined,
+): value is Record =>
+ typeof value === "object" && value !== null && !Array.isArray(value);
+
+interface TableView {
+ readonly columns: readonly string[];
+ readonly key: readonly string[];
+ readonly rows: readonly (readonly JsonValue[])[];
+}
+
+const tableViews = (version: Version): ReadonlyMap => {
+ const tables = new Map();
+ const view = version.present ? version.view : null;
+ if (!isRecord(view) || !isRecord(view.tables)) return tables;
+ for (const [name, table] of Object.entries(view.tables)) {
+ if (!isRecord(table) || table.truncated !== false) continue;
+ const strings = (value: JsonValue | undefined) =>
+ Array.isArray(value)
+ ? value.filter((item) => typeof item === "string")
+ : [];
+ tables.set(name, {
+ columns: strings(table.columns),
+ key: strings(table.primary_key),
+ rows: Array.isArray(table.rows) ? table.rows.filter(Array.isArray) : [],
+ });
+ }
+ return tables;
+};
+
+/** Column values of a row, without volatile values or the columns left out. */
+const rowWhere = (
+ table: TableView,
+ row: readonly JsonValue[],
+ keep: (column: string) => boolean,
+): Record =>
+ Object.fromEntries(
+ table.columns.flatMap((column, index) => {
+ const value = row[index] ?? null;
+ return keep(column) && !isTimestamp(value) ? [[column, value]] : [];
+ }),
+ );
+
+/** Match uniquely keyed rows separately from multiset row conditions. */
+const keyedRows = (table: TableView) =>
+ new Map(
+ table.rows.flatMap((row) => {
+ const identity = sqliteRowIdentity(table.columns, table.key, row);
+ return identity.keyed ? [[identity.value, row] as const] : [];
+ }),
+ );
+
+/** The predicates a task can express after volatile values are omitted. */
+const rowConditions = (table: TableView) => {
+ const conditions = new Map>();
+ for (const row of table.rows) {
+ if (sqliteRowIdentity(table.columns, table.key, row).keyed) continue;
+ const where = rowWhere(table, row, () => true);
+ if (Object.keys(where).length === 0) continue;
+ conditions.set(JSON.stringify(where), where);
+ }
+ return conditions;
+};
+
+const unkeyedRowSuggestions = (
+ file: string,
+ table: string,
+ baseline: TableView,
+ final: TableView,
+): Suggestion[] => {
+ const conditions = new Map([
+ ...rowConditions(final),
+ ...rowConditions(baseline),
+ ]);
+ const suggestions: Suggestion[] = [];
+ for (const where of conditions.values()) {
+ const previous = countTaskRows(baseline.columns, baseline.rows, where);
+ const count = countTaskRows(final.columns, final.rows, where);
+ if (previous === count) continue;
+ if (count === 0)
+ suggestions.push({ kind: "row_absent", file, table, where });
+ else if (previous === 0)
+ suggestions.push({ kind: "row_present", file, table, where });
+ else
+ suggestions.push({
+ kind: "row_count",
+ file,
+ table,
+ where,
+ equals: count,
+ });
+ }
+ return suggestions;
+};
+
+/** Rows a database's tables gained, changed and lost, and new row counts, without volatile values. */
+export const rowSuggestions = (
+ file: string,
+ baseline: Version,
+ final: Version,
+): Suggestion[] => {
+ const before = tableViews(baseline);
+ const suggestions: Suggestion[] = [];
+ const push = (
+ kind: "row_present" | "row_absent",
+ table: string,
+ where: Record,
+ ) => {
+ if (Object.keys(where).length > 0)
+ suggestions.push({ kind, file, table, where });
+ };
+ for (const [name, table] of tableViews(final)) {
+ const old = before.get(name) ?? { ...table, rows: [] };
+ const oldRows = keyedRows(old);
+ const newRows = keyedRows(table);
+ for (const [id, row] of newRows) {
+ const previous = oldRows.get(id);
+ if (previous === undefined)
+ push(
+ "row_present",
+ name,
+ rowWhere(table, row, (column) => !table.key.includes(column)),
+ );
+ else if (JSON.stringify(previous) !== JSON.stringify(row))
+ push(
+ "row_present",
+ name,
+ rowWhere(
+ table,
+ row,
+ (column) =>
+ table.key.includes(column) ||
+ JSON.stringify(row[table.columns.indexOf(column)]) !==
+ JSON.stringify(previous[old.columns.indexOf(column)]),
+ ),
+ );
+ }
+ for (const [id, row] of oldRows)
+ if (!newRows.has(id))
+ push(
+ "row_absent",
+ name,
+ rowWhere(old, row, (column) => old.key.includes(column)),
+ );
+ suggestions.push(...unkeyedRowSuggestions(file, name, old, table));
+ if (old.rows.length !== table.rows.length)
+ suggestions.push({
+ kind: "row_count",
+ file,
+ table: name,
+ equals: table.rows.length,
+ });
+ }
+ return suggestions;
+};
diff --git a/src/domain/desktop/appTaskDraftScalars.ts b/src/domain/desktop/appTaskDraftScalars.ts
new file mode 100644
index 0000000000..2707589b03
--- /dev/null
+++ b/src/domain/desktop/appTaskDraftScalars.ts
@@ -0,0 +1,43 @@
+import type { JsonValue } from "../jsonValue.js";
+import type { DesktopTaskCheckInput } from "./appTask.js";
+import { numberOf } from "./appTaskChecks.js";
+
+/** How far a decimal the app writes as text may drift, such as `437.80000000000001` for 437.8, and still match. */
+export const DECIMAL_TOLERANCE = 0.001;
+
+/** A decimal string as the number it spells: digits, a point and digits, as both evaluators read a numeral; undefined for anything else. */
+export const decimalOf = (value: JsonValue | undefined): number | undefined => {
+ if (typeof value !== "string" || !/^-?\d+\.\d+$/u.test(value))
+ return undefined;
+ const number = Number(value);
+ return Number.isFinite(number) ? number : undefined;
+};
+
+/** Whether a decimal's number is within the tolerance of a baseline value, read as the evaluators read a number, so a check by number would pass there too. */
+const nearBaseline = (number: number, before: JsonValue | undefined) => {
+ const old = before === undefined ? undefined : numberOf(before);
+ return old !== undefined && Math.abs(old - number) <= DECIMAL_TOLERANCE;
+};
+
+/**
+ * A `value_equals` check of a value: a decimal string as its number within
+ * `DECIMAL_TOLERANCE`, unless `before`, the value the check must fail on, is
+ * within it too; anything else exactly.
+ */
+export const valueEqualsCheck = (
+ file: string,
+ pointer: string,
+ value: JsonValue,
+ before?: JsonValue,
+): DesktopTaskCheckInput => {
+ const number = decimalOf(value);
+ return number === undefined || nearBaseline(number, before)
+ ? { kind: "value_equals", file, pointer, value }
+ : {
+ kind: "value_equals",
+ file,
+ pointer,
+ value: number,
+ tolerance: DECIMAL_TOLERANCE,
+ };
+};
diff --git a/src/domain/desktop/appTaskDraftText.ts b/src/domain/desktop/appTaskDraftText.ts
index ff1b1bd806..335db32072 100644
--- a/src/domain/desktop/appTaskDraftText.ts
+++ b/src/domain/desktop/appTaskDraftText.ts
@@ -1,6 +1,14 @@
import type { JsonValue } from "../jsonValue.js";
-import { isTimestamp, type DesktopTaskCheckInput } from "./appTask.js";
+import type { DesktopTaskCheckInput } from "./appTask.js";
import { elementText, isXmlElement } from "./appTaskChecks.js";
+import {
+ holdsSplitDate,
+ isBookkeeping,
+ isStamp,
+ isVolatileValue,
+} from "./appTaskDraftNoise.js";
+import { valueEqualsCheck } from "./appTaskDraftScalars.js";
+import { rankAttributes } from "./attributeRank.js";
import type { DesktopStateDifference } from "./stateChanges.js";
import { resolvePointer } from "./viewPointer.js";
@@ -20,11 +28,15 @@ export interface TextPlacement {
/** The pointer a check names an element by, or undefined when the final view cannot decide a check there. */
readonly name: (pointer: string) => string | undefined;
readonly limit: { readonly suggestions: number; readonly chars: number };
+ /** Whether a change at a pointer is noise, which never makes an ancestor's whole text a check. */
+ readonly noise: (pointer: string) => boolean;
}
/** Text checks for the elements whose text changed, and the pointers those checks make redundant: text and whole runs holding text in them, or an element added whole checked itself, not an attribute or name of an element both ends hold, nor an element holding no text, as an image or a break. */
export interface WholeTextSuggestions {
readonly suggestions: readonly DesktopTaskCheckInput[];
+ /** Elements whose text changed that no check took, because every place for one held more text than the limit. */
+ readonly long: readonly string[];
readonly covers: (pointer: string) => boolean;
}
@@ -128,7 +140,14 @@ const byOwnAttribute = (element: string, ends: Ends): Candidate | undefined => {
const keys = Object.keys(attributes);
for (const key of [
...keys.filter(isIdentifier),
- ...keys.filter((key) => !isIdentifier(key)),
+ ...rankAttributes(
+ keys
+ .filter((key) => !isIdentifier(key))
+ .map((key) => {
+ const value = attributes[key];
+ return { key, value: typeof value === "string" ? value : "" };
+ }),
+ ).map(({ key }) => key),
]) {
const value = attributes[key];
if (
@@ -136,7 +155,7 @@ const byOwnAttribute = (element: string, ends: Ends): Candidate | undefined => {
attributesOf(before)[key] !== value ||
/[=[\]]/u.test(key) ||
value.includes("][") ||
- isTimestamp(value)
+ isStamp(value)
)
continue;
const selector = `[name=${after.name}][@${key}=${value}]`;
@@ -150,6 +169,53 @@ const byOwnAttribute = (element: string, ends: Ends): Candidate | undefined => {
return undefined;
};
+/** How far a text check climbs from an element no attribute names to an ancestor its tag names. */
+const MAX_TAG_CLIMB = 3;
+/** A formatting run a later save may split, merge, or drop while keeping the paragraph. */
+const RUN_TAG = /^(?:a|b|c|em|i|span|strong|tspan|u)$/u;
+
+/** The tag a diff pointer's last step names, without a namespace prefix. */
+const tagOf = (pointer: string) =>
+ (
+ /\[name=(?[^\][]+)/u.exec(pointer.split("/").at(-1) ?? "")?.groups
+ ?.name ?? ""
+ )
+ .split(":")
+ .at(-1) ?? "";
+
+/**
+ * Clean places for a text check, innermost first. A formatting run is left
+ * out when a paragraph or other element remains, so the check does not depend
+ * on a run a later save can drop.
+ */
+const withoutRuns = (
+ chain: readonly string[],
+ noisy: (element: string) => boolean,
+): string[] => {
+ const clean = chain.filter((element) => !noisy(element));
+ const content = clean.filter((element) => !RUN_TAG.test(tagOf(element)));
+ return content.length > 0 ? content : clean;
+};
+
+/** Whether an element's text, or a descendant's, is a time or a value the app makes anew. */
+export const holdsNoisyText = (
+ value: JsonValue | undefined,
+ bookkeeping = false,
+): boolean => {
+ if (typeof value === "string") {
+ const text = value.trim();
+ return (
+ text.length > 0 && (isStamp(text) || isVolatileValue(text, bookkeeping))
+ );
+ }
+ return (
+ isXmlElement(value) &&
+ value.children.some((child) =>
+ holdsNoisyText(child, bookkeeping || isBookkeeping(value)),
+ )
+ );
+};
+
/**
* Where a text check goes for an element whose text changed, best first, so
* the check names the paragraph, which a later save keeps, rather than a run
@@ -159,7 +225,10 @@ const byOwnAttribute = (element: string, ends: Ends): Candidate | undefined => {
* attribute, as AbiWord's `p` by its `xid`, where one is below the document's
* root. An element passed on the way that an identifier of its own names, the
* outermost first, goes ahead of that ancestor unless an identifier names the
- * ancestor too; one that only another attribute names goes after it.
+ * ancestor too; one that only another attribute names goes after it. Where no
+ * attribute names an ancestor, as MuseScore's `` in a ``, the
+ * element itself and then up to three ancestors, nearest first; `name` keeps
+ * only one its tag names, with no step by position.
*/
const outsideRuns = (pointer: string, ends: Ends): readonly Candidate[] => {
let current = pointer;
@@ -174,11 +243,51 @@ const outsideRuns = (pointer: string, ends: Ends): readonly Candidate[] => {
let named = current;
while (!namedByAttribute(named)) {
const parent = parentBelowRoot(named);
- if (parent === undefined)
- return [{ element: current, pointer: current, identifier: false }];
+ if (parent === undefined) {
+ const stopped = parentOf(named);
+ const chain = [...passed, named].filter(
+ (element, index, all) => all.indexOf(element) === index,
+ );
+ // A record directly under the document can still hold a save time. Leave a formatting run out when the paragraph remains.
+ if (
+ stopped !== undefined &&
+ holdsNoisyText(resolvePointer(ends.final, stopped))
+ ) {
+ const narrower = withoutRuns(chain, (element) =>
+ holdsNoisyText(resolvePointer(ends.final, element)),
+ );
+ if (narrower.length > 0)
+ return narrower.map((element) => ({
+ element,
+ pointer: element,
+ identifier: false,
+ }));
+ }
+ return [current, ...[...passed.slice(1), named].slice(0, MAX_TAG_CLIMB)]
+ .filter((element, index, all) => all.indexOf(element) === index)
+ .map((element) => ({ element, pointer: element, identifier: false }));
+ }
passed.push(named);
named = parent;
}
+ const noisy = (element: string) =>
+ holdsNoisyText(resolvePointer(ends.final, element));
+ // An ancestor whose text joins a timestamp fails a later save that only restamps it.
+ if (noisy(named)) {
+ const narrower = withoutRuns(
+ [...passed].filter(
+ (element, index, all) =>
+ all.indexOf(element) === index && element !== named,
+ ),
+ noisy,
+ );
+ if (narrower.length > 0)
+ return narrower.map((element) => ({
+ element,
+ pointer: element,
+ identifier: false,
+ }));
+ }
const ranked = [
{ element: named, pointer: named, identifier: namedByIdentifier(named) },
...passed
@@ -191,6 +300,92 @@ const outsideRuns = (pointer: string, ends: Ends): readonly Candidate[] => {
];
};
+/**
+ * Where a text check goes for an element only position reaches, as a
+ * spreadsheet cell in a row no attribute names: the ancestor above the
+ * outermost step by position, as the table holding the rows, by an attribute
+ * of its own that names it alone, as `[@table:name=Q1 Sales]`, and otherwise
+ * as the diff names it; none when there is no such step or that ancestor is
+ * the document's root.
+ */
+const abovePosition = (pointer: string, ends: Ends): readonly Candidate[] => {
+ const segments = pointer.split("/");
+ const step = segments.findIndex(
+ (segment, index) =>
+ segments[index - 1] === "children" && /^\d+$/u.test(segment),
+ );
+ if (step < 0) return [];
+ const element = segments.slice(0, step - 1).join("/");
+ if (parentOf(element) === undefined) return [];
+ const own = byOwnAttribute(element, ends);
+ return [
+ ...(own === undefined ? [] : [own]),
+ { element, pointer: element, identifier: false },
+ ];
+};
+
+/** Whether an element holds only elements, as a record holds its fields, layout whitespace aside. */
+const recordLike = (value: JsonValue | undefined): boolean =>
+ isXmlElement(value) &&
+ value.children.some((child) => isXmlElement(child)) &&
+ value.children.every(
+ (child) =>
+ isXmlElement(child) || (typeof child === "string" && isWhitespace(child)),
+ );
+
+/**
+ * A tag naming a paragraph or other piece of text, whose element children are
+ * runs of it: AbiWord's, HTML's and Word's `p` (`w:p`), ODF's `text:p` and
+ * `text:h`, HTML's `h1` to `h6` and `li`, DocBook's `para`, and SVG's `text`
+ * and `tspan`, or a span or link holding further runs. Compared as written,
+ * so MuseScore's `Text`, a record of `