diff --git a/.agents/skills/reverse-engineer-anything/references/desktop-build.md b/.agents/skills/reverse-engineer-anything/references/desktop-build.md index 5618a8ad9a..8ab1876a5a 100644 --- a/.agents/skills/reverse-engineer-anything/references/desktop-build.md +++ b/.agents/skills/reverse-engineer-anything/references/desktop-build.md @@ -183,6 +183,19 @@ the first source that has the app: only at a clean exit. It writes `.vym.lock` while a map is open, so close the map before `seed`. Press A only after clicking a branch on the canvas: keys typed in the tree editor run other shortcuts. +- KmPlot: Create → Cartesian Plot, then edit the formula in the Function + tab and press Return. It stores minus as U+2212 (`−`). Its file:// title + still names the file. +- RedNotebook keeps each month in `~/.rednotebook/data/YYYY-MM.txt`, YAML + read as text, folded at about column 80: pick check phrases the fold keeps + on one line. +- Qt5 apps saving with QDom, such as KmPlot, write attributes in a random + order in each process, so check their XML by element, never by whole line. +- Parley's title shows the collection's title, so name the file in + `state_paths`. A new collection starts with 15 blank entries. While a + document is open KDE keeps an autosave and its `.lock` in + `~/.local/share/stalefiles/parley/`, and there is no File → Close: before + `seed`, use File → New, then reopen the collection from Open Recent. - Sigil unpacks the open book into `.local/share/sigil-ebook/sigil/workspace/Sigil-XXXXXX/` and deletes it on exit; naming the whole `sigil-ebook` folder seeds that stale copy, so name @@ -197,6 +210,23 @@ the first source that has the app: - Dismiss first-run dialogs, such as a welcome screen, before `seed`. A choice the app saves only later, such as Inkscape's "show this every time", may not reach the seed, so expect the dialog in the witness too. +- Some apps write their settings late: Freeplane writes + `~/.config/freeplane/1.7.x/auto.properties` only once Preferences is + confirmed, and a seed without it reopens its documentation map. Before + `seed`, open the app's preferences and confirm them, then read a probe + checkpoint's `changed_files` to see the settings file written. +- Before `seed`, click into the document to clear any selection the app + would save: Gnote saves its cursor and selection with the note. +- An app whose state folder is named after it, such as + `~/.local/share/gnote`, or after it and its version, such as TreeLine's + `~/.treeline-3.1`, is found without `state_paths`. +- TreeLine: Ctrl+A adds a child node even while a field has focus. Clicking + a node already selected starts an inline rename, so Ctrl+X then cuts its + text: cut a node from its context menu. It saves nodes sorted by their + generated uid1, so their order changes from run to run. +- After an app crashes, capture again on a fresh computer, so the seed + carries no crash-recovery state, such as MyPaint's "Recover interrupted + work?". - Give what the task changes a stable name where the app has one, such as an id in Inkscape's Object Properties, so a check can select it. - Expect apps to split text into runs: Writer's revision ids split an @@ -204,6 +234,13 @@ the first source that has the app: `P1`, and AbiWord wraps each insertion in ``. Check such text whole with `text_equals` (see Write the task), and find the paragraph by something the edit keeps, such as AbiWord's `xid`. +- Freeplane's first launch opens a documentation map: close it with Ctrl+W. + Its Ctrl+N asks for a template. +- MyPaint 2.0.1 crashes on brush strokes on the build computer: build its + seed with flood fill and layer operations. +- GnuCash's Ctrl+S on a new book opens a Save dialog with no format choice: + use File > Save As to pick XML. Close the book (File > New File) before + `seed`, so its `.LCK` lock is not seeded. ## Build the seed and the edit @@ -257,7 +294,8 @@ the first source that has the app: - The app's own compressed format works when it is gzip, such as Gnumeric's `.gnumeric`, Dia's `.dia` or Inkscape's `.svgz`: REA reads the document inside, and checks point into it as into the same - document uncompressed. A format REA reads as binary, such as one + document uncompressed. Compression may be off by default: Dia's + "Compress diagram files" is unticked, which REA reads as plain XML. A format REA reads as binary, such as one compressed with xz, has no view: save in a format the app offers that REA reads, then reopen the file once and check that a plain Ctrl+S keeps that format. @@ -331,11 +369,15 @@ Write `{"name", "prompt", "checks"}` from the draft and what you did. `text_equals` or `text_contains`. Only XML files, archives, JSON and key files have values: a text file's or SQLite database's `change` already lists its lines and rows. - `refers_to` says what a number stands for, such as `category="44"` - naming ``; use it for the prompt's words. + `refers_to` says what a number or id stands for, such as `category="44"` + naming ``, or a new KMyMoney + transaction's splits naming their account and payee; use it for the + prompt's words. `alternatives` gives a changed value's pointer by the + element's other names, such as a label beside an id, to pick another key + without another export. Skip entries whose `note` is `by_position` (no check can use that pointer) or noise (`view_state`, `renumbered`, `volatile`, - `timestamp`, `upkeep`). `values_cut` means a limit left some out. + `timestamp`, `upkeep`, `layout`). `values_cut` means a limit left some out. - Select elements by what they hold, never by position: SVG text by id, `/root/children/[name=g][@inkscape:label=Layer 1]/children/[name=text][@id=headline]/children/[name=tspan]/children/0`, or an ODF paragraph by style, @@ -358,8 +400,10 @@ Write `{"name", "prompt", "checks"}` from the draft and what you did. unchanged element is in `draft.guards` when it is a like sibling of a changed one, and otherwise use the app's usual numbering, such as another paragraph's `xid`, and let the export confirm it. -- In an ODS spreadsheet no attribute names a row or a cell. Name the sheet - in the seed, and check its whole text: `text_equals` on +- In an ODS spreadsheet no attribute names a row: the draft names it by its + label cell, `[name=table:table-row][>table:table-cell=February]`. Where no + cell names it alone, name the sheet in the seed, and check its whole + text: `text_equals` on `/entries/[name=content.xml]/view/root/children/[name=office:body]/children/[name=office:spreadsheet]/children/[name=table:table][@table:name=Q1 Sales]`, or a stretch of it with `text_contains`. Cell texts join with nothing between, so a row reads `February23100`. @@ -367,6 +411,9 @@ Write `{"name", "prompt", "checks"}` from the draft and what you did. draft compares the whole text with `text_equals`. This rejects longer or negated items, but can reject a correct edit with different ordering or spacing. Report that limitation when the task permits such variations. +- For a note placed by its own `x` and `y`, such as Xournal++'s `` + on a layer, the draft checks its text on the layer with `text_contains`, + which passes wherever the user puts it, and guards each note it kept. - Some apps keep text in an attribute, one element per run, such as Scribus's ``: add `"text_attribute": "CH"` to `text_equals` or `text_contains` on the frame, diff --git a/bridge/agentenv/capture.py b/bridge/agentenv/capture.py index 599bd2e7f0..67679331e4 100644 --- a/bridge/agentenv/capture.py +++ b/bridge/agentenv/capture.py @@ -190,16 +190,32 @@ def start(arguments: dict) -> int: argv += ["--package-file", arguments["package_file"]] # Reading and writing, so the capture never reads the end of its input between requests. reader = os.open(control, os.O_RDWR) + process = None try: - with open(os.path.join(capture, "capture.out"), "wb") as out, \ - open(os.path.join(capture, "progress.log"), "wb") as progress: - process = subprocess.Popen( - ["sh", "-c", 'done=$1; shift; "$@"; code=$?; printf "%s" "$code" > "$done.tmp" && mv "$done.tmp" "$done"', - "sh", os.path.join(capture, "exit"), *argv], - cwd=os.path.expanduser("~"), env=_environment(work), stdin=reader, stdout=out, stderr=progress, - start_new_session=True) + try: + with open(os.path.join(capture, "capture.out"), "wb") as out, \ + open(os.path.join(capture, "progress.log"), "wb") as progress: + process = subprocess.Popen( + ["sh", "-c", 'done=$1; shift; "$@"; code=$?; printf "%s" "$code" > "$done.tmp" && mv "$done.tmp" "$done"', + "sh", os.path.join(capture, "exit"), *argv], + cwd=os.path.expanduser("~"), env=_environment(work), stdin=reader, stdout=out, stderr=progress, + start_new_session=True) + except Exception as cause: + # Nothing will write exit. Record one and drop the FIFO, so stop does not wait on it. + try: + with open(os.path.join(capture, "exit"), "w", encoding="utf-8") as handle: + handle.write("1") + except OSError: + pass + try: + os.unlink(control) + except OSError: + pass + return _say({"error": f"the capture did not start: {cause}"}) finally: os.close(reader) + if process is None: + return _say({"error": "the capture did not start"}) waited = _wait(capture, 0, {"window_open"}, None, arguments.get("timeout_s", 120)) return _say({"pid": process.pid, "apt": apt, **waited}) diff --git a/docs/desktop-application-capture.md b/docs/desktop-application-capture.md index 7f02e916b1..b7cca3554c 100644 --- a/docs/desktop-application-capture.md +++ b/docs/desktop-application-capture.md @@ -328,7 +328,10 @@ dot-folder named after the package, the executable, the desktop entry's full id and its last part, and the entry's name, with whether each existed at the start. A folder in those places whose name is one of these in another letter case joins them, such as GIMP's `~/.config/GIMP` for the `gimp` package; only -the whole name is compared, so `~/.config/GIMPshop` does not. Dotfiles in the +the whole name is compared, so `~/.config/GIMPshop` does not. A file under a +folder the application creates during the capture beside one of them, with +the same name in another letter case or with a version added (`-3.1` or +`3`), as TreeLine's `~/.treeline-3.1`, is its state too. Dotfiles in the home named after the application (`.rc`, or `.` followed by `-`, `.` or `_`, for a name of three or more characters) that exist at the start join them, since an application may only read such a diff --git a/docs/desktop-application-export.md b/docs/desktop-application-export.md index 3665ec745b..2dc594fa76 100644 --- a/docs/desktop-application-export.md +++ b/docs/desktop-application-export.md @@ -160,7 +160,8 @@ The class declares: README and the result's limitations say the sandbox is off. - `paths`: the state directories holding the application's state files and the folders holding its documents, nested folders collapsed. A folder beside a - state directory whose name differs from it only in letter case is owned + state directory whose name differs from it only in letter case, or that + adds a version, as TreeLine's `~/.treeline-3.1` beside `~/.treeline`, is owned whole in the files' spelling, so a `~/.config/GIMP` profile GIMP created during the capture beside the candidate `~/.config/gimp` is owned as `.config/GIMP`, not as the folder of its first file. The home itself and @@ -355,8 +356,10 @@ export removes them from what the bundle ships: example and template files an app copies on first run do not bury the document scrubbed; one already in that form ships as captured. The class copies it at boot as it is, so the home it holds stays `/home/user`. One that does not - decompress ships as captured, its bytes searched, and one the state view did - not read is only searched. One whose document needs no edit but a member of + decompress within the scrub limit ships as captured, its bytes searched. + A document the state view did not read can still be decompressed and + recompressed under the separate 256 MiB scrub limit; its content is searched + even when its format cannot be scrubbed. One whose document needs no edit but a member of which has an extra field, where BGZF (`.vcf.gz`, `.bam`) and dictzip (`.dict.dz`) files index their blocks, also ships as captured, its bytes searched, so its blocks stay where its index says; one whose document diff --git a/docs/desktop-application-state.md b/docs/desktop-application-state.md index 36cceb829f..b069f913d6 100644 --- a/docs/desktop-application-state.md +++ b/docs/desktop-application-state.md @@ -85,7 +85,7 @@ whether the file was present, the SHA-256 of its retained bytes, and a view: | `dconf` | The `GVariant` header | Groups whose path names the application, plus the names of the other groups | | `zip` | A zip local file header, `PK\x03\x04`, whatever the name | Its file entries in archive order, each with its own format and view | | `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels | -| `xml` | An XML name or declaration that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read | +| `xml` | An XML name, a declaration or a leading element that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read | | `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB | | `text` | Other UTF-8 text | Lines, up to 5,000 | | `binary` | NUL bytes or invalid UTF-8 | None | @@ -93,6 +93,13 @@ whether the file was present, the SHA-256 of its retained bytes, and a view: A gzip file is read as the document inside it; see [compressed documents](#compressed-documents). +Text that opens with an element, such as a Freeplane map's +`` with no declaration, or with a DOCTYPE that +has no internal subset and then an element, such as a KmPlot plot's +`` before ``, is XML when the whole text +is well formed, whatever the file is called, a `.txt` file included; +otherwise it stays text. The exported verifier makes the same choice. + A JSON value holding a number JavaScript would round, such as an integer above 2^53, is not read rather than shown rounded. diff --git a/docs/desktop-application-tasks.md b/docs/desktop-application-tasks.md index 9142c7fc14..486a38cfb0 100644 --- a/docs/desktop-application-tasks.md +++ b/docs/desktop-application-tasks.md @@ -58,12 +58,34 @@ operation `describe_desktop_application_task` and predicate type JSON file's members or a key file's keys, each whole once. A value only position reaches has `note: "by_position"`, and its pointer cannot go into a check; noise an app writes as it saves has `view_state`, `renumbered`, - `volatile`, `timestamp` or `upkeep` (see `task`). `refers_to` names up to - three other elements a value equals the `key` or `id` of, with the + `volatile`, `timestamp`, `upkeep` or `layout` (see `task`). A record of + text fields added to a list of its kind, as GnuCash's + ``, is listed field by field, and a field that lists + records, as its ``, record by record, so the cut never hides + one. `refers_to` names up to + five other elements a value equals the `key` or `id` (`xmi:id` among them) + of, outside a view the app keeps of the document, as Umbrello's + `listview` mirroring its model by id, with the `attribute` holding it, as HomeBank's `category="44"` names - ``; where several elements share that key, - the one whose tag shares the attribute's stem, as `payee` and ``. What a check can - name comes first, then what only position reaches, then noise. A file lists + ``, or the id a child element holds, as a + GnuCash split's `` names the account whose `` it + is, shown with its name, `Groceries…`; + an element added whole lists those of the records of attributes it nests + too, as a KMyMoney transaction's splits name their `` and + ``, each once. Where several elements share that key, the one + whose tag shares the attribute's stem, as `payee` and ``, and of + those the one named for the attribute, as `account` an `` before + an ``. A plain number names an element only through an + attribute so named or ending in `id`, `ref` or `key`, so a list's + `count="5"` names no ``. A value that changed lists, + in `alternatives`, its pointer naming the element by each other identifier + or name-like attribute it holds alone among its tag's members, up to + three, as an Inkscape text's `[@inkscape:label=Headline]` beside its + `[@id=text12]`, for a check to name it by instead; an element its tag + alone names inside the first one a condition names, as an EPUB chapter's + `h1`, by its `[@id=chapter-two]`. What a check can + name comes first, text an edit changed before the rest, so no cut drops + the edit, then what only position reaches, then noise. A file lists at most 20 values and 8 KB, the draft 24 KB, and `values_cut` says when a limit left some out. Values cover XML, archives, JSON and key files; a text or SQLite file's change and suggestions already carry its lines and rows. @@ -74,8 +96,31 @@ operation `describe_desktop_application_task` and predicate type in `.log` remain eligible for checks. A text file gains `lines_present` for its new lines and `lines_absent` for lines it lost (when either is at most ten), and `line_count` for each line - holding a letter whose count changed; JSON and key files gain - `value_equals` and `value_absent` by pointer; SQLite tables gain rows added, + holding a letter whose count changed; a new line holding a value REA + recognizes as generated, such as a named epoch time or `ID_329023692`, gets a + `line_count` of the lines `contains` its text up to that value, as + `CHILD=VALUE]`, a name-like tag first, as Osmo's + record by `[>first_name=Delphine]`, MuseScore's `` by + `[>style=Title]`, a spreadsheet row by its label cell, or GnuCash's + transaction by `[>trn:description=Greenleaf Market]`: text of at most 64 + characters, never a time or a token or id a program generates. A pair + counts every member holding both its values, whichever attributes that + member ranks first. An element with identifiers is named by those alone; + an id the app generates, as Freeplane's `ID_329023692`, or a GUID in + any attribute, braced or not, as VYM's `uuid="{18eadb5d-…}"`, ranks + last, after a child's text, so a new node is named by its `TEXT` and a + new VYM branch by `[>heading=Sunset boat tour]`, while a member + both versions hold pairs by such an id first, so records that keep their + UUIDs and swap titles read as two title edits. A value that stands for + none, as Grisbi's `Id="(null)"` on every transaction, names nothing, so a + new transaction is named by its `Nb`. A child's text is + counted on every sibling, one an identifier names included, so a title + two records share names neither. The selector must name the + member alone at both ends, with the same values; a child's text names a + member that is in one version only when the list's length changed, so a + row whose one cell an edit changed stays that row. An element whose one + identifier changed, at its place among its tag's members, with every + other attribute and its children alike, is that element renamed, as a + MyPaint layer renamed from `Layer` to `Sun`: it is paired by its first + other attribute naming it alone at both ends, as + `[@src=data/sun.png]`, so the rename reads as its `name` changed, and + the old name gets a check that it names nothing, unless its change is + noise, as an id the app renumbered. Where that attribute's values number + the members in document order, as MyPaint 2 writes `src` as + `data/layer-00.png`, `data/layer-01.png` and on along the stack and + renumbers them when a layer moves, each version is named by its own + value of the renamed identifier instead, `[@name=Sun]` and + `[@name=Layer]`. + - A child of the document's root that its tag alone names, and that holds + a list (two or more elements of one tag and no text of its own) and has + an identifier, also gets a condition that keeps it named if the app adds + a sibling of its tag, as Dia adds an empty `Background` layer each time + it reopens and saves a diagram: an id other than `name` both ends hold, + as Inkscape's `[name=g][@id=layer1]`, which a new sibling would not + share; else the text of a child both ends hold, as Dia's + `[name=dia:layer][>dia:object=#Receive order#]`, which an empty sibling + lacks; else its `name`, which a new one may repeat. Deeper, a member of + a list named for its tag, as Gnumeric's `gnm:Sheet` in `gnm:Sheets`, + holding elements and its tag alone naming it, gets an id, else a + name-like child's text, `[name=gnm:Sheet][>gnm:Name=Budget]`, else its + `name`. Other steps, and elements with neither, stay as they are. An identifier the app numbers by position names nothing: where the members of a tag hold it as 1..N (or 0..N-1) in document order at both ends, members were inserted or removed, and every member another @@ -107,11 +195,28 @@ operation `describe_desktop_application_task` and predicate type the count did, one at least, as Planner's task `id` after a task is inserted, members are paired by their other attributes, so an insertion reads as one task added. A value two members swap, or an id appended at - the end, as Tellico's next ``, leaves the ids identity. + the end, as Tellico's next ``, leaves the ids identity. An + identifier numbering the members 0..N-1 in document order at both ends, + where members came or went, as Parley numbers its `` by + where it saves it, ranks after a child's text where a member has one, so + a new entry is `[>translation=library]` and its kept neighbours are named + alike; Tellico's ids, which count from 1, Tiled's and Rosegarden's, + whose members hold no text, stay identity. An element a change added that + holds nothing but the id of a record it added elsewhere, as Parley's + lesson membership ``, is `upkeep`, listed with `refers_to` + on that record, never a check or a count. - The text check names the paragraph rather than a run inside it: it goes up from an element sitting in its parent's text, and from one named by its tag or position alone to the nearest ancestor named by an attribute, as - AbiWord's `p` by its `xid`; where no attribute names one, to the nearest + AbiWord's `p` by its `xid`, unless it is a field its tag names in a + record of two or more fields holding text, as QElectroTech's `` + beside `` in a ``, or KmPlot's + `` beside ``, which is checked itself rather than + the record whose text joins its fields; one line of text the app lays + out between line breaks, as KmPlot's formula before the next element, is + checked, and its siblings guarded, with `normalize_whitespace: true`, + and text whose whitespace alone changed, as a formula the app moved to + an indented line of its own, gets no check; where no attribute names one, to the nearest ancestor within three levels its tag names with no step by position, as MuseScore's `VBox` around untitled `` elements; and where only position reaches the element, to the ancestor above the outermost step by @@ -124,7 +229,19 @@ operation `describe_desktop_application_task` and predicate type number instead. Free text that gains a line, such as a shopping list, uses `text_equals` on the whole text: the current XML checks cannot compare an unstructured line as a whole item. This can reject a correct - edit that puts the item elsewhere or changes spacing. A record added + edit that puts the item elsewhere or changes spacing. Text an app keeps in an attribute, one element per run, as Scribus's + ``, gets a `text_equals` + with `text_attribute` on the nearest ancestor an attribute names, as + `[name=PAGEOBJECT][@ANNAME=Headline]` with `"text_attribute": "CH"`, where + that ancestor holds no text nodes and the attribute's holders are runs, + none inside another and none carrying an identifier or a name-like + attribute beside it, as Umbrello's ``, a record whose attributes are + checked one by one, two or more or one holding words with a space: an + element's own attribute, as a Freeplane node's `TEXT` beside the nodes in + it, or a lone value, as ``, is checked where it is. + The check covers that attribute and the runs holding it; another change + in the frame, as a run's `FONT`, keeps its own check. A record added or removed whole, an element holding only elements, its fields, such as Tellico's `` or a record of repeated `` elements, never makes the list holding records of its kind a check, whose text would hold @@ -132,49 +249,136 @@ operation `describe_desktop_application_task` and predicate type position names, whose absence only the list's text shows. Text made of runs is no record: an element whose tag names a paragraph or a piece of text (`p`, Writer's `text:p` and `text:h`, `h1` to `h6`, `li`, `para`, - SVG's `text` and `tspan`, a `span` or an `a`), or whose runs split a - sentence at a space, as `Meet ` and a linked `Monday`. Nor does an - element holding a date the app stamps in parts take a text check. + SVG's `text` and `tspan`, a `span` or an `a`), one inside such an + element, as Word's `` run in its ``, one whose elements are all + inline formatting (`b`, `i`, `u`, `em`, `strong`, `bold`, `italic`, a + `span`, `a`, `code`, `sub`, `sup` and the like, Gnote's `size:` and + `link:` runs), as `olive oil`, or + whose runs split a sentence at a space, as `Meet ` and a linked + `Monday`. Nor does an element holding a date the app stamps in parts + take a text check. + - An element added to a list, where another element of its tag is at either end, gets a `value_equals` for each of its attributes, best ranked first, at most six, leaving out its selector's attributes, identifiers, times, - volatile values and view state, as HomeBank's new `` gets its - `amount`, `category` and `payee`. An element whose other attributes, two + volatile values, view state and values standing for none, as HomeBank's + new `` gets its `amount`, `category` and `payee`; a value two or + more like siblings all hold, as Grisbi's `Cu="1"`, comes after those + that vary, as its `Pa`, `Ca`, `Dt` and `Am`. Records of attributes + alone that it nests, as a KMyMoney `` its `` of + ``, get a `value_equals` of each attribute whose values vary among + the like records, its own and those under its like siblings, at most + three a record and six in all, as each split's `value` and `payee`, but + not one every split holds alike, as `price="1/1"`, nor one each holds + equal to an earlier one, as `shares` repeats `value`. Each record is + named by an identifier that the app neither generates nor numbers anew + in every record holding such a list, as KMyMoney's `S0001` in every + transaction, else by its best ranked attribute short of a coordinate or + an amount, as `[name=SPLIT][@account=A000001]`. An element whose other attributes, two or more, a like sibling holds every one of, as a new Planner task copies its neighbour's defaults, and any other element added whole, get a check - that it exists. A record added to a list that a selector names, as - Tellico's ``, gets a `text_equals` for each field, at most - five, each by its selector among the record's fields, as `[name=title]`, - leaving out a field repeating the record's identifier, a date the app - stamps in parts, and blank text or times. A record only position names, - such as Osmo's `` of fields, also gets a `text_contains` for each - new field of three or more characters, at most five, at the nearest - ancestor a check can name, each absent from that ancestor's text at the - seed. + that it exists; of those copied, an attribute naming an element the seed + holds by an id that is no plain number, as an Umbrello attribute's + `type` its data type, is the user's choice and keeps its check. A record added to a list that a selector names, as + Tellico's `` or Osmo's `` by its first name, gets + a `text_equals` for each field, at most five, each by its selector among + the record's fields, as `[name=title]`, leaving out a field repeating the + record's identifier, the field its selector names it by unless it is + the only one left, as a new VYM branch's ``, a value every + like record holds in that field, as Osmo's `None`, noise, + a field holding ids of its own, a date the app stamps in parts, and blank + text or times; a field that lists records, as GnuCash's ``, + gets its records' fields instead, each record by its own selector. An + attribute every like record holds alike, as GnuCash's `version`, is the + format's and gets no check. A record only position names also gets a + `text_contains` for each new field of three or more characters, at most + five, at the nearest ancestor a check can name, each absent from that + ancestor's text at the seed. + - Where members of a list were added or removed, a `member_count` of the + list's members of their tag, and of their kind where they share one + (`type`, `kind` or `shape`, as `[name=dia:object][@type=Flowchart - Box]`), + unless every member of the tag holds a value of its own, as Tiled's + objects a free-form `type` (`well`, `door`, `gate`), which labels each + rather than grouping them, + equal to the end's number, which differs from the seed's: at most two a + file, the shallowest lists first, after the other checks. None for a + list of one member at both ends, as a style an app adds for a run, for + members of an element a text check reads, which are runs of its text, + for an element in mixed content, or for noise. - A decimal written as text, such as `146.83`, is suggested as a number - with `tolerance: 0.001`, unless the seed's value is that close too. + with `tolerance: 0.001`, unless the seed's value, read as the evaluator + reads a number (`10` and `1e1` included), is that close too. A + `value_equals` that would pass on the seed's value is not suggested. + - An element placed by its own position, holding numeric `x` and `y` as a + like sibling does, as Xournal++ places each `` note on a layer, is + an item of its own, never a run of the others' text: one added whole + gets a `text_contains` of its text on the element holding the items, + which passes wherever it is placed, and that list's `member_count`. - Noise leaves the suggestions when the file's change holds anything else, a change only position reaches included: view state, which a tag - or attribute on the way names (the cursor, a selection, scrolling, zoom, - the viewport, windows, splitters, Inkscape's `namedview`, view and + or attribute on the way names (the cursor, a selection, as VYM's + `${selected}`; + const SEED = map("4", "", "mc:0,bo:2"); + const END = map( + "5", + branch("18eadb5d-04fc-4448-a0f9-1f9a3dedaff7", "Sunset boat tour"), + "mc:0,bo:1", + ); + const AGENDA = + "/root/children/[name=mapcenter]/children/[name=branch][@uuid={4cdcd61f-90fb-4965-91a1-849509829d26}]/children"; + + it("names the new branch by its heading, not the GUID VYM made for it, and checks that heading and the branches' count", () => { + const file = xmlDraft(SEED, END); + + expect(file.suggested).toEqual([ + { + kind: "text_equals", + file: "Documents/doc.xml", + pointer: `${AGENDA}/[name=branch][>heading=Sunset boat tour]/children/[name=heading]`, + value: "Sunset boat tour", + }, + { + kind: "member_count", + file: "Documents/doc.xml", + pointer: AGENDA, + matching: "[name=branch]", + equals: 3, + }, + ]); + }); + + it("notes the saved selection and a tree editor setting as view state and the branch count as upkeep, the new heading first", () => { + const file = xmlDraft(SEED, END); + + expect( + file.values.map(({ pointer, note }) => [ + pointer.replace(AGENDA, "~"), + note ?? null, + ]), + ).toEqual([ + [ + "~/[name=branch][>heading=Sunset boat tour]/children/[name=heading]", + null, + ], + ["/root/children/[name=setting]", "view_state"], + ["/root/attributes/branchCount", "upkeep"], + ["/root/children/[name=select]/children/0", "view_state"], + ]); + }); +}); + +describe("draftTaskFiles counts an app keeps of its records", () => { + it("notes a count in an element's text that equals its records, as GnuCash's count-data, as upkeep, never a check or a guard", () => { + const book = (count: string, added: string) => + `2${count}${added}`; + + const file = xmlDraft( + book("1", ""), + book("2", ''), + ); + + const count = + "/root/children/[name=gnc:count-data][@cd:type=transaction]/children/0"; + expect(file.values.find(({ pointer }) => pointer === count)).toMatchObject({ + note: "upkeep", + }); + expect(file.suggested.map(pointerOf)).not.toContain(count); + expect( + file.guards.map(pointerOf).filter((at) => at?.includes("count-data")), + ).toEqual([]); + }); + + it("checks a count the user entered, which tallies no tag at both ends, though it equals how many of a tag its record holds", () => { + const stock = (widgets: string) => + `Widget${widgets}Gadget5`; + + const file = xmlDraft(stock("3"), stock("1")); + + const count = file.values.find(({ after }) => after === "1"); + expect(count).toBeDefined(); + expect(count?.note).toBeUndefined(); + expect(passing(FILE, xml(stock("3")), file.suggested)).toEqual([false]); + expect(passing(FILE, xml(stock("1")), file.suggested)).toEqual([true]); + }); + + it("suggests no check of the count a new element keeps of its members, as FreeCAD's ObjectDeps of its Deps", () => { + /** Objects as FreeCAD saves them: each one's dependencies, by name, then the objects. */ + const model = (count: string, deps: string, objects: string) => + `${deps}${objects}`; + + const file = xmlDraft( + model("1", "", ""), + model( + "3", + '', + '', + ), + ); + + // Count="2" counts the two Deps it holds; Count="0" another object's none, which no tag tallies. + expect(file.suggested.map(pointerOf)).not.toContain( + "/root/children/[name=Objects]/children/[name=ObjectDeps][@Name=PlateWithHole]/attributes/Count", + ); + }); +}); + +describe("draftTaskFiles styles a user applies", () => { + const NS = 'xmlns:office="o" xmlns:text="t" xmlns:style="s"'; + /** A Writer document: its automatic styles, and named sections each holding a paragraph in a named style. */ + const minutes = (quote: string, next: string) => + `MinutesShip it, said Ana.${next}`; + + it("checks a named style the user gave a paragraph beside another edit, never upkeep, as it would an automatic one the app renumbers", () => { + const seed = minutes("Standard", "Next meeting Friday."); + const end = minutes("Quotations", "Next meeting Monday."); + const file = xmlDraft(seed, end); + + const style = file.values.find(({ after }) => after === "Quotations"); + expect(style).toBeDefined(); + expect(style?.note).toBeUndefined(); + expect( + file.suggested.some( + (check) => "value" in check && check.value === "Quotations", + ), + ).toBe(true); + // A solve that edits the date alone fails. + expect( + passing( + FILE, + xml(minutes("Standard", "Next meeting Monday.")), + file.suggested, + ), + ).toContain(false); + }); +}); + +describe("draftTaskFiles for a name shared by two lists", () => { + it("checks a category added beside a template of the same name", () => { + const file = xmlDraft( + '', + '', + ); + + expect( + file.values.some( + (value) => + value.pointer.includes("category") && value.note === "upkeep", + ), + ).toBe(false); + expect(file.suggested.map(pointerOf).join(" ")).toContain("category"); + }); +}); + +describe("draftTaskFiles for a drawing resized beside a rename", () => { + const model = (name: string, width: string) => + `${name}`; + + it("checks the width the user changed when the only other edit is a name outside the drawing", () => { + const file = xmlDraft(model("Shop", "10"), model("Store", "20")); + const width = file.suggested.find( + (check) => "pointer" in check && check.pointer.endsWith("width"), + ); + + expect(file.suggested.map(pointerOf)).toContain( + "/root/children/[name=diagram]/children/[name=widget][@width=20]/name", + ); + expect( + file.values.some( + (value) => value.pointer.includes("width") && value.note === "layout", + ), + ).toBe(false); + expect(passing(FILE, xml(model("Store", "10")), file.suggested)).toContain( + false, + ); + expect( + passing(FILE, xml(model("Store", "20")), file.suggested).every(Boolean), + ).toBe(true); + }); +}); + describe("draftTaskFiles when a sibling lock changed", () => { const text = (line: string) => ({ lines: [line], diff --git a/src/domain/desktop/appTaskDraftNoise.ts b/src/domain/desktop/appTaskDraftNoise.ts index cc1dbd2f91..fa9842f9fb 100644 --- a/src/domain/desktop/appTaskDraftNoise.ts +++ b/src/domain/desktop/appTaskDraftNoise.ts @@ -1,11 +1,14 @@ import type { JsonValue } from "../jsonValue.js"; import { isTimestamp } from "./appTask.js"; import { + isAutomaticStyle, isIdentifyingAttribute, isNameLikeKey, isViewStateName, VIEW_STATE, } from "./attributeRank.js"; +import { isMemberCount } from "./appTaskDraftMemberCount.js"; +import { idReferencesOf, mirrorsOf } from "./appTaskDraftMirrors.js"; import { isPositionalKey, positionalKeys } from "./positionalKeys.js"; import type { DesktopStateDifference } from "./stateChanges.js"; import { elementText, isXmlElement, resolvePointer } from "./viewPointer.js"; @@ -17,6 +20,7 @@ export const NOISE_REASONS = [ "volatile", "timestamp", "upkeep", + "layout", ] as const; export type NoiseReason = (typeof NOISE_REASONS)[number]; @@ -34,25 +38,24 @@ const isViewStateValue = (value: string): boolean => const namesElement = (key: string) => isNameLikeKey(key) || /^(?:[^:]*:)?key$/iu.test(key); -/** An attribute an app renumbers as it saves: a generated id such as Gnumeric's `ExprID` or Scribus's `ItemID`, or LibreOffice's `rsid`. */ -export const isRenumberedKey = (key: string): boolean => { - const local = key.split(":").at(-1) ?? key; - return ( - !isIdentifyingAttribute(key) && - /(?:[a-z]I[dD]|(?:^|[-_])[rR][sS][iI][dD])$/u.test(local) - ); -}; - const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; +export const isUuid = (value: string): boolean => UUID.test(value); /** Seconds since 1970 of 2000-01-01 and 2100-01-01: an epoch time a save stamps falls between. */ const EPOCH_RANGE = [946_684_800, 4_102_444_800] as const; -/** UUIDs and epoch-shaped values in named save metadata are likely generated. */ +/** Base64 of binary data an app encodes as it saves, as Xournal++'s page preview, a PNG (`iVBORw0KGgo`), or any run of 256 or more base64 characters mixing cases and digits, line breaks aside: no prose runs that long without a space. */ +const ENCODED = + /^(?:iVBORw0KGgo[A-Za-z0-9+/=\r\n]*|(?=[^]*[a-z])(?=[^]*[A-Z])(?=[^]*\d)[A-Za-z0-9+/\r\n]{256,}={0,2})$/u; + +/** + * UUIDs, encoded binary, and epoch-shaped values in named save metadata are + * likely generated. Numeric shape alone does not identify a timestamp. + */ export const isVolatileValue = ( value: string, bookkeeping = false, ): boolean => { - if (UUID.test(value)) return true; + if (isUuid(value) || ENCODED.test(value.trim())) return true; if (!bookkeeping) return false; if (!/^\d{10}(?:\d{3})?$/u.test(value)) return false; const seconds = Number(value.slice(0, 10)); @@ -81,7 +84,7 @@ const DATE_PART = /^(?:year|month|day|hour|minute|second)$/u; * `cdate` and `mdate`, or `created`, `modified`, `saved` or `updated`. */ const BOOKKEEPING = - /^[cm]date$|creat(?:ed|ion)|modif(?:ied|ication)|updated|saved|changed|accessed|timestamp/u; + /^[cm]date$|creat(?:ed|ion)|modif(?:ied|ication)|updated|saved|changed|accessed|entered|timestamp/u; export const isBookkeepingName = (name: string) => BOOKKEEPING.test((name.split(":").at(-1) ?? name).toLowerCase()); @@ -167,6 +170,97 @@ const stampedAlong = (elements: readonly Element[]) => { }); }; +/** A date written whole, as GnuCash's `2026-11-06 08:50:12 +0000`, which `isTimestamp` reads only in ISO 8601's own spelling. */ +const DATE_TEXT = /^\s*\d{4}-\d{2}-\d{2}\b/u; + +/** Whether a value is a date under an element bookkeeping names, as GnuCash's `…`, or such an element holding one. */ +const stampedWhole = ( + elements: readonly Element[], + value: JsonValue | undefined, +) => + elements.some(isBookkeeping) && + DATE_TEXT.test( + typeof value === "string" + ? value + : isXmlElement(value) + ? elementText(value) + : "", + ); + +/** Whether an element holds an id the app assigns, as GnuCash's ``. */ +const assignedId = (element: Element) => + /^(?:[^:]*:)?id$/iu.test(element.name) && + isRecord(element.attributes) && + element.attributes.type === "guid"; + +/** Whether text holds a control character other than a tab or line break, or the replacement character, which no user types, as GnuCash's options hold before it fills them. */ +const isGarbled = (value: string) => + [...value].some((character) => { + const code = character.codePointAt(0) ?? 0; + return ( + (code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d) || + code === 0x7f || + code === 0xfffd + ); + }); + +/** An attribute counting the ids an app hands out, as Umbrello's `uniqueid` or Tiled's `nextobjectid`. */ +const ID_COUNTER = /^(?:next|unique|last)[-_]?(?:[\p{L}_-]*)id$/iu; + +/** A key keeping the last or next id an app hands out, as Umbrello's `lastobject`, which holds the id its new element took. */ +const COUNTER_KEY = /^(?:next|unique|last)[-_]?[\p{L}_-]*$/iu; + +/** Whether a value is one a program makes: a UUID, an epoch time or garbled text, a count under the metadata, an id counter, an id the change gave a new element where a counter or that element holds it (`fresh`), or in or at an id the app assigns. */ +const volatileAt = ( + elements: readonly Element[], + key: string | undefined, + value: JsonValue | undefined, + fresh: boolean, + xml: boolean, +) => + (typeof value === "string" && + (isVolatileValue( + value, + elements.some(isBookkeeping) || + (key !== undefined && isBookkeepingName(key)), + ) || + isGarbled(value) || + fresh)) || + (key !== undefined && ID_COUNTER.test(key.split(":").at(-1) ?? key)) || + metadataCount(elements, key, value) || + elements.some(assignedId) || + (xml && isXmlElement(value) && assignedId(value)); + +/** + * The ids of the elements a change added (`id`, `xid`, `uuid`, `guid`, as + * Umbrello's new attribute's `xmi:id="ubcgF2AWfBmHh"`), which another save + * of the same edit would not repeat, so a value equal to one in an id + * counter or in that element is the app's: those of four or more + * characters, not a plain number. + */ +const freshIds = ( + change: Pick, + ends: Ends, +): ReadonlySet => + new Set( + change.added.flatMap((pointer) => { + const element = resolvePointer(ends.final, pointer); + return isXmlElement(element) + ? Object.entries(element.attributes).flatMap(([key, value]) => + /^(?:[^:]*:)?(?:x|uu|g)?id$/iu.test(key) && + typeof value === "string" && + value.length >= 4 && + !/^\d+$/u.test(value) + ? [value] + : [], + ) + : []; + }), + ); + +/** A container of an app's own bookkeeping slots, as GnuCash's `book:slots`, which it adds to and fills on reopening a book. */ +const isSlots = (element: Element) => /^(?:[^:]*:)?slots$/u.test(element.name); + /** Whether a pointer is a date the app stamps in parts, or inside one. */ export const inSplitDate = (view: JsonValue | null, pointer: string): boolean => stampedAlong(elementsAlong(view, pointer)); @@ -198,9 +292,15 @@ const showsView = (element: Element) => isViewStateName(element.name) || Object.entries(isRecord(element.attributes) ? element.attributes : {}).some( ([key, value]) => - typeof value === "string" && namesElement(key) && isViewStateValue(value), + typeof value === "string" && + namesElement(key) && + (isViewStateValue(value) || VIEW_SETTING.test(value)), ); +/** A settings path whose last part says how a panel shows, as VYM's ``. */ +const VIEW_SETTING = + /^\/(?:[\w-]+\/)+(?:visible|hidden|shown?|expanded|collapsed|geometry|state|size|zoom)$/iu; + /** A file's views at its two ends. */ interface Ends { readonly baseline: JsonValue | null; @@ -246,13 +346,21 @@ const positionalRenumbering = (ends: Ends) => { }; }; +/** A pointer to an archive entry's own size or digest, which change with any edit to it. */ +const ENTRY_META = /^\/entries\/[^/]+\/(?!view(?:\/|$))/u; + /** A pointer into an archive entry's view: the entry's part of it, as `/entries/[name=styles.xml]/view/`. */ const ENTRY_VIEW = /^\/entries\/[^/]+\/view\//u; /** - * Attributes an archive entry's elements only gained, as LibreOffice fills in - * `loext:tab-stop-distance` across `styles.xml` on save, while another entry - * holds the edit: whether each pointer is one. + * What an app keeps up as it saves, while the edit is elsewhere: attributes an + * known style defaults an archive entry's elements gained, as LibreOffice fills in + * `loext:tab-stop-distance` across `styles.xml`, while another entry holds + * the edit; and, while a change outside them is the edit, any change to a + * document's styles, `styles.xml` or its style containers and master, + * handout and notes pages, or to an attribute naming an automatic style + * at both ends (`isAutomaticStyle`), as Impress wraps its master pages' + * fields in a new `MT1` style after a reload. */ const upkeepOf = ( change: Pick, @@ -288,6 +396,41 @@ const upkeepOf = ( filled.set(entry, counts); counts.set(key, (counts.get(key) ?? 0) + 1); } + // A style a user names, as "Quotations" given to a paragraph, is an edit; one the app numbers, as `P1`, its upkeep. + const automatic = (pointer: string, key: string) => + [ends.baseline, ends.final].every((view) => { + const value = resolvePointer(view, pointer); + return ( + value === undefined || + (typeof value === "string" && isAutomaticStyle(key, value)) + ); + }); + const styled = (pointer: string) => { + const key = attributeKeyOf(pointer); + const view = + resolvePointer(ends.final, pointer) === undefined + ? ends.baseline + : ends.final; + return ( + xml(view, pointer) && + (ENTRY_VIEW.exec(pointer)?.[0] === "/entries/[name=styles.xml]/view/" || + (key !== undefined && + /style-name$/iu.test(key) && + automatic(pointer, key)) || + elementsAlong(view, pointer).some(({ name }) => STYLE_AREA.test(name))) + ); + }; + // The user's edit is elsewhere when a change no other rule calls noise lies outside the styles. + const elsewhere = [ + ...change.added, + ...change.changed, + ...change.removed, + ].some( + (pointer) => + noise(pointer) === undefined && + !styled(pointer) && + !ENTRY_META.test(pointer), + ); return (pointer: string) => { const [entry] = ENTRY_VIEW.exec(pointer) ?? []; const key = attributeKeyOf(pointer); @@ -295,39 +438,235 @@ const upkeepOf = ( entry === undefined || key === undefined ? 0 : (filled.get(entry)?.get(key) ?? 0); + const view = + resolvePointer(ends.final, pointer) === undefined + ? ends.baseline + : ends.final; + const inStyleArea = elementsAlong(view, pointer).some(({ name }) => + STYLE_AREA.test(name), + ); + const automaticName = + key !== undefined && /style-name$/iu.test(key) && automatic(pointer, key); + // One attribute gained in styles.xml is the edit. A repeated fill-in, an automatic style name, or a style area the app rewrites is upkeep. return ( - edited.size > 0 && - entry !== undefined && - !edited.has(entry) && - gained(pointer) && - count >= 2 + (elsewhere && + (automaticName || inStyleArea || (styled(pointer) && count >= 2))) || + (edited.size > 0 && + entry !== undefined && + !edited.has(entry) && + gained(pointer) && + count >= 2) ); }; }; +/** An element holding an ODF document's styles, or its master, handout and notes pages, which an app rewrites as it saves after a reload. */ +const STYLE_AREA = + /^(?:office:(?:styles|automatic-styles|master-styles|font-face-decls)|style:(?:master-page|handout-master))$/u; + +/** A number, or a list of numbers split by commas or semicolons, as Dia's `6.3225,6.15;12.5775,8.15`. */ +const COORDINATES = + /^-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?(?:[,;] ?-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)*$/u; + +/** Whether a value is a coordinate or a list of them, which an app may derive rather than its user enter. */ +export const isCoordinates = (value: JsonValue | undefined): boolean => + typeof value === "string" && COORDINATES.test(value.trim()); + +/** How far above an attribute's element the element whose text changed may sit for the attribute to be its layout. */ +const MAX_LAYOUT_CLIMB = 3; + +/** + * Attributes holding coordinates that changed inside an element whose text + * changed while their own element's did not, as Dia moves and resizes a box + * (`obj_pos`, `obj_bb`, `elem_width`) around its new text: layout the app + * derives from the edit. A list of numbers is layout; a lone number only + * where a name on its way says it is (`namesLayout`). The element whose text changed is at most three + * levels up and below the root's own children; an attribute of an element + * whose own text changed, as a spreadsheet cell's `office:value` beside its + * text, is none. + */ +const derivedLayout = ( + change: Pick, + ends: Ends, +) => { + const textAt = (view: JsonValue | null, pointer: string) => { + const element = resolvePointer(view, pointer); + return isXmlElement(element) ? elementText(element) : undefined; + }; + const known = new Map(); + const retexted = (pointer: string) => { + const before = textAt(ends.baseline, pointer); + const found = + known.get(pointer) ?? + (before !== undefined && before !== textAt(ends.final, pointer)); + known.set(pointer, found); + return found; + }; + const layout = new Set( + change.changed.filter((pointer) => { + const value = resolvePointer(ends.final, pointer); + const owner = ownerOfAttribute(pointer); + if ( + attributeKeyOf(pointer) === undefined || + !isCoordinates(value) || + retexted(owner) + ) + return false; + const segments = owner.split("/"); + const above = Array.from({ length: MAX_LAYOUT_CLIMB }, (_, index) => + segments.slice(0, -2 * (index + 1)).join("/"), + ); + const at = above.findIndex( + (element) => + /\/children\/[^/]+$/u.test(element) && + element.split("/").at(-3) !== "root" && + retexted(element), + ); + // A lone number is layout only where a name on its way says so; a list of them always is. + return ( + at >= 0 && + (/[,;]/u.test(String(value)) || + namesLayout(ends.final, pointer, [owner, ...above.slice(0, at)])) + ); + }), + ); + return (pointer: string) => layout.has(pointer); +}; + +/** A tag naming a drawing of a model rather than the model, as Umbrello's `diagrams`, `diagram`, `widgets` and `classwidget`. */ +const DRAWING = /(?:diagram|widget|canvas|scene)s?$/iu; + +/** + * Coordinates in a drawing of the model, while the model change is + * elsewhere, as Umbrello grows a class's box (`classwidget` `height` 52 to + * 67) in its diagram when an attribute is added to the class: an attribute + * holding a list of numbers, or a number a name on its way calls layout, + * below an element whose tag names a diagram, widget, canvas or scene, while + * a change no other rule calls noise lies outside every such element. + */ +const drawingLayout = ( + change: Pick, + ends: Ends, + noise: (pointer: string) => NoiseReason | undefined, +) => { + const viewOf = (pointer: string) => + resolvePointer(ends.final, pointer) === undefined + ? ends.baseline + : ends.final; + const drawn = (pointer: string) => + elementsAlong(viewOf(pointer), pointer).some(({ name }) => + DRAWING.test(name.split(":").at(-1) ?? name), + ); + // A model element added or removed can resize the drawing. A rename or + // other edit outside it does not make a deliberate resize into layout. + const elsewhere = [...change.added, ...change.removed].some( + (pointer) => + noise(pointer) === undefined && + !ENTRY_META.test(pointer) && + !drawn(pointer), + ); + return (pointer: string) => { + const value = resolvePointer(viewOf(pointer), pointer); + return ( + elsewhere && + attributeKeyOf(pointer) !== undefined && + isCoordinates(value) && + drawn(pointer) && + (/[,;]/u.test(String(value)) || + namesLayout(viewOf(pointer), pointer, [ownerOfAttribute(pointer)])) + ); + }; +}; + +/** A word in a name for where or how large something is drawn: a position, a point, a box, a corner, a width or height, an offset or a side. */ +const LAYOUT_WORD = + /^(?:pos|position|x|y|z|point|bb|bbox|bounds|rect|rectangle|corner|width|height|w|h|left|top|right|bottom|offset|origin|coords?)$/u; + +/** Whether a name holds a layout word, its parts split at punctuation and case, as Dia's `obj_pos`, `elem_width` or `dia:point`. */ +const isLayoutName = (name: string) => + name + .replaceAll(/(?\p{Ll})(?\p{Lu})/gu, "$ $") + .toLowerCase() + .split(/[^\p{L}\d]+/u) + .some((word) => LAYOUT_WORD.test(word)); + +/** + * Whether the attribute at a pointer, its element's tag, or the tag or a + * name-like attribute's value of an element between it and the element + * whose text changed, names layout, as Dia keeps `elem_width` in + * ``; an order's + * `` is no layout. + */ +const namesLayout = ( + view: JsonValue | null, + pointer: string, + elements: readonly string[], +) => + [ + attributeKeyOf(pointer) ?? "", + ...elements.flatMap((at) => { + const element = resolvePointer(view, at); + return isXmlElement(element) + ? [ + element.name, + ...Object.entries(element.attributes).flatMap(([key, value]) => + isNameLikeKey(key) && typeof value === "string" ? [value] : [], + ), + ] + : []; + }), + ].some(isLayoutName); + /** * Why each pointer into a file is noise, or undefined when it is not: a tag, * attribute or name-like attribute value on its way naming view state, as - * `[@config:name=CursorPositionX]`; an attribute an app renumbers, an - * identifier it numbers by position, or anything inside an element both ends - * hold whose renumbered attribute changed, as Gnumeric empties a formula cell - * whose shared expression it renumbers; a UUID or epoch time, or a duration or + * `[@config:name=CursorPositionX]`; an identifier numbered by position; + * a UUID or epoch time, an id + * counter, an id the change gave a new element (`freshIds`), or a duration or * count under the document's metadata; a time the app stamps, whole or in - * parts (`inSplitDate`); the known style default an archive entry gained - * while another entry holds the edit. + * parts (`inSplitDate`), or a date under a tag naming bookkeeping; an id + * the app assigns (`type="guid"`), or text of control characters; upkeep, a + * change inside an app's `slots`, a known style default an archive entry + * gained while another entry holds the edit, a text showing another field + * that changed alike (`mirrorsOf`), or an element holding nothing but the id + * of a record the change added (`idReferencesOf`); layout an edit's text + * derived (`derivedLayout`), or a drawing of the model holds while the + * model change is elsewhere (`drawingLayout`). */ export const changeNoise = ( change: Pick, ends: Ends, xml: (view: JsonValue | null, pointer: string) => boolean, ): ((pointer: string) => NoiseReason | undefined) => { - const renumbered = change.changed.flatMap((pointer) => { - const key = attributeKeyOf(pointer); - return xml(ends.final, pointer) && key !== undefined && isRenumberedKey(key) - ? [ownerOfAttribute(pointer)] - : []; - }); + const isXmlPointer = (pointer: string) => + xml( + resolvePointer(ends.final, pointer) === undefined + ? ends.baseline + : ends.final, + pointer, + ); + const xmlChange = { + added: change.added.filter((pointer) => xml(ends.final, pointer)), + changed: change.changed.filter(isXmlPointer), + removed: change.removed.filter((pointer) => xml(ends.baseline, pointer)), + }; const byPosition = positionalRenumbering(ends); + const fresh = freshIds(xmlChange, ends); + const added = new Set(change.added); + // A reference on an element both ends hold, as a split moved to the new account, is the user's. + const freshAt = ( + pointer: string, + key: string | undefined, + value: JsonValue | undefined, + ) => + typeof value === "string" && + fresh.has(value) && + ((key !== undefined && COUNTER_KEY.test(key.split(":").at(-1) ?? key)) || + pointer + .split("/") + .some((_, index, segments) => + added.has(segments.slice(0, index + 1).join("/")), + )); const known = new Map(); const base = (pointer: string): NoiseReason | undefined => { if (known.has(pointer)) return known.get(pointer); @@ -342,38 +681,40 @@ export const changeNoise = ( const reason: NoiseReason | undefined = elements.some(showsView) || (key !== undefined && isViewStateName(key)) ? "view_state" - : (key !== undefined && - (isRenumberedKey(key) || - (isIdentifyingAttribute(key) && byPosition(pointer, key)))) || - renumbered.some((element) => { - if (pointer !== element && !pointer.startsWith(`${element}/`)) - return false; - if (pointer === element) return false; - const next = resolvePointer(ends.final, pointer); - const text = - typeof next === "string" - ? next.trim() - : isXmlElement(next) - ? elementText(next).trim() - : undefined; - // Gnumeric clears a shared formula when it renumbers the cell. A formula that still has text was edited. - return text === undefined || text === ""; - }) + : key !== undefined && + isIdentifyingAttribute(key) && + byPosition(pointer, key) ? "renumbered" - : (typeof value === "string" && - isVolatileValue( - value, - elements.some(isBookkeeping) || - (key !== undefined && isBookkeepingName(key)), - )) || - metadataCount(elements, key, value) + : volatileAt( + elements, + key, + value, + freshAt(pointer, key, value), + isXml, + ) ? "volatile" - : isStamp(value ?? null) || stampedAlong(elements) + : isStamp(value ?? null) || + stampedAlong(elements) || + stampedWhole(elements, value) ? "timestamp" - : undefined; + : elements.some(isSlots) || + (isXml && isMemberCount(ends, pointer, key)) + ? "upkeep" + : undefined; known.set(pointer, reason); return reason; }; const upkeep = upkeepOf(change, ends, base, xml); - return (pointer) => base(pointer) ?? (upkeep(pointer) ? "upkeep" : undefined); + const layout = derivedLayout(change, ends); + const drawing = drawingLayout(change, ends, base); + const mirror = mirrorsOf(xmlChange, ends); + const reference = idReferencesOf(xmlChange, ends); + return (pointer) => + base(pointer) ?? + (isXmlPointer(pointer) && + (upkeep(pointer) || mirror(pointer) || reference(pointer)) + ? "upkeep" + : isXmlPointer(pointer) && (layout(pointer) || drawing(pointer)) + ? "layout" + : undefined); }; diff --git a/src/domain/desktop/appTaskDraftNotes.ts b/src/domain/desktop/appTaskDraftNotes.ts index e8cfc44004..368853be4f 100644 --- a/src/domain/desktop/appTaskDraftNotes.ts +++ b/src/domain/desktop/appTaskDraftNotes.ts @@ -34,7 +34,7 @@ export const draftNoteSchema = z.strictObject({ .string() .optional() .describe( - "by_position, view_state_only, long_text, cut_view, unreadable_view: the drafted file the note is about.", + "by_position, view_state_only, long_text, cut_view, unreadable_view, volatile_lines: the drafted file the note is about.", ), files: z .array(z.strictObject({ file: z.string(), role: z.string() })) diff --git a/src/domain/desktop/appTaskDraftPositional.test.ts b/src/domain/desktop/appTaskDraftPositional.test.ts index 20bf6c0043..4703489e9e 100644 --- a/src/domain/desktop/appTaskDraftPositional.test.ts +++ b/src/domain/desktop/appTaskDraftPositional.test.ts @@ -1,8 +1,9 @@ import { describe, expect, it } from "vitest"; import type { JsonValue } from "../jsonValue.js"; -import { passing, xml, xmlDraft } from "./appTaskDraft.fixture.js"; -import { pairMembers, resolvePointer } from "./viewPointer.js"; +import { passing, pointerOf, xml, xmlDraft } from "./appTaskDraft.fixture.js"; +import { pairMembers } from "./memberPairing.js"; +import { resolvePointer } from "./viewPointer.js"; const FILE = "Documents/doc.xml"; @@ -14,9 +15,17 @@ const membersAt = (text: string, pointer: string): JsonValue[] => { }; describe("draftTaskFiles for a change only position reaches", () => { - /** A sheet as an ODS spreadsheet keeps it: rows no attribute names, each cell's text in a paragraph, and a save counter. */ - const sheet = (february: string, total: string, cycles: string) => - `${cycles}

Month

Sales

February

${february}

Total

${total}

`; + /** A sheet as an ODS spreadsheet keeps it: rows no attribute names, each cell's text in a paragraph, labelled apart or alike, and a save counter. */ + const sheet = ( + february: string, + total: string, + cycles: string, + labelled = false, + ) => { + const label = (text: string) => + `

${labelled ? text : "Q1"}

`; + return `${cycles}

Month

Sales

${label("February")}

${february}

${label("Total")}

${total}

`; + }; it("checks the whole text of the ancestor above the step by position, by its name, and demotes the noise beside it rather than suggesting it", () => { const before = sheet("21300", "59625", "1"); @@ -29,7 +38,7 @@ describe("draftTaskFiles for a change only position reaches", () => { kind: "text_equals", file: FILE, pointer: "/root/children/[name=table][@name=Q1 Sales]", - value: "MonthSalesFebruary23100Total61425", + value: "MonthSalesQ123100Q161425", }, ]); expect(passing(FILE, xml(before), file.suggested)).toEqual([false]); @@ -37,6 +46,34 @@ describe("draftTaskFiles for a change only position reaches", () => { expect(file.notes.map(({ code }) => code)).not.toContain("view_state_only"); }); + it("names a row by its label cell, so the edited cell is checked by name, not the table's whole text", () => { + const before = sheet("21300", "59625", "1", true); + const after = sheet("23100", "61425", "2", true); + + const file = xmlDraft(before, after); + + const row = (label: string) => + `/root/children/[name=table][>row=MonthSales]/children/[name=row][>cell=${label}]/children/[name=cell][@type=float]`; + expect(file.suggested.slice(0, 2)).toEqual([ + { + kind: "text_equals", + file: FILE, + pointer: row("February"), + value: "23100", + }, + { + kind: "text_equals", + file: FILE, + pointer: row("Total"), + value: "61425", + }, + ]); + expect(passing(FILE, xml(before), file.suggested).some(Boolean)).toBe( + false, + ); + expect(passing(FILE, xml(after), file.suggested).every(Boolean)).toBe(true); + }); + it("suggests no noise when position alone reaches the edit and no ancestor's text can take a check", () => { // The table's text is past the limit a text check compares. const long = "x".repeat(2100); @@ -100,6 +137,13 @@ describe("identifiers an app numbers by position", () => { pointer: `${tasks}[@name=User testing]/name`, value: "task", }, + { + kind: "member_count", + file: FILE, + pointer: "/root/children/[name=tasks]/children", + matching: "[name=task][@type=normal]", + equals: 4, + }, ]); expect(file.values.map(({ pointer, note }) => [pointer, note])).toEqual([ [`${tasks}[@name=User testing]`, undefined], @@ -116,6 +160,13 @@ describe("identifiers an app numbers by position", () => { const tasks = "/root/children/[name=tasks]/children/[name=task]"; expect(file.suggested).toEqual([ { kind: "value_absent", file: FILE, pointer: `${tasks}[@name=Design]` }, + { + kind: "member_count", + file: FILE, + pointer: "/root/children/[name=tasks]/children", + matching: "[name=task][@type=normal]", + equals: 2, + }, ]); }); @@ -166,28 +217,41 @@ describe("identifiers an app numbers by position", () => { }); describe("an element renamed", () => { - /** A painting's layer stack as MyPaint saves it: each layer by name, its pixels in `src`, the one selected flagged. */ - const stack = (top: string) => - `${[ - [top, "00", "576", "true"], - ["Sky", "01", "0", "false"], - ["Background", "02", "0", "false"], - ] + /** + * A painting's layer stack as MyPaint saves it: each layer by name, its + * pixels in `src`, numbered along the stack as MyPaint 2 writes it or named + * by the layer, the one selected flagged; the top layer first, or second. + */ + const stack = ( + top: string, + { + numbered = true, + moved = false, + }: { numbered?: boolean; moved?: boolean } = {}, + ) => { + const layers = [ + [top, "sun", "576", "true"], + ["Sky", "sky", "0", "false"], + ["Background", "ground", "0", "false"], + ]; + const order = moved ? [1, 0, 2].map((at) => layers[at] ?? []) : layers; + return `${order .map( - ([name, layer, x, selected]) => - ``, + ([name, own, x, selected], index) => + ``, ) .join("")}`; + }; const LAYER = "/root/children/[name=stack]/children/[name=layer]"; it("pairs a layer whose name alone changed, at its place among the layers, by another attribute naming it alone rather than the one selected", () => { const layers = (top: string) => - membersAt(stack(top), "/root/children/0/children"); + membersAt(stack(top, { numbered: false }), "/root/children/0/children"); const pairing = pairMembers(layers("Sun"), layers("Untitled")); const keys = [ - "[name=layer][@src=data/layer-00.png]", + "[name=layer][@src=data/sun.png]", "[name=layer][@name=Sky]", "[name=layer][@name=Background]", ]; @@ -197,14 +261,25 @@ describe("an element renamed", () => { }); }); - it("checks the renamed layer's new name by the attribute that names it, and that the old name names nothing", () => { + it("keys a renamed layer by each version's own name where the attribute pairing it numbers the layers in order, as MyPaint's src", () => { + const layers = (top: string) => + membersAt(stack(top), "/root/children/0/children"); + + const pairing = pairMembers(layers("Sun"), layers("Untitled")); + + expect(pairing.newer.keys[0]).toBe("[name=layer][@name=Sun]"); + expect(pairing.older.keys[0]).toBe("[name=layer][@name=Untitled]"); + expect(pairing.newer.partners).toEqual([0, 1, 2]); + }); + + it("checks the renamed layer's new name by the attribute that names it, and that the old name names nothing, though a later save moves it", () => { const file = xmlDraft(stack("Untitled"), stack("Sun")); expect(file.suggested).toEqual([ { kind: "value_equals", file: FILE, - pointer: `${LAYER}[@src=data~1layer-00.png]/attributes/name`, + pointer: `${LAYER}[@name=Sun]/attributes/name`, value: "Sun", }, { @@ -213,14 +288,247 @@ describe("an element renamed", () => { pointer: `${LAYER}[@name=Untitled]`, }, ]); + expect(file.values).toMatchObject([ + { kind: "changed", before: "Untitled", after: "Sun" }, + ]); + // The move renumbers every src, which a check by src would have failed. expect( - [stack("Untitled"), stack("Sun"), stack("Moon")].map((text) => - passing(FILE, xml(text), file.suggested), - ), + [ + stack("Untitled"), + stack("Sun"), + stack("Moon"), + stack("Sun", { moved: true }), + ].map((text) => passing(FILE, xml(text), file.suggested)), ).toEqual([ [false, false], [true, true], [false, true], + [true, true], + ]); + // No layer added or removed is a guard at the unchanged count. + expect(file.guards.at(-1)).toEqual({ + kind: "member_count", + file: FILE, + pointer: "/root/children/[name=stack]/children", + matching: "[name=layer]", + equals: 3, + }); + }); +}); + +describe("members named by a child's text or a generated id", () => { + it("counts a child's text on every sibling, one an identifier names included, so a selector both match names neither", () => { + const records = (note: string) => + `SharedkeptShared${note}`; + + const file = xmlDraft(records("before"), records("after")); + + const pointers = [ + ...file.suggested.flatMap((check) => + "pointer" in check ? [check.pointer] : [], + ), + ...file.values.map(({ pointer }) => pointer), + ]; + expect( + pointers.some((pointer) => pointer.includes("[>title=Shared]")), + ).toBe(false); + expect(file.values.map(({ after }) => after)).toEqual(["after"]); + expect( + passing(FILE, xml(records("after")), file.suggested).every(Boolean), + ).toBe(true); + }); + + it("names a new node by its text rather than the id the app generated, wherever the id sits among its attributes, and pairs a kept one by that id", () => { + const map = (...nodes: string[]) => + `${nodes.join("")}`; + const node = (id: string, text: string) => + ``; + + const pairing = pairMembers( + membersAt( + map( + node("ID_1749680464", "Tent"), + node("ID_329023692", "Water filter"), + ), + "/root/children/0/children", + ), + membersAt( + map(node("ID_1749680464", "Tent")), + "/root/children/0/children", + ), + ); + + expect(pairing.newer.keys).toEqual([ + "[name=node][@ID=ID_1749680464]", + "[name=node][@TEXT=Water filter]", + ]); + }); +}); + +describe("a container in a list named for its tag", () => { + /** A workbook as Gnumeric saves one: sheets in `Sheets`, each named by its `Name` child. */ + const book = (march: string, more = "") => + `Budget402.15${march}${more}`; + + it("names a sheet its tag alone names by its name child, so a sheet added later leaves the check its own", () => { + const file = xmlDraft(book("376.9"), book("418.65")); + + expect(file.suggested.map(pointerOf)).toEqual([ + "/root/children/[name=Sheets]/children/[name=Sheet][>Name=Budget]/children/[name=Cells]/children/[name=Cell][@Col=3]", + ]); + const another = book("418.65", "Notes"); + expect(passing(FILE, xml(another), file.suggested)).toEqual([true]); + }); + + it("holds no step of a list not named for its member's tag, nor a member with no name", () => { + const plain = (text: string) => + `
Plan

${text}

`; + + const file = xmlDraft(plain("Monday"), plain("Tuesday")); + + expect(file.suggested.map(pointerOf)).toEqual([ + "/root/children/[name=body]/children/[name=section]/children/[name=p]", + ]); + }); +}); + +describe("records an app gave generated ids", () => { + const records = (first: string, second: string) => + `${first}${second}`; + + it("pairs kept records by their ids, so titles they swapped read as two title edits, not two id changes", () => { + const file = xmlDraft(records("Alpha", "Beta"), records("Beta", "Alpha")); + + expect( + file.values.map(({ pointer, kind, note }) => [ + pointer, + kind, + note ?? null, + ]), + ).toEqual( + [ + "0b5c1f0e-3c1d-4f7a-9a3e-2d7f5b8c9e10", + "7d1e2a90-5b4c-4e8f-8a21-6c3b9f0d4e57", + ].map((id) => [ + `/root/children/[name=record][@id=${id}]/children/[name=title]/children/0`, + "changed", + null, + ]), + ); + expect( + passing(FILE, xml(records("Alpha", "Beta")), file.suggested), + ).not.toContain(true); + expect( + passing(FILE, xml(records("Beta", "Alpha")), file.suggested).every( + Boolean, + ), + ).toBe(true); + }); +}); + +describe("shapes an ODF app numbers on every save", () => { + /** A Draw page's boxes, each named by the user, and numbered `id1`… by LibreOffice from `first` as it saves. */ + const page = (second: string, first = 1) => + `${[ + "Order Received", + second, + "Pack Items", + "Ship Order", + ] + .map( + (text, index) => + `${text}`, + ) + .join("")}`; + const shape = (step: number) => + `/root/children/[name=draw:custom-shape][@draw:name=Step ${String(step)}]`; + + it("names each box by the name the user gave it, not the id the save numbered, so a save numbering them anew keeps the checks and guards", () => { + const file = xmlDraft(page("Pick Items"), page("Pick and Scan")); + + expect(file.suggested).toEqual([ + { + kind: "text_equals", + file: FILE, + pointer: shape(2), + value: "Pick and Scan", + }, + ]); + expect(file.guards.map(pointerOf)).toEqual([ + shape(1), + shape(3), + "/root/children", + ]); + // Another save numbers the shapes from 5. + const renumbered = xml(page("Pick and Scan", 5)); + expect(passing(FILE, renumbered, file.suggested)).toEqual([true]); + expect(passing(FILE, renumbered, file.guards)).toEqual([true, true, true]); + // The value lists no other pointer by an id the save numbered. + expect(file.values[0]).toMatchObject({ + pointer: `${shape(2)}/children/[name=text:p]/children/0`, + }); + expect(file.values[0]?.alternatives).toBeUndefined(); + }); +}); + +describe("entries an app numbers by where it saves them", () => { + /** A Parley collection: entries numbered 0.. by save position, each two translations by language, and a lesson listing them by id. */ + const entry = (id: number, english: string, spanish: string) => + `${english}${spanish}`; + const collection = (pairs: readonly (readonly [string, string])[]) => + `${pairs.map(([english, spanish], id) => entry(id, english, spanish)).join("")}Lesson 1${pairs.map((_, id) => ``).join("")}`; + const DOG = ["dog", "perro"] as const; + const BOOK = ["book", "libro"] as const; + const APPLE = ["apple", "manzana"] as const; + const LIBRARY = ["library", "biblioteca"] as const; + const SEED = [DOG, BOOK, APPLE]; + const ENTRIES = "/root/children/[name=entries]/children"; + + it("names the new entry and the kept ones by their translations, not the id that numbers them by save position, so a save inserting the entry elsewhere keeps the checks", () => { + const file = xmlDraft(collection(SEED), collection([...SEED, LIBRARY])); + + expect(file.suggested.map(pointerOf)).toEqual([ + `${ENTRIES}/[name=entry][>translation=library]/children/[name=translation][@id=1]`, + ENTRIES, + ]); + expect(file.guards.map(pointerOf)).toEqual([ + `${ENTRIES}/[name=entry][>translation=apple]/children/[name=translation][@id=1]`, + `${ENTRIES}/[name=entry][>translation=book]/children/[name=translation][@id=1]`, + ]); + const inserted = xml(collection([DOG, BOOK, LIBRARY, APPLE])); + expect(passing(FILE, inserted, file.suggested)).toEqual([true, true]); + expect(passing(FILE, inserted, file.guards)).toEqual([true, true]); + }); + + it("notes the lesson's entry holding nothing but the new entry's id as upkeep naming that entry, never a check or a count", () => { + const file = xmlDraft(collection(SEED), collection([...SEED, LIBRARY])); + + const lesson = + "/root/children/[name=lessons]/children/[name=container]/children"; + expect( + file.values.find(({ pointer }) => pointer.startsWith(lesson)), + ).toMatchObject({ + pointer: `${lesson}/[name=entry][@id=3]`, + note: "upkeep", + refers_to: [{ attribute: "id", value: "3" }], + }); + expect( + [...file.suggested, ...file.guards].some((check) => + pointerOf(check)?.startsWith(lesson), + ), + ).toBe(false); + }); +}); + +describe("elements keyed only by an id a save numbered with six digits", () => { + const notes = (hello: string) => + `

${hello}

There

`; + + it("checks the edited paragraph by that id, counted once", () => { + const file = xmlDraft(notes("Hello"), notes("Hi")); + + expect(file.suggested.map(pointerOf)).toEqual([ + "/root/children/[name=p][@xml:id=id123456]", ]); }); }); diff --git a/src/domain/desktop/appTaskDraftScalars.ts b/src/domain/desktop/appTaskDraftScalars.ts index 2707589b03..550633362a 100644 --- a/src/domain/desktop/appTaskDraftScalars.ts +++ b/src/domain/desktop/appTaskDraftScalars.ts @@ -1,6 +1,6 @@ import type { JsonValue } from "../jsonValue.js"; import type { DesktopTaskCheckInput } from "./appTask.js"; -import { numberOf } from "./appTaskChecks.js"; +import { numberOf, sameJson } from "./appTaskChecks.js"; /** How far a decimal the app writes as text may drift, such as `437.80000000000001` for 437.8, and still match. */ export const DECIMAL_TOLERANCE = 0.001; @@ -13,12 +13,24 @@ export const decimalOf = (value: JsonValue | undefined): number | undefined => { return Number.isFinite(number) ? number : undefined; }; -/** Whether a decimal's number is within the tolerance of a baseline value, read as the evaluators read a number, so a check by number would pass there too. */ +/** Whether a decimal's number is within the tolerance of a baseline value, read as the evaluators read a number, `"10"` and `"1e1"` included, so a check by number would pass there too. */ const nearBaseline = (number: number, before: JsonValue | undefined) => { const old = before === undefined ? undefined : numberOf(before); return old !== undefined && Math.abs(old - number) <= DECIMAL_TOLERANCE; }; +/** Whether a `value_equals` passes on a value, as the evaluator decides it: by number within its tolerance, or as JSON. */ +export const passesOn = ( + check: DesktopTaskCheckInput, + value: JsonValue | undefined, +): boolean => { + if (check.kind !== "value_equals" || value === undefined) return false; + const number = numberOf(value); + return check.tolerance !== undefined && typeof check.value === "number" + ? number !== undefined && Math.abs(number - check.value) <= check.tolerance + : sameJson(value, check.value); +}; + /** * A `value_equals` check of a value: a decimal string as its number within * `DECIMAL_TOLERANCE`, unless `before`, the value the check must fail on, is diff --git a/src/domain/desktop/appTaskDraftSteps.ts b/src/domain/desktop/appTaskDraftSteps.ts new file mode 100644 index 0000000000..ebeb204bda --- /dev/null +++ b/src/domain/desktop/appTaskDraftSteps.ts @@ -0,0 +1,81 @@ +import type { JsonValue } from "../jsonValue.js"; +import { isNameLikeKey } from "./attributeRank.js"; +import { isXmlElement, resolvePointer } from "./viewPointer.js"; +import { selectorOf } from "./viewSelector.js"; + +/** Where a pointer steps into a list: the list, the member's segment and the rest of the pointer after it. */ +export interface ListStep { + readonly list: string; + readonly member: string; + readonly rest: readonly string[]; +} + +/** Where a pointer steps into a list, innermost first; by selector only, unless `positional` keeps steps by position too. */ +export const listSteps = (pointer: string, positional = false): ListStep[] => { + const segments = pointer.split("/"); + const steps: ListStep[] = []; + for (let index = segments.length - 1; index >= 2; index -= 1) { + const member = segments[index] ?? ""; + if ( + segments[index - 1] === "children" && + (positional || !/^\d+$/u.test(member)) + ) + steps.push({ + list: segments.slice(0, index).join("/"), + member, + rest: segments.slice(index + 1), + }); + } + return steps; +}; + +/** Whether a step's member is named as a field is, by a name-like attribute alone, as QElectroTech's `elementInformation[@name=label]`. */ +const namedAsField = ({ member }: ListStep) => { + const [only, ...more] = ( + selectorOf(member.replaceAll("~1", "/").replaceAll("~0", "~")) ?? [] + ).filter(({ prefix }) => prefix !== ""); + return ( + only !== undefined && + more.length === 0 && + only.prefix === "@" && + isNameLikeKey(only.key) + ); +}; + +/** Whether a like sibling of an outer step's member holds the field a step goes through, at the same path, as every component holds its `label`. */ +const sharedField = ( + view: JsonValue | null, + step: ListStep, + outer: ListStep, +) => { + const records = resolvePointer(view, outer.list); + const record = resolvePointer(view, `${outer.list}/${outer.member}`); + const inner = outer.rest.slice(0, outer.rest.length - step.rest.length); + return ( + Array.isArray(records) && + isXmlElement(record) && + records.some( + (item) => + item !== record && + isXmlElement(item) && + item.name === record.name && + resolvePointer(item, `/${inner.join("/")}`) !== undefined, + ) + ); +}; + +/** + * The record a step's member is a field of: the nearest step out whose like + * siblings hold the same field at the same path, for a member named as a + * field (`namedAsField`), as every QElectroTech component holds its + * `elementInformation[@name=label]`, or a Glade button its `label` property; + * undefined for a member that is a record of its own. + */ +export const recordOfField = ( + view: JsonValue | null, + step: ListStep, + outer: readonly ListStep[], +): ListStep | undefined => + namedAsField(step) + ? outer.find((candidate) => sharedField(view, step, candidate)) + : undefined; diff --git a/src/domain/desktop/appTaskDraftText.ts b/src/domain/desktop/appTaskDraftText.ts index 98b623d600..dd36daf912 100644 --- a/src/domain/desktop/appTaskDraftText.ts +++ b/src/domain/desktop/appTaskDraftText.ts @@ -1,11 +1,12 @@ import type { JsonValue } from "../jsonValue.js"; -import type { DesktopTaskCheckInput } from "./appTask.js"; +import { collapseWhitespace, type DesktopTaskCheckInput } from "./appTask.js"; import { holdsSplitDate, isBookkeeping, isStamp, isVolatileValue, } from "./appTaskDraftNoise.js"; +import { isPlacedItem, placedItemCheck } from "./appTaskDraftItems.js"; import { valueEqualsCheck } from "./appTaskDraftScalars.js"; import { rankAttributes } from "./attributeRank.js"; import type { DesktopStateDifference } from "./stateChanges.js"; @@ -34,6 +35,8 @@ export interface TextPlacement { /** Text checks for the elements whose text changed, and the pointers those checks make redundant: text and whole runs holding text in them, or an element added whole checked itself, not an attribute or name of an element both ends hold, nor an element holding no text, as an image or a break. */ export interface WholeTextSuggestions { readonly suggestions: readonly DesktopTaskCheckInput[]; + /** Kept positioned items associated with the selected text checks. */ + readonly guards: readonly DesktopTaskCheckInput[]; /** Elements whose text changed that no check took, because every place for one held more text than the limit. */ readonly long: readonly string[]; readonly covers: (pointer: string) => boolean; @@ -168,6 +171,34 @@ const byOwnAttribute = (element: string, ends: Ends): Candidate | undefined => { return undefined; }; +/** + * Whether the element at a pointer is a field its tag names in a record of + * two or more fields holding text, at both ends, as QElectroTech's `` + * beside `` in a ``: a check reads it, not the + * record's text, which would join its fields; a Draw shape's one paragraph + * beside its geometry is the shape's text. + */ +const isField = (pointer: string, ends: Ends) => { + const parent = parentOf(pointer); + const tag = /^\[name=(?[^\]]+)\]$/u.exec(pointer.split("/").at(-1) ?? "") + ?.groups?.tag; + return ( + parent !== undefined && + tag !== undefined && + [ends.baseline, ends.final].every((view) => { + const record = resolvePointer(view, parent); + const fields = isXmlElement(record) + ? record.children.filter(isXmlElement) + : []; + return ( + holdsFields(record) && + fields.filter((field) => elementText(field).trim() !== "").length > 1 && + fields.filter(({ name }) => name === tag).length === 1 + ); + }) + ); +}; + /** How far a text check climbs from an element no attribute names to an ancestor its tag names. */ const MAX_TAG_CLIMB = 3; /** A formatting run a later save may split, merge, or drop while keeping the paragraph. */ @@ -219,7 +250,8 @@ export const holdsNoisyText = ( * Where a text check goes for an element whose text changed, best first, so * the check names the paragraph, which a later save keeps, rather than a run * it may split or merge: up from an element in mixed content (its parent - * holds text beside it, at either end) to the nearest that is not, then, from + * holds text beside it, at either end) to the nearest that is not, which a + * field its tag names in a record of fields (`isField`) stays; then, from * one named by tag or position alone, to the nearest ancestor named by an * attribute, as AbiWord's `p` by its `xid`, where one is below the document's * root. An element passed on the way that an identifier of its own names, the @@ -238,6 +270,8 @@ const outsideRuns = (pointer: string, ends: Ends): readonly Candidate[] => { parent = parentOf(current) ) current = parent; + if (isField(current, ends)) + return [{ element: current, pointer: current, identifier: false }]; const passed: string[] = []; let named = current; while (!namedByAttribute(named)) { @@ -342,46 +376,83 @@ const recordLike = (value: JsonValue | undefined): boolean => const TEXT_TAG = /^(?:p|h|h[1-6]|para|simpara|li|text|tspan|span|a)$/u; /** - * A tag of a formatting run inside a piece of text, compared without its - * namespace and without case. `link` and `code` stay out: some records name a - * field that, and a field is not a run. + * A tag of inline formatting, by its local name: bold, italic and the like, + * a span, link or code run, or Gnote's `size:` and `link:` runs. A record + * whose fields are only such, as `olive oil`, is text. */ -const FORMAT_TAG = - /^(?:span|a|s|c|tspan|tab|line-break|soft-page-break|br|b|i|u|em|strong|bold|italic|underline|emphasis|sub|sup|small|font|mark|q|r|t|run|hyperlink)$/u; +const INLINE_TAG = + /^(?:b|i|u|s|em|strong|bold|italic|underline|strikethrough|emphasis|span|a|code|sub|sup|small|big|mark|font|tt|monospace|highlight|link)$/u; +const INLINE_PREFIX = /^(?:size|link):/u; -const isFormatTag = (tag: string) => - FORMAT_TAG.test((tag.split(":").at(-1) ?? tag).toLowerCase()); +const isInline = (name: string) => + INLINE_PREFIX.test(name) || + INLINE_TAG.test((name.split(":").at(-1) ?? name).toLowerCase()); /** * Whether an element holds fields: only elements, layout whitespace aside, as * Tellico's `` holds `` and `<publisher>`, or a record holds * repeated `<field>` elements, whatever they are called; unless it is text - * made of runs: its tag names a paragraph or other text (`TEXT_TAG`), every - * element child is a formatting run (`FORMAT_TAG`), as a list item - * `<bold>olive oil</bold>`, or a child's text opens or closes with - * whitespace, as a sentence splits into runs, `Meet ` and a linked `Monday`, - * where a field holds a value whole. + * made of runs: its tag names a paragraph or other text (`TEXT_TAG`), its + * elements are all inline formatting (`INLINE_TAG`), or a child's text opens + * or closes with a space, as a sentence splits into runs, `Meet ` and a + * linked `Monday`, where a field holds a value whole; a line break and + * indentation the app lays out after a field's text, as KmPlot's, is none. */ export const holdsFields = (value: JsonValue | undefined): boolean => recordLike(value) && isXmlElement(value) && !TEXT_TAG.test(value.name.split(":").at(-1) ?? value.name) && !value.children.every( - (child) => !isXmlElement(child) || isFormatTag(child.name), + (child) => !isXmlElement(child) || isInline(child.name), ) && !value.children.some((child) => { const text = isXmlElement(child) ? elementText(child) : ""; - return text.trim() !== "" && text.trim() !== text; + // A space, not a line break the app lays out, at either edge splits a sentence. + return /^[ \t]+\S|\S[ \t]+$/u.test(text); }); +/** Whether an element at a pointer sits in a paragraph or other text, an ancestor's tag naming one (`TEXT_TAG`), as Word's `<w:r>` run in its `<w:p>`. */ +const inText = (view: JsonValue | null, pointer: string): boolean => { + for ( + let parent = parentOf(pointer); + parent !== undefined; + parent = parentOf(parent) + ) { + const element = resolvePointer(view, parent); + if ( + isXmlElement(element) && + TEXT_TAG.test(element.name.split(":").at(-1) ?? element.name) + ) + return true; + } + return false; +}; + +/** Whether an element at a pointer is a record: it `holdsFields`, and sits in no paragraph, where it is a run of its text. */ +export const isRecordAt = ( + view: JsonValue | null, + pointer: string, + value: JsonValue | undefined, +): boolean => holdsFields(value) && !inText(view, pointer); + /** * Whether the kept ancestor of a record added or removed whole is a list of * its kind, whose whole text would fold in records the edit left alone: it * holds no text of its own, and another element of that tag with text that * both ends hold alike, as Tellico's `<collection>` holds the other entries. */ -const listOf = (holder: string, element: JsonValue, ends: Ends) => { - if (!isXmlElement(element) || !holdsFields(element) || mixedAt(holder, ends)) +const listOf = ( + holder: string, + pointer: string, + element: JsonValue, + ends: Ends, +) => { + if ( + !isXmlElement(element) || + !isRecordAt(ends.final, pointer, element) || + !isRecordAt(ends.baseline, pointer, element) || + mixedAt(holder, ends) + ) return false; const like = (view: JsonValue | null) => { const list = resolvePointer(view, holder); @@ -431,7 +502,8 @@ const textHolder = ( ) => { const kept = keptAncestor(pointer, ends); const unnamed = removed && /^\d+$/u.test(pointer.split("/").at(-1) ?? ""); - const listed = kept !== undefined && !unnamed && listOf(kept, element, ends); + const listed = + kept !== undefined && !unnamed && listOf(kept, pointer, element, ends); return { holder: listed ? undefined : kept, listed }; }; @@ -439,6 +511,8 @@ const textHolder = ( interface Placed { readonly element: string; readonly check: DesktopTaskCheckInput; + /** Positioned items can supply checks of their kept siblings. */ + readonly guards?: readonly DesktopTaskCheckInput[]; } /** What text checks read: the file, its two ends, and where checks go. */ @@ -448,6 +522,10 @@ interface TextContext { readonly place: TextPlacement; } +/** Whether text is one line the app lays out between line breaks. */ +const isLaidOutLine = (value: string) => + /[\r\n]/u.test(value) && !/[\r\n]/u.test(value.trim()); + /** * The check of an element's whole text: a decimal held as the element's one * text node, as a spreadsheet keeps a cell's number, by its value within a @@ -460,9 +538,18 @@ export const wholeTextCheck = ( value: string, before: string | undefined, ): DesktopTaskCheckInput => { - const numeric = valueEqualsCheck(file, pointer, value, before); + // A line between breaks is still that line. Trim it before the decimal test so `\\n437.8\\n` stays a number within tolerance. + const comparable = isLaidOutLine(value) ? value.trim() : value; + const earlier = + before !== undefined && isLaidOutLine(before) ? before.trim() : before; + // The element's joined text, not its first text node. A later save may split `437.8` into `437.` and a run holding `8`. When that text is a decimal, compare the number within the usual tolerance, so `437.80` and a long float still match. + const numeric = valueEqualsCheck(file, pointer, comparable, earlier); if ("tolerance" in numeric) return numeric; - return { kind: "text_equals", file, pointer, value }; + // One line the app lays out between line breaks, as KmPlot's formula before its next element, compares with its whitespace collapsed. + const laidOut = isLaidOutLine(value) + ? { value: collapseWhitespace(value), normalize_whitespace: true } + : { value }; + return { kind: "text_equals", file, pointer, ...laidOut }; }; /** The element a candidate names in the final view, whole, at the pointer `name` gives it. */ @@ -482,7 +569,8 @@ const candidateAt = ( * The text check a candidate takes: an element the final view holds whole at * the pointer `name` gives it, its text at most the limit, no timestamp and * holding no date stamped in parts, and changed from the baseline's or, for - * an element `added` whole, absent there. + * an element `added` whole, absent there; never a line the app only laid + * out anew, which a check comparing collapsed whitespace finds at the seed. */ const textCheck = ( context: TextContext, @@ -494,9 +582,16 @@ const textCheck = ( const { pointer, now } = found; const before = resolvePointer(context.ends.baseline, pointer); if (!added && !isXmlElement(before)) return undefined; + // A timestamp between other text is not part of the check. Joining across it asks for text the file never contains. + if (holdsNoisyText(now)) return undefined; const value = elementText(now); const old = isXmlElement(before) ? elementText(before) : undefined; + const relaidOut = + old !== undefined && + isLaidOutLine(value) && + collapseWhitespace(old) === collapseWhitespace(value); return value !== old && + !relaidOut && value.length <= context.place.limit.chars && !isStamp(value) && !holdsNoisyText(now) && @@ -527,6 +622,8 @@ const textHolders = (change: Listed, ends: Ends, place: TextPlacement) => { const holders = new Set<string>(); /** Elements added whole holding text, each with the ancestor checked for it, if any. */ const added = new Map<string, string | undefined>(); + /** Items placed by their own position, added whole. */ + const items: string[] = []; const note = ( pointer: string, view: JsonValue | null, @@ -549,6 +646,11 @@ const textHolders = (change: Listed, ends: Ends, place: TextPlacement) => { !isWhitespace(elementText(value)) && place.inTree(pointer) ) { + // An item placed by its own position is no run of its list's text. + if (isPlacedItem(view, pointer)) { + if (how === "added") items.push(pointer); + return; + } const { holder, listed } = textHolder( pointer, value, @@ -567,7 +669,7 @@ const textHolders = (change: Listed, ends: Ends, place: TextPlacement) => { } for (const pointer of change.added) note(pointer, ends.final, "added"); for (const pointer of change.removed) note(pointer, ends.baseline, "removed"); - return { holders, added }; + return { holders, added, items }; }; /** @@ -588,7 +690,7 @@ export const wholeTextSuggestions = ( ends: Ends, place: TextPlacement, ): WholeTextSuggestions => { - const { holders, added } = textHolders(change, ends, place); + const { holders, added, items } = textHolders(change, ends, place); const context = { file, ends, place }; const checks = new Map<string, Placed | undefined>(); const check = (candidate: Candidate) => { @@ -622,6 +724,13 @@ export const wholeTextSuggestions = ( if (found !== undefined && !placed.includes(found)) placed.push(found); }; for (const holder of holders) keep(climb(holder)); + for (const item of items) + keep( + placedItemCheck( + { file, ends, name: place.name, chars: place.limit.chars }, + item, + ), + ); for (const [element, holder] of added) if (holder === undefined || climb(holder) === undefined) keep( @@ -639,6 +748,7 @@ export const wholeTextSuggestions = ( .slice(0, place.limit.suggestions); return { suggestions: suggested.map(({ check }) => check), + guards: suggested.flatMap(({ guards }) => guards ?? []), long: [...long], covers: (pointer) => { if ( diff --git a/src/domain/desktop/appTaskDraftTextAdded.test.ts b/src/domain/desktop/appTaskDraftTextAdded.test.ts index 86344514fe..26d451c0be 100644 --- a/src/domain/desktop/appTaskDraftTextAdded.test.ts +++ b/src/domain/desktop/appTaskDraftTextAdded.test.ts @@ -27,7 +27,7 @@ describe("draftTaskFiles for lines added to free text", () => { expect( passing(FILE, xml(note(`${LIST}\nolive oil`)), checks).every(Boolean), ).toBe(true); - // Free text has no item boundary the current checks can select. + // Whole free text has no item boundary the current checks can select. expect( passing( FILE, @@ -67,3 +67,24 @@ describe("draftTaskFiles for lines added to free text", () => { expect(file.guards).toEqual([]); }); }); + +describe("draftTaskFiles for a line added to a note among others, as CherryTree saves them", () => { + const notebook = (chores: string) => + `<cherrytree><node unique_id="1" name="Weekend chores"><rich_text>${chores}</rich_text></node><node unique_id="2" name="Groceries"><rich_text>Oat milk</rich_text></node><node unique_id="3" name="Gift ideas"><rich_text>Puzzle book</rich_text></node></cherrytree>`; + const CHORES = + "Water the tomato plants\nSweep the back porch\nReplace the hallway bulb\nReturn the library books"; + + it("keeps the sibling count guard beside a whole-text check", () => { + const file = xmlDraft( + notebook(CHORES), + notebook(`${CHORES}\nClean the gutters`), + ); + expect(file.guards).toContainEqual({ + kind: "member_count", + file: "Documents/doc.xml", + pointer: "/root/children", + matching: "[name=node]", + equals: 3, + }); + }); +}); diff --git a/src/domain/desktop/appTaskDraftTree.ts b/src/domain/desktop/appTaskDraftTree.ts new file mode 100644 index 0000000000..87c853679c --- /dev/null +++ b/src/domain/desktop/appTaskDraftTree.ts @@ -0,0 +1,154 @@ +import type { JsonValue } from "../jsonValue.js"; +import type { AddedContext } from "./appTaskDraftAdded.js"; +import { changeNoise } from "./appTaskDraftNoise.js"; +import { draftValues, oneEnd, type DraftValue } from "./appTaskDraftValues.js"; +import { diffNamedMembers } from "./stateChanges.js"; +import { + entryWithoutView, + zipArrayLeavesOut, + zipEntryAt, + zipLeavesOutAt, + zipView, +} from "./zipView.js"; + +const isRecord = ( + value: JsonValue | null | undefined, +): value is Record<string, JsonValue> => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** + * The entry a pointer into a zip view goes into, when it has a view and both + * evaluators read it alike: JSON, a key file or XML. A binary or unread + * entry also holds a `null` view, which is no JSON `null`. + */ +const readableEntry = (view: JsonValue | null, pointer: string) => { + const entry = zipEntryAt(view, pointer); + return typeof entry === "object" && + entryWithoutView(entry) === undefined && + (entry.format === "json" || + entry.format === "keyfile" || + entry.format === "xml") + ? entry + : undefined; +}; + +/** + * Where checks for a pointer the diff lists start: the view of an entry it + * names whole, or its root element for XML; the pointer itself inside that; + * undefined for an entry neither evaluator reads alike, or the entry's fields. + */ +const checkRoot = ( + view: JsonValue | null, + pointer: string, +): string | undefined => { + const entry = readableEntry(view, pointer); + const [entryPointer] = /^\/entries\/[^/]+/u.exec(pointer) ?? []; + if (entry === undefined || entryPointer === undefined) return undefined; + const inner = pointer.slice(entryPointer.length).split("/").slice(1); + const root = entry.format === "xml" ? ["view", "root"] : ["view"]; + const prefix = (short: readonly string[], long: readonly string[]) => + short.length <= long.length && + short.every((part, index) => long[index] === part); + if (prefix(inner, root)) return `${entryPointer}/${root.join("/")}`; + return prefix(root, inner) ? pointer : undefined; +}; + +/** + * Where checks into a view can go: an archive's entries, or an XML file's root + * element. `root` is where checks for a pointer the diff lists start, undefined + * where the evaluators do not read alike; `xml` whether a pointer goes into an + * XML view; `leavesOut` whether the view may lack part of what a pointer + * reaches, and `arrayLeavesOut` members of the array a pointer names; + * `listed` whether a pointer is a value of the document a values list shows; + * `empty` the view of a file that holds nothing. + */ +export interface CheckTree { + readonly root: ( + view: JsonValue | null, + pointer: string, + ) => string | undefined; + readonly xml: (view: JsonValue | null, pointer: string) => boolean; + readonly leavesOut: (view: JsonValue | null, pointer: string) => boolean; + readonly arrayLeavesOut: (view: JsonValue | null, array: string) => boolean; + readonly listed: (view: JsonValue | null, pointer: string) => boolean; + readonly empty: JsonValue | null; +} + +export const ARCHIVE: CheckTree = { + root: checkRoot, + xml: (view, pointer) => readableEntry(view, pointer)?.format === "xml", + leavesOut: zipLeavesOutAt, + arrayLeavesOut: zipArrayLeavesOut, + // An entry added or removed whole, or a value inside a readable one; not an entry's size or digest. + listed: (view, pointer) => + /^\/entries\/[^/]+$/u.test(pointer) || + checkRoot(view, pointer) !== undefined, + // An archive with no entries, so each entry of one created or deleted is listed by its name. + empty: zipView([], 0), +}; + +/** An XML file's view leaves out what its `truncated` says, anywhere in the tree. */ +const truncatedXml = (view: JsonValue | null) => + isRecord(view) && view.truncated === true; + +const inXmlRoot = (pointer: string) => + pointer === "/root" || pointer.startsWith("/root/"); + +export const XML_FILE: CheckTree = { + root: (_view, pointer) => (inXmlRoot(pointer) ? pointer : undefined), + xml: () => true, + leavesOut: truncatedXml, + arrayLeavesOut: truncatedXml, + listed: (_view, pointer) => inXmlRoot(pointer), + empty: null, +}; + +export const valueFormat = ( + tree: CheckTree, + view: JsonValue | null, + pointer: string, +) => + tree === ARCHIVE && /^\/entries\/[^/]+$/u.test(pointer) + ? ("archive" as const) + : tree.xml(view, pointer) + ? ("xml" as const) + : ("json" as const); + +export const referenceRoots = ( + tree: CheckTree, + view: JsonValue | null, +): JsonValue[] => { + if (tree !== ARCHIVE) return view === null ? [] : [view]; + if (!isRecord(view) || !Array.isArray(view.entries)) return []; + return view.entries.flatMap((entry) => + isRecord(entry) && + entry.format === "xml" && + entry.view !== undefined && + entry.view !== null + ? [entry.view] + : [], + ); +}; + +/** + * The values of an archive or XML file created or deleted: what its one end + * holds against `tree.empty`, each entry, or the root element, whole once, as + * one added to a file that changed is listed, within the same limits. + */ +export const presenceValues = ( + ends: AddedContext["ends"], + tree: CheckTree, + created: boolean, +): { readonly values: DraftValue[]; readonly cut: boolean } => { + const from = created + ? { baseline: tree.empty, final: ends.final } + : { baseline: ends.baseline, final: tree.empty }; + const change = oneEnd(diffNamedMembers(from.baseline, from.final), created); + return draftValues(change, { + ends: from, + listed: tree.listed, + noise: changeNoise(change, from, tree.xml), + format: (view, pointer) => valueFormat(tree, view, pointer), + referenceRoots: (view) => referenceRoots(tree, view), + }); +}; diff --git a/src/domain/desktop/appTaskDraftValues.test.ts b/src/domain/desktop/appTaskDraftValues.test.ts index 197cab5c78..740f64ba19 100644 --- a/src/domain/desktop/appTaskDraftValues.test.ts +++ b/src/domain/desktop/appTaskDraftValues.test.ts @@ -1,8 +1,16 @@ import { describe, expect, it } from "vitest"; import type { JsonValue } from "../jsonValue.js"; -import { draftOf, statePath, xml, xmlDraft } from "./appTaskDraft.fixture.js"; -import { compactValue, draftValues } from "./appTaskDraftValues.js"; +import { + draftOf, + passing, + pointerOf, + statePath, + xml, + xmlDraft, +} from "./appTaskDraft.fixture.js"; +import { draftValues } from "./appTaskDraftValues.js"; +import { compactValue } from "./compactValue.js"; import { resolvePointer } from "./viewPointer.js"; import { zipEntry, zipView } from "./zipView.js"; @@ -24,12 +32,13 @@ it("cuts XML previews after escaping, including entity and surrogate boundaries" complete.length, complete.length + 1, ]) - expect(compactValue(element, "xml", limit)).toBe( + expect(compactValue(element, limit, "xml")).toBe( complete.length > limit ? `${complete.slice(0, limit)}…` : complete, ); expect( compactValue( { name: "n", attributes: { value: "&".repeat(1_000_000) }, children: [] }, + 600, "xml", ), ).toBe(`${('<n value="' + "&".repeat(120)).slice(0, 600)}…`); @@ -39,7 +48,7 @@ it("cuts JSON previews after escaping keys and values", () => { const value = { '😀"\n': ["a\n😀\\\ud800"] }; const complete = JSON.stringify(value); for (let limit = 0; limit <= complete.length + 1; limit += 1) - expect(compactValue(value, "json", limit)).toBe( + expect(compactValue(value, limit, "json")).toBe( complete.length > limit ? `${complete.slice(0, limit)}…` : complete, ); }); @@ -58,12 +67,13 @@ describe("draftTaskFiles values", () => { const file = xmlDraft(before, after); expect(file.values_cut).toBe(false); + // Text first, so no cut ever drops it. expect(file.values.map(({ pointer, kind }) => [kind, pointer])).toEqual([ - ["added", "/root/children/[name=ope][@wording=Weekly groceries]"], ["changed", "/root/children/[name=note]/children/0"], + ["added", "/root/children/[name=ope][@wording=Weekly groceries]"], ["removed", "/root/children/[name=tag]"], ]); - const [added, changed, removed] = file.values; + const [changed, added, removed] = file.values; expect(added).toMatchObject({ before: null, after: GROCERIES }); expect(removed).toMatchObject({ before: '<tag name="old"/>', after: null }); // A scalar's before and after are what its pointer reads at each end. @@ -570,3 +580,360 @@ describe("draftTaskFiles values of a file created or deleted", () => { ).toBeLessThanOrEqual(8 * 1024); }); }); + +describe("draftTaskFiles for a transaction added to a book, as GnuCash saves one", () => { + const GROCERIES = "98f0ee4c216a43e290133637df088843"; + const CHECKING = "63fd8736eb444c57803945f330e68a28"; + const account = (name: string, id: string) => + `<gnc:account version="2.0.0"><act:name>${name}</act:name><act:id type="guid">${id}</act:id></gnc:account>`; + const split = (id: string, value: string, to: string) => + `<trn:split><split:id type="guid">${id}</split:id><split:reconciled-state>n</split:reconciled-state><split:value>${value}</split:value><split:account type="guid">${to}</split:account></trn:split>`; + /** A transaction: its id, currency, dates, description, a slot the app keeps and its splits. */ + const transaction = ( + [id, description, posted, entered]: readonly string[], + splits: string, + ) => + `<gnc:transaction version="2.0.0"><trn:id type="guid">${id ?? ""}</trn:id><trn:currency><cmdty:id>USD</cmdty:id></trn:currency><trn:date-posted><ts:date>${posted ?? ""}</ts:date></trn:date-posted><trn:date-entered><ts:date>${entered ?? ""}</ts:date></trn:date-entered><trn:description>${description ?? ""}</trn:description><trn:slots><slot><slot:key>date-posted</slot:key></slot></trn:slots><trn:splits>${splits}</trn:splits></gnc:transaction>`; + const CORNER = transaction( + [ + "1b4e2dae15ea45c5889105a7ae939e04", + "Corner Grocer", + "2026-10-16 10:59:00 +0000", + "2026-11-06 08:49:09 +0000", + ], + split("0aa0a2a505ac40dd9c74f0abfe7e0ca1", "5275/100", GROCERIES) + + split("954a27fff0ad4d54a9a6f26436a60c54", "-5275/100", CHECKING), + ); + const greenleaf = (ids: readonly string[], entered: string) => + transaction( + [ids[0] ?? "", "Greenleaf Market", "2026-11-03 10:59:00 +0000", entered], + split(ids[1] ?? "", "6348/100", GROCERIES) + + split(ids[2] ?? "", "-6348/100", CHECKING), + ); + /** A book: options GnuCash fills on reopening, from control bytes to an id, its accounts and transactions. */ + const gnucash = (option: string, added: string, slots = "") => + `<gnc-v2 xmlns:gnc="g" xmlns:act="a" xmlns:trn="t" xmlns:split="s" xmlns:ts="ts" xmlns:cmdty="c" xmlns:slot="sl" xmlns:book="b"><gnc:book version="2.0.0"><book:slots><slot><slot:key>Default Customer TaxTable</slot:key><slot:value type="guid">${option}</slot:value></slot>${slots}</book:slots>${account("Groceries", GROCERIES)}${account("Checking Account", CHECKING)}${CORNER}${added}</gnc:book></gnc-v2>`; + const SEED = gnucash("??\u007f", ""); + const END = gnucash( + "99471669fefd47b7b6eb2131be4e6d00", + greenleaf( + [ + "a893ecc337254562a3b167b4534dc020", + "c05c3906b2634257883cbf894462acc1", + "d15c3906b2634257883cbf894462acc2", + ], + "2026-11-06 08:53:08 +0000", + ), + "<slot><slot:key>remove-color-not-set-slots</slot:key></slot>", + ); + const TRANSACTION = + "/root/children/[name=gnc:book]/children/[name=gnc:transaction][>trn:description=Greenleaf Market]"; + + it("names the transaction by its description, checks what the user entered under it, counts the transactions, and suggests none of what the app stamps or keeps", () => { + const file = xmlDraft(SEED, END); + + expect( + file.suggested.map((check) => [ + check.kind, + pointerOf(check)?.replace(TRANSACTION, "~"), + ]), + ).toEqual([ + ["text_equals", "~/children/[name=trn:date-posted]"], + [ + "text_equals", + "~/children/[name=trn:splits]/children/[name=trn:split][>split:value=6348~1100]/children/[name=split:account]", + ], + [ + "text_equals", + "~/children/[name=trn:splits]/children/[name=trn:split][>split:value=-6348~1100]/children/[name=split:account]", + ], + ["member_count", "/root/children/[name=gnc:book]/children"], + ]); + // Another save of the same entry: new ids, another time entered. + const later = gnucash( + "11471669fefd47b7b6eb2131be4e6d01", + greenleaf( + [ + "b893ecc337254562a3b167b4534dc021", + "e05c3906b2634257883cbf894462acc3", + "f15c3906b2634257883cbf894462acc4", + ], + "2026-11-09 14:02:41 +0000", + ), + ); + expect(passing("Documents/doc.xml", xml(SEED), file.suggested)).toEqual([ + false, + false, + false, + false, + ]); + expect( + passing("Documents/doc.xml", xml(later), file.suggested).every(Boolean), + ).toBe(true); + }); + + it("lists the transaction field by field and each split whole, naming the account a split's text refers to, and notes the app's ids, entry time and slots", () => { + const file = xmlDraft(SEED, END); + + const listed = file.values.map(({ pointer, note, refers_to }) => [ + pointer.replace(TRANSACTION, "~"), + note ?? null, + refers_to?.map(({ element }) => element) ?? null, + ]); + expect(listed).toEqual( + expect.arrayContaining([ + ["~/children/[name=trn:description]", null, null], + [ + "~/children/[name=trn:splits]/children/[name=trn:split][>split:value=6348~1100]", + null, + ['<gnc:account version="2.0.0"><act:name>Groceries</act:name>…'], + ], + [ + "~/children/[name=trn:splits]/children/[name=trn:split][>split:value=-6348~1100]", + null, + [ + '<gnc:account version="2.0.0"><act:name>Checking Account</act:name>…', + ], + ], + ["~/children/[name=trn:id]", "volatile", null], + ["~/children/[name=trn:date-entered]", "timestamp", null], + ["~/children/[name=trn:slots]/children/[name=slot]", "upkeep", null], + ]), + ); + const slots = file.values.filter(({ pointer }) => + pointer.includes("[name=book:slots]"), + ); + expect(slots.map(({ note }) => note)).toEqual(["upkeep", "upkeep"]); + }); +}); + +/** A split of a transaction: its id, which KMyMoney numbers anew in every transaction, amount, account and payee. */ +const split = (id: string, value: string, account: string, payee: string) => + `<SPLIT value="${value}" shares="${value}" memo="" account="${account}" price="1/1" id="${id}" payee="${payee}" reconcileflag="0"/>`; +const transaction = (id: string, date: string, splits: string) => + `<TRANSACTION memo="" entrydate="2026-11-06" postdate="${date}" id="${id}" commodity="USD"><SPLITS>${splits}</SPLITS></TRANSACTION>`; +const WATER = transaction( + "T000000000000000001", + "2026-10-12", + split("S0001", "-209/5", "A000001", "P000002") + + split("S0002", "209/5", "A000011", "P000002"), +); +const ELECTRIC = transaction( + "T000000000000000002", + "2026-10-20", + split("S0001", "-921/10", "A000001", "P000003") + + split("S0002", "921/10", "A000011", "P000003"), +); +const PAYMENT = transaction( + "T000000000000000003", + "2026-11-03", + split("S0001", "-1749/20", "A000001", "P000003") + + split("S0002", "1749/20", "A000011", "P000003"), +); +/** A book: a file version whose id is a small number, payees, accounts the institution and parent list by id, and transactions. */ +const kmy = (count: string, added: string) => + `<KMYMONEY-FILE><FILEINFO><FIXVERSION id="3"/></FILEINFO><INSTITUTION id="I000001" name="Maple Grove Credit Union"><ACCOUNTIDS><ACCOUNTID id="A000001"/></ACCOUNTIDS></INSTITUTION><PAYEES><PAYEE id="P000002" name="Northside Water Co."/><PAYEE id="P000003" name="Riverside Electric"/></PAYEES><ACCOUNTS><ACCOUNT id="AStd::Expense" name="Expense"><SUBACCOUNTS><SUBACCOUNT id="A000011"/></SUBACCOUNTS></ACCOUNT><ACCOUNT id="A000001" name="Household Checking"/><ACCOUNT id="A000011" name="Utilities"/></ACCOUNTS><TRANSACTIONS count="${count}">${WATER}${ELECTRIC}${added}</TRANSACTIONS></KMYMONEY-FILE>`; +const SEED = kmy("2", ""); +const END = kmy("3", PAYMENT); +const SPLITS = + "/root/children/[name=TRANSACTIONS]/children/[name=TRANSACTION][@postdate=2026-11-03]/children/[name=SPLITS]/children"; + +describe("draftTaskFiles for a transaction added to a book, as KMyMoney saves one", () => { + it("omits an oversized nested split field while retaining short checks and its note", () => { + const large = "x".repeat(100_000); + const end = kmy( + "3", + PAYMENT.replace( + 'memo="" account="A000001"', + `memo="${large}" account="A000001"`, + ), + ); + const file = xmlDraft(SEED, end); + expect( + file.suggested + .filter((check) => JSON.stringify(check).length >= 10_000) + .map((check) => ({ + kind: check.kind, + pointer: "pointer" in check ? check.pointer.slice(0, 220) : "", + })), + ).toEqual([]); + expect(file.suggested).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "value_equals", value: "-1749/20" }), + ]), + ); + expect(file.notes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "long_text", + pointers: expect.arrayContaining([ + `${SPLITS}/[name=SPLIT][@account=A000001]/attributes/memo`, + ]), + }), + ]), + ); + }); + + it("checks each split of the new transaction by the account it names, its amount and payee, not by the split id every transaction repeats", () => { + const file = xmlDraft(SEED, END); + + const field = (account: string, key: string, value: string) => ({ + kind: "value_equals", + file: "Documents/doc.xml", + pointer: `${SPLITS}/[name=SPLIT][@account=${account}]/attributes/${key}`, + value, + }); + // `shares` repeats `value` in every split, and `memo`, `price` and `reconcileflag` are alike in all. + expect(file.suggested.slice(0, 4)).toEqual([ + field("A000001", "value", "-1749/20"), + field("A000001", "payee", "P000003"), + field("A000011", "value", "1749/20"), + field("A000011", "payee", "P000003"), + ]); + const checks = file.suggested.slice(0, 4); + expect(passing("Documents/doc.xml", xml(SEED), checks)).toEqual([ + false, + false, + false, + false, + ]); + expect(passing("Documents/doc.xml", xml(END), checks)).toEqual([ + true, + true, + true, + true, + ]); + // A wrong amount fails. + const wrong = END.replace('value="1749/20"', 'value="1700/20"'); + expect(passing("Documents/doc.xml", xml(wrong), checks)).toEqual([ + true, + true, + false, + true, + ]); + }); +}); + +describe("draftTaskFiles for references in KMyMoney transactions", () => { + it("names the account and payee each split refers to, the account over an id list naming it too, and no element a plain count equals", () => { + const file = xmlDraft(SEED, END); + + const added = file.values.find(({ kind }) => kind === "added"); + expect( + added?.refers_to?.map(({ attribute, value, element }) => [ + attribute, + value, + element, + ]), + ).toEqual([ + [ + "account", + "A000001", + '<ACCOUNT id="A000001" name="Household Checking"/>', + ], + ["payee", "P000003", '<PAYEE id="P000003" name="Riverside Electric"/>'], + ["account", "A000011", '<ACCOUNT id="A000011" name="Utilities"/>'], + ]); + // count="3" equals FIXVERSION's id, but `count` names no element. + const count = file.values.find(({ pointer }) => + pointer.endsWith("/attributes/count"), + ); + expect(count).toMatchObject({ before: "2", after: "3" }); + expect(count?.refers_to).toBeUndefined(); + }); + + it("checks a split moved to a new account, though its value equals the id the new account took, so a solve that never moves it fails", () => { + const opened = (seed: string) => + seed + .replace( + '<SUBACCOUNT id="A000011"/>', + '<SUBACCOUNT id="A000011"/><SUBACCOUNT id="A000012"/>', + ) + .replace( + '<ACCOUNT id="A000011" name="Utilities"/>', + '<ACCOUNT id="A000011" name="Utilities"/><ACCOUNT id="A000012" name="Water"/>', + ); + const moved = opened(SEED).replace( + 'account="A000011" price="1/1" id="S0002" payee="P000002"', + 'account="A000012" price="1/1" id="S0002" payee="P000002"', + ); + const file = xmlDraft(SEED, moved); + + const account = file.values.find( + ({ pointer, after }) => + pointer.endsWith("/attributes/account") && after === "A000012", + ); + expect(account).toBeDefined(); + expect(account?.note).toBeUndefined(); + expect( + passing("Documents/doc.xml", xml(moved), file.suggested), + ).not.toContain(false); + // Opening the account alone leaves the water bill where it was. + expect( + passing("Documents/doc.xml", xml(opened(SEED)), file.suggested), + ).toContain(false); + }); +}); + +describe("draftTaskFiles values' other pointers", () => { + it("lists a changed value's pointer by each other name its element holds alone, as an Inkscape text's label beside its id", () => { + const poster = (headline: string) => + `<svg xmlns:inkscape="i"><text id="text12" inkscape:label="Headline" x="10"><tspan>${headline}</tspan></text><text id="text14" inkscape:label="Venue" x="10"><tspan>Pier 4</tspan></text></svg>`; + + const file = xmlDraft(poster("Jazz Night"), poster("Jazz Festival")); + + const [changed] = file.values; + expect(changed).toMatchObject({ + pointer: + "/root/children/[name=text][@id=text12]/children/[name=tspan]/children/0", + after: "Jazz Festival", + alternatives: [ + "/root/children/[name=text][@inkscape:label=Headline]/children/[name=tspan]/children/0", + ], + }); + // Each reads the same value at the end. + expect( + resolvePointer( + xml(poster("Jazz Festival")), + changed?.alternatives?.[0] ?? "", + ), + ).toBe("Jazz Festival"); + }); +}); + +describe("draftTaskFiles values in an archive's entries", () => { + it("lists a chapter title's pointer by the id its h1 holds, though its tag alone names it", () => { + const chapter = (title: string) => + `<html><body><h1 id="chapter-two">${title}</h1><p>The supply boat brought one letter.</p></body></html>`; + const epub = (title: string) => { + const entry = zipEntry( + { + name: "OEBPS/Text/Section0002.xhtml", + size: 1, + compressedSize: 1, + method: 8, + encrypted: false, + }, + new TextEncoder().encode(chapter(title)), + ); + return zipView([entry], 1); + }; + const [file] = draftOf({ + "Documents/book.epub": statePath( + "document", + "zip", + epub("A Letter from the Mainlnad"), + epub("A Letter from the Mainland"), + ), + }); + + const h1 = + "/entries/[name=OEBPS~1Text~1Section0002.xhtml]/view/root/children/[name=body]/children/[name=h1]"; + expect(file?.values[0]).toMatchObject({ + pointer: `${h1}/children/0`, + alternatives: [ + "/entries/[name=OEBPS~1Text~1Section0002.xhtml]/view/root/children/[name=body]/children/[name=h1][@id=chapter-two]/children/0", + ], + }); + }); +}); diff --git a/src/domain/desktop/appTaskDraftValues.ts b/src/domain/desktop/appTaskDraftValues.ts index 786e37cc94..d42bcad0ff 100644 --- a/src/domain/desktop/appTaskDraftValues.ts +++ b/src/domain/desktop/appTaskDraftValues.ts @@ -6,9 +6,20 @@ import { NOISE_REASONS, type NoiseReason, } from "./appTaskDraftNoise.js"; -import { isIdentifyingAttribute } from "./attributeRank.js"; +import { isIdentifyingAttribute, isViewStateName } from "./attributeRank.js"; import type { DesktopStateDifference } from "./stateChanges.js"; -import { isXmlElement, locatePointer, resolvePointer } from "./viewPointer.js"; +import { alternativePointers } from "./appTaskDraftAlternatives.js"; +import { isIdReference } from "./appTaskDraftMirrors.js"; +import { recordListsOf } from "./appTaskDraftNested.js"; +import { holdsFields, isRecordAt } from "./appTaskDraftText.js"; +import { attributesOf, compactValue, previewWriter } from "./compactValue.js"; +import { pairMembers, pairedAtSeed } from "./memberPairing.js"; +import { + escapeToken, + isXmlElement, + locatePointer, + resolvePointer, +} from "./viewPointer.js"; /** Why a value's pointer cannot go into a check as it is, or why the value is noise. */ export const VALUE_NOTES = ["by_position", ...NOISE_REASONS] as const; @@ -31,6 +42,7 @@ export interface DraftValue { readonly after: JsonValue; readonly note?: ValueNote; readonly refers_to?: readonly ValueReference[]; + readonly alternatives?: readonly string[]; } /** One entry of a drafted file's `values`, as the draft writes it. */ @@ -52,7 +64,7 @@ export const draftValueSchema = z.strictObject({ .enum(VALUE_NOTES) .optional() .describe( - "by_position: no selector names it, so no check can use the pointer; view_state, renumbered, volatile, timestamp or upkeep: noise the app writes, left out of the suggested checks.", + "by_position: no selector names it, so no check can use the pointer; view_state, renumbered, volatile, timestamp, upkeep or layout: noise the app writes, left out of the suggested checks.", ), refers_to: z .array( @@ -66,6 +78,12 @@ export const draftValueSchema = z.strictObject({ .describe( 'Other elements a value names by their key or id, such as category 44 naming <cat key="44" name="Groceries"/>.', ), + alternatives: z + .array(z.string()) + .optional() + .describe( + "A changed value's pointer naming its element by another identifier or name it holds alone, such as a shape's draw:name, for a check to use instead; at most three.", + ), }); /** Bounds on the values a draft lists, so a large change cannot crowd out the rest of it. */ @@ -74,7 +92,7 @@ export const VALUE_LIMITS = { fileBytes: 8 * 1024, draftBytes: 24 * 1024, chars: 600, - references: 3, + references: 5, } as const; const isRecord = ( @@ -82,154 +100,6 @@ const isRecord = ( ): value is Record<string, JsonValue> => typeof value === "object" && value !== null && !Array.isArray(value); -const cut = (text: string, max: number) => - text.length > max ? `${text.slice(0, max)}…` : text; - -/** An element's attributes; none for a value that holds none. */ -const attributesOf = (element: Record<string, JsonValue>) => { - const { attributes } = element; - return isRecord(attributes) ? attributes : {}; -}; - -/** Keep one character past the limit to distinguish an exact fit from a cut. */ -const previewWriter = (max: number) => { - const parts: string[] = []; - let length = 0; - const push = (text: string): boolean => { - const kept = text.slice(0, Math.max(0, max + 1 - length)); - parts.push(kept); - length += kept.length; - return length <= max; - }; - const escaped = (text: string, attribute = false): boolean => { - for (let index = 0; index < text.length; index += 1) { - const char = text[index] ?? ""; - if ( - !push( - char === "&" - ? "&" - : char === "<" - ? "<" - : attribute && char === '"' - ? """ - : char, - ) - ) - return false; - } - return true; - }; - const jsonString = (text: string): boolean => { - if (!push('"')) return false; - for (let index = 0; index < text.length; index += 1) { - const code = text.codePointAt(index); - const width = code !== undefined && code > 0xffff ? 2 : 1; - const encoded = JSON.stringify(text.slice(index, index + width)); - if (!push(encoded.slice(1, -1))) return false; - index += width - 1; - } - return push('"'); - }; - const startTag = ( - element: Record<string, JsonValue> & { name: string }, - ): boolean => { - if (!push("<") || !push(element.name)) return false; - const attributes = attributesOf(element); - for (const key in attributes) { - if (!Object.hasOwn(attributes, key)) continue; - if ( - !push(" ") || - !push(key) || - !push('="') || - !escaped(String(attributes[key]), true) || - !push('"') - ) - return false; - } - return true; - }; - return { - push, - escaped, - jsonString, - startTag, - result: () => cut(parts.join(""), max), - }; -}; - -/** An archive entry's own fields, without its view. */ -const isArchiveEntry = (value: JsonValue): value is Record<string, JsonValue> => - isRecord(value) && "sha256" in value && "view" in value && "name" in value; - -/** - * A value as an author reads it: a scalar as it is, cut past `max` - * characters; an element as compact XML, with every text node the XML reader - * kept, which already leaves out the whitespace an app lays out between - * elements; an archive entry as its name, format and size; anything else as - * JSON. Writing stops once past `max`, so a large subtree costs no more than - * the cut. - */ -export const compactValue = ( - value: JsonValue | undefined, - format: "json" | "xml" | "archive", - max: number = VALUE_LIMITS.chars, -): JsonValue => { - if (value === undefined) return null; - if (typeof value === "string") return cut(value, max); - if (value === null || typeof value !== "object") return value; - const { push, escaped, jsonString, startTag, result } = previewWriter(max); - const write = (node: JsonValue): boolean => { - if (format === "xml" && isXmlElement(node)) { - if (!startTag(node)) return false; - if (node.children.length === 0) return push("/>"); - return ( - push(">") && - node.children.every((child) => - typeof child === "string" ? escaped(child) : write(child), - ) && - push("</") && - push(node.name) && - push(">") - ); - } - if (format === "archive" && isArchiveEntry(node)) - return write({ - ...(node.name === undefined ? {} : { name: node.name }), - format: node.format ?? null, - size: node.size ?? null, - }); - if (Array.isArray(node)) - return ( - push("[") && - node.every( - (item, index) => (index === 0 || push(",")) && write(item), - ) && - push("]") - ); - if (isRecord(node)) { - if (!push("{")) return false; - let first = true; - for (const key in node) { - if (!Object.hasOwn(node, key)) continue; - if ( - (!first && !push(",")) || - !jsonString(key) || - !push(":") || - !write(node[key] ?? null) - ) - return false; - first = false; - } - return push("}"); - } - return typeof node === "string" - ? jsonString(node) - : push(JSON.stringify(node)); - }; - write(value); - return result(); -}; - /** The longest value a reference is resolved from. */ const MAX_REFERENCE_CHARS = 64; @@ -257,22 +127,37 @@ const indexUnder = ( }; /** - * XML elements by the value of their `key` or `id` attribute, across a view, - * and by tag: at most `PER_TAG` of a tag in document order, so a key that - * records or archive entries repeat costs no more than that. + * XML elements by the value of their `key` or `id` attribute, `xmi:id` + * among them, across a view, and by tag: at most `PER_TAG` of a tag in + * document order, so a key that records or archive entries repeat costs no + * more than that. A view the app keeps of the document, as Umbrello's + * `listview` mirroring its model by id, names nothing. */ const referenceIndex = (roots: readonly JsonValue[]) => { const index = new Map<string, Map<string, JsonValue[]>>(); const pending: JsonValue[] = [...roots]; for (let node = pending.pop(); node !== undefined; node = pending.pop()) { - if (isXmlElement(node)) + if (isXmlElement(node) && isViewStateName(node.name)) continue; + if (isXmlElement(node)) { for (const [key, value] of Object.entries(attributesOf(node))) if ( typeof value === "string" && referable(value) && - /^(?:key|id)$/iu.test(key.split(":").at(-1) ?? key) + /^(?:key|id)$/iu.test(localOf(key)) ) indexUnder(index, value, node); + // An id an app keeps as a child's text, as GnuCash's `<act:id type="guid">`, names the element it sits in. + for (const child of node.children) { + const id = leafText(child); + if ( + isXmlElement(child) && + /^id$/iu.test(localOf(child.name)) && + id !== undefined && + referable(id) + ) + indexUnder(index, id, node); + } + } const children = isXmlElement(node) ? node.children : Array.isArray(node) @@ -285,6 +170,53 @@ const referenceIndex = (roots: readonly JsonValue[]) => { return index; }; +/** A name without its namespace prefix. */ +const localOf = (name: string) => name.split(":").at(-1) ?? name; + +/** An element's text when it holds only one text node, as `<act:name>Groceries</act:name>`. */ +const leafText = (node: JsonValue): string | undefined => { + const [only] = isXmlElement(node) ? node.children : []; + return isXmlElement(node) && + node.children.length === 1 && + typeof only === "string" + ? only + : undefined; +}; + +/** An element as a reference shows it: its start tag, and its name where a child holds one, as `<gnc:account version="2.0.0"><act:name>Groceries</act:name>…`. */ +const referenceTag = ( + element: Record<string, JsonValue> & { name: string }, +) => { + const writer = previewWriter(VALUE_LIMITS.chars); + if (!writer.startTag(element)) return writer.result(); + const children = isXmlElement(element) ? element.children : []; + if (children.length === 0) { + writer.push("/>"); + return writer.result(); + } + if (!writer.push(">")) return writer.result(); + const named = children.find( + (child) => + isXmlElement(child) && + /^(?:name|title|label)$/iu.test(localOf(child.name)) && + leafText(child) !== undefined, + ); + if (isXmlElement(named)) { + if ( + !writer.push("<") || + !writer.push(named.name) || + !writer.push(">") || + !writer.escaped(leafText(named) ?? "") || + !writer.push("</") || + !writer.push(named.name) || + !writer.push(">") + ) + return writer.result(); + } + writer.push("…"); + return writer.result(); +}; + /** Whether an attribute's name and an element's tag share a stem, as HomeBank's `payee` and `<pay>`. */ const stemOf = (attribute: string, tag: string) => { const [left, right] = [attribute, tag].map((name) => @@ -298,7 +230,7 @@ const stemOf = (attribute: string, tag: string) => { ); }; -/** What a value a pointer reads names: the one other element whose `key` or `id` it equals, or, among several, the one whose tag shares the attribute's stem. */ +/** What a value a pointer reads names: the one other element whose `key` or `id` it equals, or, among several, the one whose tag shares the attribute's stem, the one named for it first; a plain number only through an attribute so named, or one ending in `id`, `ref` or `key`. */ const referencesFrom = ( index: ReadonlyMap<string, ReadonlyMap<string, readonly JsonValue[]>>, ) => { @@ -308,15 +240,33 @@ const referencesFrom = ( self: JsonValue | undefined, ): ValueReference | undefined => { if (!referable(value)) return undefined; + // A plain number, as a list's `count="5"`, names an element only through an attribute named for one. + const plain = /^-?\d+(?:\.\d+)?$/u.test(value); const found = [...(index.get(value)?.values() ?? [])] .flat() - .filter((node) => node !== self); - const hinted = + .filter( + (node) => + node !== self && + (!plain || + (attribute !== undefined && + isXmlElement(node) && + (stemOf(attribute, node.name) || + /(?:id|ref|key)$/iu.test(localOf(attribute))))), + ); + const stemmed = found.length === 1 || attribute === undefined ? found : found.filter( (node) => isXmlElement(node) && stemOf(attribute, node.name), ); + // Among several, the tag the attribute is named for, as `account` an `<ACCOUNT>` before an `<ACCOUNTID>`. + const named = stemmed.filter( + (node) => + isXmlElement(node) && + localOf(node.name).toLowerCase() === + localOf(attribute ?? "").toLowerCase(), + ); + const hinted = stemmed.length > 1 && named.length === 1 ? named : stemmed; const [element] = hinted; return hinted.length === 1 && isXmlElement(element) ? { @@ -329,17 +279,15 @@ const referencesFrom = ( return resolve; }; -const referenceTag = ( - element: Record<string, JsonValue> & { name: string }, -): string => { - const writer = previewWriter(VALUE_LIMITS.chars); - if (writer.startTag(element)) - writer.push( - element.children instanceof Array && element.children.length === 0 - ? "/>" - : ">…", - ); - return writer.result(); +/** References without repeats, the first of each attribute and value. */ +const uniqueReferences = (references: readonly ValueReference[]) => { + const seen = new Set<string>(); + return references.filter(({ attribute, value }) => { + const key = JSON.stringify([attribute, value]); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); }; /** What a values list reads: a file's two ends and how it treats each pointer. */ @@ -362,10 +310,28 @@ export interface ValueContext { const byteLength = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)).length; +/** Records of attributes an element nests whose references a value lists, as the splits of a transaction. */ +const MAX_NESTED_RECORDS = 10; + +/** The references in an element's attributes other than its own identifiers; of one holding nothing but an identifier, as Parley's `<entry id="6"/>`, that identifier's. */ +const attributeReferences = ( + resolve: ReturnType<typeof referencesFrom>, + element: JsonValue, +) => + Object.entries(isXmlElement(element) ? element.attributes : {}).flatMap( + ([key, item]) => + typeof item === "string" && + ((!isIdentifyingAttribute(key) && !/^key$/iu.test(key)) || + isIdReference(element)) + ? (resolve(item, key, element) ?? []) + : [], + ); + /** * The elements a value names: each attribute of an element by its own value, - * other than the element's own identifiers, or a scalar, with the attribute - * that holds it. + * other than the element's own identifiers, and of the records of + * attributes it nests, as a KMyMoney split's `account` and `payee`; or a + * scalar, with the attribute that holds it. Each once. */ const referencesOf = ( resolve: ReturnType<typeof referencesFrom>, @@ -374,13 +340,22 @@ const referencesOf = ( ): ValueReference[] => { const value = resolvePointer(view, pointer); if (isXmlElement(value)) - return Object.entries(attributesOf(value)).flatMap(([key, item]) => - typeof item === "string" && - !isIdentifyingAttribute(key) && - !/^key$/iu.test(key) - ? (resolve(item, key, value) ?? []) - : [], - ); + return uniqueReferences([ + ...attributeReferences(resolve, value), + ...recordListsOf(value) + .flatMap(({ records }) => records) + .slice(0, MAX_NESTED_RECORDS) + .flatMap((record) => attributeReferences(resolve, record)), + // A field holding another element's id as its text, as a split's `<split:account>`. + ...[value, ...value.children].flatMap((field) => { + const text = leafText(field); + return isXmlElement(field) && + text !== undefined && + !/^id$/iu.test(localOf(field.name)) + ? (resolve(text, field.name, value) ?? []) + : []; + }), + ]); if (typeof value !== "string") return []; const segments = pointer.split("/"); const attribute = @@ -389,6 +364,15 @@ const referencesOf = ( return [resolve(value, attribute, owner) ?? []].flat(); }; +/** What a pointer names at the seed; an attribute of a member keyed by its new name, as a renamed layer, read from the member it pairs with there. */ +const seedValue = ({ ends }: ValueContext, pointer: string) => { + const key = attributeKeyOf(pointer); + const seen = resolvePointer(ends.baseline, pointer); + if (seen !== undefined || key === undefined) return seen; + const member = pairedAtSeed(ends, pointer.split("/").slice(0, -2).join("/")); + return isXmlElement(member) ? member.attributes[key] : undefined; +}; + /** A value's entry, with the references it holds, before any limit. */ const entryOf = ( context: ValueContext, @@ -420,6 +404,12 @@ const entryOf = ( VALUE_LIMITS.references, ) : []; + const alternatives = + kind === "changed" && + note === undefined && + context.format(view, pointer) === "xml" + ? alternativePointers(view, pointer) + : []; return { pointer, kind, @@ -427,7 +417,8 @@ const entryOf = ( kind === "added" ? null : compactValue( - resolvePointer(baseline, pointer), + seedValue(context, pointer), + VALUE_LIMITS.chars, context.format(baseline, pointer), ), after: @@ -435,16 +426,71 @@ const entryOf = ( ? null : compactValue( resolvePointer(final, pointer), + VALUE_LIMITS.chars, context.format(final, pointer), ), ...(note === undefined ? {} : { note }), ...(refers.length === 0 ? {} : { refers_to: refers }), + ...(alternatives.length === 0 ? {} : { alternatives }), }; }; -/** How an entry sorts: what a check can name first, then what only position reaches, then noise. */ +/** + * The pointers a value added whole is listed by: a record of text fields in + * a list of its kind, as GnuCash's `<gnc:transaction>`, by each of its + * fields, and a field that lists such records, as its `<trn:splits>`, by + * each record, so the cut on one value never hides a field; anything else + * whole, as a document's root or a FreeCAD object of property lists. + */ +const fieldsOf = (view: JsonValue | null, pointer: string): string[] => { + const value = resolvePointer(view, pointer); + const list = resolvePointer(view, pointer.replace(/\/[^/]+$/u, "")); + if ( + !isXmlElement(value) || + !/\/children\/[^/]+$/u.test(pointer) || + !Array.isArray(list) || + list.filter((item) => isXmlElement(item) && item.name === value.name) + .length < 2 || + !isRecordAt(view, pointer, value) || + !value.children.some((child) => leafText(child) !== undefined) + ) + return [pointer]; + const members = (element: JsonValue) => { + const { keys } = pairMembers( + isXmlElement(element) ? element.children : [], + [], + ).newer; + return (isXmlElement(element) ? element.children : []).flatMap( + (child, index) => + isXmlElement(child) + ? [{ child, key: escapeToken(keys[index] ?? String(index)) }] + : [], + ); + }; + return members(value).flatMap(({ child, key }) => { + const at = `${pointer}/children/${key}`; + const records = members(child); + return records.length > 0 && + records.every( + ({ child: record }) => + record.name === records[0]?.child.name && holdsFields(record), + ) + ? records.map((record) => `${at}/children/${record.key}`) + : [at]; + }); +}; + +/** How an entry sorts: what a check can name first, text before attributes and whole elements, so a cut never drops the edit; then what only position reaches; then noise. */ const rank = (entry: DraftValue) => - entry.note === undefined ? 0 : entry.note === "by_position" ? 1 : 2; + entry.note === undefined + ? typeof entry.after === "string" && + !entry.after.startsWith("<") && + /\/children\/\d+$/u.test(entry.pointer) + ? 0 + : 1 + : entry.note === "by_position" + ? 2 + : 3; /** * The values a file's net change added, changed and removed, at the pointers @@ -481,6 +527,11 @@ export const draftValues = ( const entries = [ ...change.added .filter((pointer) => context.listed(final, pointer)) + .flatMap((pointer) => + context.format(final, pointer) === "xml" + ? fieldsOf(final, pointer) + : [pointer], + ) .map((pointer) => entryOf(context, references, pointer, "added")), ...change.changed .filter((pointer) => context.listed(final, pointer)) diff --git a/src/domain/desktop/appTaskPython.ts b/src/domain/desktop/appTaskPython.ts index 5624fabb66..a8b50a59fb 100644 --- a/src/domain/desktop/appTaskPython.ts +++ b/src/domain/desktop/appTaskPython.ts @@ -175,7 +175,8 @@ def _xml_text(entry, view, pointer: str, found) -> bool: def _near(at: str, found, expected, tolerance, join: bool): """A value compared within a tolerance: numbers, and numeral strings, by their difference. An element's text is joined only when join says the view is XML.""" - number = _number(_element_text(found) if join else found) + read = _element_text(found) if join else found + number = _number(read.strip() if isinstance(read, str) else read) within = f"within {_shown(tolerance)} of {_shown(expected)}" if number is None: return False, f"{at} is {_shown(found)}, not a number" @@ -327,6 +328,8 @@ def _members(check, entry): if kind == "unresolved": return False, f"{at}: {found}" if kind == "none": + if _array_leaves_out(form, view, pointer): + return _undecided(check) return False, f"{at} has no value" if found is None else f"{at}: {found}" if not isinstance(found, list): return False, f"{at} is {_shown(found)}, not an array" diff --git a/src/domain/desktop/appTaskPythonPointer.ts b/src/domain/desktop/appTaskPythonPointer.ts index 0f9fec80f9..b2e0c15be0 100644 --- a/src/domain/desktop/appTaskPythonPointer.ts +++ b/src/domain/desktop/appTaskPythonPointer.ts @@ -41,11 +41,41 @@ def _selector(token: str): return conditions +_ISO_STAMP = re.compile( + r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:[.,]\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?$", re.ASCII +) +_ASCTIME_STAMP = re.compile( + r"^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [ 0-3]?\d \d{2}:\d{2}:\d{2} \d{4}$", re.ASCII +) + + +def _is_saved_time(text: str) -> bool: + stripped = text.strip() + return _ISO_STAMP.fullmatch(stripped) is not None or _ASCTIME_STAMP.fullmatch(stripped) is not None + + +def _text_without_stamps(node) -> str: + """An element's text with save-time leaves removed, matching the selector that named it.""" + if isinstance(node, str): + return "" if _is_saved_time(node) else node + if not _is_element(node): + return "" + return "".join(_text_without_stamps(child) for child in node["children"]) + + def _has_child_text(member, key: str, value: str) -> bool: - """Whether an element has a child element named key whose text, its whitespace collapsed, is value.""" - return _is_element(member) and any( - _is_element(child) and child["name"] == key and _collapse_whitespace(_element_text(child)) == value - for child in member["children"]) + """Whether an element has a child element named key whose text, its whitespace collapsed, is value. A nested + save time is not part of the name. A child that is only a time still matches that time.""" + if not _is_element(member): + return False + for child in member["children"]: + if not _is_element(child) or child["name"] != key: + continue + plain = _collapse_whitespace(_text_without_stamps(child)) + text = plain if plain else _collapse_whitespace(_element_text(child)) + if text == value: + return True + return False def _selects(member, conditions) -> bool: diff --git a/src/domain/desktop/attributeRank.ts b/src/domain/desktop/attributeRank.ts index d75cc6c4df..50895f1814 100644 --- a/src/domain/desktop/attributeRank.ts +++ b/src/domain/desktop/attributeRank.ts @@ -4,12 +4,14 @@ export const isIdentifyingAttribute = (key: string): boolean => /** * Words in a name for how the app showed, printed or kept the document rather - * than what it holds: its cursor, selection, scroll, zoom, windows, view and - * configuration settings, printer, and a save's history, edit time and count, - * generator and statistics. + * than what it holds: its cursor, selection (VYM's `<select>`), the layer, + * frame, view or colour it has current, scroll, zoom, windows, view and + * configuration settings, a list or tree view mirroring the document, + * printer, a save's history, edit time and count, generator and statistics, + * and the preview or thumbnail it renders. */ export const VIEW_STATE = - /cursor|selection|selected|scroll|zoom|viewport|window|splitter|toplevel|topleft|uidata|namedview|view-?settings|config(?:uration)?-?settings|printer|history|edit(?:ing)?-?(?:time|duration|cycles?)|last-?saved|generator|document-?statistic/iu; + /cursor|^select$|selection|selected|current[-_]?(?:layer|frame|view|colou?r|page|tab|tool|index|item|row|column|cell|sheet|slide|object|scene|track)|scroll|zoom|viewport|window|splitter|toplevel|topleft|uidata|namedview|view-?settings|(?:list|tree)-?view|config(?:uration)?-?settings|printer|history|edit(?:ing)?-?(?:time|duration|cycles?)|last-?saved|generator|document-?statistic|preview|thumbnail/iu; /** Whether a tag or attribute name is about how the app showed the document rather than what it holds. */ export const isViewStateName = (name: string): boolean => VIEW_STATE.test(name); @@ -24,24 +26,69 @@ const STYLE_NAME = /(?:style|font|family|class|type)-?name$/u; const HEX_TOKEN = /^[0-9a-f-]{8,}$/iu; const LONG_TOKEN = /^(?=.*\d)[A-Za-z0-9+/=_-]{16,}$/u; +/** Whether a value stands for no value, as Grisbi writes `Id="(null)"` on every transaction: it names nothing. */ +export const isPlaceholder = (value: string): boolean => + /^\(?(?:null|nil|none|undefined)\)?$/iu.test(value.trim()); + +/** Umbrello's ids: `u` and twelve letters and digits mixing cases at random, as `ubcgF2AWfBmHh`, with a digit or four or more changes of case. */ +const isRandomCaseId = (value: string): boolean => + /^u(?=[^]*[A-Z])(?=[^]*[a-z])[A-Za-z\d]{12}$/u.test(value) && + (/\d/u.test(value) || + (value.match(/\p{Ll}(?=\p{Lu})|\p{Lu}(?=\p{Ll})/gu)?.length ?? 0) >= 4); + +/** Whether a value is a GUID, braced or not, as VYM's `uuid="{18eadb5d-04fc-4448-a0f9-1f9a3dedaff7}"` on each branch. */ +export const isGuid = (value: string): boolean => + /^\{?[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\}?$/iu.test(value); + +/** Whether a value is a uid an app generates: a GUID, or 32 hex digits without dashes, as TreeLine's uuid1 `fc11808dd9f511f181b7edd4f1920549`. */ +export const isGeneratedUid = (value: string): boolean => + isGuid(value) || /^(?=[^]*[a-f])[0-9a-f]{32}$/iu.test(value); + +/** Whether a value is a word a program makes, a hex run, GUID, long token or an id of random case, rather than one a user typed. */ +export const isGeneratedToken = (value: string): boolean => + HEX_TOKEN.test(value) || + isGuid(value) || + LONG_TOKEN.test(value) || + isRandomCaseId(value); + +/** Whether an identifier's value is one an app generates for a new element, as Freeplane's `ID_329023692` or a GUID, which another save of the same edit would not repeat. */ +export const isGeneratedId = (value: string): boolean => + /^[A-Za-z]*[_-]?\d{6,}$/u.test(value) || isGeneratedToken(value); + +/** Whether an identifier is one an ODF app numbers anew as it saves, as LibreOffice's `xml:id="id2"` and `draw:id="id2"` on a shape: where the shape came in that save, not which shape it is. */ +export const isSaveNumberedId = (key: string, value: string): boolean => + /^(?:xml|draw):id$/iu.test(key) && /^id\d+$/u.test(value); + /** Whether a key names what its element is called rather than a style or font it uses. */ export const isNameLikeKey = (key: string): boolean => { const local = localName(key); return NAME_LIKE.test(local) && !STYLE_NAME.test(local); }; +/** A length with its unit, as ODF's `svg:height="2.629cm"`: a coordinate, however it is spelled. */ +const MEASURE = /^-?\d+(?:\.\d+)?(?:cm|mm|in|pt|pc|px|em|%)$/u; + +/** Whether an attribute names an automatic style an app numbers as it saves, as LibreOffice's `presentation:style-name="pr1"` or `text:style-name="MT1"`, which it shares and renumbers on reload. */ +export const isAutomaticStyle = (key: string, value: string): boolean => + /style-name$/iu.test(key) && /^[A-Za-z]{1,4}\d+$/u.test(value); + /** * How well an attribute names its element across saves, best first: 0 a * name-like key, 1 a worded value that is no generated token, 2 an integer of - * up to five digits, 3 anything else, such as a coordinate, a long number or a - * token, which a later save may change, as it may view state such as which - * layer is `selected`. + * up to five digits, 3 anything else, such as a coordinate or a length with + * its unit, a long number, a token or an automatic style name (`pr1`), which + * a later save may change, as it may view state such as which layer is + * `selected`. */ export const attributeTier = (key: string, value: string): 0 | 1 | 2 | 3 => { - if (isViewStateName(key)) return 3; + if ( + isViewStateName(key) || + isAutomaticStyle(key, value) || + MEASURE.test(value) + ) + return 3; if (isNameLikeKey(key)) return 0; - if (/\p{L}/u.test(value) && !HEX_TOKEN.test(value) && !LONG_TOKEN.test(value)) - return 1; + if (/\p{L}/u.test(value) && !isGeneratedToken(value)) return 1; return /^-?\d{1,5}$/u.test(value) ? 2 : 3; }; diff --git a/src/domain/desktop/compactValue.ts b/src/domain/desktop/compactValue.ts new file mode 100644 index 0000000000..1e4aabb28d --- /dev/null +++ b/src/domain/desktop/compactValue.ts @@ -0,0 +1,155 @@ +import type { JsonValue } from "../jsonValue.js"; +import { isXmlElement } from "./viewPointer.js"; + +const isRecord = ( + value: JsonValue | undefined, +): value is Record<string, JsonValue> => + typeof value === "object" && value !== null && !Array.isArray(value); + +export const cut = (text: string, max: number) => + text.length > max ? `${text.slice(0, max)}…` : text; + +/** An element's attributes; none for a value that holds none. */ +export const attributesOf = (element: Record<string, JsonValue>) => { + const { attributes } = element; + return isRecord(attributes) ? attributes : {}; +}; + +/** Keep one character past the limit to distinguish an exact fit from a cut. */ +export const previewWriter = (max: number) => { + const parts: string[] = []; + let length = 0; + const push = (text: string): boolean => { + const kept = text.slice(0, Math.max(0, max + 1 - length)); + parts.push(kept); + length += kept.length; + return length <= max; + }; + const escaped = (text: string, attribute = false): boolean => { + for (let index = 0; index < text.length; index += 1) { + const char = text[index] ?? ""; + if ( + !push( + char === "&" + ? "&" + : char === "<" + ? "<" + : attribute && char === '"' + ? """ + : char, + ) + ) + return false; + } + return true; + }; + const jsonString = (text: string): boolean => { + if (!push('"')) return false; + for (let index = 0; index < text.length; index += 1) { + const code = text.codePointAt(index); + const width = code !== undefined && code > 0xffff ? 2 : 1; + const encoded = JSON.stringify(text.slice(index, index + width)); + if (!push(encoded.slice(1, -1))) return false; + index += width - 1; + } + return push('"'); + }; + const startTag = ( + element: Record<string, JsonValue> & { name: string }, + ): boolean => { + if (!push("<") || !push(element.name)) return false; + const attributes = attributesOf(element); + for (const key in attributes) { + if (!Object.hasOwn(attributes, key)) continue; + if ( + !push(" ") || + !push(key) || + !push('="') || + !escaped(String(attributes[key]), true) || + !push('"') + ) + return false; + } + return true; + }; + return { + push, + escaped, + jsonString, + startTag, + result: () => cut(parts.join(""), max), + }; +}; + +/** An archive entry's own fields, without its view. */ +const isArchiveEntry = (value: JsonValue): value is Record<string, JsonValue> => + isRecord(value) && "sha256" in value && "view" in value && "name" in value; + +/** + * A value as an author reads it: a scalar as it is, cut past `max` + * characters; an element as compact XML, with every text node the XML reader + * kept, which already leaves out the whitespace an app lays out between + * elements; an archive entry as its name, format and size; anything else as + * JSON. Writing stops once past `max`, so a large subtree costs no more than + * the cut. + */ +export const compactValue = ( + value: JsonValue | undefined, + max: number, + format: "json" | "xml" | "archive", +): JsonValue => { + if (value === undefined) return null; + if (typeof value === "string") return cut(value, max); + if (value === null || typeof value !== "object") return value; + const { push, escaped, jsonString, startTag, result } = previewWriter(max); + const write = (node: JsonValue): boolean => { + if (format === "xml" && isXmlElement(node)) { + if (!startTag(node)) return false; + if (node.children.length === 0) return push("/>"); + return ( + push(">") && + node.children.every((child) => + typeof child === "string" ? escaped(child) : write(child), + ) && + push("</") && + push(node.name) && + push(">") + ); + } + if (format === "archive" && isArchiveEntry(node)) + return write({ + ...(node.name === undefined ? {} : { name: node.name }), + format: node.format ?? null, + size: node.size ?? null, + }); + if (Array.isArray(node)) + return ( + push("[") && + node.every( + (item, index) => (index === 0 || push(",")) && write(item), + ) && + push("]") + ); + if (isRecord(node)) { + if (!push("{")) return false; + let first = true; + for (const key in node) { + if (!Object.hasOwn(node, key)) continue; + if ( + (!first && !push(",")) || + !jsonString(key) || + !push(":") || + !write(node[key] ?? null) + ) + return false; + first = false; + } + return push("}"); + } + return typeof node === "string" + ? jsonString(node) + : push(JSON.stringify(node)); + }; + write(value); + return result(); +}; diff --git a/src/domain/desktop/jsonRecordPairing.ts b/src/domain/desktop/jsonRecordPairing.ts new file mode 100644 index 0000000000..940fff7bd7 --- /dev/null +++ b/src/domain/desktop/jsonRecordPairing.ts @@ -0,0 +1,86 @@ +import type { JsonValue } from "../jsonValue.js"; +import { isGeneratedId } from "./attributeRank.js"; +import { isXmlElement } from "./viewPointer.js"; + +/** A field naming a JSON record, as TreeLine's `uid`: `id`, `uid`, `uuid`, `guid`, `key`, or a name ending in `_id`, `-id` or `Id`. */ +const ID_FIELD = /^(?:_?id|uid|uuid|guid|key)$|[_-]id$|[a-z]Id$/u; +/** The record's own id, ahead of a relationship such as `parent_id`, which also matches `ID_FIELD`. */ +const OWN_ID = ["uid", "uuid", "guid", "id", "key"]; + +const isJsonRecord = (value: JsonValue): value is Record<string, JsonValue> => + typeof value === "object" && + value !== null && + !Array.isArray(value) && + !isXmlElement(value); + +/** Whether every member holds a string at a key a selector can carry, each its own. */ +const namesEach = (members: readonly JsonValue[], key: string) => { + const values = members.map((member) => + isJsonRecord(member) ? member[key] : undefined, + ); + return ( + values.every( + (value) => + typeof value === "string" && value !== "" && !value.includes("]["), + ) && new Set(values).size === values.length + ); +}; + +/** One version's side of a pairing: each member's key, and the other version's member it pairs with. */ +interface Side { + readonly keys: readonly string[]; + readonly partners: readonly (number | undefined)[]; +} + +/** + * Pair the JSON records of two versions of an array by an id field each + * holds alone at both ends (`ID_FIELD`), as TreeLine's nodes by + * `[uid=7584d494d9f511f1a873edd4f1920549]`, whatever order the app saves + * them in; a member only one version holds is named by its id too, unless + * the app generated it (`isGeneratedId`), when another save would not + * repeat it and only position names the member. Undefined when the members + * are not all JSON records or no such field names each. + */ +export const pairRecordsById = ( + newer: readonly JsonValue[], + older: readonly JsonValue[], +): { readonly newer: Side; readonly older: Side } | undefined => { + const [first] = [...newer, ...older]; + if (!isJsonRecord(first ?? null) || ![...newer, ...older].every(isJsonRecord)) + return undefined; + const key = Object.keys(first ?? {}) + .filter( + (field) => + ID_FIELD.test(field) && + // `[@attr]` and `[>child]` are selector marks, not a JSON field's name. + !/^[@>]|[=[\]]/u.test(field) && + namesEach(newer, field) && + namesEach(older, field), + ) + .toSorted((left, right) => { + const rank = (field: string) => { + const own = OWN_ID.indexOf(field.toLowerCase()); + return own < 0 ? OWN_ID.length : own; + }; + return rank(left) - rank(right); + })[0]; + if (key === undefined) return undefined; + const idOf = (member: JsonValue) => + isJsonRecord(member) ? String(member[key]) : ""; + const side = ( + members: readonly JsonValue[], + others: readonly JsonValue[], + ): Side => { + const where = new Map(others.map((other, index) => [idOf(other), index])); + const partners = members.map((member) => where.get(idOf(member))); + return { + keys: members.map((member, index) => + partners[index] === undefined && isGeneratedId(idOf(member)) + ? String(index) + : `[${key}=${idOf(member)}]`, + ), + partners, + }; + }; + return { newer: side(newer, older), older: side(older, newer) }; +}; diff --git a/src/domain/desktop/memberPairing.ts b/src/domain/desktop/memberPairing.ts new file mode 100644 index 0000000000..8e46021a18 --- /dev/null +++ b/src/domain/desktop/memberPairing.ts @@ -0,0 +1,565 @@ +import type { JsonValue } from "../jsonValue.js"; +import { isTimestamp } from "./appTask.js"; +import { + isGeneratedId, + isGuid, + isIdentifyingAttribute, + isPlaceholder, + isSaveNumberedId, + rankAttributes, +} from "./attributeRank.js"; +import { + isPositionalKey, + numberedInOrder, + positionalKeys, + savePositionKeys, +} from "./positionalKeys.js"; +import { + childConditions, + condition, + countUnspelledPairs, + holdingCondition, + MAX_PAIRED, + pairSuffix, + type Attribute, + type Candidates, +} from "./selectorCandidates.js"; +import { pairRecordsById } from "./jsonRecordPairing.js"; +import { resolvePointer } from "./viewPointer.js"; +import { selectorOf } from "./viewSelector.js"; + +const isRecord = ( + value: JsonValue | null | undefined, +): value is Record<string, JsonValue> => + typeof value === "object" && value !== null && !Array.isArray(value); + +const own = (record: Record<string, JsonValue>, key: string) => + Object.hasOwn(record, key) ? record[key] : undefined; + +/** A value a selector can carry and that stays the same between saves. */ +const selectable = (value: string) => + !value.includes("][") && !isTimestamp(value) && !isPlaceholder(value); + +/** An attribute name a selector can carry. */ +const attributeName = (key: string) => key !== "" && !/[=[\]]/u.test(key); + +/** Two-attribute selectors from the attributes `paired`, the best ranked pairs first. */ +const pairsOf = (paired: readonly (Attribute & { tier: number })[]) => + paired + .flatMap((first, index) => + paired.slice(index + 1).map((second) => ({ + tier: first.tier + second.tier, + suffix: pairSuffix(first, second), + })), + ) + .toSorted((left, right) => left.tier - right.tier) + .map(({ suffix }) => suffix); + +/** + * The selectors that could name a member, most general first: `[name=N]`; + * then its identifiers, a `name` attribute first; then each other attribute + * by itself, a name-like key, a worded value or a small integer first; then + * two of the best `MAX_PAIRED` of those together; then a coordinate, a long + * number or a token by itself, and an id an ODF app numbers anew on each save + * (`isSaveNumberedId`); then a child element's text, as Osmo's record + * by `[>first_name=Delphine]` (`childConditions`), for an element no + * identifier names; last, an id the app generates (`isGeneratedId`), as + * Freeplane's `ID_329023692`, or a GUID in any attribute, as VYM's `uuid`, + * after an identifier numbering the members by save position + * (`savePositionKeys`), as Parley's `<entry id="6">`, where a child's text + * names the member. + * All are counted over every sibling, so a selector several siblings match + * is never taken for one naming a member alone; but an element with + * identifying attributes is keyed by those alone, + * so one whose identifiers are new is another element, whatever else it + * shares. An identifier the app numbers by position (`positional`) names no + * member. The ids number the name among those seen, so a name is spelled + * into a selector only once one is chosen, not per attribute. + */ +const candidatesOf = ( + member: JsonValue, + names: Map<string, number>, + positional: ReadonlySet<string>, + saved: ReadonlySet<string>, +): Candidates | undefined => { + if (!isRecord(member)) return undefined; + const name = own(member, "name"); + if (typeof name !== "string" || !selectable(name)) return undefined; + const attributes = own(member, "attributes"); + const usable = ( + isRecord(attributes) ? Object.entries(attributes) : [] + ).flatMap(([key, value]) => + typeof value === "string" && + attributeName(key) && + selectable(value) && + !isPositionalKey(positional, name, key) + ? [{ key, value }] + : [], + ); + // An id the app generates names a kept member, but no new one another save would key alike, so it ranks last. + const children = childConditions(member); + // An id numbering the members by save position names one after a child's text does. + const placed = + children.length === 0 + ? [] + : usable.filter( + ({ key }) => + isIdentifyingAttribute(key) && isPositionalKey(saved, name, key), + ); + // An id the app numbers anew on each save names no member past it: it ranks with the tokens. + const numbered = usable.filter(({ key, value }) => + isSaveNumberedId(key, value), + ); + const generated = usable.filter( + (field) => + !placed.includes(field) && + !numbered.includes(field) && + ((isIdentifyingAttribute(field.key) && isGeneratedId(field.value)) || + isGuid(field.value)), + ); + const identities = usable.filter( + (field) => + isIdentifyingAttribute(field.key) && + !generated.includes(field) && + !numbered.includes(field) && + !placed.includes(field), + ); + const named = (key: string) => key.toLowerCase() === "name"; + const ranked = [ + ...rankAttributes( + usable.filter( + (field) => + !isIdentifyingAttribute(field.key) && !generated.includes(field), + ), + ), + ...numbered.map((field) => ({ ...field, tier: 3 as const })), + ]; + const pairable = ranked.filter(({ tier }) => tier < 3); + const suffixes = [ + "", + ...[ + ...identities.filter(({ key }) => named(key)), + ...identities.filter(({ key }) => !named(key)), + ...pairable, + ].map(condition), + ...pairsOf(pairable.slice(0, MAX_PAIRED)), + ...ranked.filter(({ tier }) => tier === 3).map(condition), + ]; + // Counted on every member, one its identifiers key included, since the evaluator matches any member holding the child. + suffixes.push( + ...children, + ...placed.map(condition), + ...generated.map(condition), + ); + const number = names.get(name) ?? names.size; + names.set(name, number); + // A suffix is empty or opens with `[`, so the number before it cannot run into it. + const ids = suffixes.map((suffix) => `${String(number)}${suffix}`); + const choosable = + identities.length > 0 ? identities.length + 1 : suffixes.length; + return { + name, + number, + ids, + choosable, + byChild: + suffixes.length - children.length - placed.length - generated.length, + identified: identities.length + 1, + generated: generated.length, + pairable: pairable.map(({ key }) => key), + }; +}; + +/** + * The candidates a member present in both versions pairs by, in order: its + * tag and stable identifiers, then an id the app generated, which a kept + * member keeps, as a record's UUID however its title changes, then the rest + * it may be keyed by. A new member is keyed with that id last (`unmatched`). + */ +const pairingOrder = ({ + ids, + choosable, + identified, + generated, +}: Candidates): number[] => { + const last = ids.length - generated; + const ranks = Array.from({ length: ids.length }, (_, at) => at); + return [ + ...ranks.slice(0, Math.min(identified, choosable)), + ...ranks.slice(last), + ...ranks.slice(Math.min(identified, choosable), Math.min(choosable, last)), + ]; +}; + +/** The selector a member's candidate at `index` spells: its name, then its id past the name's number. */ +const selectorAt = ({ name, number, ids }: Candidates, index: number) => + `[name=${name}]${(ids[index] ?? "").slice(String(number).length)}`; + +const countIds = (members: readonly (Candidates | undefined)[]) => { + const counts = new Map<string, number>(); + for (const candidates of members) + for (const id of candidates?.ids ?? []) + counts.set(id, (counts.get(id) ?? 0) + 1); + return counts; +}; + +/** Each member's place among the members of its tag, as the tag and how many come before it; none for a member with no tag. */ +const placesOf = (members: readonly JsonValue[]) => { + const seen = new Map<string, number>(); + return members.map((member) => { + const name = isRecord(member) ? own(member, "name") : undefined; + if (typeof name !== "string") return undefined; + const before = seen.get(name) ?? 0; + seen.set(name, before + 1); + return `${String(before)}\u0000${name}`; + }); +}; + +/** + * Whether two members are one element renamed: the same tag and attribute + * keys, one identifier's value changed and every other attribute and the + * children alike, as MyPaint renames a layer. + */ +const renamed = (newer: JsonValue, older: JsonValue) => { + if (!isRecord(newer) || !isRecord(older)) return false; + const [after, before] = [own(newer, "attributes"), own(older, "attributes")]; + if ( + own(newer, "name") !== own(older, "name") || + !isRecord(after) || + !isRecord(before) + ) + return false; + const keys = Object.keys(after); + const changed = keys.filter((key) => own(before, key) !== after[key]); + const [key] = changed; + return ( + keys.length === Object.keys(before).length && + changed.length === 1 && + key !== undefined && + Object.hasOwn(before, key) && + isIdentifyingAttribute(key) && + JSON.stringify(own(newer, "children")) === + JSON.stringify(own(older, "children")) + ); +}; + +/** An element's attributes; none for any other value. */ +const attributesOf = (member: JsonValue) => { + const attributes = isRecord(member) ? own(member, "attributes") : undefined; + return isRecord(attributes) ? attributes : {}; +}; + +/** One version's members as a pairing reads and fills them. */ +interface Pairing { + readonly members: readonly JsonValue[]; + readonly candidates: readonly (Candidates | undefined)[]; + readonly counts: ReadonlyMap<string, number>; + readonly keys: (string | undefined)[]; + readonly partners: (number | undefined)[]; +} + +/** + * The keys of a member `renamed` where the attribute it pairs by numbers the + * members in order, as MyPaint's `src`, which a move renumbers: each + * version's own value of the renamed identifier, as `[@name=Crescent Moon]` + * and `[@name=Sketch]`, where each names its member alone; undefined + * otherwise, and the pairing keeps the attribute. + */ +const renameKeys = ( + sides: { readonly newer: Pairing; readonly older: Pairing }, + at: { readonly index: number; readonly match: number; readonly id: string }, +) => { + const { newer, older } = sides; + const candidates = newer.candidates[at.index]; + const [after, before] = [ + attributesOf(newer.members[at.index] ?? null), + attributesOf(older.members[at.match] ?? null), + ]; + const paired = + candidates === undefined + ? undefined + : selectorOf(at.id.slice(String(candidates.number).length))?.[0]?.key; + const key = Object.keys(after).find((name) => after[name] !== before[name]); + if ( + candidates === undefined || + paired === undefined || + key === undefined || + !numberedInOrder(newer.members, candidates.name, paired) || + !numberedInOrder(older.members, candidates.name, paired) + ) + return undefined; + const spelled = (side: Pairing, value: JsonValue | undefined) => { + const suffix = typeof value === "string" ? condition({ key, value }) : ""; + return suffix !== "" && + side.counts.get(`${String(candidates.number)}${suffix}`) === 1 + ? `[name=${candidates.name}]${suffix}` + : undefined; + }; + const [mine, theirs] = [ + spelled(newer, after[key]), + spelled(older, before[key]), + ]; + return mine === undefined || theirs === undefined + ? undefined + : { newer: mine, older: theirs }; +}; + +/** + * Pair, as one element `renamed`, a member of each version left unpaired at + * the same place among the members of its tag, by its first selector past + * its identifiers that names it alone in both versions with the same value, + * as a layer by its `src`; left unpaired when none does. Where that selector + * numbers the members in order, each version keys it by its own name + * (`renameKeys`). + */ +const pairRenamed = (newer: Pairing, older: Pairing) => { + const unpaired = new Map<string, number>(); + placesOf(older.members).forEach((place, index) => { + if (place !== undefined && older.partners[index] === undefined) + unpaired.set(place, index); + }); + placesOf(newer.members).forEach((place, index) => { + const match = place === undefined ? undefined : unpaired.get(place); + const candidates = newer.candidates[index]; + const theirs = match === undefined ? undefined : older.candidates[match]; + if ( + match === undefined || + candidates === undefined || + theirs === undefined || + newer.partners[index] !== undefined || + !renamed(newer.members[index] ?? null, older.members[match] ?? null) + ) + return; + const at = candidates.ids.findIndex( + (id, position) => + position >= candidates.choosable && + newer.counts.get(id) === 1 && + older.counts.get(id) === 1 && + theirs.ids.includes(id), + ); + if (at < 0) return; + const keys = renameKeys( + { newer, older }, + { index, match, id: candidates.ids[at] ?? "" }, + ); + newer.keys[index] = keys?.newer ?? selectorAt(candidates, at); + older.keys[match] = keys?.older ?? selectorAt(candidates, at); + newer.partners[index] = match; + older.partners[match] = index; + }); +}; + +/** Whether pairing a list holds its members' tag-only keys: the children of a document's root element, where an app may add a container of the tag, as Dia a layer. */ +/** Where a list sits, for its members' `holdingCondition`: whether it is the document root's children, and the tag the step to its element names, when one does. */ +export interface ListPlace { + readonly root: boolean; + readonly owner?: string; +} + +/** A list's place, read from its pointer, as `/root/children/[name=gnm:Sheets]/children` sits in `gnm:Sheets`. */ +export const holdsTagOnly = (list: string): ListPlace => { + const owner = selectorOf( + (list.split("/").at(-2) ?? "").replaceAll("~1", "/").replaceAll("~0", "~"), + )?.find(({ prefix, key }) => prefix === "" && key === "name")?.value; + return { + root: /(?:^|\/)root\/children$/u.test(list), + ...(owner === undefined ? {} : { owner }), + }; +}; + +/** Whether a tag is the singular of a list's, as `gnm:Sheet` of `gnm:Sheets`, so the list holds or could hold several. */ +const pluralOf = (owner: string | undefined, tag: string) => { + const [list, one] = [owner?.toLowerCase() ?? "", tag.toLowerCase()]; + return ( + list === `${one}s` || + list === `${one}es` || + (one.endsWith("y") && list === `${one.slice(0, -1)}ies`) + ); +}; + +/** + * Give each member of a pair its tag alone names a `holdingCondition`, so it + * stays named if a sibling of its tag appears: in the root's children, and + * in a list named for its tag, as `gnm:Sheets` holds a `gnm:Sheet`. + */ +const holdTagOnly = (newer: Pairing, older: Pairing, place: ListPlace) => { + newer.partners.forEach((match, index) => { + const candidates = newer.candidates[index]; + const deep = + candidates !== undefined && pluralOf(place.owner, candidates.name); + if ( + match === undefined || + candidates === undefined || + newer.keys[index] !== `[name=${candidates.name}]` || + (!place.root && !deep) + ) + return; + const held = holdingCondition( + newer.members[index] ?? null, + older.members[match] ?? null, + !place.root, + ); + if (held !== undefined) + newer.keys[index] = older.keys[match] = + `[name=${candidates.name}]${held}`; + }); +}; + +/** One version's side of a pairing: each member's key, and the other version's member it pairs with. */ +interface PairedSide { + readonly keys: readonly string[]; + readonly partners: readonly (number | undefined)[]; +} + +/** + * Pair the members of two versions of an array one to one. A member pairs by + * its first candidate selector naming it alone in its version and one unpaired + * member of the other, whichever before it name nothing there; then one + * element renamed by another selector (`pairRenamed`). One with no partner + * keeps its first selector naming it alone and nothing in the other version. + * Members left without a selector are keyed and paired by position; so is an + * identifier the app numbers by position, as `positionalKeys` finds. Given + * the list's `place`, a pair its tag alone names also gets a + * `holdingCondition` (`holdTagOnly`). + */ +const pairElements = ( + newer: readonly JsonValue[], + older: readonly JsonValue[], + place?: ListPlace, +): { readonly newer: PairedSide; readonly older: PairedSide } => { + const names = new Map<string, number>(); + const positional = positionalKeys(newer, older); + const saved = savePositionKeys(newer, older); + const newCandidates = newer.map((member) => + candidatesOf(member, names, positional, saved), + ); + const oldCandidates = older.map((member) => + candidatesOf(member, names, positional, saved), + ); + const newCounts = countIds(newCandidates); + const oldCounts = countIds(oldCandidates); + countUnspelledPairs([ + { members: newer, candidates: newCandidates, counts: newCounts }, + { members: older, candidates: oldCandidates, counts: oldCounts }, + ]); + const count = (counts: ReadonlyMap<string, number>, id: string) => + counts.get(id) ?? 0; + const oldById = new Map<string, number>(); + oldCandidates.forEach((candidates, index) => { + for (const at of candidates === undefined ? [] : pairingOrder(candidates)) { + const id = candidates?.ids[at] ?? ""; + if (count(oldCounts, id) === 1) oldById.set(id, index); + } + }); + const newKeys: (string | undefined)[] = newer.map(() => undefined); + const oldKeys: (string | undefined)[] = older.map(() => undefined); + const newPartners: (number | undefined)[] = newer.map(() => undefined); + const oldPartners: (number | undefined)[] = older.map(() => undefined); + newCandidates.forEach((candidates, index) => { + if (candidates === undefined) return; + for (const at of pairingOrder(candidates)) { + const id: string = candidates.ids[at] ?? ""; + const match = count(newCounts, id) === 1 ? oldById.get(id) : undefined; + if (match === undefined || oldPartners[match] !== undefined) continue; + newKeys[index] = oldKeys[match] = selectorAt(candidates, at); + newPartners[index] = match; + oldPartners[match] = index; + return; + } + }); + const sides = [ + { + members: newer, + candidates: newCandidates, + counts: newCounts, + keys: newKeys, + partners: newPartners, + }, + { + members: older, + candidates: oldCandidates, + counts: oldCounts, + keys: oldKeys, + partners: oldPartners, + }, + ] as const; + pairRenamed(...sides); + if (place !== undefined) holdTagOnly(...sides, place); + /** + * The first selector naming a member alone in its version and nothing in + * the other; by a child's text only where the list's length changed, since + * otherwise a member whose naming child an edit changed is that member. + */ + const unmatched = ( + candidates: Candidates | undefined, + here: ReadonlyMap<string, number>, + there: ReadonlyMap<string, number>, + ) => { + if (candidates === undefined) return undefined; + const last = + newer.length === older.length ? candidates.byChild : candidates.choosable; + const at = candidates.ids + .slice(0, Math.min(candidates.choosable, last)) + .findIndex((id) => count(here, id) === 1 && count(there, id) === 0); + return at < 0 ? undefined : selectorAt(candidates, at); + }; + newCandidates.forEach((candidates, index) => { + newKeys[index] ??= unmatched(candidates, newCounts, oldCounts); + }); + oldCandidates.forEach((candidates, index) => { + oldKeys[index] ??= unmatched(candidates, oldCounts, newCounts); + }); + for (let index = 0; index < Math.min(newer.length, older.length); index += 1) + if (newKeys[index] === undefined && oldKeys[index] === undefined) { + newPartners[index] = index; + oldPartners[index] = index; + } + const side = ( + keys: readonly (string | undefined)[], + partners: readonly (number | undefined)[], + ): PairedSide => ({ + keys: keys.map((key, index) => key ?? String(index)), + partners, + }); + return { + newer: side(newKeys, newPartners), + older: side(oldKeys, oldPartners), + }; +}; + +/** Pair the members of two versions of an array one to one: JSON records by an id field each holds (`pairRecordsById`), others by selector (`pairElements`). */ +export const pairMembers = ( + newer: readonly JsonValue[], + older: readonly JsonValue[], + place?: ListPlace, +): { readonly newer: PairedSide; readonly older: PairedSide } => + pairRecordsById(newer, older) ?? pairElements(newer, older, place); + +/** + * The member a pointer names at the end, as its seed version: the other + * version's member it pairs with in the same list, as a layer `renameKeys` + * keys by its new name pairs with the layer of its old one; undefined when + * the pointer names no member of a list. + */ +export const pairedAtSeed = ( + ends: { + readonly baseline: JsonValue | null; + readonly final: JsonValue | null; + }, + member: string, +): JsonValue | undefined => { + const list = /^(?<list>.*\/children)\/[^/]+$/u.exec(member)?.groups?.list; + const newer = + list === undefined ? undefined : resolvePointer(ends.final, list); + const older = + list === undefined ? undefined : resolvePointer(ends.baseline, list); + const element = resolvePointer(ends.final, member); + if (list === undefined || !Array.isArray(newer) || !Array.isArray(older)) + return undefined; + const index = newer.indexOf(element ?? null); + const partner = + index < 0 + ? undefined + : pairMembers(newer, older, holdsTagOnly(list)).newer.partners[index]; + return partner === undefined ? undefined : older[partner]; +}; diff --git a/src/domain/desktop/positionalKeys.ts b/src/domain/desktop/positionalKeys.ts index b5a46187b2..901cce4cfe 100644 --- a/src/domain/desktop/positionalKeys.ts +++ b/src/domain/desktop/positionalKeys.ts @@ -122,9 +122,62 @@ export const positionalKeys = ( return keys; }; +/** + * Identifiers numbering a list's members of a tag 0..N-1 in document order at + * both ends, where members came or went, as Parley numbers its entries by + * where they are saved: a place among the members as much as a name, which a + * member with a child's text to name it by ranks after that text. + */ +export const savePositionKeys = ( + newer: readonly JsonValue[], + older: readonly JsonValue[], +): ReadonlySet<string> => { + const before = attributesByTag(older); + const fromZero = (members: readonly Record<string, string>[]) => + Object.keys(members[0] ?? {}).filter( + (key) => + isIdentifyingAttribute(key) && + members.every((member, index) => member[key] === String(index)), + ); + const keys = new Set<string>(); + for (const [tag, members] of attributesByTag(newer)) { + const old = before.get(tag) ?? []; + if (old.length === members.length) continue; + const numbered = new Set(fromZero(old)); + for (const key of fromZero(members)) + if (numbered.has(key)) keys.add(tagKey(tag, key)); + } + return keys; +}; + /** Whether `positionalKeys` holds an identifier of members with a tag. */ export const isPositionalKey = ( keys: ReadonlySet<string>, tag: string, key: string, ) => keys.has(tagKey(tag, key)); + +/** + * Whether an attribute's values on the members with a tag end in numbers that + * count up or down by one in document order, as MyPaint names each layer's + * `src` `data/layer-00.png`, `data/layer-01.png` and on along the stack on + * every save: they say where a member sits, not which it is. + */ +export const numberedInOrder = ( + members: readonly JsonValue[], + tag: string, + key: string, +): boolean => { + const numbers = (attributesByTag(members).get(tag) ?? []).map((attributes) => + Number( + /(?<n>\d+)\D*$/u.exec(attributes[key] ?? "")?.groups?.n ?? Number.NaN, + ), + ); + const [first, second] = numbers; + if (first === undefined || second === undefined) return false; + const step = second - first; + return ( + Math.abs(step) === 1 && + numbers.every((number, index) => number === first + step * index) + ); +}; diff --git a/src/domain/desktop/selectorCandidates.test.ts b/src/domain/desktop/selectorCandidates.test.ts new file mode 100644 index 0000000000..ac6c1d6ec9 --- /dev/null +++ b/src/domain/desktop/selectorCandidates.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; + +import { childConditions } from "./selectorCandidates.js"; + +describe("childConditions", () => { + it("stops reading a child's text once it is past what a selector names a member by, however long the text", () => { + const element = (text: string) => ({ + name: "record", + attributes: {}, + children: [ + { name: "title", attributes: {}, children: ["Budget"] }, + { name: "body", attributes: {}, children: [text] }, + ], + }); + const huge = element(`a b ${"x".repeat(4 * 1024 * 1024)}`); + + const started = performance.now(); + const conditions = childConditions(huge); + const elapsed = performance.now() - started; + + expect(conditions).toEqual(["[>title=Budget]"]); + // Building the whole collapsed text took about half a second. + expect(elapsed).toBeLessThan(100); + }); +}); diff --git a/src/domain/desktop/selectorCandidates.ts b/src/domain/desktop/selectorCandidates.ts index a29d29123a..6c18a01eff 100644 --- a/src/domain/desktop/selectorCandidates.ts +++ b/src/domain/desktop/selectorCandidates.ts @@ -1,5 +1,13 @@ import type { JsonValue } from "../jsonValue.js"; -import { spellCondition, type Condition } from "./viewSelector.js"; +import { isTimestamp } from "./appTask.js"; +import { + isGeneratedId, + isIdentifyingAttribute, + isNameLikeKey, + isPlaceholder, + rankAttributes, +} from "./attributeRank.js"; +import { selectorOf, spellCondition, type Condition } from "./viewSelector.js"; const isRecord = ( value: JsonValue | null | undefined, @@ -12,15 +20,19 @@ const own = (record: Record<string, JsonValue>, key: string) => /** * A member's selectors as its name and the ids they are counted by, the * number its name is counted by and then the conditions after the name; how - * many of them, from the first, may key it; and the keys of its attributes - * that pair, best ranked first, its pairs spelled from the first - * `MAX_PAIRED`. + * many of them, from the first, may key it, where those by a child's text + * start, and those by its stable identifiers end; how many ids the app + * generated close the list; and the keys of its attributes that pair, best + * ranked first, its pairs spelled from the first `MAX_PAIRED`. */ export interface Candidates { readonly name: string; readonly number: number; readonly ids: readonly string[]; readonly choosable: number; + readonly byChild: number; + readonly identified: number; + readonly generated: number; readonly pairable: readonly string[]; } @@ -39,6 +51,74 @@ export const pairSuffix = (first: Attribute, second: Attribute) => ? `${condition(first)}${condition(second)}` : `${condition(second)}${condition(first)}`; +/** The longest child text a selector names a member by. */ +const MAX_CHILD_TEXT = 64; + +/** + * An element's text with its whitespace collapsed as `collapseWhitespace` + * does, as `[>CHILD=VALUE]` reads it, whatever elements hold it, so every + * member a selector matches counts; collapsed as it is read, and undefined + * once longer than a selector names a member by. + */ +const shortText = (element: JsonValue): string | undefined => { + let text = ""; + let space = false; + /** Add a text node's characters, collapsed; false once the text is past the limit, read no further. */ + const append = (node: string) => { + for (const character of node) + if (" \t\r\n".includes(character)) space = text !== ""; + else { + text += `${space ? " " : ""}${character}`; + space = false; + if (text.length > MAX_CHILD_TEXT) return false; + } + return true; + }; + const pending: JsonValue[] = [element]; + for (let node = pending.pop(); node !== undefined; node = pending.pop()) + if (typeof node === "string") { + if (isTimestamp(node)) continue; + if (!append(node)) return undefined; + } else if (isRecord(node) && Array.isArray(node.children)) + for (let index = node.children.length - 1; index >= 0; index -= 1) + pending.push(node.children[index] ?? null); + return text; +}; + +/** + * The `[>CHILD=VALUE]` conditions that could name an element by a child's + * text, best first by `attributeTier`: a name-like tag, as Osmo's + * `first_name`, then worded text, then a small integer, then other numbers. + * A child whose text is long, blank, a time, or a token or id a program + * generates names nothing a later save keeps. + */ +export const childConditions = (member: JsonValue): string[] => { + const children = isRecord(member) ? own(member, "children") : undefined; + const found = (Array.isArray(children) ? children : []).flatMap((child) => { + const key = isRecord(child) ? own(child, "name") : undefined; + const value = typeof key === "string" ? shortText(child) : undefined; + return typeof key === "string" && + value !== undefined && + key !== "" && + !/[=[\]]/u.test(key) && + /[\p{L}\d]/u.test(value) && + !value.includes("][") && + !isTimestamp(value) && + !isGeneratedId(value) && + !isPlaceholder(value) && + !/\d{10}/u.test(value) + ? [{ key, value }] + : []; + }); + return [ + ...new Set( + rankAttributes(found).map(({ key, value }) => + spellCondition({ prefix: ">", key, value }), + ), + ), + ]; +}; + /** One version's members, their candidates, and how many members each id names. */ interface CountedVersion { readonly members: readonly JsonValue[]; @@ -128,3 +208,83 @@ export const countUnspelledPairs = (versions: readonly CountedVersion[]) => { if (spelled(id)) counts.set(id, (counts.get(id) ?? 0) + 1); }); }; + +/** An identifier that holds its value at both versions and a selector can carry. */ +const keptIdentifier = ( + after: Record<string, JsonValue>, + before: Record<string, JsonValue>, + name: string, +) => { + const value = after[name]; + return typeof value === "string" && + value === before[name] && + !/[=[\]]/u.test(name) && + !value.includes("][") && + !isTimestamp(value) + ? condition({ key: name, value }) + : undefined; +}; + +/** Whether an element holds elements and no text of its own, as Gnumeric's `gnm:Sheet` its name and cells. */ +const holdsElements = (member: JsonValue) => { + const children = isRecord(member) ? own(member, "children") : undefined; + const all = Array.isArray(children) ? children : []; + return ( + all.some((child) => isRecord(child)) && + all.every((child) => typeof child !== "string" || child.trim() === "") + ); +}; + +/** Whether an element holds a list, two or more elements of one tag and no text of its own, as Dia's layer its boxes. */ +const isContainer = (member: JsonValue) => { + const children = isRecord(member) ? own(member, "children") : undefined; + const all = Array.isArray(children) ? children : []; + const tags = all.flatMap((child) => { + const tag = isRecord(child) ? own(child, "name") : undefined; + return typeof tag === "string" ? [tag] : []; + }); + return ( + new Set(tags).size < tags.length && + all.every((child) => typeof child !== "string" || child.trim() === "") + ); +}; + +/** + * A condition that keeps a member its tag alone names its own once the app + * adds a sibling of that tag, as Dia adds an empty `Background` layer each + * time it reopens a diagram, for a list's container with an identifier, a + * kind a document can hold several of: an id both versions hold, which a new + * sibling would not share; else a child's text both hold, which an empty one + * lacks; else its `name`, which a new one may repeat, as Dia's do. `nested`, + * for a member of a list named for its tag, as a `gnm:Sheet` in + * `gnm:Sheets`, takes any element holding elements, and only a name-like + * child, as `[>gnm:Name=Budget]`. Undefined when none is there. + */ +export const holdingCondition = ( + newer: JsonValue, + older: JsonValue, + nested = false, +): string | undefined => { + const attributesOf = (member: JsonValue) => { + const attributes = isRecord(member) ? own(member, "attributes") : undefined; + return isRecord(attributes) ? attributes : {}; + }; + const [after, before] = [attributesOf(newer), attributesOf(older)]; + const identifiers = (nested ? holdsElements : isContainer)(newer) + ? Object.keys(after).filter(isIdentifyingAttribute) + : []; + const named = (key: string) => key.toLowerCase() === "name"; + const theirs = new Set(childConditions(older)); + const children = childConditions(newer).filter( + (spelled) => + theirs.has(spelled) && + (!nested || isNameLikeKey(selectorOf(spelled)?.[0]?.key ?? "")), + ); + const kept = (keys: readonly string[]) => + keys.flatMap((key) => keptIdentifier(after, before, key) ?? [])[0]; + return identifiers.length === 0 && !(nested && holdsElements(newer)) + ? undefined + : (kept(identifiers.filter((key) => !named(key))) ?? + children[0] ?? + kept(identifiers.filter(named))); +}; diff --git a/src/domain/desktop/stateChanges.ts b/src/domain/desktop/stateChanges.ts index f56ebaf683..3783ffdc3e 100644 --- a/src/domain/desktop/stateChanges.ts +++ b/src/domain/desktop/stateChanges.ts @@ -1,6 +1,7 @@ import type { JsonValue } from "../jsonValue.js"; import type { DesktopStateFormat } from "./stateFormats.js"; -import { memberLookup, pairMembers } from "./viewPointer.js"; +import { holdsTagOnly, pairMembers } from "./memberPairing.js"; +import { memberLookup } from "./viewPointer.js"; import { zipLeavesOut } from "./zipView.js"; const MAX_LISTED = 100; @@ -76,10 +77,14 @@ interface Noting extends WalkOptions { const keyedMembers = ( from: readonly JsonValue[], other: JsonValue | undefined, - older: boolean, + at: { readonly older: boolean; readonly list: string }, ) => { const others = Array.isArray(other) ? other : []; - const pairing = older ? pairMembers(others, from) : pairMembers(from, others); + const hold = holdsTagOnly(at.list); + const pairing = at.older + ? pairMembers(others, from, hold) + : pairMembers(from, others, hold); + const older = at.older; const side = older ? pairing.older : pairing.newer; const lookup = memberLookup(other); return side.keys.map((key, index) => { @@ -119,7 +124,10 @@ const walk = ( path.length -= segment.length + 1; }; if (Array.isArray(from) && into.keyed === true) { - const members = keyedMembers(from, other, into.older === true); + const members = keyedMembers(from, other, { + older: into.older === true, + list: path.segments.map((segment) => `/${segment}`).join(""), + }); from.forEach((item, index) => { const member = members[index]; if (member !== undefined) visit(member.key, item, member.counterpart); @@ -320,7 +328,7 @@ export const diffStateViews = ( options: { readonly named?: boolean } = {}, ): DesktopStateDifference => { const difference = - format === "xml" && options.named === true + (format === "xml" || format === "json") && options.named === true ? namedOrPositional(before, after) : compareViews(format, before, after); // Only REA's own readers write coverage flags; JSON and key files are data. diff --git a/src/domain/desktop/stateFormats.test.ts b/src/domain/desktop/stateFormats.test.ts index 7c270a2fb1..5300c93860 100644 --- a/src/domain/desktop/stateFormats.test.ts +++ b/src/domain/desktop/stateFormats.test.ts @@ -41,6 +41,18 @@ describe("detectStateFormat", () => { ).toBe("binary"); }); + it("reads Unicode XML names in unnamed formats and rejects malformed XML", () => { + for (const [text, format] of [ + ['<地图 名称="甲"><节点/></地图>', "xml"], + ['<!DOCTYPE 地图 SYSTEM "map.dtd"><地图/>', "xml"], + ["<地图><节点></地图>", "text"], + ["<ada@example.test>\nBuy milk", "text"], + ] as const) { + expect(detectStateFormat("map.dat", bytes(text))).toBe(format); + expect(textSyntaxFormat("map.dat", text)).toBe(format); + } + }); + it("tells JSON, XML, key files and plain text apart", () => { expect(detectStateFormat("/h/a.json", bytes('{"a":1}'))).toBe("json"); expect(detectStateFormat("/h/notes", bytes("{not json"))).toBe("text"); @@ -164,6 +176,43 @@ describe("detectStateFormat with a DOCTYPE", () => { ).toBe("text"); }); + it("reads a file that opens with a DOCTYPE naming no internal subset and then an element as XML, as KmPlot's plot, whatever it is called", () => { + const plot = (doctype: string, body = '<function type="cartesian"/>') => + bytes(`${doctype}\n<kmpdoc version="4">\n ${body}\n</kmpdoc>\n`); + + expect( + detectStateFormat( + "/h/Documents/lesson_plots.fkt", + plot("<!DOCTYPE kmpdoc>"), + ), + ).toBe("xml"); + expect( + textSyntaxFormat( + "/h/Documents/lesson_plots.fkt", + '<!DOCTYPE kmpdoc>\n<kmpdoc version="4"/>\n', + ), + ).toBe("xml"); + expect( + textSyntaxFormat( + "/h/Documents/state.txt", + '<!DOCTYPE state SYSTEM "state[1].dtd"><state/>', + ), + ).toBe("xml"); + // A DOCTYPE that can declare entities, or text that is not well formed, stays text. + expect( + detectStateFormat( + "/h/Documents/lesson_plots.fkt", + plot('<!DOCTYPE kmpdoc [<!ENTITY e "x">]>', "<f>&e;</f>"), + ), + ).toBe("text"); + expect( + detectStateFormat( + "/h/Documents/lesson_plots.fkt", + plot("<!DOCTYPE kmpdoc>", "<function>"), + ), + ).toBe("text"); + }); + it.each([ '<!DOCTYPE abiword PUBLIC "-//ABISOURCE//DTD AWML 1.0 Strict//EN" "http://www.abisource.com/awml.dtd"><abiword/>', '<!DOCTYPE abiword [<!ENTITY lol "lol">]><abiword>&lol;</abiword>', diff --git a/src/domain/desktop/stateFormats.ts b/src/domain/desktop/stateFormats.ts index 6ded600d9f..8c0adb4b0f 100644 --- a/src/domain/desktop/stateFormats.ts +++ b/src/domain/desktop/stateFormats.ts @@ -7,6 +7,7 @@ import { safeParseJson } from "../safeJson.js"; import { declaredDocumentType, isWellFormedXml, + opensLikeXml, parseXmlDocument, } from "../xmlDocument.js"; @@ -222,6 +223,59 @@ const namedJson = (path: string, trimmed: string): boolean => const namedXml = (path: string, trimmed: string): boolean => XML_EXTENSIONS.test(path) || trimmed.startsWith("<?xml"); +/** + * Whether text opens with an element, as a Freeplane map's `<map version=…>` + * without a declaration, or with a DOCTYPE that has no internal subset and + * then an element, as KmPlot's `<!DOCTYPE kmpdoc><kmpdoc …>`, so it is XML + * if it is well formed, whatever the file is called. A DOCTYPE with an + * internal subset, which may declare entities, never opens one. + */ +/** + * Whether a DOCTYPE before `>` opens an internal subset. A `[` inside a + * quoted system or public id, as `state[1].dtd`, does not. + */ +const doctypeOpensSubset = (text: string, from: number): boolean => { + let quote = ""; + for (let index = from; index < text.length; index += 1) { + const character = text[index] ?? ""; + if (quote !== "") { + if (character === quote) quote = ""; + continue; + } + if (character === '"' || character === "'") quote = character; + else if (character === "[") return true; + else if (character === ">") return false; + } + return false; +}; + +const opensElement = (trimmed: string): boolean => { + if (!trimmed.startsWith("<!DOCTYPE") && opensLikeXml(trimmed)) return true; + const declared = /^(?:<\?xml[^>]*>\s*)?<!DOCTYPE\b/u.exec(trimmed); + if (declared === null) return false; + let quote = ""; + for (let index = declared[0].length; index < trimmed.length; index += 1) { + const character = trimmed[index] ?? ""; + if (quote !== "") { + if (character === quote) quote = ""; + continue; + } + if (character === '"' || character === "'") quote = character; + else if (character === "[") return false; + else if (character === ">") { + const rest = trimmed.slice(index + 1).trimStart(); + return opensLikeXml(rest); + } + } + return false; +}; + +/** Whether `opening` has a DOCTYPE whose internal subset starts outside quotes. */ +export const doctypeHasInternalSubset = (opening: string): boolean => { + const declared = /^(?:<\?xml[^>]*>\s*)?<!DOCTYPE\b/u.exec(opening); + return declared !== null && doctypeOpensSubset(opening, declared[0].length); +}; + /** * Whether decoded text is JSON or XML as `detectStateFormat` reads it, told * by syntax alone so no value or document is built: JSON a token at a time @@ -233,7 +287,10 @@ export const textSyntaxFormat = ( ): "json" | "xml" | "text" => { const trimmed = text.trimStart(); if (namedJson(path, trimmed) && isJsonText(text)) return "json"; - if (namedXml(path, trimmed) && isWellFormedXml(text, XML_POLICY)) + if ( + (namedXml(path, trimmed) || opensElement(trimmed)) && + isWellFormedXml(text, XML_POLICY) + ) return "xml"; return "text"; }; @@ -258,7 +315,8 @@ export const detectStateFormat = ( : safeParseJson(text).ok) ) return "json"; - if (namedXml(path, trimmed)) + // Check syntax without constructing a document for format detection. + if (namedXml(path, trimmed) || opensElement(trimmed)) return isWellFormedXml(text, XML_POLICY) ? "xml" : "text"; return isKeyfile(text) ? "keyfile" : "text"; }; diff --git a/src/domain/desktop/stateRoles.test.ts b/src/domain/desktop/stateRoles.test.ts index b0e1104069..bfe4c55938 100644 --- a/src/domain/desktop/stateRoles.test.ts +++ b/src/domain/desktop/stateRoles.test.ts @@ -317,3 +317,27 @@ describe("isApplicationDconfGroup", () => { expect(isApplicationDconfGroup("org/mousepadx", names)).toBe(false); }); }); + +describe("stateReasons for a state directory the app names with its version", () => { + it("counts TreeLine's ~/.treeline-3.1 beside ~/.treeline, and a numbered folder alike, never another app's longer name", () => { + const facts: DesktopRoleFacts = { + home: "/home/ada", + stateDirectories: ["/home/ada/.treeline"], + stateTails: [".treeline"], + selectedRoots: [], + selections: [], + titles: [], + openedFiles: new Set(), + }; + const reasons = (path: string) => stateReasons(facts, path, [], false); + + expect(reasons("/home/ada/.treeline-3.1/settings.ini")).toEqual([ + "state_directory", + ]); + expect(reasons("/home/ada/.treeline3/settings.ini")).toEqual([ + "state_directory", + ]); + expect(reasons("/home/ada/.treelines/settings.ini")).toEqual([]); + expect(reasons("/home/ada/.treeline-beta/settings.ini")).toEqual([]); + }); +}); diff --git a/src/domain/desktop/stateRoles.ts b/src/domain/desktop/stateRoles.ts index 26bde09bf1..7c0f370b38 100644 --- a/src/domain/desktop/stateRoles.ts +++ b/src/domain/desktop/stateRoles.ts @@ -90,7 +90,12 @@ const hiddenTail = (path: string): string | null => { return index < 0 ? null : path.slice(index + 1); }; -/** The state directory a path lies in, or the folder beside one whose name differs only in letter case that the application may create during the capture, as the path spells it. */ +/** + * The state directory a path lies in, or the folder beside one that the + * application may create during the capture, as the path spells it: one + * whose name differs only in letter case, or that adds a version, as + * TreeLine's `~/.treeline-3.1` beside `~/.treeline`. + */ export const stateDirectoryOf = ( path: string, directories: readonly string[], @@ -99,9 +104,13 @@ export const stateDirectoryOf = ( const parent = directory.slice(0, directory.lastIndexOf("/") + 1); if (!path.startsWith(parent)) continue; const folder = path.slice(parent.length).split("/")[0] ?? ""; + const name = directory.slice(parent.length).toLowerCase(); + const own = folder.toLowerCase(); if ( folder.length > 0 && - folder.toLowerCase() === directory.slice(parent.length).toLowerCase() + (own === name || + (own.startsWith(name) && + /^-?\d+(?:\.\d+)*$/u.test(own.slice(name.length)))) ) return `${parent}${folder}`; } diff --git a/src/domain/desktop/viewPointer.test.ts b/src/domain/desktop/viewPointer.test.ts index df54089dda..5e9bcb46fd 100644 --- a/src/domain/desktop/viewPointer.test.ts +++ b/src/domain/desktop/viewPointer.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from "vitest"; import type { JsonValue } from "../jsonValue.js"; -import { locatePointer, memberLookup, pairMembers } from "./viewPointer.js"; +import { pairMembers } from "./memberPairing.js"; +import { locatePointer, memberLookup } from "./viewPointer.js"; const element = ( name: string, diff --git a/src/domain/desktop/viewPointer.ts b/src/domain/desktop/viewPointer.ts index d6330df5cf..a42ad5b3a2 100644 --- a/src/domain/desktop/viewPointer.ts +++ b/src/domain/desktop/viewPointer.ts @@ -1,15 +1,5 @@ import type { JsonValue } from "../jsonValue.js"; import { collapseWhitespace, isTimestamp } from "./appTask.js"; -import { isIdentifyingAttribute, rankAttributes } from "./attributeRank.js"; -import { isPositionalKey, positionalKeys } from "./positionalKeys.js"; -import { - condition, - countUnspelledPairs, - MAX_PAIRED, - pairSuffix, - type Attribute, - type Candidates, -} from "./selectorCandidates.js"; import { bracketed, selectorOf, type Condition } from "./viewSelector.js"; /** The arrays a pointer stepped into, each as the pointer to it. */ @@ -92,15 +82,23 @@ const fieldOf = (member: JsonValue, { prefix, key }: Condition) => { return typeof value === "string" ? value : undefined; }; +/** An element's text with save-time leaves removed, so a name around a timestamp still matches after the time changes. */ +const textWithoutStamps = (node: JsonValue): string => { + if (typeof node === "string") return isTimestamp(node) ? "" : node; + if (!isRecord(node) || !Array.isArray(node.children)) return ""; + return node.children.map((child) => textWithoutStamps(child)).join(""); +}; + /** Whether an element has a child element named `key` whose text, its whitespace collapsed, is `value`. */ const hasChildText = (member: JsonValue, { key, value }: Condition) => isXmlElement(member) && - member.children.some( - (child) => - isXmlElement(child) && - child.name === key && - collapseWhitespace(elementText(child)) === value, - ); + member.children.some((child) => { + if (!isXmlElement(child) || child.name !== key) return false; + // A nested save time is not part of the name. A child that is only a time still matches that time. + const plain = collapseWhitespace(textWithoutStamps(child)); + const text = plain === "" ? collapseWhitespace(elementText(child)) : plain; + return text === value; + }); /** Whether an array member meets every condition; selectors compare strings only. */ const selects = (member: JsonValue, conditions: readonly Condition[]) => @@ -166,6 +164,8 @@ export const locatePointer = ( view: JsonValue | null, pointer: string, ): Located => { + if (pointer === "") + return { kind: "value", value: view, searched: [], byPosition: [] }; let current: JsonValue = view; const segments = pointer.slice(1).split("/"); const steps = { searched: [] as string[], byPosition: [] as string[] }; @@ -298,293 +298,3 @@ export const resolvePointer = ( const located = locatePointer(view, pointer); return located.kind === "value" ? located.value : undefined; }; - -/** A value a selector can carry and that stays the same between saves. */ -const selectable = (value: string) => - !value.includes("][") && !isTimestamp(value); - -/** An attribute name a selector can carry. */ -const attributeName = (key: string) => key !== "" && !/[=[\]]/u.test(key); - -/** Two-attribute selectors from the attributes `paired`, the best ranked pairs first. */ -const pairsOf = (paired: readonly (Attribute & { tier: number })[]) => - paired - .flatMap((first, index) => - paired.slice(index + 1).map((second) => ({ - tier: first.tier + second.tier, - suffix: pairSuffix(first, second), - })), - ) - .toSorted((left, right) => left.tier - right.tier) - .map(({ suffix }) => suffix); - -/** - * The selectors that could name a member, most general first: `[name=N]`; - * then its identifiers, a `name` attribute first; then each other attribute - * by itself, a name-like key, a worded value or a small integer first; then - * two of the best `MAX_PAIRED` of those together; then a coordinate, a long - * number or a token by itself. All are counted over every sibling, so a - * selector several siblings match is never taken for one naming a member - * alone; but an element with identifying attributes is keyed by those alone, - * so one whose identifiers are new is another element, whatever else it - * shares. An identifier the app numbers by position (`positional`) names no - * member. The ids number the name among those seen, so a name is spelled into - * a selector only once one is chosen, not per attribute. - */ -const candidatesOf = ( - member: JsonValue, - names: Map<string, number>, - positional: ReadonlySet<string>, -): Candidates | undefined => { - if (!isRecord(member)) return undefined; - const name = own(member, "name"); - if (typeof name !== "string" || !selectable(name)) return undefined; - const attributes = own(member, "attributes"); - const usable = ( - isRecord(attributes) ? Object.entries(attributes) : [] - ).flatMap(([key, value]) => - typeof value === "string" && - attributeName(key) && - selectable(value) && - !isPositionalKey(positional, name, key) - ? [{ key, value }] - : [], - ); - const identities = usable.filter(({ key }) => isIdentifyingAttribute(key)); - const named = (key: string) => key.toLowerCase() === "name"; - const ranked = rankAttributes( - usable.filter(({ key }) => !isIdentifyingAttribute(key)), - ); - const pairable = ranked.filter(({ tier }) => tier < 3); - const suffixes = [ - "", - ...[ - ...identities.filter(({ key }) => named(key)), - ...identities.filter(({ key }) => !named(key)), - ...pairable, - ].map(condition), - ...pairsOf(pairable.slice(0, MAX_PAIRED)), - ...ranked.filter(({ tier }) => tier === 3).map(condition), - ]; - const number = names.get(name) ?? names.size; - names.set(name, number); - // A suffix is empty or opens with `[`, so the number before it cannot run into it. - const ids = suffixes.map((suffix) => `${String(number)}${suffix}`); - const choosable = - identities.length > 0 ? identities.length + 1 : suffixes.length; - return { - name, - number, - ids, - choosable, - pairable: pairable.map(({ key }) => key), - }; -}; - -/** The selector a member's candidate at `index` spells: its name, then its id past the name's number. */ -const selectorAt = ({ name, number, ids }: Candidates, index: number) => - `[name=${name}]${(ids[index] ?? "").slice(String(number).length)}`; - -const countIds = (members: readonly (Candidates | undefined)[]) => { - const counts = new Map<string, number>(); - for (const candidates of members) - for (const id of candidates?.ids ?? []) - counts.set(id, (counts.get(id) ?? 0) + 1); - return counts; -}; - -/** Each member's place among the members of its tag, as the tag and how many come before it; none for a member with no tag. */ -const placesOf = (members: readonly JsonValue[]) => { - const seen = new Map<string, number>(); - return members.map((member) => { - const name = isRecord(member) ? own(member, "name") : undefined; - if (typeof name !== "string") return undefined; - const before = seen.get(name) ?? 0; - seen.set(name, before + 1); - return `${String(before)}\u0000${name}`; - }); -}; - -/** - * Whether two members are one element renamed: the same tag and attribute - * keys, one identifier's value changed and every other attribute and the - * children alike, as MyPaint renames a layer. - */ -const renamed = (newer: JsonValue, older: JsonValue) => { - if (!isRecord(newer) || !isRecord(older)) return false; - const [after, before] = [own(newer, "attributes"), own(older, "attributes")]; - if ( - own(newer, "name") !== own(older, "name") || - !isRecord(after) || - !isRecord(before) - ) - return false; - const keys = Object.keys(after); - const changed = keys.filter((key) => own(before, key) !== after[key]); - const [key] = changed; - return ( - keys.length === Object.keys(before).length && - changed.length === 1 && - key !== undefined && - Object.hasOwn(before, key) && - isIdentifyingAttribute(key) && - JSON.stringify(own(newer, "children")) === - JSON.stringify(own(older, "children")) - ); -}; - -/** One version's members as a pairing reads and fills them. */ -interface Pairing { - readonly members: readonly JsonValue[]; - readonly candidates: readonly (Candidates | undefined)[]; - readonly counts: ReadonlyMap<string, number>; - readonly keys: (string | undefined)[]; - readonly partners: (number | undefined)[]; -} - -/** - * Pair, as one element `renamed`, a member of each version left unpaired at - * the same place among the members of its tag, by its first selector past - * its identifiers that names it alone in both versions with the same value, - * as a layer by its `src`; left unpaired when none does. - */ -const pairRenamed = (newer: Pairing, older: Pairing) => { - const unpaired = new Map<string, number>(); - placesOf(older.members).forEach((place, index) => { - if (place !== undefined && older.partners[index] === undefined) - unpaired.set(place, index); - }); - placesOf(newer.members).forEach((place, index) => { - const match = place === undefined ? undefined : unpaired.get(place); - const candidates = newer.candidates[index]; - const theirs = match === undefined ? undefined : older.candidates[match]; - if ( - match === undefined || - candidates === undefined || - theirs === undefined || - newer.partners[index] !== undefined || - !renamed(newer.members[index] ?? null, older.members[match] ?? null) - ) - return; - const at = candidates.ids.findIndex( - (id, position) => - position >= candidates.choosable && - newer.counts.get(id) === 1 && - older.counts.get(id) === 1 && - theirs.ids.includes(id), - ); - if (at < 0) return; - newer.keys[index] = older.keys[match] = selectorAt(candidates, at); - newer.partners[index] = match; - older.partners[match] = index; - }); -}; - -/** One version's side of a pairing: each member's key, and the other version's member it pairs with. */ -interface PairedSide { - readonly keys: readonly string[]; - readonly partners: readonly (number | undefined)[]; -} - -/** - * Pair the members of two versions of an array one to one. A member pairs by - * its first candidate selector naming it alone in its version and one unpaired - * member of the other, whichever before it name nothing there; then one - * element renamed by another selector (`pairRenamed`). One with no partner - * keeps its first selector naming it alone and nothing in the other version. - * Members left without a selector are keyed and paired by position; so is an - * identifier the app numbers by position, as `positionalKeys` finds. - */ -export const pairMembers = ( - newer: readonly JsonValue[], - older: readonly JsonValue[], -): { readonly newer: PairedSide; readonly older: PairedSide } => { - const names = new Map<string, number>(); - const positional = positionalKeys(newer, older); - const newCandidates = newer.map((member) => - candidatesOf(member, names, positional), - ); - const oldCandidates = older.map((member) => - candidatesOf(member, names, positional), - ); - const newCounts = countIds(newCandidates); - const oldCounts = countIds(oldCandidates); - countUnspelledPairs([ - { members: newer, candidates: newCandidates, counts: newCounts }, - { members: older, candidates: oldCandidates, counts: oldCounts }, - ]); - const count = (counts: ReadonlyMap<string, number>, id: string) => - counts.get(id) ?? 0; - const oldById = new Map<string, number>(); - oldCandidates.forEach((candidates, index) => { - for (const id of candidates?.ids.slice(0, candidates.choosable) ?? []) - if (count(oldCounts, id) === 1) oldById.set(id, index); - }); - const newKeys: (string | undefined)[] = newer.map(() => undefined); - const oldKeys: (string | undefined)[] = older.map(() => undefined); - const newPartners: (number | undefined)[] = newer.map(() => undefined); - const oldPartners: (number | undefined)[] = older.map(() => undefined); - newCandidates.forEach((candidates, index) => { - if (candidates === undefined) return; - for (const [at, id] of candidates.ids - .slice(0, candidates.choosable) - .entries()) { - const match = count(newCounts, id) === 1 ? oldById.get(id) : undefined; - if (match === undefined || oldPartners[match] !== undefined) continue; - newKeys[index] = oldKeys[match] = selectorAt(candidates, at); - newPartners[index] = match; - oldPartners[match] = index; - return; - } - }); - pairRenamed( - { - members: newer, - candidates: newCandidates, - counts: newCounts, - keys: newKeys, - partners: newPartners, - }, - { - members: older, - candidates: oldCandidates, - counts: oldCounts, - keys: oldKeys, - partners: oldPartners, - }, - ); - /** The first selector naming a member alone in its version and nothing in the other. */ - const unmatched = ( - candidates: Candidates | undefined, - here: ReadonlyMap<string, number>, - there: ReadonlyMap<string, number>, - ) => { - if (candidates === undefined) return undefined; - const at = candidates.ids - .slice(0, candidates.choosable) - .findIndex((id) => count(here, id) === 1 && count(there, id) === 0); - return at < 0 ? undefined : selectorAt(candidates, at); - }; - newCandidates.forEach((candidates, index) => { - newKeys[index] ??= unmatched(candidates, newCounts, oldCounts); - }); - oldCandidates.forEach((candidates, index) => { - oldKeys[index] ??= unmatched(candidates, oldCounts, newCounts); - }); - for (let index = 0; index < Math.min(newer.length, older.length); index += 1) - if (newKeys[index] === undefined && oldKeys[index] === undefined) { - newPartners[index] = index; - oldPartners[index] = index; - } - const side = ( - keys: readonly (string | undefined)[], - partners: readonly (number | undefined)[], - ): PairedSide => ({ - keys: keys.map((key, index) => key ?? String(index)), - partners, - }); - return { - newer: side(newKeys, newPartners), - older: side(oldKeys, oldPartners), - }; -}; diff --git a/src/domain/xmlDocument.ts b/src/domain/xmlDocument.ts index 75a129925c..427722f3d1 100644 --- a/src/domain/xmlDocument.ts +++ b/src/domain/xmlDocument.ts @@ -29,6 +29,17 @@ const declaresInternalSubset = (declaration: string): boolean => /** XML 1.0's `NameStartChar`, and the further characters `NameChar` allows. */ const NAME_START = String.raw`:A-Z_a-z\u{C0}-\u{D6}\u{D8}-\u{F6}\u{F8}-\u{2FF}\u{370}-\u{37D}\u{37F}-\u{1FFF}\u{200C}\u{200D}\u{2070}-\u{218F}\u{2C00}-\u{2FEF}\u{3001}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFFD}\u{10000}-\u{EFFFF}`; const NAME_MORE = String.raw`\-.0-9\u{B7}\u{300}-\u{36F}\u{203F}\u{2040}`; +/** XML element opening grammar shared by the desktop format readers. */ +export const XML_ELEMENT_OPENING = String.raw`<[${NAME_START}][${NAME_START}${NAME_MORE}]*(?=[ \t\r\n]|/?>)`; +const XML_OPENING = new RegExp( + `^(?:<\\?xml\\b|<!DOCTYPE\\b|${XML_ELEMENT_OPENING})`, + "u", +); + +/** Whether text begins with a possible XML declaration, DOCTYPE or start tag. */ +export const opensLikeXml = (source: string): boolean => + XML_OPENING.test(source); + /** XML 1.0's `PubidChar` but the apostrophe, which only a double-quoted literal may hold. */ const PUBID = String.raw`\n\r a-zA-Z0-9\-()+,./:=?;!*#@$_%`; const SPACE = String.raw`[ \t\r\n]`; @@ -161,6 +172,48 @@ export const xmlElementRanges = ( } }; +/** Inclusive source ranges of attribute quotes, parsed without reading a DTD. */ +export const xmlAttributeQuoteRanges = ( + source: string, +): Result< + readonly { readonly start: number; readonly end: number }[], + { + readonly error: XmlDocumentParseError; + readonly ranges: readonly { + readonly start: number; + readonly end: number; + }[]; + readonly examined: number; + } +> => { + const ranges: { start: number; end: number }[] = []; + let reader: SaxesParser<{ xmlns: true }> | undefined; + try { + checkXmlSyntax(source, "allow", (syntax) => { + reader = syntax; + syntax.on("attribute", () => { + const end = syntax.position - 1; + const quote = source[end]; + if (quote !== '"' && quote !== "'") + throw new XmlDocumentParseError("malformed"); + const start = source.lastIndexOf(quote, end - 1); + if (start < 0) throw new XmlDocumentParseError("malformed"); + ranges.push({ start, end }); + }); + }); + return ok(ranges); + } catch (cause: unknown) { + return err({ + error: + cause instanceof XmlDocumentParseError + ? cause + : new XmlDocumentParseError("malformed", cause), + ranges, + examined: Math.max(0, (reader?.position ?? 0) - 1), + }); + } +}; + /** One boundary owns strict XML syntax, under a DOCTYPE policy, and the source-located DOM projection. */ export const parseXmlDocument = ( source: string, diff --git a/tests/boundary/mcp/buildCaptureDraftMcp.test.ts b/tests/boundary/mcp/buildCaptureDraftMcp.test.ts index 82c7752412..3c16a43efa 100644 --- a/tests/boundary/mcp/buildCaptureDraftMcp.test.ts +++ b/tests/boundary/mcp/buildCaptureDraftMcp.test.ts @@ -110,6 +110,15 @@ it("returns each changed file's values, with what a value names, and guards outs check("check-1", `${item("eggs")}/attributes/cat`, "1"), check("check-2", `${item("eggs")}/attributes/qty`, "12"), check("check-3", `${item("bread")}/attributes/qty`, "3"), + // An item was added, so the list's count is a check too. + { + id: "check-4", + kind: "member_count", + file: LIST, + pointer: "/root/children", + matching: "[name=item]", + equals: 3, + }, ]); // The same fields on the nearest other items, which the user left alone. expect(draft?.guards).toEqual([ diff --git a/tests/boundary/providers/agentEnvCapture.test.ts b/tests/boundary/providers/agentEnvCapture.test.ts new file mode 100644 index 0000000000..fe29449430 --- /dev/null +++ b/tests/boundary/providers/agentEnvCapture.test.ts @@ -0,0 +1,58 @@ +import { execFileSync } from "node:child_process"; +import { chmod, mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +describe("agentenv capture launch", () => { + it("records an exit when the capture process never starts, so stop does not wait", async () => { + const root = await createTestTempDirectory("agentenv-capture-"); + const bin = join(root, "bin"); + const work = join(root, "work"); + const capture = join(root, "capture"); + await mkdir(bin); + await mkdir(work); + await writeFile(join(bin, "rea"), "#!/bin/sh\nexit 0\n"); + await chmod(join(bin, "rea"), 0o755); + const script = ` +import importlib.util, json, os, sys +spec = importlib.util.spec_from_file_location("capture", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +def boom(*_args, **_kwargs): + raise OSError("cannot start") +module.subprocess.Popen = boom +module.start({"work": sys.argv[2], "capture": sys.argv[3], "desktop_entry": "notes.desktop"}) +print(json.dumps({"exit": module._exit_code(sys.argv[3]), "control": os.path.exists(os.path.join(sys.argv[3], "control"))})) +`; + const output = execFileSync( + "python3", + [ + "-I", + "-B", + "-c", + script, + fileURLToPath( + new URL("../../../bridge/agentenv/capture.py", import.meta.url), + ), + work, + capture, + ], + { + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}` }, + encoding: "utf8", + }, + ); + const lines = output.trim().split("\n"); + const recorded = JSON.parse(lines.at(-1) ?? "{}") as { + exit: number | null; + control: boolean; + }; + + expect(recorded.exit).toBe(1); + expect(recorded.control).toBe(false); + }); +}); diff --git a/tests/boundary/providers/desktopExportPython.test.ts b/tests/boundary/providers/desktopExportPython.test.ts index 37a7be0c11..d3da1d8c30 100644 --- a/tests/boundary/providers/desktopExportPython.test.ts +++ b/tests/boundary/providers/desktopExportPython.test.ts @@ -67,6 +67,23 @@ for name, text in json.load(sys.stdin): .split(/\r?\n/u); }; +it("detects Unicode XML names in both state readers while rejecting non-XML text", () => { + const files = [ + ["map.dat", '<地图 名称="甲"><节点/></地图>'], + ["map.dat", '<!DOCTYPE 地图 SYSTEM "map.dtd"><地图/>'], + ["map.dat", "<máp/>"], + ["map.dat", "<地图><节点></地图>"], + ["note.dat", "<ada@example.test>\nBuy milk"], + ] as const; + const expected = ["xml", "xml", "xml", "text", "text"]; + expect(pythonFormats(files)).toEqual(expected); + expect( + files.map(([name, text]) => + detectStateFormat(name, new TextEncoder().encode(text)), + ), + ).toEqual(expected); +}); + it("detects key files by the state view's rule rather than a name or any group header", () => { const files = [ ["list.txt", "Groceries\n[x]\nmilk\n"], diff --git a/tests/boundary/providers/desktopStateHeap.test.ts b/tests/boundary/providers/desktopStateHeap.test.ts index 2f3431aa98..b93f2673be 100644 --- a/tests/boundary/providers/desktopStateHeap.test.ts +++ b/tests/boundary/providers/desktopStateHeap.test.ts @@ -50,7 +50,7 @@ console.log(JSON.stringify({ wide: deeperThan(wide, 500), edge: deeperThan(neste it("pairs elements with a long tag and many attributes in a small heap, spelling a selector only once chosen", async () => { // Spelling the tag into a selector per attribute needed more than 140 MiB here. const pairing = await inSmallHeap( - "viewPointer.ts", + "memberPairing.ts", "pairMembers", `const tag = "t".repeat(40_000); const element = (id) => { diff --git a/tests/boundary/providers/desktopXmlTextPython.test.ts b/tests/boundary/providers/desktopXmlTextPython.test.ts index 50768e9f6a..0647729fb5 100644 --- a/tests/boundary/providers/desktopXmlTextPython.test.ts +++ b/tests/boundary/providers/desktopXmlTextPython.test.ts @@ -146,6 +146,48 @@ describe("XML with a DOCTYPE in the exported module", () => { }); }); + it("reads text that opens with an element as XML when it is well formed, whatever the file is called, alike in both", async () => { + const saved = (name: string, text: string) => ({ + name, + bytes: new TextEncoder().encode(text), + }); + const files = [ + saved( + "weekend-trip.mm", + '<map version="freeplane 1.7.0">\n<node TEXT="Weekend Trip" ID="ID_1"><node TEXT="Packing" ID="ID_2"/></node>\n</map>\n', + ), + saved("notes.txt", "<b>bold</b> and <i>italic</b>\n"), + saved("plain.txt", " <todo><item>milk</item></todo>\n"), + ]; + await expectSameFileViews(files); + + const formats = await Promise.all( + files.map(async (file) => (await typescriptFileView(file)).format), + ); + expect(formats).toEqual(["xml", "text", "xml"]); + }); + + it("reads text that opens with a DOCTYPE naming no internal subset and then an element as XML, as KmPlot's plots, alike in both", async () => { + const saved = (name: string, text: string) => ({ + name, + bytes: new TextEncoder().encode(text), + }); + const files = [ + saved( + "lesson_plots.fkt", + '<!DOCTYPE kmpdoc>\n<kmpdoc version="4">\n <function type="cartesian"><equation-0 step="0.05">h(x) = 2x + 1</equation-0></function>\n</kmpdoc>\n', + ), + saved("entities.fkt", '<!DOCTYPE a [<!ENTITY e "boom">]>\n<a>&e;</a>\n'), + saved("broken.fkt", "<!DOCTYPE a>\n<a><b></a>\n"), + ]; + await expectSameFileViews(files); + + const formats = await Promise.all( + files.map(async (file) => (await typescriptFileView(file)).format), + ); + expect(formats).toEqual(["xml", "text", "text"]); + }); + it("never reads the DTD a DOCTYPE names: neither its entities nor its default attributes apply", async () => { const folder = await createTestTempDirectory("desktop-xml-dtd-"); const dtd = join(folder, "a.dtd");