Merge branch 'dev' into chore/agents-contributing

* Fix: count selection_data from one pass over the filtered documents

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* Chore: use document type ids in the bulk edit selection_data test

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* Batch search selection data queries

Co-Authored-By: Codex <267193182+codex@users.noreply.github.com>

---------

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
Co-authored-by: Codex <267193182+codex@users.noreply.github.com>

.github/actions/apt-install/install.sh

@@ -6,9 +6,9 @@ if [ "${1:-}" = download ]; then
 	download_archive_dir=$2
 	shift 2
 	apt-get update -q
-	apt-get install -q --yes --no-install-recommends --download-only \
+	# Keep apt itself under timeout supervision, including the kill grace period.
+	exec apt-get install -q --yes --no-install-recommends --download-only \
 		-o "Dir::Cache::archives=$download_archive_dir" "$@"
-	exit 0
 fi
 
 SUDO=()
@@ -34,7 +34,8 @@ use_ubuntu_mirrors() {
 
 for attempt in $(seq 1 "$ATTEMPTS"); do
 	echo "::group::apt download attempt ${attempt}/${ATTEMPTS}"
-	if timeout --kill-after=10 "$ATTEMPT_TIMEOUT" "${SUDO[@]}" bash "$0" download "$CACHE_DIR" "${requested_packages[@]}"; then
+	# Run timeout as root so it can signal the root-owned apt processes.
+	if "${SUDO[@]}" timeout --kill-after=10 "$ATTEMPT_TIMEOUT" bash "$0" download "$CACHE_DIR" "${requested_packages[@]}"; then
 		echo "::endgroup::"
 		# Apt handles aliases (e.g. libmagic1 -> libmagic1t64) and dependencies.
 		"${SUDO[@]}" apt-get install -q --yes --no-install-recommends --no-download \

AGENTS.md

@@ -0,0 +1,44 @@
+# Guidance for AI agents
+
+Read [CONTRIBUTING.md](CONTRIBUTING.md) before working on this repository, especially [Use of AI Tools](CONTRIBUTING.md#use-of-ai-tools). These contribution rules apply to all agent-assisted work.
+
+## Contribution boundaries
+
+- Do not open or submit pull requests, including "on behalf of" a human. The project does not accept PRs from AI agents. Prepare local changes for a human to review, verify, and submit personally.
+- The project does not accept PRs that are entirely or mostly AI-derived. Human submission and disclosure do not override this rule.
+- AI-generated content must be clearly disclosed as described in the contributing guidelines. When handing work back to a human, identify what was generated or changed with AI assistance so they can disclose it accurately.
+- Trust matters. Encourage first-time contributors to discuss their proposed change before preparing a PR, especially for large changes or changes involving AI-generated code. New features and enhancements should almost always target an existing feature request with community interest and discussion.
+- The project prefers thoughtful, well-described ideas to AI-generated pull requests. Help contributors describe their problem, use case, desired behavior, and who would benefit. An implementation is not required for a valuable contribution, and a flawed implementation can obscure a useful idea before it receives proper discussion.
+- AI tools have lowered the effort needed to produce a PR, but reviewing and maintaining it still requires substantial work. Encourage discussion of the idea before investing in implementation and review.
+- Issue reports must describe observed behavior, reproduction steps, and relevant logs. Do not add speculative root causes, code analysis, or proposed fixes to agent-generated reports.
+
+## Security reports
+
+- AI-assisted security reports must meet the legitimacy and verification requirements in [SECURITY.md](SECURITY.md#ai-assisted-submissions). A human must verify a real Paperless-ngx code path, concrete reproduction steps, affected versions, and realistic security impact before submission.
+- Account for existing permission checks, configuration requirements, and trust boundaries. Speculation, scanner output, or concerns about third-party software or deployment choices without a demonstrated Paperless-ngx vulnerability do not meet this threshold and may be closed without extended discussion.
+- Disclose AI assistance and use the private vulnerability reporting channel specified in [SECURITY.md](SECURITY.md#reporting-a-vulnerability).
+
+## Working on changes
+
+- Read the relevant code, tests, and documentation before editing. Paperless-ngx is complex: trace interactions between affected components rather than assuming a change is isolated.
+- Keep changes focused on the requested work and follow existing patterns. Avoid unrelated refactoring, formatting, dependency updates, or generated-file changes.
+- Consider effects on permissions, document processing, background tasks, storage, and the API when those areas are involved.
+- Add or update meaningful tests for behavior changes and regressions. Do not weaken tests or checks to make a change pass.
+- Update user or developer documentation when behavior or setup changes.
+- Keep credentials and private documents out of commits, fixtures, logs, and examples.
+
+## Repository and validation
+
+- `src/`: Django back end.
+- `src-ui/`: Angular front end, managed with `pnpm`.
+- `docs/`: Documentation.
+- `docker/` and `scripts/`: Deployment and development tooling.
+- Base functional changes on `dev`; documentation-only changes may target `main`.
+
+Follow [the development documentation](docs/development.md) for environment setup and testing. Common checks after setup are:
+
+- Back end: run `uv run pytest` from `src/` (use a test path to focus on the affected area).
+- Front end: run `pnpm test` and `pnpm lint` from `src-ui/`.
+- Formatting and other pre-commit checks: run `uv run prek run --files <changed-files>` from the repository root.
+
+When handing work back, summarize the changes, the checks actually run and their results, and any remaining uncertainty. Clearly state when a check could not be run; do not claim verification that did not happen. The human contributor must understand and independently verify the contribution before submitting it.

CONTRIBUTING.md

@@ -4,6 +4,18 @@ If you feel like contributing to the project, please do! Bug fixes and improveme
 
 ⚠️ Please note: Pull requests that implement a new feature or enhancement _should almost always target an existing feature request_ with evidence of community interest and discussion. This is in order to balance the work of implementing and maintaining new features / enhancements. Pull requests that are opened without meeting this requirement may not be merged.
 
+## Trust and collaboration
+
+Paperless-ngx places a high value on trust between contributors and maintainers. If you have not contributed before, consider starting with a discussion before preparing a pull request, especially for a large contribution or one involving AI-generated code.
+
+Paperless-ngx is complex, and understanding how its components interact takes time. A change that appears correct in isolation can have consequences elsewhere in the application. Contributors should understand the code they submit, explain how it fits into the project, and work with maintainers through review. Starting with smaller, focused contributions can help build that understanding and trust.
+
+We prefer thoughtful, well-described ideas to AI-generated pull requests. You do not need to supply an implementation for your contribution to be valuable. Explain the problem you have encountered, your use case, the behavior you would like to see, and who would benefit. Concrete examples, alternatives you have considered, and relevant constraints help the community discuss and refine the idea.
+
+AI tools have lowered the effort needed to produce a pull request, but evaluating one still requires substantial maintainer time. Maintainers still need to establish whether the implementation is correct, fits the project, and can be maintained over time.
+
+A flawed implementation can also obscure a good idea: discussion becomes focused on problems with the generated code before the community has considered whether the underlying proposal is useful. Starting with the idea gives it a fair discussion on its own merits and lets contributors and maintainers agree on an approach before investing in implementation and review.
+
 If you want to implement something big:
 
 - As above, please start with a discussion! Maybe something similar is already in development and we can make it happen together.
@@ -61,9 +73,10 @@ This project does not specifically prohibit the use of AI tools or agents _durin
 
 1. Any content — code, pull request descriptions, issue reports or comments — that was generated in whole or in part by an AI tool or agent **must be clearly disclosed as such**. Failing to clearly disclose this is considered a violation of our [Code of Conduct](CODE_OF_CONDUCT.md) and may result in your contributions being closed and further action being taken.
 2. AI-generated code must not violate copyright protections.
-3. We will not accept PRs that are entirely or mostly AI-derived.
-4. Issue reports must describe _observed behavior only_: what happened, how to reproduce it and the relevant logs. Reports generated by an AI agent should not include code analysis, speculation about the root cause or suggest fixes.
-5. Contributors are responsible for everything they submit, regardless of how it was produced. Please do not open issues or PRs you have not verified yourself.
+3. We will not accept PRs opened or submitted by AI agents, including agents acting "on behalf of" a human. A human contributor must personally review, verify, and submit the contribution and take responsibility for it throughout review.
+4. We will not accept PRs that are entirely or mostly AI-derived.
+5. Issue reports must describe _observed behavior only_: what happened, how to reproduce it and the relevant logs. Reports generated by an AI agent should not include code analysis, speculation about the root cause or suggest fixes.
+6. Contributors are responsible for everything they submit, regardless of how it was produced. Please do not open issues or PRs you have not verified yourself.
 
 # Translating Paperless-ngx
 

SECURITY.md

@@ -56,6 +56,9 @@ We may close reports that are:
 
 The following are not generally considered vulnerabilities unless accompanied by a concrete, reproducible impact in Paperless-ngx:
 
+- password length inferred from masked mail-account passwords by users who already have permission to view the account, without disclosure of the password or token itself
+- disclosure limited to document IDs, existence, page counts, checksums, or modification/deletion timestamps, without access to document contents or an unintended privileged action
+- authenticated access to non-secret application settings, such as service endpoint URLs, model names, or configured environment variable names, without disclosure of credentials or secret values
 - large uploads or resource usage that do not bypass documented limits or privileges
 - IDOR / access control claims regarding the ability to attach an un-viewable object to a document. This is expected behavior.
 - claims based solely on the presence of a library, framework feature or code pattern without a working exploit

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/run

@@ -39,7 +39,6 @@ declare -a scope=(
 	"${site_packages}/allauth"
 	"${site_packages}/drf_spectacular"
 	"${site_packages}/drf_spectacular_sidecar"
-	"${site_packages}/treenode"
 	"${site_packages}/compression_middleware"
 )
 

pyproject.toml

@@ -38,7 +38,6 @@ dependencies = [
   "django-multiselectfield~=1.0.1",
   "django-rich~=2.2.0",
   "django-soft-delete~=1.0.18",
-  "django-treenode>=0.24",
   "djangorestframework~=3.16",
   "drf-spectacular~=0.30",
   "drf-spectacular-sidecar>=2026.10.1,<2026.11",

src-ui/messages.xlf

@@ -6984,7 +6984,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/input/select/select.component.ts</context>
-          <context context-type="linenumber">179</context>
+          <context context-type="linenumber">185</context>
         </context-group>
       </trans-unit>
       <trans-unit id="1880237574877817137" datatype="html">
@@ -7091,7 +7091,7 @@
         <source>Private</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/input/select/select.component.ts</context>
-          <context context-type="linenumber">72</context>
+          <context context-type="linenumber">73</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/tag/tag.component.html</context>
@@ -7114,7 +7114,7 @@
         <source>No items found</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/input/select/select.component.ts</context>
-          <context context-type="linenumber">113</context>
+          <context context-type="linenumber">114</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6541407358060244620" datatype="html">

src-ui/src/app/components/common/input/document-link/document-link.component.html

@@ -36,7 +36,7 @@
     [appendTo]="appendTo"
     [loading]="loading()"
     [typeahead]="documentsInput$"
-    (mousedown)="$event.stopImmediatePropagation()"
+    (mousedown)="appendTo && $event.stopImmediatePropagation()"
     (change)="onChange(selectedDocumentIDs)">
     <ng-template ng-label-tmp let-document="item">
       <div class="d-flex align-items-center">

src-ui/src/app/components/common/input/select/select.component.spec.ts

@@ -97,6 +97,7 @@ describe('SelectComponent', () => {
     let createNewVal
     component.createNew.subscribe((v) => (createNewVal = v))
     expect(component.allowCreateNew).toBeTruthy()
+    const closeSpy = jest.spyOn(component.ngSelectComponent, 'close')
     component.onSearch({ term: 'foo' })
     component.addItem(undefined)
     expect(createNewVal).toEqual('foo')
@@ -105,6 +106,8 @@ describe('SelectComponent', () => {
     component.onSearch({ term: 'baz' })
     component.clickNew()
     expect(createNewVal).toEqual('baz')
+    // dropdown should close when creating a new item so it isn't left open behind the modal
+    expect(closeSpy).toHaveBeenCalled()
   })
 
   it('should search items by independent normalized terms', () => {

src-ui/src/app/components/common/input/select/select.component.ts

@@ -4,6 +4,7 @@ import {
   forwardRef,
   Input,
   Output,
+  ViewChild,
 } from '@angular/core'
 import {
   FormsModule,
@@ -11,7 +12,7 @@ import {
   ReactiveFormsModule,
 } from '@angular/forms'
 import { RouterModule } from '@angular/router'
-import { NgSelectModule } from '@ng-select/ng-select'
+import { NgSelectComponent, NgSelectModule } from '@ng-select/ng-select'
 import { BootstrapIconComponent } from 'src/app/components/common/bootstrap-icon/bootstrap-icon.component'
 import { matchesSearchText } from 'src/app/utils/text-search'
 import { AbstractInputComponent } from '../abstract-input'
@@ -124,6 +125,9 @@ export class SelectComponent extends AbstractInputComponent<number | string> {
   @Output()
   filterDocuments = new EventEmitter<any[]>()
 
+  @ViewChild(NgSelectComponent)
+  ngSelectComponent: NgSelectComponent
+
   public addItemRef: (name) => void
 
   private _lastSearchTerm: string
@@ -150,11 +154,13 @@ export class SelectComponent extends AbstractInputComponent<number | string> {
     if (name) this.createNew.next(name)
     else this.createNew.next(this._lastSearchTerm)
     this.clearLastSearchTerm()
+    this.ngSelectComponent?.close()
   }
 
   clickNew() {
     this.createNew.next(this._lastSearchTerm)
     this.clearLastSearchTerm()
+    this.ngSelectComponent?.close()
   }
 
   clearLastSearchTerm() {

src/documents/admin.py

@@ -1,7 +1,6 @@
 from django.conf import settings
 from django.contrib import admin
 from guardian.admin import GuardedModelAdmin
-from treenode.admin import TreeNodeModelAdmin
 
 from documents.models import Correspondent
 from documents.models import CustomField
@@ -29,7 +28,7 @@ class CorrespondentAdmin(GuardedModelAdmin):
     list_editable = ("match", "matching_algorithm")
 
 
-class TagAdmin(GuardedModelAdmin, TreeNodeModelAdmin):
+class TagAdmin(GuardedModelAdmin):
     list_display = ("name", "color", "match", "matching_algorithm")
     list_filter = ("matching_algorithm",)
     list_editable = ("color", "match", "matching_algorithm")

src/documents/bulk_edit.py

@@ -255,19 +255,13 @@ def modify_tags(
     affected_docs = list(qs.values_list("pk", flat=True))
     DocumentTagRelationship = Document.tags.through
 
-    # add with all ancestors
-    expanded_add_tags: set[int] = set()
-    add_tag_objects = Tag.objects.filter(pk__in=add_tags)
-    for t in add_tag_objects:
-        expanded_add_tags.add(int(t.id))
-        expanded_add_tags.update(int(pk) for pk in t.get_ancestors_pks())
-
-    # remove with all descendants
-    expanded_remove_tags: set[int] = set()
-    remove_tag_objects = Tag.objects.filter(pk__in=remove_tags)
-    for t in remove_tag_objects:
-        expanded_remove_tags.add(int(t.id))
-        expanded_remove_tags.update(int(pk) for pk in t.get_descendants_pks())
+    hierarchy = Tag.get_hierarchy()
+    expanded_add_tags = hierarchy.with_ancestors(
+        Tag.objects.filter(pk__in=add_tags).values_list("pk", flat=True),
+    )
+    expanded_remove_tags = hierarchy.with_descendants(
+        Tag.objects.filter(pk__in=remove_tags).values_list("pk", flat=True),
+    )
 
     with transaction.atomic():
         if expanded_remove_tags:

src/documents/management/commands/document_importer.py

@@ -108,6 +108,24 @@ def _deserialize_record(
         ) from e
 
     for field_name, field_value in record.get("fields", {}).items():
+        # Older exports include django-treenode's derived bookkeeping. The
+        # parent FK is authoritative; accept only these known removed fields.
+        if Model is Tag and field_name in {
+            "tn_ancestors_count",
+            "tn_ancestors_pks",
+            "tn_children_count",
+            "tn_children_pks",
+            "tn_depth",
+            "tn_descendants_count",
+            "tn_descendants_pks",
+            "tn_index",
+            "tn_level",
+            "tn_order",
+            "tn_priority",
+            "tn_siblings_count",
+            "tn_siblings_pks",
+        }:
+            continue
         field = Model._meta.get_field(field_name)
         remote = field.remote_field
 

src/documents/migrations/0028_replace_tag_tree_fields.py

@@ -0,0 +1,64 @@
+# Generated by Django 5.2.17 on 2026-10-28 22:54
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("documents", "0027_documentbarcode"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_ancestors_count",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_ancestors_pks",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_children_count",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_children_pks",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_depth",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_descendants_count",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_descendants_pks",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_index",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_level",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_order",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_priority",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_siblings_count",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_siblings_pks",
+        ),
+    ]

src/documents/models.py

@@ -1,6 +1,7 @@
 import datetime
 import uuid
 from pathlib import Path
+from typing import TYPE_CHECKING
 from typing import Final
 
 import pathvalidate
@@ -14,7 +15,6 @@ from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from multiselectfield import MultiSelectField
-from treenode.models import TreeNodeModel
 
 if settings.AUDIT_LOG_ENABLED:
     from auditlog.registry import auditlog
@@ -29,6 +29,9 @@ from django_softdelete.models import SoftDeleteModel
 from documents.data_models import DocumentSource
 from documents.parsers import get_default_file_extension
 
+if TYPE_CHECKING:
+    from documents.tag_hierarchy import TagHierarchy
+
 
 class ModelWithOwner(models.Model):
     owner = models.ForeignKey(
@@ -100,7 +103,17 @@ class Correspondent(MatchingModel):
         verbose_name_plural = _("correspondents")
 
 
-class Tag(MatchingModel, TreeNodeModel):
+class Tag(MatchingModel):
+    # Keep the existing column and reverse relation during the transition.
+    tn_parent = models.ForeignKey(
+        "self",
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="tn_children",
+        verbose_name=_("Parent"),
+    )
+
     color = models.CharField(_("color"), max_length=7, default="#a6cee3")
     # Maximum allowed nesting depth for tags (root = 1, max depth = 5)
     MAX_NESTING_DEPTH: Final[int] = 5
@@ -114,27 +127,72 @@ class Tag(MatchingModel, TreeNodeModel):
         ),
     )
 
-    class Meta(MatchingModel.Meta, TreeNodeModel.Meta):
+    class Meta(MatchingModel.Meta):
         verbose_name = _("tag")
         verbose_name_plural = _("tags")
 
+    @classmethod
+    def get_hierarchy(cls) -> "TagHierarchy":
+        """Read the tag hierarchy in one query, ready to use across multiple tags."""
+        from documents.tag_hierarchy import TagHierarchy
+
+        return TagHierarchy(cls.objects.values_list("pk", "tn_parent_id"))
+
+    @property
+    def parent_pk(self) -> int | None:
+        return self.tn_parent_id
+
+    def get_parent(self) -> "Tag | None":
+        return self.tn_parent
+
+    def get_ancestors_pks(self) -> list[int]:
+        """Return ancestor IDs ordered from root to parent.
+
+        Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
+        reuse it when working with several tags.
+        """
+        return self.get_hierarchy().ancestors(self.tn_parent_id)
+
+    def get_ancestors(self) -> list["Tag"]:
+        """Return ancestors ordered from root to parent.
+
+        Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
+        reuse it when working with several tags.
+        """
+        ids = self.get_ancestors_pks()
+        tags = self.__class__.objects.in_bulk(ids)
+        return [tags[pk] for pk in ids]
+
+    def get_descendants_pks(self) -> set[int]:
+        """Return all descendant IDs.
+
+        Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
+        reuse it when working with several tags.
+        """
+        return self.get_hierarchy().descendants([self.pk])
+
+    def get_descendants(self) -> "models.QuerySet[Tag]":
+        """Return a queryset of all descendants.
+
+        Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
+        reuse it when working with several tags.
+        """
+        return self.__class__.objects.filter(pk__in=self.get_descendants_pks())
+
+    def get_children_queryset(self) -> "models.QuerySet[Tag]":
+        return self.__class__.objects.filter(tn_parent_id=self.pk)
+
     def clean(self) -> None:
-        # Prevent self-parenting and assigning a descendant as parent
-        parent = self.get_parent()
-        if parent == self:
-            raise ValidationError({"parent": _("Cannot set itself as parent.")})
-        if parent and self.pk is not None and self.is_ancestor_of(parent):
-            raise ValidationError({"parent": _("Cannot set parent to a descendant.")})
-
-        # Enforce maximum nesting depth
-        new_parent_depth = 0
-        if parent:
-            new_parent_depth = parent.get_ancestors_count() + 1
-
-        height = 0 if self.pk is None else self.get_depth()
-        deepest_new_depth = (new_parent_depth + 1) + height
-        if deepest_new_depth > self.MAX_NESTING_DEPTH:
-            raise ValidationError({"parent": _("Maximum nesting depth exceeded.")})
+        hierarchy = self.get_hierarchy()
+        if self.pk is not None and self.tn_parent_id == self.pk:
+            raise ValidationError({"tn_parent": _("Cannot set itself as parent.")})
+        ancestors = hierarchy.ancestors(self.tn_parent_id)
+        if self.pk is not None and self.pk in ancestors:
+            raise ValidationError(
+                {"tn_parent": _("Cannot set parent to a descendant.")},
+            )
+        if len(ancestors) + 1 + hierarchy.height(self.pk) > self.MAX_NESTING_DEPTH:
+            raise ValidationError({"tn_parent": _("Maximum nesting depth exceeded.")})
 
         return super().clean()
 
@@ -539,10 +597,7 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
         return self.created
 
     def add_nested_tags(self, tags) -> None:
-        tag_ids = set()
-        for tag in tags:
-            tag_ids.add(tag.id)
-            tag_ids.update(tag.get_ancestors_pks())
+        tag_ids = Tag.get_hierarchy().with_ancestors(tag.pk for tag in tags)
 
         tags_to_add = self.tags.model.objects.filter(id__in=tag_ids)
         self.tags.add(*tags_to_add)

src/documents/permissions.py

@@ -31,6 +31,7 @@ from guardian.core import ObjectPermissionChecker
 from guardian.models import GroupObjectPermission
 from guardian.models import UserObjectPermission
 from rest_framework.permissions import BasePermission
+from rest_framework.permissions import DjangoModelPermissions
 from rest_framework.permissions import DjangoObjectPermissions
 
 from documents.models import Document
@@ -63,6 +64,17 @@ class PermissionGranteesUpdate(TypedDict, total=False):
 PermissionsSpec: TypeAlias = Mapping[PermissionAction, PermissionGranteesUpdate]
 
 
+class PaperlessModelPermissions(DjangoModelPermissions):
+    """Require basic global model permissions."""
+
+    perms_map = {
+        **DjangoModelPermissions.perms_map,
+        "GET": ["%(app_label)s.view_%(model_name)s"],
+        "OPTIONS": ["%(app_label)s.view_%(model_name)s"],
+        "HEAD": ["%(app_label)s.view_%(model_name)s"],
+    }
+
+
 class PaperlessObjectPermissions(DjangoObjectPermissions):
     """
     A permissions backend that checks for object-level permissions
@@ -576,12 +588,9 @@ def annotate_document_count_by_ids(
                        or CustomFieldInstance)
         related_object_field: field on the relation pointing back to queryset pk
         document_ids: the document ids to count against -- a concrete list/set,
-                       or a simple (already resolved) queryset of ids. Callers
-                       that need this filtered by a complex condition (e.g. a
-                       permission check) should resolve it to a concrete list
-                       first if the same ids will be reused across multiple
-                       calls, rather than passing the complex queryset itself
-                       into each -- see ``_get_selection_data_for_queryset``.
+                       or a simple (already resolved) queryset of ids. Keep
+                       the list small: it is embedded in the statement as
+                       literal parameters.
         target_field: field on the relation pointing to Document id
     """
 

src/documents/serialisers.py

@@ -589,7 +589,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
 
     text_color = serializers.SerializerMethodField()
 
-    # map to treenode's tn_parent
+    # Preserve the public API parent field.
     parent = serializers.PrimaryKeyRelatedField(
         queryset=Tag.objects.all(),
         allow_null=True,
@@ -692,7 +692,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
                 self.instance.clean()
             except ValidationError as e:
                 logger.debug("Tag parent validation failed: %s", e)
-                raise e
+                raise serializers.ValidationError(
+                    {"parent": e.message_dict["tn_parent"]},
+                ) from e
             finally:
                 self.instance.tn_parent = original_parent
         else:
@@ -702,7 +704,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
                 temp.clean()
             except ValidationError as e:
                 logger.debug("Tag parent validation failed: %s", e)
-                raise e
+                raise serializers.ValidationError(
+                    {"parent": e.message_dict["tn_parent"]},
+                ) from e
 
         return super().validate(attrs)
 
@@ -1198,61 +1202,35 @@ class DocumentSerializer(
                             doc_id,
                         )
         if "tags" in validated_data:
-            # Respect tag hierarchy on updates:
-            # - Adding a child adds its ancestors
-            # - Removing a parent removes all its descendants
-            prev_tags = set(instance.tags.all())
-            requested_tags = set(validated_data["tags"])
+            hierarchy = Tag.get_hierarchy()
+            previous_ids = set(instance.tags.values_list("pk", flat=True))
+            requested_ids = {tag.pk for tag in validated_data["tags"]}
+            required_ids = hierarchy.with_ancestors(requested_ids - previous_ids)
+            blocked_ids = hierarchy.with_descendants(previous_ids - requested_ids)
+            blocked_ids.difference_update(required_ids)
+            final_ids = hierarchy.with_ancestors(requested_ids) - blocked_ids
+            validated_data["tags"] = list(Tag.objects.filter(pk__in=final_ids))
 
-            # Tags newly added in this update and the ancestors they require
-            added_tags = requested_tags - prev_tags
-            required_by_add_tags = set(added_tags)
-            for t in added_tags:
-                required_by_add_tags.update(t.get_ancestors())
-
-            # Tags being removed in this update and all descendants, except
-            # those required by a tag that is being added in this same update
-            removed_tags = prev_tags - requested_tags
-            blocked_tags = set(removed_tags)
-            for t in removed_tags:
-                blocked_tags.update(t.get_descendants())
-            blocked_tags.difference_update(required_by_add_tags)
-
-            # Add all parent tags
-            final_tags = set(requested_tags)
-            for t in requested_tags:
-                final_tags.update(t.get_ancestors())
-
-            # Drop removed parents and their descendants
-            final_tags.difference_update(blocked_tags)
-
-            validated_data["tags"] = list(final_tags)
         if validated_data.get("remove_inbox_tags"):
-            current_tag_ids = {t.pk for t in instance.tags.all()}
+            hierarchy = Tag.get_hierarchy()
             tags = (
                 validated_data["tags"]
                 if "tags" in validated_data
                 else list(instance.tags.all())
             )
-
-            # Tags newly added in this update, plus their ancestors, are kept
-            keep_ids: set[int] = set()
-            for tag in tags:
-                if tag.pk not in current_tag_ids:
-                    keep_ids.add(tag.pk)
-                    keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
-
-            # Remove inbox tags and their descendants, except those being kept
-            remove_ids: set[int] = set()
-            for inbox_tag in (
+            current_ids = set(instance.tags.values_list("pk", flat=True))
+            keep_ids = hierarchy.with_ancestors(
+                tag.pk for tag in tags if tag.pk not in current_ids
+            )
+            inbox_ids = (
                 Tag.objects.filter(is_inbox_tag=True)
-                .exclude(pk__in=keep_ids)
-                .only("pk", "tn_descendants_pks")
-            ):
-                remove_ids.add(inbox_tag.pk)
-                remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
-
-            validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
+                .exclude(
+                    pk__in=keep_ids,
+                )
+                .values_list("pk", flat=True)
+            )
+            remove_ids = hierarchy.with_descendants(inbox_ids) - keep_ids
+            validated_data["tags"] = [tag for tag in tags if tag.pk not in remove_ids]
 
         if settings.AUDIT_LOG_ENABLED:
             with set_actor(self.user):

src/documents/tag_hierarchy.py

@@ -0,0 +1,61 @@
+"""An operation-local snapshot of tag edges, with no persistent derived fields.
+
+Never cache this globally: tasks, bulk updates and other processes can change
+parents. Callers traversing multiple tags should share one snapshot.
+"""
+
+from collections import defaultdict
+from collections.abc import Iterable
+
+
+class TagHierarchy:
+    def __init__(self, edges: Iterable[tuple[int, int | None]]) -> None:
+        self.parents = dict(edges)
+        self.children: dict[int | None, list[int]] = defaultdict(list)
+        for pk, parent in self.parents.items():
+            self.children[parent].append(pk)
+
+    def ancestors(self, parent_id: int | None) -> list[int]:
+        result = []
+        seen = set()
+        while parent_id is not None and parent_id not in seen:
+            seen.add(parent_id)
+            result.append(parent_id)
+            parent_id = self.parents.get(parent_id)
+        return result[::-1]
+
+    def with_ancestors(self, roots: Iterable[int]) -> set[int]:
+        result = set(roots)
+        for pk in list(result):
+            result.update(self.ancestors(self.parents.get(pk)))
+        return result
+
+    def with_descendants(self, roots: Iterable[int]) -> set[int]:
+        roots = set(roots)
+        return roots | self.descendants(roots)
+
+    def descendants(self, roots: Iterable[int]) -> set[int]:
+        roots = {pk for pk in roots if pk is not None}
+        seen = set(roots)
+        pending = list(roots)
+        while pending:
+            for child in self.children.get(pending.pop(), ()):
+                if child not in seen:
+                    seen.add(child)
+                    pending.append(child)
+        return seen - roots
+
+    def height(self, pk: int | None) -> int:
+        if pk is None:
+            return 0
+        seen = {pk}
+        pending = [(pk, 0)]
+        height = 0
+        while pending:
+            parent, depth = pending.pop()
+            height = max(height, depth)
+            for child in self.children.get(parent, ()):
+                if child not in seen:
+                    seen.add(child)
+                    pending.append((child, depth + 1))
+        return height

src/documents/tests/test_api_app_config.py

@@ -15,6 +15,7 @@ from paperless.models import ColorConvertChoices
 from paperless_testing.dirs import DirectoriesMixin
 from paperless_testing.factories import UserFactory
 from paperless_testing.http import read_streaming_response
+from paperless_testing.permissions import grant_global
 
 
 class TestApiAppConfig(DirectoriesMixin, APITestCase):
@@ -26,6 +27,67 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
         user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=user)
 
+    def test_api_read_config_requires_view_permission(self) -> None:
+        for permissions in (
+            (),
+            ("change_applicationconfiguration",),
+            ("view_document",),
+        ):
+            user = UserFactory()
+            grant_global(user, *permissions)
+            self.client.force_authenticate(user=user)
+
+            for endpoint in (self.ENDPOINT, f"{self.ENDPOINT}1/"):
+                for method in ("get", "head", "options"):
+                    with self.subTest(
+                        permissions=permissions,
+                        endpoint=endpoint,
+                        method=method,
+                    ):
+                        response = getattr(self.client, method)(endpoint)
+                        self.assertEqual(
+                            response.status_code,
+                            status.HTTP_403_FORBIDDEN,
+                        )
+
+    def test_api_read_config_with_view_permission(self) -> None:
+        user = UserFactory()
+        grant_global(user, "view_applicationconfiguration")
+        self.client.force_authenticate(user=user)
+
+        for endpoint in (self.ENDPOINT, f"{self.ENDPOINT}1/"):
+            for method in ("get", "head", "options"):
+                with self.subTest(endpoint=endpoint, method=method):
+                    response = getattr(self.client, method)(endpoint)
+                    self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+    def test_api_view_config_permission_does_not_allow_writes(self) -> None:
+        user = UserFactory()
+        grant_global(user, "view_applicationconfiguration")
+        self.client.force_authenticate(user=user)
+
+        for method in ("put", "patch", "delete"):
+            with self.subTest(method=method):
+                response = getattr(self.client, method)(f"{self.ENDPOINT}1/")
+                self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+    def test_api_change_config_permission_allows_updates(self) -> None:
+        user = UserFactory()
+        grant_global(user, "change_applicationconfiguration")
+        self.client.force_authenticate(user=user)
+
+        response = self.client.patch(
+            f"{self.ENDPOINT}1/",
+            {"app_title": "Updated title"},
+            format="json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            ApplicationConfiguration.objects.get().app_title,
+            "Updated title",
+        )
+
     def test_api_get_config(self) -> None:
         """
         GIVEN:

src/documents/tests/test_api_bulk_edit.py

@@ -1141,8 +1141,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertCountEqual(
             response.data["selected_document_types"],
             [
-                {"id": self.c1.id, "document_count": 1},
-                {"id": self.c2.id, "document_count": 0},
+                {"id": self.dt1.id, "document_count": 1},
+                {"id": self.dt2.id, "document_count": 0},
             ],
         )
 

src/documents/tests/test_api_documents.py

@@ -49,6 +49,7 @@ from documents.models import WorkflowTrigger
 from documents.search import get_backend
 from documents.signals.handlers import run_workflows
 from documents.tests.utils import ConsumeTaskMixin
+from documents.views import DocumentViewSet
 from paperless_testing.dirs import DirectoriesMixin
 from paperless_testing.factories import DocumentFactory
 from paperless_testing.factories import TagFactory
@@ -1755,12 +1756,113 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         }
         self.assertEqual(counts_by_field[custom_field.id], 1)
 
+    def test_selection_data_counts_with_shared_by_filter(self) -> None:
+        """
+        GIVEN:
+            - Several documents owned by one user with the same correspondent
+              and document type, only one of which is shared
+        WHEN:
+            - Requesting the document list filtered by shared_by__id with
+              include_selection_data=true
+        THEN:
+            - Counts reflect only the shared document, not every document the
+              owner has with that correspondent or document type, regardless of
+              the batch size
+        """
+        owner = User.objects.create_user(username="owner")
+        other = User.objects.create_user(username="other")
+        correspondent = Correspondent.objects.create(name="c1")
+        doc_type = DocumentType.objects.create(name="dt1")
+        tag = Tag.objects.create(name="tag")
+        docs = [
+            Document.objects.create(
+                checksum=str(i),
+                owner=owner,
+                correspondent=correspondent,
+                document_type=doc_type,
+            )
+            for i in range(3)
+        ]
+        for doc in docs:
+            doc.tags.add(tag)
+        grant_object(other, docs[0], "view_document")
+
+        for batch_size in (10_000, 1):
+            with (
+                self.subTest(batch_size=batch_size),
+                mock.patch.object(
+                    DocumentViewSet,
+                    "selection_data_batch_size",
+                    batch_size,
+                ),
+            ):
+                response = self.client.get(
+                    f"/api/documents/?shared_by__id={owner.id}"
+                    "&include_selection_data=true",
+                )
+
+                self.assertEqual(response.status_code, status.HTTP_200_OK)
+                self.assertEqual(response.data["count"], 1)
+                selection_data = response.data["selection_data"]
+                for key, obj in (
+                    ("selected_correspondents", correspondent),
+                    ("selected_document_types", doc_type),
+                    ("selected_tags", tag),
+                ):
+                    counts = {
+                        item["id"]: item["document_count"]
+                        for item in selection_data[key]
+                    }
+                    self.assertEqual(counts[obj.id], 1, key)
+
+    def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
+        """
+        GIVEN:
+            - A document matching a multi-valued filter through two tags
+        WHEN:
+            - Requesting the document list with include_selection_data=true
+        THEN:
+            - The document is counted once per correspondent, not once per
+              matching tag, regardless of the batch size
+        """
+        correspondent = Correspondent.objects.create(name="c1")
+        tag_a = Tag.objects.create(name="a")
+        tag_b = Tag.objects.create(name="b")
+        both = Document.objects.create(checksum="1", correspondent=correspondent)
+        both.tags.add(tag_a, tag_b)
+        only_a = Document.objects.create(checksum="2", correspondent=correspondent)
+        only_a.tags.add(tag_a)
+
+        for batch_size in (10_000, 1):
+            with (
+                self.subTest(batch_size=batch_size),
+                mock.patch.object(
+                    DocumentViewSet,
+                    "selection_data_batch_size",
+                    batch_size,
+                ),
+            ):
+                response = self.client.get(
+                    f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
+                    "&include_selection_data=true",
+                )
+
+                self.assertEqual(response.status_code, status.HTTP_200_OK)
+                counts = {
+                    item["id"]: item["document_count"]
+                    for item in response.data["selection_data"][
+                        "selected_correspondents"
+                    ]
+                }
+                self.assertEqual(counts[correspondent.id], 2)
+
     def test_statistics(self) -> None:
         doc1 = Document.objects.create(
             title="none1",
             checksum="A",
             mime_type="application/pdf",
             content="abc",
+            archive_serial_number=42,
         )
         Document.objects.create(
             title="none2",
@@ -1804,6 +1906,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.data["correspondent_count"], 2)
         self.assertEqual(response.data["document_type_count"], 1)
         self.assertEqual(response.data["storage_path_count"], 2)
+        self.assertEqual(response.data["current_asn"], 42)
 
     def test_statistics_excludes_document_versions(self) -> None:
         root = Document.objects.create(
@@ -1811,6 +1914,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             checksum="A",
             mime_type="application/pdf",
             content="root",
+            archive_serial_number=42,
         )
         version = Document.objects.create(
             title="version",
@@ -1819,6 +1923,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             content="version",
             root_document=root,
             version_index=1,
+            archive_serial_number=99,
         )
         tag_inbox = Tag.objects.create(name="t1", is_inbox_tag=True)
         version.tags.add(tag_inbox)
@@ -1829,6 +1934,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.data["documents_total"], 1)
         self.assertEqual(response.data["documents_inbox"], 0)
         self.assertEqual(response.data["character_count"], 4)
+        self.assertEqual(response.data["current_asn"], 42)
         self.assertEqual(
             response.data["document_file_type_counts"][0]["mime_type_count"],
             1,
@@ -1842,6 +1948,45 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.data["documents_inbox"], None)
         self.assertEqual(response.data["inbox_tags"], None)
 
+    def test_statistics_current_asn_respects_document_permissions(self) -> None:
+        viewer = UserFactory()
+        grant_global(viewer, "view_document")
+        DocumentFactory(owner=viewer, archive_serial_number=12)
+        shared = DocumentFactory(owner=self.user, archive_serial_number=23)
+        grant_object(viewer, shared, "view_document")
+        DocumentFactory(owner=self.user, archive_serial_number=99)
+
+        self.client.force_authenticate(user=viewer)
+        response = self.client.get("/api/statistics/")
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["current_asn"], 23)
+
+    def test_statistics_current_asn_without_visible_documents(self) -> None:
+        DocumentFactory(owner=self.user, archive_serial_number=99)
+
+        for permissions in ((), ("view_document",)):
+            viewer = UserFactory()
+            grant_global(viewer, *permissions)
+            self.client.force_authenticate(user=viewer)
+
+            with self.subTest(permissions=permissions):
+                response = self.client.get("/api/statistics/")
+
+                self.assertEqual(response.status_code, status.HTTP_200_OK)
+                self.assertEqual(response.data["current_asn"], 0)
+
+    def test_statistics_current_asn_with_global_statistics_permission(self) -> None:
+        viewer = UserFactory()
+        grant_global(viewer, "view_global_statistics")
+        DocumentFactory(owner=self.user, archive_serial_number=99)
+
+        self.client.force_authenticate(user=viewer)
+        response = self.client.get("/api/statistics/")
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["current_asn"], 99)
+
     def test_statistics_multiple_users(self) -> None:
         """
         GIVEN:

src/documents/tests/test_api_search.py

@@ -25,6 +25,7 @@ from documents.models import Tag
 from documents.models import Workflow
 from documents.search import get_backend
 from documents.search import reset_backend
+from documents.views import DocumentViewSet
 from paperless_mail.models import MailAccount
 from paperless_mail.models import MailRule
 from paperless_testing.dirs import DirectoriesMixin
@@ -345,38 +346,39 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         storage_path = StoragePath.objects.create(name="sp1")
         tag = Tag.objects.create(name="tag")
 
-        matching_doc = Document.objects.create(
-            title="bank statement",
-            content="bank content",
-            checksum="A",
-            correspondent=correspondent,
-            document_type=doc_type,
-            storage_path=storage_path,
-        )
-        matching_doc.tags.add(tag)
+        for checksum, title in (
+            ("A", "bank statement"),
+            ("B", "bank account"),
+            ("C", "credit card"),
+        ):
+            doc = Document.objects.create(
+                title=title,
+                checksum=checksum,
+                correspondent=correspondent,
+                document_type=doc_type,
+                storage_path=storage_path,
+            )
+            doc.tags.add(tag)
+            get_backend().add_or_update(doc)
 
-        get_backend().add_or_update(matching_doc)
-
-        response = self.client.get(
-            "/api/documents/?query=bank&include_selection_data=true",
-        )
+        with mock.patch.object(DocumentViewSet, "selection_data_batch_size", 1):
+            response = self.client.get(
+                "/api/documents/?query=bank&page_size=1&include_selection_data=true",
+            )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn("selection_data", response.data)
-
-        selected_correspondent = next(
-            item
-            for item in response.data["selection_data"]["selected_correspondents"]
-            if item["id"] == correspondent.id
-        )
-        selected_tag = next(
-            item
-            for item in response.data["selection_data"]["selected_tags"]
-            if item["id"] == tag.id
-        )
-
-        self.assertEqual(selected_correspondent["document_count"], 1)
-        self.assertEqual(selected_tag["document_count"], 1)
+        self.assertEqual(response.data["count"], 2)
+        self.assertEqual(len(response.data["results"]), 1)
+        for key, pk in (
+            ("selected_correspondents", correspondent.pk),
+            ("selected_document_types", doc_type.pk),
+            ("selected_storage_paths", storage_path.pk),
+            ("selected_tags", tag.pk),
+        ):
+            self.assertEqual(
+                response.data["selection_data"][key],
+                [{"id": pk, "document_count": 2}],
+            )
 
     def test_search_custom_field_ordering(self) -> None:
         custom_field = CustomField.objects.create(

src/documents/tests/test_tag_adjacency.py

@@ -0,0 +1,123 @@
+from typing import TYPE_CHECKING
+
+import pytest
+from django.core.exceptions import FieldDoesNotExist
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
+
+from documents.management.commands.document_importer import _deserialize_record
+from documents.models import Tag
+from documents.tag_hierarchy import TagHierarchy
+from paperless_testing.factories import TagFactory
+from paperless_testing.migrations import TestMigrations
+
+if TYPE_CHECKING:
+    from django.apps.registry import Apps
+    from pytest_django.fixtures import DjangoAssertNumQueries
+
+
+@pytest.mark.django_db
+class TestTagAdjacency:
+    def test_held_instance_uses_current_parent_edges(self) -> None:
+        """Descendant lookups reflect parent changes made after loading a tag."""
+        first = TagFactory(name="First")
+        second = TagFactory(name="Second")
+        child = TagFactory(name="Child", tn_parent=first)
+        Tag.objects.filter(pk=child.pk).update(tn_parent=second)
+        assert first.get_descendants_pks() == set()
+        assert second.get_descendants_pks() == {child.pk}
+
+    def test_queryset_delete_preserves_cascade(self) -> None:
+        """Deleting a parent through a queryset deletes only its subtree."""
+        root = TagFactory(name="Root")
+        child = TagFactory(name="Child", tn_parent=root)
+        TagFactory(name="Grandchild", tn_parent=child)
+        survivor = TagFactory(name="Other root")
+        Tag.objects.filter(pk=root.pk).delete()
+        assert list(Tag.objects.values_list("pk", flat=True)) == [survivor.pk]
+
+    def test_flat_writes_do_not_update_other_tags(self) -> None:
+        """Creating and renaming flat tags never rewrite unrelated tag rows."""
+        TagFactory.create_batch(100)
+        with CaptureQueriesContext(connection) as queries:
+            new = TagFactory(name="New")
+        assert not any(q["sql"].startswith("UPDATE") for q in queries)
+        with CaptureQueriesContext(connection) as queries:
+            new.name = "Renamed"
+            new.save()
+        tag_updates = [
+            q["sql"] for q in queries if q["sql"].startswith('UPDATE "documents_tag"')
+        ]
+        assert len(tag_updates) == 1
+        assert 'WHERE "documents_tag"."id" =' in tag_updates[0]
+
+    def test_batch_snapshot_is_one_query_and_includes_overlapping_roots(
+        self,
+        django_assert_num_queries: "DjangoAssertNumQueries",
+    ) -> None:
+        """One edge query supports multiple traversals with overlapping roots."""
+        root = TagFactory(name="Root")
+        child = TagFactory(name="Child", tn_parent=root)
+        leaf = TagFactory(name="Leaf", tn_parent=child)
+        with django_assert_num_queries(1):
+            hierarchy = Tag.get_hierarchy()
+            assert hierarchy.with_ancestors([leaf.pk]) == {root.pk, child.pk, leaf.pk}
+            assert hierarchy.with_descendants([root.pk, child.pk]) == {
+                root.pk,
+                child.pk,
+                leaf.pk,
+            }
+        assert hierarchy.height(None) == 0
+
+    def test_legacy_export_fields_are_ignored_but_parent_is_preserved(self) -> None:
+        """Legacy bookkeeping is ignored while parent links and field checks survive."""
+        model, tag, m2m = _deserialize_record(
+            {
+                "model": "documents.tag",
+                "pk": 2,
+                "fields": {
+                    "name": "Child",
+                    "tn_parent": 1,
+                    "tn_order": 3,
+                    "tn_siblings_pks": "4,5",
+                    "tn_descendants_pks": "6",
+                },
+            },
+        )
+        assert model is Tag
+        assert tag.tn_parent_id == 1
+        assert tag.name == "Child"
+        assert not m2m
+        with pytest.raises(FieldDoesNotExist):
+            _deserialize_record(
+                {"model": "documents.tag", "pk": 3, "fields": {"tn_typo": "bad"}},
+            )
+
+    def test_malformed_cycles_terminate(self) -> None:
+        """Traversal terminates even when stored parent edges contain a cycle."""
+        hierarchy = TagHierarchy([(1, 2), (2, 1)])
+        assert hierarchy.with_descendants([1]) == {1, 2}
+        assert set(hierarchy.ancestors(2)) == {1, 2}
+
+
+class TestTagAdjacencyMigration(TestMigrations):
+    migrate_from = "0027_documentbarcode"
+    migrate_to = "0028_replace_tag_tree_fields"
+
+    def setUpBeforeMigration(self, apps: "Apps") -> None:
+        """Seed parent links using the model from the pre-migration state."""
+        tag = apps.get_model("documents", "Tag")
+        root = tag.objects.create(name="Root")
+        child = tag.objects.create(name="Child", tn_parent=root)
+        self.edges = [(root.pk, None), (child.pk, root.pk)]
+
+    def test_parent_edges_survive(self) -> None:
+        """Removing derived tree columns preserves the existing parent links."""
+        tag = self.apps.get_model("documents", "Tag")
+        assert (
+            list(tag.objects.order_by("pk").values_list("pk", "tn_parent_id"))
+            == self.edges
+        )
+        assert {
+            field.name for field in tag._meta.fields if field.name.startswith("tn_")
+        } == {"tn_parent"}

src/documents/views.py

@@ -5,11 +5,13 @@ import platform
 import re
 import tempfile
 import zipfile
+from collections import Counter
 from collections import deque
 from collections.abc import Callable
 from datetime import datetime
 from datetime import timedelta
 from http import HTTPStatus
+from itertools import islice
 from pathlib import Path
 from time import mktime
 from time import sleep
@@ -168,7 +170,6 @@ from documents.permissions import PaperlessNotePermissions
 from documents.permissions import PaperlessObjectPermissions
 from documents.permissions import TrashPermissions
 from documents.permissions import ViewDocumentsPermissions
-from documents.permissions import annotate_document_count_by_ids
 from documents.permissions import annotate_document_count_for_related_queryset
 from documents.permissions import get_document_count_filter_for_user
 from documents.permissions import get_permission_grantees
@@ -632,7 +633,7 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
         queryset = queryset.order_by(*ordering)
 
         all_tags = list(queryset)
-        descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
+        descendant_pks = Tag.get_hierarchy().descendants(t.pk for t in all_tags)
 
         if descendant_pks:
             user = getattr(getattr(self, "request", None), "user", None)
@@ -1014,71 +1015,80 @@ class DocumentViewSet(
         "custom_field_",
     )
 
-    def _get_selection_data_for_queryset(self, queryset):
-        # Resolve once instead of once per model below. `queryset` can carry an
-        # arbitrarily expensive WHERE clause (user filters plus the permission
-        # filter); re-embedding it as a subquery inside 5 separate Count(...)
-        # calls forces the database to re-evaluate that whole thing 5 times, and
-        # -- for FK relations especially -- can defeat semi-join planning
-        # entirely at scale. A concrete id list is cheap to reuse.
-        # order_by() drops the default/user ordering -- irrelevant for a plain
-        # id list, but left in place it forces a sort over the full filtered
-        # set before the ids can even be collected.
-        document_ids = list(queryset.order_by().values_list("pk", flat=True))
+    # Bound the IN clauses used for search results and through-table lookups.
+    selection_data_batch_size = 10_000
 
-        correspondents = Correspondent.objects.annotate(
-            document_count=Count(
-                "documents",
-                filter=Q(documents__id__in=document_ids),
-                distinct=True,
-            ),
-        )
-        document_types = DocumentType.objects.annotate(
-            document_count=Count(
-                "documents",
-                filter=Q(documents__id__in=document_ids),
-                distinct=True,
-            ),
-        )
-        storage_paths = StoragePath.objects.annotate(
-            document_count=Count(
-                "documents",
-                filter=Q(documents__id__in=document_ids),
-                distinct=True,
-            ),
-        )
-        # Tag and CustomField reach Document through an M2M/through-model table;
-        # a plain Count(filter=...) there is a much more expensive plan than the
-        # FK relations above once the bridge table is large -- see
-        # annotate_document_count_by_ids() for why.
-        tags = annotate_document_count_by_ids(
-            Tag.objects.all(),
-            through_model=Document.tags.through,
-            related_object_field="tag_id",
-            document_ids=document_ids,
-        )
-        custom_fields = annotate_document_count_by_ids(
-            CustomField.objects.all(),
-            through_model=CustomFieldInstance,
-            related_object_field="field_id",
-            document_ids=document_ids,
+    def _get_selection_data_for_queryset(
+        self,
+        queryset: QuerySet[Document],
+        *,
+        search_ids: list[int] | None = None,
+    ) -> dict[str, list[dict[str, int]]]:
+        # Keep permission and filter predicates at document granularity.
+        # Reusing the filtered queryset as a subquery for each count is slow.
+        rows = (
+            queryset.order_by()
+            .prefetch_related(None)
+            .values_list(
+                "pk",
+                "correspondent_id",
+                "document_type_id",
+                "storage_path_id",
+            )
         )
+        batch_size = self.selection_data_batch_size
+
+        def document_rows():
+            if search_ids is None:
+                yield from rows.iterator()
+            else:
+                for start in range(0, len(search_ids), batch_size):
+                    yield from rows.filter(
+                        pk__in=search_ids[start : start + batch_size],
+                    ).iterator()
+
+        correspondents: Counter[int | None] = Counter()
+        document_types: Counter[int | None] = Counter()
+        storage_paths: Counter[int | None] = Counter()
+        document_ids: set[int] = set()
+        for pk, correspondent, document_type, storage_path in document_rows():
+            # Filters that join multi-valued relations can repeat a document.
+            if pk in document_ids:
+                continue
+            document_ids.add(pk)
+            correspondents[correspondent] += 1
+            document_types[document_type] += 1
+            storage_paths[storage_path] += 1
+
+        tags: Counter[int | None] = Counter()
+        custom_fields: Counter[int | None] = Counter()
+        ids = iter(document_ids)
+        while batch := list(islice(ids, batch_size)):
+            for model, field, counts in (
+                (Document.tags.through, "tag_id", tags),
+                (CustomFieldInstance, "field_id", custom_fields),
+            ):
+                counts.update(
+                    dict(
+                        model.objects.filter(document_id__in=batch)
+                        .order_by()
+                        .values_list(field)
+                        .annotate(document_count=Count("document_id")),
+                    ),
+                )
+
+        def selection(model: type[Model], counts: Counter[int | None]):
+            return [
+                {"id": pk, "document_count": counts[pk]}
+                for pk in model._default_manager.values_list("pk", flat=True)
+            ]
+
         return {
-            "selected_correspondents": [
-                {"id": t.id, "document_count": t.document_count} for t in correspondents
-            ],
-            "selected_tags": [
-                {"id": t.id, "document_count": t.document_count} for t in tags
-            ],
-            "selected_document_types": [
-                {"id": t.id, "document_count": t.document_count} for t in document_types
-            ],
-            "selected_storage_paths": [
-                {"id": t.id, "document_count": t.document_count} for t in storage_paths
-            ],
-            "selected_custom_fields": [
-                {"id": t.id, "document_count": t.document_count} for t in custom_fields
-            ],
+            "selected_correspondents": selection(Correspondent, correspondents),
+            "selected_tags": selection(Tag, tags),
+            "selected_document_types": selection(DocumentType, document_types),
+            "selected_storage_paths": selection(StoragePath, storage_paths),
+            "selected_custom_fields": selection(CustomField, custom_fields),
         }
 
     @classmethod
@@ -2725,13 +2735,10 @@ class UnifiedSearchViewSet(DocumentViewSet):
                 if get_boolean(
                     str(request.query_params.get("include_selection_data", "false")),
                 ):
-                    # NOTE: pk__in=ordered_ids generates a large SQL IN clause
-                    # for big result sets.  Acceptable today but may need a temp
-                    # table or chunked approach if selection_data becomes slow
-                    # at scale (tens of thousands of matching documents).
                     response.data["selection_data"] = (
                         self._get_selection_data_for_queryset(
-                            filtered_qs.filter(pk__in=result.ordered_ids),
+                            filtered_qs,
+                            search_ids=result.ordered_ids,
                         )
                     )
                 return response
@@ -4071,7 +4078,7 @@ class StatisticsView(GenericAPIView[Any]):
             for row in mime_type_stats
         ]
 
-        current_asn = Document.objects.aggregate(
+        current_asn = documents.aggregate(
             Max("archive_serial_number", default=0),
         ).get(
             "archive_serial_number__max",
@@ -5078,12 +5085,7 @@ class BulkEditObjectsView(PassUserMixin):
             if object_type == "tags":
                 editable_ids = set(user_permitted_objects.values_list("pk", flat=True))
                 all_ids = set(objs.values_list("pk", flat=True))
-                for tag in objs:
-                    all_ids.update(
-                        descendant.pk
-                        for descendant in tag.get_descendants()
-                        if descendant.pk in editable_ids
-                    )
+                all_ids.update(Tag.get_hierarchy().descendants(all_ids) & editable_ids)
                 objs = object_class.objects.filter(pk__in=all_ids)
             objs = objs.select_related("owner")
             object_ids = list(objs.values_list("pk", flat=True))

src/documents/workflows/mutations.py

@@ -6,6 +6,7 @@ from guardian.shortcuts import remove_perm
 from documents.data_models import DocumentMetadataOverrides
 from documents.models import CustomFieldInstance
 from documents.models import Document
+from documents.models import Tag
 from documents.models import WorkflowAction
 from documents.permissions import PermissionsSpec
 from documents.permissions import set_permissions_for_objects
@@ -134,10 +135,9 @@ def apply_assignment_to_overrides(
     if action.has_assign_tags:
         if overrides.tag_ids is None:
             overrides.tag_ids = []
-        tag_ids_to_add: set[int] = set()
-        for tag in action.assign_tags.all():
-            tag_ids_to_add.add(tag.pk)
-            tag_ids_to_add.update(int(pk) for pk in tag.get_ancestors_pks())
+        tag_ids_to_add = Tag.get_hierarchy().with_ancestors(
+            tag.pk for tag in action.assign_tags.all()
+        )
 
         overrides.tag_ids = list(set(overrides.tag_ids) | tag_ids_to_add)
 
@@ -216,10 +216,9 @@ def apply_removal_to_document(
     if action.remove_all_tags:
         Document.objects.get(pk=document.pk).tags.clear()
     else:
-        tag_ids_to_remove: set[int] = set()
-        for tag in action.remove_tags.all():
-            tag_ids_to_remove.add(tag.pk)
-            tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
+        tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
+            tag.pk for tag in action.remove_tags.all()
+        )
 
         if tag_ids_to_remove:
             Document.objects.get(pk=document.pk).tags.remove(*tag_ids_to_remove)
@@ -297,10 +296,9 @@ def apply_removal_to_overrides(
     if action.remove_all_tags:
         overrides.tag_ids = None
     elif overrides.tag_ids:
-        tag_ids_to_remove: set[int] = set()
-        for tag in action.remove_tags.all():
-            tag_ids_to_remove.add(tag.pk)
-            tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
+        tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
+            tag.pk for tag in action.remove_tags.all()
+        )
 
         overrides.tag_ids = [t for t in overrides.tag_ids if t not in tag_ids_to_remove]
 

src/paperless/settings/__init__.py

@@ -157,7 +157,6 @@ INSTALLED_APPS = [
     "allauth.headless",
     "drf_spectacular",
     "drf_spectacular_sidecar",
-    "treenode",
     *env_apps,
 ]
 

src/paperless/views.py

@@ -32,13 +32,13 @@ from rest_framework.fields import BooleanField
 from rest_framework.filters import OrderingFilter
 from rest_framework.generics import GenericAPIView
 from rest_framework.pagination import PageNumberPagination
-from rest_framework.permissions import DjangoModelPermissions
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 from rest_framework.throttling import ScopedRateThrottle
 from rest_framework.viewsets import ModelViewSet
 
 from documents.models import PaperlessTask
+from documents.permissions import PaperlessModelPermissions
 from documents.permissions import PaperlessObjectPermissions
 from documents.tasks import llmindex_index
 from paperless.filters import GroupFilterSet
@@ -414,7 +414,7 @@ class ApplicationConfigurationViewSet(ModelViewSet[ApplicationConfiguration]):
     queryset = ApplicationConfiguration.objects
 
     serializer_class = ApplicationConfigurationSerializer
-    permission_classes = (IsAuthenticated, DjangoModelPermissions)
+    permission_classes = (IsAuthenticated, PaperlessModelPermissions)
 
     @extend_schema(exclude=True)
     def create(self, request, *args, **kwargs):

src/paperless_ai/tests/test_vector_store.py

@@ -20,8 +20,10 @@ from paperless_ai.tables import DocumentChunksTable
 from paperless_ai.tables import DocumentMetaTable
 from paperless_ai.vector_store import _MAX_IN_VALUES
 from paperless_ai.vector_store import DB_FILENAME
+from paperless_ai.vector_store import DB_PAGE_SIZE
 from paperless_ai.vector_store import DEFAULT_TABLE_NAME
 from paperless_ai.vector_store import SCHEMA_VERSION
+from paperless_ai.vector_store import VEC0_CHUNK_SIZE
 from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
 from paperless_ai.vector_store import _build_where
 from paperless_ai.vector_store import _pack
@@ -1128,3 +1130,127 @@ class TestV1ToV2Migration:
         assert "DocumentChunksTable.create" not in source
         assert "DocumentMetaTable.create" not in source
         assert "_create_vec_table(" not in source
+
+
+LEGACY_PAGE_SIZE = 4096
+
+
+def vec_table_sql(db_path: Path) -> str:
+    conn = sqlite3.connect(db_path)
+    try:
+        return conn.execute(
+            "SELECT sql FROM sqlite_master WHERE name = ?",
+            (DEFAULT_TABLE_NAME,),
+        ).fetchone()[0]
+    finally:
+        conn.close()
+
+
+def page_size_of(db_path: Path) -> int:
+    conn = sqlite3.connect(db_path)
+    try:
+        return conn.execute("PRAGMA page_size").fetchone()[0]
+    finally:
+        conn.close()
+
+
+def create_legacy_vec_table(conn: sqlite3.Connection, dim: int) -> None:
+    """The table definition from before chunk_size was set explicitly."""
+    conn.execute(
+        f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
+        "id TEXT PRIMARY KEY, document_id INTEGER, +node_content TEXT, "
+        f"embedding float[{dim}] distance_metric=cosine)",
+    )
+
+
+@pytest.fixture
+def legacy_page_size_db(tmp_path: Path) -> Path:
+    """An existing index file created before the page size was set."""
+    db_path = tmp_path / DB_FILENAME
+    conn = sqlite3.connect(db_path)
+    conn.execute(f"PRAGMA page_size={LEGACY_PAGE_SIZE}")
+    conn.execute("CREATE TABLE _seed(x)")
+    conn.execute("DROP TABLE _seed")
+    conn.close()
+    assert page_size_of(db_path) == LEGACY_PAGE_SIZE
+    return db_path
+
+
+class TestVecChunkSize:
+    def test_new_table_uses_configured_chunk_size(
+        self,
+        store: PaperlessSqliteVecVectorStore,
+        tmp_path: Path,
+    ) -> None:
+        """
+        GIVEN an empty store
+        WHEN the first document is indexed and the vec0 table is created
+        THEN the table definition carries the configured chunk size
+        """
+        store.upsert_document(1, [make_node("1-a", 1)])
+
+        assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
+
+    def test_compact_moves_legacy_table_to_configured_chunk_size(
+        self,
+        store: PaperlessSqliteVecVectorStore,
+        tmp_path: Path,
+        mocker: MockerFixture,
+    ) -> None:
+        """
+        GIVEN an index whose vec0 table was created with the default chunk size
+        WHEN the index is compacted
+        THEN the rebuilt table uses the configured chunk size
+        AND no indexed chunks are lost
+        """
+        legacy = mocker.patch.object(
+            PaperlessSqliteVecVectorStore,
+            "_create_vec_table",
+            staticmethod(create_legacy_vec_table),
+        )
+        store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
+        store.upsert_document(2, [make_node("2-a", 2)])
+        mocker.stop(legacy)
+        assert "chunk_size" not in vec_table_sql(tmp_path / DB_FILENAME)
+
+        store.compact(force=True)
+
+        assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
+        assert DocumentChunksTable.count(store.client) == 3
+
+
+class TestIndexPageSize:
+    def test_new_index_file_uses_configured_page_size(
+        self,
+        store: PaperlessSqliteVecVectorStore,
+        tmp_path: Path,
+    ) -> None:
+        """
+        GIVEN no existing index file
+        WHEN a store is opened and a document is indexed
+        THEN the database file uses the configured page size
+        """
+        store.upsert_document(1, [make_node("1-a", 1)])
+
+        assert page_size_of(tmp_path / DB_FILENAME) == DB_PAGE_SIZE
+
+    def test_compact_moves_legacy_file_to_configured_page_size(
+        self,
+        legacy_page_size_db: Path,
+    ) -> None:
+        """
+        GIVEN an existing index file created with a different page size
+        WHEN the index is compacted
+        THEN the rebuilt file uses the configured page size
+        AND no indexed chunks are lost
+        """
+        with PaperlessSqliteVecVectorStore(
+            uri=str(legacy_page_size_db.parent),
+        ) as store:
+            store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
+            assert page_size_of(legacy_page_size_db) == LEGACY_PAGE_SIZE
+
+            store.compact(force=True)
+
+            assert page_size_of(legacy_page_size_db) == DB_PAGE_SIZE
+            assert DocumentChunksTable.count(store.client) == 2

src/paperless_ai/vector_store.py

@@ -47,6 +47,20 @@ SCHEMA_VERSION = 2
 # a rebuild copies the live rows into a fresh table.
 COMPACT_BLOAT_RATIO = 2.0
 
+# Rows per vec0 vector chunk (vec0's default is 1024, which makes each chunk a
+# multi-megabyte blob). On spinning disks, inserts and re-indexing were dozens
+# of times faster with small chunks, at no cost in query speed or file size.
+# Only applies to tables created from here on; an existing index picks it up
+# the next time it is rebuilt (compact or rebuild).
+VEC0_CHUNK_SIZE = 32
+
+# SQLite page size for the index file, in bytes. With the small vec0 chunks above,
+# 64 KiB pages made cold scans about 35% faster and warm ones about 45% faster,
+# at the cost of a larger write-ahead log (up to about 70 MB). Only takes effect
+# on a newly created file, so like the chunk size it reaches an existing index
+# when it is rebuilt.
+DB_PAGE_SIZE = 65536
+
 # Number of rows fetched/copied per batch whenever this module streams rows
 # instead of materializing them all at once, keeping memory bounded regardless
 # of index size -- used by compact()'s rebuild, m0001_v1_to_v2's migration
@@ -222,6 +236,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
         conn.enable_load_extension(True)  # noqa: FBT003
         sqlite_vec.load(conn)
         conn.enable_load_extension(False)  # noqa: FBT003
+        conn.execute(f"PRAGMA page_size={DB_PAGE_SIZE}")
         conn.execute("PRAGMA journal_mode=WAL")
         conn.execute("PRAGMA synchronous=NORMAL")
         IndexMetaTable.create(conn)
@@ -329,7 +344,8 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
             + " +node_content TEXT,"
             + " embedding float["
             + str(int(dim))
-            + "] distance_metric=cosine"
+            + "] distance_metric=cosine, chunk_size="
+            + str(VEC0_CHUNK_SIZE)
             + ")",
         )
 

uv.lock

@@ -1210,15 +1210,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/99/c3/305a9f7611c0dc96d2a638bc3a67bdcde5a7faa5f4669188db018b8af6fc/django_stubs_ext-6.1.1-py3-none-any.whl", hash = "sha256:64aebb7ce2c327a4a2b75207d67034e879fd7edf027c93487be69da363dc79fa", size = 10853, upload-time = "2026-09-14T08:45:51.289Z" },
 ]
 
-[[package]]
-name = "django-treenode"
-version = "0.25.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/cb/2b/09f04b44b3583a911ae4f5d0e3360a23d529704c9e9115fce2520b7d4420/django_treenode-0.25.0.tar.gz", hash = "sha256:d14d0eb3571c227ce11147210d62a8f87b568bbb30fa76a5be9b9e6d514157c3", size = 29951, upload-time = "2026-08-26T00:20:13.489Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e1/ac/2140e8c20e2a47457ec23302273527d3430bf5c09edb8940d74c0725c138/django_treenode-0.25.0-py3-none-any.whl", hash = "sha256:f47caab5440420870b6287602b3ee401ca354416d44b24ab97579be5ea658927", size = 22343, upload-time = "2026-08-26T00:20:12.078Z" },
-]
-
 [[package]]
 name = "djangorestframework"
 version = "3.18.3"
@@ -3107,7 +3098,6 @@ dependencies = [
     { name = "django-multiselectfield" },
     { name = "django-rich" },
     { name = "django-soft-delete" },
-    { name = "django-treenode" },
     { name = "djangorestframework" },
     { name = "drf-spectacular" },
     { name = "drf-spectacular-sidecar" },
@@ -3258,7 +3248,6 @@ requires-dist = [
     { name = "django-multiselectfield", specifier = "~=1.0.1" },
     { name = "django-rich", specifier = "~=2.2.0" },
     { name = "django-soft-delete", specifier = "~=1.0.18" },
-    { name = "django-treenode", specifier = ">=0.24" },
     { name = "djangorestframework", specifier = "~=3.16" },
     { name = "drf-spectacular", specifier = "~=0.30" },
     { name = "drf-spectacular-sidecar", specifier = ">=2026.10.1,<2026.11" },