Compare commits

...
1 Commits
Author SHA1 Message Date
shamoonandCodex 9c3e543415 Performance: replace django-treenode with adjacency-list tags
Co-Authored-By: Codex <267193182+codex@users.noreply.github.com>
2026-10-28 15:59:41 -07:00
14 changed files with 348 additions and 121 deletions

No files matched your search

@@ -39,7 +39,6 @@ declare -a scope=(
"${site_packages}/allauth"
"${site_packages}/drf_spectacular"
"${site_packages}/drf_spectacular_sidecar"
"${site_packages}/treenode"
"${site_packages}/compression_middleware"
)
-1
View File
@@ -38,7 +38,6 @@ dependencies = [
"django-multiselectfield~=1.0.1",
"django-rich~=2.2.0",
"django-soft-delete~=1.0.18",
"django-treenode>=0.24",
"djangorestframework~=3.16",
"drf-spectacular~=0.30",
"drf-spectacular-sidecar>=2026.7.1,<2026.10",
+1 -2
View File
@@ -1,7 +1,6 @@
from django.conf import settings
from django.contrib import admin
from guardian.admin import GuardedModelAdmin
from treenode.admin import TreeNodeModelAdmin
from documents.models import Correspondent
from documents.models import CustomField
@@ -29,7 +28,7 @@ class CorrespondentAdmin(GuardedModelAdmin):
list_editable = ("match", "matching_algorithm")
class TagAdmin(GuardedModelAdmin, TreeNodeModelAdmin):
class TagAdmin(GuardedModelAdmin):
list_display = ("name", "color", "match", "matching_algorithm")
list_filter = ("matching_algorithm",)
list_editable = ("color", "match", "matching_algorithm")
+7 -13
View File
@@ -255,19 +255,13 @@ def modify_tags(
affected_docs = list(qs.values_list("pk", flat=True))
DocumentTagRelationship = Document.tags.through
# add with all ancestors
expanded_add_tags: set[int] = set()
add_tag_objects = Tag.objects.filter(pk__in=add_tags)
for t in add_tag_objects:
expanded_add_tags.add(int(t.id))
expanded_add_tags.update(int(pk) for pk in t.get_ancestors_pks())
# remove with all descendants
expanded_remove_tags: set[int] = set()
remove_tag_objects = Tag.objects.filter(pk__in=remove_tags)
for t in remove_tag_objects:
expanded_remove_tags.add(int(t.id))
expanded_remove_tags.update(int(pk) for pk in t.get_descendants_pks())
hierarchy = Tag.get_hierarchy()
expanded_add_tags = hierarchy.with_ancestors(
Tag.objects.filter(pk__in=add_tags).values_list("pk", flat=True),
)
expanded_remove_tags = hierarchy.with_descendants(
Tag.objects.filter(pk__in=remove_tags).values_list("pk", flat=True),
)
with transaction.atomic():
if expanded_remove_tags:
@@ -108,6 +108,24 @@ def _deserialize_record(
) from e
for field_name, field_value in record.get("fields", {}).items():
# Older exports include django-treenode's derived bookkeeping. The
# parent FK is authoritative; accept only these known removed fields.
if Model is Tag and field_name in {
"tn_ancestors_count",
"tn_ancestors_pks",
"tn_children_count",
"tn_children_pks",
"tn_depth",
"tn_descendants_count",
"tn_descendants_pks",
"tn_index",
"tn_level",
"tn_order",
"tn_priority",
"tn_siblings_count",
"tn_siblings_pks",
}:
continue
field = Model._meta.get_field(field_name)
remote = field.remote_field
@@ -0,0 +1,64 @@
# Generated by Django 5.2.17 on 2026-10-28 22:54
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("documents", "0027_documentbarcode"),
]
operations = [
migrations.RemoveField(
model_name="tag",
name="tn_ancestors_count",
),
migrations.RemoveField(
model_name="tag",
name="tn_ancestors_pks",
),
migrations.RemoveField(
model_name="tag",
name="tn_children_count",
),
migrations.RemoveField(
model_name="tag",
name="tn_children_pks",
),
migrations.RemoveField(
model_name="tag",
name="tn_depth",
),
migrations.RemoveField(
model_name="tag",
name="tn_descendants_count",
),
migrations.RemoveField(
model_name="tag",
name="tn_descendants_pks",
),
migrations.RemoveField(
model_name="tag",
name="tn_index",
),
migrations.RemoveField(
model_name="tag",
name="tn_level",
),
migrations.RemoveField(
model_name="tag",
name="tn_order",
),
migrations.RemoveField(
model_name="tag",
name="tn_priority",
),
migrations.RemoveField(
model_name="tag",
name="tn_siblings_count",
),
migrations.RemoveField(
model_name="tag",
name="tn_siblings_pks",
),
]
+53 -23
View File
@@ -14,7 +14,6 @@ from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from multiselectfield import MultiSelectField
from treenode.models import TreeNodeModel
if settings.AUDIT_LOG_ENABLED:
from auditlog.registry import auditlog
@@ -100,7 +99,17 @@ class Correspondent(MatchingModel):
verbose_name_plural = _("correspondents")
class Tag(MatchingModel, TreeNodeModel):
class Tag(MatchingModel):
# Keep the existing column and reverse relation during the transition.
tn_parent = models.ForeignKey(
"self",
blank=True,
null=True,
on_delete=models.CASCADE,
related_name="tn_children",
verbose_name=_("Parent"),
)
color = models.CharField(_("color"), max_length=7, default="#a6cee3")
# Maximum allowed nesting depth for tags (root = 1, max depth = 5)
MAX_NESTING_DEPTH: Final[int] = 5
@@ -114,27 +123,51 @@ class Tag(MatchingModel, TreeNodeModel):
),
)
class Meta(MatchingModel.Meta, TreeNodeModel.Meta):
class Meta(MatchingModel.Meta):
verbose_name = _("tag")
verbose_name_plural = _("tags")
@classmethod
def get_hierarchy(cls):
from documents.tag_hierarchy import TagHierarchy
return TagHierarchy(cls.objects.values_list("pk", "tn_parent_id"))
@property
def parent_pk(self):
return self.tn_parent_id
def get_parent(self):
return self.tn_parent
def get_ancestors_pks(self):
return self.get_hierarchy().ancestors(self.tn_parent_id)
def get_ancestors(self):
ids = self.get_ancestors_pks()
tags = self.__class__.objects.in_bulk(ids)
return [tags[pk] for pk in ids]
def get_descendants_pks(self):
return self.get_hierarchy().descendants([self.pk])
def get_descendants(self):
return self.__class__.objects.filter(pk__in=self.get_descendants_pks())
def get_children_queryset(self):
return self.__class__.objects.filter(tn_parent_id=self.pk)
def clean(self) -> None:
# Prevent self-parenting and assigning a descendant as parent
parent = self.get_parent()
if parent == self:
raise ValidationError({"parent": _("Cannot set itself as parent.")})
if parent and self.pk is not None and self.is_ancestor_of(parent):
raise ValidationError({"parent": _("Cannot set parent to a descendant.")})
# Enforce maximum nesting depth
new_parent_depth = 0
if parent:
new_parent_depth = parent.get_ancestors_count() + 1
height = 0 if self.pk is None else self.get_depth()
deepest_new_depth = (new_parent_depth + 1) + height
if deepest_new_depth > self.MAX_NESTING_DEPTH:
raise ValidationError({"parent": _("Maximum nesting depth exceeded.")})
hierarchy = self.get_hierarchy()
if self.pk is not None and self.tn_parent_id == self.pk:
raise ValidationError({"tn_parent": _("Cannot set itself as parent.")})
ancestors = hierarchy.ancestors(self.tn_parent_id)
if self.pk is not None and self.pk in ancestors:
raise ValidationError(
{"tn_parent": _("Cannot set parent to a descendant.")},
)
if len(ancestors) + 1 + hierarchy.height(self.pk) > self.MAX_NESTING_DEPTH:
raise ValidationError({"tn_parent": _("Maximum nesting depth exceeded.")})
return super().clean()
@@ -539,10 +572,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
return self.created
def add_nested_tags(self, tags) -> None:
tag_ids = set()
for tag in tags:
tag_ids.add(tag.id)
tag_ids.update(tag.get_ancestors_pks())
tag_ids = Tag.get_hierarchy().with_ancestors(tag.pk for tag in tags)
tags_to_add = self.tags.model.objects.filter(id__in=tag_ids)
self.tags.add(*tags_to_add)
+28 -50
View File
@@ -589,7 +589,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
text_color = serializers.SerializerMethodField()
# map to treenode's tn_parent
# Preserve the public API parent field.
parent = serializers.PrimaryKeyRelatedField(
queryset=Tag.objects.all(),
allow_null=True,
@@ -692,7 +692,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
self.instance.clean()
except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e)
raise e
raise serializers.ValidationError(
{"parent": e.message_dict["tn_parent"]},
) from e
finally:
self.instance.tn_parent = original_parent
else:
@@ -702,7 +704,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
temp.clean()
except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e)
raise e
raise serializers.ValidationError(
{"parent": e.message_dict["tn_parent"]},
) from e
return super().validate(attrs)
@@ -1198,61 +1202,35 @@ class DocumentSerializer(
doc_id,
)
if "tags" in validated_data:
# Respect tag hierarchy on updates:
# - Adding a child adds its ancestors
# - Removing a parent removes all its descendants
prev_tags = set(instance.tags.all())
requested_tags = set(validated_data["tags"])
hierarchy = Tag.get_hierarchy()
previous_ids = set(instance.tags.values_list("pk", flat=True))
requested_ids = {tag.pk for tag in validated_data["tags"]}
required_ids = hierarchy.with_ancestors(requested_ids - previous_ids)
blocked_ids = hierarchy.with_descendants(previous_ids - requested_ids)
blocked_ids.difference_update(required_ids)
final_ids = hierarchy.with_ancestors(requested_ids) - blocked_ids
validated_data["tags"] = list(Tag.objects.filter(pk__in=final_ids))
# Tags newly added in this update and the ancestors they require
added_tags = requested_tags - prev_tags
required_by_add_tags = set(added_tags)
for t in added_tags:
required_by_add_tags.update(t.get_ancestors())
# Tags being removed in this update and all descendants, except
# those required by a tag that is being added in this same update
removed_tags = prev_tags - requested_tags
blocked_tags = set(removed_tags)
for t in removed_tags:
blocked_tags.update(t.get_descendants())
blocked_tags.difference_update(required_by_add_tags)
# Add all parent tags
final_tags = set(requested_tags)
for t in requested_tags:
final_tags.update(t.get_ancestors())
# Drop removed parents and their descendants
final_tags.difference_update(blocked_tags)
validated_data["tags"] = list(final_tags)
if validated_data.get("remove_inbox_tags"):
current_tag_ids = {t.pk for t in instance.tags.all()}
hierarchy = Tag.get_hierarchy()
tags = (
validated_data["tags"]
if "tags" in validated_data
else list(instance.tags.all())
)
# Tags newly added in this update, plus their ancestors, are kept
keep_ids: set[int] = set()
for tag in tags:
if tag.pk not in current_tag_ids:
keep_ids.add(tag.pk)
keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
# Remove inbox tags and their descendants, except those being kept
remove_ids: set[int] = set()
for inbox_tag in (
current_ids = set(instance.tags.values_list("pk", flat=True))
keep_ids = hierarchy.with_ancestors(
tag.pk for tag in tags if tag.pk not in current_ids
)
inbox_ids = (
Tag.objects.filter(is_inbox_tag=True)
.exclude(pk__in=keep_ids)
.only("pk", "tn_descendants_pks")
):
remove_ids.add(inbox_tag.pk)
remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
.exclude(
pk__in=keep_ids,
)
.values_list("pk", flat=True)
)
remove_ids = hierarchy.with_descendants(inbox_ids) - keep_ids
validated_data["tags"] = [tag for tag in tags if tag.pk not in remove_ids]
if settings.AUDIT_LOG_ENABLED:
with set_actor(self.user):
+61
View File
@@ -0,0 +1,61 @@
"""An operation-local snapshot of tag edges, with no persistent derived fields.
Never cache this globally: tasks, bulk updates and other processes can change
parents. Callers traversing multiple tags should share one snapshot.
"""
from collections import defaultdict
from collections.abc import Iterable
class TagHierarchy:
def __init__(self, edges: Iterable[tuple[int, int | None]]) -> None:
self.parents = dict(edges)
self.children: dict[int | None, list[int]] = defaultdict(list)
for pk, parent in self.parents.items():
self.children[parent].append(pk)
def ancestors(self, parent_id: int | None) -> list[int]:
result = []
seen = set()
while parent_id is not None and parent_id not in seen:
seen.add(parent_id)
result.append(parent_id)
parent_id = self.parents.get(parent_id)
return result[::-1]
def with_ancestors(self, roots: Iterable[int]) -> set[int]:
result = set(roots)
for pk in list(result):
result.update(self.ancestors(self.parents.get(pk)))
return result
def with_descendants(self, roots: Iterable[int]) -> set[int]:
roots = set(roots)
return roots | self.descendants(roots)
def descendants(self, roots: Iterable[int]) -> set[int]:
roots = {pk for pk in roots if pk is not None}
seen = set(roots)
pending = list(roots)
while pending:
for child in self.children.get(pending.pop(), ()):
if child not in seen:
seen.add(child)
pending.append(child)
return seen - roots
def height(self, pk: int | None) -> int:
if pk is None:
return 0
seen = {pk}
pending = [(pk, 0)]
height = 0
while pending:
parent, depth = pending.pop()
height = max(height, depth)
for child in self.children.get(parent, ()):
if child not in seen:
seen.add(child)
pending.append((child, depth + 1))
return height
+104
View File
@@ -0,0 +1,104 @@
from django.core.exceptions import FieldDoesNotExist
from django.db import connection
from django.test import TestCase
from django.test.utils import CaptureQueriesContext
from documents.management.commands.document_importer import _deserialize_record
from documents.models import Tag
from documents.tag_hierarchy import TagHierarchy
from paperless_testing.migrations import TestMigrations
class TestTagAdjacency(TestCase):
def test_held_instance_uses_current_parent_edges(self):
first = Tag.objects.create(name="First")
second = Tag.objects.create(name="Second")
child = Tag.objects.create(name="Child", tn_parent=first)
Tag.objects.filter(pk=child.pk).update(tn_parent=second)
assert first.get_descendants_pks() == set()
assert second.get_descendants_pks() == {child.pk}
def test_queryset_delete_preserves_cascade(self):
root = Tag.objects.create(name="Root")
child = Tag.objects.create(name="Child", tn_parent=root)
Tag.objects.create(name="Grandchild", tn_parent=child)
survivor = Tag.objects.create(name="Other root")
Tag.objects.filter(pk=root.pk).delete()
assert list(Tag.objects.values_list("pk", flat=True)) == [survivor.pk]
def test_flat_writes_do_not_update_other_tags(self):
Tag.objects.bulk_create([Tag(name=f"Tag {i}") for i in range(100)])
with CaptureQueriesContext(connection) as queries:
new = Tag.objects.create(name="New")
assert not any(q["sql"].startswith("UPDATE") for q in queries)
with CaptureQueriesContext(connection) as queries:
new.name = "Renamed"
new.save()
tag_updates = [
q["sql"] for q in queries if q["sql"].startswith('UPDATE "documents_tag"')
]
assert len(tag_updates) == 1
assert 'WHERE "documents_tag"."id" =' in tag_updates[0]
def test_batch_snapshot_is_one_query_and_includes_overlapping_roots(self):
root = Tag.objects.create(name="Root")
child = Tag.objects.create(name="Child", tn_parent=root)
leaf = Tag.objects.create(name="Leaf", tn_parent=child)
with self.assertNumQueries(1):
hierarchy = Tag.get_hierarchy()
assert hierarchy.with_ancestors([leaf.pk]) == {root.pk, child.pk, leaf.pk}
assert hierarchy.with_descendants([root.pk, child.pk]) == {
root.pk,
child.pk,
leaf.pk,
}
assert hierarchy.height(None) == 0
def test_legacy_export_fields_are_ignored_but_parent_is_preserved(self):
model, tag, m2m = _deserialize_record(
{
"model": "documents.tag",
"pk": 2,
"fields": {
"name": "Child",
"tn_parent": 1,
"tn_order": 3,
"tn_siblings_pks": "4,5",
"tn_descendants_pks": "6",
},
},
)
assert model is Tag
assert tag.tn_parent_id == 1
assert tag.name == "Child"
assert not m2m
with self.assertRaises(FieldDoesNotExist):
_deserialize_record(
{"model": "documents.tag", "pk": 3, "fields": {"tn_typo": "bad"}},
)
def test_malformed_cycles_terminate(self):
hierarchy = TagHierarchy([(1, 2), (2, 1)])
assert hierarchy.with_descendants([1]) == {1, 2}
assert set(hierarchy.ancestors(2)) == {1, 2}
class TestTagAdjacencyMigration(TestMigrations):
migrate_from = "0027_documentbarcode"
migrate_to = "0028_replace_tag_tree_fields"
def setUpBeforeMigration(self, apps):
tag = apps.get_model("documents", "Tag")
root = tag.objects.create(name="Root")
child = tag.objects.create(name="Child", tn_parent=root)
self.edges = [(root.pk, None), (child.pk, root.pk)]
def test_parent_edges_survive(self):
tag = self.apps.get_model("documents", "Tag")
assert (
list(tag.objects.order_by("pk").values_list("pk", "tn_parent_id"))
== self.edges
)
assert {
field.name for field in tag._meta.fields if field.name.startswith("tn_")
} == {"tn_parent"}
+2 -7
View File
@@ -627,7 +627,7 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
queryset = queryset.order_by(*ordering)
all_tags = list(queryset)
descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
descendant_pks = Tag.get_hierarchy().descendants(t.pk for t in all_tags)
if descendant_pks:
user = getattr(getattr(self, "request", None), "user", None)
@@ -5060,12 +5060,7 @@ class BulkEditObjectsView(PassUserMixin):
if object_type == "tags":
editable_ids = set(user_permitted_objects.values_list("pk", flat=True))
all_ids = set(objs.values_list("pk", flat=True))
for tag in objs:
all_ids.update(
descendant.pk
for descendant in tag.get_descendants()
if descendant.pk in editable_ids
)
all_ids.update(Tag.get_hierarchy().descendants(all_ids) & editable_ids)
objs = object_class.objects.filter(pk__in=all_ids)
objs = objs.select_related("owner")
object_ids = list(objs.values_list("pk", flat=True))
+10 -12
View File
@@ -6,6 +6,7 @@ from guardian.shortcuts import remove_perm
from documents.data_models import DocumentMetadataOverrides
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Tag
from documents.models import WorkflowAction
from documents.permissions import PermissionsSpec
from documents.permissions import set_permissions_for_objects
@@ -131,10 +132,9 @@ def apply_assignment_to_overrides(
if action.has_assign_tags:
if overrides.tag_ids is None:
overrides.tag_ids = []
tag_ids_to_add: set[int] = set()
for tag in action.assign_tags.all():
tag_ids_to_add.add(tag.pk)
tag_ids_to_add.update(int(pk) for pk in tag.get_ancestors_pks())
tag_ids_to_add = Tag.get_hierarchy().with_ancestors(
tag.pk for tag in action.assign_tags.all()
)
overrides.tag_ids = list(set(overrides.tag_ids) | tag_ids_to_add)
@@ -213,10 +213,9 @@ def apply_removal_to_document(
if action.remove_all_tags:
Document.objects.get(pk=document.pk).tags.clear()
else:
tag_ids_to_remove: set[int] = set()
for tag in action.remove_tags.all():
tag_ids_to_remove.add(tag.pk)
tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
tag.pk for tag in action.remove_tags.all()
)
if tag_ids_to_remove:
Document.objects.get(pk=document.pk).tags.remove(*tag_ids_to_remove)
@@ -294,10 +293,9 @@ def apply_removal_to_overrides(
if action.remove_all_tags:
overrides.tag_ids = None
elif overrides.tag_ids:
tag_ids_to_remove: set[int] = set()
for tag in action.remove_tags.all():
tag_ids_to_remove.add(tag.pk)
tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
tag.pk for tag in action.remove_tags.all()
)
overrides.tag_ids = [t for t in overrides.tag_ids if t not in tag_ids_to_remove]
-1
View File
@@ -157,7 +157,6 @@ INSTALLED_APPS = [
"allauth.headless",
"drf_spectacular",
"drf_spectacular_sidecar",
"treenode",
*env_apps,
]
Generated
-11
View File
@@ -1210,15 +1210,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/99/c3/305a9f7611c0dc96d2a638bc3a67bdcde5a7faa5f4669188db018b8af6fc/django_stubs_ext-6.1.1-py3-none-any.whl", hash = "sha256:64aebb7ce2c327a4a2b75207d67034e879fd7edf027c93487be69da363dc79fa", size = 10853, upload-time = "2026-09-14T08:45:51.289Z" },
]
[[package]]
name = "django-treenode"
version = "0.25.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/cb/2b/09f04b44b3583a911ae4f5d0e3360a23d529704c9e9115fce2520b7d4420/django_treenode-0.25.0.tar.gz", hash = "sha256:d14d0eb3571c227ce11147210d62a8f87b568bbb30fa76a5be9b9e6d514157c3", size = 29951, upload-time = "2026-08-26T00:20:13.489Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e1/ac/2140e8c20e2a47457ec23302273527d3430bf5c09edb8940d74c0725c138/django_treenode-0.25.0-py3-none-any.whl", hash = "sha256:f47caab5440420870b6287602b3ee401ca354416d44b24ab97579be5ea658927", size = 22343, upload-time = "2026-08-26T00:20:12.078Z" },
]
[[package]]
name = "djangorestframework"
version = "3.18.3"
@@ -3107,7 +3098,6 @@ dependencies = [
{ name = "django-multiselectfield" },
{ name = "django-rich" },
{ name = "django-soft-delete" },
{ name = "django-treenode" },
{ name = "djangorestframework" },
{ name = "drf-spectacular" },
{ name = "drf-spectacular-sidecar" },
@@ -3258,7 +3248,6 @@ requires-dist = [
{ name = "django-multiselectfield", specifier = "~=1.0.1" },
{ name = "django-rich", specifier = "~=2.2.0" },
{ name = "django-soft-delete", specifier = "~=1.0.18" },
{ name = "django-treenode", specifier = ">=0.24" },
{ name = "djangorestframework", specifier = "~=3.16" },
{ name = "drf-spectacular", specifier = "~=0.30" },
{ name = "drf-spectacular-sidecar", specifier = ">=2026.7.1,<2026.10" },