Performance: load a cached Hugging Face embedding model without contacting the Hub

Every chat question and AI suggestion request builds the embedding model, and sentence-transformers
checks the Hub for a newer revision each time. With the model already cached that check cost about
2 seconds per request (0.2 seconds without it), and it fails outright when the Hub is unreachable.

Load from the local cache first and only download when the model is missing. A full rebuild still
checks the Hub, so a new model release is picked up together with re-embedding every document
instead of silently mixing revisions.

docs/administration.md

@@ -565,6 +565,8 @@ Specify `rebuild` to build the index from scratch from all documents in the data
 this the first time you enable the feature, or after changing the embedding backend or
 model.
 
+When using the `huggingface` embedding backend, a model that is already downloaded is loaded from the local cache without contacting the Hugging Face Hub. Only `rebuild` checks the Hub for a newer revision of the model, so a model update is picked up together with re-embedding every document. If the model is not cached yet, it is downloaded on first use.
+
 Specify `update` to incrementally index new and changed documents. This is what the
 scheduled task runs.
 

docs/configuration.md

@@ -501,20 +501,20 @@ See the [Django project documentation on the settings](https://docs.djangoprojec
 
 #### [`PAPERLESS_ALLOWED_HOSTS=<comma-separated-list>`](#PAPERLESS_ALLOWED_HOSTS) {#PAPERLESS_ALLOWED_HOSTS}
 
-: If you expose Paperless to the internet, set `PAPERLESS_URL` to its public URL
-or list its public hostname in `PAPERLESS_ALLOWED_HOSTS` to help protect against
-HTTP Host header attacks.
+: If you're planning on putting Paperless on the open internet, then
+you really should set this value to the domain name you're using.
+Failing to do so leaves you open to HTTP host header attacks.
 You can read more about this in [the Django project's documentation](https://docs.djangoproject.com/en/4.1/topics/security/#host-header-validation)
 
-    Separate multiple hostnames with commas, as in `example.com,www.example.com`.
+    Just remember that this is a comma-separated list, so
+    "example.com" is fine, as is "example.com,www.example.com", but
+    NOT " example.com" or "example.com,"
 
-    - With neither setting configured, Paperless accepts any host.
-    - With only `PAPERLESS_URL` set, Paperless accepts the hostname from that URL
-      and `localhost` (used by the Docker health check).
-    - `PAPERLESS_ALLOWED_HOSTS` adds hosts to `localhost` and the hostname from
-      `PAPERLESS_URL`, if set. A blank list adds no hosts.
-    - With `PAPERLESS_ALLOWED_HOSTS=*`, Paperless accepts any host, even when
-      `PAPERLESS_URL` is set.
+    Can also be set using PAPERLESS_URL (see above).
+
+    "localhost" is always allowed for docker healthcheck
+
+    Defaults to "\*", which is all hosts.
 
 #### [`PAPERLESS_CORS_ALLOWED_HOSTS=<comma-separated-list>`](#PAPERLESS_CORS_ALLOWED_HOSTS) {#PAPERLESS_CORS_ALLOWED_HOSTS}
 
@@ -873,6 +873,14 @@ If both the [PAPERLESS_ACCOUNT_DEFAULT_GROUPS](#PAPERLESS_ACCOUNT_DEFAULT_GROUPS
 
     Defaults to 1209600 (2 weeks)
 
+#### [`PAPERLESS_SESSION_CLEANUP_TASK_CRON=<cron expression>`](#PAPERLESS_SESSION_CLEANUP_TASK_CRON) {#PAPERLESS_SESSION_CLEANUP_TASK_CRON}
+
+: Configures the schedule to remove expired sessions from the session store.
+Django does not prune these automatically, so without this they would
+accumulate indefinitely.
+
+    Defaults to `0 3 * * *`, once per day at 03:00.
+
 #### [`PAPERLESS_TOKEN_THROTTLE_RATE=<rate>`](#PAPERLESS_TOKEN_THROTTLE_RATE) {#PAPERLESS_TOKEN_THROTTLE_RATE}
 
 : Rate limit for the API token authentication endpoint (`/api/token/`), used to mitigate brute-force login attempts.

src/documents/tasks.py

@@ -12,6 +12,7 @@ from celery import Task
 from celery import shared_task
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
+from django.contrib.sessions.models import Session
 from django.db import models
 from django.db import transaction
 from django.db.models.signals import post_save
@@ -925,3 +926,17 @@ def cleanup_expired_share_link_bundles() -> None:
             )
     if count:
         logger.info("Deleted %s expired share link bundle(s)", count)
+
+
+@shared_task
+def clear_expired_sessions() -> None:
+    """
+    Remove expired database sessions in batches to limit the size of each delete.
+    """
+    expired = Session.objects.filter(expire_date__lt=timezone.now())
+    while session_keys := list(
+        expired.order_by("expire_date").values_list("pk", flat=True)[:500],
+    ):
+        expired.filter(pk__in=session_keys).delete()
+
+    logger.info("Cleared expired sessions")

src/documents/tests/test_tasks.py

@@ -5,8 +5,11 @@ from unittest import mock
 
 import pytest
 from django.conf import settings
+from django.contrib.sessions.models import Session
+from django.db import connection
 from django.test import TestCase
 from django.test import override_settings
+from django.test.utils import CaptureQueriesContext
 from django.utils import timezone
 
 from documents import tasks
@@ -612,3 +615,36 @@ class TestApplyAISuggestionsTask(DirectoriesMixin, TestCase):
 
         apply_suggestions.assert_not_called()
         self.assertIn("no longer exists", "".join(cm.output))
+
+
+@pytest.mark.django_db
+class TestClearExpiredSessions:
+    def test_clears_only_expired_sessions_in_batches(self) -> None:
+        """
+        GIVEN:
+            - More than one batch of expired sessions and one still-valid session
+        WHEN:
+            - The clear_expired_sessions task runs
+        THEN:
+            - Only expired sessions are removed, using multiple deletes
+        """
+        now = timezone.now()
+        Session.objects.bulk_create(
+            Session(
+                session_key=f"expired-{i}",
+                session_data="",
+                expire_date=now - timedelta(days=1),
+            )
+            for i in range(501)
+        )
+        Session.objects.create(
+            session_key="valid",
+            session_data="",
+            expire_date=now + timedelta(days=1),
+        )
+
+        with CaptureQueriesContext(connection) as queries:
+            tasks.clear_expired_sessions()
+
+        assert set(Session.objects.values_list("session_key", flat=True)) == {"valid"}
+        assert sum(query["sql"].startswith("DELETE FROM") for query in queries) == 2

src/paperless/settings/__init__.py

@@ -450,10 +450,7 @@ CORS_EXPOSE_HEADERS = [
     "Content-Disposition",
 ]
 
-if "PAPERLESS_ALLOWED_HOSTS" in os.environ or os.getenv("PAPERLESS_URL"):
-    ALLOWED_HOSTS = get_list_from_env("PAPERLESS_ALLOWED_HOSTS")
-else:
-    ALLOWED_HOSTS = ["*"]
+ALLOWED_HOSTS = get_list_from_env("PAPERLESS_ALLOWED_HOSTS", default=["*"])
 if ALLOWED_HOSTS != ["*"]:
     # always allow localhost. Necessary e.g. for healthcheck in docker.
     ALLOWED_HOSTS.append("localhost")
@@ -463,15 +460,13 @@ def _parse_paperless_url():
     global CSRF_TRUSTED_ORIGINS, CORS_ALLOWED_ORIGINS, ALLOWED_HOSTS
     url = os.getenv("PAPERLESS_URL")
     if url:
-        hostname = (urlparse(url).hostname or "").removesuffix(".")
-        if not hostname or hostname == "*" or hostname.startswith("."):
+        hostname = urlparse(url).hostname
+        if not hostname:
             raise ImproperlyConfigured(
-                f"PAPERLESS_URL {url!r} has no valid hostname, expected a URL like https://example.com",
+                f"PAPERLESS_URL {url!r} has no hostname, expected a URL like https://example.com",
             )
         CSRF_TRUSTED_ORIGINS.append(url)
         CORS_ALLOWED_ORIGINS.append(url)
-        if ":" in hostname and not hostname.startswith("["):
-            hostname = f"[{hostname}]"  # Add brackets around IPv6 addresses
         ALLOWED_HOSTS.append(hostname)
 
     return url

src/paperless/settings/custom.py

@@ -165,6 +165,17 @@ def parse_beat_schedule() -> dict:
                 "expires": 23.0 * 60.0 * 60.0,
             },
         },
+        {
+            "name": "Clear expired sessions",
+            "env_key": "PAPERLESS_SESSION_CLEANUP_TASK_CRON",
+            # Default daily at 03:00
+            "env_default": "0 3 * * *",
+            "task": "documents.tasks.clear_expired_sessions",
+            "options": {
+                # 1 hour before default schedule sends again
+                "expires": 23.0 * 60.0 * 60.0,
+            },
+        },
     ]
     for task in tasks:
         # Either get the environment setting or use the default

src/paperless/tests/settings/test_custom_parsers.py

@@ -182,6 +182,7 @@ def make_expected_schedule(
     workflow_expire = 59.0 * 60.0
     llm_index_expire = 23.0 * 60.0 * 60.0
     share_link_cleanup_expire = 23.0 * 60.0 * 60.0
+    session_cleanup_expire = 23.0 * 60.0 * 60.0
 
     schedule: dict[str, Any] = {
         "Check all e-mail accounts": {
@@ -248,6 +249,14 @@ def make_expected_schedule(
                 "headers": {"trigger_source": "scheduled"},
             },
         },
+        "Clear expired sessions": {
+            "task": "documents.tasks.clear_expired_sessions",
+            "schedule": crontab(minute=0, hour="3"),
+            "options": {
+                "expires": session_cleanup_expire,
+                "headers": {"trigger_source": "scheduled"},
+            },
+        },
     }
 
     overrides = overrides or {}
@@ -298,6 +307,7 @@ class TestParseBeatSchedule:
                     "PAPERLESS_WORKFLOW_SCHEDULED_TASK_CRON": "disable",
                     "PAPERLESS_LLM_INDEX_TASK_CRON": "disable",
                     "PAPERLESS_SHARE_LINK_BUNDLE_CLEANUP_CRON": "disable",
+                    "PAPERLESS_SESSION_CLEANUP_TASK_CRON": "disable",
                 },
                 {},
                 id="all-disabled",

src/paperless/tests/settings/test_settings.py

@@ -1,8 +1,4 @@
-import json
 import os
-import subprocess
-import sys
-from pathlib import Path
 from unittest import TestCase
 from unittest import mock
 
@@ -172,66 +168,8 @@ class TestPaperlessURLSettings(TestCase):
             self.assertIn(url, settings.CORS_ALLOWED_ORIGINS)
 
 
-@pytest.mark.parametrize(
-    ("url", "allowed_hosts", "expected"),
-    [
-        (None, None, ["*"]),
-        (None, "", ["localhost"]),
-        ("https://example.com:8443", None, ["localhost", "example.com"]),
-        ("https://example.com.", "", ["localhost", "example.com"]),
-        ("http://[::1]:8000", None, ["localhost", "[::1]"]),
-        ("https://example.com", "*", ["*", "example.com"]),
-        (
-            "https://example.com",
-            "lan.example.com",
-            ["lan.example.com", "localhost", "example.com"],
-        ),
-    ],
-)
-def test_allowed_hosts_at_startup(
-    tmp_path: Path,
-    url: str | None,
-    allowed_hosts: str | None,
-    expected: list[str],
-) -> None:
-    config = tmp_path / "paperless.conf"
-    config.touch()
-    env = {
-        **os.environ,
-        "PAPERLESS_CONFIGURATION_PATH": str(config),
-        "PAPERLESS_LOGGING_DIR": str(tmp_path),
-    }
-    for name, value in (
-        ("PAPERLESS_URL", url),
-        ("PAPERLESS_ALLOWED_HOSTS", allowed_hosts),
-    ):
-        if value is None:
-            env.pop(name, None)
-        else:
-            env[name] = value
-
-    result = subprocess.run(
-        [
-            sys.executable,
-            "-c",
-            "import json; from paperless.settings import ALLOWED_HOSTS; print(json.dumps(ALLOWED_HOSTS))",
-        ],
-        cwd=Path(__file__).resolve().parents[3],
-        env=env,
-        capture_output=True,
-        text=True,
-        timeout=30,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-    assert json.loads(result.stdout) == expected
-
-
-@pytest.mark.parametrize(
-    "url",
-    ["example.com", "https://", "/just/a/path", "https://*", "https://.example.com"],
-)
-def test_paperless_url_without_valid_hostname_is_rejected(
+@pytest.mark.parametrize("url", ["example.com", "https://", "/just/a/path"])
+def test_paperless_url_without_hostname_is_rejected(
     monkeypatch: pytest.MonkeyPatch,
     url: str,
 ) -> None:
@@ -241,9 +179,9 @@ def test_paperless_url_without_valid_hostname_is_rejected(
     WHEN:
         - The URL is parsed
     THEN:
-        - A configuration error is raised for missing or wildcard hostnames
+        - A configuration error is raised instead of adding None to ALLOWED_HOSTS
     """
     monkeypatch.setenv("PAPERLESS_URL", url)
 
-    with pytest.raises(ImproperlyConfigured, match="no valid hostname"):
+    with pytest.raises(ImproperlyConfigured, match="no hostname"):
         _parse_paperless_url()

src/paperless_ai/embedding.py

@@ -1,4 +1,5 @@
 import re
+from contextlib import suppress
 from typing import TYPE_CHECKING
 
 from django.conf import settings
@@ -20,7 +21,17 @@ OCR_LEADER_REGEX = re.compile(r"[._\-\u00b7]{4,}")
 HORIZONTAL_WHITESPACE_REGEX = re.compile(r"[ \t\u00a0]+")
 
 
-def get_embedding_model(config: AIConfig) -> "BaseEmbedding":
+def get_embedding_model(
+    config: AIConfig,
+    *,
+    refresh: bool = False,
+) -> "BaseEmbedding":
+    """
+    Build the embedding model for the configured backend.
+
+    For Hugging Face, a cached model loads offline by default, saving seconds per
+    call. Pass ``refresh=True`` to check the Hub for a newer model revision.
+    """
     match config.llm_embedding_backend:
         case LLMEmbeddingBackend.OPENAI_LIKE:
             from llama_index.embeddings.openai_like import OpenAILikeEmbedding
@@ -52,11 +63,15 @@ def get_embedding_model(config: AIConfig) -> "BaseEmbedding":
         case LLMEmbeddingBackend.HUGGINGFACE:
             from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 
-            return HuggingFaceEmbedding(
-                model_name=config.llm_embedding_model
+            hf_kwargs = {
+                "model_name": config.llm_embedding_model
                 or "sentence-transformers/all-MiniLM-L6-v2",
-                cache_folder=str(settings.DATA_DIR / "hf_cache"),
-            )
+                "cache_folder": str(settings.DATA_DIR / "hf_cache"),
+            }
+            if not refresh:
+                with suppress(OSError):
+                    return HuggingFaceEmbedding(**hf_kwargs, local_files_only=True)
+            return HuggingFaceEmbedding(**hf_kwargs)
         case LLMEmbeddingBackend.OLLAMA:
             from llama_index.embeddings.ollama import OllamaEmbedding
             from ollama import AsyncClient

src/paperless_ai/indexing.py

@@ -470,7 +470,7 @@ def update_llm_index(
         logger.warning("No documents found to index.")
 
     chunk_size = config.llm_embedding_chunk_size
-    embed_model = get_embedding_model(config)
+    embed_model = get_embedding_model(config, refresh=rebuild)
 
     with write_store(embed_model_name=model_name) as store:
         if rebuild or not store.table_exists():

src/paperless_ai/tests/test_ai_indexing.py

@@ -212,6 +212,10 @@ def test_update_llm_index(
 
         ai_config.assert_called_once()
         build_document_node.assert_called_once_with(real_document, chunk_size=512)
+        indexing.get_embedding_model.assert_called_once_with(
+            mock_config,
+            refresh=True,
+        )
 
 
 @pytest.mark.django_db
@@ -336,6 +340,8 @@ def test_update_llm_index_partial_update(
 
         indexing.update_llm_index(rebuild=False)
 
+    assert indexing.get_embedding_model.call_args.kwargs == {"refresh": False}
+
     with indexing.get_vector_store() as store:
         assert store.table_exists(), (
             "Expected the vector store table to exist after incremental update"

src/paperless_ai/tests/test_embedding.py

@@ -1,5 +1,6 @@
 from unittest.mock import ANY
 from unittest.mock import MagicMock
+from unittest.mock import call
 from unittest.mock import patch
 
 import pytest
@@ -167,10 +168,45 @@ def test_get_embedding_model_huggingface(mock_ai_config):
         MockHuggingFaceEmbedding.assert_called_once_with(
             model_name="sentence-transformers/all-MiniLM-L6-v2",
             cache_folder=str(settings.DATA_DIR / "hf_cache"),
+            local_files_only=True,
         )
         assert model == MockHuggingFaceEmbedding.return_value
 
 
+def test_get_embedding_model_huggingface_downloads_when_not_cached(mock_ai_config):
+    mock_ai_config.return_value.llm_embedding_backend = LLMEmbeddingBackend.HUGGINGFACE
+    mock_ai_config.return_value.llm_embedding_model = "some/model"
+    downloaded = MagicMock()
+
+    with patch(
+        "llama_index.embeddings.huggingface.HuggingFaceEmbedding",
+        side_effect=[OSError("not cached"), downloaded],
+    ) as MockHuggingFaceEmbedding:
+        model = get_embedding_model(mock_ai_config.return_value)
+
+    cache_folder = str(settings.DATA_DIR / "hf_cache")
+    assert MockHuggingFaceEmbedding.call_args_list == [
+        call(model_name="some/model", cache_folder=cache_folder, local_files_only=True),
+        call(model_name="some/model", cache_folder=cache_folder),
+    ]
+    assert model is downloaded
+
+
+def test_get_embedding_model_huggingface_refresh_checks_the_hub(mock_ai_config):
+    mock_ai_config.return_value.llm_embedding_backend = LLMEmbeddingBackend.HUGGINGFACE
+    mock_ai_config.return_value.llm_embedding_model = "some/model"
+
+    with patch(
+        "llama_index.embeddings.huggingface.HuggingFaceEmbedding",
+    ) as MockHuggingFaceEmbedding:
+        get_embedding_model(mock_ai_config.return_value, refresh=True)
+
+    MockHuggingFaceEmbedding.assert_called_once_with(
+        model_name="some/model",
+        cache_folder=str(settings.DATA_DIR / "hf_cache"),
+    )
+
+
 def test_get_embedding_model_ollama(mock_ai_config):
     mock_ai_config.return_value.llm_embedding_backend = LLMEmbeddingBackend.OLLAMA
     mock_ai_config.return_value.llm_embedding_model = "embeddinggemma"