mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-18 02:22:20 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b57251e6c6 | ||
|
|
7977093e83 | ||
|
|
237521ec4b | ||
|
|
0596579d4b | ||
|
|
a22579e311 | ||
|
|
199d1a88e9 |
No files matched your search
+3
-2
@@ -151,8 +151,9 @@ typing = [
|
||||
required-version = ">=0.9.0"
|
||||
prerelease = "disallow"
|
||||
environments = [
|
||||
"sys_platform == 'darwin'",
|
||||
"sys_platform == 'linux'",
|
||||
"sys_platform == 'darwin' and platform_machine == 'arm64' and platform_python_implementation == 'CPython'",
|
||||
"sys_platform == 'linux' and platform_machine == 'aarch64' and platform_python_implementation == 'CPython'",
|
||||
"sys_platform == 'linux' and platform_machine == 'x86_64' and platform_python_implementation == 'CPython'",
|
||||
]
|
||||
package = false
|
||||
[tool.uv.sources]
|
||||
|
||||
+2
-2
@@ -4151,14 +4151,14 @@
|
||||
<source>Ask a question about this document...</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/chat/chat/chat.component.ts</context>
|
||||
<context context-type="linenumber">83</context>
|
||||
<context context-type="linenumber">87</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="224049052163028286" datatype="html">
|
||||
<source>Ask a question about a document...</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/chat/chat/chat.component.ts</context>
|
||||
<context context-type="linenumber">84</context>
|
||||
<context context-type="linenumber">88</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="8700121026680200191" datatype="html">
|
||||
|
||||
@@ -188,6 +188,62 @@ describe('ChatComponent', () => {
|
||||
expect(component.loading()).toBe(false)
|
||||
})
|
||||
|
||||
it('should reveal a complete answer within one second', () => {
|
||||
component.input.set('Hello')
|
||||
component.sendMessage()
|
||||
|
||||
const answer = 'A'.repeat(660)
|
||||
mockStream$.next(answer)
|
||||
mockStream$.complete()
|
||||
|
||||
expect(component.messages()[1].content.length).toBeGreaterThan(0)
|
||||
expect(component.messages()[1].content.length).toBeLessThan(answer.length)
|
||||
expect(component.loading()).toBe(true)
|
||||
|
||||
jest.advanceTimersByTime(800)
|
||||
expect(component.messages()[1].content).toBe(answer)
|
||||
expect(component.loading()).toBe(false)
|
||||
})
|
||||
|
||||
it('should give a long answer more time to reveal', () => {
|
||||
component.input.set('Hello')
|
||||
component.sendMessage()
|
||||
|
||||
const answer = 'A'.repeat(3000)
|
||||
mockStream$.next(answer)
|
||||
mockStream$.complete()
|
||||
|
||||
jest.advanceTimersByTime(800)
|
||||
expect(component.messages()[1].content.length).toBeLessThan(answer.length)
|
||||
expect(component.loading()).toBe(true)
|
||||
|
||||
jest.advanceTimersByTime(2500)
|
||||
expect(component.messages()[1].content).toBe(answer)
|
||||
expect(component.loading()).toBe(false)
|
||||
})
|
||||
|
||||
it('should reveal fast incoming chunks without making them pop in', () => {
|
||||
component.input.set('Hello')
|
||||
component.sendMessage()
|
||||
|
||||
let answer = ''
|
||||
for (let i = 0; i < 20; i++) {
|
||||
answer += 'A'.repeat(100)
|
||||
const visibleBefore = component.messages()[1].content.length
|
||||
mockStream$.next(answer)
|
||||
jest.advanceTimersByTime(10)
|
||||
expect(
|
||||
component.messages()[1].content.length - visibleBefore
|
||||
).toBeLessThan(100)
|
||||
}
|
||||
expect(component.messages()[1].content.length).toBeLessThan(answer.length)
|
||||
|
||||
mockStream$.complete()
|
||||
jest.advanceTimersByTime(5200)
|
||||
expect(component.messages()[1].content).toBe(answer)
|
||||
expect(component.loading()).toBe(false)
|
||||
})
|
||||
|
||||
it('should show a slow response message while waiting', () => {
|
||||
component.input.set('Hello')
|
||||
component.sendMessage()
|
||||
|
||||
@@ -21,6 +21,9 @@ import {
|
||||
import { pngxPopperOptions } from 'src/app/utils/popper-options'
|
||||
|
||||
const SLOW_RESPONSE_DELAY_MS = 8000
|
||||
const TYPEWRITER_MIN_DURATION_MS = 750
|
||||
const TYPEWRITER_MAX_DURATION_MS = 3000
|
||||
const TYPEWRITER_INTERVAL_MS = 16
|
||||
|
||||
@Component({
|
||||
selector: 'pngx-chat',
|
||||
@@ -73,6 +76,7 @@ export class ChatComponent implements OnInit {
|
||||
private typewriterBuffer: string[] = []
|
||||
private typewriterActive = false
|
||||
private typewriterTimeout: ReturnType<typeof setTimeout>
|
||||
private typewriterDeadline = 0
|
||||
private streamSubscription: Subscription
|
||||
private streamingMessage: ChatMessage
|
||||
private streamFinished = false
|
||||
@@ -134,7 +138,7 @@ export class ChatComponent implements OnInit {
|
||||
if (nextResponse.content.length < lastVisibleContent.length) {
|
||||
this.resetTypewriter(assistantMessage, nextResponse.content)
|
||||
lastVisibleContent = nextResponse.content
|
||||
} else {
|
||||
} else if (nextResponse.content.length > lastVisibleContent.length) {
|
||||
const visibleDelta = nextResponse.content.substring(
|
||||
lastVisibleContent.length
|
||||
)
|
||||
@@ -214,6 +218,12 @@ export class ChatComponent implements OnInit {
|
||||
if (!chunk) return
|
||||
|
||||
this.typewriterBuffer.push(...chunk.split(''))
|
||||
// Longer bursts need more time to remain readable as the chat scrolls.
|
||||
const revealDuration = Math.min(
|
||||
TYPEWRITER_MAX_DURATION_MS,
|
||||
Math.max(TYPEWRITER_MIN_DURATION_MS, this.typewriterBuffer.length)
|
||||
)
|
||||
this.typewriterDeadline = Date.now() + revealDuration
|
||||
|
||||
if (!this.typewriterActive) {
|
||||
this.typewriterActive = true
|
||||
@@ -222,19 +232,29 @@ export class ChatComponent implements OnInit {
|
||||
}
|
||||
|
||||
playTypewriter(message: ChatMessage): void {
|
||||
if (this.typewriterBuffer.length === 0) {
|
||||
this.typewriterActive = false
|
||||
if (this.streamFinished) this.finishStreaming(message)
|
||||
return
|
||||
}
|
||||
|
||||
const nextChar = this.typewriterBuffer.shift()
|
||||
message.content += nextChar
|
||||
// Spread the queued text across the updates left before the deadline
|
||||
const remainingUpdates = Math.max(
|
||||
1,
|
||||
Math.ceil((this.typewriterDeadline - Date.now()) / TYPEWRITER_INTERVAL_MS)
|
||||
)
|
||||
const charactersToShow = Math.ceil(
|
||||
this.typewriterBuffer.length / remainingUpdates
|
||||
)
|
||||
message.content += this.typewriterBuffer
|
||||
.splice(0, charactersToShow)
|
||||
.join('')
|
||||
this.notifyMessagesChanged()
|
||||
this.scrollToBottom()
|
||||
|
||||
// 10ms per character
|
||||
this.typewriterTimeout = setTimeout(() => this.playTypewriter(message), 10)
|
||||
if (this.typewriterBuffer.length === 0) {
|
||||
this.typewriterActive = false
|
||||
if (this.streamFinished) this.finishStreaming(message)
|
||||
} else {
|
||||
this.typewriterTimeout = setTimeout(
|
||||
() => this.playTypewriter(message),
|
||||
TYPEWRITER_INTERVAL_MS
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private notifyMessagesChanged(): void {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Queued document bulk operations."""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from typing import Any
|
||||
|
||||
from auditlog.context import set_actor
|
||||
@@ -112,6 +113,8 @@ def execute_bulk_operation(
|
||||
if operation == "run_workflow" and not user.has_perm("documents.view_workflow"):
|
||||
raise PermissionDenied("Insufficient permissions to run this workflow")
|
||||
|
||||
actor_context = set_actor(user) if settings.AUDIT_LOG_ENABLED else nullcontext()
|
||||
|
||||
if operation not in MODIFIED_FIELDS:
|
||||
execution_parameters = parameters.copy()
|
||||
if operation in USER_OPERATIONS:
|
||||
@@ -119,7 +122,7 @@ def execute_bulk_operation(
|
||||
if operation in TRIGGER_SOURCE_OPERATIONS:
|
||||
execution_parameters["trigger_source"] = trigger_source
|
||||
modified_field = LEGACY_MODIFIED_FIELDS.get(operation)
|
||||
with set_actor(user):
|
||||
with actor_context:
|
||||
old_values = (
|
||||
snapshot_field(document_ids, modified_field)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field
|
||||
@@ -146,7 +149,7 @@ def execute_bulk_operation(
|
||||
modified_field = MODIFIED_FIELDS[operation]
|
||||
# Keep metadata writes and their audit entries together. Follow-up processing
|
||||
# runs after commit, within this same tracked task.
|
||||
with set_actor(user), transaction.atomic():
|
||||
with actor_context, transaction.atomic():
|
||||
old_values = (
|
||||
snapshot_field(document_ids, modified_field)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field
|
||||
|
||||
@@ -4,14 +4,18 @@ from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from asgiref.sync import async_to_sync
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.views import _iterate_in_sync_thread
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.permissions import grant_global
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
@@ -62,7 +66,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
),
|
||||
mock.patch(
|
||||
"paperless_ai.chat.stream_chat_with_documents",
|
||||
return_value=iter(chunks),
|
||||
return_value=(c for c in chunks),
|
||||
),
|
||||
):
|
||||
resp = self.client.post(
|
||||
@@ -74,7 +78,11 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
assert not resp.has_header("Content-Encoding")
|
||||
assert list(resp.streaming_content) == [c.encode() for c in chunks]
|
||||
|
||||
async def read_parts() -> list[bytes]:
|
||||
return [part async for part in resp.streaming_content]
|
||||
|
||||
assert async_to_sync(read_parts)() == [c.encode() for c in chunks]
|
||||
|
||||
def test_missing_question_is_rejected(self) -> None:
|
||||
with mock.patch(
|
||||
@@ -155,3 +163,74 @@ class TestChatStreamingViewUnrestrictedFlag:
|
||||
assert (
|
||||
mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
|
||||
)
|
||||
|
||||
|
||||
class TestIterateInSyncThread:
|
||||
@pytest.fixture
|
||||
def events(self) -> list[str]:
|
||||
return []
|
||||
|
||||
@pytest.fixture
|
||||
def chunks(self, events: list[str]) -> Generator[str, None, None]:
|
||||
def generate() -> Generator[str, None, None]:
|
||||
try:
|
||||
for chunk in ("first", "second"):
|
||||
events.append(f"produced {chunk}")
|
||||
yield chunk
|
||||
finally:
|
||||
events.append("closed")
|
||||
|
||||
return generate()
|
||||
|
||||
def test_yields_each_chunk_before_pulling_the_next(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The wrapper is consumed
|
||||
THEN:
|
||||
- Each chunk is delivered before the next one is generated,
|
||||
rather than the whole generator being drained up front
|
||||
"""
|
||||
|
||||
async def consume() -> None:
|
||||
async for chunk in _iterate_in_sync_thread(chunks):
|
||||
events.append(f"received {chunk}")
|
||||
|
||||
async_to_sync(consume)()
|
||||
|
||||
assert events == [
|
||||
"produced first",
|
||||
"received first",
|
||||
"produced second",
|
||||
"received second",
|
||||
"closed",
|
||||
]
|
||||
|
||||
def test_closing_early_closes_the_source_generator(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The consumer closes the wrapper after the first chunk, as
|
||||
happens when the client disconnects
|
||||
THEN:
|
||||
- The source generator is closed, so its cleanup runs
|
||||
"""
|
||||
|
||||
async def consume_one() -> None:
|
||||
wrapper = _iterate_in_sync_thread(chunks)
|
||||
await anext(wrapper)
|
||||
await wrapper.aclose()
|
||||
|
||||
async_to_sync(consume_one)()
|
||||
|
||||
assert events == ["produced first", "closed"]
|
||||
@@ -123,6 +123,41 @@ def test_document_action_runs_with_user_in_worker(bulk_setup):
|
||||
assert result == {"operation": "rotate", "document_count": 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_delete_task_without_audit_log(bulk_setup, settings):
|
||||
user, _, doc, _ = bulk_setup
|
||||
settings.AUDIT_LOG_ENABLED = False
|
||||
with mock.patch(
|
||||
"documents.bulk_operations.set_actor",
|
||||
side_effect=AssertionError("audit context must not be entered"),
|
||||
):
|
||||
result = execute_bulk_operation("delete", [doc.pk], {}, user.pk)
|
||||
assert result == {"operation": "delete", "document_count": 1}
|
||||
assert not Document.objects.filter(pk=doc.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_metadata_task_without_audit_log(bulk_setup, settings):
|
||||
user, _, doc, correspondent = bulk_setup
|
||||
settings.AUDIT_LOG_ENABLED = False
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.bulk_operations.set_actor",
|
||||
side_effect=AssertionError("audit context must not be entered"),
|
||||
),
|
||||
mock.patch("documents.bulk_operations.bulk_update_documents"),
|
||||
):
|
||||
result = execute_bulk_operation(
|
||||
"set_correspondent",
|
||||
[doc.pk],
|
||||
{"correspondent": correspondent.pk},
|
||||
user.pk,
|
||||
)
|
||||
doc.refresh_from_db()
|
||||
assert doc.correspondent_id == correspondent.pk
|
||||
assert result == {"operation": "set_correspondent", "document_count": 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_document_action_queue_contains_only_serializable_parameters(bulk_setup):
|
||||
user, client, doc, _ = bulk_setup
|
||||
|
||||
+29
-7
@@ -7,7 +7,9 @@ import tempfile
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Generator
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
@@ -27,6 +29,7 @@ from urllib.parse import urlparse
|
||||
import httpx
|
||||
import magic
|
||||
import pathvalidate
|
||||
from asgiref.sync import sync_to_async
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import User
|
||||
@@ -2416,6 +2419,24 @@ class DocumentViewSet(
|
||||
)
|
||||
|
||||
|
||||
async def _iterate_in_sync_thread(
|
||||
chunks: Generator[str, None, None],
|
||||
) -> AsyncIterator[str]:
|
||||
"""Serve a blocking sync generator to an ASGI server chunk by chunk.
|
||||
|
||||
Under ASGI, Django drains a sync iterator with ``sync_to_async(list)``
|
||||
before sending anything. Pull each chunk in the request's sync thread
|
||||
instead, where the generator's ORM access belongs.
|
||||
"""
|
||||
sentinel = object() # StopIteration can't cross an asyncio Future
|
||||
next_chunk = sync_to_async(next)
|
||||
try:
|
||||
while (chunk := await next_chunk(chunks, sentinel)) is not sentinel:
|
||||
yield chunk
|
||||
finally:
|
||||
await sync_to_async(chunks.close)()
|
||||
|
||||
|
||||
class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]):
|
||||
q = serializers.CharField(required=True, max_length=4000)
|
||||
document_id = serializers.IntegerField(required=False, allow_null=True)
|
||||
@@ -2474,16 +2495,17 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
user=request.user,
|
||||
)
|
||||
|
||||
response = StreamingHttpResponse(
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
return StreamingHttpResponse(
|
||||
_iterate_in_sync_thread(
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
),
|
||||
),
|
||||
content_type="text/event-stream",
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
|
||||
@@ -2,7 +2,7 @@ msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: paperless-ngx\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2026-11-17 00:24+0000\n"
|
||||
"POT-Creation-Date: 2026-11-17 20:48+0000\n"
|
||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||
"Last-Translator: \n"
|
||||
"Language-Team: English\n"
|
||||
@@ -1669,8 +1669,8 @@ msgid "workflow runs"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:483 documents/serialisers.py:844
|
||||
#: documents/serialisers.py:2893 documents/views.py:356 documents/views.py:622
|
||||
#: documents/views.py:2761 paperless_mail/serialisers.py:145
|
||||
#: documents/serialisers.py:2893 documents/views.py:359 documents/views.py:625
|
||||
#: documents/views.py:2783 paperless_mail/serialisers.py:145
|
||||
msgid "Insufficient permissions."
|
||||
msgstr ""
|
||||
|
||||
@@ -1710,7 +1710,7 @@ msgstr ""
|
||||
msgid "Duplicate document identifiers are not allowed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2979 documents/views.py:4748
|
||||
#: documents/serialisers.py:2979 documents/views.py:4770
|
||||
#, python-format
|
||||
msgid "Documents not found: %(ids)s"
|
||||
msgstr ""
|
||||
@@ -1961,40 +1961,40 @@ msgstr ""
|
||||
msgid ", "
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:311
|
||||
#: documents/views.py:314
|
||||
msgid "Specify only one of text, title_search, query, or more_like_id."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:349 documents/views.py:2758
|
||||
#: documents/views.py:352 documents/views.py:2780
|
||||
msgid "Invalid more_like_id"
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1681
|
||||
#: documents/views.py:1684
|
||||
msgid "Invalid AI configuration."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1692
|
||||
#: documents/views.py:1695
|
||||
msgid "AI backend request timed out."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1704
|
||||
#: documents/views.py:1707
|
||||
msgid "AI backend rejected the request. Check logs for details."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4764
|
||||
#: documents/views.py:4786
|
||||
#, python-format
|
||||
msgid "Insufficient permissions to share document %(id)s."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4811
|
||||
#: documents/views.py:4833
|
||||
msgid "Bundle is already being processed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4877
|
||||
#: documents/views.py:4899
|
||||
msgid "The share link bundle is still being prepared. Please try again later."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4891
|
||||
#: documents/views.py:4913
|
||||
msgid "The share link bundle is unavailable."
|
||||
msgstr ""
|
||||
|
||||
|
||||
Reference in new issue
Block a user