Compare commits

...
Author SHA1 Message Date
shamoon e427818360 Fix test_bulk_edit_query_at_exactly_the_cap_is_accepted 2026-11-07 08:22:34 -08:00
shamoon 2f623bad7b Add back the queued notificaiton + link 2026-11-07 08:20:54 -08:00
shamoon 6a162abbfa prevent refreshes from changing destination 2026-11-07 08:10:04 -08:00
shamoon 0bd07aa61d This was a leftover 2026-11-07 08:10:04 -08:00
shamoon 358a291455 Shit, lets move delete back 2026-11-07 08:10:03 -08:00
shamoon cfa997b167 Update app-frame.component.ts 2026-11-07 08:10:03 -08:00
shamoon e52956c382 Hook up the refresh stuff on ws for bulk operations 2026-11-07 08:10:03 -08:00
shamoon f13f83aff5 Track task_id query param in tasks, add filtering by id 2026-11-07 08:10:02 -08:00
shamoon 631b19ef56 Actually we always get the task_id 2026-11-07 08:10:02 -08:00
shamoon 14ec3a2582 Update frontend bulk operatoins to pass from_webui, get the task id 2026-11-07 08:10:02 -08:00
shamoon 02a3d5ebe1 Ping via ws after bulk operations complete 2026-11-07 08:10:01 -08:00
shamoon a07a2f15fb Backend tests for the bulk operatoins stuff, task tracking 2026-11-07 08:10:01 -08:00
shamoon 570a4af601 Include bulk_operations for celery 2026-11-07 08:10:01 -08:00
shamoon dc02ca4707 Include from_webui for delete serializer 2026-11-07 08:10:00 -08:00
shamoon ee55d0adca Task tracking for bulk operations 2026-11-07 08:10:00 -08:00
shamoon bc89e00771 Make bulk edit use the bulk_operations callbacks, and unify the finish 2026-11-07 08:10:00 -08:00
shamoon 27bf4b05fa Create a bulk_operations module, move snapshot_field and has_document_operation_permissions 2026-11-07 08:09:59 -08:00
shamoon 0736c07d8e Fix: use filtered documents for statistics ASN max (#14710) 2026-11-07 07:52:23 -08:00
shamoon 53d9aa34a1 FIx: make config view permissions consistent (#14709) 2026-11-07 07:52:12 -08:00
GitHub Actions 19fdc3b815 Auto translate strings 2026-11-07 02:45:43 +00:00
5357be2d43 Performance: stop embedding every document ID in selection_data queries (#14597)
* Fix: count selection_data from one pass over the filtered documents

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* Chore: use document type ids in the bulk edit selection_data test

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* Batch search selection data queries

Co-Authored-By: Codex <267193182+codex@users.noreply.github.com>

---------

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
Co-authored-by: Codex <267193182+codex@users.noreply.github.com>
2026-11-06 18:44:18 -08:00
GitHub Actions 2642cefd35 Auto translate strings 2026-11-06 20:20:35 +00:00
shamoonandCodex 8aa99a381a Performance: replace django-treenode with adjacency-list tags (#14612)
---------

Co-authored-by: Codex <267193182+codex@users.noreply.github.com>
2026-11-06 20:18:59 +00:00
Trenton H d14cfc93f4 Performance: use small vec0 chunks and larger pages for the LLM index (#14678)
The vec0 default of 1024 rows per vector chunk makes each chunk a
multi-megabyte blob. On spinning disks, indexing a document took about
2.5 seconds and re-indexing one about 13 seconds. With 32 rows per
chunk the same operations take about 35 and 180 milliseconds, with the
write-ahead log a third of the size and no cost in query speed or file
size.

Pair that with 64 KiB SQLite pages, which made cold scans about 35%
faster and warm ones about 45% faster at no write cost once the chunks
are small. The page size is set before the file enters WAL mode, so it
applies to new files.
2026-11-06 10:20:03 -08:00
shamoon 53e386ec81 Chore: avoid stale lock from failed apt process on CI (#14701) 2026-11-06 09:29:56 -08:00
44 changed files with 2526 additions and 955 deletions

No files matched your search

+4 -3
View File
@@ -6,9 +6,9 @@ if [ "${1:-}" = download ]; then
download_archive_dir=$2
shift 2
apt-get update -q
apt-get install -q --yes --no-install-recommends --download-only \
# Keep apt itself under timeout supervision, including the kill grace period.
exec apt-get install -q --yes --no-install-recommends --download-only \
-o "Dir::Cache::archives=$download_archive_dir" "$@"
exit 0
fi
SUDO=()
@@ -34,7 +34,8 @@ use_ubuntu_mirrors() {
for attempt in $(seq 1 "$ATTEMPTS"); do
echo "::group::apt download attempt ${attempt}/${ATTEMPTS}"
if timeout --kill-after=10 "$ATTEMPT_TIMEOUT" "${SUDO[@]}" bash "$0" download "$CACHE_DIR" "${requested_packages[@]}"; then
# Run timeout as root so it can signal the root-owned apt processes.
if "${SUDO[@]}" timeout --kill-after=10 "$ATTEMPT_TIMEOUT" bash "$0" download "$CACHE_DIR" "${requested_packages[@]}"; then
echo "::endgroup::"
# Apt handles aliases (e.g. libmagic1 -> libmagic1t64) and dependencies.
"${SUDO[@]}" apt-get install -q --yes --no-install-recommends --no-download \
@@ -39,7 +39,6 @@ declare -a scope=(
"${site_packages}/allauth"
"${site_packages}/drf_spectacular"
"${site_packages}/drf_spectacular_sidecar"
"${site_packages}/treenode"
"${site_packages}/compression_middleware"
)
-1
View File
@@ -38,7 +38,6 @@ dependencies = [
"django-multiselectfield~=1.0.1",
"django-rich~=2.2.0",
"django-soft-delete~=1.0.18",
"django-treenode>=0.24",
"djangorestframework~=3.16",
"drf-spectacular~=0.30",
"drf-spectacular-sidecar>=2026.10.1,<2026.11",
@@ -7,10 +7,15 @@ import {
import { ComponentFixture, TestBed } from '@angular/core/testing'
import { FormsModule } from '@angular/forms'
import { By } from '@angular/platform-browser'
import { Router } from '@angular/router'
import {
ActivatedRoute,
convertToParamMap,
ParamMap,
Router,
} from '@angular/router'
import { RouterTestingModule } from '@angular/router/testing'
import { NgbModal, NgbModalRef, NgbModule } from '@ng-bootstrap/ng-bootstrap'
import { of, throwError } from 'rxjs'
import { BehaviorSubject, of, throwError } from 'rxjs'
import { routes } from 'src/app/app-routing.module'
import {
PaperlessTask,
@@ -159,6 +164,7 @@ const sectionCountResponse = {
}
describe('TasksComponent', () => {
let queryParams: BehaviorSubject<ParamMap>
let component: TasksComponent
let fixture: ComponentFixture<TasksComponent>
let tasksService: TasksService
@@ -169,6 +175,7 @@ describe('TasksComponent', () => {
let toastService: ToastService
beforeEach(async () => {
queryParams = new BehaviorSubject(convertToParamMap({}))
TestBed.configureTestingModule({
imports: [
NgbModule,
@@ -181,6 +188,17 @@ describe('TasksComponent', () => {
ConfirmDialogComponent,
],
providers: [
{
provide: ActivatedRoute,
useValue: {
queryParamMap: queryParams.asObservable(),
snapshot: {
get queryParamMap() {
return queryParams.value
},
},
},
},
{
provide: PermissionsService,
useValue: {
@@ -238,6 +256,71 @@ describe('TasksComponent', () => {
flushTaskLists(25, count)
}
it('consumes a task link on load into the normal resettable Task ID filter', () => {
jest
.spyOn(tasksService, 'list')
.mockReturnValue(of({ count: 0, results: [] }))
jest
.spyOn(tasksService, 'statusCounts')
.mockReturnValue(of(sectionCountResponse))
jest.spyOn(router, 'navigate').mockResolvedValue(true)
queryParams.next(convertToParamMap({ task_id: 'linked-task' }))
fixture.destroy()
fixture = TestBed.createComponent(TasksComponent)
component = fixture.componentInstance
fixture.detectChanges()
expect(router.navigate).toHaveBeenCalledTimes(1)
expect(router.navigate).toHaveBeenCalledWith(
[],
expect.objectContaining({
queryParams: { task_id: null },
replaceUrl: true,
})
)
expect(component.filterTargetName).toBe('Task ID')
expect(component.filterText).toBe('linked-task')
expect(component.isFiltered).toBe(true)
component.setSection(TaskSection.InProgress)
expect(tasksService.list).toHaveBeenLastCalledWith(1, 25, {
task_id: 'linked-task',
status: [PaperlessTaskStatus.Pending, PaperlessTaskStatus.Started],
})
component.resetFilters()
expect(component.filterText).toBe('')
expect(component.isFiltered).toBe(false)
expect(tasksService.list).toHaveBeenLastCalledWith(1, 25, {
acknowledged: false,
status: [PaperlessTaskStatus.Pending, PaperlessTaskStatus.Started],
})
expect(router.navigate).toHaveBeenCalledTimes(1)
})
it('supports manually filtering by Task ID and clearing the text filter', () => {
jest
.spyOn(tasksService, 'list')
.mockReturnValue(of({ count: 0, results: [] }))
jest
.spyOn(tasksService, 'statusCounts')
.mockReturnValue(of(sectionCountResponse))
component.setSection(TaskSection.InProgress)
component.setFilterTarget(TaskFilterTargetID.TaskId)
component.filterText = tasks[0].task_id
jest.advanceTimersByTime(150)
expect(tasksService.list).toHaveBeenLastCalledWith(1, 25, {
task_id: tasks[0].task_id,
status: [PaperlessTaskStatus.Pending, PaperlessTaskStatus.Started],
})
component.pagedTasks.set(tasks)
expect(component.tasksForSection(TaskSection.NeedsAttention)).toEqual([
tasks[0],
])
component.resetFilter()
expect(tasksService.list).toHaveBeenLastCalledWith(1, 25, {
acknowledged: false,
status: [PaperlessTaskStatus.Pending, PaperlessTaskStatus.Started],
})
})
it('should display task sections with counts', () => {
expect(component.selectedSection).toBe(TaskSection.All)
expect(component.selectedTaskType).toBeNull()
@@ -1,7 +1,7 @@
import { JsonPipe, NgTemplateOutlet } from '@angular/common'
import { Component, inject, OnDestroy, OnInit, signal } from '@angular/core'
import { FormsModule, ReactiveFormsModule } from '@angular/forms'
import { Router, RouterLink } from '@angular/router'
import { ActivatedRoute, Router, RouterLink } from '@angular/router'
import {
NgbCollapseModule,
NgbDropdownModule,
@@ -47,11 +47,13 @@ export enum TaskSection {
export enum TaskFilterTargetID {
Name,
Result,
TaskId,
}
const FILTER_TARGETS = [
{ id: TaskFilterTargetID.Name, name: $localize`Name` },
{ id: TaskFilterTargetID.Result, name: $localize`Result` },
{ id: TaskFilterTargetID.TaskId, name: $localize`Task ID` },
]
const SECTION_LABELS = {
@@ -161,6 +163,7 @@ export class TasksComponent
tasksService = inject(TasksService)
private modalService = inject(NgbModal)
private readonly router = inject(Router)
private readonly route = inject(ActivatedRoute)
private readonly toastService = inject(ToastService)
readonly TaskSection = TaskSection
@@ -275,6 +278,17 @@ export class TasksComponent
}
ngOnInit() {
const taskId = this.route.snapshot.queryParamMap.get('task_id')
if (taskId) {
this.filterTargetID = TaskFilterTargetID.TaskId
this._filterText = taskId
this.router.navigate([], {
relativeTo: this.route,
queryParams: { task_id: null },
queryParamsHandling: 'merge',
replaceUrl: true,
})
}
this.tasksService.reload()
this.reloadPage()
timer(5000, 5000)
@@ -636,6 +650,9 @@ export class TasksComponent
const query = this._filterText.toLowerCase()
if (this.filterTargetID === TaskFilterTargetID.TaskId) {
return task.task_id.toLowerCase() === query
}
if (this.filterTargetID == TaskFilterTargetID.Name) {
return [
this.taskDisplayName(task),
@@ -706,9 +723,15 @@ export class TasksComponent
}
if (this._filterText.length) {
params[
this.filterTargetID === TaskFilterTargetID.Name ? 'name' : 'result'
] = this._filterText
const parameter = {
[TaskFilterTargetID.Name]: 'name',
[TaskFilterTargetID.Result]: 'result',
[TaskFilterTargetID.TaskId]: 'task_id',
}[this.filterTargetID]
params[parameter] = this._filterText
if (this.filterTargetID === TaskFilterTargetID.TaskId) {
delete params.acknowledged
}
}
return params
@@ -11,8 +11,10 @@ import { ActivatedRoute, Router } from '@angular/router'
import { RouterTestingModule } from '@angular/router/testing'
import { NgbModal, NgbModalModule, NgbModule } from '@ng-bootstrap/ng-bootstrap'
import { provideUiTour } from 'ngx-ui-tour-ng-bootstrap'
import { of, throwError } from 'rxjs'
import { of, Subject, throwError } from 'rxjs'
import { routes } from 'src/app/app-routing.module'
import { BulkOperationStatus } from 'src/app/data/bulk-operation'
import { PaperlessTaskStatus } from 'src/app/data/paperless-task'
import { SavedView } from 'src/app/data/saved-view'
import { HideableSidebarItemID, SETTINGS_KEYS } from 'src/app/data/ui-settings'
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
@@ -21,10 +23,11 @@ import {
DjangoMessageLevel,
DjangoMessagesService,
} from 'src/app/services/django-messages.service'
import { DocumentListViewService } from 'src/app/services/document-list-view.service'
import { OpenDocumentsService } from 'src/app/services/open-documents.service'
import {
PermissionType,
PermissionsService,
PermissionType,
} from 'src/app/services/permissions.service'
import { RemoteVersionService } from 'src/app/services/rest/remote-version.service'
import { SavedViewService } from 'src/app/services/rest/saved-view.service'
@@ -95,6 +98,7 @@ describe('AppFrameComponent', () => {
let savedViewSpy
let modalService: NgbModal
let maybeRefreshSpy
let bulkEvents: Subject<BulkOperationStatus>
let tasksService: TasksService
beforeEach(async () => {
@@ -182,6 +186,10 @@ describe('AppFrameComponent', () => {
savedViewSpy = jest.spyOn(savedViewService, 'reload')
maybeRefreshSpy = jest.spyOn(savedViewService, 'maybeRefreshDocumentCounts')
bulkEvents = new Subject<BulkOperationStatus>()
jest
.spyOn(TestBed.inject(WebsocketStatusService), 'onBulkOperationStatus')
.mockReturnValue(bulkEvents.asObservable())
fixture = TestBed.createComponent(AppFrameComponent)
component = fixture.componentInstance
@@ -190,6 +198,71 @@ describe('AppFrameComponent', () => {
fixture.detectChanges()
})
it('refreshes and notifies on completion without client-side task tracking', () => {
const list = TestBed.inject(DocumentListViewService)
const reload = jest.spyOn(list, 'reload').mockImplementation(() => {})
jest.spyOn(list, 'reduceSelectionToFilter').mockImplementation(() => {})
const tasksReload = jest
.spyOn(tasksService, 'reload')
.mockImplementation(() => {})
const success = jest.spyOn(toastService, 'showSuccess')
const refresh = jest
.spyOn(openDocumentsService, 'refreshDocument')
.mockImplementation(() => {})
jest
.spyOn(openDocumentsService, 'getOpenDocuments')
.mockReturnValue([document])
tasksReload.mockClear()
maybeRefreshSpy.mockClear()
bulkEvents.next({
task_id: 'bulk-task',
operation: 'set_permissions',
status: PaperlessTaskStatus.Started,
})
expect(reload).not.toHaveBeenCalled()
bulkEvents.next({
task_id: 'bulk-task',
operation: 'set_permissions',
status: PaperlessTaskStatus.Success,
})
expect(reload).toHaveBeenCalledTimes(1)
expect(reload).toHaveBeenCalledWith(null, false)
expect(refresh).toHaveBeenCalledWith(document.id)
expect(maybeRefreshSpy).toHaveBeenCalledTimes(1)
expect(tasksReload).toHaveBeenCalledTimes(1)
expect(success).toHaveBeenCalledWith('Bulk operation completed.')
fixture.destroy()
bulkEvents.next({
task_id: 'bulk-task',
operation: 'set_permissions',
status: PaperlessTaskStatus.Success,
})
expect(reload).toHaveBeenCalledTimes(1)
})
it('refreshes and reports failed or revoked bulk operations', () => {
const list = TestBed.inject(DocumentListViewService)
const reload = jest.spyOn(list, 'reload').mockImplementation(() => {})
jest.spyOn(list, 'reduceSelectionToFilter').mockImplementation(() => {})
jest.spyOn(tasksService, 'reload').mockImplementation(() => {})
const error = jest.spyOn(toastService, 'showError')
const success = jest.spyOn(toastService, 'showSuccess')
for (const status of [
PaperlessTaskStatus.Failure,
PaperlessTaskStatus.Revoked,
]) {
bulkEvents.next({
task_id: 'bulk-task',
operation: 'set_permissions',
status,
error: 'Could not update',
})
}
expect(reload).toHaveBeenCalledTimes(2)
expect(error).toHaveBeenCalledWith(expect.any(String), 'Could not update')
expect(success).not.toHaveBeenCalled()
})
it('should initialize the saved view service', () => {
expect(savedViewSpy).toHaveBeenCalled()
})
@@ -8,12 +8,14 @@ import {
import { NgClass } from '@angular/common'
import {
Component,
DestroyRef,
HostListener,
inject,
OnInit,
signal,
viewChild,
} from '@angular/core'
import { takeUntilDestroyed } from '@angular/core/rxjs-interop'
import { FormsModule } from '@angular/forms'
import { ActivatedRoute, Router, RouterModule } from '@angular/router'
import {
@@ -24,10 +26,11 @@ import {
NgbPopoverModule,
} from '@ng-bootstrap/ng-bootstrap'
import { TourNgBootstrap } from 'ngx-ui-tour-ng-bootstrap'
import { Observable } from 'rxjs'
import { filter, Observable } from 'rxjs'
import { first } from 'rxjs/operators'
import { BootstrapIconComponent } from 'src/app/components/common/bootstrap-icon/bootstrap-icon.component'
import { Document } from 'src/app/data/document'
import { PaperlessTaskStatus } from 'src/app/data/paperless-task'
import { SavedView } from 'src/app/data/saved-view'
import {
CollapsibleSection,
@@ -41,6 +44,7 @@ import {
DjangoMessageLevel,
DjangoMessagesService,
} from 'src/app/services/django-messages.service'
import { DocumentListViewService } from 'src/app/services/document-list-view.service'
import { OpenDocumentsService } from 'src/app/services/open-documents.service'
import {
PermissionAction,
@@ -101,6 +105,8 @@ export class AppFrameComponent
extends ComponentWithPermissions
implements OnInit, ComponentCanDeactivate
{
private destroyRef = inject(DestroyRef)
private documentList = inject(DocumentListViewService)
router = inject(Router)
private activatedRoute = inject(ActivatedRoute)
private openDocumentsService = inject(OpenDocumentsService)
@@ -164,6 +170,44 @@ export class AppFrameComponent
}
ngOnInit(): void {
this.websocketStatusService
.onBulkOperationStatus()
.pipe(
filter(({ status }) =>
[
PaperlessTaskStatus.Success,
PaperlessTaskStatus.Failure,
PaperlessTaskStatus.Revoked,
].includes(status)
),
takeUntilDestroyed(this.destroyRef)
)
.subscribe(({ status, error }) => {
this.documentList.reload(null, false)
this.documentList.reduceSelectionToFilter()
// maybe someday we can optimize to only refresh affected documents
this.openDocumentsService.getOpenDocuments().forEach((doc) => {
this.openDocumentsService.refreshDocument(doc.id)
})
this.savedViewService.maybeRefreshDocumentCounts()
if (
this.permissionsService.currentUserCan(
PermissionAction.View,
PermissionType.PaperlessTask
)
) {
this.tasksService.reload()
}
if (status === PaperlessTaskStatus.Success) {
this.toastService.showSuccess($localize`Bulk operation completed.`)
} else {
this.toastService.showError(
$localize`Bulk operation failed. Some changes may already have been applied.`,
error
)
}
})
this.lastScrollY = window.scrollY
this.detectClassicScrollbars()
@@ -559,18 +559,19 @@ describe('BulkEditorComponent', () => {
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
req.flush({ result: 'OK', task_id: 'queued-task' })
expect(req.request.body).toEqual({
from_webui: true,
documents: [3, 4],
method: 'modify_tags',
parameters: { add_tags: [101], remove_tags: [] },
})
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should execute modify tags bulk operation for all filtered documents', () => {
@@ -607,17 +608,18 @@ describe('BulkEditorComponent', () => {
const req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
req.flush({ result: 'OK', task_id: 'queued-task' })
expect(req.request.body).toEqual({
from_webui: true,
all: true,
filters: { title_search: 'apple' },
excluded_documents: [4],
method: 'modify_tags',
parameters: { add_tags: [101], remove_tags: [] },
})
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
)
})
it('should execute modify tags bulk operation with confirmation dialog if enabled', () => {
@@ -644,12 +646,12 @@ describe('BulkEditorComponent', () => {
httpTestingController
.expectOne(`${environment.apiBaseUrl}documents/bulk_edit/`)
.flush(true)
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should set modal dialog text accordingly for tag edit confirmation', () => {
@@ -729,18 +731,19 @@ describe('BulkEditorComponent', () => {
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
req.flush({ result: 'OK', task_id: 'queued-task' })
expect(req.request.body).toEqual({
from_webui: true,
documents: [3, 4],
method: 'set_correspondent',
parameters: { correspondent: 101 },
})
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should execute modify correspondent bulk operation with confirmation dialog if enabled', () => {
@@ -767,12 +770,12 @@ describe('BulkEditorComponent', () => {
httpTestingController
.expectOne(`${environment.apiBaseUrl}documents/bulk_edit/`)
.flush(true)
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should set modal dialog text accordingly for correspondent edit confirmation', () => {
@@ -827,18 +830,19 @@ describe('BulkEditorComponent', () => {
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
req.flush({ result: 'OK', task_id: 'queued-task' })
expect(req.request.body).toEqual({
from_webui: true,
documents: [3, 4],
method: 'set_document_type',
parameters: { document_type: 101 },
})
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should execute modify document type bulk operation with confirmation dialog if enabled', () => {
@@ -865,12 +869,12 @@ describe('BulkEditorComponent', () => {
httpTestingController
.expectOne(`${environment.apiBaseUrl}documents/bulk_edit/`)
.flush(true)
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should set modal dialog text accordingly for document type edit confirmation', () => {
@@ -925,18 +929,19 @@ describe('BulkEditorComponent', () => {
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
req.flush({ result: 'OK', task_id: 'queued-task' })
expect(req.request.body).toEqual({
from_webui: true,
documents: [3, 4],
method: 'set_storage_path',
parameters: { storage_path: 101 },
})
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should execute modify storage path bulk operation with confirmation dialog if enabled', () => {
@@ -963,12 +968,12 @@ describe('BulkEditorComponent', () => {
httpTestingController
.expectOne(`${environment.apiBaseUrl}documents/bulk_edit/`)
.flush(true)
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should set modal dialog text accordingly for storage path edit confirmation', () => {
@@ -1023,18 +1028,19 @@ describe('BulkEditorComponent', () => {
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
req.flush({ result: 'OK', task_id: 'queued-task' })
expect(req.request.body).toEqual({
from_webui: true,
documents: [3, 4],
method: 'modify_custom_fields',
parameters: { add_custom_fields: [101], remove_custom_fields: [102] },
})
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should execute modify custom fields bulk operation with confirmation dialog if enabled', () => {
@@ -1061,12 +1067,12 @@ describe('BulkEditorComponent', () => {
httpTestingController
.expectOne(`${environment.apiBaseUrl}documents/bulk_edit/`)
.flush(true)
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
// coverage for modal messages
component.setCustomFields({
@@ -1540,6 +1546,47 @@ describe('BulkEditorComponent', () => {
)
})
it.each([true, false])(
'shows queued feedback and gates the task link on permission (%s)',
(canViewTasks) => {
jest
.spyOn(permissionsService, 'currentUserCan')
.mockReturnValue(canViewTasks)
const queued = jest.spyOn(toastService, 'show')
let modal: NgbModalRef
modalService.activeInstances.subscribe((instances) => {
if (instances.length) modal = instances[0]
})
const reload = jest.spyOn(documentListViewService, 'reload')
component.setPermissions()
const close = jest.spyOn(modal, 'close')
modal.componentInstance.confirmClicked.emit({
permissions: {},
merge: true,
})
const req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush({ result: 'OK', task_id: 'queued-operation' })
expect(close).toHaveBeenCalledTimes(1)
const notification = queued.mock.calls.at(-1)[0]
expect(notification.content).toBe('Bulk operation queued.')
if (canViewTasks) {
expect(notification.actionName).toBe('View task')
notification.action()
expect(router.navigate).toHaveBeenCalledWith(['/tasks'], {
queryParams: { task_id: 'queued-operation' },
})
} else {
expect(notification.action).toBeUndefined()
expect(notification.actionName).toBeUndefined()
}
expect(reload).not.toHaveBeenCalled()
expect(documentListViewService.selected.size).toBe(0)
expect(documentListViewService.allSelected).toBe(false)
}
)
it('should support bulk permissions update', () => {
let modal: NgbModalRef
modalService.activeInstances.subscribe((m) => (modal = m[0]))
@@ -1572,8 +1619,9 @@ describe('BulkEditorComponent', () => {
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
req.flush({ result: 'OK', task_id: 'queued-task' })
expect(req.request.body).toEqual({
from_webui: true,
documents: [3, 4],
method: 'set_permissions',
parameters: {
@@ -1581,12 +1629,12 @@ describe('BulkEditorComponent', () => {
merge: true,
},
})
httpTestingController.match(
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should not attempt to retrieve objects if user does not have permissions', () => {
@@ -1887,15 +1935,18 @@ describe('BulkEditorComponent', () => {
expect(toastServiceShowErrorSpy).toHaveBeenCalled()
expect(listReloadSpy).not.toHaveBeenCalled()
modal.componentInstance.succeeded.emit()
expect(toastServiceShowInfoSpy).toHaveBeenCalled()
expect(listReloadSpy).toHaveBeenCalled()
httpTestingController.match(
modal.componentInstance.succeeded.emit({
result: 'OK',
task_id: 'custom-fields-task',
})
expect(toastServiceShowInfoSpy).not.toHaveBeenCalled()
expect(listReloadSpy).not.toHaveBeenCalled()
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
) // list reload
httpTestingController.match(
)
httpTestingController.expectNone(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
)
})
it('should create share link bundle and enable manage callback', () => {
@@ -104,11 +104,11 @@ export class BulkEditorComponent
private correspondentService = inject(CorrespondentService)
list = inject(DocumentListViewService)
private documentService = inject(DocumentService)
private trashService = inject(TrashService)
private modalService = inject(NgbModal)
private openDocumentService = inject(OpenDocumentsService)
private settings = inject(SettingsService)
private toastService = inject(ToastService)
private trashService = inject(TrashService)
private storagePathService = inject(StoragePathService)
private customFieldService = inject(CustomFieldsService)
public readonly permissionService = inject(PermissionsService)
@@ -299,11 +299,33 @@ export class BulkEditorComponent
.bulkEdit(overrideSelection ?? this.getSelectionQuery(), method, args)
.pipe(first())
.subscribe({
next: () => this.handleOperationSuccess(modal),
next: ({ task_id }) => this.handleOperationQueued(modal, task_id),
error: (error) => this.handleOperationError(modal, error),
})
}
private handleOperationQueued(modal: NgbModalRef, taskId: string) {
modal?.close()
this.list.selectNone()
const canViewTasks = this.permissionService.currentUserCan(
PermissionAction.View,
PermissionType.PaperlessTask
)
this.toastService.show({
content: $localize`Bulk operation queued.`,
delay: 5000,
...(canViewTasks
? {
actionName: $localize`View task`,
action: () =>
this.router.navigate(['/tasks'], {
queryParams: { task_id: taskId },
}),
}
: {}),
})
}
private executeDocumentAction(
modal: NgbModalRef,
request: Observable<any>,
@@ -1088,14 +1110,9 @@ export class BulkEditorComponent
dialog.selection = this.getSelectionQuery()
dialog.selectionCount = this.getSelectionSize()
dialog.succeeded.subscribe((result) => {
this.toastService.showSuccess($localize`Custom fields updated.`)
this.list.reload()
this.list.reduceSelectionToFilter()
this.list.selected.forEach((id) => {
this.openDocumentService.refreshDocument(id)
})
})
dialog.succeeded.subscribe(({ task_id }) =>
this.handleOperationQueued(null, task_id)
)
dialog.failed.subscribe((error) => {
this.toastService.showError(
$localize`Error updating custom fields.`,
@@ -59,7 +59,7 @@ describe('CustomFieldsBulkEditDialogComponent', () => {
it('should emit succeeded event and close modal on successful save', () => {
const editSpy = jest
.spyOn(documentService, 'bulkEdit')
.mockReturnValue(of('Success'))
.mockReturnValue(of({ result: 'OK', task_id: 'custom-fields-task' }))
const successSpy = jest.spyOn(component.succeeded, 'emit')
component.selection = [1, 2]
@@ -16,6 +16,7 @@ import { NumberComponent } from 'src/app/components/common/input/number/number.c
import { SelectComponent } from 'src/app/components/common/input/select/select.component'
import { TextComponent } from 'src/app/components/common/input/text/text.component'
import { UrlComponent } from 'src/app/components/common/input/url/url.component'
import { BulkOperationResponse } from 'src/app/data/bulk-operation'
import { CustomField, CustomFieldDataType } from 'src/app/data/custom-field'
import {
DocumentSelectionQuery,
@@ -49,7 +50,7 @@ export class CustomFieldsBulkEditDialogComponent {
CustomFieldDataType = CustomFieldDataType
@Output()
succeeded = new EventEmitter()
succeeded = new EventEmitter<BulkOperationResponse>()
@Output()
failed = new EventEmitter()
@@ -103,6 +104,8 @@ export class CustomFieldsBulkEditDialogComponent {
}
public save() {
if (this.networkActive) return
this.networkActive = true
this.documentService
.bulkEdit(this.selection, 'modify_custom_fields', {
add_custom_fields: this.form.value,
@@ -110,11 +113,12 @@ export class CustomFieldsBulkEditDialogComponent {
})
.pipe(first())
.subscribe({
next: () => {
next: (response) => {
this.activeModal.close()
this.succeeded.emit()
this.succeeded.emit(response)
},
error: (error) => {
this.networkActive = false
this.failed.emit(error)
},
})
+13
View File
@@ -0,0 +1,13 @@
import { PaperlessTaskStatus } from './paperless-task'
export interface BulkOperationResponse {
result: string
task_id: string
}
export interface BulkOperationStatus {
task_id: string
operation: string
status: PaperlessTaskStatus
error?: string
}
@@ -124,6 +124,42 @@ describe('DocumentListViewService', () => {
})
})
it.each([false, true])(
'preserves the current route on a background refresh (saved view: %s)',
(savedView) => {
if (savedView) documentListViewService.activateSavedView(view)
const navigate = jest.mocked(router.navigate)
navigate.mockClear()
documentListViewService.reload(null, false)
httpTestingController
.expectOne((req) => req.url === `${environment.apiBaseUrl}documents/`)
.flush(full_results)
expect(documentListViewService.documents).toEqual(full_results.results)
expect(navigate).not.toHaveBeenCalled()
}
)
it('preserves the route when a background refresh falls back to the first page', () => {
jest
.spyOn(documentListViewService, 'saveDocumentListView')
.mockImplementation(() => {})
documentListViewService.currentPage = 2
httpTestingController
.expectOne((req) => req.url === `${environment.apiBaseUrl}documents/`)
.flush(full_results)
const navigate = jest.mocked(router.navigate)
navigate.mockClear()
documentListViewService.reload(null, false)
httpTestingController
.expectOne((req) => req.url === `${environment.apiBaseUrl}documents/`)
.flush('Not found', { status: 404, statusText: 'Not found' })
httpTestingController
.expectOne((req) => req.url === `${environment.apiBaseUrl}documents/`)
.flush(full_results)
expect(documentListViewService.currentPage).toBe(1)
expect(navigate).not.toHaveBeenCalled()
})
it('should reload the list', () => {
expect(documentListViewService.currentPage).toEqual(1)
documentListViewService.reload()
@@ -380,7 +380,7 @@ export class DocumentListViewService {
queryParams: paramsFromViewState(activeListViewState),
replaceUrl: !this.router.routerState.snapshot.url.includes('?'), // in case navigating from params-less /documents
})
} else if (this._activeSavedViewId) {
} else if (updateQueryParams && this._activeSavedViewId) {
this.router.navigate([], {
queryParams: paramsFromViewState(activeListViewState, true),
queryParamsHandling: 'merge',
@@ -399,7 +399,7 @@ export class DocumentListViewService {
// this happens when applying a filter: the current page might not be available anymore due to the reduced result set.
activeListViewState.currentPage = 1
this.markChanged()
this.reload()
this.reload(onFinish, updateQueryParams)
} else if (
activeListViewState.sortField.indexOf('custom_field') === 0 &&
!this.settings
@@ -248,6 +248,7 @@ describe(`DocumentService`, () => {
)
expect(req.request.method).toEqual('POST')
expect(req.request.body).toEqual({
from_webui: true,
documents: ids,
method,
parameters,
@@ -271,6 +272,7 @@ describe(`DocumentService`, () => {
)
expect(req.request.method).toEqual('POST')
expect(req.request.body).toEqual({
from_webui: true,
all: true,
filters: { title__icontains: 'apple' },
excluded_documents: [2, 3],
@@ -14,6 +14,7 @@ import { DocumentSuggestions } from 'src/app/data/document-suggestions'
import { FilterRule } from 'src/app/data/filter-rule'
import { Results, SelectionData } from 'src/app/data/results'
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
import { BulkOperationResponse } from '../../data/bulk-operation'
import { queryParamsFromFilterRules } from '../../utils/query-params'
import {
PermissionAction,
@@ -337,11 +338,15 @@ export class DocumentService extends AbstractPaperlessService<Document> {
method: DocumentBulkEditMethod,
args: any
) {
return this.http.post(this.getResourceUrl(null, 'bulk_edit'), {
...selection,
method: method,
parameters: args,
})
return this.http.post<BulkOperationResponse>(
this.getResourceUrl(null, 'bulk_edit'),
{
...selection,
method: method,
from_webui: true,
parameters: args,
}
)
}
deleteDocuments(selection: DocumentSelectionQuery) {
@@ -438,6 +438,20 @@ describe('ConsumerStatusService', () => {
expect(updated).toBeTruthy()
})
it('forwards bulk operation completion events', () => {
const received = jest.fn()
websocketStatusService.onBulkOperationStatus().subscribe(received)
const data = {
task_id: 'bulk-task',
operation: 'set_permissions',
status: 'success',
}
websocketStatusService.connect()
server.send({ type: WebsocketStatusType.BULK_OPERATION_STATUS, data })
websocketStatusService.disconnect()
expect(received).toHaveBeenCalledWith(data)
})
it('should ignore keep-alive heartbeat messages from the server', () => {
let updated = false
let deleted = false
@@ -1,6 +1,7 @@
import { Injectable, inject, signal } from '@angular/core'
import { Subject } from 'rxjs'
import { environment } from 'src/environments/environment'
import { BulkOperationStatus } from '../data/bulk-operation'
import { User } from '../data/user'
import { WebsocketDocumentUpdatedMessage } from '../data/websocket-document-updated-message'
import { WebsocketDocumentsDeletedMessage } from '../data/websocket-documents-deleted-message'
@@ -12,6 +13,7 @@ export enum WebsocketStatusType {
DOCUMENTS_DELETED = 'documents_deleted',
DOCUMENT_UPDATED = 'document_updated',
HEARTBEAT = 'heartbeat',
BULK_OPERATION_STATUS = 'bulk_operation_status',
}
// see ProgressStatusOptions in src/documents/plugins/helpers.py
@@ -116,6 +118,13 @@ export class WebsocketStatusService {
private readonly documentDeletedSubject = new Subject<boolean>()
private readonly documentUpdatedSubject =
new Subject<WebsocketDocumentUpdatedMessage>()
private readonly bulkOperationStatusSubject =
new Subject<BulkOperationStatus>()
onBulkOperationStatus() {
return this.bulkOperationStatusSubject.asObservable()
}
private readonly connectionStatusSubject = new Subject<boolean>()
private get(taskId: string, filename?: string) {
@@ -192,6 +201,7 @@ export class WebsocketStatusService {
| WebsocketProgressMessage
| WebsocketDocumentsDeletedMessage
| WebsocketDocumentUpdatedMessage
| BulkOperationStatus
} = JSON.parse(ev.data)
switch (type) {
@@ -209,6 +219,12 @@ export class WebsocketStatusService {
this.handleProgressUpdate(messageData as WebsocketProgressMessage)
break
case WebsocketStatusType.BULK_OPERATION_STATUS:
this.bulkOperationStatusSubject.next(
messageData as BulkOperationStatus
)
break
case WebsocketStatusType.HEARTBEAT:
// keep-alive from the server, see paperless.consumers.StatusConsumer
break
+1 -2
View File
@@ -1,7 +1,6 @@
from django.conf import settings
from django.contrib import admin
from guardian.admin import GuardedModelAdmin
from treenode.admin import TreeNodeModelAdmin
from documents.models import Correspondent
from documents.models import CustomField
@@ -29,7 +28,7 @@ class CorrespondentAdmin(GuardedModelAdmin):
list_editable = ("match", "matching_algorithm")
class TagAdmin(GuardedModelAdmin, TreeNodeModelAdmin):
class TagAdmin(GuardedModelAdmin):
list_display = ("name", "color", "match", "matching_algorithm")
list_filter = ("matching_algorithm",)
list_editable = ("color", "match", "matching_algorithm")
+60 -49
View File
@@ -48,6 +48,7 @@ from documents.versioning import get_root_document
from documents.versioning import versions_newest_first
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Mapping
from django.contrib.auth.models import User
@@ -131,9 +132,24 @@ def _scratch_path(name: str) -> Path:
return Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) / name
def _finish_metadata_edit(
document_ids: list[int],
update_callback: Callable[[list[int]], None] | None,
) -> None:
if update_callback is not None:
update_callback(document_ids)
else:
bulk_update_documents.apply_async(
kwargs={"document_ids": document_ids},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
def set_correspondent(
doc_ids: list[int],
correspondent: Correspondent,
*,
update_callback: Callable[[list[int]], None] | None = None,
) -> Literal["OK"]:
if correspondent:
correspondent = Correspondent.objects.only("pk").get(id=correspondent)
@@ -146,15 +162,17 @@ def set_correspondent(
affected_docs = list(qs.values_list("pk", flat=True))
qs.update(correspondent=correspondent)
bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
_finish_metadata_edit(affected_docs, update_callback)
return "OK"
def set_storage_path(doc_ids: list[int], storage_path: StoragePath) -> Literal["OK"]:
def set_storage_path(
doc_ids: list[int],
storage_path: StoragePath,
*,
update_callback: Callable[[list[int]], None] | None = None,
) -> Literal["OK"]:
if storage_path:
storage_path = StoragePath.objects.only("pk").get(id=storage_path)
@@ -168,15 +186,17 @@ def set_storage_path(doc_ids: list[int], storage_path: StoragePath) -> Literal["
affected_docs = list(qs.values_list("pk", flat=True))
qs.update(storage_path=storage_path)
bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
_finish_metadata_edit(affected_docs, update_callback)
return "OK"
def set_document_type(doc_ids: list[int], document_type: DocumentType) -> Literal["OK"]:
def set_document_type(
doc_ids: list[int],
document_type: DocumentType,
*,
update_callback: Callable[[list[int]], None] | None = None,
) -> Literal["OK"]:
if document_type:
document_type = DocumentType.objects.only("pk").get(id=document_type)
@@ -188,15 +208,17 @@ def set_document_type(doc_ids: list[int], document_type: DocumentType) -> Litera
affected_docs = list(qs.values_list("pk", flat=True))
qs.update(document_type=document_type)
bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
_finish_metadata_edit(affected_docs, update_callback)
return "OK"
def add_tag(doc_ids: list[int], tag: int) -> Literal["OK"]:
def add_tag(
doc_ids: list[int],
tag: int,
*,
update_callback: Callable[[list[int]], None] | None = None,
) -> Literal["OK"]:
tag_obj = Tag.objects.get(pk=tag)
tags_to_add = [tag_obj, *tag_obj.get_ancestors()]
@@ -217,15 +239,17 @@ def add_tag(doc_ids: list[int], tag: int) -> Literal["OK"]:
DocumentTagRelationship.objects.bulk_create(to_create)
if affected_docs:
bulk_update_documents.apply_async(
kwargs={"document_ids": list(affected_docs)},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
_finish_metadata_edit(list(affected_docs), update_callback)
return "OK"
def remove_tag(doc_ids: list[int], tag: int) -> Literal["OK"]:
def remove_tag(
doc_ids: list[int],
tag: int,
*,
update_callback: Callable[[list[int]], None] | None = None,
) -> Literal["OK"]:
tag_obj = Tag.objects.get(pk=tag)
tag_ids = [tag_obj.id, *tag_obj.get_descendants_pks()]
@@ -238,10 +262,7 @@ def remove_tag(doc_ids: list[int], tag: int) -> Literal["OK"]:
qs.delete()
if affected_docs:
bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
_finish_metadata_edit(affected_docs, update_callback)
return "OK"
@@ -250,24 +271,20 @@ def modify_tags(
doc_ids: list[int],
add_tags: list[int],
remove_tags: list[int],
*,
update_callback: Callable[[list[int]], None] | None = None,
) -> Literal["OK"]:
qs = Document.objects.filter(id__in=doc_ids).only("pk")
affected_docs = list(qs.values_list("pk", flat=True))
DocumentTagRelationship = Document.tags.through
# add with all ancestors
expanded_add_tags: set[int] = set()
add_tag_objects = Tag.objects.filter(pk__in=add_tags)
for t in add_tag_objects:
expanded_add_tags.add(int(t.id))
expanded_add_tags.update(int(pk) for pk in t.get_ancestors_pks())
# remove with all descendants
expanded_remove_tags: set[int] = set()
remove_tag_objects = Tag.objects.filter(pk__in=remove_tags)
for t in remove_tag_objects:
expanded_remove_tags.add(int(t.id))
expanded_remove_tags.update(int(pk) for pk in t.get_descendants_pks())
hierarchy = Tag.get_hierarchy()
expanded_add_tags = hierarchy.with_ancestors(
Tag.objects.filter(pk__in=add_tags).values_list("pk", flat=True),
)
expanded_remove_tags = hierarchy.with_descendants(
Tag.objects.filter(pk__in=remove_tags).values_list("pk", flat=True),
)
with transaction.atomic():
if expanded_remove_tags:
@@ -299,10 +316,7 @@ def modify_tags(
)
if affected_docs:
bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
_finish_metadata_edit(affected_docs, update_callback)
return "OK"
@@ -311,6 +325,8 @@ def modify_custom_fields(
doc_ids: list[int],
add_custom_fields: list[int] | dict,
remove_custom_fields: list[int],
*,
update_callback: Callable[[list[int]], None] | None = None,
) -> Literal["OK"]:
qs = Document.objects.filter(id__in=doc_ids).only("pk")
affected_docs = list(qs.values_list("pk", flat=True))
@@ -373,10 +389,7 @@ def modify_custom_fields(
field_id__in=remove_custom_fields,
).hard_delete()
bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
_finish_metadata_edit(affected_docs, update_callback)
return "OK"
@@ -457,6 +470,7 @@ def set_permissions(
*,
owner: User | None = None,
merge: bool = False,
update_callback: Callable[[list[int]], None] | None = None,
) -> Literal["OK"]:
qs = Document.objects.filter(id__in=doc_ids).select_related("owner")
@@ -474,10 +488,7 @@ def set_permissions(
merge=merge,
)
bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
_finish_metadata_edit(affected_docs, update_callback)
return "OK"
+107
View File
@@ -0,0 +1,107 @@
"""Queued document metadata edits."""
from typing import Any
from auditlog.context import set_actor
from celery import shared_task
from django.conf import settings
from django.contrib.auth.models import User
from django.db import transaction
from rest_framework.exceptions import PermissionDenied
from documents import bulk_edit
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.permissions import has_document_operation_permissions
from documents.tasks import bulk_update_documents
MODIFIED_FIELDS = {
"set_correspondent": "correspondent",
"set_document_type": "document_type",
"set_storage_path": "storage_path",
"add_tag": "tags",
"remove_tag": "tags",
"modify_tags": "tags",
"modify_custom_fields": "custom_fields",
"set_permissions": None,
}
QUEUED_OPERATIONS = frozenset(MODIFIED_FIELDS)
def snapshot_field(doc_ids: list[int], field: str) -> dict[int, Any]:
"""
Returns each document's current value of field, for the audit log.
Tags and custom fields are one row per value, so they are gathered
into a sorted list of pks per document (empty when there are none).
Reading them through Document.values() instead would join those rows
and return one arbitrary value per document.
"""
if field == "tags":
rows = (
Document.tags.through.objects.filter(document_id__in=doc_ids)
.order_by("tag_id")
.values_list("document_id", "tag_id")
)
elif field == "custom_fields":
rows = (
CustomFieldInstance.objects.filter(document_id__in=doc_ids)
.order_by("pk")
.values_list("document_id", "pk")
)
else:
return dict(
Document.objects.filter(pk__in=doc_ids).values_list("pk", field),
)
values: dict[int, list[int]] = {doc_id: [] for doc_id in doc_ids}
for doc_id, pk in rows:
values[doc_id].append(pk)
return values
@shared_task
def execute_bulk_operation(
operation: str,
document_ids: list[int],
parameters: dict,
user_id: int,
) -> dict:
if operation not in QUEUED_OPERATIONS:
raise ValueError(f"Unsupported bulk operation: {operation}")
method = getattr(bulk_edit, operation)
user = User.objects.get(pk=user_id)
if not has_document_operation_permissions(
user=user,
documents=document_ids,
method=method,
parameters=parameters,
):
raise PermissionDenied("Insufficient permissions to execute bulk operation")
affected_ids: list[int] = []
modified_field = MODIFIED_FIELDS[operation]
# Keep metadata writes and their audit entries together. Follow-up processing
# runs after commit, within this same tracked task.
with set_actor(user), transaction.atomic():
old_values = (
snapshot_field(document_ids, modified_field)
if settings.AUDIT_LOG_ENABLED and modified_field
else {}
)
method(document_ids, **parameters, update_callback=affected_ids.extend)
if settings.AUDIT_LOG_ENABLED and modified_field:
from auditlog.models import LogEntry
new_values = snapshot_field(document_ids, modified_field)
for doc in Document.objects.filter(pk__in=document_ids):
LogEntry.objects.log_create(
instance=doc,
changes={modified_field: [old_values[doc.pk], new_values[doc.pk]]},
action=LogEntry.Action.UPDATE,
actor=user,
additional_data={"reason": f"Bulk edit: {operation}"},
)
if affected_ids:
bulk_update_documents(list(dict.fromkeys(affected_ids)))
return {"operation": operation, "document_count": len(document_ids)}
@@ -108,6 +108,24 @@ def _deserialize_record(
) from e
for field_name, field_value in record.get("fields", {}).items():
# Older exports include django-treenode's derived bookkeeping. The
# parent FK is authoritative; accept only these known removed fields.
if Model is Tag and field_name in {
"tn_ancestors_count",
"tn_ancestors_pks",
"tn_children_count",
"tn_children_pks",
"tn_depth",
"tn_descendants_count",
"tn_descendants_pks",
"tn_index",
"tn_level",
"tn_order",
"tn_priority",
"tn_siblings_count",
"tn_siblings_pks",
}:
continue
field = Model._meta.get_field(field_name)
remote = field.remote_field
@@ -0,0 +1,64 @@
# Generated by Django 5.2.17 on 2026-10-28 22:54
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("documents", "0027_documentbarcode"),
]
operations = [
migrations.RemoveField(
model_name="tag",
name="tn_ancestors_count",
),
migrations.RemoveField(
model_name="tag",
name="tn_ancestors_pks",
),
migrations.RemoveField(
model_name="tag",
name="tn_children_count",
),
migrations.RemoveField(
model_name="tag",
name="tn_children_pks",
),
migrations.RemoveField(
model_name="tag",
name="tn_depth",
),
migrations.RemoveField(
model_name="tag",
name="tn_descendants_count",
),
migrations.RemoveField(
model_name="tag",
name="tn_descendants_pks",
),
migrations.RemoveField(
model_name="tag",
name="tn_index",
),
migrations.RemoveField(
model_name="tag",
name="tn_level",
),
migrations.RemoveField(
model_name="tag",
name="tn_order",
),
migrations.RemoveField(
model_name="tag",
name="tn_priority",
),
migrations.RemoveField(
model_name="tag",
name="tn_siblings_count",
),
migrations.RemoveField(
model_name="tag",
name="tn_siblings_pks",
),
]
+78 -23
View File
@@ -1,6 +1,7 @@
import datetime
import uuid
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Final
import pathvalidate
@@ -14,7 +15,6 @@ from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from multiselectfield import MultiSelectField
from treenode.models import TreeNodeModel
if settings.AUDIT_LOG_ENABLED:
from auditlog.registry import auditlog
@@ -29,6 +29,9 @@ from django_softdelete.models import SoftDeleteModel
from documents.data_models import DocumentSource
from documents.parsers import get_default_file_extension
if TYPE_CHECKING:
from documents.tag_hierarchy import TagHierarchy
class ModelWithOwner(models.Model):
owner = models.ForeignKey(
@@ -100,7 +103,17 @@ class Correspondent(MatchingModel):
verbose_name_plural = _("correspondents")
class Tag(MatchingModel, TreeNodeModel):
class Tag(MatchingModel):
# Keep the existing column and reverse relation during the transition.
tn_parent = models.ForeignKey(
"self",
blank=True,
null=True,
on_delete=models.CASCADE,
related_name="tn_children",
verbose_name=_("Parent"),
)
color = models.CharField(_("color"), max_length=7, default="#a6cee3")
# Maximum allowed nesting depth for tags (root = 1, max depth = 5)
MAX_NESTING_DEPTH: Final[int] = 5
@@ -114,27 +127,72 @@ class Tag(MatchingModel, TreeNodeModel):
),
)
class Meta(MatchingModel.Meta, TreeNodeModel.Meta):
class Meta(MatchingModel.Meta):
verbose_name = _("tag")
verbose_name_plural = _("tags")
@classmethod
def get_hierarchy(cls) -> "TagHierarchy":
"""Read the tag hierarchy in one query, ready to use across multiple tags."""
from documents.tag_hierarchy import TagHierarchy
return TagHierarchy(cls.objects.values_list("pk", "tn_parent_id"))
@property
def parent_pk(self) -> int | None:
return self.tn_parent_id
def get_parent(self) -> "Tag | None":
return self.tn_parent
def get_ancestors_pks(self) -> list[int]:
"""Return ancestor IDs ordered from root to parent.
Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
reuse it when working with several tags.
"""
return self.get_hierarchy().ancestors(self.tn_parent_id)
def get_ancestors(self) -> list["Tag"]:
"""Return ancestors ordered from root to parent.
Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
reuse it when working with several tags.
"""
ids = self.get_ancestors_pks()
tags = self.__class__.objects.in_bulk(ids)
return [tags[pk] for pk in ids]
def get_descendants_pks(self) -> set[int]:
"""Return all descendant IDs.
Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
reuse it when working with several tags.
"""
return self.get_hierarchy().descendants([self.pk])
def get_descendants(self) -> "models.QuerySet[Tag]":
"""Return a queryset of all descendants.
Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
reuse it when working with several tags.
"""
return self.__class__.objects.filter(pk__in=self.get_descendants_pks())
def get_children_queryset(self) -> "models.QuerySet[Tag]":
return self.__class__.objects.filter(tn_parent_id=self.pk)
def clean(self) -> None:
# Prevent self-parenting and assigning a descendant as parent
parent = self.get_parent()
if parent == self:
raise ValidationError({"parent": _("Cannot set itself as parent.")})
if parent and self.pk is not None and self.is_ancestor_of(parent):
raise ValidationError({"parent": _("Cannot set parent to a descendant.")})
# Enforce maximum nesting depth
new_parent_depth = 0
if parent:
new_parent_depth = parent.get_ancestors_count() + 1
height = 0 if self.pk is None else self.get_depth()
deepest_new_depth = (new_parent_depth + 1) + height
if deepest_new_depth > self.MAX_NESTING_DEPTH:
raise ValidationError({"parent": _("Maximum nesting depth exceeded.")})
hierarchy = self.get_hierarchy()
if self.pk is not None and self.tn_parent_id == self.pk:
raise ValidationError({"tn_parent": _("Cannot set itself as parent.")})
ancestors = hierarchy.ancestors(self.tn_parent_id)
if self.pk is not None and self.pk in ancestors:
raise ValidationError(
{"tn_parent": _("Cannot set parent to a descendant.")},
)
if len(ancestors) + 1 + hierarchy.height(self.pk) > self.MAX_NESTING_DEPTH:
raise ValidationError({"tn_parent": _("Maximum nesting depth exceeded.")})
return super().clean()
@@ -539,10 +597,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
return self.created
def add_nested_tags(self, tags) -> None:
tag_ids = set()
for tag in tags:
tag_ids.add(tag.id)
tag_ids.update(tag.get_ancestors_pks())
tag_ids = Tag.get_hierarchy().with_ancestors(tag.pk for tag in tags)
tags_to_add = self.tags.model.objects.filter(id__in=tag_ids)
self.tags.add(*tags_to_add)
+112 -6
View File
@@ -31,6 +31,7 @@ from guardian.core import ObjectPermissionChecker
from guardian.models import GroupObjectPermission
from guardian.models import UserObjectPermission
from rest_framework.permissions import BasePermission
from rest_framework.permissions import DjangoModelPermissions
from rest_framework.permissions import DjangoObjectPermissions
from documents.models import Document
@@ -63,6 +64,17 @@ class PermissionGranteesUpdate(TypedDict, total=False):
PermissionsSpec: TypeAlias = Mapping[PermissionAction, PermissionGranteesUpdate]
class PaperlessModelPermissions(DjangoModelPermissions):
"""Require basic global model permissions."""
perms_map = {
**DjangoModelPermissions.perms_map,
"GET": ["%(app_label)s.view_%(model_name)s"],
"OPTIONS": ["%(app_label)s.view_%(model_name)s"],
"HEAD": ["%(app_label)s.view_%(model_name)s"],
}
class PaperlessObjectPermissions(DjangoObjectPermissions):
"""
A permissions backend that checks for object-level permissions
@@ -576,12 +588,9 @@ def annotate_document_count_by_ids(
or CustomFieldInstance)
related_object_field: field on the relation pointing back to queryset pk
document_ids: the document ids to count against -- a concrete list/set,
or a simple (already resolved) queryset of ids. Callers
that need this filtered by a complex condition (e.g. a
permission check) should resolve it to a concrete list
first if the same ids will be reused across multiple
calls, rather than passing the complex queryset itself
into each -- see ``_get_selection_data_for_queryset``.
or a simple (already resolved) queryset of ids. Keep
the list small: it is embedded in the statement as
literal parameters.
target_field: field on the relation pointing to Document id
"""
@@ -730,3 +739,100 @@ class AcknowledgeTasksPermissions(GlobalPermissionsByMethod):
"OPTIONS": ["documents.change_paperlesstask"],
"POST": ["documents.change_paperlesstask"],
}
def has_document_operation_permissions(
*,
user: User,
documents: list[int],
method,
parameters: dict[str, Any],
) -> bool:
from documents import bulk_edit
if not user.is_active:
return False
if user.is_superuser:
return True
root_docs = {
get_root_document(doc)
for doc in Document.objects.select_related(
"owner",
"root_document__owner",
).filter(pk__in=documents)
}
user_is_owner_of_all_documents = all(
(doc.owner == user or doc.owner is None) for doc in root_docs
)
# check global and object permissions for all documents
has_perms = (
user.has_perm(
"documents.change_document",
)
and not Document.global_objects.filter(pk__in=documents)
.exclude(pk__in=permitted_document_ids(user, perm="change_document"))
.exists()
)
# check ownership for methods that change original document
if (
(
has_perms
and method
in [
bulk_edit.set_permissions,
bulk_edit.delete,
bulk_edit.rotate,
bulk_edit.delete_pages,
bulk_edit.edit_pdf,
bulk_edit.merge_as_versions,
bulk_edit.remove_password,
]
)
or (
method in [bulk_edit.merge, bulk_edit.split]
and parameters.get("delete_originals")
)
or (method == bulk_edit.edit_pdf and parameters.get("update_document"))
):
has_perms = has_perms and user_is_owner_of_all_documents
# check global add permissions for methods that create documents
if (
has_perms
and (
method in [bulk_edit.split, bulk_edit.merge]
or (
method in [bulk_edit.edit_pdf, bulk_edit.remove_password]
and not parameters.get("update_document")
)
)
and not user.has_perm("documents.add_document")
):
has_perms = False
# check global delete permissions for methods that delete documents
if (
has_perms
and (
method == bulk_edit.delete
# Sources stop being documents of their own, and removing one
# again afterwards needs delete_document
or method == bulk_edit.merge_as_versions
or (
method in [bulk_edit.merge, bulk_edit.split]
and parameters.get("delete_originals")
)
or (
method in [bulk_edit.edit_pdf, bulk_edit.remove_password]
and parameters.get("delete_original")
and not parameters.get("update_document")
)
)
and not user.has_perm("documents.delete_document")
):
has_perms = False
return has_perms
+28 -50
View File
@@ -589,7 +589,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
text_color = serializers.SerializerMethodField()
# map to treenode's tn_parent
# Preserve the public API parent field.
parent = serializers.PrimaryKeyRelatedField(
queryset=Tag.objects.all(),
allow_null=True,
@@ -692,7 +692,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
self.instance.clean()
except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e)
raise e
raise serializers.ValidationError(
{"parent": e.message_dict["tn_parent"]},
) from e
finally:
self.instance.tn_parent = original_parent
else:
@@ -702,7 +704,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
temp.clean()
except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e)
raise e
raise serializers.ValidationError(
{"parent": e.message_dict["tn_parent"]},
) from e
return super().validate(attrs)
@@ -1198,61 +1202,35 @@ class DocumentSerializer(
doc_id,
)
if "tags" in validated_data:
# Respect tag hierarchy on updates:
# - Adding a child adds its ancestors
# - Removing a parent removes all its descendants
prev_tags = set(instance.tags.all())
requested_tags = set(validated_data["tags"])
hierarchy = Tag.get_hierarchy()
previous_ids = set(instance.tags.values_list("pk", flat=True))
requested_ids = {tag.pk for tag in validated_data["tags"]}
required_ids = hierarchy.with_ancestors(requested_ids - previous_ids)
blocked_ids = hierarchy.with_descendants(previous_ids - requested_ids)
blocked_ids.difference_update(required_ids)
final_ids = hierarchy.with_ancestors(requested_ids) - blocked_ids
validated_data["tags"] = list(Tag.objects.filter(pk__in=final_ids))
# Tags newly added in this update and the ancestors they require
added_tags = requested_tags - prev_tags
required_by_add_tags = set(added_tags)
for t in added_tags:
required_by_add_tags.update(t.get_ancestors())
# Tags being removed in this update and all descendants, except
# those required by a tag that is being added in this same update
removed_tags = prev_tags - requested_tags
blocked_tags = set(removed_tags)
for t in removed_tags:
blocked_tags.update(t.get_descendants())
blocked_tags.difference_update(required_by_add_tags)
# Add all parent tags
final_tags = set(requested_tags)
for t in requested_tags:
final_tags.update(t.get_ancestors())
# Drop removed parents and their descendants
final_tags.difference_update(blocked_tags)
validated_data["tags"] = list(final_tags)
if validated_data.get("remove_inbox_tags"):
current_tag_ids = {t.pk for t in instance.tags.all()}
hierarchy = Tag.get_hierarchy()
tags = (
validated_data["tags"]
if "tags" in validated_data
else list(instance.tags.all())
)
# Tags newly added in this update, plus their ancestors, are kept
keep_ids: set[int] = set()
for tag in tags:
if tag.pk not in current_tag_ids:
keep_ids.add(tag.pk)
keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
# Remove inbox tags and their descendants, except those being kept
remove_ids: set[int] = set()
for inbox_tag in (
current_ids = set(instance.tags.values_list("pk", flat=True))
keep_ids = hierarchy.with_ancestors(
tag.pk for tag in tags if tag.pk not in current_ids
)
inbox_ids = (
Tag.objects.filter(is_inbox_tag=True)
.exclude(pk__in=keep_ids)
.only("pk", "tn_descendants_pks")
):
remove_ids.add(inbox_tag.pk)
remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
.exclude(
pk__in=keep_ids,
)
.values_list("pk", flat=True)
)
remove_ids = hierarchy.with_descendants(inbox_ids) - keep_ids
validated_data["tags"] = [tag for tag in tags if tag.pk not in remove_ids]
if settings.AUDIT_LOG_ENABLED:
with set_actor(self.user):
+40 -3
View File
@@ -8,6 +8,7 @@ from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
from asgiref.sync import async_to_sync
from celery import shared_task
from celery.signals import before_task_publish
from celery.signals import task_failure
@@ -16,6 +17,7 @@ from celery.signals import task_prerun
from celery.signals import task_revoked
from celery.signals import worker_process_init
from celery.signals import worker_process_shutdown
from channels.layers import get_channel_layer
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
@@ -1064,6 +1066,7 @@ TRACKED_TASKS: dict[str, PaperlessTask.TaskType] = {
"documents.tasks.build_share_link_bundle": PaperlessTask.TaskType.BUILD_SHARE_LINK,
"documents.bulk_edit.delete": PaperlessTask.TaskType.BULK_DELETE,
"documents.tasks.apply_ai_suggestions": PaperlessTask.TaskType.APPLY_AI_SUGGESTIONS,
"documents.bulk_operations.execute_bulk_operation": PaperlessTask.TaskType.BULK_UPDATE,
}
_CELERY_STATE_TO_STATUS: dict[str, PaperlessTask.Status] = {
@@ -1081,8 +1084,8 @@ def _extract_input_data(
For consume_file tasks this includes the filename, MIME type, and any
non-null overrides from the DocumentMetadataOverrides object. For
mail_fetch tasks it captures the account_ids list. All other task
types store no input data and return {}.
mail_fetch tasks it captures the account_ids list. Bulk operations store
the operation name and document count, without copying the full selection.
"""
if task_type == PaperlessTask.TaskType.CONSUME_FILE:
input_doc = task_kwargs.get("input_doc")
@@ -1111,6 +1114,12 @@ def _extract_input_data(
data["overrides"] = override_dict
return data
if "operation" in task_kwargs and "user_id" in task_kwargs:
return {
"operation": task_kwargs["operation"],
"document_count": len(task_kwargs.get("document_ids", [])),
}
if task_type == PaperlessTask.TaskType.MAIL_FETCH:
account_ids = task_kwargs.get("account_ids")
if account_ids is not None:
@@ -1147,7 +1156,9 @@ def _extract_owner_id(
task_type: PaperlessTask.TaskType,
task_kwargs: dict,
) -> int | None:
"""Return the owner_id from consume_file overrides, or None for all other task types."""
"""Use the initiator for bulk operations and the document owner for consumption."""
if "operation" in task_kwargs:
return task_kwargs.get("user_id")
if task_type != PaperlessTask.TaskType.CONSUME_FILE:
return None
overrides = task_kwargs.get("overrides")
@@ -1228,6 +1239,29 @@ def task_prerun_handler(sender=None, task_id=None, task=None, **kwargs) -> None:
logger.exception("Setting PaperlessTask started failed")
def _send_bulk_operation_status(task_id: str) -> None:
"""Notify the initiator, task records still happen."""
try:
task = PaperlessTask.objects.filter(task_id=task_id).first()
if task is None or task.owner_id is None or "operation" not in task.input_data:
return
async_to_sync(get_channel_layer().group_send)(
f"bulk_operations_{task.owner_id}",
{
"type": "bulk_operation_status",
"data": {
"task_id": task.task_id,
"status": task.status,
"operation": task.input_data["operation"],
"error": (task.result_data or {}).get("error_message", ""),
},
},
)
except Exception:
# Just a notification failure
logger.exception("Sending bulk operation status failed")
@task_postrun.connect
def task_postrun_handler(
sender=None,
@@ -1285,6 +1319,7 @@ def task_postrun_handler(
changed_fields.append("status")
task_instance.save(update_fields=changed_fields)
_send_bulk_operation_status(task_id)
except Exception: # pragma: no cover
logger.exception("Updating PaperlessTask failed")
@@ -1350,6 +1385,7 @@ def task_failure_handler(
date_started - date_created
).total_seconds()
task_qs.update(**update_fields)
_send_bulk_operation_status(task_id)
except Exception: # pragma: no cover
logger.exception("Updating PaperlessTask on failure failed")
@@ -1393,6 +1429,7 @@ def task_revoked_handler(
"reason": (reason + " Check worker logs for details."),
},
)
_send_bulk_operation_status(task_id)
except Exception: # pragma: no cover
logger.exception("Updating PaperlessTask on revocation failed")
+61
View File
@@ -0,0 +1,61 @@
"""An operation-local snapshot of tag edges, with no persistent derived fields.
Never cache this globally: tasks, bulk updates and other processes can change
parents. Callers traversing multiple tags should share one snapshot.
"""
from collections import defaultdict
from collections.abc import Iterable
class TagHierarchy:
def __init__(self, edges: Iterable[tuple[int, int | None]]) -> None:
self.parents = dict(edges)
self.children: dict[int | None, list[int]] = defaultdict(list)
for pk, parent in self.parents.items():
self.children[parent].append(pk)
def ancestors(self, parent_id: int | None) -> list[int]:
result = []
seen = set()
while parent_id is not None and parent_id not in seen:
seen.add(parent_id)
result.append(parent_id)
parent_id = self.parents.get(parent_id)
return result[::-1]
def with_ancestors(self, roots: Iterable[int]) -> set[int]:
result = set(roots)
for pk in list(result):
result.update(self.ancestors(self.parents.get(pk)))
return result
def with_descendants(self, roots: Iterable[int]) -> set[int]:
roots = set(roots)
return roots | self.descendants(roots)
def descendants(self, roots: Iterable[int]) -> set[int]:
roots = {pk for pk in roots if pk is not None}
seen = set(roots)
pending = list(roots)
while pending:
for child in self.children.get(pending.pop(), ()):
if child not in seen:
seen.add(child)
pending.append(child)
return seen - roots
def height(self, pk: int | None) -> int:
if pk is None:
return 0
seen = {pk}
pending = [(pk, 0)]
height = 0
while pending:
parent, depth = pending.pop()
height = max(height, depth)
for child in self.children.get(parent, ()):
if child not in seen:
seen.add(child)
pending.append((child, depth + 1))
return height
@@ -15,6 +15,7 @@ from paperless.models import ColorConvertChoices
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_global
class TestApiAppConfig(DirectoriesMixin, APITestCase):
@@ -26,6 +27,67 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
user = UserFactory(username="temp_admin", superuser=True)
self.client.force_authenticate(user=user)
def test_api_read_config_requires_view_permission(self) -> None:
for permissions in (
(),
("change_applicationconfiguration",),
("view_document",),
):
user = UserFactory()
grant_global(user, *permissions)
self.client.force_authenticate(user=user)
for endpoint in (self.ENDPOINT, f"{self.ENDPOINT}1/"):
for method in ("get", "head", "options"):
with self.subTest(
permissions=permissions,
endpoint=endpoint,
method=method,
):
response = getattr(self.client, method)(endpoint)
self.assertEqual(
response.status_code,
status.HTTP_403_FORBIDDEN,
)
def test_api_read_config_with_view_permission(self) -> None:
user = UserFactory()
grant_global(user, "view_applicationconfiguration")
self.client.force_authenticate(user=user)
for endpoint in (self.ENDPOINT, f"{self.ENDPOINT}1/"):
for method in ("get", "head", "options"):
with self.subTest(endpoint=endpoint, method=method):
response = getattr(self.client, method)(endpoint)
self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_api_view_config_permission_does_not_allow_writes(self) -> None:
user = UserFactory()
grant_global(user, "view_applicationconfiguration")
self.client.force_authenticate(user=user)
for method in ("put", "patch", "delete"):
with self.subTest(method=method):
response = getattr(self.client, method)(f"{self.ENDPOINT}1/")
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_api_change_config_permission_allows_updates(self) -> None:
user = UserFactory()
grant_global(user, "change_applicationconfiguration")
self.client.force_authenticate(user=user)
response = self.client.patch(
f"{self.ENDPOINT}1/",
{"app_title": "Updated title"},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
ApplicationConfiguration.objects.get().app_title,
"Updated title",
)
def test_api_get_config(self) -> None:
"""
GIVEN:
+87 -13
View File
@@ -7,6 +7,7 @@ from django.test import override_settings
from rest_framework import status
from rest_framework.test import APITestCase
from documents.bulk_operations import execute_bulk_operation
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
@@ -29,9 +30,18 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.user = user
self.client.force_authenticate(user=user)
patcher = mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
patcher = mock.patch("documents.bulk_operations.bulk_update_documents")
self.async_task = patcher.start()
self.addCleanup(patcher.stop)
# Exercise accepted requests and worker execution together in these
# operation/validation tests. Queue-only behavior is tested separately.
enqueue = mock.patch(
"documents.views.execute_bulk_operation.apply_async",
side_effect=lambda *, kwargs, headers: self.run_operation(kwargs),
)
enqueue.start()
self.addCleanup(enqueue.stop)
self.c1 = Correspondent.objects.create(name="c1")
self.c2 = Correspondent.objects.create(name="c2")
self.dt1 = DocumentType.objects.create(name="dt1")
@@ -61,11 +71,16 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.cf1 = CustomField.objects.create(name="cf1", data_type="string")
self.cf2 = CustomField.objects.create(name="cf2", data_type="string")
@staticmethod
def run_operation(kwargs):
execute_bulk_operation(**kwargs)
return mock.Mock(id="test-bulk-operation")
def setup_mock(self, m, method_name, return_value="OK") -> None:
m.return_value = return_value
m.__name__ = method_name
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.bulk_operations.bulk_update_documents")
def test_api_set_correspondent(self, bulk_update_task_mock) -> None:
self.assertNotEqual(self.doc1.correspondent, self.c1)
response = self.client.post(
@@ -84,11 +99,11 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(self.doc1.correspondent, self.c1)
bulk_update_task_mock.assert_called_once()
self.assertCountEqual(
bulk_update_task_mock.call_args.kwargs["kwargs"]["document_ids"],
bulk_update_task_mock.call_args.args[0],
[self.doc1.pk],
)
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.bulk_operations.bulk_update_documents")
def test_api_unset_correspondent(self, bulk_update_task_mock) -> None:
self.doc1.correspondent = self.c1
self.doc1.save()
@@ -110,7 +125,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.doc1.refresh_from_db()
self.assertIsNone(self.doc1.correspondent)
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.bulk_operations.bulk_update_documents")
def test_api_set_type(self, bulk_update_task_mock) -> None:
self.assertNotEqual(self.doc1.document_type, self.dt1)
response = self.client.post(
@@ -129,11 +144,11 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(self.doc1.document_type, self.dt1)
bulk_update_task_mock.assert_called_once()
self.assertCountEqual(
bulk_update_task_mock.call_args.kwargs["kwargs"]["document_ids"],
bulk_update_task_mock.call_args.args[0],
[self.doc1.pk],
)
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.bulk_operations.bulk_update_documents")
def test_api_unset_type(self, bulk_update_task_mock) -> None:
self.doc1.document_type = self.dt1
self.doc1.save()
@@ -154,11 +169,11 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertIsNone(self.doc1.document_type)
bulk_update_task_mock.assert_called_once()
self.assertCountEqual(
bulk_update_task_mock.call_args.kwargs["kwargs"]["document_ids"],
bulk_update_task_mock.call_args.args[0],
[self.doc1.pk],
)
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.bulk_operations.bulk_update_documents")
def test_api_add_tag(self, bulk_update_task_mock) -> None:
self.assertFalse(self.doc1.tags.filter(pk=self.t1.pk).exists())
@@ -180,11 +195,11 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
bulk_update_task_mock.assert_called_once()
self.assertCountEqual(
bulk_update_task_mock.call_args.kwargs["kwargs"]["document_ids"],
bulk_update_task_mock.call_args.args[0],
[self.doc1.pk],
)
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.bulk_operations.bulk_update_documents")
def test_api_remove_tag(self, bulk_update_task_mock) -> None:
self.doc1.tags.add(self.t1)
@@ -223,6 +238,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertListEqual(args[0], [self.doc1.id, self.doc3.id])
self.assertEqual(kwargs["add_tags"], [self.t1.id])
self.assertEqual(kwargs["remove_tags"], [self.t2.id])
@@ -277,6 +295,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertListEqual(args[0], [self.doc1.id, self.doc3.id])
self.assertEqual(kwargs["add_custom_fields"], [self.cf1.id])
self.assertEqual(kwargs["remove_custom_fields"], [self.cf2.id])
@@ -345,6 +366,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertListEqual(args[0], [self.doc1.id, self.doc3.id])
self.assertEqual(kwargs["add_custom_fields"], {self.cf1.id: "foo"})
self.assertEqual(kwargs["remove_custom_fields"], [self.cf2.id])
@@ -506,6 +530,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertEqual(args[0], [self.doc1.id])
self.assertEqual(len(kwargs), 0)
@@ -520,6 +547,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertEqual(args[0], [self.doc1.id])
self.assertEqual(len(kwargs), 0)
@@ -544,6 +574,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertEqual(args[0], [self.doc1.id])
self.assertEqual(len(kwargs), 0)
@@ -558,6 +591,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertEqual(args[0], [self.doc1.id])
self.assertEqual(kwargs, {"remote_ocr": False})
@@ -580,6 +616,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertEqual(args[0], [self.doc1.id])
self.assertEqual(kwargs, {"remote_ocr": True})
@@ -609,6 +648,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertListEqual(args[0], [self.doc1.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@@ -639,6 +681,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertListEqual(args[0], [self.doc1.id])
self.assertEqual(kwargs["storage_path"], None)
@@ -761,6 +806,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertEqual(args[0], [self.doc2.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@@ -784,6 +832,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertCountEqual(args[0], [self.doc1.id, self.doc3.id, self.doc5.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@@ -820,6 +871,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertCountEqual(args[0], [first_duplicate.id, second_duplicate.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@@ -856,6 +910,11 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
get_backend.return_value.search_ids.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value
for key, value in kwargs.items()
if key != "update_callback"
}
self.assertEqual(args[0], [self.doc2.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@@ -1141,8 +1200,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertCountEqual(
response.data["selected_document_types"],
[
{"id": self.c1.id, "document_count": 1},
{"id": self.c2.id, "document_count": 0},
{"id": self.dt1.id, "document_count": 1},
{"id": self.dt2.id, "document_count": 0},
],
)
@@ -1232,6 +1291,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
self.assertEqual(len(kwargs["set_permissions"]["view"]["users"]), 2)
@@ -1644,6 +1706,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
self.assertEqual(kwargs["degrees"], 90)
self.assertEqual(kwargs["source_mode"], "latest_version")
@@ -1930,6 +1995,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
self.assertEqual(kwargs["metadata_document_id"], self.doc3.id)
self.assertEqual(kwargs["source_mode"], "latest_version")
@@ -2137,6 +2205,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertCountEqual(args[0], [self.doc2.id])
self.assertEqual(kwargs["operations"], [{"page": 1}])
self.assertEqual(kwargs["source_mode"], "explicit_selection")
@@ -2482,6 +2553,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
kwargs = {
key: value for key, value in kwargs.items() if key != "update_callback"
}
self.assertCountEqual(args[0], [self.doc2.id])
self.assertEqual(kwargs["password"], "secret")
self.assertTrue(kwargs["update_document"])
+145
View File
@@ -49,6 +49,7 @@ from documents.models import WorkflowTrigger
from documents.search import get_backend
from documents.signals.handlers import run_workflows
from documents.tests.utils import ConsumeTaskMixin
from documents.views import DocumentViewSet
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import TagFactory
@@ -1755,12 +1756,113 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
}
self.assertEqual(counts_by_field[custom_field.id], 1)
def test_selection_data_counts_with_shared_by_filter(self) -> None:
"""
GIVEN:
- Several documents owned by one user with the same correspondent
and document type, only one of which is shared
WHEN:
- Requesting the document list filtered by shared_by__id with
include_selection_data=true
THEN:
- Counts reflect only the shared document, not every document the
owner has with that correspondent or document type, regardless of
the batch size
"""
owner = User.objects.create_user(username="owner")
other = User.objects.create_user(username="other")
correspondent = Correspondent.objects.create(name="c1")
doc_type = DocumentType.objects.create(name="dt1")
tag = Tag.objects.create(name="tag")
docs = [
Document.objects.create(
checksum=str(i),
owner=owner,
correspondent=correspondent,
document_type=doc_type,
)
for i in range(3)
]
for doc in docs:
doc.tags.add(tag)
grant_object(other, docs[0], "view_document")
for batch_size in (10_000, 1):
with (
self.subTest(batch_size=batch_size),
mock.patch.object(
DocumentViewSet,
"selection_data_batch_size",
batch_size,
),
):
response = self.client.get(
f"/api/documents/?shared_by__id={owner.id}"
"&include_selection_data=true",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["count"], 1)
selection_data = response.data["selection_data"]
for key, obj in (
("selected_correspondents", correspondent),
("selected_document_types", doc_type),
("selected_tags", tag),
):
counts = {
item["id"]: item["document_count"]
for item in selection_data[key]
}
self.assertEqual(counts[obj.id], 1, key)
def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
"""
GIVEN:
- A document matching a multi-valued filter through two tags
WHEN:
- Requesting the document list with include_selection_data=true
THEN:
- The document is counted once per correspondent, not once per
matching tag, regardless of the batch size
"""
correspondent = Correspondent.objects.create(name="c1")
tag_a = Tag.objects.create(name="a")
tag_b = Tag.objects.create(name="b")
both = Document.objects.create(checksum="1", correspondent=correspondent)
both.tags.add(tag_a, tag_b)
only_a = Document.objects.create(checksum="2", correspondent=correspondent)
only_a.tags.add(tag_a)
for batch_size in (10_000, 1):
with (
self.subTest(batch_size=batch_size),
mock.patch.object(
DocumentViewSet,
"selection_data_batch_size",
batch_size,
),
):
response = self.client.get(
f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
"&include_selection_data=true",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
counts = {
item["id"]: item["document_count"]
for item in response.data["selection_data"][
"selected_correspondents"
]
}
self.assertEqual(counts[correspondent.id], 2)
def test_statistics(self) -> None:
doc1 = Document.objects.create(
title="none1",
checksum="A",
mime_type="application/pdf",
content="abc",
archive_serial_number=42,
)
Document.objects.create(
title="none2",
@@ -1804,6 +1906,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(response.data["correspondent_count"], 2)
self.assertEqual(response.data["document_type_count"], 1)
self.assertEqual(response.data["storage_path_count"], 2)
self.assertEqual(response.data["current_asn"], 42)
def test_statistics_excludes_document_versions(self) -> None:
root = Document.objects.create(
@@ -1811,6 +1914,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
checksum="A",
mime_type="application/pdf",
content="root",
archive_serial_number=42,
)
version = Document.objects.create(
title="version",
@@ -1819,6 +1923,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
content="version",
root_document=root,
version_index=1,
archive_serial_number=99,
)
tag_inbox = Tag.objects.create(name="t1", is_inbox_tag=True)
version.tags.add(tag_inbox)
@@ -1829,6 +1934,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(response.data["documents_total"], 1)
self.assertEqual(response.data["documents_inbox"], 0)
self.assertEqual(response.data["character_count"], 4)
self.assertEqual(response.data["current_asn"], 42)
self.assertEqual(
response.data["document_file_type_counts"][0]["mime_type_count"],
1,
@@ -1842,6 +1948,45 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(response.data["documents_inbox"], None)
self.assertEqual(response.data["inbox_tags"], None)
def test_statistics_current_asn_respects_document_permissions(self) -> None:
viewer = UserFactory()
grant_global(viewer, "view_document")
DocumentFactory(owner=viewer, archive_serial_number=12)
shared = DocumentFactory(owner=self.user, archive_serial_number=23)
grant_object(viewer, shared, "view_document")
DocumentFactory(owner=self.user, archive_serial_number=99)
self.client.force_authenticate(user=viewer)
response = self.client.get("/api/statistics/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["current_asn"], 23)
def test_statistics_current_asn_without_visible_documents(self) -> None:
DocumentFactory(owner=self.user, archive_serial_number=99)
for permissions in ((), ("view_document",)):
viewer = UserFactory()
grant_global(viewer, *permissions)
self.client.force_authenticate(user=viewer)
with self.subTest(permissions=permissions):
response = self.client.get("/api/statistics/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["current_asn"], 0)
def test_statistics_current_asn_with_global_statistics_permission(self) -> None:
viewer = UserFactory()
grant_global(viewer, "view_global_statistics")
DocumentFactory(owner=self.user, archive_serial_number=99)
self.client.force_authenticate(user=viewer)
response = self.client.get("/api/statistics/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["current_asn"], 99)
def test_statistics_multiple_users(self) -> None:
"""
GIVEN:
+31 -29
View File
@@ -25,6 +25,7 @@ from documents.models import Tag
from documents.models import Workflow
from documents.search import get_backend
from documents.search import reset_backend
from documents.views import DocumentViewSet
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
from paperless_testing.dirs import DirectoriesMixin
@@ -345,38 +346,39 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
storage_path = StoragePath.objects.create(name="sp1")
tag = Tag.objects.create(name="tag")
matching_doc = Document.objects.create(
title="bank statement",
content="bank content",
checksum="A",
correspondent=correspondent,
document_type=doc_type,
storage_path=storage_path,
)
matching_doc.tags.add(tag)
for checksum, title in (
("A", "bank statement"),
("B", "bank account"),
("C", "credit card"),
):
doc = Document.objects.create(
title=title,
checksum=checksum,
correspondent=correspondent,
document_type=doc_type,
storage_path=storage_path,
)
doc.tags.add(tag)
get_backend().add_or_update(doc)
get_backend().add_or_update(matching_doc)
response = self.client.get(
"/api/documents/?query=bank&include_selection_data=true",
)
with mock.patch.object(DocumentViewSet, "selection_data_batch_size", 1):
response = self.client.get(
"/api/documents/?query=bank&page_size=1&include_selection_data=true",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertIn("selection_data", response.data)
selected_correspondent = next(
item
for item in response.data["selection_data"]["selected_correspondents"]
if item["id"] == correspondent.id
)
selected_tag = next(
item
for item in response.data["selection_data"]["selected_tags"]
if item["id"] == tag.id
)
self.assertEqual(selected_correspondent["document_count"], 1)
self.assertEqual(selected_tag["document_count"], 1)
self.assertEqual(response.data["count"], 2)
self.assertEqual(len(response.data["results"]), 1)
for key, pk in (
("selected_correspondents", correspondent.pk),
("selected_document_types", doc_type.pk),
("selected_storage_paths", storage_path.pk),
("selected_tags", tag.pk),
):
self.assertEqual(
response.data["selection_data"][key],
[{"id": pk, "document_count": 2}],
)
def test_search_custom_field_ordering(self) -> None:
custom_field = CustomField.objects.create(
@@ -149,10 +149,10 @@ class TestPostSelectionPathsEnforceTheCap:
assert str(_MAX_QUERY_LENGTH + 1) in message
parse_spy.assert_not_called()
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.views.execute_bulk_operation.apply_async")
def test_bulk_edit_query_at_exactly_the_cap_is_accepted(
self,
bulk_update_task_mock: mock.MagicMock,
enqueue_mock: mock.MagicMock,
admin_client: APIClient,
searchable_document: Document,
) -> None:
@@ -163,12 +163,12 @@ class TestPostSelectionPathsEnforceTheCap:
- Its `filters.query` is exactly `_MAX_QUERY_LENGTH` characters
long
THEN:
- The cap check accepts it and the request reaches the real
bulk-edit method (its Celery dispatch is mocked out here,
same as every other bulk-edit test, since nothing here is
testing that method itself)
- The cap check accepts it and queues the metadata operation.
Mock the executor dispatch so this exercises query validation
without requiring a broker.
"""
query = "a" * _MAX_QUERY_LENGTH
enqueue_mock.return_value.id = "query-cap-task"
response = admin_client.post(
"/api/documents/bulk_edit/",
@@ -183,6 +183,11 @@ class TestPostSelectionPathsEnforceTheCap:
)
assert response.status_code == status.HTTP_200_OK
assert response.data == {"result": "OK", "task_id": "query-cap-task"}
enqueue_mock.assert_called_once()
assert (
enqueue_mock.call_args.kwargs["kwargs"]["operation"] == "set_document_type"
)
def test_bulk_download_query_one_over_the_cap_is_a_400(
self,
+281
View File
@@ -0,0 +1,281 @@
"""Request acceptance, worker execution, and task tracking."""
import uuid
from unittest import mock
import pytest
from auditlog.models import LogEntry
from django.contrib.auth.models import User
from rest_framework.exceptions import PermissionDenied
from rest_framework.test import APIClient
from documents.bulk_operations import execute_bulk_operation
from documents.models import Correspondent
from documents.models import Document
from documents.models import PaperlessTask
from documents.signals.handlers import before_task_publish_handler
from documents.signals.handlers import task_failure_handler
from documents.signals.handlers import task_postrun_handler
from paperless_testing.factories import UserFactory
@pytest.fixture
def bulk_setup(paperless_dirs):
user = UserFactory(superuser=True)
client = APIClient()
client.force_authenticate(user)
doc = Document.objects.create(title="Example", checksum="bulk-op-example")
correspondent = Correspondent.objects.create(name="Example correspondent")
return user, client, doc, correspondent
@pytest.mark.django_db
@pytest.mark.parametrize(
"endpoint,operation,parameters",
[
("bulk_edit", "set_correspondent", {"correspondent": None}),
],
)
def test_request_only_queues(bulk_setup, endpoint, operation, parameters):
user, client, doc, _ = bulk_setup
task_id = str(uuid.uuid4())
data = {"documents": [doc.pk], "from_webui": True}
if endpoint == "bulk_edit":
data.update(method=operation, parameters=parameters)
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
enqueue.return_value.id = task_id
response = client.post(f"/api/documents/{endpoint}/", data, format="json")
assert response.status_code == 200
assert response.data == {"result": "OK", "task_id": task_id}
assert Document.objects.filter(pk=doc.pk).exists()
assert enqueue.call_args.kwargs["kwargs"] == {
"operation": operation,
"document_ids": [doc.pk],
"parameters": parameters,
"user_id": user.pk,
}
assert enqueue.call_args.kwargs["headers"]["trigger_source"] == "web_ui"
@pytest.mark.django_db
def test_metadata_task_includes_audit_and_followup(bulk_setup, settings):
user, _, doc, correspondent = bulk_setup
settings.AUDIT_LOG_ENABLED = True
with (
mock.patch("documents.bulk_operations.bulk_update_documents") as followup,
mock.patch(
"documents.bulk_edit.bulk_update_documents.apply_async",
) as extra_task,
):
def check_followup(ids):
doc.refresh_from_db()
assert doc.correspondent_id == correspondent.pk
assert LogEntry.objects.filter(object_pk=str(doc.pk), actor=user).exists()
assert ids == [doc.pk]
followup.side_effect = check_followup
result = execute_bulk_operation(
"set_correspondent",
[doc.pk],
{"correspondent": correspondent.pk},
user.pk,
)
followup.assert_called_once()
extra_task.assert_not_called()
assert result["document_count"] == 1
@pytest.mark.django_db
def test_permissions_rechecked_on_worker(bulk_setup):
user, _, doc, correspondent = bulk_setup
User.objects.filter(pk=user.pk).update(is_superuser=False)
with pytest.raises(PermissionDenied):
execute_bulk_operation(
"set_correspondent",
[doc.pk],
{"correspondent": correspondent.pk},
user.pk,
)
doc.refresh_from_db()
assert doc.correspondent_id is None
@pytest.mark.django_db
def test_queue_failure_does_not_change_documents(bulk_setup):
_, client, doc, correspondent = bulk_setup
with mock.patch(
"documents.views.execute_bulk_operation.apply_async",
side_effect=RuntimeError("broker unavailable"),
):
response = client.post(
"/api/documents/bulk_edit/",
{
"documents": [doc.pk],
"method": "set_correspondent",
"parameters": {"correspondent": correspondent.pk},
},
format="json",
)
assert response.status_code == 400
doc.refresh_from_db()
assert doc.correspondent_id is None
@pytest.mark.django_db
@pytest.mark.parametrize("failed", [False, True])
def test_tracked_outcome_records_initiator_and_result(bulk_setup, failed):
user, _, doc, _ = bulk_setup
task_id = str(uuid.uuid4())
before_task_publish_handler(
headers={
"task": "documents.bulk_operations.execute_bulk_operation",
"id": task_id,
},
body=(
(),
{
"operation": "set_permissions",
"document_ids": [doc.pk],
"user_id": user.pk,
},
{},
),
)
task = PaperlessTask.objects.get(task_id=task_id)
assert task.owner_id == user.pk
assert task.task_type == PaperlessTask.TaskType.BULK_UPDATE
if failed:
task_failure_handler(task_id=task_id, exception=ValueError("Could not update"))
else:
task_postrun_handler(
task_id=task_id,
state="SUCCESS",
retval={"document_count": 1},
)
task.refresh_from_db()
assert task.status == ("failure" if failed else "success")
if failed:
assert task.result_data["error_message"] == "Could not update"
else:
assert task.result_data == {"document_count": 1}
@pytest.mark.django_db
def test_large_filtered_selection_is_one_task(bulk_setup):
_, client, _, _ = bulk_setup
Document.objects.bulk_create(
[
Document(title="bulk-large-selection", checksum=f"large-{i}")
for i in range(10_000)
],
)
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
enqueue.return_value.id = str(uuid.uuid4())
response = client.post(
"/api/documents/bulk_edit/",
{
"all": True,
"filters": {"title__icontains": "bulk-large-selection"},
"method": "set_correspondent",
"parameters": {"correspondent": None},
},
format="json",
)
assert response.status_code == 200
enqueue.assert_called_once()
assert len(enqueue.call_args.kwargs["kwargs"]["document_ids"]) == 10_000
@pytest.mark.django_db
@pytest.mark.parametrize("failed", [False, True])
def test_tracked_outcome_notifies_only_initiator(bulk_setup, failed):
user, _, doc, _ = bulk_setup
task_id = str(uuid.uuid4())
before_task_publish_handler(
headers={
"task": "documents.bulk_operations.execute_bulk_operation",
"id": task_id,
},
body=(
(),
{
"operation": "set_permissions",
"document_ids": [doc.pk],
"user_id": user.pk,
},
{},
),
)
task = PaperlessTask.objects.get(task_id=task_id)
assert task.owner_id == user.pk
assert task.task_type == PaperlessTask.TaskType.BULK_UPDATE
with mock.patch("documents.signals.handlers.async_to_sync") as send:
if failed:
task_failure_handler(
task_id=task_id,
exception=ValueError("Could not update"),
)
else:
task_postrun_handler(
task_id=task_id,
state="SUCCESS",
retval={"document_count": 1},
)
group, event = send.return_value.call_args.args
assert group == f"bulk_operations_{user.pk}"
assert event["data"]["status"] == ("failure" if failed else "success")
assert event["data"]["task_id"] == task_id
@pytest.mark.anyio
@pytest.mark.parametrize("anyio_backend", ["asyncio"])
async def test_completion_socket_is_private(anyio_backend):
from types import SimpleNamespace
from channels.layers import get_channel_layer
from channels.testing import WebsocketCommunicator
from paperless.consumers import StatusConsumer
own = WebsocketCommunicator(StatusConsumer.as_asgi(), "/ws/status/")
other = WebsocketCommunicator(StatusConsumer.as_asgi(), "/ws/status/")
own.scope["user"] = SimpleNamespace(id=1, is_authenticated=True)
other.scope["user"] = SimpleNamespace(id=2, is_authenticated=True)
assert (await own.connect())[0]
assert (await other.connect())[0]
try:
event = {
"type": "bulk_operation_status",
"data": {"task_id": "example", "status": "success"},
}
await get_channel_layer().group_send("bulk_operations_1", event)
assert await own.receive_json_from() == event
assert await other.receive_nothing(timeout=0.02)
finally:
await own.disconnect()
await other.disconnect()
@pytest.mark.django_db
@pytest.mark.parametrize("endpoint", ["delete", "bulk_edit"])
def test_trash_remains_synchronous(bulk_setup, endpoint):
_, client, doc, _ = bulk_setup
data = {"documents": [doc.pk]}
if endpoint == "bulk_edit":
data.update(method="delete", parameters={})
with (
mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue,
mock.patch("documents.search.get_backend"),
):
response = client.post(
f"/api/documents/{endpoint}/",
data,
format="json",
HTTP_ACCEPT="application/json; version=9",
)
assert response.status_code == 200
assert response.data == {"result": "OK"}
assert not Document.objects.filter(pk=doc.pk).exists()
assert Document.deleted_objects.filter(pk=doc.pk).exists()
enqueue.assert_not_called()
+123
View File
@@ -0,0 +1,123 @@
from typing import TYPE_CHECKING
import pytest
from django.core.exceptions import FieldDoesNotExist
from django.db import connection
from django.test.utils import CaptureQueriesContext
from documents.management.commands.document_importer import _deserialize_record
from documents.models import Tag
from documents.tag_hierarchy import TagHierarchy
from paperless_testing.factories import TagFactory
from paperless_testing.migrations import TestMigrations
if TYPE_CHECKING:
from django.apps.registry import Apps
from pytest_django.fixtures import DjangoAssertNumQueries
@pytest.mark.django_db
class TestTagAdjacency:
def test_held_instance_uses_current_parent_edges(self) -> None:
"""Descendant lookups reflect parent changes made after loading a tag."""
first = TagFactory(name="First")
second = TagFactory(name="Second")
child = TagFactory(name="Child", tn_parent=first)
Tag.objects.filter(pk=child.pk).update(tn_parent=second)
assert first.get_descendants_pks() == set()
assert second.get_descendants_pks() == {child.pk}
def test_queryset_delete_preserves_cascade(self) -> None:
"""Deleting a parent through a queryset deletes only its subtree."""
root = TagFactory(name="Root")
child = TagFactory(name="Child", tn_parent=root)
TagFactory(name="Grandchild", tn_parent=child)
survivor = TagFactory(name="Other root")
Tag.objects.filter(pk=root.pk).delete()
assert list(Tag.objects.values_list("pk", flat=True)) == [survivor.pk]
def test_flat_writes_do_not_update_other_tags(self) -> None:
"""Creating and renaming flat tags never rewrite unrelated tag rows."""
TagFactory.create_batch(100)
with CaptureQueriesContext(connection) as queries:
new = TagFactory(name="New")
assert not any(q["sql"].startswith("UPDATE") for q in queries)
with CaptureQueriesContext(connection) as queries:
new.name = "Renamed"
new.save()
tag_updates = [
q["sql"] for q in queries if q["sql"].startswith('UPDATE "documents_tag"')
]
assert len(tag_updates) == 1
assert 'WHERE "documents_tag"."id" =' in tag_updates[0]
def test_batch_snapshot_is_one_query_and_includes_overlapping_roots(
self,
django_assert_num_queries: "DjangoAssertNumQueries",
) -> None:
"""One edge query supports multiple traversals with overlapping roots."""
root = TagFactory(name="Root")
child = TagFactory(name="Child", tn_parent=root)
leaf = TagFactory(name="Leaf", tn_parent=child)
with django_assert_num_queries(1):
hierarchy = Tag.get_hierarchy()
assert hierarchy.with_ancestors([leaf.pk]) == {root.pk, child.pk, leaf.pk}
assert hierarchy.with_descendants([root.pk, child.pk]) == {
root.pk,
child.pk,
leaf.pk,
}
assert hierarchy.height(None) == 0
def test_legacy_export_fields_are_ignored_but_parent_is_preserved(self) -> None:
"""Legacy bookkeeping is ignored while parent links and field checks survive."""
model, tag, m2m = _deserialize_record(
{
"model": "documents.tag",
"pk": 2,
"fields": {
"name": "Child",
"tn_parent": 1,
"tn_order": 3,
"tn_siblings_pks": "4,5",
"tn_descendants_pks": "6",
},
},
)
assert model is Tag
assert tag.tn_parent_id == 1
assert tag.name == "Child"
assert not m2m
with pytest.raises(FieldDoesNotExist):
_deserialize_record(
{"model": "documents.tag", "pk": 3, "fields": {"tn_typo": "bad"}},
)
def test_malformed_cycles_terminate(self) -> None:
"""Traversal terminates even when stored parent edges contain a cycle."""
hierarchy = TagHierarchy([(1, 2), (2, 1)])
assert hierarchy.with_descendants([1]) == {1, 2}
assert set(hierarchy.ancestors(2)) == {1, 2}
class TestTagAdjacencyMigration(TestMigrations):
migrate_from = "0027_documentbarcode"
migrate_to = "0028_replace_tag_tree_fields"
def setUpBeforeMigration(self, apps: "Apps") -> None:
"""Seed parent links using the model from the pre-migration state."""
tag = apps.get_model("documents", "Tag")
root = tag.objects.create(name="Root")
child = tag.objects.create(name="Child", tn_parent=root)
self.edges = [(root.pk, None), (child.pk, root.pk)]
def test_parent_edges_survive(self) -> None:
"""Removing derived tree columns preserves the existing parent links."""
tag = self.apps.get_model("documents", "Tag")
assert (
list(tag.objects.order_by("pk").values_list("pk", "tn_parent_id"))
== self.edges
)
assert {
field.name for field in tag._meta.fields if field.name.startswith("tn_")
} == {"tn_parent"}
+113 -197
View File
@@ -5,11 +5,14 @@ import platform
import re
import tempfile
import zipfile
from collections import Counter
from collections import deque
from collections.abc import Callable
from datetime import datetime
from datetime import timedelta
from http import HTTPStatus
from inspect import signature
from itertools import islice
from pathlib import Path
from time import mktime
from time import sleep
@@ -104,6 +107,9 @@ from documents import bulk_edit
from documents.bulk_download import ArchiveOnlyStrategy
from documents.bulk_download import OriginalAndArchiveStrategy
from documents.bulk_download import OriginalsOnlyStrategy
from documents.bulk_operations import QUEUED_OPERATIONS
from documents.bulk_operations import execute_bulk_operation
from documents.bulk_operations import snapshot_field
from documents.caching import get_llm_suggestion_cache
from documents.caching import get_metadata_cache
from documents.caching import get_suggestion_cache
@@ -168,10 +174,10 @@ from documents.permissions import PaperlessNotePermissions
from documents.permissions import PaperlessObjectPermissions
from documents.permissions import TrashPermissions
from documents.permissions import ViewDocumentsPermissions
from documents.permissions import annotate_document_count_by_ids
from documents.permissions import annotate_document_count_for_related_queryset
from documents.permissions import get_document_count_filter_for_user
from documents.permissions import get_permission_grantees
from documents.permissions import has_document_operation_permissions
from documents.permissions import has_global_statistics_permission
from documents.permissions import has_perms_owner_aware
from documents.permissions import has_system_status_permission
@@ -632,7 +638,7 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
queryset = queryset.order_by(*ordering)
all_tags = list(queryset)
descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
descendant_pks = Tag.get_hierarchy().descendants(t.pk for t in all_tags)
if descendant_pks:
user = getattr(getattr(self, "request", None), "user", None)
@@ -1014,71 +1020,80 @@ class DocumentViewSet(
"custom_field_",
)
def _get_selection_data_for_queryset(self, queryset):
# Resolve once instead of once per model below. `queryset` can carry an
# arbitrarily expensive WHERE clause (user filters plus the permission
# filter); re-embedding it as a subquery inside 5 separate Count(...)
# calls forces the database to re-evaluate that whole thing 5 times, and
# -- for FK relations especially -- can defeat semi-join planning
# entirely at scale. A concrete id list is cheap to reuse.
# order_by() drops the default/user ordering -- irrelevant for a plain
# id list, but left in place it forces a sort over the full filtered
# set before the ids can even be collected.
document_ids = list(queryset.order_by().values_list("pk", flat=True))
# Bound the IN clauses used for search results and through-table lookups.
selection_data_batch_size = 10_000
correspondents = Correspondent.objects.annotate(
document_count=Count(
"documents",
filter=Q(documents__id__in=document_ids),
distinct=True,
),
)
document_types = DocumentType.objects.annotate(
document_count=Count(
"documents",
filter=Q(documents__id__in=document_ids),
distinct=True,
),
)
storage_paths = StoragePath.objects.annotate(
document_count=Count(
"documents",
filter=Q(documents__id__in=document_ids),
distinct=True,
),
)
# Tag and CustomField reach Document through an M2M/through-model table;
# a plain Count(filter=...) there is a much more expensive plan than the
# FK relations above once the bridge table is large -- see
# annotate_document_count_by_ids() for why.
tags = annotate_document_count_by_ids(
Tag.objects.all(),
through_model=Document.tags.through,
related_object_field="tag_id",
document_ids=document_ids,
)
custom_fields = annotate_document_count_by_ids(
CustomField.objects.all(),
through_model=CustomFieldInstance,
related_object_field="field_id",
document_ids=document_ids,
def _get_selection_data_for_queryset(
self,
queryset: QuerySet[Document],
*,
search_ids: list[int] | None = None,
) -> dict[str, list[dict[str, int]]]:
# Keep permission and filter predicates at document granularity.
# Reusing the filtered queryset as a subquery for each count is slow.
rows = (
queryset.order_by()
.prefetch_related(None)
.values_list(
"pk",
"correspondent_id",
"document_type_id",
"storage_path_id",
)
)
batch_size = self.selection_data_batch_size
def document_rows():
if search_ids is None:
yield from rows.iterator()
else:
for start in range(0, len(search_ids), batch_size):
yield from rows.filter(
pk__in=search_ids[start : start + batch_size],
).iterator()
correspondents: Counter[int | None] = Counter()
document_types: Counter[int | None] = Counter()
storage_paths: Counter[int | None] = Counter()
document_ids: set[int] = set()
for pk, correspondent, document_type, storage_path in document_rows():
# Filters that join multi-valued relations can repeat a document.
if pk in document_ids:
continue
document_ids.add(pk)
correspondents[correspondent] += 1
document_types[document_type] += 1
storage_paths[storage_path] += 1
tags: Counter[int | None] = Counter()
custom_fields: Counter[int | None] = Counter()
ids = iter(document_ids)
while batch := list(islice(ids, batch_size)):
for model, field, counts in (
(Document.tags.through, "tag_id", tags),
(CustomFieldInstance, "field_id", custom_fields),
):
counts.update(
dict(
model.objects.filter(document_id__in=batch)
.order_by()
.values_list(field)
.annotate(document_count=Count("document_id")),
),
)
def selection(model: type[Model], counts: Counter[int | None]):
return [
{"id": pk, "document_count": counts[pk]}
for pk in model._default_manager.values_list("pk", flat=True)
]
return {
"selected_correspondents": [
{"id": t.id, "document_count": t.document_count} for t in correspondents
],
"selected_tags": [
{"id": t.id, "document_count": t.document_count} for t in tags
],
"selected_document_types": [
{"id": t.id, "document_count": t.document_count} for t in document_types
],
"selected_storage_paths": [
{"id": t.id, "document_count": t.document_count} for t in storage_paths
],
"selected_custom_fields": [
{"id": t.id, "document_count": t.document_count} for t in custom_fields
],
"selected_correspondents": selection(Correspondent, correspondents),
"selected_tags": selection(Tag, tags),
"selected_document_types": selection(DocumentType, document_types),
"selected_storage_paths": selection(StoragePath, storage_paths),
"selected_custom_fields": selection(CustomField, custom_fields),
}
@classmethod
@@ -2725,13 +2740,10 @@ class UnifiedSearchViewSet(DocumentViewSet):
if get_boolean(
str(request.query_params.get("include_selection_data", "false")),
):
# NOTE: pk__in=ordered_ids generates a large SQL IN clause
# for big result sets. Acceptable today but may need a temp
# table or chunked approach if selection_data becomes slow
# at scale (tens of thousands of matching documents).
response.data["selection_data"] = (
self._get_selection_data_for_queryset(
filtered_qs.filter(pk__in=result.ordered_ids),
filtered_qs,
search_ids=result.ordered_ids,
)
)
return response
@@ -2975,98 +2987,29 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
"merge_as_versions",
}
def _has_document_permissions(
self,
*,
user: User,
documents: list[int],
method,
parameters: dict[str, Any],
) -> bool:
if user.is_superuser:
return True
def _has_document_permissions(self, **kwargs) -> bool:
return has_document_operation_permissions(**kwargs)
root_docs = {
get_root_document(doc)
for doc in Document.objects.select_related(
"owner",
"root_document__owner",
).filter(pk__in=documents)
}
user_is_owner_of_all_documents = all(
(doc.owner == user or doc.owner is None) for doc in root_docs
def _queue_bulk_operation(self, method, documents, parameters, from_webui):
if "update_callback" in parameters:
raise ValueError("Unsupported bulk operation parameter: update_callback")
signature(method).bind(documents, **parameters)
task = execute_bulk_operation.apply_async(
kwargs={
"operation": method.__name__,
"document_ids": documents,
"parameters": parameters,
"user_id": self.request.user.pk,
},
headers={
"trigger_source": (
PaperlessTask.TriggerSource.WEB_UI
if from_webui
else PaperlessTask.TriggerSource.API_UPLOAD
),
},
)
# check global and object permissions for all documents
has_perms = (
user.has_perm(
"documents.change_document",
)
and not Document.global_objects.filter(pk__in=documents)
.exclude(pk__in=permitted_document_ids(user, perm="change_document"))
.exists()
)
# check ownership for methods that change original document
if (
(
has_perms
and method
in [
bulk_edit.set_permissions,
bulk_edit.delete,
bulk_edit.rotate,
bulk_edit.delete_pages,
bulk_edit.edit_pdf,
bulk_edit.merge_as_versions,
bulk_edit.remove_password,
]
)
or (
method in [bulk_edit.merge, bulk_edit.split]
and parameters.get("delete_originals")
)
or (method == bulk_edit.edit_pdf and parameters.get("update_document"))
):
has_perms = has_perms and user_is_owner_of_all_documents
# check global add permissions for methods that create documents
if (
has_perms
and (
method in [bulk_edit.split, bulk_edit.merge]
or (
method in [bulk_edit.edit_pdf, bulk_edit.remove_password]
and not parameters.get("update_document")
)
)
and not user.has_perm("documents.add_document")
):
has_perms = False
# check global delete permissions for methods that delete documents
if (
has_perms
and (
method == bulk_edit.delete
# Sources stop being documents of their own, and removing one
# again afterwards needs delete_document
or method == bulk_edit.merge_as_versions
or (
method in [bulk_edit.merge, bulk_edit.split]
and parameters.get("delete_originals")
)
or (
method in [bulk_edit.edit_pdf, bulk_edit.remove_password]
and parameters.get("delete_original")
and not parameters.get("update_document")
)
)
and not user.has_perm("documents.delete_document")
):
has_perms = False
return has_perms
return Response({"result": "OK", "task_id": task.id})
def _execute_document_action(
self,
@@ -3134,6 +3077,7 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
name="BulkEditDocumentsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(required=False),
},
),
},
@@ -3163,37 +3107,7 @@ class BulkEditView(DocumentOperationPermissionMixin):
serializer_class = BulkEditSerializer
@staticmethod
def _snapshot_field(doc_ids: list[int], field: str) -> dict[int, Any]:
"""
Returns each document's current value of field, for the audit log.
Tags and custom fields are one row per value, so they are gathered
into a sorted list of pks per document (empty when there are none).
Reading them through Document.values() instead would join those rows
and return one arbitrary value per document.
"""
if field == "tags":
rows = (
Document.tags.through.objects.filter(document_id__in=doc_ids)
.order_by("tag_id")
.values_list("document_id", "tag_id")
)
elif field == "custom_fields":
rows = (
CustomFieldInstance.objects.filter(document_id__in=doc_ids)
.order_by("pk")
.values_list("document_id", "pk")
)
else:
return dict(
Document.objects.filter(pk__in=doc_ids).values_list("pk", field),
)
values: dict[int, list[int]] = {doc_id: [] for doc_id in doc_ids}
for doc_id, pk in rows:
values[doc_id].append(pk)
return values
_snapshot_field = staticmethod(snapshot_field)
def post(self, request, *args, **kwargs):
request_method = request.data.get("method")
@@ -3239,6 +3153,13 @@ class BulkEditView(DocumentOperationPermissionMixin):
return HttpResponseForbidden("Insufficient permissions")
try:
if method.__name__ in QUEUED_OPERATIONS:
return self._queue_bulk_operation(
method,
documents,
parameters,
from_webui,
)
modified_field = self.MODIFIED_FIELD_BY_METHOD.get(method.__name__, None)
if settings.AUDIT_LOG_ENABLED and modified_field:
old_values = self._snapshot_field(documents, modified_field)
@@ -4071,7 +3992,7 @@ class StatisticsView(GenericAPIView[Any]):
for row in mime_type_stats
]
current_asn = Document.objects.aggregate(
current_asn = documents.aggregate(
Max("archive_serial_number", default=0),
).get(
"archive_serial_number__max",
@@ -5078,12 +4999,7 @@ class BulkEditObjectsView(PassUserMixin):
if object_type == "tags":
editable_ids = set(user_permitted_objects.values_list("pk", flat=True))
all_ids = set(objs.values_list("pk", flat=True))
for tag in objs:
all_ids.update(
descendant.pk
for descendant in tag.get_descendants()
if descendant.pk in editable_ids
)
all_ids.update(Tag.get_hierarchy().descendants(all_ids) & editable_ids)
objs = object_class.objects.filter(pk__in=all_ids)
objs = objs.select_related("owner")
object_ids = list(objs.values_list("pk", flat=True))
+10 -12
View File
@@ -6,6 +6,7 @@ from guardian.shortcuts import remove_perm
from documents.data_models import DocumentMetadataOverrides
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Tag
from documents.models import WorkflowAction
from documents.permissions import PermissionsSpec
from documents.permissions import set_permissions_for_objects
@@ -134,10 +135,9 @@ def apply_assignment_to_overrides(
if action.has_assign_tags:
if overrides.tag_ids is None:
overrides.tag_ids = []
tag_ids_to_add: set[int] = set()
for tag in action.assign_tags.all():
tag_ids_to_add.add(tag.pk)
tag_ids_to_add.update(int(pk) for pk in tag.get_ancestors_pks())
tag_ids_to_add = Tag.get_hierarchy().with_ancestors(
tag.pk for tag in action.assign_tags.all()
)
overrides.tag_ids = list(set(overrides.tag_ids) | tag_ids_to_add)
@@ -216,10 +216,9 @@ def apply_removal_to_document(
if action.remove_all_tags:
Document.objects.get(pk=document.pk).tags.clear()
else:
tag_ids_to_remove: set[int] = set()
for tag in action.remove_tags.all():
tag_ids_to_remove.add(tag.pk)
tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
tag.pk for tag in action.remove_tags.all()
)
if tag_ids_to_remove:
Document.objects.get(pk=document.pk).tags.remove(*tag_ids_to_remove)
@@ -297,10 +296,9 @@ def apply_removal_to_overrides(
if action.remove_all_tags:
overrides.tag_ids = None
elif overrides.tag_ids:
tag_ids_to_remove: set[int] = set()
for tag in action.remove_tags.all():
tag_ids_to_remove.add(tag.pk)
tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
tag.pk for tag in action.remove_tags.all()
)
overrides.tag_ids = [t for t in overrides.tag_ids if t not in tag_ids_to_remove]
File diff suppressed because it is too large. Load diff
+15
View File
@@ -45,12 +45,21 @@ class StatusConsumer(AsyncWebsocketConsumer):
await self.close()
return
await self.channel_layer.group_add("status_updates", self.channel_name)
await self.channel_layer.group_add(
f"bulk_operations_{self.scope['user'].id}",
self.channel_name,
)
await self.accept()
self._start_heartbeat()
async def disconnect(self, code: int) -> None:
await self._stop_heartbeat()
await self.channel_layer.group_discard("status_updates", self.channel_name)
if self._authenticated():
await self.channel_layer.group_discard(
f"bulk_operations_{self.scope['user'].id}",
self.channel_name,
)
def _start_heartbeat(self) -> None:
self.heartbeat_task = asyncio.create_task(self._heartbeat_loop())
@@ -84,3 +93,9 @@ class StatusConsumer(AsyncWebsocketConsumer):
await self.close()
elif await self._can_view(event["data"]):
await self.send(json.dumps(event))
async def bulk_operation_status(self, event: dict) -> None:
if not self._authenticated():
await self.close()
else:
await self.send(json.dumps(event))
+1 -2
View File
@@ -157,7 +157,6 @@ INSTALLED_APPS = [
"allauth.headless",
"drf_spectacular",
"drf_spectacular_sidecar",
"treenode",
*env_apps,
]
@@ -691,7 +690,7 @@ logging.config.dictConfig(LOGGING)
# https://docs.celeryq.dev/en/stable/userguide/configuration.html
# Tasks outside tasks.py must be loaded even when Django skips checks
CELERY_IMPORTS = ("documents.bulk_edit",)
CELERY_IMPORTS = ("documents.bulk_edit", "documents.bulk_operations")
CELERY_BROKER_URL = _CELERY_REDIS_URL
CELERY_RESULT_BACKEND = _CELERY_REDIS_URL
+2 -2
View File
@@ -32,13 +32,13 @@ from rest_framework.fields import BooleanField
from rest_framework.filters import OrderingFilter
from rest_framework.generics import GenericAPIView
from rest_framework.pagination import PageNumberPagination
from rest_framework.permissions import DjangoModelPermissions
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.throttling import ScopedRateThrottle
from rest_framework.viewsets import ModelViewSet
from documents.models import PaperlessTask
from documents.permissions import PaperlessModelPermissions
from documents.permissions import PaperlessObjectPermissions
from documents.tasks import llmindex_index
from paperless.filters import GroupFilterSet
@@ -414,7 +414,7 @@ class ApplicationConfigurationViewSet(ModelViewSet[ApplicationConfiguration]):
queryset = ApplicationConfiguration.objects
serializer_class = ApplicationConfigurationSerializer
permission_classes = (IsAuthenticated, DjangoModelPermissions)
permission_classes = (IsAuthenticated, PaperlessModelPermissions)
@extend_schema(exclude=True)
def create(self, request, *args, **kwargs):
+126
View File
@@ -20,8 +20,10 @@ from paperless_ai.tables import DocumentChunksTable
from paperless_ai.tables import DocumentMetaTable
from paperless_ai.vector_store import _MAX_IN_VALUES
from paperless_ai.vector_store import DB_FILENAME
from paperless_ai.vector_store import DB_PAGE_SIZE
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
from paperless_ai.vector_store import SCHEMA_VERSION
from paperless_ai.vector_store import VEC0_CHUNK_SIZE
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
from paperless_ai.vector_store import _build_where
from paperless_ai.vector_store import _pack
@@ -1128,3 +1130,127 @@ class TestV1ToV2Migration:
assert "DocumentChunksTable.create" not in source
assert "DocumentMetaTable.create" not in source
assert "_create_vec_table(" not in source
LEGACY_PAGE_SIZE = 4096
def vec_table_sql(db_path: Path) -> str:
conn = sqlite3.connect(db_path)
try:
return conn.execute(
"SELECT sql FROM sqlite_master WHERE name = ?",
(DEFAULT_TABLE_NAME,),
).fetchone()[0]
finally:
conn.close()
def page_size_of(db_path: Path) -> int:
conn = sqlite3.connect(db_path)
try:
return conn.execute("PRAGMA page_size").fetchone()[0]
finally:
conn.close()
def create_legacy_vec_table(conn: sqlite3.Connection, dim: int) -> None:
"""The table definition from before chunk_size was set explicitly."""
conn.execute(
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
"id TEXT PRIMARY KEY, document_id INTEGER, +node_content TEXT, "
f"embedding float[{dim}] distance_metric=cosine)",
)
@pytest.fixture
def legacy_page_size_db(tmp_path: Path) -> Path:
"""An existing index file created before the page size was set."""
db_path = tmp_path / DB_FILENAME
conn = sqlite3.connect(db_path)
conn.execute(f"PRAGMA page_size={LEGACY_PAGE_SIZE}")
conn.execute("CREATE TABLE _seed(x)")
conn.execute("DROP TABLE _seed")
conn.close()
assert page_size_of(db_path) == LEGACY_PAGE_SIZE
return db_path
class TestVecChunkSize:
def test_new_table_uses_configured_chunk_size(
self,
store: PaperlessSqliteVecVectorStore,
tmp_path: Path,
) -> None:
"""
GIVEN an empty store
WHEN the first document is indexed and the vec0 table is created
THEN the table definition carries the configured chunk size
"""
store.upsert_document(1, [make_node("1-a", 1)])
assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
def test_compact_moves_legacy_table_to_configured_chunk_size(
self,
store: PaperlessSqliteVecVectorStore,
tmp_path: Path,
mocker: MockerFixture,
) -> None:
"""
GIVEN an index whose vec0 table was created with the default chunk size
WHEN the index is compacted
THEN the rebuilt table uses the configured chunk size
AND no indexed chunks are lost
"""
legacy = mocker.patch.object(
PaperlessSqliteVecVectorStore,
"_create_vec_table",
staticmethod(create_legacy_vec_table),
)
store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
store.upsert_document(2, [make_node("2-a", 2)])
mocker.stop(legacy)
assert "chunk_size" not in vec_table_sql(tmp_path / DB_FILENAME)
store.compact(force=True)
assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
assert DocumentChunksTable.count(store.client) == 3
class TestIndexPageSize:
def test_new_index_file_uses_configured_page_size(
self,
store: PaperlessSqliteVecVectorStore,
tmp_path: Path,
) -> None:
"""
GIVEN no existing index file
WHEN a store is opened and a document is indexed
THEN the database file uses the configured page size
"""
store.upsert_document(1, [make_node("1-a", 1)])
assert page_size_of(tmp_path / DB_FILENAME) == DB_PAGE_SIZE
def test_compact_moves_legacy_file_to_configured_page_size(
self,
legacy_page_size_db: Path,
) -> None:
"""
GIVEN an existing index file created with a different page size
WHEN the index is compacted
THEN the rebuilt file uses the configured page size
AND no indexed chunks are lost
"""
with PaperlessSqliteVecVectorStore(
uri=str(legacy_page_size_db.parent),
) as store:
store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
assert page_size_of(legacy_page_size_db) == LEGACY_PAGE_SIZE
store.compact(force=True)
assert page_size_of(legacy_page_size_db) == DB_PAGE_SIZE
assert DocumentChunksTable.count(store.client) == 2
+17 -1
View File
@@ -47,6 +47,20 @@ SCHEMA_VERSION = 2
# a rebuild copies the live rows into a fresh table.
COMPACT_BLOAT_RATIO = 2.0
# Rows per vec0 vector chunk (vec0's default is 1024, which makes each chunk a
# multi-megabyte blob). On spinning disks, inserts and re-indexing were dozens
# of times faster with small chunks, at no cost in query speed or file size.
# Only applies to tables created from here on; an existing index picks it up
# the next time it is rebuilt (compact or rebuild).
VEC0_CHUNK_SIZE = 32
# SQLite page size for the index file, in bytes. With the small vec0 chunks above,
# 64 KiB pages made cold scans about 35% faster and warm ones about 45% faster,
# at the cost of a larger write-ahead log (up to about 70 MB). Only takes effect
# on a newly created file, so like the chunk size it reaches an existing index
# when it is rebuilt.
DB_PAGE_SIZE = 65536
# Number of rows fetched/copied per batch whenever this module streams rows
# instead of materializing them all at once, keeping memory bounded regardless
# of index size -- used by compact()'s rebuild, m0001_v1_to_v2's migration
@@ -222,6 +236,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
conn.enable_load_extension(True) # noqa: FBT003
sqlite_vec.load(conn)
conn.enable_load_extension(False) # noqa: FBT003
conn.execute(f"PRAGMA page_size={DB_PAGE_SIZE}")
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
IndexMetaTable.create(conn)
@@ -329,7 +344,8 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
+ " +node_content TEXT,"
+ " embedding float["
+ str(int(dim))
+ "] distance_metric=cosine"
+ "] distance_metric=cosine, chunk_size="
+ str(VEC0_CHUNK_SIZE)
+ ")",
)
Generated
-11
View File
@@ -1210,15 +1210,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/99/c3/305a9f7611c0dc96d2a638bc3a67bdcde5a7faa5f4669188db018b8af6fc/django_stubs_ext-6.1.1-py3-none-any.whl", hash = "sha256:64aebb7ce2c327a4a2b75207d67034e879fd7edf027c93487be69da363dc79fa", size = 10853, upload-time = "2026-09-14T08:45:51.289Z" },
]
[[package]]
name = "django-treenode"
version = "0.25.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/cb/2b/09f04b44b3583a911ae4f5d0e3360a23d529704c9e9115fce2520b7d4420/django_treenode-0.25.0.tar.gz", hash = "sha256:d14d0eb3571c227ce11147210d62a8f87b568bbb30fa76a5be9b9e6d514157c3", size = 29951, upload-time = "2026-08-26T00:20:13.489Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e1/ac/2140e8c20e2a47457ec23302273527d3430bf5c09edb8940d74c0725c138/django_treenode-0.25.0-py3-none-any.whl", hash = "sha256:f47caab5440420870b6287602b3ee401ca354416d44b24ab97579be5ea658927", size = 22343, upload-time = "2026-08-26T00:20:12.078Z" },
]
[[package]]
name = "djangorestframework"
version = "3.18.3"
@@ -3107,7 +3098,6 @@ dependencies = [
{ name = "django-multiselectfield" },
{ name = "django-rich" },
{ name = "django-soft-delete" },
{ name = "django-treenode" },
{ name = "djangorestframework" },
{ name = "drf-spectacular" },
{ name = "drf-spectacular-sidecar" },
@@ -3258,7 +3248,6 @@ requires-dist = [
{ name = "django-multiselectfield", specifier = "~=1.0.1" },
{ name = "django-rich", specifier = "~=2.2.0" },
{ name = "django-soft-delete", specifier = "~=1.0.18" },
{ name = "django-treenode", specifier = ">=0.24" },
{ name = "djangorestframework", specifier = "~=3.16" },
{ name = "drf-spectacular", specifier = "~=0.30" },
{ name = "drf-spectacular-sidecar", specifier = ">=2026.10.1,<2026.11" },