mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-07 05:05:56 +00:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62648fffe8 | ||
|
|
b0c8ce25fd | ||
|
|
3c4f5d8959 | ||
|
|
85c52cc27f | ||
|
|
9b3d484e24 | ||
|
|
3a5ba814b6 | ||
|
|
26552f54b4 | ||
|
|
de9d7e54bd | ||
|
|
a5c460a567 | ||
|
|
cfca79e7fc | ||
|
|
9d59f3aaa2 | ||
|
|
332c308f84 | ||
|
|
a0b8da7687 | ||
|
|
2642cefd35 | ||
|
|
8aa99a381a | ||
|
|
d14cfc93f4 | ||
|
|
53e386ec81 |
No files matched your search
@@ -6,9 +6,9 @@ if [ "${1:-}" = download ]; then
|
||||
download_archive_dir=$2
|
||||
shift 2
|
||||
apt-get update -q
|
||||
apt-get install -q --yes --no-install-recommends --download-only \
|
||||
# Keep apt itself under timeout supervision, including the kill grace period.
|
||||
exec apt-get install -q --yes --no-install-recommends --download-only \
|
||||
-o "Dir::Cache::archives=$download_archive_dir" "$@"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SUDO=()
|
||||
@@ -34,7 +34,8 @@ use_ubuntu_mirrors() {
|
||||
|
||||
for attempt in $(seq 1 "$ATTEMPTS"); do
|
||||
echo "::group::apt download attempt ${attempt}/${ATTEMPTS}"
|
||||
if timeout --kill-after=10 "$ATTEMPT_TIMEOUT" "${SUDO[@]}" bash "$0" download "$CACHE_DIR" "${requested_packages[@]}"; then
|
||||
# Run timeout as root so it can signal the root-owned apt processes.
|
||||
if "${SUDO[@]}" timeout --kill-after=10 "$ATTEMPT_TIMEOUT" bash "$0" download "$CACHE_DIR" "${requested_packages[@]}"; then
|
||||
echo "::endgroup::"
|
||||
# Apt handles aliases (e.g. libmagic1 -> libmagic1t64) and dependencies.
|
||||
"${SUDO[@]}" apt-get install -q --yes --no-install-recommends --no-download \
|
||||
|
||||
@@ -39,7 +39,6 @@ declare -a scope=(
|
||||
"${site_packages}/allauth"
|
||||
"${site_packages}/drf_spectacular"
|
||||
"${site_packages}/drf_spectacular_sidecar"
|
||||
"${site_packages}/treenode"
|
||||
"${site_packages}/compression_middleware"
|
||||
)
|
||||
|
||||
|
||||
+30
-5
@@ -493,7 +493,7 @@ fields and permissions, which will be merged.
|
||||
|
||||
#### Types {#workflow-trigger-types}
|
||||
|
||||
Currently, there are four events that correspond to workflow trigger 'types':
|
||||
Currently, there are five events that correspond to workflow trigger 'types':
|
||||
|
||||
1. **Consumption Started**: _before_ a document is consumed, so events can include filters by source (mail, consumption
|
||||
folder or API), file path, file name, mail rule
|
||||
@@ -505,8 +505,10 @@ Currently, there are four events that correspond to workflow trigger 'types':
|
||||
4. **Scheduled**: a scheduled trigger that can be used to run workflows at a specific time. The date used can be either the document
|
||||
added, created, updated date or you can specify a (date) custom field. You can also specify a day offset from the date (positive
|
||||
offsets will trigger after the date, negative offsets will trigger before).
|
||||
5. **Manual**: a trigger that only runs when you explicitly choose to run the workflow against one or more existing documents. See
|
||||
[running a workflow manually](#running-a-workflow-manually) below.
|
||||
|
||||
The following flow diagram illustrates the four document trigger types:
|
||||
The following flow diagram illustrates the document trigger types:
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
@@ -526,6 +528,10 @@ flowchart TD
|
||||
matching
|
||||
trigger(s)"}
|
||||
|
||||
manual{"Matching
|
||||
'Manual'
|
||||
trigger(s)"}
|
||||
|
||||
A[New Document] --> consumption
|
||||
consumption --> |Yes| C[Workflow Actions Run]
|
||||
consumption --> |No| D
|
||||
@@ -543,6 +549,10 @@ flowchart TD
|
||||
scheduled --> |Yes| N[Workflow Actions Run]
|
||||
scheduled --> |No| O[Document Saved]
|
||||
N --> O
|
||||
P[User Runs Workflow] --> manual
|
||||
manual --> |Yes| Q[Workflow Actions Run]
|
||||
manual --> |No| R[Document Saved]
|
||||
Q --> R
|
||||
```
|
||||
|
||||
#### Filters {#workflow-trigger-filters}
|
||||
@@ -554,9 +564,9 @@ Workflows allow you to filter by:
|
||||
- File path, including wildcards. Note that enabling `PAPERLESS_CONSUMER_RECURSIVE` would allow, for
|
||||
example, automatically assigning documents to different owners based on the upload directory.
|
||||
- Mail rule. Choosing this option will force 'mail fetch' to be the workflow source.
|
||||
- Content matching (`Added`, `Updated` and `Scheduled` triggers only). Filter document content using the matching settings.
|
||||
- Content matching (`Added`, `Updated`, `Scheduled` and `Manual` triggers only). Filter document content using the matching settings.
|
||||
|
||||
There are also 'advanced' filters available for `Added`, `Updated` and `Scheduled` triggers:
|
||||
There are also 'advanced' filters available for `Added`, `Updated`, `Scheduled` and `Manual` triggers:
|
||||
|
||||
- Any Tags: Filter for documents with any of the specified tags.
|
||||
- All Tags: Filter for documents with all of the specified tags.
|
||||
@@ -569,6 +579,21 @@ There are also 'advanced' filters available for `Added`, `Updated` and `Schedule
|
||||
- Not Storage paths: Filter documents without any of these storage paths.
|
||||
- Custom field query: Filter documents with a custom field query (the same as used for the document list filters).
|
||||
|
||||
#### Running a workflow manually {#running-a-workflow-manually}
|
||||
|
||||
A workflow with a **Manual** trigger does not run automatically. Instead, it can be run on demand against existing
|
||||
documents from the web UI:
|
||||
|
||||
- From the document list, select one or more documents and choose **Run workflow** from the "Actions" menu.
|
||||
- From an individual document, choose **Run workflow** from the "Actions" menu.
|
||||
|
||||
You will be asked which workflow to run. By default the workflow's trigger filters still apply, so documents that do
|
||||
not match are skipped. You can instead choose to **run on all selected documents**, which ignores the trigger filters
|
||||
and runs the workflow's actions on every selected document.
|
||||
|
||||
Manual runs happen in the background. As with scheduled workflows, the document list is not refreshed automatically when
|
||||
they finish, so you may need to reload the list to see the results.
|
||||
|
||||
### Workflow Actions
|
||||
|
||||
#### Types {#workflow-action-types}
|
||||
@@ -649,7 +674,7 @@ new version of the same document, this time with normal OCR/text extraction appl
|
||||
a password-protected file added with this trigger will briefly exist as an un-OCR'd version before
|
||||
the properly processed version is created.
|
||||
|
||||
**Document Added**, **Document Updated**, **Scheduled**: these triggers run against a document that
|
||||
**Document Added**, **Document Updated**, **Scheduled**, **Manual**: these triggers run against a document that
|
||||
already exists, so password removal happens immediately: the decrypted content is queued for
|
||||
consumption as a new version right away. Note that if the document's initial consumption also
|
||||
happened while it was still encrypted, that original version will likewise be missing OCR text.
|
||||
|
||||
@@ -38,7 +38,6 @@ dependencies = [
|
||||
"django-multiselectfield~=1.0.1",
|
||||
"django-rich~=2.2.0",
|
||||
"django-soft-delete~=1.0.18",
|
||||
"django-treenode>=0.24",
|
||||
"djangorestframework~=3.16",
|
||||
"drf-spectacular~=0.30",
|
||||
"drf-spectacular-sidecar>=2026.10.1,<2026.11",
|
||||
|
||||
@@ -118,6 +118,7 @@ import personFill from 'bootstrap-icons/icons/person-fill.svg'
|
||||
import personLock from 'bootstrap-icons/icons/person-lock.svg'
|
||||
import personSquare from 'bootstrap-icons/icons/person-square.svg'
|
||||
import person from 'bootstrap-icons/icons/person.svg'
|
||||
import playCircle from 'bootstrap-icons/icons/play-circle.svg'
|
||||
import playFill from 'bootstrap-icons/icons/play-fill.svg'
|
||||
import plusCircle from 'bootstrap-icons/icons/plus-circle.svg'
|
||||
import plus from 'bootstrap-icons/icons/plus.svg'
|
||||
@@ -279,6 +280,7 @@ export const icons: Record<string, string> = {
|
||||
personFillLock,
|
||||
personLock,
|
||||
personSquare,
|
||||
playCircle,
|
||||
playFill,
|
||||
plus,
|
||||
plusCircle,
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
<div class="modal-header">
|
||||
<h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
|
||||
<button type="button" class="btn-close" aria-label="Close" (click)="cancel()">
|
||||
</button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
@if (message) {
|
||||
<p class="text-break" [innerHTML]="message"></p>
|
||||
}
|
||||
<div class="mb-3">
|
||||
<label class="form-label" for="runWorkflowSelect" i18n>Workflow</label>
|
||||
<select class="form-select" id="runWorkflowSelect" [(ngModel)]="workflowId" (ngModelChange)="workflowSelected()">
|
||||
<option [ngValue]="null" disabled i18n>Select a workflow</option>
|
||||
@for (workflow of workflows; track workflow.id) {
|
||||
<option [ngValue]="workflow.id">{{workflow.name}}</option>
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" id="runWorkflowForce" [(ngModel)]="force" />
|
||||
<label class="form-check-label" for="runWorkflowForce" i18n>Run on all selected documents</label>
|
||||
<div class="form-text" i18n>Ignore the workflow's trigger filters and run on every selected document.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn" [class]="cancelBtnClass" (click)="cancel()" [disabled]="!buttonsEnabled">
|
||||
<span class="d-inline-block" style="padding-bottom: 1px;">{{cancelBtnCaption}}</span>
|
||||
</button>
|
||||
<button type="button" class="btn" [class]="btnClass" (click)="confirm()" [disabled]="!confirmButtonEnabled || !buttonsEnabled">
|
||||
{{btnCaption}}
|
||||
</button>
|
||||
</div>
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing'
|
||||
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
|
||||
import { RunWorkflowConfirmDialogComponent } from './run-workflow-confirm-dialog.component'
|
||||
|
||||
describe('RunWorkflowConfirmDialogComponent', () => {
|
||||
let component: RunWorkflowConfirmDialogComponent
|
||||
let fixture: ComponentFixture<RunWorkflowConfirmDialogComponent>
|
||||
|
||||
beforeEach(async () => {
|
||||
TestBed.configureTestingModule({
|
||||
providers: [NgbActiveModal],
|
||||
imports: [RunWorkflowConfirmDialogComponent],
|
||||
}).compileComponents()
|
||||
|
||||
fixture = TestBed.createComponent(RunWorkflowConfirmDialogComponent)
|
||||
component = fixture.componentInstance
|
||||
component.workflows = [{ id: 5, name: 'Manual workflow' } as any]
|
||||
fixture.detectChanges()
|
||||
})
|
||||
|
||||
it('should disable confirm until a workflow is selected and not force by default', () => {
|
||||
expect(component.confirmButtonEnabled).toBeFalsy()
|
||||
expect(component.force).toBeFalsy()
|
||||
|
||||
component.workflowId = 5
|
||||
component.workflowSelected()
|
||||
expect(component.confirmButtonEnabled).toBeTruthy()
|
||||
|
||||
component.workflowId = null
|
||||
component.workflowSelected()
|
||||
expect(component.confirmButtonEnabled).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should render an option per workflow', () => {
|
||||
const options = fixture.nativeElement.querySelectorAll(
|
||||
'#runWorkflowSelect option'
|
||||
)
|
||||
// placeholder option + one per workflow
|
||||
expect(options).toHaveLength(2)
|
||||
expect(options[1].textContent).toContain('Manual workflow')
|
||||
})
|
||||
})
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
import { Component } from '@angular/core'
|
||||
import { FormsModule } from '@angular/forms'
|
||||
import { Workflow } from 'src/app/data/workflow'
|
||||
import { ConfirmDialogComponent } from '../confirm-dialog.component'
|
||||
|
||||
@Component({
|
||||
selector: 'pngx-run-workflow-confirm-dialog',
|
||||
templateUrl: './run-workflow-confirm-dialog.component.html',
|
||||
imports: [FormsModule],
|
||||
})
|
||||
export class RunWorkflowConfirmDialogComponent extends ConfirmDialogComponent {
|
||||
public workflows: Workflow[] = []
|
||||
|
||||
public workflowId: number = null
|
||||
|
||||
// i.e. ignore the workflow's trigger filters
|
||||
public force: boolean = false
|
||||
|
||||
override confirmButtonEnabled = false
|
||||
|
||||
public workflowSelected(): void {
|
||||
this.confirmButtonEnabled = !!this.workflowId
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -164,7 +164,7 @@
|
||||
<pngx-input-text i18n-title title="Filter path" formControlName="filter_path" horizontal="true" i18n-hint hint="Apply to documents that match this path. Wildcards specified as * are allowed. Case-normalized." [error]="error?.filter_path"></pngx-input-text>
|
||||
<pngx-input-select i18n-title title="Filter mail rule" [items]="mailRules()" horizontal="true" [allowNull]="true" formControlName="filter_mailrule" i18n-hint hint="Apply to documents consumed via this mail rule." [error]="error?.filter_mailrule"></pngx-input-select>
|
||||
}
|
||||
@if (formGroup.get('type').value === WorkflowTriggerType.DocumentAdded || formGroup.get('type').value === WorkflowTriggerType.DocumentUpdated || formGroup.get('type').value === WorkflowTriggerType.Scheduled) {
|
||||
@if (formGroup.get('type').value === WorkflowTriggerType.DocumentAdded || formGroup.get('type').value === WorkflowTriggerType.DocumentUpdated || formGroup.get('type').value === WorkflowTriggerType.Scheduled || formGroup.get('type').value === WorkflowTriggerType.Manual) {
|
||||
<pngx-input-select i18n-title title="Content matching algorithm" horizontal="true" [items]="getMatchingAlgorithms()" formControlName="matching_algorithm"></pngx-input-select>
|
||||
@if (matchingPatternRequired(formGroup)) {
|
||||
<pngx-input-text i18n-title title="Content matching pattern" horizontal="true" formControlName="match" [error]="error?.match"></pngx-input-text>
|
||||
@@ -175,7 +175,7 @@
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
@if (formGroup.get('type').value === WorkflowTriggerType.DocumentAdded || formGroup.get('type').value === WorkflowTriggerType.DocumentUpdated || formGroup.get('type').value === WorkflowTriggerType.Scheduled) {
|
||||
@if (formGroup.get('type').value === WorkflowTriggerType.DocumentAdded || formGroup.get('type').value === WorkflowTriggerType.DocumentUpdated || formGroup.get('type').value === WorkflowTriggerType.Scheduled || formGroup.get('type').value === WorkflowTriggerType.Manual) {
|
||||
<div class="row mt-3">
|
||||
<div class="col">
|
||||
<div class="trigger-filters mb-3">
|
||||
|
||||
+4
@@ -123,6 +123,10 @@ export const WORKFLOW_TYPE_OPTIONS = [
|
||||
id: WorkflowTriggerType.Scheduled,
|
||||
name: $localize`Scheduled`,
|
||||
},
|
||||
{
|
||||
id: WorkflowTriggerType.Manual,
|
||||
name: $localize`Manual`,
|
||||
},
|
||||
]
|
||||
|
||||
export const WORKFLOW_ACTION_OPTIONS = [
|
||||
|
||||
@@ -49,6 +49,12 @@
|
||||
<pngx-bs-icon width="1em" height="1em" name="arrow-counterclockwise" class="me-2"></pngx-bs-icon><span i18n>Reprocess</span>
|
||||
</button>
|
||||
|
||||
@if (manualWorkflows().length > 0) {
|
||||
<button ngbDropdownItem (click)="runWorkflow()" [disabled]="!userCanEdit || !userIsOwner">
|
||||
<pngx-bs-icon width="1em" height="1em" name="play-circle" class="me-2"></pngx-bs-icon><span i18n>Run workflow</span>
|
||||
</button>
|
||||
}
|
||||
|
||||
<button ngbDropdownItem (click)="printDocument()" [hidden]="useNativePdfViewer || isMobile">
|
||||
<pngx-bs-icon width="1em" height="1em" name="printer" class="me-2"></pngx-bs-icon><span i18n>Print</span>
|
||||
</button>
|
||||
|
||||
@@ -1030,6 +1030,27 @@ describe('DocumentDetailComponent', () => {
|
||||
expect(modalCloseSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should run a workflow on the document, confirm and close modal', () => {
|
||||
initNormally()
|
||||
const bulkEditSpy = jest.spyOn(documentService, 'bulkEdit')
|
||||
bulkEditSpy.mockReturnValue(of(true))
|
||||
let openModal: NgbModalRef
|
||||
modalService.activeInstances.subscribe((modal) => (openModal = modal[0]))
|
||||
const toastSpy = jest.spyOn(toastService, 'showInfo')
|
||||
component.runWorkflow()
|
||||
const modalCloseSpy = jest.spyOn(openModal, 'close')
|
||||
openModal.componentInstance.workflowId = 5
|
||||
openModal.componentInstance.force = true
|
||||
openModal.componentInstance.confirmClicked.next()
|
||||
expect(bulkEditSpy).toHaveBeenCalledWith(
|
||||
{ documents: [doc.id] },
|
||||
'run_workflow',
|
||||
{ workflow: 5, filter_documents: false }
|
||||
)
|
||||
expect(toastSpy).toHaveBeenCalled()
|
||||
expect(modalCloseSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should pass remote OCR choice when reprocessing', () => {
|
||||
initNormally()
|
||||
const reprocessSpy = jest.spyOn(documentService, 'reprocessDocuments')
|
||||
|
||||
@@ -68,6 +68,8 @@ import { StoragePath } from 'src/app/data/storage-path'
|
||||
import { Tag } from 'src/app/data/tag'
|
||||
import { SETTINGS_KEYS, SuggestionSource } from 'src/app/data/ui-settings'
|
||||
import { User } from 'src/app/data/user'
|
||||
import { Workflow } from 'src/app/data/workflow'
|
||||
import { WorkflowTriggerType } from 'src/app/data/workflow-trigger'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
|
||||
import { DocumentTitlePipe } from 'src/app/pipes/document-title.pipe'
|
||||
@@ -93,6 +95,7 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service'
|
||||
import { StoragePathService } from 'src/app/services/rest/storage-path.service'
|
||||
import { TagService } from 'src/app/services/rest/tag.service'
|
||||
import { UserService } from 'src/app/services/rest/user.service'
|
||||
import { WorkflowService } from 'src/app/services/rest/workflow.service'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
import { ToastService } from 'src/app/services/toast.service'
|
||||
import { TrashService } from 'src/app/services/trash.service'
|
||||
@@ -104,6 +107,7 @@ import { DocumentDetailFieldID } from '../admin/settings/settings.component'
|
||||
import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component'
|
||||
import { PasswordRemovalConfirmDialogComponent } from '../common/confirm-dialog/password-removal-confirm-dialog/password-removal-confirm-dialog.component'
|
||||
import { ReprocessConfirmDialogComponent } from '../common/confirm-dialog/reprocess-confirm-dialog/reprocess-confirm-dialog.component'
|
||||
import { RunWorkflowConfirmDialogComponent } from '../common/confirm-dialog/run-workflow-confirm-dialog/run-workflow-confirm-dialog.component'
|
||||
import { CustomFieldsDropdownComponent } from '../common/custom-fields-dropdown/custom-fields-dropdown.component'
|
||||
import { CorrespondentEditDialogComponent } from '../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component'
|
||||
import { DocumentTypeEditDialogComponent } from '../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
|
||||
@@ -233,6 +237,7 @@ export class DocumentDetailComponent
|
||||
private componentRouterService = inject(ComponentRouterService)
|
||||
private deviceDetectorService = inject(DeviceDetectorService)
|
||||
private savedViewService = inject(SavedViewService)
|
||||
private readonly workflowService = inject(WorkflowService)
|
||||
private readonly websocketStatusService = inject(WebsocketStatusService)
|
||||
private readonly useNativePdfViewerSetting = this.settings.getSignal<boolean>(
|
||||
SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER
|
||||
@@ -295,6 +300,7 @@ export class DocumentDetailComponent
|
||||
readonly correspondents = signal<Correspondent[]>(undefined)
|
||||
readonly documentTypes = signal<DocumentType[]>(undefined)
|
||||
readonly storagePaths = signal<StoragePath[]>(undefined)
|
||||
readonly manualWorkflows = signal<Workflow[]>([])
|
||||
|
||||
documentForm: FormGroup = new FormGroup({
|
||||
title: new FormControl(''),
|
||||
@@ -836,6 +842,19 @@ export class DocumentDetailComponent
|
||||
.pipe(first(), takeUntil(this.unsubscribeNotifier))
|
||||
.subscribe((result) => this.users.set(result.results))
|
||||
}
|
||||
if (
|
||||
this.permissionsService.currentUserCan(
|
||||
PermissionAction.View,
|
||||
PermissionType.Workflow
|
||||
)
|
||||
) {
|
||||
this.workflowService
|
||||
.list(1, 100000, null, null, {
|
||||
trigger_type: WorkflowTriggerType.Manual,
|
||||
})
|
||||
.pipe(first(), takeUntil(this.unsubscribeNotifier))
|
||||
.subscribe((result) => this.manualWorkflows.set(result.results))
|
||||
}
|
||||
|
||||
this.getCustomFields()
|
||||
|
||||
@@ -1559,6 +1578,39 @@ export class DocumentDetailComponent
|
||||
})
|
||||
}
|
||||
|
||||
runWorkflow() {
|
||||
const modal = this.modalService.open(RunWorkflowConfirmDialogComponent, {
|
||||
backdrop: 'static',
|
||||
})
|
||||
modal.componentInstance.title = $localize`Run workflow`
|
||||
modal.componentInstance.message = $localize`Run a workflow on "${this.document().title}".`
|
||||
modal.componentInstance.btnCaption = $localize`Run`
|
||||
modal.componentInstance.workflows = this.manualWorkflows()
|
||||
modal.componentInstance.confirmClicked.subscribe(() => {
|
||||
modal.componentInstance.buttonsEnabled.set(false)
|
||||
this.documentsService
|
||||
.bulkEdit({ documents: [this.document().id] }, 'run_workflow', {
|
||||
workflow: modal.componentInstance.workflowId,
|
||||
filter_documents: !modal.componentInstance.force,
|
||||
})
|
||||
.subscribe({
|
||||
next: () => {
|
||||
this.toastService.showInfo(
|
||||
$localize`Workflow will run on "${this.document().title}" in the background.`
|
||||
)
|
||||
modal.close()
|
||||
},
|
||||
error: (error) => {
|
||||
modal.componentInstance.buttonsEnabled.set(true)
|
||||
this.toastService.showError(
|
||||
$localize`Error executing operation`,
|
||||
error
|
||||
)
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private getSelectedNonLatestVersionId(): number | null {
|
||||
const versions = this.document()?.versions ?? []
|
||||
if (!versions.length || !this.selectedVersionId()) {
|
||||
|
||||
@@ -98,6 +98,11 @@
|
||||
<button ngbDropdownItem (click)="mergeSelectedAsVersions()" [disabled]="!userOwnsAll || !userCanEditAll || !userCanDelete || list.allSelected || list.selectedCount < 2">
|
||||
<pngx-bs-icon name="journal-bookmark-fill" class="me-2"></pngx-bs-icon><ng-container i18n>Merge as versions</ng-container>
|
||||
</button>
|
||||
@if (manualWorkflows.length > 0) {
|
||||
<button ngbDropdownItem (click)="runWorkflowSelected()" [disabled]="!canRunWorkflow">
|
||||
<pngx-bs-icon name="play-circle" class="me-2"></pngx-bs-icon><ng-container i18n>Run workflow</ng-container>
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -17,6 +17,7 @@ import { FILTER_TITLE } from 'src/app/data/filter-rule-type'
|
||||
import { Results } from 'src/app/data/results'
|
||||
import { StoragePath } from 'src/app/data/storage-path'
|
||||
import { Tag } from 'src/app/data/tag'
|
||||
import { WorkflowTriggerType } from 'src/app/data/workflow-trigger'
|
||||
import { FilterPipe } from 'src/app/pipes/filter.pipe'
|
||||
import { DocumentListViewService } from 'src/app/services/document-list-view.service'
|
||||
import {
|
||||
@@ -36,6 +37,7 @@ import { ShareLinkBundleService } from 'src/app/services/rest/share-link-bundle.
|
||||
import { StoragePathService } from 'src/app/services/rest/storage-path.service'
|
||||
import { TagService } from 'src/app/services/rest/tag.service'
|
||||
import { UserService } from 'src/app/services/rest/user.service'
|
||||
import { WorkflowService } from 'src/app/services/rest/workflow.service'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
import { ToastService } from 'src/app/services/toast.service'
|
||||
import { TrashService } from 'src/app/services/trash.service'
|
||||
@@ -83,6 +85,7 @@ describe('BulkEditorComponent', () => {
|
||||
let customFieldsService: CustomFieldsService
|
||||
let httpTestingController: HttpTestingController
|
||||
let shareLinkBundleService: ShareLinkBundleService
|
||||
let workflowService: WorkflowService
|
||||
let router: Router
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -173,6 +176,22 @@ describe('BulkEditorComponent', () => {
|
||||
delete: jest.fn(),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: WorkflowService,
|
||||
useValue: {
|
||||
list: jest.fn(() =>
|
||||
of({
|
||||
results: [
|
||||
{
|
||||
id: 5,
|
||||
name: 'Manual workflow',
|
||||
triggers: [{ type: WorkflowTriggerType.Manual }],
|
||||
},
|
||||
],
|
||||
})
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: Router,
|
||||
useValue: { navigate: jest.fn().mockResolvedValue(true) },
|
||||
@@ -195,6 +214,7 @@ describe('BulkEditorComponent', () => {
|
||||
customFieldsService = TestBed.inject(CustomFieldsService)
|
||||
httpTestingController = TestBed.inject(HttpTestingController)
|
||||
shareLinkBundleService = TestBed.inject(ShareLinkBundleService)
|
||||
workflowService = TestBed.inject(WorkflowService)
|
||||
router = TestBed.inject(Router)
|
||||
|
||||
fixture = TestBed.createComponent(BulkEditorComponent)
|
||||
@@ -652,6 +672,50 @@ describe('BulkEditorComponent', () => {
|
||||
) // listAllFilteredIds
|
||||
})
|
||||
|
||||
it('should load manual workflows on init', () => {
|
||||
jest.spyOn(permissionsService, 'currentUserCan').mockReturnValue(true)
|
||||
fixture.detectChanges()
|
||||
expect(workflowService.list).toHaveBeenCalledWith(1, 100000, null, null, {
|
||||
trigger_type: WorkflowTriggerType.Manual,
|
||||
})
|
||||
expect(component.manualWorkflows).toHaveLength(1)
|
||||
expect(component.manualWorkflows[0].id).toEqual(5)
|
||||
})
|
||||
|
||||
it('should run a workflow on the selection via the run workflow dialog', () => {
|
||||
let modal: NgbModalRef
|
||||
modalService.activeInstances.subscribe((m) => (modal = m[0]))
|
||||
jest.spyOn(permissionsService, 'currentUserCan').mockReturnValue(true)
|
||||
jest
|
||||
.spyOn(documentListViewService, 'documents', 'get')
|
||||
.mockReturnValue([{ id: 3 }, { id: 4 }])
|
||||
jest
|
||||
.spyOn(documentListViewService, 'selected', 'get')
|
||||
.mockReturnValue(new Set([3, 4]))
|
||||
fixture.detectChanges()
|
||||
component.runWorkflowSelected()
|
||||
expect(modal).not.toBeUndefined()
|
||||
expect(modal.componentInstance.workflows).toEqual(component.manualWorkflows)
|
||||
modal.componentInstance.workflowId = 5
|
||||
modal.componentInstance.force = true
|
||||
modal.componentInstance.confirm()
|
||||
const req = httpTestingController.expectOne(
|
||||
`${environment.apiBaseUrl}documents/bulk_edit/`
|
||||
)
|
||||
expect(req.request.body.method).toEqual('run_workflow')
|
||||
expect(req.request.body.parameters).toEqual({
|
||||
workflow: 5,
|
||||
filter_documents: false,
|
||||
})
|
||||
req.flush(true)
|
||||
httpTestingController.match(
|
||||
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
|
||||
) // list reload
|
||||
httpTestingController.match(
|
||||
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
|
||||
) // listAllFilteredIds
|
||||
})
|
||||
|
||||
it('should set modal dialog text accordingly for tag edit confirmation', () => {
|
||||
let modal: NgbModalRef
|
||||
modalService.activeInstances.subscribe((m) => (modal = m[m.length - 1]))
|
||||
|
||||
@@ -34,6 +34,7 @@ import { CustomField } from 'src/app/data/custom-field'
|
||||
import { MatchingModel } from 'src/app/data/matching-model'
|
||||
import { SelectionDataItem } from 'src/app/data/results'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { Workflow } from 'src/app/data/workflow'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import { DocumentListViewService } from 'src/app/services/document-list-view.service'
|
||||
import { OpenDocumentsService } from 'src/app/services/open-documents.service'
|
||||
@@ -53,6 +54,7 @@ import {
|
||||
} from 'src/app/services/rest/document.service'
|
||||
import { SavedViewService } from 'src/app/services/rest/saved-view.service'
|
||||
import { ShareLinkBundleService } from 'src/app/services/rest/share-link-bundle.service'
|
||||
import { WorkflowService } from 'src/app/services/rest/workflow.service'
|
||||
import { StoragePathService } from 'src/app/services/rest/storage-path.service'
|
||||
import { TagService } from 'src/app/services/rest/tag.service'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
@@ -64,6 +66,7 @@ import { MergeAsVersionsConfirmDialogComponent } from '../../common/confirm-dial
|
||||
import { MergeConfirmDialogComponent } from '../../common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
|
||||
import { ReprocessConfirmDialogComponent } from '../../common/confirm-dialog/reprocess-confirm-dialog/reprocess-confirm-dialog.component'
|
||||
import { RotateConfirmDialogComponent } from '../../common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
|
||||
import { RunWorkflowConfirmDialogComponent } from '../../common/confirm-dialog/run-workflow-confirm-dialog/run-workflow-confirm-dialog.component'
|
||||
import { CorrespondentEditDialogComponent } from '../../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component'
|
||||
import { CustomFieldEditDialogComponent } from '../../common/edit-dialog/custom-field-edit-dialog/custom-field-edit-dialog.component'
|
||||
import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
|
||||
@@ -81,6 +84,7 @@ import { PermissionsDialogComponent } from '../../common/permissions-dialog/perm
|
||||
import { ShareLinkBundleDialogComponent } from '../../common/share-link-bundle-dialog/share-link-bundle-dialog.component'
|
||||
import { ComponentWithPermissions } from '../../with-permissions/with-permissions.component'
|
||||
import { CustomFieldsBulkEditDialogComponent } from './custom-fields-bulk-edit-dialog/custom-fields-bulk-edit-dialog.component'
|
||||
import { WorkflowTriggerType } from 'src/app/data/workflow-trigger'
|
||||
|
||||
@Component({
|
||||
selector: 'pngx-bulk-editor',
|
||||
@@ -114,8 +118,11 @@ export class BulkEditorComponent
|
||||
public readonly permissionService = inject(PermissionsService)
|
||||
private savedViewService = inject(SavedViewService)
|
||||
private readonly shareLinkBundleService = inject(ShareLinkBundleService)
|
||||
private readonly workflowService = inject(WorkflowService)
|
||||
private readonly router = inject(Router)
|
||||
|
||||
public manualWorkflows: Workflow[] = []
|
||||
|
||||
tagSelectionModel = new FilterableDropdownSelectionModel(true)
|
||||
correspondentSelectionModel = new FilterableDropdownSelectionModel()
|
||||
documentTypeSelectionModel = new FilterableDropdownSelectionModel()
|
||||
@@ -258,6 +265,19 @@ export class BulkEditorComponent
|
||||
(result) => (this.customFieldsSelectionModel.items = result.results)
|
||||
)
|
||||
}
|
||||
if (
|
||||
this.permissionService.currentUserCan(
|
||||
PermissionAction.View,
|
||||
PermissionType.Workflow
|
||||
)
|
||||
) {
|
||||
this.workflowService
|
||||
.list(1, 100000, null, null, {
|
||||
trigger_type: WorkflowTriggerType.Manual,
|
||||
})
|
||||
.pipe(first())
|
||||
.subscribe((result) => (this.manualWorkflows = result.results))
|
||||
}
|
||||
|
||||
this.downloadForm
|
||||
.get('downloadFileTypeArchive')
|
||||
@@ -286,6 +306,28 @@ export class BulkEditorComponent
|
||||
this.unsubscribeNotifier.complete()
|
||||
}
|
||||
|
||||
get canRunWorkflow(): boolean {
|
||||
return this.userCanEditAll && this.manualWorkflows.length > 0
|
||||
}
|
||||
|
||||
runWorkflowSelected() {
|
||||
const modal = this.modalService.open(RunWorkflowConfirmDialogComponent, {
|
||||
backdrop: 'static',
|
||||
})
|
||||
modal.componentInstance.title = $localize`Run workflow`
|
||||
modal.componentInstance.message = $localize`Run a workflow on ${this.getSelectionSize()} selected document(s).`
|
||||
modal.componentInstance.btnCaption = $localize`Run`
|
||||
modal.componentInstance.workflows = this.manualWorkflows
|
||||
modal.componentInstance.confirmClicked
|
||||
.pipe(takeUntil(this.unsubscribeNotifier))
|
||||
.subscribe(() => {
|
||||
this.executeBulkEditMethod(modal, 'run_workflow', {
|
||||
workflow: modal.componentInstance.workflowId,
|
||||
filter_documents: !modal.componentInstance.force,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private executeBulkEditMethod(
|
||||
modal: NgbModalRef,
|
||||
method: DocumentBulkEditMethod,
|
||||
|
||||
@@ -12,6 +12,7 @@ export enum WorkflowTriggerType {
|
||||
DocumentAdded = 2,
|
||||
DocumentUpdated = 3,
|
||||
Scheduled = 4,
|
||||
Manual = 5,
|
||||
}
|
||||
|
||||
export enum ScheduleDateField {
|
||||
|
||||
@@ -38,6 +38,7 @@ export type DocumentBulkEditMethod =
|
||||
| 'modify_tags'
|
||||
| 'modify_custom_fields'
|
||||
| 'set_permissions'
|
||||
| 'run_workflow'
|
||||
|
||||
export interface MergeDocumentsRequest {
|
||||
metadata_document_id?: number
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from django.conf import settings
|
||||
from django.contrib import admin
|
||||
from guardian.admin import GuardedModelAdmin
|
||||
from treenode.admin import TreeNodeModelAdmin
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
@@ -29,7 +28,7 @@ class CorrespondentAdmin(GuardedModelAdmin):
|
||||
list_editable = ("match", "matching_algorithm")
|
||||
|
||||
|
||||
class TagAdmin(GuardedModelAdmin, TreeNodeModelAdmin):
|
||||
class TagAdmin(GuardedModelAdmin):
|
||||
list_display = ("name", "color", "match", "matching_algorithm")
|
||||
list_filter = ("matching_algorithm",)
|
||||
list_editable = ("color", "match", "matching_algorithm")
|
||||
|
||||
+27
-13
@@ -42,6 +42,7 @@ from documents.plugins.helpers import DocumentsStatusManager
|
||||
from documents.tasks import bulk_update_documents
|
||||
from documents.tasks import consume_file
|
||||
from documents.tasks import remove_document_from_index
|
||||
from documents.tasks import run_workflows_for_documents
|
||||
from documents.tasks import update_document_content_maybe_archive_file
|
||||
from documents.versioning import get_latest_version_for_root
|
||||
from documents.versioning import get_root_document
|
||||
@@ -255,19 +256,13 @@ def modify_tags(
|
||||
affected_docs = list(qs.values_list("pk", flat=True))
|
||||
DocumentTagRelationship = Document.tags.through
|
||||
|
||||
# add with all ancestors
|
||||
expanded_add_tags: set[int] = set()
|
||||
add_tag_objects = Tag.objects.filter(pk__in=add_tags)
|
||||
for t in add_tag_objects:
|
||||
expanded_add_tags.add(int(t.id))
|
||||
expanded_add_tags.update(int(pk) for pk in t.get_ancestors_pks())
|
||||
|
||||
# remove with all descendants
|
||||
expanded_remove_tags: set[int] = set()
|
||||
remove_tag_objects = Tag.objects.filter(pk__in=remove_tags)
|
||||
for t in remove_tag_objects:
|
||||
expanded_remove_tags.add(int(t.id))
|
||||
expanded_remove_tags.update(int(pk) for pk in t.get_descendants_pks())
|
||||
hierarchy = Tag.get_hierarchy()
|
||||
expanded_add_tags = hierarchy.with_ancestors(
|
||||
Tag.objects.filter(pk__in=add_tags).values_list("pk", flat=True),
|
||||
)
|
||||
expanded_remove_tags = hierarchy.with_descendants(
|
||||
Tag.objects.filter(pk__in=remove_tags).values_list("pk", flat=True),
|
||||
)
|
||||
|
||||
with transaction.atomic():
|
||||
if expanded_remove_tags:
|
||||
@@ -451,6 +446,25 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
|
||||
return "OK"
|
||||
|
||||
|
||||
def run_workflow(
|
||||
doc_ids: list[int],
|
||||
workflow: int,
|
||||
*,
|
||||
filter_documents: bool = True,
|
||||
) -> Literal["OK"]:
|
||||
"""
|
||||
Manually run a single workflow against the given documents.
|
||||
|
||||
Done asynchronously via a Celery task.
|
||||
"""
|
||||
run_workflows_for_documents.delay(
|
||||
document_ids=doc_ids,
|
||||
workflow_id=workflow,
|
||||
filter_documents=filter_documents,
|
||||
)
|
||||
return "OK"
|
||||
|
||||
|
||||
def set_permissions(
|
||||
doc_ids: list[int],
|
||||
set_permissions: PermissionsSpec,
|
||||
|
||||
@@ -35,6 +35,7 @@ from django_filters.rest_framework import DateTimeFilter
|
||||
from django_filters.rest_framework import Filter
|
||||
from django_filters.rest_framework import FilterSet
|
||||
from django_filters.rest_framework import MultipleChoiceFilter
|
||||
from django_filters.rest_framework import NumberFilter
|
||||
from drf_spectacular.utils import extend_schema_field
|
||||
from guardian.utils import get_group_obj_perms_model
|
||||
from guardian.utils import get_user_obj_perms_model
|
||||
@@ -52,6 +53,7 @@ from documents.models import ShareLink
|
||||
from documents.models import ShareLinkBundle
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.permissions import annotate_authorizing_fields
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import permitted_object_ids
|
||||
@@ -249,6 +251,16 @@ class CustomFieldFilterSet(FilterSet):
|
||||
}
|
||||
|
||||
|
||||
class WorkflowFilterSet(FilterSet):
|
||||
trigger_type = NumberFilter(field_name="triggers__type", distinct=True)
|
||||
|
||||
class Meta:
|
||||
model = Workflow
|
||||
fields = {
|
||||
"id": ID_KWARGS,
|
||||
}
|
||||
|
||||
|
||||
@extend_schema_field(serializers.CharField)
|
||||
class CustomFieldsFilter(Filter):
|
||||
def filter(self, qs, value):
|
||||
|
||||
@@ -108,6 +108,24 @@ def _deserialize_record(
|
||||
) from e
|
||||
|
||||
for field_name, field_value in record.get("fields", {}).items():
|
||||
# Older exports include django-treenode's derived bookkeeping. The
|
||||
# parent FK is authoritative; accept only these known removed fields.
|
||||
if Model is Tag and field_name in {
|
||||
"tn_ancestors_count",
|
||||
"tn_ancestors_pks",
|
||||
"tn_children_count",
|
||||
"tn_children_pks",
|
||||
"tn_depth",
|
||||
"tn_descendants_count",
|
||||
"tn_descendants_pks",
|
||||
"tn_index",
|
||||
"tn_level",
|
||||
"tn_order",
|
||||
"tn_priority",
|
||||
"tn_siblings_count",
|
||||
"tn_siblings_pks",
|
||||
}:
|
||||
continue
|
||||
field = Model._meta.get_field(field_name)
|
||||
remote = field.remote_field
|
||||
|
||||
|
||||
@@ -684,6 +684,7 @@ def document_matches_workflow(
|
||||
trigger_type == WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED
|
||||
or trigger_type == WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED
|
||||
or trigger_type == WorkflowTrigger.WorkflowTriggerType.SCHEDULED
|
||||
or trigger_type == WorkflowTrigger.WorkflowTriggerType.MANUAL
|
||||
):
|
||||
trigger_matched, reason = existing_document_matches_workflow(
|
||||
document,
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# Generated by Django 5.2.17 on 2026-10-28 22:54
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0027_documentbarcode"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_ancestors_count",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_ancestors_pks",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_children_count",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_children_pks",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_depth",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_descendants_count",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_descendants_pks",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_index",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_level",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_order",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_priority",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_siblings_count",
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name="tag",
|
||||
name="tn_siblings_pks",
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,43 @@
|
||||
# Generated by Django 5.2.18 on 2026-11-06 17:12
|
||||
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0028_replace_tag_tree_fields"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="workflowrun",
|
||||
name="type",
|
||||
field=models.PositiveSmallIntegerField(
|
||||
choices=[
|
||||
(1, "Consumption Started"),
|
||||
(2, "Document Added"),
|
||||
(3, "Document Updated"),
|
||||
(4, "Scheduled"),
|
||||
(5, "Manual"),
|
||||
],
|
||||
null=True,
|
||||
verbose_name="workflow trigger type",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="workflowtrigger",
|
||||
name="type",
|
||||
field=models.PositiveSmallIntegerField(
|
||||
choices=[
|
||||
(1, "Consumption Started"),
|
||||
(2, "Document Added"),
|
||||
(3, "Document Updated"),
|
||||
(4, "Scheduled"),
|
||||
(5, "Manual"),
|
||||
],
|
||||
default=1,
|
||||
verbose_name="Workflow Trigger Type",
|
||||
),
|
||||
),
|
||||
]
|
||||
+79
-23
@@ -1,6 +1,7 @@
|
||||
import datetime
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Final
|
||||
|
||||
import pathvalidate
|
||||
@@ -14,7 +15,6 @@ from django.db import models
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from multiselectfield import MultiSelectField
|
||||
from treenode.models import TreeNodeModel
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
from auditlog.registry import auditlog
|
||||
@@ -29,6 +29,9 @@ from django_softdelete.models import SoftDeleteModel
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.parsers import get_default_file_extension
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.tag_hierarchy import TagHierarchy
|
||||
|
||||
|
||||
class ModelWithOwner(models.Model):
|
||||
owner = models.ForeignKey(
|
||||
@@ -100,7 +103,17 @@ class Correspondent(MatchingModel):
|
||||
verbose_name_plural = _("correspondents")
|
||||
|
||||
|
||||
class Tag(MatchingModel, TreeNodeModel):
|
||||
class Tag(MatchingModel):
|
||||
# Keep the existing column and reverse relation during the transition.
|
||||
tn_parent = models.ForeignKey(
|
||||
"self",
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="tn_children",
|
||||
verbose_name=_("Parent"),
|
||||
)
|
||||
|
||||
color = models.CharField(_("color"), max_length=7, default="#a6cee3")
|
||||
# Maximum allowed nesting depth for tags (root = 1, max depth = 5)
|
||||
MAX_NESTING_DEPTH: Final[int] = 5
|
||||
@@ -114,27 +127,72 @@ class Tag(MatchingModel, TreeNodeModel):
|
||||
),
|
||||
)
|
||||
|
||||
class Meta(MatchingModel.Meta, TreeNodeModel.Meta):
|
||||
class Meta(MatchingModel.Meta):
|
||||
verbose_name = _("tag")
|
||||
verbose_name_plural = _("tags")
|
||||
|
||||
@classmethod
|
||||
def get_hierarchy(cls) -> "TagHierarchy":
|
||||
"""Read the tag hierarchy in one query, ready to use across multiple tags."""
|
||||
from documents.tag_hierarchy import TagHierarchy
|
||||
|
||||
return TagHierarchy(cls.objects.values_list("pk", "tn_parent_id"))
|
||||
|
||||
@property
|
||||
def parent_pk(self) -> int | None:
|
||||
return self.tn_parent_id
|
||||
|
||||
def get_parent(self) -> "Tag | None":
|
||||
return self.tn_parent
|
||||
|
||||
def get_ancestors_pks(self) -> list[int]:
|
||||
"""Return ancestor IDs ordered from root to parent.
|
||||
|
||||
Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
|
||||
reuse it when working with several tags.
|
||||
"""
|
||||
return self.get_hierarchy().ancestors(self.tn_parent_id)
|
||||
|
||||
def get_ancestors(self) -> list["Tag"]:
|
||||
"""Return ancestors ordered from root to parent.
|
||||
|
||||
Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
|
||||
reuse it when working with several tags.
|
||||
"""
|
||||
ids = self.get_ancestors_pks()
|
||||
tags = self.__class__.objects.in_bulk(ids)
|
||||
return [tags[pk] for pk in ids]
|
||||
|
||||
def get_descendants_pks(self) -> set[int]:
|
||||
"""Return all descendant IDs.
|
||||
|
||||
Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
|
||||
reuse it when working with several tags.
|
||||
"""
|
||||
return self.get_hierarchy().descendants([self.pk])
|
||||
|
||||
def get_descendants(self) -> "models.QuerySet[Tag]":
|
||||
"""Return a queryset of all descendants.
|
||||
|
||||
Each call reads the full hierarchy. Use Tag.get_hierarchy() once and
|
||||
reuse it when working with several tags.
|
||||
"""
|
||||
return self.__class__.objects.filter(pk__in=self.get_descendants_pks())
|
||||
|
||||
def get_children_queryset(self) -> "models.QuerySet[Tag]":
|
||||
return self.__class__.objects.filter(tn_parent_id=self.pk)
|
||||
|
||||
def clean(self) -> None:
|
||||
# Prevent self-parenting and assigning a descendant as parent
|
||||
parent = self.get_parent()
|
||||
if parent == self:
|
||||
raise ValidationError({"parent": _("Cannot set itself as parent.")})
|
||||
if parent and self.pk is not None and self.is_ancestor_of(parent):
|
||||
raise ValidationError({"parent": _("Cannot set parent to a descendant.")})
|
||||
|
||||
# Enforce maximum nesting depth
|
||||
new_parent_depth = 0
|
||||
if parent:
|
||||
new_parent_depth = parent.get_ancestors_count() + 1
|
||||
|
||||
height = 0 if self.pk is None else self.get_depth()
|
||||
deepest_new_depth = (new_parent_depth + 1) + height
|
||||
if deepest_new_depth > self.MAX_NESTING_DEPTH:
|
||||
raise ValidationError({"parent": _("Maximum nesting depth exceeded.")})
|
||||
hierarchy = self.get_hierarchy()
|
||||
if self.pk is not None and self.tn_parent_id == self.pk:
|
||||
raise ValidationError({"tn_parent": _("Cannot set itself as parent.")})
|
||||
ancestors = hierarchy.ancestors(self.tn_parent_id)
|
||||
if self.pk is not None and self.pk in ancestors:
|
||||
raise ValidationError(
|
||||
{"tn_parent": _("Cannot set parent to a descendant.")},
|
||||
)
|
||||
if len(ancestors) + 1 + hierarchy.height(self.pk) > self.MAX_NESTING_DEPTH:
|
||||
raise ValidationError({"tn_parent": _("Maximum nesting depth exceeded.")})
|
||||
|
||||
return super().clean()
|
||||
|
||||
@@ -539,10 +597,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
||||
return self.created
|
||||
|
||||
def add_nested_tags(self, tags) -> None:
|
||||
tag_ids = set()
|
||||
for tag in tags:
|
||||
tag_ids.add(tag.id)
|
||||
tag_ids.update(tag.get_ancestors_pks())
|
||||
tag_ids = Tag.get_hierarchy().with_ancestors(tag.pk for tag in tags)
|
||||
|
||||
tags_to_add = self.tags.model.objects.filter(id__in=tag_ids)
|
||||
self.tags.add(*tags_to_add)
|
||||
@@ -1477,6 +1532,7 @@ class WorkflowTrigger(models.Model):
|
||||
DOCUMENT_ADDED = 2, _("Document Added")
|
||||
DOCUMENT_UPDATED = 3, _("Document Updated")
|
||||
SCHEDULED = 4, _("Scheduled")
|
||||
MANUAL = 5, _("Manual")
|
||||
|
||||
class DocumentSourceChoices(models.IntegerChoices):
|
||||
CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder")
|
||||
|
||||
@@ -589,7 +589,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
|
||||
|
||||
text_color = serializers.SerializerMethodField()
|
||||
|
||||
# map to treenode's tn_parent
|
||||
# Preserve the public API parent field.
|
||||
parent = serializers.PrimaryKeyRelatedField(
|
||||
queryset=Tag.objects.all(),
|
||||
allow_null=True,
|
||||
@@ -692,7 +692,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
|
||||
self.instance.clean()
|
||||
except ValidationError as e:
|
||||
logger.debug("Tag parent validation failed: %s", e)
|
||||
raise e
|
||||
raise serializers.ValidationError(
|
||||
{"parent": e.message_dict["tn_parent"]},
|
||||
) from e
|
||||
finally:
|
||||
self.instance.tn_parent = original_parent
|
||||
else:
|
||||
@@ -702,7 +704,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
|
||||
temp.clean()
|
||||
except ValidationError as e:
|
||||
logger.debug("Tag parent validation failed: %s", e)
|
||||
raise e
|
||||
raise serializers.ValidationError(
|
||||
{"parent": e.message_dict["tn_parent"]},
|
||||
) from e
|
||||
|
||||
return super().validate(attrs)
|
||||
|
||||
@@ -1198,61 +1202,35 @@ class DocumentSerializer(
|
||||
doc_id,
|
||||
)
|
||||
if "tags" in validated_data:
|
||||
# Respect tag hierarchy on updates:
|
||||
# - Adding a child adds its ancestors
|
||||
# - Removing a parent removes all its descendants
|
||||
prev_tags = set(instance.tags.all())
|
||||
requested_tags = set(validated_data["tags"])
|
||||
hierarchy = Tag.get_hierarchy()
|
||||
previous_ids = set(instance.tags.values_list("pk", flat=True))
|
||||
requested_ids = {tag.pk for tag in validated_data["tags"]}
|
||||
required_ids = hierarchy.with_ancestors(requested_ids - previous_ids)
|
||||
blocked_ids = hierarchy.with_descendants(previous_ids - requested_ids)
|
||||
blocked_ids.difference_update(required_ids)
|
||||
final_ids = hierarchy.with_ancestors(requested_ids) - blocked_ids
|
||||
validated_data["tags"] = list(Tag.objects.filter(pk__in=final_ids))
|
||||
|
||||
# Tags newly added in this update and the ancestors they require
|
||||
added_tags = requested_tags - prev_tags
|
||||
required_by_add_tags = set(added_tags)
|
||||
for t in added_tags:
|
||||
required_by_add_tags.update(t.get_ancestors())
|
||||
|
||||
# Tags being removed in this update and all descendants, except
|
||||
# those required by a tag that is being added in this same update
|
||||
removed_tags = prev_tags - requested_tags
|
||||
blocked_tags = set(removed_tags)
|
||||
for t in removed_tags:
|
||||
blocked_tags.update(t.get_descendants())
|
||||
blocked_tags.difference_update(required_by_add_tags)
|
||||
|
||||
# Add all parent tags
|
||||
final_tags = set(requested_tags)
|
||||
for t in requested_tags:
|
||||
final_tags.update(t.get_ancestors())
|
||||
|
||||
# Drop removed parents and their descendants
|
||||
final_tags.difference_update(blocked_tags)
|
||||
|
||||
validated_data["tags"] = list(final_tags)
|
||||
if validated_data.get("remove_inbox_tags"):
|
||||
current_tag_ids = {t.pk for t in instance.tags.all()}
|
||||
hierarchy = Tag.get_hierarchy()
|
||||
tags = (
|
||||
validated_data["tags"]
|
||||
if "tags" in validated_data
|
||||
else list(instance.tags.all())
|
||||
)
|
||||
|
||||
# Tags newly added in this update, plus their ancestors, are kept
|
||||
keep_ids: set[int] = set()
|
||||
for tag in tags:
|
||||
if tag.pk not in current_tag_ids:
|
||||
keep_ids.add(tag.pk)
|
||||
keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
|
||||
|
||||
# Remove inbox tags and their descendants, except those being kept
|
||||
remove_ids: set[int] = set()
|
||||
for inbox_tag in (
|
||||
current_ids = set(instance.tags.values_list("pk", flat=True))
|
||||
keep_ids = hierarchy.with_ancestors(
|
||||
tag.pk for tag in tags if tag.pk not in current_ids
|
||||
)
|
||||
inbox_ids = (
|
||||
Tag.objects.filter(is_inbox_tag=True)
|
||||
.exclude(pk__in=keep_ids)
|
||||
.only("pk", "tn_descendants_pks")
|
||||
):
|
||||
remove_ids.add(inbox_tag.pk)
|
||||
remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
|
||||
|
||||
validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
|
||||
.exclude(
|
||||
pk__in=keep_ids,
|
||||
)
|
||||
.values_list("pk", flat=True)
|
||||
)
|
||||
remove_ids = hierarchy.with_descendants(inbox_ids) - keep_ids
|
||||
validated_data["tags"] = [tag for tag in tags if tag.pk not in remove_ids]
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
with set_actor(self.user):
|
||||
@@ -1851,6 +1829,7 @@ class BulkEditSerializer(
|
||||
"modify_tags",
|
||||
"modify_custom_fields",
|
||||
"set_permissions",
|
||||
"run_workflow",
|
||||
*LEGACY_DOCUMENT_ACTION_METHODS,
|
||||
],
|
||||
label="Method",
|
||||
@@ -1940,6 +1919,8 @@ class BulkEditSerializer(
|
||||
return bulk_edit.reprocess
|
||||
elif method == "set_permissions":
|
||||
return bulk_edit.set_permissions
|
||||
elif method == "run_workflow":
|
||||
return bulk_edit.run_workflow
|
||||
elif method == "rotate":
|
||||
return bulk_edit.rotate
|
||||
elif method == "merge":
|
||||
@@ -2179,6 +2160,27 @@ class BulkEditSerializer(
|
||||
else:
|
||||
parameters["remote_ocr"] = False
|
||||
|
||||
def _validate_parameters_run_workflow(self, parameters) -> None:
|
||||
if "workflow" not in parameters:
|
||||
raise serializers.ValidationError("workflow not specified")
|
||||
try:
|
||||
workflow = Workflow.objects.get(id=parameters["workflow"])
|
||||
except (Workflow.DoesNotExist, ValueError, TypeError):
|
||||
raise serializers.ValidationError("Workflow does not exist")
|
||||
if not workflow.triggers.filter(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.MANUAL,
|
||||
).exists():
|
||||
raise serializers.ValidationError(
|
||||
"Workflow does not have a manual trigger",
|
||||
)
|
||||
if "filter_documents" in parameters:
|
||||
if not isinstance(parameters["filter_documents"], bool):
|
||||
raise serializers.ValidationError(
|
||||
"filter_documents must be a boolean",
|
||||
)
|
||||
else:
|
||||
parameters["filter_documents"] = True
|
||||
|
||||
def validate_parameters_remove_password(self, parameters):
|
||||
if "password" not in parameters:
|
||||
raise serializers.ValidationError("password not specified")
|
||||
@@ -2245,6 +2247,8 @@ class BulkEditSerializer(
|
||||
self.validate_parameters_remove_password(parameters)
|
||||
elif method == bulk_edit.reprocess:
|
||||
self._validate_parameters_reprocess(parameters)
|
||||
elif method == bulk_edit.run_workflow:
|
||||
self._validate_parameters_run_workflow(parameters)
|
||||
|
||||
return attrs
|
||||
|
||||
|
||||
@@ -862,6 +862,8 @@ def run_workflows(
|
||||
logging_group: uuid.UUID | None = None,
|
||||
overrides: DocumentMetadataOverrides | None = None,
|
||||
original_file: Path | None = None,
|
||||
*,
|
||||
skip_matching: bool = False,
|
||||
) -> tuple[DocumentMetadataOverrides, str] | None:
|
||||
"""
|
||||
Execute workflows matching a document for the given trigger. When `overrides` is provided
|
||||
@@ -872,6 +874,10 @@ def run_workflows(
|
||||
`document.source_path` (Document) or `document.original_file` (ConsumableDocument).
|
||||
|
||||
Passing `workflow_to_run` skips the workflow query (currently only used by scheduled runs).
|
||||
|
||||
Passing `skip_matching` runs the workflow's actions without evaluating its trigger
|
||||
filters against the document. It is only meaningful together with `workflow_to_run`
|
||||
(e.g. a manual run the user has explicitly forced onto the selected documents).
|
||||
"""
|
||||
|
||||
use_overrides = overrides is not None
|
||||
@@ -921,7 +927,11 @@ def run_workflows(
|
||||
)
|
||||
break
|
||||
|
||||
if matching.document_matches_workflow(document, workflow, trigger_type):
|
||||
if skip_matching or matching.document_matches_workflow(
|
||||
document,
|
||||
workflow,
|
||||
trigger_type,
|
||||
):
|
||||
action: WorkflowAction
|
||||
has_move_to_trash_action = False
|
||||
for action in workflow.actions.order_by("order", "pk"):
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""An operation-local snapshot of tag edges, with no persistent derived fields.
|
||||
|
||||
Never cache this globally: tasks, bulk updates and other processes can change
|
||||
parents. Callers traversing multiple tags should share one snapshot.
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
class TagHierarchy:
|
||||
def __init__(self, edges: Iterable[tuple[int, int | None]]) -> None:
|
||||
self.parents = dict(edges)
|
||||
self.children: dict[int | None, list[int]] = defaultdict(list)
|
||||
for pk, parent in self.parents.items():
|
||||
self.children[parent].append(pk)
|
||||
|
||||
def ancestors(self, parent_id: int | None) -> list[int]:
|
||||
result = []
|
||||
seen = set()
|
||||
while parent_id is not None and parent_id not in seen:
|
||||
seen.add(parent_id)
|
||||
result.append(parent_id)
|
||||
parent_id = self.parents.get(parent_id)
|
||||
return result[::-1]
|
||||
|
||||
def with_ancestors(self, roots: Iterable[int]) -> set[int]:
|
||||
result = set(roots)
|
||||
for pk in list(result):
|
||||
result.update(self.ancestors(self.parents.get(pk)))
|
||||
return result
|
||||
|
||||
def with_descendants(self, roots: Iterable[int]) -> set[int]:
|
||||
roots = set(roots)
|
||||
return roots | self.descendants(roots)
|
||||
|
||||
def descendants(self, roots: Iterable[int]) -> set[int]:
|
||||
roots = {pk for pk in roots if pk is not None}
|
||||
seen = set(roots)
|
||||
pending = list(roots)
|
||||
while pending:
|
||||
for child in self.children.get(pending.pop(), ()):
|
||||
if child not in seen:
|
||||
seen.add(child)
|
||||
pending.append(child)
|
||||
return seen - roots
|
||||
|
||||
def height(self, pk: int | None) -> int:
|
||||
if pk is None:
|
||||
return 0
|
||||
seen = {pk}
|
||||
pending = [(pk, 0)]
|
||||
height = 0
|
||||
while pending:
|
||||
parent, depth = pending.pop()
|
||||
height = max(height, depth)
|
||||
for child in self.children.get(parent, ()):
|
||||
if child not in seen:
|
||||
seen.add(child)
|
||||
pending.append((child, depth + 1))
|
||||
return height
|
||||
@@ -52,6 +52,7 @@ from documents.models import ShareLink
|
||||
from documents.models import ShareLinkBundle
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowRun
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.plugins.base import ConsumeTaskPlugin
|
||||
@@ -690,6 +691,47 @@ def check_scheduled_workflows() -> None:
|
||||
)
|
||||
|
||||
|
||||
@shared_task
|
||||
def run_workflows_for_documents(
|
||||
document_ids: list[int],
|
||||
workflow_id: int,
|
||||
*,
|
||||
filter_documents: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Run a single workflow manually against the given documents.
|
||||
|
||||
``filter_documents`` controls whether the workflow's trigger filters are applied.
|
||||
When True, only documents matching the existing filters will have the workflow run.
|
||||
When False, the workflow will be run on all specified documents regardless of the filters.
|
||||
``
|
||||
"""
|
||||
try:
|
||||
workflow = Workflow.objects.get(id=workflow_id)
|
||||
except Workflow.DoesNotExist:
|
||||
logger.warning(
|
||||
f"Manual workflow run requested for missing workflow {workflow_id}",
|
||||
)
|
||||
return
|
||||
|
||||
for document in Document.objects.filter(
|
||||
id__in=document_ids,
|
||||
root_document__isnull=True,
|
||||
):
|
||||
run_workflows(
|
||||
trigger_type=WorkflowTrigger.WorkflowTriggerType.MANUAL,
|
||||
document=document,
|
||||
workflow_to_run=workflow,
|
||||
skip_matching=not filter_documents,
|
||||
)
|
||||
# Manual runs dont send a document_updated signal, so push a websocket
|
||||
# update here to ensure clients are refreshed
|
||||
send_websocket_document_updated(
|
||||
sender=None,
|
||||
document=document,
|
||||
)
|
||||
|
||||
|
||||
def update_document_parent_tags(tag: Tag, new_parent: Tag) -> None:
|
||||
"""
|
||||
When a tag's parent changes, ensure all documents containing the tag also have
|
||||
|
||||
@@ -14,7 +14,11 @@ from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowAction
|
||||
from documents.models import WorkflowTrigger
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.permissions import grant_all_global
|
||||
from paperless_testing.permissions import grant_global
|
||||
@@ -583,6 +587,85 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(args[0], [self.doc1.id])
|
||||
self.assertEqual(kwargs, {"remote_ocr": True})
|
||||
|
||||
def _make_manual_workflow(self) -> Workflow:
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.MANUAL,
|
||||
)
|
||||
action = WorkflowAction.objects.create()
|
||||
workflow = Workflow.objects.create(name="Manual workflow", order=0)
|
||||
workflow.triggers.add(trigger)
|
||||
workflow.actions.add(action)
|
||||
return workflow
|
||||
|
||||
def _run_workflow(self, parameters, documents=None):
|
||||
return self.client.post(
|
||||
"/api/documents/bulk_edit/",
|
||||
json.dumps(
|
||||
{
|
||||
"documents": documents or [self.doc1.id],
|
||||
"method": "run_workflow",
|
||||
"parameters": parameters,
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.run_workflow")
|
||||
def test_run_workflow(self, m) -> None:
|
||||
"""A valid run_workflow call passes the docs, workflow id and filter flag through."""
|
||||
self.setup_mock(m, "run_workflow")
|
||||
workflow = self._make_manual_workflow()
|
||||
|
||||
response = self._run_workflow({"workflow": workflow.id})
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
args, kwargs = m.call_args
|
||||
self.assertEqual(args[0], [self.doc1.id])
|
||||
self.assertEqual(kwargs, {"workflow": workflow.id, "filter_documents": True})
|
||||
|
||||
self._run_workflow({"workflow": workflow.id, "filter_documents": False})
|
||||
self.assertFalse(m.call_args.kwargs["filter_documents"])
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.run_workflow")
|
||||
def test_run_workflow_invalid_parameters(self, m) -> None:
|
||||
"""Invalid run_workflow parameters are rejected before dispatch."""
|
||||
self.setup_mock(m, "run_workflow")
|
||||
manual = self._make_manual_workflow()
|
||||
non_manual = Workflow.objects.create(name="Non-manual", order=0)
|
||||
non_manual.triggers.add(
|
||||
WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
),
|
||||
)
|
||||
for label, parameters in (
|
||||
("missing workflow", {}),
|
||||
("nonexistent workflow", {"workflow": 9999}),
|
||||
("no manual trigger", {"workflow": non_manual.id}),
|
||||
("non-bool filter", {"workflow": manual.id, "filter_documents": "nope"}),
|
||||
):
|
||||
with self.subTest(label):
|
||||
response = self._run_workflow(parameters)
|
||||
self.assertEqual(
|
||||
response.status_code,
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
m.assert_not_called()
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.run_workflow")
|
||||
def test_run_workflow_requires_ownership(self, m) -> None:
|
||||
"""A non-owner without document ownership is forbidden (actions mutate docs)."""
|
||||
self.setup_mock(m, "run_workflow")
|
||||
user = UserFactory(username="no_perms")
|
||||
grant_all_global(user)
|
||||
doc = DocumentFactory(owner=UserFactory(username="other"))
|
||||
workflow = self._make_manual_workflow()
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
response = self._run_workflow({"workflow": workflow.id}, documents=[doc.id])
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
|
||||
m.assert_not_called()
|
||||
|
||||
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")
|
||||
def test_api_set_storage_path(self, m) -> None:
|
||||
"""
|
||||
|
||||
@@ -100,6 +100,30 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
|
||||
self.action.assign_correspondent.pk,
|
||||
)
|
||||
|
||||
def test_api_get_workflows_filter_trigger_type(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A consumption workflow and a manual workflow
|
||||
WHEN:
|
||||
- Workflows are requested filtered by trigger_type
|
||||
THEN:
|
||||
- Only workflows with a matching trigger are returned
|
||||
"""
|
||||
manual_trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.MANUAL,
|
||||
)
|
||||
manual_workflow = Workflow.objects.create(name="Manual", order=1)
|
||||
manual_workflow.triggers.add(manual_trigger)
|
||||
|
||||
response = self.client.get(
|
||||
f"{self.ENDPOINT}?trigger_type={WorkflowTrigger.WorkflowTriggerType.MANUAL}",
|
||||
format="json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["count"], 1)
|
||||
self.assertEqual(response.data["results"][0]["id"], manual_workflow.id)
|
||||
|
||||
def test_api_get_workflow_actions_ordered(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from django.core.exceptions import FieldDoesNotExist
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
|
||||
from documents.management.commands.document_importer import _deserialize_record
|
||||
from documents.models import Tag
|
||||
from documents.tag_hierarchy import TagHierarchy
|
||||
from paperless_testing.factories import TagFactory
|
||||
from paperless_testing.migrations import TestMigrations
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.apps.registry import Apps
|
||||
from pytest_django.fixtures import DjangoAssertNumQueries
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTagAdjacency:
|
||||
def test_held_instance_uses_current_parent_edges(self) -> None:
|
||||
"""Descendant lookups reflect parent changes made after loading a tag."""
|
||||
first = TagFactory(name="First")
|
||||
second = TagFactory(name="Second")
|
||||
child = TagFactory(name="Child", tn_parent=first)
|
||||
Tag.objects.filter(pk=child.pk).update(tn_parent=second)
|
||||
assert first.get_descendants_pks() == set()
|
||||
assert second.get_descendants_pks() == {child.pk}
|
||||
|
||||
def test_queryset_delete_preserves_cascade(self) -> None:
|
||||
"""Deleting a parent through a queryset deletes only its subtree."""
|
||||
root = TagFactory(name="Root")
|
||||
child = TagFactory(name="Child", tn_parent=root)
|
||||
TagFactory(name="Grandchild", tn_parent=child)
|
||||
survivor = TagFactory(name="Other root")
|
||||
Tag.objects.filter(pk=root.pk).delete()
|
||||
assert list(Tag.objects.values_list("pk", flat=True)) == [survivor.pk]
|
||||
|
||||
def test_flat_writes_do_not_update_other_tags(self) -> None:
|
||||
"""Creating and renaming flat tags never rewrite unrelated tag rows."""
|
||||
TagFactory.create_batch(100)
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
new = TagFactory(name="New")
|
||||
assert not any(q["sql"].startswith("UPDATE") for q in queries)
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
new.name = "Renamed"
|
||||
new.save()
|
||||
tag_updates = [
|
||||
q["sql"] for q in queries if q["sql"].startswith('UPDATE "documents_tag"')
|
||||
]
|
||||
assert len(tag_updates) == 1
|
||||
assert 'WHERE "documents_tag"."id" =' in tag_updates[0]
|
||||
|
||||
def test_batch_snapshot_is_one_query_and_includes_overlapping_roots(
|
||||
self,
|
||||
django_assert_num_queries: "DjangoAssertNumQueries",
|
||||
) -> None:
|
||||
"""One edge query supports multiple traversals with overlapping roots."""
|
||||
root = TagFactory(name="Root")
|
||||
child = TagFactory(name="Child", tn_parent=root)
|
||||
leaf = TagFactory(name="Leaf", tn_parent=child)
|
||||
with django_assert_num_queries(1):
|
||||
hierarchy = Tag.get_hierarchy()
|
||||
assert hierarchy.with_ancestors([leaf.pk]) == {root.pk, child.pk, leaf.pk}
|
||||
assert hierarchy.with_descendants([root.pk, child.pk]) == {
|
||||
root.pk,
|
||||
child.pk,
|
||||
leaf.pk,
|
||||
}
|
||||
assert hierarchy.height(None) == 0
|
||||
|
||||
def test_legacy_export_fields_are_ignored_but_parent_is_preserved(self) -> None:
|
||||
"""Legacy bookkeeping is ignored while parent links and field checks survive."""
|
||||
model, tag, m2m = _deserialize_record(
|
||||
{
|
||||
"model": "documents.tag",
|
||||
"pk": 2,
|
||||
"fields": {
|
||||
"name": "Child",
|
||||
"tn_parent": 1,
|
||||
"tn_order": 3,
|
||||
"tn_siblings_pks": "4,5",
|
||||
"tn_descendants_pks": "6",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert model is Tag
|
||||
assert tag.tn_parent_id == 1
|
||||
assert tag.name == "Child"
|
||||
assert not m2m
|
||||
with pytest.raises(FieldDoesNotExist):
|
||||
_deserialize_record(
|
||||
{"model": "documents.tag", "pk": 3, "fields": {"tn_typo": "bad"}},
|
||||
)
|
||||
|
||||
def test_malformed_cycles_terminate(self) -> None:
|
||||
"""Traversal terminates even when stored parent edges contain a cycle."""
|
||||
hierarchy = TagHierarchy([(1, 2), (2, 1)])
|
||||
assert hierarchy.with_descendants([1]) == {1, 2}
|
||||
assert set(hierarchy.ancestors(2)) == {1, 2}
|
||||
|
||||
|
||||
class TestTagAdjacencyMigration(TestMigrations):
|
||||
migrate_from = "0027_documentbarcode"
|
||||
migrate_to = "0028_replace_tag_tree_fields"
|
||||
|
||||
def setUpBeforeMigration(self, apps: "Apps") -> None:
|
||||
"""Seed parent links using the model from the pre-migration state."""
|
||||
tag = apps.get_model("documents", "Tag")
|
||||
root = tag.objects.create(name="Root")
|
||||
child = tag.objects.create(name="Child", tn_parent=root)
|
||||
self.edges = [(root.pk, None), (child.pk, root.pk)]
|
||||
|
||||
def test_parent_edges_survive(self) -> None:
|
||||
"""Removing derived tree columns preserves the existing parent links."""
|
||||
tag = self.apps.get_model("documents", "Tag")
|
||||
assert (
|
||||
list(tag.objects.order_by("pk").values_list("pk", "tn_parent_id"))
|
||||
== self.edges
|
||||
)
|
||||
assert {
|
||||
field.name for field in tag._meta.fields if field.name.startswith("tn_")
|
||||
} == {"tn_parent"}
|
||||
@@ -2288,6 +2288,56 @@ class TestWorkflows(
|
||||
doc.refresh_from_db()
|
||||
self.assertEqual(doc.owner, self.user2)
|
||||
|
||||
def _make_manual_workflow(self, *, require_tag: Tag | None = None) -> Workflow:
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.MANUAL,
|
||||
)
|
||||
if require_tag is not None:
|
||||
trigger.filter_has_all_tags.add(require_tag)
|
||||
action = WorkflowAction.objects.create(assign_owner=self.user2)
|
||||
workflow = Workflow.objects.create(name="Manual workflow", order=0)
|
||||
workflow.triggers.add(trigger)
|
||||
workflow.actions.add(action)
|
||||
return workflow
|
||||
|
||||
@mock.patch("documents.tasks.send_websocket_document_updated")
|
||||
def test_manual_workflow_run_applies_and_notifies(self, mock_ws) -> None:
|
||||
"""A manual run applies the workflow's actions and pushes a websocket update."""
|
||||
workflow = self._make_manual_workflow()
|
||||
doc = DocumentFactory()
|
||||
|
||||
tasks.run_workflows_for_documents([doc.pk], workflow.pk)
|
||||
|
||||
doc.refresh_from_db()
|
||||
self.assertEqual(doc.owner, self.user2)
|
||||
self.assertEqual(mock_ws.call_args.kwargs["document"].pk, doc.pk)
|
||||
|
||||
def test_manual_workflow_run_respects_filters(self) -> None:
|
||||
"""With filter_documents=True, a document failing the trigger filters is skipped."""
|
||||
workflow = self._make_manual_workflow(require_tag=self.t1)
|
||||
doc = DocumentFactory()
|
||||
|
||||
tasks.run_workflows_for_documents([doc.pk], workflow.pk, filter_documents=True)
|
||||
|
||||
doc.refresh_from_db()
|
||||
self.assertIsNone(doc.owner)
|
||||
|
||||
def test_manual_workflow_run_force_ignores_filters(self) -> None:
|
||||
"""With filter_documents=False, actions run regardless of the trigger filters."""
|
||||
workflow = self._make_manual_workflow(require_tag=self.t1)
|
||||
doc = DocumentFactory()
|
||||
|
||||
tasks.run_workflows_for_documents([doc.pk], workflow.pk, filter_documents=False)
|
||||
|
||||
doc.refresh_from_db()
|
||||
self.assertEqual(doc.owner, self.user2)
|
||||
|
||||
def test_manual_workflow_run_missing_workflow(self) -> None:
|
||||
"""A missing workflow id logs a warning rather than raising."""
|
||||
doc = DocumentFactory()
|
||||
with self.assertLogs("paperless.tasks", level="WARNING"):
|
||||
tasks.run_workflows_for_documents([doc.pk], 9999)
|
||||
|
||||
@mock.patch("documents.tasks.send_websocket_document_updated")
|
||||
def test_workflow_scheduled_trigger_sends_websocket_update(
|
||||
self,
|
||||
|
||||
@@ -140,6 +140,7 @@ from documents.filters import ShareLinkFilterSet
|
||||
from documents.filters import StoragePathFilterSet
|
||||
from documents.filters import TagFilterSet
|
||||
from documents.filters import TitleContentFilter
|
||||
from documents.filters import WorkflowFilterSet
|
||||
from documents.mail import EmailAttachment
|
||||
from documents.mail import send_email
|
||||
from documents.matching import match_correspondents
|
||||
@@ -632,7 +633,7 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
|
||||
queryset = queryset.order_by(*ordering)
|
||||
|
||||
all_tags = list(queryset)
|
||||
descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
|
||||
descendant_pks = Tag.get_hierarchy().descendants(t.pk for t in all_tags)
|
||||
|
||||
if descendant_pks:
|
||||
user = getattr(getattr(self, "request", None), "user", None)
|
||||
@@ -3020,6 +3021,7 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
bulk_edit.edit_pdf,
|
||||
bulk_edit.merge_as_versions,
|
||||
bulk_edit.remove_password,
|
||||
bulk_edit.run_workflow,
|
||||
]
|
||||
)
|
||||
or (
|
||||
@@ -3159,6 +3161,7 @@ class BulkEditView(DocumentOperationPermissionMixin):
|
||||
"edit_pdf": None,
|
||||
"reprocess": "checksum",
|
||||
"remove_password": None,
|
||||
"run_workflow": None,
|
||||
}
|
||||
|
||||
serializer_class = BulkEditSerializer
|
||||
@@ -5078,12 +5081,7 @@ class BulkEditObjectsView(PassUserMixin):
|
||||
if object_type == "tags":
|
||||
editable_ids = set(user_permitted_objects.values_list("pk", flat=True))
|
||||
all_ids = set(objs.values_list("pk", flat=True))
|
||||
for tag in objs:
|
||||
all_ids.update(
|
||||
descendant.pk
|
||||
for descendant in tag.get_descendants()
|
||||
if descendant.pk in editable_ids
|
||||
)
|
||||
all_ids.update(Tag.get_hierarchy().descendants(all_ids) & editable_ids)
|
||||
objs = object_class.objects.filter(pk__in=all_ids)
|
||||
objs = objs.select_related("owner")
|
||||
object_ids = list(objs.values_list("pk", flat=True))
|
||||
@@ -5194,6 +5192,8 @@ class WorkflowViewSet(ModelViewSet[Workflow]):
|
||||
|
||||
serializer_class = WorkflowSerializer
|
||||
pagination_class = StandardPagination
|
||||
filter_backends = (DjangoFilterBackend,)
|
||||
filterset_class = WorkflowFilterSet
|
||||
|
||||
model = Workflow
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from guardian.shortcuts import remove_perm
|
||||
from documents.data_models import DocumentMetadataOverrides
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import Tag
|
||||
from documents.models import WorkflowAction
|
||||
from documents.permissions import PermissionsSpec
|
||||
from documents.permissions import set_permissions_for_objects
|
||||
@@ -134,10 +135,9 @@ def apply_assignment_to_overrides(
|
||||
if action.has_assign_tags:
|
||||
if overrides.tag_ids is None:
|
||||
overrides.tag_ids = []
|
||||
tag_ids_to_add: set[int] = set()
|
||||
for tag in action.assign_tags.all():
|
||||
tag_ids_to_add.add(tag.pk)
|
||||
tag_ids_to_add.update(int(pk) for pk in tag.get_ancestors_pks())
|
||||
tag_ids_to_add = Tag.get_hierarchy().with_ancestors(
|
||||
tag.pk for tag in action.assign_tags.all()
|
||||
)
|
||||
|
||||
overrides.tag_ids = list(set(overrides.tag_ids) | tag_ids_to_add)
|
||||
|
||||
@@ -216,10 +216,9 @@ def apply_removal_to_document(
|
||||
if action.remove_all_tags:
|
||||
Document.objects.get(pk=document.pk).tags.clear()
|
||||
else:
|
||||
tag_ids_to_remove: set[int] = set()
|
||||
for tag in action.remove_tags.all():
|
||||
tag_ids_to_remove.add(tag.pk)
|
||||
tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
|
||||
tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
|
||||
tag.pk for tag in action.remove_tags.all()
|
||||
)
|
||||
|
||||
if tag_ids_to_remove:
|
||||
Document.objects.get(pk=document.pk).tags.remove(*tag_ids_to_remove)
|
||||
@@ -297,10 +296,9 @@ def apply_removal_to_overrides(
|
||||
if action.remove_all_tags:
|
||||
overrides.tag_ids = None
|
||||
elif overrides.tag_ids:
|
||||
tag_ids_to_remove: set[int] = set()
|
||||
for tag in action.remove_tags.all():
|
||||
tag_ids_to_remove.add(tag.pk)
|
||||
tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
|
||||
tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
|
||||
tag.pk for tag in action.remove_tags.all()
|
||||
)
|
||||
|
||||
overrides.tag_ids = [t for t in overrides.tag_ids if t not in tag_ids_to_remove]
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -157,7 +157,6 @@ INSTALLED_APPS = [
|
||||
"allauth.headless",
|
||||
"drf_spectacular",
|
||||
"drf_spectacular_sidecar",
|
||||
"treenode",
|
||||
*env_apps,
|
||||
]
|
||||
|
||||
|
||||
@@ -20,8 +20,10 @@ from paperless_ai.tables import DocumentChunksTable
|
||||
from paperless_ai.tables import DocumentMetaTable
|
||||
from paperless_ai.vector_store import _MAX_IN_VALUES
|
||||
from paperless_ai.vector_store import DB_FILENAME
|
||||
from paperless_ai.vector_store import DB_PAGE_SIZE
|
||||
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
|
||||
from paperless_ai.vector_store import SCHEMA_VERSION
|
||||
from paperless_ai.vector_store import VEC0_CHUNK_SIZE
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
from paperless_ai.vector_store import _build_where
|
||||
from paperless_ai.vector_store import _pack
|
||||
@@ -1128,3 +1130,127 @@ class TestV1ToV2Migration:
|
||||
assert "DocumentChunksTable.create" not in source
|
||||
assert "DocumentMetaTable.create" not in source
|
||||
assert "_create_vec_table(" not in source
|
||||
|
||||
|
||||
LEGACY_PAGE_SIZE = 4096
|
||||
|
||||
|
||||
def vec_table_sql(db_path: Path) -> str:
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE name = ?",
|
||||
(DEFAULT_TABLE_NAME,),
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def page_size_of(db_path: Path) -> int:
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
return conn.execute("PRAGMA page_size").fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def create_legacy_vec_table(conn: sqlite3.Connection, dim: int) -> None:
|
||||
"""The table definition from before chunk_size was set explicitly."""
|
||||
conn.execute(
|
||||
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
|
||||
"id TEXT PRIMARY KEY, document_id INTEGER, +node_content TEXT, "
|
||||
f"embedding float[{dim}] distance_metric=cosine)",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def legacy_page_size_db(tmp_path: Path) -> Path:
|
||||
"""An existing index file created before the page size was set."""
|
||||
db_path = tmp_path / DB_FILENAME
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute(f"PRAGMA page_size={LEGACY_PAGE_SIZE}")
|
||||
conn.execute("CREATE TABLE _seed(x)")
|
||||
conn.execute("DROP TABLE _seed")
|
||||
conn.close()
|
||||
assert page_size_of(db_path) == LEGACY_PAGE_SIZE
|
||||
return db_path
|
||||
|
||||
|
||||
class TestVecChunkSize:
|
||||
def test_new_table_uses_configured_chunk_size(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN an empty store
|
||||
WHEN the first document is indexed and the vec0 table is created
|
||||
THEN the table definition carries the configured chunk size
|
||||
"""
|
||||
store.upsert_document(1, [make_node("1-a", 1)])
|
||||
|
||||
assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
|
||||
|
||||
def test_compact_moves_legacy_table_to_configured_chunk_size(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN an index whose vec0 table was created with the default chunk size
|
||||
WHEN the index is compacted
|
||||
THEN the rebuilt table uses the configured chunk size
|
||||
AND no indexed chunks are lost
|
||||
"""
|
||||
legacy = mocker.patch.object(
|
||||
PaperlessSqliteVecVectorStore,
|
||||
"_create_vec_table",
|
||||
staticmethod(create_legacy_vec_table),
|
||||
)
|
||||
store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
|
||||
store.upsert_document(2, [make_node("2-a", 2)])
|
||||
mocker.stop(legacy)
|
||||
assert "chunk_size" not in vec_table_sql(tmp_path / DB_FILENAME)
|
||||
|
||||
store.compact(force=True)
|
||||
|
||||
assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
|
||||
assert DocumentChunksTable.count(store.client) == 3
|
||||
|
||||
|
||||
class TestIndexPageSize:
|
||||
def test_new_index_file_uses_configured_page_size(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN no existing index file
|
||||
WHEN a store is opened and a document is indexed
|
||||
THEN the database file uses the configured page size
|
||||
"""
|
||||
store.upsert_document(1, [make_node("1-a", 1)])
|
||||
|
||||
assert page_size_of(tmp_path / DB_FILENAME) == DB_PAGE_SIZE
|
||||
|
||||
def test_compact_moves_legacy_file_to_configured_page_size(
|
||||
self,
|
||||
legacy_page_size_db: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN an existing index file created with a different page size
|
||||
WHEN the index is compacted
|
||||
THEN the rebuilt file uses the configured page size
|
||||
AND no indexed chunks are lost
|
||||
"""
|
||||
with PaperlessSqliteVecVectorStore(
|
||||
uri=str(legacy_page_size_db.parent),
|
||||
) as store:
|
||||
store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
|
||||
assert page_size_of(legacy_page_size_db) == LEGACY_PAGE_SIZE
|
||||
|
||||
store.compact(force=True)
|
||||
|
||||
assert page_size_of(legacy_page_size_db) == DB_PAGE_SIZE
|
||||
assert DocumentChunksTable.count(store.client) == 2
|
||||
@@ -47,6 +47,20 @@ SCHEMA_VERSION = 2
|
||||
# a rebuild copies the live rows into a fresh table.
|
||||
COMPACT_BLOAT_RATIO = 2.0
|
||||
|
||||
# Rows per vec0 vector chunk (vec0's default is 1024, which makes each chunk a
|
||||
# multi-megabyte blob). On spinning disks, inserts and re-indexing were dozens
|
||||
# of times faster with small chunks, at no cost in query speed or file size.
|
||||
# Only applies to tables created from here on; an existing index picks it up
|
||||
# the next time it is rebuilt (compact or rebuild).
|
||||
VEC0_CHUNK_SIZE = 32
|
||||
|
||||
# SQLite page size for the index file, in bytes. With the small vec0 chunks above,
|
||||
# 64 KiB pages made cold scans about 35% faster and warm ones about 45% faster,
|
||||
# at the cost of a larger write-ahead log (up to about 70 MB). Only takes effect
|
||||
# on a newly created file, so like the chunk size it reaches an existing index
|
||||
# when it is rebuilt.
|
||||
DB_PAGE_SIZE = 65536
|
||||
|
||||
# Number of rows fetched/copied per batch whenever this module streams rows
|
||||
# instead of materializing them all at once, keeping memory bounded regardless
|
||||
# of index size -- used by compact()'s rebuild, m0001_v1_to_v2's migration
|
||||
@@ -222,6 +236,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
conn.enable_load_extension(True) # noqa: FBT003
|
||||
sqlite_vec.load(conn)
|
||||
conn.enable_load_extension(False) # noqa: FBT003
|
||||
conn.execute(f"PRAGMA page_size={DB_PAGE_SIZE}")
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
IndexMetaTable.create(conn)
|
||||
@@ -329,7 +344,8 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
+ " +node_content TEXT,"
|
||||
+ " embedding float["
|
||||
+ str(int(dim))
|
||||
+ "] distance_metric=cosine"
|
||||
+ "] distance_metric=cosine, chunk_size="
|
||||
+ str(VEC0_CHUNK_SIZE)
|
||||
+ ")",
|
||||
)
|
||||
|
||||
|
||||
@@ -1210,15 +1210,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/99/c3/305a9f7611c0dc96d2a638bc3a67bdcde5a7faa5f4669188db018b8af6fc/django_stubs_ext-6.1.1-py3-none-any.whl", hash = "sha256:64aebb7ce2c327a4a2b75207d67034e879fd7edf027c93487be69da363dc79fa", size = 10853, upload-time = "2026-09-14T08:45:51.289Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "django-treenode"
|
||||
version = "0.25.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cb/2b/09f04b44b3583a911ae4f5d0e3360a23d529704c9e9115fce2520b7d4420/django_treenode-0.25.0.tar.gz", hash = "sha256:d14d0eb3571c227ce11147210d62a8f87b568bbb30fa76a5be9b9e6d514157c3", size = 29951, upload-time = "2026-08-26T00:20:13.489Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/ac/2140e8c20e2a47457ec23302273527d3430bf5c09edb8940d74c0725c138/django_treenode-0.25.0-py3-none-any.whl", hash = "sha256:f47caab5440420870b6287602b3ee401ca354416d44b24ab97579be5ea658927", size = 22343, upload-time = "2026-08-26T00:20:12.078Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "djangorestframework"
|
||||
version = "3.18.3"
|
||||
@@ -3107,7 +3098,6 @@ dependencies = [
|
||||
{ name = "django-multiselectfield" },
|
||||
{ name = "django-rich" },
|
||||
{ name = "django-soft-delete" },
|
||||
{ name = "django-treenode" },
|
||||
{ name = "djangorestframework" },
|
||||
{ name = "drf-spectacular" },
|
||||
{ name = "drf-spectacular-sidecar" },
|
||||
@@ -3258,7 +3248,6 @@ requires-dist = [
|
||||
{ name = "django-multiselectfield", specifier = "~=1.0.1" },
|
||||
{ name = "django-rich", specifier = "~=2.2.0" },
|
||||
{ name = "django-soft-delete", specifier = "~=1.0.18" },
|
||||
{ name = "django-treenode", specifier = ">=0.24" },
|
||||
{ name = "djangorestframework", specifier = "~=3.16" },
|
||||
{ name = "drf-spectacular", specifier = "~=0.30" },
|
||||
{ name = "drf-spectacular-sidecar", specifier = ">=2026.10.1,<2026.11" },
|
||||
|
||||
Reference in new issue
Block a user