alter defaults for workers and threads to allow more parallel tasks #446

Documentation changes

.dockerignore

@@ -1,3 +1,4 @@
+**/__pycache__
 /src-ui/.vscode
 /src-ui/node_modules
 /src-ui/dist
@@ -10,3 +11,9 @@
 .pytest_cache
 /dist
 /scripts
+/resources
+**/tests
+**/*.spec.ts
+**/htmlcov
+/src/.pytest_cache
+.idea

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,42 @@
+---
+name: Bug report
+about: Something is not working
+title: "[BUG] Concise description of the issue"
+labels: ''
+assignees: ''
+
+---
+
+<!---
+=> Before opening an issue, please check the documentation and see if it helps you resolve your issue: https://paperless-ng.readthedocs.io/en/latest/troubleshooting.html
+=> Please also make sure that you followed the installation instructions.
+=> Please search the issues and look for similar issues before opening a bug report.
+-->
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Webserver logs**
+```
+If available, post any logs from the web server related to your issue.
+```
+
+**Relevant information**
+ - Host OS of the machine running paperless: [e.g. Archlinux / Ubuntu 20.04]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 1.0.0]
+ - Installation method: [docker / bare metal]
+ - Any configuration changes you made in `docker-compose.yml`, `docker-compose.env` or `paperless.conf`.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[Feature Request] Consice and clear description of your feature request"
+labels: ''
+assignees: ''
+
+---
+
+<!--
+=> We already have lots of feature requests open. Please search the existing requests first and look for feature requests similar to yours.
+-->
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.

.github/ISSUE_TEMPLATE/other.md

@@ -0,0 +1,14 @@
+---
+name: Other
+about: Anything that is not a feature request or bug.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+<!--
+
+=> Discussions, Feedback and other suggestions belong in the "Disussion" section and not on the issue tracker.
+
+-->

.github/workflows/ansible.yml

@@ -0,0 +1,64 @@
+---
+name: Ansible Role
+
+on: [push, pull_request]
+
+jobs:
+  # https://molecule.readthedocs.io/en/latest/ci.html#github-actions
+  test:
+    runs-on: ubuntu-latest
+    # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context
+    if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/'))
+    steps:
+      - name: Check out the codebase
+        uses: actions/checkout@v2
+        with:
+          path: "${{ github.repository }}"
+      - name: Set up Python
+        uses: actions/setup-python@v2
+      - name: Set up Docker
+        uses: docker-practice/actions-setup-docker@master
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install molecule[ansible,docker]
+          ansible --version
+          docker --version
+          molecule --version
+          python --version
+      - name: Test fresh installation with molecule
+        run: |
+          cd ansible
+          molecule test -s fresh
+        working-directory: "${{ github.repository }}"
+      - name: Test release update with molecule
+        run: |
+          cd ansible
+          molecule test -s update
+        working-directory: "${{ github.repository }}"
+  # # https://galaxy.ansible.com/docs/contributing/importing.html
+  # release:
+  #   runs-on: ubuntu-latest
+  #   needs:
+  #     - test
+  #   # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context
+  #   if: contains(github.ref, 'refs/tags/')
+  #   steps:
+  #     - name: Check out the codebase
+  #       uses: actions/checkout@v2
+  #       with:
+  #         path: "${{ github.repository }}"
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v2
+  #     - name: Install dependencies
+  #       run: |
+  #         python3 -m pip install --upgrade ansible-base
+  #         ansible --version
+  #         python --version
+  #     - name: Trigger a new import on Galaxy
+  #     # TODO Check if source if pulled from cwd or imported from github
+  #     # https://github.com/ansible/ansible/blob/devel/lib/ansible/cli/galaxy.py
+  #       run: |
+  #         cd ansible
+  #         ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2)
+  #       working-directory: "${{ github.repository }}"

.github/workflows/ci.yml

@@ -0,0 +1,291 @@
+name: ci
+
+on: [push, pull_request]
+
+jobs:
+  documentation:
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      -
+        name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      -
+        name: Persistent Github pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip3.8}
+      -
+        name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev
+          pip install --upgrade pipenv
+          pipenv install --system --dev --ignore-pipfile
+      -
+        name: Make documentation
+        run: |
+          cd docs/
+          make html
+      -
+        name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: documentation
+          path: docs/_build/html/
+
+  tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ['3.6', '3.7', '3.8']
+      fail-fast: false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "${{ matrix.python-version }}"
+      -
+        name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      -
+        name: Persistent Github pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip${{ matrix.python-version }}
+      -
+        name: Prepare tests
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng
+          pip install --upgrade pipenv
+          pipenv install --system --dev --ignore-pipfile
+      -
+        name: Tests
+        run: |
+          cd src/
+          pytest
+      -
+        name: Codestyle
+        run: |
+          cd src/
+          pycodestyle
+      -
+        name: Publish coverage results
+        if: matrix.python-version == '3.8'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # https://github.com/coveralls-clients/coveralls-python/issues/251
+        run: |
+          cd src/
+          coveralls --service=github
+
+  frontend:
+    runs-on: ubuntu-20.04
+    steps:
+    -
+      name: Checkout
+      uses: actions/checkout@v2
+    -
+      uses: actions/setup-node@v2
+      with:
+        node-version: '15'
+    -
+      name: Build frontend
+      run: ./compile-frontend.sh
+    -
+      name: Upload artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: frontend-compiled
+        path: src/documents/static/frontend/
+
+  build-release:
+    needs: [frontend, documentation, tests]
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      -
+        name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext liblept5
+          pip3 install -r requirements.txt
+      -
+        name: Download frontend artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: frontend-compiled
+          path: src/documents/static/frontend/
+      -
+        name: Download documentation artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: documentation
+          path: docs/_build/html/
+      -
+        name: Move files
+        run: |
+          mkdir dist
+          mkdir dist/paperless-ng
+          mkdir dist/paperless-ng/scripts
+          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ng/
+          cp paperless.conf.example dist/paperless-ng/paperless.conf
+          cp docker/ dist/paperless-ng/docker -r
+          cp scripts/*.service scripts/*.sh dist/paperless-ng/scripts/
+          cp src/ dist/paperless-ng/src -r
+          cp docs/_build/html/ dist/paperless-ng/docs -r
+      -
+        name: Compile messages
+        run: |
+          cd dist/paperless-ng/src
+          python3 manage.py compilemessages
+      -
+        name: Collect static files
+        run: |
+          cd dist/paperless-ng/src
+          python3 manage.py collectstatic --no-input
+      -
+        name: Make release package
+        run: |
+          cd dist
+          find . -name __pycache__ | xargs rm -r
+          tar -cJf paperless-ng.tar.xz paperless-ng/
+      -
+        name: Upload release artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: release
+          path: dist/paperless-ng.tar.xz
+
+  publish-release:
+    runs-on: ubuntu-latest
+    needs: build-release
+    if: contains(github.ref, 'refs/tags/ng-')
+    steps:
+      -
+        name: Download release artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: release
+          path: ./
+      -
+        name: Get version
+        id: get_version
+        run: |
+          echo ::set-output name=version::${GITHUB_REF#refs/tags/ng-}
+      -
+        name: Create release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ng-${{ steps.get_version.outputs.version }}
+          release_name: Paperless-ng ${{ steps.get_version.outputs.version }}
+          draft: false
+          prerelease: false
+          body: |
+            For a complete list of changes, see the changelog at https://paperless-ng.readthedocs.io/en/latest/changelog.html.
+      -
+        name: Upload release archive
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: ./paperless-ng.tar.xz
+          asset_name: paperless-ng-${{ steps.get_version.outputs.version }}.tar.xz
+          asset_content_type: application/x-xz
+
+  # build and push image to docker hub.
+  build-docker-image:
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-'))
+    runs-on: ubuntu-latest
+    needs: [frontend, tests]
+    steps:
+      -
+        name: Prepare
+        id: prepare
+        run: |
+          IMAGE_NAME=jonaswinkler/paperless-ng
+          if [[ $GITHUB_REF == refs/tags/ng-* ]]; then
+            TAGS=${IMAGE_NAME}:${GITHUB_REF#refs/tags/ng-},${IMAGE_NAME}:latest
+            INSPECT_TAG=${IMAGE_NAME}:latest
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            TAGS=${IMAGE_NAME}:${GITHUB_REF#refs/heads/}
+            INSPECT_TAG=${TAGS}
+          else
+            exit 1
+          fi
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=inspect_tag::${INSPECT_TAG}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Download frontend artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: frontend-compiled
+          path: src/documents/static/frontend/
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          push: true
+          tags: ${{ steps.prepare.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+      -
+        name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ steps.prepare.outputs.inspect_tag }}

.gitignore

@@ -65,8 +65,8 @@ target/
 .virtualenv
 virtualenv
 /venv
-docker-compose.env
-docker-compose.yml
+/docker-compose.env
+/docker-compose.yml
 
 # Used for development
 scripts/import-for-development
@@ -85,3 +85,4 @@ scripts/nuke
 
 # this is where the compiled frontend is moved to.
 /src/documents/static/frontend/
+/docs/.vscode/settings.json

.readthedocs.yml

@@ -0,0 +1,16 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.7
+  install:
+    - requirements: docs/requirements.txt

.travis.yml

@@ -1,51 +0,0 @@
-language: python
-
-dist: focal
-os: linux
-
-jobs:
-  include:
-    - name: "Paperless on Python 3.6"
-      python: "3.6"
-
-    - name: "Paperless on Python 3.7"
-      python: "3.7"
-
-    - name: "Paperless on Python 3.8"
-      python: "3.8"
-
-    - name: "Documentation"
-      script:
-        - cd docs/
-        - make html
-      after_success: true
-
-    - name: "Front end"
-      language: node_js
-      node_js:
-        - 15
-      before_install: true
-      install:
-        - cd src-ui/
-        - npm install -g @angular/cli
-        - npm install
-      script:
-        - ng build --prod
-      after_success: true
-
-
-before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng
-
-install:
-  - pip install --upgrade pipenv
-  - pipenv install --system --dev
-
-script:
-  - cd src/
-  - pipenv run pytest --cov
-  - pipenv run pycodestyle
-
-after_success:
-  - pipenv run coveralls

Dockerfile

@@ -1,45 +1,79 @@
+FROM ubuntu:20.04 AS jbig2enc
+
+WORKDIR /usr/src/jbig2enc
+
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential automake libtool libleptonica-dev zlib1g-dev git ca-certificates
+
+RUN git clone https://github.com/agl/jbig2enc .
+RUN ./autogen.sh
+RUN ./configure && make
+
 FROM python:3.7-slim
 
 WORKDIR /usr/src/paperless/
 
 COPY requirements.txt ./
 
-#Dependencies
+# Binary dependencies
 RUN apt-get update \
   && apt-get -y --no-install-recommends install \
-		build-essential \
+  	# Basic dependencies
 		curl \
-		ghostscript \
+		file \
+		# fonts for text file thumbnail generation
+		fonts-liberation \
+		# for making translations further down
+		gettext \
 		gnupg \
-		icc-profiles-free \
 		imagemagick \
+		# for Numpy
 		libatlas-base-dev \
-		liblept5 \
-		libmagic-dev \
-		libpoppler-cpp-dev \
-		libpq-dev \
-		libqpdf-dev \
-		libxml2 \
+		libxslt1-dev \
+		mime-support \
+		# thumbnail size reduction
 		optipng \
+		sudo \
+		tzdata \
+  	# OCRmyPDF dependencies
+		ghostscript \
+		icc-profiles-free \
+		liblept5 \
+		libxml2 \
 		pngquant \
 		qpdf \
-		sudo \
 		tesseract-ocr \
 		tesseract-ocr-eng \
 		tesseract-ocr-deu \
 		tesseract-ocr-fra \
 		tesseract-ocr-ita \
 		tesseract-ocr-spa \
-		tzdata \
 		unpaper \
 		zlib1g \
-	&& pip3 install --upgrade supervisor setuptools \
-	&& pip install --no-cache-dir -r requirements.txt \
+		&& rm -rf /var/lib/apt/lists/*
+
+# This pulls in updated dependencies from bullseye to fix some issues with file type detection.
+# TODO: Remove this once bullseye releases.
+RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \
+  && apt-get update \
+  && apt-get install --no-install-recommends -y file libmagic-dev \
+  && rm -rf /var/lib/apt/lists/* \
+  && rm /etc/apt/sources.list.d/bullseye.list
+
+# Python dependencies
+RUN apt-get update \
+  && apt-get -y --no-install-recommends install \
+		build-essential \
+		libpoppler-cpp-dev \
+		libpq-dev \
+		libqpdf-dev \
+	&& python3 -m pip install --upgrade --no-cache-dir supervisor \
+  && python3 -m pip install --no-cache-dir -r requirements.txt \
 	&& apt-get -y purge build-essential libqpdf-dev \
 	&& apt-get -y autoremove --purge \
 	&& rm -rf /var/lib/apt/lists/* \
 	&& mkdir /var/log/supervisord /var/run/supervisord
 
+
 # copy scripts
 # this fixes issues with imagemagick and PDF
 COPY docker/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
@@ -47,6 +81,12 @@ COPY docker/gunicorn.conf.py ./
 COPY docker/supervisord.conf /etc/supervisord.conf
 COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh
 
+# copy jbig2enc
+COPY --from=jbig2enc /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
+COPY --from=jbig2enc /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
+COPY --from=jbig2enc /usr/src/jbig2enc/src/*.h /usr/local/include/
+
+
 # copy app
 COPY src/ ./src/
 
@@ -60,8 +100,11 @@ WORKDIR /usr/src/paperless/src/
 
 RUN sudo -HEu paperless python3 manage.py collectstatic --clear --no-input
 
+RUN sudo -HEu paperless python3 manage.py compilemessages
+
 VOLUME ["/usr/src/paperless/data", "/usr/src/paperless/media", "/usr/src/paperless/consume", "/usr/src/paperless/export"]
 ENTRYPOINT ["/sbin/docker-entrypoint.sh"]
+EXPOSE 8000
 CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf"]
 
 LABEL maintainer="Jonas Winkler <dev@jpwinkler.de>"

Pipfile

@@ -8,9 +8,6 @@ url = "https://www.piwheels.org/simple"
 verify_ssl = true
 name = "piwheels"
 
-[requires]
-python_version = "3.6"
-
 [packages]
 dateparser = "~=0.7.6"
 django = "~=3.1.3"
@@ -26,8 +23,9 @@ imap-tools = "*"
 langdetect = "*"
 pdftotext = "*"
 pathvalidate = "*"
-pillow = "*"
-pikepdf = "*"
+# pinned to 8.1.0, since aarch64 wheels might not be available beyond that https://github.com/python-pillow/Pillow/issues/5202
+pillow = "==8.1.0"
+pikepdf = "~=2.2.5"
 python-gnupg = "*"
 python-dotenv = "*"
 python-dateutil = "*"
@@ -35,13 +33,14 @@ python-Levenshtein = "*"
 python-magic = "*"
 psycopg2-binary = "*"
 redis = "*"
-scikit-learn="~=0.23.2"
+scikit-learn="~=0.24.0"
 whitenoise = "~=5.2.0"
 watchdog = "*"
 whoosh="~=2.7.4"
 inotifyrecursive = "~=0.3.4"
-ocrmypdf = "*"
+ocrmypdf = "~=11.4.5"
 tqdm = "*"
+tika = "*"
 
 [dev-packages]
 coveralls = "*"
@@ -53,6 +52,6 @@ pytest-django = "*"
 pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
-sphinx = "~=3.3"
+sphinx = "~=3.4.2"
 sphinx_rtd_theme = "*"
 tox = "*"

Pipfile.lock

@@ -1,12 +1,10 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "3d576f289958226a7583e4c471c7f8c11bff6933bf093185f623cfb381a92412"
+            "sha256": "3c85a487240f18b3feb44f8899395696cb79630f320f0df9ef5ee37b914c89f2"
         },
         "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3.6"
-        },
+        "requires": {},
         "sources": [
             {
                 "name": "pypi",
@@ -44,6 +42,13 @@
             ],
             "version": "==1.17.12"
         },
+        "certifi": {
+            "hashes": [
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
+            ],
+            "version": "==2020.12.5"
+        },
         "cffi": {
             "hashes": [
                 "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e",
@@ -89,50 +94,40 @@
         },
         "chardet": {
             "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
+                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
             "markers": "python_version >= '3.1'",
-            "version": "==3.0.4"
+            "version": "==4.0.0"
         },
         "coloredlogs": {
             "hashes": [
-                "sha256:346f58aad6afd48444c2468618623638dadab76e4e70d5e10822676f2d32226a",
-                "sha256:a1fab193d2053aa6c0a97608c4342d031f1f93a3d1218432c59322441d31a505",
-                "sha256:b0c2124367d4f72bd739f48e1f61491b4baf145d6bda33b606b4a53cb3f96a97"
+                "sha256:5e78691e2673a8e294499e1832bb13efcfb44a86b92e18109fa18951093218ab",
+                "sha256:b7f630a8297a66984b6bae0f6a1b0e0afb9f2f6838ea3bfa58f50d3d13e133d6"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==14.0"
+            "version": "==15.0"
         },
         "cryptography": {
             "hashes": [
-                "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538",
-                "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f",
-                "sha256:257dab4f368fae15f378ea9a4d2799bf3696668062de0e9fa0ebb7a738a6917d",
-                "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77",
-                "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b",
-                "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33",
-                "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e",
-                "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb",
-                "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e",
-                "sha256:59f7d4cfea9ef12eb9b14b83d79b432162a0a24a91ddc15c2c9bf76a68d96f2b",
-                "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7",
-                "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297",
-                "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d",
-                "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7",
-                "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b",
-                "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7",
-                "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4",
-                "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8",
-                "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b",
-                "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851",
-                "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13",
-                "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b",
-                "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3",
-                "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"
+                "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d",
+                "sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7",
+                "sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901",
+                "sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c",
+                "sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244",
+                "sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6",
+                "sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5",
+                "sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e",
+                "sha256:982f661bffc7a24b6d4f8ebe3291f17cf3833a0941c6f4d9d55c790b9aa2cdb3",
+                "sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c",
+                "sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0",
+                "sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812",
+                "sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a",
+                "sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030",
+                "sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==3.2.1"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==3.3.1"
         },
         "dateparser": {
             "hashes": [
@@ -144,19 +139,19 @@
         },
         "django": {
             "hashes": [
-                "sha256:5c866205f15e7a7123f1eec6ab939d22d5bde1416635cab259684af66d8e48a2",
-                "sha256:edb10b5c45e7e9c0fb1dc00b76ec7449aca258a39ffd613dbd078c51d19c9f03"
+                "sha256:2d78425ba74c7a1a74b196058b261b9733a8570782f4e2828974777ccca7edf7",
+                "sha256:efa2ab96b33b20c2182db93147a0c3cd7769d418926f9e9f140a60dca7c64ca9"
             ],
             "index": "pypi",
-            "version": "==3.1.4"
+            "version": "==3.1.5"
         },
         "django-cors-headers": {
             "hashes": [
-                "sha256:9322255c296d5f75089571f29e520c83ff9693df17aa3cf9f6a4bea7c6740169",
-                "sha256:db82b2840f667d47872ae3e4a4e0a0d72fbecb42779b8aa233fa8bb965f7836a"
+                "sha256:5665fc1b1aabf1b678885cf6f8f8bd7da36ef0a978375e767d491b48d3055d8f",
+                "sha256:ba898dd478cd4be3a38ebc3d8729fa4d044679f8c91b2684edee41129d7e968a"
             ],
             "index": "pypi",
-            "version": "==3.5.0"
+            "version": "==3.6.0"
         },
         "django-extensions": {
             "hashes": [
@@ -192,7 +187,8 @@
         },
         "djangorestframework": {
             "hashes": [
-                "sha256:0209bafcb7b5010fdfec784034f059d512256424de2a0f084cb82b096d6dd6a7"
+                "sha256:0209bafcb7b5010fdfec784034f059d512256424de2a0f084cb82b096d6dd6a7",
+                "sha256:0898182b4737a7b584a2c73735d89816343369f259fea932d90dc78e35d8ac33"
             ],
             "index": "pypi",
             "version": "==3.12.2"
@@ -223,19 +219,28 @@
         },
         "humanfriendly": {
             "hashes": [
-                "sha256:175ffa628aa76da2c17369a5da5856084562cc66dfe7f82ae93ca3ef175277a6",
-                "sha256:3c9ab8d28e88e6cc998e41963357736dafd555ee5bb666b50e42f6ce28dd3e3d"
+                "sha256:066562956639ab21ff2676d1fda0b5987e985c534fc76700a19bd54bcb81121d",
+                "sha256:d5c731705114b9ad673754f3317d9fa4c23212f36b29bdc4272a892eafc9bc72"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==9.0"
+            "version": "==9.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:4a57a6379512ade94fa99e2fa46d3cd0f2f553040548d0e2958c6ed90ee48226",
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.10"
         },
         "imap-tools": {
             "hashes": [
-                "sha256:72bf46dc135b039a5d5b59f4e079242ac15eac02a30038e8cb2dec7b153cab65",
-                "sha256:75dc1c72dd76d9e577df26a1e0ec3a809b5eebce77678851458dcd2eae127ac9"
+                "sha256:7d2d25b35117a3750c3b561dd93cc2fcb24cdc457830a049796c639f4371e317",
+                "sha256:80088839cd1959f20c44206cdad4463ca1e7647ff67cf5b0e31e810fb6aaa6c4"
             ],
             "index": "pypi",
-            "version": "==0.33.0"
+            "version": "==0.34.0"
         },
         "img2pdf": {
             "hashes": [
@@ -246,11 +251,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013",
-                "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"
+                "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771",
+                "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"
             ],
             "markers": "python_version < '3.8'",
-            "version": "==3.1.1"
+            "version": "==3.4.0"
         },
         "inotify-simple": {
             "hashes": [
@@ -270,11 +275,11 @@
         },
         "joblib": {
             "hashes": [
-                "sha256:698c311779f347cf6b7e6b8a39bb682277b8ee4aba8cf9507bc0cf4cd4737b72",
-                "sha256:9e284edd6be6b71883a63c9b7f124738a3c16195513ad940eae7e3438de885d5"
+                "sha256:75ead23f13484a2a414874779d69ade40d4fa1abe62b222a23cd50d4bc822f6f",
+                "sha256:7ad866067ac1fdec27d51c8678ea760601b70e32ff1881d4dc8e1171f2b64b24"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.17.0"
+            "version": "==1.0.0"
         },
         "langdetect": {
             "hashes": [
@@ -332,67 +337,60 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db",
-                "sha256:09c12096d843b90eafd01ea1b3307e78ddd47a55855ad402b157b6c4862197ce",
-                "sha256:13d166f77d6dc02c0a73c1101dd87fdf01339febec1030bd810dcd53fff3b0f1",
-                "sha256:141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512",
-                "sha256:16c1b388cc31a9baa06d91a19366fb99ddbe1c7b205293ed072211ee5bac1ed2",
-                "sha256:18bed2bcb39e3f758296584337966e68d2d5ba6aab7e038688ad53c8f889f757",
-                "sha256:1aeef46a13e51931c0b1cf8ae1168b4a55ecd282e6688fdb0a948cc5a1d5afb9",
-                "sha256:27d3f3b9e3406579a8af3a9f262f5339005dd25e0ecf3cf1559ff8a49ed5cbf2",
-                "sha256:2a2740aa9733d2e5b2dfb33639d98a64c3b0f24765fed86b0fd2aec07f6a0a08",
-                "sha256:4377e10b874e653fe96985c05feed2225c912e328c8a26541f7fc600fb9c637b",
-                "sha256:448ebb1b3bf64c0267d6b09a7cba26b5ae61b6d2dbabff7c91b660c7eccf2bdb",
-                "sha256:50e86c076611212ca62e5a59f518edafe0c0730f7d9195fec718da1a5c2bb1fc",
-                "sha256:5734bdc0342aba9dfc6f04920988140fb41234db42381cf7ccba64169f9fe7ac",
-                "sha256:5ddd1dfa2be066595c1993165b4cae84b9866b12339d0c903db7f21a094324a3",
-                "sha256:64324f64f90a9e4ef732be0928be853eee378fd6a01be21a0a8469c4f2682c83",
-                "sha256:6ae6c680f3ebf1cf7ad1d7748868b39d9f900836df774c453c11c5440bc15b36",
-                "sha256:6d7593a705d662be5bfe24111af14763016765f43cb6923ed86223f965f52387",
-                "sha256:8cac8790a6b1ddf88640a9267ee67b1aee7a57dfa2d2dd33999d080bc8ee3a0f",
-                "sha256:8ece138c3a16db8c1ad38f52eb32be6086cc72f403150a79336eb2045723a1ad",
-                "sha256:9eeb7d1d04b117ac0d38719915ae169aa6b61fca227b0b7d198d43728f0c879c",
-                "sha256:a09f98011236a419ee3f49cedc9ef27d7a1651df07810ae430a6b06576e0b414",
-                "sha256:a5d897c14513590a85774180be713f692df6fa8ecf6483e561a6d47309566f37",
-                "sha256:ad6f2ff5b1989a4899bf89800a671d71b1612e5ff40866d1f4d8bcf48d4e5764",
-                "sha256:c42c4b73121caf0ed6cd795512c9c09c52a7287b04d105d112068c1736d7c753",
-                "sha256:cb1017eec5257e9ac6209ac172058c430e834d5d2bc21961dceeb79d111e5909",
-                "sha256:d6c7bb82883680e168b55b49c70af29b84b84abb161cbac2800e8fcb6f2109b6",
-                "sha256:e452dc66e08a4ce642a961f134814258a082832c78c90351b75c41ad16f79f63",
-                "sha256:e5b6ed0f0b42317050c88022349d994fe72bfe35f5908617512cd8c8ef9da2a9",
-                "sha256:e9b30d4bd69498fc0c3fe9db5f62fffbb06b8eb9321f92cc970f2969be5e3949",
-                "sha256:ec149b90019852266fec2341ce1db513b843e496d5a8e8cdb5ced1923a92faab",
-                "sha256:edb01671b3caae1ca00881686003d16c2209e07b7ef8b7639f1867852b948f7c",
-                "sha256:f0d3929fe88ee1c155129ecd82f981b8856c5d97bcb0d5f23e9b4242e79d1de3",
-                "sha256:f29454410db6ef8126c83bd3c968d143304633d45dc57b51252afbd79d700893",
-                "sha256:fe45becb4c2f72a0907c1d0246ea6449fe7a9e2293bb0e11c4e9a32bb0930a15",
-                "sha256:fedbd128668ead37f33917820b704784aff695e0019309ad446a6d0b065b57e4"
+                "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94",
+                "sha256:06fab248a088e439402141ea04f0fffb203723148f6ee791e9c75b3e9e82f080",
+                "sha256:0eef32ca3132a48e43f6a0f5a82cb508f22ce5a3d6f67a8329c81c8e226d3f6e",
+                "sha256:1ded4fce9cfaaf24e7a0ab51b7a87be9038ea1ace7f34b841fe3b6894c721d1c",
+                "sha256:2e55195bc1c6b705bfd8ad6f288b38b11b1af32f3c8289d6c50d47f950c12e76",
+                "sha256:2ea52bd92ab9f768cc64a4c3ef8f4b2580a17af0a5436f6126b08efbd1838371",
+                "sha256:36674959eed6957e61f11c912f71e78857a8d0604171dfd9ce9ad5cbf41c511c",
+                "sha256:384ec0463d1c2671170901994aeb6dce126de0a95ccc3976c43b0038a37329c2",
+                "sha256:39b70c19ec771805081578cc936bbe95336798b7edf4732ed102e7a43ec5c07a",
+                "sha256:400580cbd3cff6ffa6293df2278c75aef2d58d8d93d3c5614cd67981dae68ceb",
+                "sha256:43d4c81d5ffdff6bae58d66a3cd7f54a7acd9a0e7b18d97abb255defc09e3140",
+                "sha256:50a4a0ad0111cc1b71fa32dedd05fa239f7fb5a43a40663269bb5dc7877cfd28",
+                "sha256:603aa0706be710eea8884af807b1b3bc9fb2e49b9f4da439e76000f3b3c6ff0f",
+                "sha256:6149a185cece5ee78d1d196938b2a8f9d09f5a5ebfbba66969302a778d5ddd1d",
+                "sha256:6373751c4b6fd325606d29dd98dc2bf7092485ad20aafbfc6a177acd3b89059e",
+                "sha256:759e4095edc3c1b3ac031f34d9459fa781777a93ccc633a472a5468587a190ff",
+                "sha256:7fb43004bce0ca31d8f13a6eb5e943fa73371381e53f7074ed21a4cb786c32f8",
+                "sha256:811daee36a58dc79cf3d8bdd4a490e4277d0e4b7d103a001a4e73ddb48e7e6aa",
+                "sha256:8b5e972b43c8fc27d56550b4120fe6257fdc15f9301914380b27f74856299fea",
+                "sha256:99abf4f353c3d1a0c7a5f27699482c987cf663b1eac20db59b8c7b061eabd7fc",
+                "sha256:a0d53e51a6cb6f0d9082decb7a4cb6dfb33055308c4c44f53103c073f649af73",
+                "sha256:a12ff4c8ddfee61f90a1633a4c4afd3f7bcb32b11c52026c92a12e1325922d0d",
+                "sha256:a4646724fba402aa7504cd48b4b50e783296b5e10a524c7a6da62e4a8ac9698d",
+                "sha256:a76f502430dd98d7546e1ea2250a7360c065a5fdea52b2dffe8ae7180909b6f4",
+                "sha256:a9d17f2be3b427fbb2bce61e596cf555d6f8a56c222bd2ca148baeeb5e5c783c",
+                "sha256:ab83f24d5c52d60dbc8cd0528759532736b56db58adaa7b5f1f76ad551416a1e",
+                "sha256:aeb9ed923be74e659984e321f609b9ba54a48354bfd168d21a2b072ed1e833ea",
+                "sha256:c843b3f50d1ab7361ca4f0b3639bf691569493a56808a0b0c54a051d260b7dbd",
+                "sha256:cae865b1cae1ec2663d8ea56ef6ff185bad091a5e33ebbadd98de2cfa3fa668f",
+                "sha256:cc6bd4fd593cb261332568485e20a0712883cf631f6f5e8e86a52caa8b2b50ff",
+                "sha256:cf2402002d3d9f91c8b01e66fbb436a4ed01c6498fffed0e4c7566da1d40ee1e",
+                "sha256:d051ec1c64b85ecc69531e1137bb9751c6830772ee5c1c426dbcfe98ef5788d7",
+                "sha256:d6631f2e867676b13026e2846180e2c13c1e11289d67da08d71cacb2cd93d4aa",
+                "sha256:dbd18bcf4889b720ba13a27ec2f2aac1981bd41203b3a3b27ba7a33f88ae4827",
+                "sha256:df609c82f18c5b9f6cb97271f03315ff0dbe481a2a02e56aeb1b1a985ce38e60"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==1.19.4"
+            "version": "==1.19.5"
         },
         "ocrmypdf": {
             "hashes": [
-                "sha256:91e7394172cedb3be801a229dbd3d308fb5ae80cbc3a77879fa7954beea407b1",
-                "sha256:e550b8e884150accab7ea41f4a576b5844594cb5cbd6ed514fbf1206720343ad"
+                "sha256:416a9c4321bfc844f250694b8c68ebb538f60609bbc8686bd9f84a13c5127d68",
+                "sha256:f45fc7e844e6026d6080a623a2936be120fc077d99aaa599df022acf35fb31e6"
             ],
             "index": "pypi",
-            "version": "==11.3.4"
-        },
-        "pathtools": {
-            "hashes": [
-                "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0",
-                "sha256:d77d982475e87f32b82157a43b09f0a5ef3e66c1d8f3c7eb8d2580e783cd8202"
-            ],
-            "version": "==0.1.2"
+            "version": "==11.4.5"
         },
         "pathvalidate": {
             "hashes": [
-                "sha256:1697c8ea71ff4c48e7aa0eda72fe4581404be8f41e51a17363ef682dd6824d35",
-                "sha256:32d30dbacb711c16bb188b12ce7e9a46b41785f50a12f64500f747480a4b6ee3"
+                "sha256:378c8b319838a255c00ab37f664686b75f0aabea4444d6c5a34effbec6738285",
+                "sha256:cae8ad5cd9223c5c1f4bc4e2ef0cd4c5e89acd2d698fdb7610ee108b9be654d2"
             ],
             "index": "pypi",
-            "version": "==2.3.0"
+            "version": "==2.3.2"
         },
         "pdfminer.six": {
             "hashes": [
@@ -411,65 +409,69 @@
         },
         "pikepdf": {
             "hashes": [
-                "sha256:0829bd5dacd73bb4a37e7575bae523f49603479755563c92ddb55c206700cab1",
-                "sha256:0d2b631077cd6af6e4d1b396208020705842610a6f13fab489d5f9c47916baa2",
-                "sha256:21c98af08fae4ac9fbcad02b613b6768a4ca300fda4cba867f4a4b6f73c2d04b",
-                "sha256:2240372fed30124ddc35b0c15a613f2b687a426ea2f150091e0a0c58cca7a495",
-                "sha256:2a97f5f1403e058d217d7f6861cf51fca200c5687bce0d052f5f2fa89b5bfa22",
-                "sha256:3faaefca0ae80d19891acec8b0dd5e6235f59f2206d82375eb80d090285e9557",
-                "sha256:48ef45b64882901c0d69af3b85d16a19bd0f3e95b43e614fefb53521d8caf36c",
-                "sha256:5212fe41f2323fc7356ba67caa39737fe13080562cff37bcbb74a8094076c8d0",
-                "sha256:56859c32170663c57bd0658189ce44e180533eebe813853446cd6413810be9eb",
-                "sha256:5f8fd1cb3478c5534222018aca24fbbd2bc74460c899bda988ec76722c13caa9",
-                "sha256:74300a32c41b3d578772f6933f23a88b19f74484185e71e5225ce2f7ea5aea78",
-                "sha256:8cbc946bdd217148f4a9c029fcea62f4ae0f67d5346de4c865f4718cd0ddc37f",
-                "sha256:9ceefd30076f732530cf84a1be2ecb2fa9931af932706ded760a6d37c73b96ad",
-                "sha256:ad69c170fda41b07a4c6b668a3128e7a759f50d9aebcfcde0ccff1358abe0423",
-                "sha256:b715fe182189fb6870fab5b0383bb2fb278c88c46eade346b0f4c1ed8818c09d",
-                "sha256:bb01ecf95083ffcb9ad542dc5342ccc1059e46f1395fd966629d36d9cc766b4a",
-                "sha256:bd6328547219cf48cefb4e0a1bc54442910594de1c5a5feae847d9ff3c629031",
-                "sha256:edb128379bb1dea76b5bdbdacf5657a6e4754bacc2049640762725590d8ed905",
-                "sha256:f8e687900557fcd4c51b4e72b9e337fdae9e2c81049d1d80b624bb2e88b5769d",
-                "sha256:fe0ca120e3347c851c34a91041d574f3c588d832023906d8ae18d66d042e8a52",
-                "sha256:fe8e0152672f24d8bfdecc725f97e9013f2de1b41849150959526ca3562bd3ef"
+                "sha256:0e67e5beeeed5422b3b8e862e4777fed5a4cd3c72e711e2a449a65d9ee641448",
+                "sha256:138155ae1f71634cd6eca79f5517f77b2067ef0bd5b627ea9414e308fe868dc5",
+                "sha256:15cf648dd760a47c55a4106b601b92bb653ae98155b10f04310553629c6695dd",
+                "sha256:1d6a011ae4c501c78509caf19cbe152c2e3cb5c267f7b47bc3db8cd3436585a7",
+                "sha256:211f529313953e44ae42eb896c2b688668385e6e8f9d04d21484bddb3c42b34c",
+                "sha256:22049ad288d603a7fc68e90a0722770d307886788373ddfe71fbf614ced0f5b2",
+                "sha256:24f7c371f6ecbee8f0ae30030992fc75cd32cd575dcfca8d466a03a8290377ca",
+                "sha256:26cdf561632866d584fedb6b1c1fce78cefa49b5cae54c65aa6a6ca5fe6de4ac",
+                "sha256:2c37afcd21a2eb1da1773687e853327fa8ec7d2c5cd90cdcd70180f55f0221e1",
+                "sha256:65b8ec6403814f51e1b9c7e18a8ff26087fcc7a199b1405583e5ff9eb931db56",
+                "sha256:66a03103aadb2e2738271cb18c89837ac3980fa0b4687195c4c150228b7e79de",
+                "sha256:6e8f0124354c53a66f83ec5a18111b760aeff1a64db3a86e7ee5fed8e8624707",
+                "sha256:70f2836cd468aa25bc8b09a2b9561364bd75d3e6ddb0e50a25d248d7da6cff25",
+                "sha256:82cebf68952cfb65c86d880eb782a0c558b37531cdae59f2e11fcd0f2bb4669c",
+                "sha256:84ad3e8fd5f3251fb5b534614da64b04a264ce9348f0fe35b781c0fb378b0f82",
+                "sha256:af13fbc022efa85d1ae161129d4cde66493479db52b9adb74d525b890a078208",
+                "sha256:c1d40fb8f8192c75f54f0e74a569ccf45e4e13bed8da78a78a5b488be29979bf",
+                "sha256:d147ec1ab58512871fdf40a161809f698eaa75720b4a230198e7e028582b20a1",
+                "sha256:dedad1f68d6b0b54000f7f99386351f1c6e19c8cf70a9700d8dd06b9809c54fb",
+                "sha256:e72c3f5b624b9c7341fd6a7e657926d4cf12a7ea453681ffd7332cabc3530c62",
+                "sha256:eb75f22e261b3bc69b6fc9a17b1d6966c95e79d3e792b7737a018a2bf6a2b07f"
             ],
             "index": "pypi",
-            "version": "==2.2.0"
+            "version": "==2.2.5"
         },
         "pillow": {
             "hashes": [
-                "sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a",
-                "sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae",
-                "sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce",
-                "sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e",
-                "sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140",
-                "sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb",
-                "sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021",
-                "sha256:5a3342d34289715928c914ee7f389351eb37fa4857caa9297fc7948f2ed3e53d",
-                "sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6",
-                "sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302",
-                "sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c",
-                "sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271",
-                "sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09",
-                "sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3",
-                "sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015",
-                "sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3",
-                "sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544",
-                "sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8",
-                "sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792",
-                "sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0",
-                "sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3",
-                "sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8",
-                "sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11",
-                "sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7",
-                "sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11",
-                "sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e",
-                "sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039",
-                "sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5",
-                "sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"
+                "sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6",
+                "sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded",
+                "sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865",
+                "sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174",
+                "sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032",
+                "sha256:3de6b2ee4f78c6b3d89d184ade5d8fa68af0848f9b6b6da2b9ab7943ec46971a",
+                "sha256:47c0d93ee9c8b181f353dbead6530b26980fe4f5485aa18be8f1fd3c3cbc685e",
+                "sha256:5e2fe3bb2363b862671eba632537cd3a823847db4d98be95690b7e382f3d6378",
+                "sha256:604815c55fd92e735f9738f65dabf4edc3e79f88541c221d292faec1904a4b17",
+                "sha256:6c5275bd82711cd3dcd0af8ce0bb99113ae8911fc2952805f1d012de7d600a4c",
+                "sha256:731ca5aabe9085160cf68b2dbef95fc1991015bc0a3a6ea46a371ab88f3d0913",
+                "sha256:7612520e5e1a371d77e1d1ca3a3ee6227eef00d0a9cddb4ef7ecb0b7396eddf7",
+                "sha256:7916cbc94f1c6b1301ac04510d0881b9e9feb20ae34094d3615a8a7c3db0dcc0",
+                "sha256:81c3fa9a75d9f1afafdb916d5995633f319db09bd773cb56b8e39f1e98d90820",
+                "sha256:887668e792b7edbfb1d3c9d8b5d8c859269a0f0eba4dda562adb95500f60dbba",
+                "sha256:8c183b5c60544b49e0a66f924b18c526dfd37774811b627f70836fe01711abd3",
+                "sha256:93a473b53cc6e0b3ce6bf51b1b95b7b1e7e6084be3a07e40f79b42e83503fbf2",
+                "sha256:96d4dc103d1a0fa6d47c6c55a47de5f5dafd5ef0114fa10c85a1fd8e0216284b",
+                "sha256:a3d3e086474ef12ef13d42e5f9b7bbf09d39cf6bd4940f982263d6954b13f6a9",
+                "sha256:b02a0b9f332086657852b1f7cb380f6a42403a6d9c42a4c34a561aa4530d5234",
+                "sha256:b09e10ec453de97f9a23a5aa5e30b334195e8d2ddd1ce76cc32e52ba63c8b31d",
+                "sha256:b6f00ad5ebe846cc91763b1d0c6d30a8042e02b2316e27b05de04fa6ec831ec5",
+                "sha256:bba80df38cfc17f490ec651c73bb37cd896bc2400cfba27d078c2135223c1206",
+                "sha256:c3d911614b008e8a576b8e5303e3db29224b455d3d66d1b2848ba6ca83f9ece9",
+                "sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8",
+                "sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59",
+                "sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d",
+                "sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7",
+                "sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a",
+                "sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0",
+                "sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b",
+                "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d",
+                "sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae"
             ],
             "index": "pypi",
-            "version": "==8.0.1"
+            "version": "==8.1.0"
         },
         "pluggy": {
             "hashes": [
@@ -574,10 +576,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
-                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
+                "sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4",
+                "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"
             ],
-            "version": "==2020.4"
+            "version": "==2020.5"
         },
         "redis": {
             "hashes": [
@@ -638,73 +640,90 @@
         },
         "reportlab": {
             "hashes": [
-                "sha256:0008b5baa39d7e3a8132c4b47ecae88d6858ad386518e754e5e7b8025ee4722b",
-                "sha256:0ad5a540c336941272fe161ef3a9830da3d4b3a65a195531cebd3cad5db58b2a",
-                "sha256:0c965a5691686d746f558ee1c52aa9c63a01a0e13cba61ffc661573948e32f61",
-                "sha256:0fd568fa5615ae99f76289c52ff230207852ee942d4934f6c893c93d2a79544e",
-                "sha256:1117d905a3404c696869c7aabec9454b43ed6acbbc73f9256c6fcea23e7ae93e",
-                "sha256:1ea7c388e91ad9d823655ad6a13751ff67e8a0e7cf4065cf051b4c931cdd9450",
-                "sha256:26c0ee8f62652cc7fcdc47a1cb3b34775a4d625738025c1a7edb8718bda5a315",
-                "sha256:368c5b3fc3d5a541cb9dcacefa563fdb445365f517e3cbf64b4326631d1cf13c",
-                "sha256:451d42fdcdd7d84587d6d9c8f5d9a7d0e997305efb606705063ca1fe8bcca551",
-                "sha256:47394acba4da8e56ef8e55d8eb483b868521696ba49ab0f0fcf8a1a4a5ac6e49",
-                "sha256:51b16e297f7b937fc530dd151e4b38f1d305b01c9aa10657bc32a5d2901b8ad7",
-                "sha256:51c0cdcf606ded0a7b4b50050400f25125ea797fbfc3c817135993b38f8b764e",
-                "sha256:55c672c579618843e0fd00140fb71f1ffebc4f1c542ac385c4f4999f2f5398d9",
-                "sha256:5c34a96ecfbf595caf16178a06abcd26a5f8720e01fe1285d4c97333382cfaeb",
-                "sha256:61aa89a00754b18c4f2956b8bff831f1fd3affef6476dc63462d92211941605e",
-                "sha256:62234d29c97279917903e4587faf240a5dea4617be250db55386ff268eb5a7c5",
-                "sha256:670f2a8dcc23bf798c39b95c64bf76ee387549b962f76783670821978a226663",
-                "sha256:69387f171f6c7b55109caa6d061b17a18f2f9e724a0212c07cd692aeb369dd19",
-                "sha256:6c5c8871b659f7c2975382d7b61f3c182701fa9eb62cf649c3c73ba8fc5e2595",
-                "sha256:80139ceb3a568f5be908094f1701fd05391b71425e8b69aaed0d30db647ca2aa",
-                "sha256:80661a76d0019b5e2c315ccd3bc7093d754067d6142b36a3a0ec4f416073d23b",
-                "sha256:85a2236f324ae336da7f4b183fa99bed261bcc00ac1255ee91a504e68b086d00",
-                "sha256:89a3acd98bd4478d6bbc5cb32e0665ea546c98bff8b58d5e1014659daa6ef75a",
-                "sha256:8a39119fcab146bde41fd1c6d148f9ee1e2cca10c6f9c2b7eb4dd710a3a2c6ac",
-                "sha256:9c31c2526401da6cc92018f68483f2aac0a731cb98435445ea4b72d46b438c84",
-                "sha256:9e8ae1c3b8a1697147c5c97f00d66ab1c54d88c4615b0cdd9b1a667d7baf3eb7",
-                "sha256:a479c38ab2b997ce05d3bef906783ac20cf4cb224a154e80c9018c5e4d943a35",
-                "sha256:a79aab8d069543d5085d58260f18705a08acd92a4501a41261913fddc2137d46",
-                "sha256:b0a8314383de853599ca531dfe55eaa49bb8d6b0bb663b2f8479b7a0f3385ea2",
-                "sha256:b3d9926e64bd8008007b2d9819d7b30179b069ce95431d5060f71afc36885389",
-                "sha256:c2a9a77ce4f25ffb52d705be82a9f41b47f6b0da23870ebc3587709e7242da30",
-                "sha256:c578dd0799f70fb577474cd383f035c6e1057e4fe837278113f9cfa6eee4b076",
-                "sha256:c5abd9d0023ad20030524ab0d5fa39d77aed025519b1fa426304ab2dd0328b89",
-                "sha256:ced96125525ba21311e9512adf391170b9e149f89e27e45b06ff07b70f97a0b2",
-                "sha256:d692fb88d6ef5e75242b00009b54953a0425eaa8bd3a36db9db8b396785e1f57",
-                "sha256:d70c2104286459658e61388af9eee838b612986bd8a36e1d21ba36152983ac15",
-                "sha256:de47c65c10ac6f0d2addb28f1b1657b1c707aca014d09d01b3b728cf19e8f791",
-                "sha256:e6e7592527791841db0820a72c6afae52655a05b0b6d4df184fd2bafe82ee1ee",
-                "sha256:e8a7e95ee6ea5566291b59ede5b9fadce809dca43ebfbfe11e3ff3d6492c6f0e",
-                "sha256:f041759138b3a95508c4281b3db3bf9bb28636d84c554272a58a5ca7c9f9bbf4",
-                "sha256:f39c7fc1fa2e4a1d9747a3effd70731a9d0e9eb5738247fa089c059eff19d43e",
-                "sha256:f65ac89ee0ba569f5279360eae08783f7f2e95c9810a9846c957fbd5950f4896"
+                "sha256:009fa61710647cdc62eb373345248d8ebb93583a058990f7c4f9be46d90aa5b1",
+                "sha256:04a08d284da86882ec3a41a7c719833362ef891b09ee8e2fbb47cee352aa684a",
+                "sha256:07bff6742fba612da8d1b1f783c436338c6fdc6962828159827d5ca7d2b67935",
+                "sha256:09fb11ab1500e679fc1b01199d2fed24435499856e75043a9ac0d31dd48fd881",
+                "sha256:18a876449c9000c391dd3415ebc8454cd7bb9e488977b894886a2d7d018f16cd",
+                "sha256:18eec161411026dde49767bee4e5e8eeb8014879554811a62581dc7433628d5b",
+                "sha256:19353aead39fc115a4d6c598d6fb9fa26da7e69160a0443ebb49b02903e704e8",
+                "sha256:1b85c20e89c22ae902ca973df2afdd2d64d27dc4ffd2b29ebad8c805a213756b",
+                "sha256:1da3d7a35f918cee905facfa94bd00ae6091cadc06dca1b0b31b69ae02d41d1d",
+                "sha256:1e484ce83dae26cb40fcbd312d45b8ba921de7856a00339d867dd4ecf145a1e7",
+                "sha256:33f3cfdc492575f8af3225701301a7e62fc478358729820c9e0091aff5831378",
+                "sha256:3b0026c1129147befd4e5a8cf25da8dea1096fce371e7b2412e36d7254019c06",
+                "sha256:3d7713dddaa8081ed709a1fa2456a43f6a74b0f07d605da8441fd53fef334f69",
+                "sha256:3e2b4d69763103b9dc9b54c0952dc3cee05cedd06e28c0987fad7f84705b12c0",
+                "sha256:4ca5233a19a5ceca23546290f43addec2345789c7d65bb32f8b2668aa148351f",
+                "sha256:5214a289cf01ebbd65e49bae83709671dd9edb601891cf0ae8abf85f3c0b392f",
+                "sha256:52f8237654acbc78ea2fa6fb4a6a06e5b023b6da93f7889adfe2deba09473fad",
+                "sha256:5ed00894e0f8281c0b7c0494b4d3067c641fd90c8e5cf933089ec4cc9a48e491",
+                "sha256:6191961533d49c9d860964d42bada4d7ac3bb28502d984feb8034093f2012fa8",
+                "sha256:6f3ad2b1afe99c436563cd436d8693d4a12e2c4bd45f70c7705759ff7837fe53",
+                "sha256:739b743b7ca1ba4b4d64c321de6fccb49b562d0507ea06c817d9cc4faed5cd22",
+                "sha256:792efba0c0c6e4ee94f6dc95f305451733ee9230a1c7d51cb8e5301a549e0dfb",
+                "sha256:79d63ca40231ca3860859b39a92daa5219035ba9553da89a5e1b218550744121",
+                "sha256:83b28104edd58ad65748d2d0e60e0d97e3b91b3e90b4573ea6fe60de6811972c",
+                "sha256:85650446538cd2f606ca234634142a7ccd74cb6db7cfec250f76a4242e0f2431",
+                "sha256:8850eba6de6eb813036eb8dce353e40d60c8af48bbce107de82770b10d3aa525",
+                "sha256:9da445cb79e3f740756924c053edc952cde11a65ff5af8acfda3c0a1317136ef",
+                "sha256:9fabd5fbd24f5971085ffe53150d663f158f7d3050b25c95736e29ebf676d454",
+                "sha256:a0c377bc45e73c3f15f55d7de69fab270d174749d5b454ab0de502b15430ec2a",
+                "sha256:a1d3f7022a920d4a5e165d264581f1862e1c1b877ceeabb96fe98cec98125ae5",
+                "sha256:a315edef5c5610b0c75790142f49487e89ea34397fc247ae8aa890fe6d6dd057",
+                "sha256:a755cca2dcf023130b03bb671670301a992157d5c3151d838c0b68ef89894536",
+                "sha256:b1b20208ecdfffd7ca027955c4fe8972b28b30a4b3b80cf25099a08d3b20ed7c",
+                "sha256:b26d6f416891cef93411d6d478a25db275766081a5fb66368248293ef459f3be",
+                "sha256:b4ba4c30af7044ee987e61c88a5ffb76031ca0c53666bc85d823b7de55ddbc75",
+                "sha256:b71faf3b6e4d7058e1af1b8afedaf39a962db4a219affc8177009d8244ec10d4",
+                "sha256:cfa854bea525f8c913cb77e2bda724d94b965a0eb3bcfc4a645a9baa29bb86e2",
+                "sha256:dd9687359e466086b9f6fe6d8069034017f8b6ca3080944fae5709767ca6814e",
+                "sha256:de0c675fc2998a7eaa929c356ba49c84f53a892e9ab25e8ee7d8ebbbdcb2ac16",
+                "sha256:e2b4e33fea2ce9d3a14ea39191b169e41eb2ac995274f54ac8fd27519974bce8",
+                "sha256:f3d4a1a273dc141e03b72a553c11bc14dd7a27ec7654a071edcf83eb04f004bc",
+                "sha256:ff547cf4c1de7e104cad1a378431ff81efcb03e90e40871ee686107da5b91442"
             ],
-            "version": "==3.5.56"
+            "version": "==3.5.59"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==2.25.1"
         },
         "scikit-learn": {
             "hashes": [
-                "sha256:090bbf144fd5823c1f2efa3e1a9bf180295b24294ca8f478e75b40ed54f8036e",
-                "sha256:0a127cc70990d4c15b1019680bfedc7fec6c23d14d3719fdf9b64b22d37cdeca",
-                "sha256:0d39748e7c9669ba648acf40fb3ce96b8a07b240db6888563a7cb76e05e0d9cc",
-                "sha256:1b8a391de95f6285a2f9adffb7db0892718950954b7149a70c783dc848f104ea",
-                "sha256:20766f515e6cd6f954554387dfae705d93c7b544ec0e6c6a5d8e006f6f7ef480",
-                "sha256:2aa95c2f17d2f80534156215c87bee72b6aa314a7f8b8fe92a2d71f47280570d",
-                "sha256:5ce7a8021c9defc2b75620571b350acc4a7d9763c25b7593621ef50f3bd019a2",
-                "sha256:6c28a1d00aae7c3c9568f61aafeaad813f0f01c729bee4fd9479e2132b215c1d",
-                "sha256:7671bbeddd7f4f9a6968f3b5442dac5f22bf1ba06709ef888cc9132ad354a9ab",
-                "sha256:914ac2b45a058d3f1338d7736200f7f3b094857758895f8667be8a81ff443b5b",
-                "sha256:98508723f44c61896a4e15894b2016762a55555fbf09365a0bb1870ecbd442de",
-                "sha256:a64817b050efd50f9abcfd311870073e500ae11b299683a519fbb52d85e08d25",
-                "sha256:cb3e76380312e1f86abd20340ab1d5b3cc46a26f6593d3c33c9ea3e4c7134028",
-                "sha256:d0dcaa54263307075cb93d0bee3ceb02821093b1b3d25f66021987d305d01dce",
-                "sha256:d9a1ce5f099f29c7c33181cc4386660e0ba891b21a60dc036bf369e3a3ee3aec",
-                "sha256:da8e7c302003dd765d92a5616678e591f347460ac7b53e53d667be7dfe6d1b10",
-                "sha256:daf276c465c38ef736a79bd79fc80a249f746bcbcae50c40945428f7ece074f8"
+                "sha256:076369634ee72b5a5941440661e2f306ff4ac30903802dc52031c7e9199ac640",
+                "sha256:18f7131e62265bf2691ed1d0303c640313894ccfe4278427478c6b2f45094b53",
+                "sha256:26f66b3726b54dfb76ea51c5d9c2431ed17ebc066cb4527662b9e851a3e7ba61",
+                "sha256:2951f87d35e72f007701c6e028aa230f6df6212a3194677c0c950486066a454d",
+                "sha256:2a5348585aa793bc8cc5a72f8e9067c9380834b0aadbd55f924843b071f13282",
+                "sha256:3eeff086f7329521d27249a082ea3c48c085cedb110db5f65968ab55c3ba2e09",
+                "sha256:4395e91b3548005f4a645018435b5a94f8cce232b5b70753020e606c6a750656",
+                "sha256:44e452ea8491225c5783d49577aad0f36202dfd52aec7f82c0fdfe5fbd5f7400",
+                "sha256:490436b44b3a1957cb625e871764b0aa330b34cc416aea4abc6c38ca63d0d682",
+                "sha256:5e6e3c042cea83f2e20a45e563b8eabc1f8f72446251fe23ebefdf111a173a33",
+                "sha256:66f27bf21202a850bcd7b6303916e4907f6e22ec59a14974ede4955aed5c7ed0",
+                "sha256:743b6edd98c98991be46c08e6b21df3861d5ae915f91d59f988384d93f7263e7",
+                "sha256:758619e49cd7c17282e6cc60d5cc73c02c072b47c9a10010bb3bb47e0d976e50",
+                "sha256:7f654befc5ad413690cc58f3f34a3e906caf825195ce0fda00a8e9565e1403e6",
+                "sha256:800aaf63f8838c00e85db2267dd226f89858594843fd03932a9eda95746d2c40",
+                "sha256:80ca024154b84b6ac4cfc86930ba13fdc348a209753bf2c16129db6f9eb8a80b",
+                "sha256:890d7d588f65acb0c4f6c083347c9076916bda5e6bd8400f06244b1afc1009af",
+                "sha256:905d8934d1e27a686698864a5863ff2c0e13a2ae1adb78a8a848aacc8a49927d",
+                "sha256:a83fcd9d59c42a2f66b307e3b0b0f08aa8e6e45be33da055697ea499f0e4f7c2",
+                "sha256:afeb06dc69847927634e58579b9cdc72e1390b79497336b2324b1b173f33bd47",
+                "sha256:b0d13fd56d26cf3de0314a4fd48037108c638fe126d813f5c1222bb0f08b6a76",
+                "sha256:c08b27cb78ee8d2dc781a7affed09859441f5b624f9f92da59ac0791c8774dfc",
+                "sha256:c912247e42114f389858ae05d63f4359d4e667ea72aaabee191aee9ad3f9774a",
+                "sha256:d7fe05fcb44eadd6d6c874c768f085f5de1239db3a3b7be4d3d23d12e4120589",
+                "sha256:d819d625832fb2969911a243e009cfa135cb8ef1e150866e417d6e9d75290087",
+                "sha256:e534f5f3796db6781c87e9835dcd51b7854c8c5a379c9210b93605965c1941fd"
             ],
             "index": "pypi",
-            "version": "==0.23.2"
+            "version": "==0.24.0"
         },
         "scipy": {
             "hashes": [
@@ -769,13 +788,30 @@
             "markers": "python_version >= '3.5'",
             "version": "==2.1.0"
         },
-        "tqdm": {
+        "tika": {
             "hashes": [
-                "sha256:38b658a3e4ecf9b4f6f8ff75ca16221ae3378b2e175d846b6b33ea3a20852cf5",
-                "sha256:d4f413aecb61c9779888c64ddf0c62910ad56dcbe857d8922bb505d4dbff0df1"
+                "sha256:c2c50f405622f74531841104f9e85c17511aede11de8e5385eab1a29a31f191b",
+                "sha256:d1f2eddb93caa9a2857569486aa2bc0320d0bf1796cdbe03066954cbc4b4bf62"
             ],
             "index": "pypi",
-            "version": "==4.54.1"
+            "version": "==1.24"
+        },
+        "tqdm": {
+            "hashes": [
+                "sha256:4621f6823bab46a9cc33d48105753ccbea671b68bab2c50a9f0be23d4065cb5a",
+                "sha256:fe3d08dd00a526850568d542ff9de9bbc2a09a791da3c334f3213d8d0bbbca65"
+            ],
+            "index": "pypi",
+            "version": "==4.56.0"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.7.4.3"
         },
         "tzlocal": {
             "hashes": [
@@ -784,13 +820,36 @@
             ],
             "version": "==2.1"
         },
+        "urllib3": {
+            "hashes": [
+                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
+                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.2"
+        },
         "watchdog": {
             "hashes": [
-                "sha256:3caefdcc8f06a57fdc5ef2d22aa7c0bfda4f55e71a0bee74cbf3176d97536ef3",
-                "sha256:e38bffc89b15bafe2a131f0e1c74924cf07dcec020c2e0a26cccd208831fcd43"
+                "sha256:016b01495b9c55b5d4126ed8ae75d93ea0d99377084107c33162df52887cee18",
+                "sha256:101532b8db506559e52a9b5d75a308729b3f68264d930670e6155c976d0e52a0",
+                "sha256:27d9b4666938d5d40afdcdf2c751781e9ce36320788b70208d0f87f7401caf93",
+                "sha256:2f1ade0d0802503fda4340374d333408831cff23da66d7e711e279ba50fe6c4a",
+                "sha256:376cbc2a35c0392b0fe7ff16fbc1b303fd99d4dd9911ab5581ee9d69adc88982",
+                "sha256:57f05e55aa603c3b053eed7e679f0a83873c540255b88d58c6223c7493833bac",
+                "sha256:5f1f3b65142175366ba94c64d8d4c8f4015825e0beaacee1c301823266b47b9b",
+                "sha256:602dbd9498592eacc42e0632c19781c3df1728ef9cbab555fab6778effc29eeb",
+                "sha256:68744de2003a5ea2dfbb104f9a74192cf381334a9e2c0ed2bbe1581828d50b61",
+                "sha256:85e6574395aa6c1e14e0f030d9d7f35c2340a6cf95d5671354ce876ac3ffdd4d",
+                "sha256:b1d723852ce90a14abf0ec0ca9e80689d9509ee4c9ee27163118d87b564a12ac",
+                "sha256:d948ad9ab9aba705f9836625b32e965b9ae607284811cd98334423f659ea537a",
+                "sha256:e2a531e71be7b5cc3499ae2d1494d51b6a26684bcc7c3146f63c810c00e8a3cc",
+                "sha256:e7c73edef48f4ceeebb987317a67e0080e5c9228601ff67b3c4062fa020403c7",
+                "sha256:ee21aeebe6b3e51e4ba64564c94cee8dbe7438b9cb60f0bb350c4fa70d1b52c2",
+                "sha256:f1d0e878fd69129d0d68b87cee5d9543f20d8018e82998efb79f7e412d42154a",
+                "sha256:f84146f7864339c8addf2c2b9903271df21d18d2c721e9a77f779493234a82b5"
             ],
             "index": "pypi",
-            "version": "==0.10.4"
+            "version": "==1.0.2"
         },
         "wcwidth": {
             "hashes": [
@@ -873,61 +932,76 @@
         },
         "chardet": {
             "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
+                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
             "markers": "python_version >= '3.1'",
-            "version": "==3.0.4"
+            "version": "==4.0.0"
         },
         "coverage": {
             "hashes": [
-                "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516",
-                "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259",
-                "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9",
-                "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097",
-                "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0",
-                "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f",
-                "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7",
-                "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c",
-                "sha256:3188a7dfd96f734a7498f37cde6598b1e9c084f1ca68bc1aa04e88db31168ab6",
-                "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5",
-                "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7",
-                "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729",
-                "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978",
-                "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9",
-                "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f",
-                "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9",
-                "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822",
-                "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418",
-                "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82",
-                "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f",
-                "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d",
-                "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221",
-                "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4",
-                "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21",
-                "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709",
-                "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54",
-                "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d",
-                "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270",
-                "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24",
-                "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751",
-                "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a",
-                "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237",
-                "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7",
-                "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636",
-                "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8",
-                "sha256:ef221855191457fffeb909d5787d1807800ab4d0111f089e6c93ee68f577634d"
+                "sha256:08b3ba72bd981531fd557f67beee376d6700fba183b167857038997ba30dd297",
+                "sha256:262066798d786ad67a13c7ba869e3ce0e39609f99f6d6c80160ad602c4808e32",
+                "sha256:2757fa64e11ec12220968f65d086b7a29b6583d16e9a544c889b22ba98555ef1",
+                "sha256:3102bb2c206700a7d28181dbe04d66b30780cde1d1c02c5f3c165cf3d2489497",
+                "sha256:3498b27d8236057def41de3585f317abae235dd3a11d33e01736ffedb2ef8606",
+                "sha256:378ac77af41350a8c6b8801a66021b52da8a05fd77e578b7380e876c0ce4f528",
+                "sha256:38f16b1317b8dd82df67ed5daa5f5e7c959e46579840d77a67a4ceb9cef0a50b",
+                "sha256:3911c2ef96e5ddc748a3c8b4702c61986628bb719b8378bf1e4a6184bbd48fe4",
+                "sha256:3a3c3f8863255f3c31db3889f8055989527173ef6192a283eb6f4db3c579d830",
+                "sha256:3b14b1da110ea50c8bcbadc3b82c3933974dbeea1832e814aab93ca1163cd4c1",
+                "sha256:535dc1e6e68fad5355f9984d5637c33badbdc987b0c0d303ee95a6c979c9516f",
+                "sha256:6f61319e33222591f885c598e3e24f6a4be3533c1d70c19e0dc59e83a71ce27d",
+                "sha256:723d22d324e7997a651478e9c5a3120a0ecbc9a7e94071f7e1954562a8806cf3",
+                "sha256:76b2775dda7e78680d688daabcb485dc87cf5e3184a0b3e012e1d40e38527cc8",
+                "sha256:782a5c7df9f91979a7a21792e09b34a658058896628217ae6362088b123c8500",
+                "sha256:7e4d159021c2029b958b2363abec4a11db0ce8cd43abb0d9ce44284cb97217e7",
+                "sha256:8dacc4073c359f40fcf73aede8428c35f84639baad7e1b46fce5ab7a8a7be4bb",
+                "sha256:8f33d1156241c43755137288dea619105477961cfa7e47f48dbf96bc2c30720b",
+                "sha256:8ffd4b204d7de77b5dd558cdff986a8274796a1e57813ed005b33fd97e29f059",
+                "sha256:93a280c9eb736a0dcca19296f3c30c720cb41a71b1f9e617f341f0a8e791a69b",
+                "sha256:9a4f66259bdd6964d8cf26142733c81fb562252db74ea367d9beb4f815478e72",
+                "sha256:9a9d4ff06804920388aab69c5ea8a77525cf165356db70131616acd269e19b36",
+                "sha256:a2070c5affdb3a5e751f24208c5c4f3d5f008fa04d28731416e023c93b275277",
+                "sha256:a4857f7e2bc6921dbd487c5c88b84f5633de3e7d416c4dc0bb70256775551a6c",
+                "sha256:a607ae05b6c96057ba86c811d9c43423f35e03874ffb03fbdcd45e0637e8b631",
+                "sha256:a66ca3bdf21c653e47f726ca57f46ba7fc1f260ad99ba783acc3e58e3ebdb9ff",
+                "sha256:ab110c48bc3d97b4d19af41865e14531f300b482da21783fdaacd159251890e8",
+                "sha256:b239711e774c8eb910e9b1ac719f02f5ae4bf35fa0420f438cdc3a7e4e7dd6ec",
+                "sha256:be0416074d7f253865bb67630cf7210cbc14eb05f4099cc0f82430135aaa7a3b",
+                "sha256:c46643970dff9f5c976c6512fd35768c4a3819f01f61169d8cdac3f9290903b7",
+                "sha256:c5ec71fd4a43b6d84ddb88c1df94572479d9a26ef3f150cef3dacefecf888105",
+                "sha256:c6e5174f8ca585755988bc278c8bb5d02d9dc2e971591ef4a1baabdf2d99589b",
+                "sha256:c89b558f8a9a5a6f2cfc923c304d49f0ce629c3bd85cb442ca258ec20366394c",
+                "sha256:cc44e3545d908ecf3e5773266c487ad1877be718d9dc65fc7eb6e7d14960985b",
+                "sha256:cc6f8246e74dd210d7e2b56c76ceaba1cc52b025cd75dbe96eb48791e0250e98",
+                "sha256:cd556c79ad665faeae28020a0ab3bda6cd47d94bec48e36970719b0b86e4dcf4",
+                "sha256:ce6f3a147b4b1a8b09aae48517ae91139b1b010c5f36423fa2b866a8b23df879",
+                "sha256:ceb499d2b3d1d7b7ba23abe8bf26df5f06ba8c71127f188333dddcf356b4b63f",
+                "sha256:cef06fb382557f66d81d804230c11ab292d94b840b3cb7bf4450778377b592f4",
+                "sha256:e448f56cfeae7b1b3b5bcd99bb377cde7c4eb1970a525c770720a352bc4c8044",
+                "sha256:e52d3d95df81c8f6b2a1685aabffadf2d2d9ad97203a40f8d61e51b70f191e4e",
+                "sha256:eb33c4c858d06bd8d79713c7628d3f2b50fb1c62071e2e88cb44876be03eabe1",
+                "sha256:ee2f1d1c223c3d2c24e3afbb2dd38be3f03b1a8d6a83ee3d9eb8c36a52bee899",
+                "sha256:f2c6888eada180814b8583c3e793f3f343a692fc802546eed45f40a001b1169f",
+                "sha256:f51dbba78d68a44e99d484ca8c8f604f17e957c1ca09c3ebc2c7e3bbd9ba0448",
+                "sha256:f54de00baf200b4539a5a092a759f000b5f45fd226d6d25a76b0dff71177a714",
+                "sha256:fa10fee7e32213f5c7b0d6428ea92e3a3fdd6d725590238a3f92c0de1c78b9d2",
+                "sha256:fabeeb121735d47d8eab8671b6b031ce08514c86b7ad8f7d5490a7b6dcd6267d",
+                "sha256:fac3c432851038b3e6afe086f777732bcf7f6ebbfd90951fa04ee53db6d0bcdd",
+                "sha256:fda29412a66099af6d6de0baa6bd7c52674de177ec2ad2630ca264142d69c6c7",
+                "sha256:ff1330e8bc996570221b450e2d539134baa9465f5cb98aff0e0f73f34172e0ae"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==5.3"
+            "version": "==5.3.1"
         },
         "coveralls": {
             "hashes": [
-                "sha256:2301a19500b06649d2ec4f2858f9c69638d7699a4c63027c5d53daba666147cc",
-                "sha256:b990ba1f7bc4288e63340be0433698c1efe8217f78c689d254c2540af3d38617"
+                "sha256:5399c0565ab822a70a477f7031f6c88a9dd196b3de2877b3facb43b51bd13434",
+                "sha256:f8384968c57dee4b7133ae701ecdad88e85e30597d496dcba0d7fbb470dca41f"
             ],
             "index": "pypi",
-            "version": "==2.2.0"
+            "version": "==3.0.0"
         },
         "distlib": {
             "hashes": [
@@ -961,19 +1035,19 @@
         },
         "factory-boy": {
             "hashes": [
-                "sha256:d8626622550c8ba31392f9e19fdbcef9f139cf1ad643c5923f20490a7b3e2e3d",
-                "sha256:ded73e49135c24bd4d3f45bf1eb168f8d290090f5cf4566b8df3698317dc9c08"
+                "sha256:1d3db4b44b8c8c54cdd8b83ae4bdb9aeb121e464400035f1f03ae0e1eade56a4",
+                "sha256:401cc00ff339a022f84d64a4339503d1689e8263a4478d876e58a3295b155c5b"
             ],
             "index": "pypi",
-            "version": "==3.1.0"
+            "version": "==3.2.0"
         },
         "faker": {
             "hashes": [
-                "sha256:1fcb415562ee6e2395b041e85fa6901d4708d30b84d54015226fa754ed0822c3",
-                "sha256:e8beccb398ee9b8cc1a91d9295121d66512b6753b4846eb1e7370545d46b3311"
+                "sha256:47ac7d62d5bad8c16422a91f121430ab7656d40ca8fea9c84bcdbdf92e739b03",
+                "sha256:6bc44606d44f711e1d89ad9a5b42394cc6f7eedaffc765ddb5b2d22084c15733"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==5.0.1"
+            "version": "==5.5.0"
         },
         "filelock": {
             "hashes": [
@@ -1002,19 +1076,19 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013",
-                "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"
+                "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771",
+                "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"
             ],
             "markers": "python_version < '3.8'",
-            "version": "==3.1.1"
+            "version": "==3.4.0"
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:7b51f0106c8ec564b1bef3d9c588bc694ce2b92125bbb6278f4f2f5b54ec3592",
-                "sha256:a3d34a8464ce1d5d7c92b0ea4e921e696d86f2aa212e684451cb1482c8d84ed5"
+                "sha256:4743f090ed8946e713745ec0e660249ef9fb0b9843eacc5b5ff931d2fd5aa67f",
+                "sha256:ea17df80a0ff04b5dbd3d96dbeab1842acfd1c6c902eaeb8c8858abf2720161e"
             ],
             "markers": "python_version < '3.7'",
-            "version": "==3.3.0"
+            "version": "==5.0.0"
         },
         "iniconfig": {
             "hashes": [
@@ -1077,11 +1151,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236",
-                "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"
+                "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858",
+                "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.7"
+            "version": "==20.8"
         },
         "pluggy": {
             "hashes": [
@@ -1093,11 +1167,11 @@
         },
         "py": {
             "hashes": [
-                "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2",
-                "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"
+                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
+                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.9.0"
+            "version": "==1.10.0"
         },
         "pycodestyle": {
             "hashes": [
@@ -1109,11 +1183,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:ccf3acacf3782cbed4a989426012f1c535c9a90d3a7fc3f16d231b9372d2b716",
-                "sha256:f275b6c0909e5dafd2d6269a656aa90fa58ebf4a74f8fcf9053195d226b24a08"
+                "sha256:bc9591213a8f0e0ca1a5e68a479b4887fdc3e75d0774e5c71c31920c427de435",
+                "sha256:df49d09b498e83c1a73128295860250b0b7edd4c723a32e9bc0d295c7c2ec337"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==2.7.3"
+            "version": "==2.7.4"
         },
         "pyparsing": {
             "hashes": [
@@ -1125,11 +1199,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe",
-                "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"
+                "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8",
+                "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"
             ],
             "index": "pypi",
-            "version": "==6.1.2"
+            "version": "==6.2.1"
         },
         "pytest-cov": {
             "hashes": [
@@ -1174,11 +1248,11 @@
         },
         "pytest-xdist": {
             "hashes": [
-                "sha256:7c629016b3bb006b88ac68e2b31551e7becf173c76b977768848e2bbed594d90",
-                "sha256:82d938f1a24186520e2d9d3a64ef7d9ac7ecdf1a0659e095d18e596b8cbd0672"
+                "sha256:1d8edbb1a45e8e1f8e44b1260583107fc23f8bc8da6d18cb331ff61d41258ecf",
+                "sha256:f127e11e84ad37cc1de1088cb2990f3c354630d428af3f71282de589c5bb779b"
             ],
             "index": "pypi",
-            "version": "==2.1.0"
+            "version": "==2.2.0"
         },
         "python-dateutil": {
             "hashes": [
@@ -1190,18 +1264,18 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
-                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
+                "sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4",
+                "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"
             ],
-            "version": "==2020.4"
+            "version": "==2020.5"
         },
         "requests": {
             "hashes": [
-                "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8",
-                "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==2.25.0"
+            "version": "==2.25.1"
         },
         "six": {
             "hashes": [
@@ -1220,19 +1294,19 @@
         },
         "sphinx": {
             "hashes": [
-                "sha256:1e8d592225447104d1172be415bc2972bd1357e3e12fdc76edf2261105db4300",
-                "sha256:d4e59ad4ea55efbb3c05cde3bfc83bfc14f0c95aa95c3d75346fcce186a47960"
+                "sha256:41cad293f954f7d37f803d97eb184158cfd90f51195131e94875bc07cd08b93c",
+                "sha256:c314c857e7cd47c856d2c5adff514ac2e6495f8b8e0f886a8a37e9305dfea0d8"
             ],
             "index": "pypi",
-            "version": "==3.3.1"
+            "version": "==3.4.3"
         },
         "sphinx-rtd-theme": {
             "hashes": [
-                "sha256:22c795ba2832a169ca301cd0a083f7a434e09c538c70beb42782c073651b707d",
-                "sha256:373413d0f82425aaa28fb288009bf0d0964711d347763af2f1b65cafcb028c82"
+                "sha256:eda689eda0c7301a80cf122dad28b1861e5605cbf455558f3775e1e8200e83a5",
+                "sha256:fa6bebd5ab9a73da8e102509a86f3fcc36dec04a0b52ea80e5a033b2aba00113"
             ],
             "index": "pypi",
-            "version": "==0.5.0"
+            "version": "==0.5.1"
         },
         "sphinxcontrib-applehelp": {
             "hashes": [
@@ -1306,11 +1380,20 @@
         },
         "tox": {
             "hashes": [
-                "sha256:42ce19ce5dc2f6d6b1fdc5666c476e1f1e2897359b47e0aa3a5b774f335d57c2",
-                "sha256:4321052bfe28f9d85082341ca8e233e3ea901fdd14dab8a5d3fbd810269fbaf6"
+                "sha256:5efda30ad73e662c3844ac51ce1381bf28f61063773e06996aa8b6277133a7c0",
+                "sha256:8cccede64802e78aa6c69f81051b25f0706639d1cbbb34d9366ce00c70ee054f"
             ],
             "index": "pypi",
-            "version": "==3.20.1"
+            "version": "==3.21.0"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.7.4.3"
         },
         "urllib3": {
             "hashes": [
@@ -1322,11 +1405,11 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:54b05fc737ea9c9ee9f8340f579e5da5b09fb64fd010ab5757eb90268616907c",
-                "sha256:b7a8ec323ee02fb2312f098b6b4c9de99559b462775bc8fe3627a73706603c1b"
+                "sha256:205a7577275dd0d9223c730dd498e21a8910600085c3dee97412b041fc4b853b",
+                "sha256:7992b8de87e544a4ab55afc2240bf8388c4e3b5765d03784dad384bfdf9097ee"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.2.2"
+            "version": "==20.3.0"
         },
         "zipp": {
             "hashes": [

README.md

@@ -1,15 +1,18 @@
-[![Build Status](https://travis-ci.org/jonaswinkler/paperless-ng.svg?branch=master)](https://travis-ci.org/jonaswinkler/paperless-ng)
+[![ci](https://github.com/jonaswinkler/paperless-ng/workflows/ci/badge.svg)](https://github.com/jonaswinkler/paperless-ng/actions)
 [![Documentation Status](https://readthedocs.org/projects/paperless-ng/badge/?version=latest)](https://paperless-ng.readthedocs.io/en/latest/?badge=latest)
+[![Gitter](https://badges.gitter.im/paperless-ng/community.svg)](https://gitter.im/paperless-ng/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Docker Hub Pulls](https://img.shields.io/docker/pulls/jonaswinkler/paperless-ng.svg)](https://hub.docker.com/r/jonaswinkler/paperless-ng)
 [![Coverage Status](https://coveralls.io/repos/github/jonaswinkler/paperless-ng/badge.svg?branch=master)](https://coveralls.io/github/jonaswinkler/paperless-ng?branch=master)
 
 # Paperless-ng
 
-[Paperless](https://github.com/the-paperless-project/paperless) is an application by Daniel Quinn and others that indexes your scanned documents and allows you to easily search for documents and store metadata alongside your documents.
+[Paperless](https://github.com/the-paperless-project/paperless) is an application by Daniel Quinn and contributors that indexes your scanned documents and allows you to easily search for documents and store metadata alongside your documents.
 
 Paperless-ng is a fork of the original project, adding a new interface and many other changes under the hood. For a detailed list of changes, have a look at the changelog in the documentation.
 
-This project is still in development and some things may not work as expected.
+# Survey
+
+If you already used Paperless-ng for a bit, would like to give some anonymous feedback, and help me decide on what to focus on next: I've created a survey, [see here](https://github.com/jonaswinkler/paperless-ng/issues/402). Thank you!
 
 # How it Works
 
@@ -30,6 +33,8 @@ Here's what you get:
 # Features
 
 * Performs OCR on your documents, adds selectable text to image only documents and adds tags, correspondents and document types to your documents.
+* Supports PDF documents, images, plain text files, and Office documents (Word, Excel, Powerpoint, and LibreOffice equivalents).
+	* Office document support is optional and provided by Apache Tika (see [configuration](https://paperless-ng.readthedocs.io/en/latest/configuration.html#tika-settings))
 * Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and can be configured freely.
 * Single page application front end. Should be pretty snappy. Will be mobile friendly in the future.
 	* Includes a dashboard that shows basic statistics and has document upload.
@@ -39,50 +44,21 @@ Here's what you get:
 	* Auto completion suggests relevant words from your documents.
 	* Results are sorted by relevance to your search query.
 	* Highlighting shows you which parts of the document matched the query.
+	* Searching for similar documents ("More like this")
 * Email processing: Paperless adds documents from your email accounts.
 	* Configure multiple accounts and filters for each account.
 	* When adding documents from mails, paperless can move these mails to a new folder, mark them as read, flag them or delete them.
 * Machine learning powered document matching.
 	* Paperless learns from your documents and will be able to automatically assign tags, correspondents and types to documents once you've stored a few documents in paperless.
 * A task processor that processes documents in parallel and also tells you when something goes wrong. On modern multi core systems, consumption is blazing fast.
-* Code cleanup in many, MANY areas. Some of the code from OG paperless was just overly complicated.
-* More tests, more stability.
 
-If you want to see some screenshots of paperless-ng in action, [some are available in the documentation](https://paperless-ng.readthedocs.io/en/latest/screenshots.html).
+If you want to see some screenshots of paperless-ng in action, [some are available in the documentation](https://paperless-ng.readthedocs.io/en/latest/screenshots.html). However, some parts of the UI have changed since I took these.
 
 For a complete list of changes from paperless, check out the [changelog](https://paperless-ng.readthedocs.io/en/latest/changelog.html)
 
-# Roadmap for 1.0
-
-- **Bulk editing**. Add/remove metadata from multiple documents at once.
-
-- Make the front end nice (except mobile).
-- Test coverage at 90%.
-- Fix whatever bugs I and you find.
-
-## Roadmap for versions beyond 1.0
-
-These are things that I want to add to paperless eventually. They are sorted by priority.
-
-- **More search.** The search backend is incredibly versatile and customizable. Searching is the most important feature of this project and thus, I want to implement things like:
-  - Group and limit search results by correspondent, show “more from this” links in the results.
-  - Ability to search for “Similar documents” in the search results
-- **Nested tags**. Organize tags in a hierarchical structure. This will combine the benefits of folders and tags in one coherent system.
-- **An interactive consumer** that shows its progress for documents it processes on the web page.
-	- With live updates ans websockets. This already works on a dev branch, but requires a lot of new dependencies, which I'm not particular happy about.
-	- Notifications when a document was added with buttons to open the new document right away.
-- **Arbitrary tag colors**. Allow the selection of any color with a color picker.
-- **More file types**. Possibly allow more file types to be processed by paperless, such as office .odt, .doc, .docx documents.
-
-Apart from that, paperless is pretty much feature complete.
-
-## On the chopping block.
-
-- **GnuPG encrypion.** [Here's a note about encryption in paperless](https://paperless-ng.readthedocs.io/en/latest/administration.html#managing-encryption). The gist of it is that I don't see which attacks this implementation protects against. It gives a false sense of security to users who don't care about how it works.
-
 # Getting started
 
-The recommended way to deploy paperless is docker-compose. Don't clone the repository, grab the latest release to get started instead. The dockerfiles archive contains just the docker files which will pull the image from docker hub. The source archive contains everything you need to build the docker image yourself (i.e. if you want to run on Raspberry Pi).
+The recommended way to deploy paperless is docker-compose. The files in the /docker/hub directory are configured to pull the image from Docker Hub.
 
 Read the [documentation](https://paperless-ng.readthedocs.io/en/latest/setup.html#installation) on how to get started.
 
@@ -96,6 +72,12 @@ Read the section about [migration](https://paperless-ng.readthedocs.io/en/latest
 
 The documentation for Paperless-ng is available on [ReadTheDocs](https://paperless-ng.readthedocs.io/).
 
+# Translation
+
+Paperless is currently available in English, German, Dutch and French. Translation is coordinated at transifex: https://www.transifex.com/paperless/paperless-ng
+
+If you want to see paperless in your own language, request that language at transifex and you can start translating after I approve the language.
+
 # Suggestions? Questions? Something not working?
 
 Please open an issue and start a discussion about it!
@@ -116,7 +98,6 @@ Paperless has been around a while now, and people are starting to build stuff on
 These projects also exist, but their status and compatibility with paperless-ng is unknown.
 
 * [Paperless Desktop](https://github.com/thomasbrueggemann/paperless-desktop): A desktop UI for your Paperless installation.  Runs on Mac, Linux, and Windows.
-* [ansible-role-paperless](https://github.com/ovv/ansible-role-paperless): An easy way to get Paperless running via Ansible.
 * [paperless-cli](https://github.com/stgarf/paperless-cli): A golang command line binary to interact with a Paperless instance.
 
 # Important Note

ansible/README.md

@@ -0,0 +1,114 @@
+Ansible Role: paperless-ng
+==========================
+
+Installs and configures paperless-ng EDMS on Debian/Ubuntu systems.
+
+Requirements
+------------
+
+No special system requirements. Ansible 2.7 or newer is required.
+
+Note that this role requires root access, so either run it in a playbook with a global `become: yes`, or invoke the role in your playbook like:
+
+    - hosts: all
+      roles:
+        - role: ansible
+          become: yes
+
+Role Variables
+--------------
+
+Most configuration variables from paperless-ng itself are available and accept their respective arguments.
+Every `PAPERLESS_*` configuration varaible is lowercased and instead prefixed with `paperlessng_*` in `defaults/main.yml`.
+
+For a full listing including explainations and allowed values, see the current [documentation](https://paperless-ng.readthedocs.io/en/ng-0.9.14/configuration.html).
+
+Additional variables available in this role are listed below, along with default values:
+
+    paperlessng_version: 0.9.14
+
+The [release](https://github.com/jonaswinkler/paperless-ng/releases) archive version of paperless-ng to install.
+
+    paperlessng_redis_host: localhost
+    paperlessng_redis_port: 6379
+
+Seperate configuration values that combine into `PAPERLESS_REDIS`.
+
+    paperlessng_db_type: sqlite
+
+Database to use. Default is file-based SQLite.
+
+    paperlessng_db_host: localhost
+    paperlessng_db_port: 5432
+    paperlessng_db_name: paperlessng
+    paperlessng_db_user: paperlessng
+    paperlessng_db_pass: paperlessng
+    paperlessng_db_sslmode: prefer
+
+Database configuration (only applicable if `paperlessng_db_type == 'postgresql'`).
+
+    paperlessng_directory: /opt/paperless-ng
+
+Root directory paperless-ng is installed into.
+
+    paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv"
+
+Directory used for the virtual environment for paperless-ng.
+
+    paperlessng_ocr_languages:
+      - eng
+
+List of OCR languages to install and configure (`apt search tesseract-ocr-*`).
+
+    paperlessng_use_jbig2enc: True
+
+Whether to install and use [jbig2enc](https://github.com/agl/jbig2enc) for OCRmyPDF.
+
+    paperlessng_big2enc_lossy: False
+
+Whether to use jbig2enc's lossy compression mode.
+
+    paperlessng_superuser_name: paperlessng
+    paperlessng_superuser_email: paperlessng@example.com
+    paperlessng_superuser_password: paperlessng
+
+Credentials of the initial superuser in paperless-ng.
+
+    paperlessng_system_user: paperlessng
+    paperlessng_system_group: paperlessng
+
+System user and group to run the paperless-ng services as (will be created if required).
+
+    paperlessng_listen_address: 127.0.0.1
+    paperlessng_listen_port: 8000
+
+Address and port for the paperless-ng service to listen on.
+
+Dependencies
+------------
+
+No ansible dependencies.
+
+Example Playbook
+----------------
+`playbook.yml`:
+
+    - hosts: all
+      become: yes
+      vars_files:
+        - vars/main.yml
+      roles:
+        - ansible
+
+`vars/main.yml`:
+
+    paperlessng_media_root: /mnt/media/smbshare
+
+    paperlessng_db_type: postgresql
+    paperlessng_db_pass: PLEASEPROVIDEASTRONGPASSWORDHERE
+
+    paperless_secret_key: AGAINPLEASECHANGETHISNOW
+
+    paperlessng_ocr_languages:
+      - eng
+      - deu

ansible/defaults/main.yml

@@ -0,0 +1,78 @@
+---
+paperlessng_version: 0.9.14
+
+# Required services
+paperlessng_redis_host: localhost
+paperlessng_redis_port: 6379
+paperlessng_db_type: sqlite # or postgresql
+# Below entries only apply for paperlessng_db_type=='postgresql'
+paperlessng_db_host: localhost
+paperlessng_db_port: 5432
+paperlessng_db_name: paperlessng
+paperlessng_db_user: paperlessng
+paperlessng_db_pass: paperlessng
+paperlessng_db_sslmode: prefer
+
+# Paths and folders
+paperlessng_directory: /opt/paperless-ng
+paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption"
+paperlessng_data_dir: "{{ paperlessng_directory }}/data"
+paperlessng_media_root: "{{ paperlessng_directory }}/media"
+paperlessng_static_dir: "{{ paperlessng_directory }}/static"
+paperlessng_filename_format:
+paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv"
+
+# Hosting & Security
+paperless_secret_key: PLEASECHANGETHISFORTHELOVEOFGOD
+paperless_allowed_hosts: "*"
+paperless_cors_allowed_hosts: http://localhost:8000
+paperless_force_script_name:
+paperless_static_url: /static/
+paperless_auto_login_username:
+paperless_cookie_prefix: ""
+paperless_enable_http_remote_user: False
+
+# OCR settings
+paperlessng_ocr_languages:
+  - eng
+paperlessng_ocr_mode: skip
+paperlessng_ocr_output_type: pdfa
+paperlessng_ocr_pages: 0
+paperlessng_ocr_image_dpi:
+# see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr
+paperlessng_ocr_user_args:
+  #- "deskew": True  # https://github.com/jonaswinkler/paperless-ng/issues/231
+  - "optimize": 1
+paperlessng_use_jbig2enc: True
+paperlessng_big2enc_lossy: False
+
+# Tika settings
+paperlessng_tika_enabled: False
+paperlessng_tika_endpoint: http://localhost:9998
+paperlessng_tika_gotenberg_endpoint: http://localhost:3000
+
+# Software tweaks
+paperlessng_time_zone: Europe/Berlin
+paperlessng_consumer_polling: 0
+paperlessng_consumer_delete_duplicates: False
+paperlessng_consumer_recursive: False
+paperlessng_consumer_subdirs_as_tags: False
+paperlessng_optimize_thumbnails: True
+paperlessng_post_consume_script:
+paperlessng_filename_date_order:
+paperlessng_filename_parse_transforms:
+paperlessng_thumbnail_font_name: /usr/share/fonts/liberation/LiberationSerif-Regular.ttf
+paperlessng_ignore_dates: ""
+
+# Superuser settings
+paperlessng_superuser_name: paperlessng
+paperlessng_superuser_email: paperlessng@example.com
+paperlessng_superuser_password: paperlessng
+
+# System user settings
+paperlessng_system_user: paperlessng
+paperlessng_system_group: paperlessng
+
+# Webserver settings
+paperlessng_listen_address: 127.0.0.1
+paperlessng_listen_port: 8000

ansible/meta/main.yml

@@ -0,0 +1,17 @@
+dependencies: []
+
+galaxy_info:
+  author: C0nsultant
+  description: Bare-metal deployment of paperless-ng DMS
+  license: license (GPLv3)
+  min_ansible_version: 2.7
+
+  platforms:
+    - name: Debian
+      versions:
+        - buster
+    - name: Ubuntu
+      versions:
+        - focal
+
+  galaxy_tags: [EDMS, django, python, web]

ansible/molecule/fresh/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: fresh installation
+  hosts: all
+  tasks:
+    - name: install paperless-ng with default parameters
+      include_role:
+        name: ansible

ansible/molecule/fresh/molecule.yml

@@ -0,0 +1,35 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: ubuntu_focal
+    image: jrei/systemd-ubuntu:20.04
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+      - /run/lock
+    override_command: False
+  # ubuntu 18.04 bionic works except that
+  #   the default redis configuration expects IPv6 which is not enabled in docker by default
+  #   the default Python environment is configured for ASCII instead of UTF-8
+  # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies
+  - name: debian_buster
+    image: jrei/systemd-debian:10
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+      - /run/lock
+    override_command: False
+  # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies
+provisioner:
+  name: ansible
+verifier:
+  name: ansible

ansible/molecule/fresh/verify.yml

@@ -0,0 +1,60 @@
+---
+- name: Verify
+  hosts: all
+  gather_facts: false
+
+  vars_files:
+    - ../../defaults/main.yml
+
+  tasks:
+    - name: check if webserver is up
+      uri:
+        url: http://localhost:8000
+        status_code: [200, 302]
+        return_content: yes
+      register: landingpage
+      failed_when: "'Sign in</button>' not in landingpage.content"
+
+    - name: check if document posting works
+      uri:
+        url: http://localhost:8000/api/documents/post_document/
+        method: POST
+        body_format: form-multipart
+        body:
+          document:
+            content: FOO
+            filename: document.txt
+            mime_type: text/plain
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: post_document
+      failed_when: "'OK' not in post_document.content"
+
+    - name: verify uploaded document has been accepted
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'Consuming document.txt' not in logs.content"
+
+    # assumes txt consumption finished by now, might have to sleep a bit
+    - name: verify uploaded document has been consumed
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'document consumption finished' not in logs.content"
+
+    - name: verify uploaded document is avaiable
+      uri:
+        url: http://localhost:8000/api/documents/1/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: document
+      failed_when: "'Not found.' in document.content or 'FOO' not in document.content"

ansible/molecule/update/converge.yml

@@ -0,0 +1,11 @@
+---
+- name: update previous release to newest release
+  hosts: all
+  tasks:
+    - name: set current version as installation target
+      set_fact:
+        paperlessng_version: 0.9.14
+
+    - name: update to newest paperless-ng release
+      include_role:
+        name: ansible

ansible/molecule/update/molecule.yml

@@ -0,0 +1,35 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: ubuntu_focal
+    image: jrei/systemd-ubuntu:20.04
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+      - /run/lock
+    override_command: False
+  # ubuntu 18.04 bionic works except that
+  #   the default redis configuration expects IPv6 which is not enabled in docker by default
+  #   the default Python environment is configured for ASCII instead of UTF-8
+  # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies
+  - name: debian_buster
+    image: jrei/systemd-debian:10
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+      - /run/lock
+    override_command: False
+  # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies
+provisioner:
+  name: ansible
+verifier:
+  name: ansible

ansible/molecule/update/prepare.yml

@@ -0,0 +1,10 @@
+- name: install previous release
+  hosts: all
+  tasks:
+    - name: set previous version as installation target
+      set_fact:
+        paperlessng_version: 0.9.13
+
+    - name: install previous paperless-ng release
+      include_role:
+        name: ansible

ansible/molecule/update/verify.yml

@@ -0,0 +1,60 @@
+---
+- name: Verify
+  hosts: all
+  gather_facts: false
+
+  vars_files:
+    - ../../defaults/main.yml
+
+  tasks:
+    - name: check if webserver is up
+      uri:
+        url: http://localhost:8000
+        status_code: [200, 302]
+        return_content: yes
+      register: landingpage
+      failed_when: "'Sign in</button>' not in landingpage.content"
+
+    - name: check if document posting works
+      uri:
+        url: http://localhost:8000/api/documents/post_document/
+        method: POST
+        body_format: form-multipart
+        body:
+          document:
+            content: FOO
+            filename: document.txt
+            mime_type: text/plain
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: post_document
+      failed_when: "'OK' not in post_document.content"
+
+    - name: verify uploaded document has been accepted
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'Consuming document.txt' not in logs.content"
+
+    # assumes txt consumption finished by now, might have to sleep a bit
+    - name: verify uploaded document has been consumed
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'document consumption finished' not in logs.content"
+
+    - name: verify uploaded document is avaiable
+      uri:
+        url: http://localhost:8000/api/documents/1/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: document
+      failed_when: "'Not found.' in document.content or 'FOO' not in document.content"

ansible/tasks/main.yml

@@ -0,0 +1,454 @@
+---
+- name: verify operating system
+  fail:
+    msg: Sorry, only Debian and Ubuntu supported at the moment.
+  when: not(ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu')
+
+- name: install base dependencies
+  apt:
+    update_cache: yes
+    pkg:
+      # paperless-ng
+      - python3-pip
+      - python3-dev
+      - fonts-liberation
+      - imagemagick
+      - optipng
+      - gnupg
+      - libpoppler-cpp-dev
+      - libpq-dev
+      - libmagic-dev
+      - mime-support
+      # OCRmyPDF
+      - unpaper
+      - ghostscript
+      - icc-profiles-free
+      - qpdf
+      - liblept5
+      - libxml2
+      - pngquant
+      - zlib1g
+      - tesseract-ocr
+      # dev
+      - sudo
+      - build-essential
+      - python3-setuptools
+      - python3-wheel
+      - python3-virtualenv
+
+- name: install ocr languages
+  apt:
+    pkg: "{{ paperlessng_ocr_languages | map('regex_replace', '^(.*)$', 'tesseract-ocr-\\1') | list }}"
+
+- name: set up notesalexp repository key (for jbig2enc)
+  apt_key:
+    url: https://notesalexp.org/debian/alexp_key.asc
+    state: present
+  when: paperlessng_use_jbig2enc
+
+- name: set up notesalexp repository (for jbig2enc)
+  apt_repository:
+    repo: "deb https://notesalexp.org/debian/{{ ansible_distribution_release }}/ {{ ansible_distribution_release }} main"
+    state: present
+  when: paperlessng_use_jbig2enc
+
+- name: set up notesalexp repository pinning
+  copy:
+    content: |
+      Package: *
+      Pin: release o=notesalexp.org
+      Pin-Priority: 1
+
+      Package: jbig2enc
+      Pin: release o=notesalexp.org
+      Pin-Priority: 500
+    dest: /etc/apt/preferences.d/notesalexp
+  when: paperlessng_use_jbig2enc
+
+- name: install jbig2enc
+  apt:
+    pkg: jbig2enc
+    update_cache: yes
+  when: paperlessng_use_jbig2enc
+
+- name: install redis
+  apt:
+    pkg: redis-server
+  when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1'
+
+- name: enable redis
+  systemd:
+    name: redis-server
+    enabled: yes
+    masked: no
+    state: started
+  when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1'
+
+- name: create paperless system group
+  group:
+    name: "{{ paperlessng_system_group }}"
+
+- name: create paperless system user
+  user:
+    name: "{{ paperlessng_system_user }}"
+    groups:
+      - "{{ paperlessng_system_group }}"
+    shell: /usr/sbin/nologin
+    # GNUPG_HOME required due to paperless db.py
+    create_home: yes
+
+- name: check for paperless-ng installation
+  command:
+    cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html'
+  changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  failed_when: false
+  ignore_errors: yes
+  register: paperlessng_current_version
+
+- name: register current state
+  set_fact:
+    fresh_installation: '{{ "No such file or directory" in paperlessng_current_version.stderr }}'
+    update_installation: '{{ "No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string }}'
+    reconfigure_only: '{{ paperlessng_current_version.stdout == paperlessng_version | string }}'
+
+- name: backup current paperless-ng installation
+  copy:
+    src: "{{ paperlessng_directory }}"
+    remote_src: yes
+    dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/"
+  when: update_installation
+
+- name: remove current paperless sources
+  file:
+    path: "{{ paperlessng_directory }}/{{ item }}"
+    state: absent
+  with_items:
+    - docker
+    - docs
+    - scripts
+    - src
+    - static
+  when: update_installation
+
+- name: create temporary directory
+  tempfile:
+    state: directory
+  register: tempdir
+  when: not reconfigure_only
+
+- name: extract paperless-ng
+  unarchive:
+    src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
+    remote_src: yes
+    dest: "{{ tempdir.path }}"
+  when: not reconfigure_only
+
+- name: change owner and permissions of paperless-ng
+  command:
+    cmd: "{{ item }}"
+    warn: false
+  with_items:
+    - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
+    - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
+    - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
+  when: not reconfigure_only
+
+- name: move paperless-ng
+  command:
+    cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}"
+  when: not reconfigure_only
+
+- name: remove temporary directory
+  file:
+    path: "{{ tempdir.path }}"
+    state: absent
+  when: not reconfigure_only
+
+- name: create paperless-ng directories and set permissions
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ paperlessng_system_user }}"
+    group: "{{ paperlessng_system_group }}"
+    mode: "750"
+  with_items:
+    - "{{ paperlessng_directory }}"
+    - "{{ paperlessng_consumption_dir }}"
+    - "{{ paperlessng_data_dir }}"
+    - "{{ paperlessng_media_root }}"
+    - "{{ paperlessng_static_dir }}"
+
+- name: rename initial config
+  command:
+    cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template"
+    removes: "{{ paperlessng_directory }}/paperless.conf"
+
+- name: configure paperless-ng
+  lineinfile:
+    path: "{{ paperlessng_directory }}/paperless.conf.template"
+    regexp: "^#?{{ item.regexp }}="
+    line: "{{ item.line }}"
+  with_items:
+    # Required services
+    - regexp: PAPERLESS_REDIS
+      line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}"
+    # Paths and folders
+    - regexp: PAPERLESS_CONSUMPTION_DIR
+      line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}"
+    - regexp: PAPERLESS_DATA_DIR
+      line: "PAPERLESS_DATA_DIR={{ paperlessng_data_dir }}"
+    - regexp: PAPERLESS_MEDIA_ROOT
+      line: "PAPERLESS_MEDIA_ROOT={{ paperlessng_media_root }}"
+    - regexp: PAPERLESS_STATICDIR
+      line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}"
+    - regexp: PAPERLESS_FILENAME_FORMAT
+      line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}"
+    # Hosting & Security
+    - regexp: PAPERLESS_SECRET_KEY
+      line: "PAPERLESS_SECRET_KEY={{ paperless_secret_key }}"
+    - regexp: PAPERLESS_ALLOWED_HOSTS
+      line: "PAPERLESS_ALLOWED_HOSTS={{ paperless_allowed_hosts }}"
+    - regexp: PAPERLESS_CORS_ALLOWED_HOSTS
+      line: "PAPERLESS_CORS_ALLOWED_HOSTS={{ paperless_cors_allowed_hosts }}"
+    - regexp: PAPERLESS_FORCE_SCRIPT_NAME
+      line: "PAPERLESS_FORCE_SCRIPT_NAME={{ paperless_force_script_name }}"
+    - regexp: PAPERLESS_STATIC_URL
+      line: "PAPERLESS_STATIC_URL={{ paperless_static_url }}"
+    - regexp: PAPERLESS_AUTO_LOGIN_USERNAME
+      line: "PAPERLESS_AUTO_LOGIN_USERNAME={{ paperless_auto_login_username }}"
+    - regexp: PAPERLESS_COOKIE_PREFIX
+      line: "PAPERLESS_COOKIE_PREFIX={{ paperless_cookie_prefix }}"
+    - regexp: PAPERLESS_ENABLE_HTTP_REMOTE_USER
+      line: "PAPERLESS_ENABLE_HTTP_REMOTE_USER={{ paperless_enable_http_remote_user }}"
+    # OCR settings
+    - regexp: PAPERLESS_OCR_LANGUAGE
+      line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}"
+    - regexp: PAPERLESS_OCR_MODE
+      line: "PAPERLESS_OCR_MODE={{ paperlessng_ocr_mode }}"
+    - regexp: PAPERLESS_OCR_OUTPUT_TYPE
+      line: "PAPERLESS_OCR_OUTPUT_TYPE={{ paperlessng_ocr_output_type }}"
+    - regexp: PAPERLESS_OCR_PAGES
+      line: "PAPERLESS_OCR_PAGES={{ paperlessng_ocr_pages }}"
+    - regexp: PAPERLESS_OCR_IMAGE_DPI
+      line: "PAPERLESS_OCR_IMAGE_DPI={{ paperlessng_ocr_image_dpi }}"
+    - regexp: PAPERLESS_OCR_USER_ARGS
+      line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocr_user_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}"
+    # Tika settings
+    - regexp: PAPERLESS_TIKA_ENABLED
+      line: "PAPERLESS_TIKA_ENABLED={{ paperlessng_tika_enabled }}"
+    - regexp: PAPERLESS_TIKA_ENDPOINT
+      line: "PAPERLESS_TIKA_ENDPOINT={{ paperlessng_tika_endpoint }}"
+    - regexp: PAPERLESS_TIKA_GOTENBERG_ENDPOINT
+      line: "PAPERLESS_TIKA_GOTENBERG_ENDPOINT={{ paperlessng_tika_endpoint }}"
+    # Software tweaks
+    - regexp: PAPERLESS_TIME_ZONE
+      line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}"
+    - regexp: PAPERLESS_CONSUMER_POLLING
+      line: "PAPERLESS_CONSUMER_POLLING={{ paperlessng_consumer_polling }}"
+    - regexp: PAPERLESS_CONSUMER_DELETE_DUPLICATES
+      line: "PAPERLESS_CONSUMER_DELETE_DUPLICATES={{ paperlessng_consumer_delete_duplicates }}"
+    - regexp: PAPERLESS_CONSUMER_RECURSIVE
+      line: "PAPERLESS_CONSUMER_RECURSIVE={{ paperlessng_consumer_recursive }}"
+    - regexp: PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS
+      line: "PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS={{ paperlessng_consumer_subdirs_as_tags }}"
+    - regexp: PAPERLESS_OPTIMIZE_THUMBNAILS
+      line: "PAPERLESS_OPTIMIZE_THUMBNAILS={{ paperlessng_optimize_thumbnails }}"
+    - regexp: PAPERLESS_POST_CONSUME_SCRIPT
+      line: "PAPERLESS_POST_CONSUME_SCRIPT={{ paperlessng_post_consume_script }}"
+    - regexp: PAPERLESS_FILENAME_DATE_ORDER
+      line: "PAPERLESS_FILENAME_DATE_ORDER={{ paperlessng_filename_date_order }}"
+    - regexp: PAPERLESS_THUMBNAIL_FONT_NAME
+      line: "PAPERLESS_THUMBNAIL_FONT_NAME={{ paperlessng_thumbnail_font_name }}"
+    - regexp: PAPERLESS_IGNORE_DATES
+      line: "PAPERLESS_IGNORE_DATES={{ paperlessng_ignore_dates }}"
+  no_log: yes
+
+- name: configure paperless-ng database [sqlite]
+  lineinfile:
+    path: "{{ paperlessng_directory }}/paperless.conf.template"
+    regexp: "^#?PAPERLESS_DBHOST=(.*)$"
+    line: '#PAPERLESS_DBHOST=\1'
+    backrefs: yes
+  when: paperlessng_db_type == 'sqlite'
+
+- name: configure paperless-ng database [postgresql]
+  lineinfile:
+    path: "{{ paperlessng_directory }}/paperless.conf.template"
+    regexp: "^#?{{ item.regexp }}="
+    line: "{{ item.line }}"
+  with_items:
+    - regexp: PAPERLESS_DBHOST
+      line: "PAPERLESS_DBHOST={{ paperlessng_db_host }}"
+    - regexp: PAPERLESS_DBPORT
+      line: "PAPERLESS_DBPORT={{ paperlessng_db_port }}"
+    - regexp: PAPERLESS_DBNAME
+      line: "PAPERLESS_DBNAME={{ paperlessng_db_name }}"
+    - regexp: PAPERLESS_DBUSER
+      line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}"
+    - regexp: PAPERLESS_DBPASS
+      line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}"
+    - regexp: PAPERLESS_DBSSLMODE
+      line: "PAPERLESS_DBSSLMODE={{ paperlessng_db_sslmode }}"
+  when: paperlessng_db_type == 'postgresql'
+  no_log: yes
+
+- name: deploy paperless-ng configuration
+  copy:
+    src: "{{ paperlessng_directory }}/paperless.conf.template"
+    remote_src: yes
+    dest: /etc/paperless.conf
+    owner: root
+    group: root
+    mode: "0644"
+  register: configuration
+
+- name: create paperlessng venv
+  become: yes
+  become_user: "{{ paperlessng_system_user }}"
+  command:
+    cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3"
+    creates: "{{ paperlessng_virtualenv }}"
+  register: venv
+
+- name: install paperlessng requirements
+  become: yes
+  become_user: "{{ paperlessng_system_user }}"
+  pip:
+    requirements: "{{ paperlessng_directory }}/requirements.txt"
+    executable: "{{ paperlessng_virtualenv }}/bin/pip3"
+    extra_args: --upgrade
+  when: not reconfigure_only
+
+- name: migrate database schema
+  become: yes
+  become_user: "{{ paperlessng_system_user }}"
+  command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate"
+  register: database_schema
+  changed_when: '"No migrations to apply." not in database_schema.stdout'
+  when: not reconfigure_only
+
+- name: configure paperless superuser
+  become: yes
+  become_user: "{{ paperlessng_system_user }}"
+  # "manage.py createsuperuser" only works on interactive TTYs
+  vars:
+    creation_script: |
+      from django.contrib.auth.models import User
+      from django.contrib.auth.hashers import get_hasher
+
+      if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists():
+          user = User.objects.get(username='{{ paperlessng_superuser_name }}')
+          old = user.__dict__.copy()
+
+          user.is_superuser = True
+          user.email = '{{ paperlessng_superuser_email }}'
+          user.set_password('{{ paperlessng_superuser_password }}')
+          user.save()
+          new = user.__dict__
+
+          algorithm, iterations, old_salt, old_hash = old['password'].split('$')
+          new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations))
+          _, _, _, new_hash = new_password_old_salt.split('$')
+          if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']):
+              print('changed')
+      else:
+          User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}')
+          print('changed')
+  command: '{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c "{{ creation_script }}"'
+  register: superuser
+  changed_when: superuser.stdout == 'changed'
+  no_log: yes
+
+- name: set ownership and permissions on paperlessng venv
+  file:
+    path: "{{ paperlessng_virtualenv }}"
+    state: directory
+    recurse: yes
+    owner: "{{ paperlessng_system_user }}"
+    group: "{{ paperlessng_system_group }}"
+    mode: g-w,o-rwx
+  when: venv.changed or not reconfigure_only
+
+- name: configure ghostscript for PDF
+  lineinfile:
+    path: /etc/ImageMagick-6/policy.xml
+    regexp: '(\s+)<policy domain="coder" rights=".*" pattern="PDF" />'
+    line: '\1<policy domain="coder" rights="read|write" pattern="PDF" />'
+    backrefs: yes
+
+- name: configure systemd services
+  ini_file:
+    path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}"
+    section: "Service"
+    option: "{{ item[1].option  }}"
+    value: "{{ item[1].value }}"
+  with_nested:
+    - [
+        paperless-consumer.service,
+        paperless-scheduler.service,
+        paperless-webserver.service,
+      ]
+    - [
+        # https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+        { option: "User", value: "{{ paperlessng_system_user }}" },
+        { option: "Group", value: "{{ paperlessng_system_group }}" },
+        { option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", },
+        { option: "ProtectSystem", value: "full" },
+        { option: "NoNewPrivileges", value: "true" },
+        { option: "PrivateUsers", value: "true" },
+        { option: "PrivateDevices", value: "true" },
+      ]
+
+- name: configure paperless-consumer service
+  ini_file:
+    path: "{{ paperlessng_directory }}/scripts/paperless-consumer.service"
+    section: "Service"
+    option: "ExecStart"
+    value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py document_consumer"
+
+- name: configure paperless-scheduler service
+  ini_file:
+    path: "{{ paperlessng_directory }}/scripts/paperless-scheduler.service"
+    section: "Service"
+    option: "ExecStart"
+    value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py qcluster"
+
+- name: configure paperless-webserver service
+  ini_file:
+    path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service"
+    section: "Service"
+    option: "ExecStart"
+    value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}"
+
+- name: copy systemd services
+  copy:
+    src: "{{ paperlessng_directory }}/scripts/{{ item }}"
+    remote_src: yes
+    dest: "/etc/systemd/system/{{ item }}"
+  with_items:
+    - paperless-consumer.service
+    - paperless-scheduler.service
+    - paperless-webserver.service
+  register: paperless_services
+
+- name: reload systemd daemon
+  systemd:
+    name: "{{ item }}"
+    state: restarted
+    daemon_reload: yes
+  with_items:
+    - paperless-consumer
+    - paperless-scheduler
+    - paperless-webserver
+  when: paperless_services.changed or configuration.changed
+
+- name: enable paperlessng services
+  systemd:
+    name: "{{ item }}"
+    enabled: yes
+    masked: no
+    state: started
+  with_items:
+    - paperless-consumer
+    - paperless-scheduler
+    - paperless-webserver

compile-frontend.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+cd src-ui
+npm install
+./node_modules/.bin/ng build --prod

crowdin.yml

@@ -0,0 +1,5 @@
+files:
+  - source: /src/locale/en-us/LC_MESSAGES/django.po
+    translation: /src/locale/%two_letters_code%/LC_MESSAGES/django.po
+  - source: /src-ui/messages.xlf
+    translation: /src-ui/src/locale/messages.%two_letters_code%.xlf

docker/compose/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=paperless

docker/compose/docker-compose.env

@@ -7,7 +7,7 @@
 # Additional languages to install for text recognition, separated by a
 # whitespace. Note that this is
 # different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the
-# default language used when guessing the language from the OCR output.
+# language used for OCR.
 # The container installs English, German, Italian, Spanish and French by
 # default.
 # See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster

docker/compose/docker-compose.postgres-tika.yml

@@ -0,0 +1,90 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Instead of SQLite (default), PostgreSQL is used as the database server.
+# - Apache Tika and Gotenberg servers are started with paperless and paperless
+#   is configured to use these services. These provide support for consuming
+#   Office documents (Word, Excel, Power Point and their LibreOffice counter-
+#   parts.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: redis:6.0
+    restart: unless-stopped
+
+  db:
+    image: postgres:13
+    restart: unless-stopped
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: paperless
+      POSTGRES_USER: paperless
+      POSTGRES_PASSWORD: paperless
+
+  webserver:
+    image: jonaswinkler/paperless-ng:latest
+    restart: unless-stopped
+    depends_on:
+      - db
+      - broker
+      - gotenberg
+      - tika
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_DBHOST: db
+      PAPERLESS_TIKA_ENABLED: 1
+      PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
+      PAPERLESS_TIKA_ENDPOINT: http://tika:9998
+
+  gotenberg:
+    image: thecodingmachine/gotenberg
+    restart: unless-stopped
+    environment:
+      DISABLE_GOOGLE_CHROME: 1
+
+  tika:
+    image: apache/tika
+    restart: unless-stopped
+
+volumes:
+  data:
+  media:
+  pgdata:

docker/compose/docker-compose.postgres.yml

@@ -0,0 +1,72 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Instead of SQLite (default), PostgreSQL is used as the database server.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: redis:6.0
+    restart: unless-stopped
+
+  db:
+    image: postgres:13
+    restart: unless-stopped
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: paperless
+      POSTGRES_USER: paperless
+      POSTGRES_PASSWORD: paperless
+
+  webserver:
+    image: jonaswinkler/paperless-ng:latest
+    restart: unless-stopped
+    depends_on:
+      - db
+      - broker
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_DBHOST: db
+
+
+volumes:
+  data:
+  media:
+  pgdata:

docker/compose/docker-compose.sqlite-tika.yml

@@ -0,0 +1,78 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# SQLite is used as the database. The SQLite file is stored in the data volume.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Apache Tika and Gotenberg servers are started with paperless and paperless
+#   is configured to use these services. These provide support for consuming
+#   Office documents (Word, Excel, Power Point and their LibreOffice counter-
+#   parts.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: redis:6.0
+    restart: unless-stopped
+
+  webserver:
+    image: jonaswinkler/paperless-ng:latest
+    restart: unless-stopped
+    depends_on:
+      - broker
+      - gotenberg
+      - tika
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_TIKA_ENABLED: 1
+      PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
+      PAPERLESS_TIKA_ENDPOINT: http://tika:9998
+
+  gotenberg:
+    image: thecodingmachine/gotenberg
+    restart: unless-stopped
+    environment:
+      DISABLE_GOOGLE_CHROME: 1
+
+  tika:
+    image: apache/tika
+    restart: unless-stopped
+
+volumes:
+  data:
+  media:

docker/compose/docker-compose.sqlite.yml

@@ -0,0 +1,56 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# SQLite is used as the database. The SQLite file is stored in the data volume.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: redis:6.0
+    restart: unless-stopped
+
+  webserver:
+    image: jonaswinkler/paperless-ng:latest
+    restart: unless-stopped
+    depends_on:
+      - broker
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+
+
+volumes:
+  data:
+  media:

docker/docker-entrypoint.sh

@@ -62,6 +62,7 @@ migrations() {
 		# simultaneously. This also ensures that the db is ready when the command
 		# of the current container starts.
 		flock 200
+		echo "Apply database migrations..."
 		sudo -HEu paperless python3 manage.py migrate
 	)  200>/usr/src/paperless/data/migration_lock
 
@@ -78,13 +79,19 @@ initialize() {
 		fi
 	done
 
+	echo "creating directory /tmp/paperless"
+	mkdir -p /tmp/paperless
+
 	chown -R paperless:paperless ../
+	chown -R paperless:paperless /tmp/paperless
 
 	migrations
 
 }
 
 install_languages() {
+	echo "Installing languages..."
+
 	local langs="$1"
 	read -ra langs <<<"$langs"
 
@@ -119,6 +126,8 @@ install_languages() {
     done
 }
 
+echo "Paperless-ng docker container starting..."
+
 # Install additional languages if specified
 if [[ ! -z "$PAPERLESS_OCR_LANGUAGES"  ]]; then
 		install_languages "$PAPERLESS_OCR_LANGUAGES"
@@ -127,8 +136,10 @@ fi
 initialize
 
 if [[ "$1" != "/"* ]]; then
+	echo Executing management command "$@"
 	exec sudo -HEu paperless python3 manage.py "$@"
 else
+	echo Executing "$@"
 	exec "$@"
 fi
 

docker/gunicorn.conf.py

@@ -1,4 +1,4 @@
-bind = '127.0.0.1:8000'
+bind = '0.0.0.0:8000'
 backlog = 2048
 workers = 3
 worker_class = 'sync'

docker/hub/docker-compose.postgres.yml

@@ -1,44 +0,0 @@
-version: "3.4"
-services:
-  broker:
-    image: redis:6.0
-    restart: always
-
-  db:
-    image: postgres:13
-    restart: always
-    volumes:
-      - pgdata:/var/lib/postgresql/data
-    environment:
-      POSTGRES_DB: paperless
-      POSTGRES_USER: paperless
-      POSTGRES_PASSWORD: paperless
-
-  webserver:
-    image: jonaswinkler/paperless-ng:0.9.9
-    restart: always
-    depends_on:
-      - db
-      - broker
-    ports:
-      - 8000:8000
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-    volumes:
-      - data:/usr/src/paperless/data
-      - media:/usr/src/paperless/media
-      - ./export:/usr/src/paperless/export
-      - ./consume:/usr/src/paperless/consume
-    env_file: docker-compose.env
-    environment:
-      PAPERLESS_REDIS: redis://broker:6379
-      PAPERLESS_DBHOST: db
-
-
-volumes:
-  data:
-  media:
-  pgdata:

docker/hub/docker-compose.sqlite.yml

@@ -1,31 +0,0 @@
-version: "3.4"
-services:
-  broker:
-    image: redis:6.0
-    restart: always
-
-  webserver:
-    image: jonaswinkler/paperless-ng:0.9.9
-    restart: always
-    depends_on:
-      - broker
-    ports:
-      - 8000:8000
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-    volumes:
-      - data:/usr/src/paperless/data
-      - media:/usr/src/paperless/media
-      - ./export:/usr/src/paperless/export
-      - ./consume:/usr/src/paperless/consume
-    env_file: docker-compose.env
-    environment:
-      PAPERLESS_REDIS: redis://broker:6379
-
-
-volumes:
-  data:
-  media:

docker/local/docker-compose.postgres.yml

@@ -1,44 +0,0 @@
-version: "3.4"
-services:
-  broker:
-    image: redis:6.0
-    restart: always
-
-  db:
-    image: postgres:13
-    restart: always
-    volumes:
-      - pgdata:/var/lib/postgresql/data
-    environment:
-      POSTGRES_DB: paperless
-      POSTGRES_USER: paperless
-      POSTGRES_PASSWORD: paperless
-
-  webserver:
-    build: .
-    restart: always
-    depends_on:
-      - db
-      - broker
-    ports:
-      - 8000:8000
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-    volumes:
-      - data:/usr/src/paperless/data
-      - media:/usr/src/paperless/media
-      - ./export:/usr/src/paperless/export
-      - ./consume:/usr/src/paperless/consume
-    env_file: docker-compose.env
-    environment:
-      PAPERLESS_REDIS: redis://broker:6379
-      PAPERLESS_DBHOST: db
-
-
-volumes:
-  data:
-  media:
-  pgdata:

docker/local/docker-compose.sqlite.yml

@@ -1,31 +0,0 @@
-version: "3.4"
-services:
-  broker:
-    image: redis:6.0
-    restart: always
-
-  webserver:
-    build: .
-    restart: always
-    depends_on:
-      - broker
-    ports:
-      - 8000:8000
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-    volumes:
-      - data:/usr/src/paperless/data
-      - media:/usr/src/paperless/media
-      - ./export:/usr/src/paperless/export
-      - ./consume:/usr/src/paperless/consume
-    env_file: docker-compose.env
-    environment:
-      PAPERLESS_REDIS: redis://broker:6379
-
-
-volumes:
-  data:
-  media:

docker/supervisord.conf

@@ -8,7 +8,7 @@ loglevel=info                ; log level; default info; others: debug,warn,trace
 user=root
 
 [program:gunicorn]
-command=gunicorn -c /usr/src/paperless/gunicorn.conf.py -b 0.0.0.0:8000 paperless.wsgi
+command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.wsgi
 user=paperless
 
 stdout_logfile=/dev/stdout

docs/administration.rst

@@ -20,6 +20,8 @@ Options available to any installation of paperless:
     metadata to a specific folder. You may import your documents into a
     fresh instance of paperless again or store your documents in another
     DMS with this export.
+*   The document exporter is also able to update an already existing export.
+    Therefore, incremental backups with ``rsync`` are entirely possible.
 
 Options available to docker installations:
 
@@ -48,16 +50,16 @@ Options available to bare-metal and non-docker installations:
 Restoring
 =========
 
-
-
-
 .. _administration-updating:
 
-Updating paperless
+Updating Paperless
 ##################
 
+Docker Route
+============
+
 If a new release of paperless-ng is available, upgrading depends on how you
-installed paperless-ng in the first place. The releases are available at
+installed paperless-ng in the first place. The releases are available at the
 `release page <https://github.com/jonaswinkler/paperless-ng/releases>`_.
 
 First of all, ensure that paperless is stopped.
@@ -69,87 +71,106 @@ First of all, ensure that paperless is stopped.
 
 After that, :ref:`make a backup <administration-backup>`.
 
-A.  If you used the dockerfiles archive, simply download the files of the new release,
-    adjust the settings in the files (i.e., the path to your consumption directory),
-    and replace your existing docker-compose files. Then start paperless as usual,
-    which will pull the new image, and update your database, if necessary:
+A.  If you pull the image from the docker hub, all you need to do is:
 
     .. code:: shell-session
 
-        $ cd /path/to/paperless
+        $ docker-compose pull
         $ docker-compose up
 
-    If you see everything working, you can start paperless-ng with "-d" to have it
-    run in the background.
+    The docker-compose files refer to the ``latest`` version, which is always the latest
+    stable release.
 
-    .. hint::
-
-        The released docker-compose files specify exact versions to be pulled from the hub.
-        This is to ensure that if the docker-compose files should change at some point
-        (i.e., services updates/configured differently), you wont run into trouble due to
-        docker pulling the ``latest`` image and running it in an older environment.
-        
-B.  If you built the image yourself, grab the new archive and replace your current
-    paperless folder with the new contents.
-
-    After that, make the necessary adjustments to the docker-compose.yml (i.e.,
-    adjust your consumption directory).
-
-    Build and start the new image with:
+B.  If you built the image yourself, do the following:
 
     .. code:: shell-session
 
-        $ cd /path/to/paperless
+        $ git pull
+        $ ./compile-frontend.sh
         $ docker-compose build
         $ docker-compose up
 
-    If you see everything working, you can start paperless-ng with "-d" to have it
-    run in the background.
+Running ``docker-compose up`` will also apply any new database migrations.
+If you see everything working, press CTRL+C once to gracefully stop paperless.
+Then you can start paperless-ng with ``-d`` to have it run in the background.
 
-.. hint::
+    .. note::
 
-    You can usually keep your ``docker-compose.env`` file, since this file will
-    never include mandatory configuration options. However, it is worth checking
-    out the new version of this file, since it might have new recommendations
-    on what to configure.
+        In version 0.9.14, the update process was changed. In 0.9.13 and earlier, the
+        docker-compose files specified exact versions and pull won't automatically
+        update to newer versions. In order to enable updates as described above, either
+        get the new ``docker-compose.yml`` file from `here <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/compose>`_
+        or edit the ``docker-compose.yml`` file, find the line that says
+        
+            .. code::
 
+                image: jonaswinkler/paperless-ng:0.9.x
+        
+        and replace the version with ``latest``:
 
-Updating paperless without docker
-=================================
+            .. code::
+
+                image: jonaswinkler/paperless-ng:latest
+        
+Bare Metal Route
+================
 
 After grabbing the new release and unpacking the contents, do the following:
 
 1.  Update dependencies. New paperless version may require additional
-    dependencies. The dependencies required are listed in the section about 
+    dependencies. The dependencies required are listed in the section about
     :ref:`bare metal installations <setup-bare_metal>`.
 
-2.  Update python requirements. If you use Pipenv, this is done with the following steps.
+2.  Update python requirements. Keep in mind to activate your virtual environment
+    before that, if you use one.
 
     .. code:: shell-session
 
-        $ pip install --upgrade pipenv
-        $ cd /path/to/paperless
-        $ pipenv clean
-        $ pipenv install
+        $ pip install -r requirements.txt
 
-    This creates a new virtual environment (or uses your existing environment)
-    and installs all dependencies into it.
-
-3.  Collect static files.
+3.  Migrate the database.
 
     .. code:: shell-session
 
         $ cd src
-        $ pipenv run python3 manage.py collectstatic --clear
-    
-4.  Migrate the database.
+        $ python3 manage.py migrate
+
+    This might not actually do anything. Not every new paperless version comes with new
+    database migrations.
+
+Ansible Route
+=============
+
+Most of the update process is automated when using the ansible role.
+
+1.  Backup your defined role variables file outside the paperless source-tree:
 
     .. code:: shell-session
 
-        $ cd src
-        $ pipenv run python3 manage.py migrate
+        $ cp ansible/vars.yml ~/vars.yml.old
+
+2.  Pull the release tag you want to update to:
+
+    .. code:: shell-session
+
+        $ git fetch --all
+        $ git checkout ng-0.9.14
+
+3.  Update the role variable definitions ``ansible/vars.yml`` (where appropriate).
+
+4.  Run the ansible playbook you created created during :ref:`installation <setup-ansible>` again:
+
+    .. note::
+
+        When ansible detects that an update run is in progress, it backs up the entire ``paperlessng_directory`` to ``paperlessng_directory-TIMESTAMP``.
+        Updates can be rolled back by simply moving the timestamped folder back to the original location.
+        If the update succeeds and you want to continue using the new release, please don't forget to delete the backup folder.
+
+    .. code:: shell-session
+
+        $ ansible-playbook playbook.yml
+
 
-        
 Management utilities
 ####################
 
@@ -166,7 +187,7 @@ or
 .. code:: shell-session
 
     $ cd /path/to/paperless/src
-    $ pipenv run python manage.py <command> <arguments>
+    $ python3 manage.py <command> <arguments>
 
 depending on whether you use docker or not.
 
@@ -183,8 +204,13 @@ backup or migration to another DMS.
 
 .. code::
 
-    document_exporter target
+    document_exporter target [-c] [-f] [-d]
 
+    optional arguments:
+    -c, --compare-checksums
+    -f, --use-filename-format
+    -d, --delete
+    
 ``target`` is a folder to which the data gets written. This includes documents,
 thumbnails and a ``manifest.json`` file. The manifest contains all metadata from
 the database (correspondents, tags, etc).
@@ -193,6 +219,24 @@ When you use the provided docker compose script, specify ``../export`` as the
 target. This path inside the container is automatically mounted on your host on
 the folder ``export``.
 
+If the target directory already exists and contains files, paperless will assume
+that the contents of the export directory are a previous export and will attempt
+to update the previous export. Paperless will only export changed and added files.
+Paperless determines whether a file has changed by inspecting the file attributes
+"date/time modified" and "size". If that does not work out for you, specify
+``--compare-checksums`` and paperless will attempt to compare file checksums instead.
+This is slower.
+
+Paperless will not remove any existing files in the export directory. If you want
+paperless to also remove files that do not belong to the current export such as files
+from deleted documents, specify ``--delete``. Be careful when pointing paperless to
+a directory that already contains other files.
+
+The filenames generated by this command follow the format
+``[date created] [correspondent] [title].[extension]``.
+If you want paperless to use ``PAPERLESS_FILENAME_FORMAT`` for exported filenames
+instead, specify ``--use-filename-format``.
+
 
 .. _utilities-importer:
 
@@ -387,7 +431,7 @@ Documents can be stored in Paperless using GnuPG encryption.
     Furthermore, the entire text content of the documents is stored plain in the
     database, even if your documents are encrypted. Filenames are not encrypted as
     well.
-    
+
     Also, the web server provides transparent access to your encrypted documents.
 
     Consider running paperless on an encrypted filesystem instead, which will then
@@ -410,6 +454,3 @@ Basic usage to disable encryption of your document store:
 .. code::
 
     decrypt_documents [--passphrase SECR3TP4SSPHRA$E]
-
-
-.. _Pipenv: https://pipenv.pypa.io/en/latest/

docs/changelog.rst

@@ -5,6 +5,163 @@
 Changelog
 *********
 
+paperless-ng 1.0.0
+##################
+
+Nothing special about this release, but since there are relatively few bug reports coming in, I think that this is reasonably stable.
+
+* Document export
+
+  * The document exporter has been rewritten to support updating an already existing export in place.
+    This enables incremental backups with ``rsync``.
+  * The document exporter supports naming exported files according to ``PAPERLESS_FILENAME_FORMAT``.
+  * The document exporter locks the media directory and the database during execution to ensure that
+    the resulting export is consistent.
+  * See the :ref:`updated documentation <utilities-exporter>` for more details.
+
+* Other changes and additions
+
+  * Added a language selector to the settings.
+  * Added date format options to the settings.
+  * Range selection with shift clicking is now possible in the document list.
+  * Filtering correspondent, type and tag management pages by name.
+  * Focus "Name" field in dialogs by default.
+
+
+paperless-ng 0.9.14
+###################
+
+Starting with this version, releases are getting built automatically. This release also comes with changes on how to install and
+update paperless.
+
+* Paperless now uses GitHub Actions to make releases and build docker images.
+
+  * Docker images are available for amd64, armhf, and aarch64.
+  * When you pull an image from Docker Hub, Docker will automatically select the correct image for you.
+
+* Changes to docker installations and updates
+
+  * The ``-dockerfiles.tar.xz`` release archive is gone. Instead, simply grab the docker files from ``/docker/compose`` in the repository
+    if you wish to install paperless by pulling from the hub.
+  * The docker compose files in ``/docker/compose`` were changed to always use the ``latest`` version automatically. In order to do further
+    updates, simply do a ``docker-compose pull``. The documentation has been updated.
+  * The docker compose files were changed to restart paperless on system boot only if it was running before shutdown.
+  * Documentation of the docker-compose files about what they do.
+
+* Changes to bare metal installations and updates
+
+  * The release archive is built exactly like before. However, the release now comes with already compiled translation messages and
+    collected static files. Therefore, the update steps ``compilemessages`` and ``collectstatic`` are now obsolete.
+
+* Other changes
+
+  * A new configuration option ``PAPERLESS_IGNORE_DATES`` was added by `jayme-github`_. This can be used to instruct paperless to ignore
+    certain dates (such as your date of birth) when guessing the date from the document content. This was actually introduced in 0.9.12,
+    I just forgot to mention it in the changelog.
+  * The filter drop downs now display selected entries on top of all other entries.
+  * The PostgreSQL client now supports setting an explicit ``sslmode`` to force encryption of the connection to PostgreSQL.
+  * The docker images now come with ``jbig2enc``, which is a lossless image encoder for PDF documents and decreases the size of certain
+    PDF/A documents.
+  * When using any of the manual matching algorithms, paperless now logs messages about when and why these matching algorithms matched.
+  * The default settings for parallelization in paperless were adjusted to always leave one CPU core free.
+  * Added an option to the frontend to choose which method to use for displaying PDF documents.
+
+* Fixes
+
+  * An issue with the tika parser not picking up files from the consumption directory was fixed.
+  * A couple changes to the dark mode and fixes to several other layout issues.
+  * An issue with the drop downs for correspondents, tags and types not properly supporting filtering with special characters was fixed.
+  * Fixed an issue with filenames of downloaded files: Dates where off by one day due to timezone issues.
+  * Searching will continue to work even when the index returns non-existing documents. This resulted in "Document does not exist" errors
+    before. Instead, a warning is logged, indicating the issue.
+  * An issue with the consumer crashing when invalid regular expression were used was fixed.
+
+paperless-ng 0.9.13
+###################
+
+* Fixed an issue with Paperless not starting due to the new Tika integration when ``USERMAP_UID`` and ``USERMAP_GID`` was used
+  in the ``docker-compose.env`` file.
+
+paperless-ng 0.9.12
+###################
+
+* Paperless localization
+
+  * Thanks to the combined efforts of many users, Paperless is now available in English, Dutch, French and German.
+
+* Thanks to `Jo Vandeginste`_, Paperless has optional support for Office documents such as .docx, .doc, .odt and more.
+
+  * See the :ref:`configuration<configuration-tika>` on how to enable this feature. This feature requires two additional services
+    (one for parsing Office documents and metadata extraction and another for converting Office documents to PDF), and is therefore
+    not enabled on default installations.
+  * As with all other documents, paperless converts Office documents to PDF and stores both the original as well as the archived PDF.
+
+* Dark mode
+
+  * Thanks to `Michael Shamoon`_, paperless now has a dark mode. Configuration is available in the settings.
+
+* Other changes and additions
+
+  * The PDF viewer now uses a local copy of some dependencies instead of fetching them from the internet. Thanks to `slorenz`_.
+  * Revamped search bar styling thanks to `Michael Shamoon`_.
+  * Sorting in the document list by clicking on table headers.
+  * A button was added to the document detail page that assigns a new ASN to a document.
+  * Form field validation: When providing invalid input in a form (such as a duplicate ASN or no name), paperless now has visual
+    indicators and clearer error messages about what's wrong.
+  * Paperless disables buttons with network actions (such as save and delete) when a network action is active. This indicates that
+    something is happening and prevents double clicking.
+  * When using "Save & next", the title field is focussed automatically to better support keyboard editing.
+  * E-Mail: Added filter rule parameters to allow inline attachments (watch out for mails with inlined images!) and attachment filename filters
+    with wildcards.
+  * Support for remote user authentication thanks to `Michael Shamoon`_. This is useful for hiding Paperless behind single sign on applications
+    such as `authelia <https://www.authelia.com/>`_.
+  * "Clear filters" has been renamed to "Reset filters" and now correctly restores the default filters on saved views. Thanks to `Michael Shamoon`_
+
+* Fixes
+
+  * Paperless was unable to save views when "Not assigned" was chosen in one of the filter dropdowns.
+  * Clearer error messages when pre and post consumption scripts do not exist.
+  * The post consumption script is executed later in the consumption process. Before the change, an ID was passed to the script referring to
+    a document that did not yet exist in the database.
+
+paperless-ng 0.9.11
+###################
+
+* Fixed an issue with the docker image not starting at all due to a configuration change of the web server.
+
+
+paperless-ng 0.9.10
+###################
+
+* Bulk editing
+
+  * Thanks to `Michael Shamoon`_, we've got a new interface for the bulk editor.
+  * There are some configuration options in the settings to alter the behavior.
+
+* Other changes and additions
+
+  * Thanks to `zjean`_, paperless now publishes a webmanifest, which is useful for adding the application to home screens on mobile devices.
+  * The Paperless-ng logo now navigates to the dashboard.
+  * Filter for documents that don't have any correspondents, types or tags assigned.
+  * Tags, types and correspondents are now sorted case insensitive.
+  * Lots of preparation work for localization support.
+
+* Fixes
+
+  * Added missing dependencies for Raspberry Pi builds.
+  * Fixed an issue with plain text file consumption: Thumbnail generation failed due to missing fonts.
+  * An issue with the search index reporting missing documents after bulk deletes was fixed.
+  * Issue with the tag selector not clearing input correctly.
+  * The consumer used to stop working when encountering an incomplete classifier model file.
+
+.. note::
+
+  The bulk delete operations did not update the search index. Therefore, documents that you deleted remained in the index and
+  caused the search to return messages about missing documents when searching. Further bulk operations will properly update
+  the index.
+
+  However, this change is not retroactive: If you used the delete method of the bulk editor, you need to reindex your search index
+  by :ref:`running the management command document_index with the argument reindex <administration-index>`.
 
 paperless-ng 0.9.9
 ##################
@@ -57,12 +214,12 @@ paperless-ng 0.9.7
 
   * Thanks to the hard work of `Michael Shamoon`_, paperless now comes with a much more streamlined UI for
     filtering documents.
-  
+
   * `Michael Shamoon`_ replaced the document preview with another component. This should fix compatibility with Safari browsers.
 
   * Added buttons to the management pages to quickly show all documents with one specific tag, correspondent, or title.
-  
-  * Paperless now stores your saved views on the server and associates them with your user account. 
+
+  * Paperless now stores your saved views on the server and associates them with your user account.
     This means that you can access your views on multiple devices and have separate views for different users.
     You will have to recreate your views.
 
@@ -80,7 +237,7 @@ paperless-ng 0.9.7
     This option enables you to be logged in into multiple instances by specifying different cookie names for each instance.
 
 * Fixes
-  
+
   * Sometimes paperless would assign dates in the future to newly consumed documents.
   * The filename format fields ``{created_month}`` and ``{created_day}`` now use a leading zero for single digit values.
   * The filename format field ``{tags}`` can no longer be used without arguments.
@@ -927,6 +1084,9 @@ bulk of the work on this big change.
 
 * Initial release
 
+.. _slorenz: https://github.com/sisao
+.. _Jo Vandeginste: https://github.com/jovandeginste
+.. _zjean: https://github.com/zjean
 .. _rYR79435: https://github.com/rYR79435
 .. _Michael Shamoon: https://github.com/shamoon
 .. _jayme-github: http://github.com/jayme-github

docs/configuration.rst

@@ -53,6 +53,12 @@ PAPERLESS_DBPASS=<password>
 
     Defaults to "paperless".
 
+PAPERLESS_DBSSLMODE=<mode>
+    SSL mode to use when connecting to PostgreSQL.
+
+    See `the official documentation about sslmode <https://www.postgresql.org/docs/current/libpq-ssl.html>`_.
+
+    Default is ``prefer``.
 
 Paths and folders
 #################
@@ -162,6 +168,12 @@ PAPERLESS_COOKIE_PREFIX=<str>
 
     Defaults to ``""``, which does not alter the cookie names.
 
+PAPERLESS_ENABLE_HTTP_REMOTE_USER=<bool>
+    Allows authentication via HTTP_REMOTE_USER which is used by some SSO
+    applications.
+
+    Defaults to `false` which disables this feature.
+
 .. _configuration-ocr:
 
 OCR settings
@@ -210,20 +222,20 @@ PAPERLESS_OCR_MODE=<mode>
         into images and puts the OCRed text on top. This works for all documents,
         however, the resulting document may be significantly larger and text
         won't appear as sharp when zoomed in.
-    
+
     The default is ``skip``, which only performs OCR when necessary and always
     creates archived documents.
 
 PAPERLESS_OCR_OUTPUT_TYPE=<type>
     Specify the the type of PDF documents that paperless should produce.
-    
+
     *   ``pdf``: Modify the PDF document as little as possible.
     *   ``pdfa``: Convert PDF documents into PDF/A-2b documents, which is a
         subset of the entire PDF specification and meant for storing
         documents long term.
     *   ``pdfa-1``, ``pdfa-2``, ``pdfa-3`` to specify the exact version of
         PDF/A you wish to use.
-    
+
     If not specified, ``pdfa`` is used. Remember that paperless also keeps
     the original input file as well as the archived version.
 
@@ -255,7 +267,7 @@ PAPERLESS_OCR_IMAGE_DPI=<num>
     present in an image.
 
 
-PAPERLESS_OCR_USER_ARG=<json>
+PAPERLESS_OCR_USER_ARGS=<json>
     OCRmyPDF offers many more options. Use this parameter to specify any
     additional arguments you wish to pass to OCRmyPDF. Since Paperless uses
     the API of OCRmyPDF, you have to specify these in a format that can be
@@ -275,9 +287,69 @@ PAPERLESS_OCR_USER_ARG=<json>
 
     .. code:: json
 
-        {"deskew": true, "optimize": 3, "unpaper_args": "--pre-rotate 90"}    
-    
-    
+        {"deskew": true, "optimize": 3, "unpaper_args": "--pre-rotate 90"}
+
+.. _configuration-tika:
+
+Tika settings
+#############
+
+Paperless can make use of `Tika <https://tika.apache.org/>`_ and
+`Gotenberg <https://thecodingmachine.github.io/gotenberg/>`_ for parsing and
+converting "Office" documents (such as ".doc", ".xlsx" and ".odt"). If you
+wish to use this, you must provide a Tika server and a Gotenberg server,
+configure their endpoints, and enable the feature.
+
+PAPERLESS_TIKA_ENABLED=<bool>
+    Enable (or disable) the Tika parser.
+
+    Defaults to false.
+
+PAPERLESS_TIKA_ENDPOINT=<url>
+    Set the endpoint URL were Paperless can reach your Tika server.
+
+    Defaults to "http://localhost:9998".
+
+PAPERLESS_TIKA_GOTENBERG_ENDPOINT=<url>
+    Set the endpoint URL were Paperless can reach your Gotenberg server.
+
+    Defaults to "http://localhost:3000".
+
+If you run paperless on docker, you can add those services to the docker-compose
+file (see the provided ``docker-compose.tika.yml`` file for reference). The changes
+requires are as follows:
+
+.. code:: yaml
+
+    services:
+        # ...
+
+        webserver:
+            # ...
+
+            environment:
+                # ...
+
+                PAPERLESS_TIKA_ENABLED: 1
+                PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
+                PAPERLESS_TIKA_ENDPOINT: http://tika:9998
+        
+        # ...
+
+        gotenberg:
+            image: thecodingmachine/gotenberg
+            restart: unless-stopped
+            environment:
+                DISABLE_GOOGLE_CHROME: 1
+
+        tika:
+            image: apache/tika
+            restart: unless-stopped
+
+Add the configuration variables to the environment of the webserver (alternatively
+put the configuration in the ``docker-compose.env`` file) and add the additional
+services below the webserver service. Watch out for indentation.
+
 Software tweaks
 ###############
 
@@ -304,8 +376,25 @@ PAPERLESS_THREADS_PER_WORKER=<num>
         use a higher thread per worker count.
 
     The default is a balance between the two, according to your CPU core count,
-    with a slight favor towards threads per worker, and using as much cores as
-    possible.
+    with a slight favor towards threads per worker:
+
+    +----------------+---------+---------+
+    | CPU core count | Workers | Threads |
+    +----------------+---------+---------+
+    |              1 |       1 |       1 |
+    +----------------+---------+---------+
+    |              2 |       2 |       1 |
+    +----------------+---------+---------+
+    |              4 |       2 |       2 |
+    +----------------+---------+---------+
+    |              6 |       2 |       3 |
+    +----------------+---------+---------+
+    |              8 |       2 |       4 |
+    +----------------+---------+---------+
+    |             12 |       3 |       4 |
+    +----------------+---------+---------+
+    |             16 |       4 |       4 |
+    +----------------+---------+---------+
 
     If you only specify PAPERLESS_TASK_WORKERS, paperless will adjust
     PAPERLESS_THREADS_PER_WORKER automatically.
@@ -319,11 +408,14 @@ PAPERLESS_TIME_ZONE=<timezone>
     Defaults to UTC.
 
 
+.. _configuration-polling:
+
 PAPERLESS_CONSUMER_POLLING=<num>
     If paperless won't find documents added to your consume folder, it might
     not be able to automatically detect filesystem changes. In that case,
     specify a polling interval in seconds here, which will then cause paperless
-    to periodically check your consumption directory for changes.
+    to periodically check your consumption directory for changes. This will also
+    disable listening for file system changes with ``inotify``.
 
     Defaults to 0, which disables polling and uses filesystem notifications.
 
@@ -348,6 +440,9 @@ PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=<bool>
     E.g. <CONSUMPTION_DIR>/foo/bar/file.pdf will add the tags "foo" and "bar" to
     the consumed file. Paperless will create any tags that don't exist yet.
 
+    This is useful for sorting documents with certain tags such as ``car`` or
+    ``todo`` prior to consumption. These folders won't be deleted.
+
     PAPERLESS_CONSUMER_RECURSIVE must be enabled for this to work.
 
     Defaults to false.
@@ -400,6 +495,28 @@ PAPERLESS_FILENAME_DATE_ORDER=<format>
 
     Defaults to none, which disables this feature.
 
+PAPERLESS_THUMBNAIL_FONT_NAME=<filename>
+    Paperless creates thumbnails for plain text files by rendering the content
+    of the file on an image and uses a predefined font for that. This
+    font can be changed here.
+
+    Note that this won't have any effect on already generated thumbnails.
+
+    Defaults to ``/usr/share/fonts/liberation/LiberationSerif-Regular.ttf``.
+
+PAPERLESS_IGNORE_DATES=<string>
+    Paperless parses a documents creation date from filename and file content.
+    You may specify a comma separated list of dates that should be ignored during
+    this process. This is useful for special dates (like date of birth) that appear
+    in documents regularly but are very unlikely to be the documents creation date.
+
+    You may specify dates in a multitude of formats supported by dateparser (see
+    https://dateparser.readthedocs.io/en/latest/#popular-formats) but as the dates
+    need to be comma separated, the options are limited.
+    Example: "2020-12-02,22.04.1999"
+
+    Defaults to an empty string to not ignore any dates.
+
 
 Binaries
 ########

docs/extending.rst

@@ -25,8 +25,9 @@ This section describes the steps you need to take to start development on paperl
 
     *   Python 3.6.
     *   All dependencies listed in the :ref:`Bare metal route <setup-bare_metal>`
-    *   redis. You can either install redis or use the included scritps/start-redis.sh
-        to use docker to fire up a redis instance.
+    *   redis. You can either install redis or use the included scritps/start-services.sh
+        to use docker to fire up a redis instance (and some other services such as tika,
+        gotenberg and a postgresql server).
 
 Back end development
 ====================
@@ -38,7 +39,7 @@ Install the python dependencies by performing ``pipenv install --dev`` in the sr
 This will also create a virtual environment, which you can enter with ``pipenv shell`` or
 execute one-shot commands in with ``pipenv run``.
 
-In ``src/paperless.conf``, enable debug mode.
+Copy ``paperless.conf.example`` to ``paperless.conf`` and enable debug mode.
 
 Configure the IDE to use the src/ folder as the base source folder. Configure the following
 launch configurations in your IDE:
@@ -102,18 +103,11 @@ In order to build the front end and serve it as part of django, execute
 
 .. code:: shell-session
 
-    $ ng build --prod --output-path ../src/documents/static/frontend/
+    $ ng build --prod
 
 This will build the front end and put it in a location from which the Django server will serve
 it as static content. This way, you can verify that authentication is working.
 
-Making a release
-================
-
-Execute the ``make-release.sh <ver>`` script.
-
-This will test and assemble everything and also build and tag a docker image.
-
 
 Extending Paperless
 ===================

docs/faq.rst

@@ -52,6 +52,8 @@ out of that folder to use them elsewhere. Here are a couple notes about that.
 *   PDF documents, PNG images, JPEG images, TIFF images and GIF images are processed with OCR and converted into PDF documents.
 *   Plain text documents are supported as well and are added verbatim
     to paperless.
+*   With the optional Tika integration enabled (see :ref:`Configuration <configuration-tika>`), Paperless also supports various
+    Office documents (.docx, .doc, odt, .ppt, .pptx, .odp, .xls, .xlsx, .ods).
 
 Paperless determines the type of a file by inspecting its content. The
 file extensions do not matter.
@@ -73,10 +75,8 @@ in your browser and paperless has to do much less work to serve the data.
 
 **Q:** *How do I install paperless-ng on Raspberry Pi?*
 
-**A:** There is no docker image for ARM available. If you know how to build
-that automatically, I'm all ears. For now, you have to grab the latest release
-archive from the project page and build the image yourself. The release comes
-with the front end already compiled, so you don't have to do this on the Pi.
+**A:** Docker images are available for arm and arm64 hardware, so just follow
+the docker-compose instructions, or go the bare metal route.
 
 **Q:** *How do I run this on unRaid?*
 

docs/scanners.rst

@@ -10,6 +10,9 @@ scanner you use, but sometimes finding a scanner that will write to an FTP,
 NFS, or SMB server can be difficult.  This page is here to help you find one
 that works right for you based on recommendations from other Paperless users.
 
+Physical scanners
+=================
+
 +---------+----------------+-----+-----+-----+----------------+
 | Brand   | Model          | Supports        | Recommended By |
 +---------+----------------+-----+-----+-----+----------------+
@@ -25,6 +28,8 @@ that works right for you based on recommendations from other Paperless users.
 +---------+----------------+-----+-----+-----+----------------+
 | Fujitsu | `ix500`_       | yes |     | yes | `eonist`_      |
 +---------+----------------+-----+-----+-----+----------------+
+| Epson   | `WF-7710DWF`_  | yes |     | yes | `Skylinar`_    |
++---------+----------------+-----+-----+-----+----------------+
 | Fujitsu | `S1300i`_      | yes |     | yes | `jonaswinkler`_|
 +---------+----------------+-----+-----+-----+----------------+
 
@@ -32,7 +37,8 @@ that works right for you based on recommendations from other Paperless users.
 .. _MFC-J6930DW: https://www.brother.ca/en/p/MFCJ6930DW
 .. _MFC-J5910DW: https://www.brother.co.uk/printers/inkjet-printers/mfcj5910dw
 .. _MFC-9142CDN: https://www.brother.co.uk/printers/laser-printers/mfc9140cdn
-.. _ix500: https://www.fujitsu.com/global/products/computing/peripheral/scanners/scansnap/ix500/
+.. _ix500: http://www.fujitsu.com/us/products/computing/peripheral/scanners/scansnap/ix500/
+.. _WF-7710DWF: https://www.epson.de/en/products/printers/inkjet-printers/for-home/workforce-wf-7710dwf
 .. _S1300i: https://www.fujitsu.com/global/products/computing/peripheral/scanners/soho/s1300i/
 
 .. _danielquinn: https://github.com/danielquinn
@@ -40,4 +46,27 @@ that works right for you based on recommendations from other Paperless users.
 .. _bmsleight: https://github.com/bmsleight
 .. _eonist: https://github.com/eonist
 .. _REOLDEV: https://github.com/REOLDEV
+.. _Skylinar: https://github.com/Skylinar
 .. _jonaswinkler: https://github.com/jonaswinkler
+
+Mobile phone software
+=====================
+
+You can use your phone to "scan" documents. The regular camera app will work, but may have too low contrast for OCR to work well. Apps specifically for scanning are recommended.
+
++-------------------+----------------+-----+-----+-----+-------+--------+----------------+
+| Name              | OS             | Supports                         | Recommended By |
++-------------------+----------------+-----+-----+-----+-------+--------+----------------+
+|                   |                | FTP | NFS | SMB | Email | WebDav |                |
++===================+================+=====+=====+=====+=======+========+================+
+| `Office Lens`_    | Android        | ?   | ?   | ?   | ?     | ?      | `jonaswinkler`_|
++-------------------+----------------+-----+-----+-----+-------+--------+----------------+
+| `Genius Scan`_    | Android        | yes | no  | yes | yes   | yes    | `hannahswain`_ |
++-------------------+----------------+-----+-----+-----+-------+--------+----------------+
+
+On Android, you can use these applications in combination with one of the :ref:`Paperless-ng compatible apps <usage-mobile_upload>` to "Share" the documents produced by these scanner apps with paperless.
+
+.. _Office Lens: https://play.google.com/store/apps/details?id=com.microsoft.office.officelens
+.. _Genius Scan: https://play.google.com/store/apps/details?id=com.thegrizzlylabs.geniusscan.free
+
+.. _hannahswain: https://github.com/hannahswain

docs/setup.rst

@@ -3,35 +3,6 @@
 Setup
 *****
 
-Download
-########
-
-Go to the project page on GitHub and download the
-`latest release <https://github.com/jonaswinkler/paperless-ng/releases>`_.
-There are multiple options available.
-
-*   Download the dockerfiles archive if you want to pull paperless from
-    Docker Hub.
-
-*   Download the dist archive and extract it if you want to build the docker image
-    yourself or want to install paperless without docker.
-
-.. hint::
-
-    In contrast to paperless, the recommended way to get and update paperless-ng
-    is not to pull the entire git repository. Paperless-ng includes artifacts
-    that need to be compiled, and that's already done for you in the release.
-
-.. admonition:: Want to try out paperless-ng before migrating?
-
-    The release contains a file ``.env`` which sets the docker-compose project
-    name to "paperless", which is the same as before and instructs docker-compose
-    to reuse and upgrade your paperless volumes.
-
-    Just rename the project name in that file to anything else and docker-compose
-    will create fresh volumes for you!
-
-
 Overview of Paperless-ng
 ########################
 
@@ -49,45 +20,45 @@ Paperless consists of the following components:
     .. code:: shell-session
 
         $ cd /path/to/paperless/src/
-        $ pipenv run gunicorn -c /usr/src/paperless/gunicorn.conf.py -b 0.0.0.0:8000 paperless.wsgi
+        $ gunicorn -c ../gunicorn.conf.py -b 0.0.0.0:8000 paperless.wsgi
 
     or by any other means such as Apache ``mod_wsgi``.
 
 *   **The consumer:** This is what watches your consumption folder for documents.
-    However, the consumer itself does not consume really consume your documents anymore.
-    It rather notifies a task processor that a new file is ready for consumption.
+    However, the consumer itself does not really consume your documents.
+    Now it notifies a task processor that a new file is ready for consumption.
     I suppose it should be named differently.
-    This also used to check your emails, but that's now gone elsewhere as well.
+    This was also used to check your emails, but that's now done elsewhere as well.
 
     Start the consumer with the management command ``document_consumer``:
 
     .. code:: shell-session
 
         $ cd /path/to/paperless/src/
-        $ pipenv run python3 manage.py document_consumer
+        $ python3 manage.py document_consumer
 
     .. _setup-task_processor:
 
 *   **The task processor:** Paperless relies on `Django Q <https://django-q.readthedocs.io/en/latest/>`_
-    for doing much of the heavy lifting. This is a task queue that accepts tasks from
-    multiple sources and processes tasks in parallel. It also comes with a scheduler that executes
+    for doing most of the heavy lifting. This is a task queue that accepts tasks from
+    multiple sources and processes these in parallel. It also comes with a scheduler that executes
     certain commands periodically.
 
     This task processor is responsible for:
 
     *   Consuming documents. When the consumer finds new documents, it notifies the task processor to
         start a consumption task.
-    *   Consuming emails. It periodically checks your configured accounts for new mails and
-        produces consumption tasks for any documents it finds.
     *   The task processor also performs the consumption of any documents you upload through
         the web interface.
-    *   Maintain the search index and the automatic matching algorithm. These are things that paperless
+    *   Consuming emails. It periodically checks your configured accounts for new emails and
+        notifies the task processor to consume the attachment of an email.
+    *   Maintaining the search index and the automatic matching algorithm. These are things that paperless
         needs to do from time to time in order to operate properly.
 
     This allows paperless to process multiple documents from your consumption folder in parallel! On
-    a modern multi core system, consumption with full ocr is blazing fast.
+    a modern multi core system, this makes the consumption process with full OCR blazingly fast.
 
-    The task processor comes with a built-in admin interface that you can use to see whenever any of the
+    The task processor comes with a built-in admin interface that you can use to check whenever any of the
     tasks fail and inspect the errors (i.e., wrong email credentials, errors during consuming a specific
     file, etc).
 
@@ -96,11 +67,11 @@ Paperless consists of the following components:
     .. code:: shell-session
 
         $ cd /path/to/paperless/src/
-        $ pipenv run python3 manage.py qcluster
+        $ python3 manage.py qcluster
 
 *   A `redis <https://redis.io/>`_ message broker: This is a really lightweight service that is responsible
-    for getting the tasks from the webserver and consumer to the task scheduler. These run in different
-    processes (maybe even on different machines!), and therefore, this is necessary.
+    for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different
+    process (maybe even on different machines!), and therefore, this is necessary.
 
 *   Optional: A database server. Paperless supports both PostgreSQL and SQLite for storing its data.
 
@@ -108,50 +79,67 @@ Paperless consists of the following components:
 Installation
 ############
 
-You can go multiple routes with setting up and running Paperless:
+You can go multiple routes to setup and run Paperless:
 
-* The `docker route`_
-* The `bare metal route`_
+* :ref:`Pull the image from Docker Hub <setup-docker_hub>`
+* :ref:`Build the Docker image yourself <setup-docker_build>`
+* :ref:`Install Paperless directly on your system manually (bare metal) <setup-bare_metal>`
+* :ref:`Use ansible to install Paperless on your system automatically (bare metal) <setup-ansible>`
 
-The `docker route`_ is quick & easy. This is the recommended route. This configures all the stuff
-from above automatically so that it just works and uses sensible defaults for all configuration options.
+The Docker routes are quick & easy. These are the recommended routes. This configures all the stuff
+from the above automatically so that it just works and uses sensible defaults for all configuration options.
+Here you find a cheat-sheet for docker beginners: `CLI Basics <https://sehn.tech/post/devops-with-docker/>`_ 
 
-The `bare metal route`_ is more complicated to setup but makes it easier
+The bare metal route is complicated to setup but makes it easier
 should you want to contribute some code back. You need to configure and
 run the above mentioned components yourself.
 
-.. _setup-docker_route:
+The ansible route combines benefits of both options:
+the setup process is fully automated, reproducible and `idempotent <https://docs.ansible.com/ansible/latest/reference_appendices/glossary.html#Idempotency>`_,
+it includes the same sensible defaults, and it simultaneously provides the flexibility of a bare metal installation.
 
-Docker Route
-============
+.. _CLI Basics: https://sehn.tech/post/devops-with-docker/
+.. _idempotent: https://docs.ansible.com/ansible/latest/reference_appendices/glossary.html#Idempotency
 
-1.  Install `Docker`_ and `docker-compose`_. [#compose]_
+.. _setup-docker_hub:
 
-    .. caution::
+Install Paperless from Docker Hub
+=================================
 
-        If you want to use the included ``docker-compose.*.yml`` file, you
-        need to have at least Docker version **17.09.0** and docker-compose
-        version **1.17.0**.
+1.  Login with your user and create a folder in your home-directory `mkdir -v ~/paperless-ng` to have a place for your configuration files and consumption directory.
 
-        See the `Docker installation guide`_ on how to install the current
-        version of Docker for your operating system or Linux distribution of
-        choice. To get an up-to-date version of docker-compose, follow the
-        `docker-compose installation guide`_ if your package repository doesn't
-        include it.
-
-        .. _Docker installation guide: https://docs.docker.com/engine/installation/
-        .. _docker-compose installation guide: https://docs.docker.com/compose/install/
-
-2.  Copy either ``docker-compose.sqlite.yml`` or ``docker-compose.postgres.yml`` to
-    ``docker-compose.yml``, depending on which database backend you want to use.
+2.  Go to the `/docker/compose directory on the project page <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/compose>`_
+    and download one of the `docker-compose.*.yml` files, depending on which database backend you
+    want to use. Rename this file to `docker-compose.yml`.
+    If you want to enable optional support for Office documents, download a file with `-tika` in the file name.
+    Download the ``docker-compose.env`` file and the ``.env`` file as well and store them
+    in the same directory.
 
     .. hint::
 
         For new installations, it is recommended to use PostgreSQL as the database
         backend.
 
-2.  Modify ``docker-compose.yml`` to your preferences. You may want to change the path
-    to the consumption directory in this file. Find the line that specifies where
+3.  Install `Docker`_ and `docker-compose`_.
+
+    .. caution::
+
+        If you want to use the included ``docker-compose.*.yml`` file, you
+        need to have at least Docker version **17.09.0** and docker-compose
+        version **1.17.0**.  
+        To check do: `docker-compose -v` or `docker -v`
+
+        See the `Docker installation guide`_ on how to install the current
+        version of Docker for your operating system or Linux distribution of
+        choice. To get the latest version of docker-compose, follow the
+        `docker-compose installation guide`_ if your package repository doesn't
+        include it.
+
+        .. _Docker installation guide: https://docs.docker.com/engine/installation/
+        .. _docker-compose installation guide: https://docs.docker.com/compose/install/
+
+4.  Modify ``docker-compose.yml`` to your preferences. You may want to change the path
+    to the consumption directory. Find the line that specifies where
     to mount the consumption directory:
 
     .. code::
@@ -167,24 +155,35 @@ Docker Route
     Don't change the part after the colon or paperless wont find your documents.
 
 
-3.  Modify ``docker-compose.env``, following the comments in the file. The
+5.  Modify ``docker-compose.env``, following the comments in the file. The
     most important change is to set ``USERMAP_UID`` and ``USERMAP_GID``
     to the uid and gid of your user on the host system. This ensures that
     both the docker container and you on the host machine have write access
     to the consumption directory. If your UID and GID on the host system is
     1000 (the default for the first normal user on most systems), it will
-    work out of the box without any modifications.
+    work out of the box without any modifications. `id "username"` to check.
 
     .. note::
 
-        You can use any settings from the file ``paperless.conf`` in this file.
-        Have a look at :ref:`configuration` to see whats available.
+        You can copy any setting from the file ``paperless.conf.example`` and paste it here.
+        Have a look at :ref:`configuration` to see what's available.
 
-4.  Run ``docker-compose up -d``. This will create and start the necessary
-    containers. This will also build the image of paperless if you grabbed the
-    source archive.
+    .. caution::
 
-5.  To be able to login, you will need a super user. To create it, execute the
+        Some file systems such as NFS network shares don't support file system
+        notifications with ``inotify``. When storing the consumption directory
+        on such a file system, paperless will not pick up new files
+        with the default configuration. You will need to use ``PAPERLESS_CONSUMER_POLLING``,
+        which will disable inotify. See :ref:`here <configuration-polling>`.
+        
+6.  Now head over to: https://hub.docker.com/r/jonaswinkler/paperless-ng and choose your preferred
+    image and copy the link. To download this image do a `docker pull` followed by the link. Do this within the directory with the .yml files.
+    Depending on your network connection and CPU this will take a while. You have time to get a beverage.
+
+7.  Run ``docker-compose up -d``. This will create and start the necessary
+    containers, but your are not done yet!
+
+8.  To be able to login, you will need a super user. To create it, execute the
     following command:
 
     .. code-block:: shell-session
@@ -192,21 +191,59 @@ Docker Route
         $ docker-compose run --rm webserver createsuperuser
 
     This will prompt you to set a username, an optional e-mail address and
-    finally a password.
+    finally a password (at least 8 characters).
 
-6.  The default ``docker-compose.yml`` exports the webserver on your local port
+9.  The default ``docker-compose.yml`` exports the webserver on your local port
     8000. If you haven't adapted this, you should now be able to visit your
-    Paperless instance at ``http://127.0.0.1:8000``. You can login with the
-    user and password you just created.
+    Paperless instance at ``http://127.0.0.1:8000`` or your servers IP-Address:8000. 
+    Use the login credentials you have created with the previous step.
 
 .. _Docker: https://www.docker.com/
 .. _docker-compose: https://docs.docker.com/compose/install/
 
-.. [#compose] You of course don't have to use docker-compose, but it
-   simplifies deployment immensely. If you know your way around Docker, feel
-   free to tinker around without using compose!
+.. _setup-docker_build:
 
-.. _`setup-bare_metal`:
+Build the docker image yourself
+===============================
+
+1.  Clone the entire repository of paperless:
+
+    .. code:: shell-session
+
+        git clone https://github.com/jonaswinkler/paperless-ng
+
+    The master branch always reflects the latest stable version.
+
+2.  Copy one of the ``docker/compose/docker-compose.*.yml`` to ``docker-compose.yml`` in the root folder,
+    depending on which database backend you want to use. Copy
+    ``docker-compose.env`` into the project root as well.
+
+3.  In the ``docker-compose.yml`` file, find the line that instructs docker-compose to pull the paperless image from Docker Hub:
+
+    .. code:: yaml
+
+        webserver:
+            image: jonaswinkler/paperless-ng:latest
+    
+    and replace it with a line that instructs docker-compose to build the image from the current working directory instead:
+
+    .. code:: yaml
+
+        webserver:
+            build: .
+
+4.  Run the ``compile-frontend.sh`` script. This requires ``node`` and ``npm >= v15``.
+
+5.  Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run
+    ``docker-compose up -d`` to start the containers, do
+
+    .. code:: shell-session
+
+        $ docker-compose build
+
+    before that to build the image.
+
+.. _setup-bare_metal:
 
 Bare Metal Route
 ================
@@ -218,15 +255,17 @@ writing. Windows is not and will never be supported.
 1.  Install dependencies. Paperless requires the following packages.
 
     *   ``python3`` 3.6, 3.7, 3.8 (3.9 is untested).
-    *   ``python3-pip``, optionally ``pipenv`` for package installation
+    *   ``python3-pip``
     *   ``python3-dev``
 
+    *   ``fonts-liberation`` for generating thumbnails for plain text files
     *   ``imagemagick`` >= 6 for PDF conversion
-    *   ``optipng`` for optimising thumbnails
+    *   ``optipng`` for optimizing thumbnails
     *   ``gnupg`` for handling encrypted documents
     *   ``libpoppler-cpp-dev`` for PDF to text conversion
-    *   ``libmagic-dev`` for mime type detection
     *   ``libpq-dev`` for PostgreSQL
+    *   ``libmagic-dev`` for mime type detection
+    *   ``mime-support`` for mime type detection
 
     These dependencies are required for OCRmyPDF, which is used for text recognition.
 
@@ -241,17 +280,22 @@ writing. Windows is not and will never be supported.
     *   ``tesseract-ocr`` >= 4.0.0 for OCR
     *   ``tesseract-ocr`` language packs (``tesseract-ocr-eng``, ``tesseract-ocr-deu``, etc)
 
+    On Raspberry Pi, these libraries are required as well:
+
+    *   ``libatlas-base-dev``
+    *   ``libxslt1-dev``
+
     You will also need ``build-essential``, ``python3-setuptools`` and ``python3-wheel``
-    for installing some of the python dependencies. You can remove that
-    again after installation.
+    for installing some of the python dependencies.
 
 2.  Install ``redis`` >= 5.0 and configure it to start automatically.
 
 3.  Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
     to use PostgreSQL, SQLite is avialable as well.
 
-4.  Get the release archive. If you pull the git repo as it is, you also have to compile the front end by yourself.
-    Extract the frontend to a place from where you wish to execute it, such as ``/opt/paperless``.
+4.  Get the release archive from `<https://github.com/jonaswinkler/paperless-ng/releases>`_.
+    If you clone the git repo as it is, you also have to compile the front end by yourself.
+    Extract the archive to a place from where you wish to execute it, such as ``/opt/paperless``.
 
 5.  Configure paperless. See :ref:`configuration` for details. Edit the included ``paperless.conf`` and adjust the
     settings to your needs. Required settings for getting paperless running are:
@@ -264,7 +308,7 @@ writing. Windows is not and will never be supported.
         paperless stores its data. If you like, you can point both to the same directory.
     *   ``PAPERLESS_SECRET_KEY`` should be a random sequence of characters. It's used for authentication. Failure
         to do so allows third parties to forge authentication credentials.
-    
+
     Many more adjustments can be made to paperless, especially the OCR part. The following options are recommended
     for everyone:
 
@@ -273,23 +317,25 @@ writing. Windows is not and will never be supported.
 
 6.  Setup permissions. Create a system users under which you wish to run paperless. Ensure that these directories exist
     and that the user has write permissions to the following directories
-    
+
     *   ``/opt/paperless/media``
     *   ``/opt/paperless/data``
     *   ``/opt/paperless/consume``
 
     Adjust as necessary if you configured different folders.
 
-7.  Install python requirements. Paperless comes with both Pipfiles for ``pipenv`` as well as with a ``requirements.txt``.
-    Both will install exactly the same requirements. It is up to you if you wish to use a virtual environment or not.
+7.  Install python requirements from the ``requirements.txt`` file.
+    It is up to you if you wish to use a virtual environment or not.
+
+    .. code:: shell-session
+
+        pip3 install -r requirements.txt
+
 
 8.  Go to ``/opt/paperless/src``, and execute the following commands:
 
     .. code:: bash
 
-        # This collects static files from paperless and django.
-        python3 manage.py collectstatic --clear --no-input
-        
         # This creates the database schema.
         python3 manage.py migrate
 
@@ -302,13 +348,14 @@ writing. Windows is not and will never be supported.
 
         # This collects static files from paperless and django.
         python3 manage.py runserver
-    
+
     and pointing your browser to http://localhost:8000/.
 
     .. warning::
 
         This is a development server which should not be used in
-        production.
+        production. It is not audited for security and performance
+        is inferior to production ready web servers.
 
     .. hint::
 
@@ -323,6 +370,11 @@ writing. Windows is not and will never be supported.
     ``consumer`` script to watch the input folder, and the ``scheduler``
     script to run tasks such as email checking and document consumption.
 
+    You may need to adjust the path to the ``gunicorn`` executable. This
+    will be installed as part of the python dependencies, and is either located
+    in the ``bin`` folder of your virtual environment, or in ``~/.local/bin/`` if
+    no virtual environment is used.
+
     These services rely on redis and optionally the database server, but
     don't need to be started in any particular order. The example files
     depend on redis being started. If you use a database server, you should
@@ -355,13 +407,133 @@ writing. Windows is not and will never be supported.
     .. code::
 
         <policy domain="coder" rights="none" pattern="PDF" />
-    
+
     to
 
     .. code::
 
         <policy domain="coder" rights="read|write" pattern="PDF" />
 
+13. Optional: Install the `jbig2enc <https://ocrmypdf.readthedocs.io/en/latest/jbig2.html>`_
+    encoder. This will reduce the size of generated PDF documents. You'll most likely need
+    to compile this by yourself, because this software has been patented until around 2017 and
+    binary packages are not available for most distributions.
+
+.. _setup-ansible:
+
+Install Paperless using ansible
+===============================
+    .. note::
+
+        This role currently only supports Debian 10 Buster and Ubuntu 20.04 Focal or later as target hosts.
+
+1.  Install ansible 2.7+ on the management node. 
+    This may be the target host paperless-ng is being installed on or any remote host which can access the target host.
+    For further details, check the ansible `inventory <https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html>`_ documentation.
+
+    On Debian and Ubuntu, the official repositories should provide a suitable version:
+
+    .. code:: bash
+
+        apt install ansible
+        ansible --version
+
+    Alternatively, you can install the most recent ansible release using PyPI:
+
+    .. code:: bash
+
+        python3 -m pip install ansible
+        ansible --version
+
+    Make sure your taget hosts are accessible:
+
+    .. code:: sh
+
+        ansible -m ping YourAnsibleTargetHostGoesHere
+
+2.  Clone the repository of paperless-ng:
+
+    .. code:: sh
+
+        git clone https://github.com/jonaswinkler/paperless-ng
+
+    Checkout the latest release tag:
+
+    .. code:: sh
+
+        cd paperless-ng
+        git checkout ng-0.9.14
+
+3.  Create an ansible ``playbook.yml`` in the paperless-ng root directory:
+
+    .. code:: yaml
+
+        - hosts: YourAnsibleTargetHostGoesHere
+          become: yes
+          vars_files:
+            - ansible/vars.yml
+          roles:
+            - ansible
+
+    Optional: If you also want to use PostgreSQL on the target system, install and add (for example) the `geerlingguy.postgresql <https://github.com/geerlingguy/ansible-role-postgresql>`_ role:
+
+    .. code:: sh
+
+        ansible-galaxy install geerlingguy.postgresql
+
+    .. code:: yaml
+
+        - hosts: YourAnsibleTargetHostGoesHere
+          become: yes
+          vars_files:
+            - ansible/vars.yml
+          roles:
+            - geerlingguy.postgresql
+            - ansible
+
+    Optional: If you also want to use a reverse proxy on the target system, install and add (for example) the `geerlingguy.nginx <https://github.com/geerlingguy/ansible-role-nginx>`_ role:
+
+    .. code:: sh
+
+        ansible-galaxy install geerlingguy.nginx
+
+    .. code:: yaml
+
+        - hosts: YourAnsibleTargetHostGoesHere
+          become: yes
+          vars_files:
+            - ansible/vars.yml
+          roles:
+            - geerlingguy.postgresql
+            - ansible
+            - geerlingguy.nginx
+
+4.  Create ``ansible/vars.yml`` to configure your ansible deployment:
+
+    .. code:: yaml
+
+        paperless_secret_key: PleaseGenerateAStrongKeyForThis
+
+        paperlessng_superuser_name: YourUserName
+        paperlessng_superuser_email: name@domain.tld
+        paperlessng_superuser_password: YourDesiredPasswordUsedForFirstLogin
+
+        paperlessng_ocr_languages:
+            - eng
+            - deu
+
+    For all of the available options, please check ``ansible/README.md`` and :ref:`configuration`.
+
+    Optional configurations for the above-mentioned PostgreSQL and nginx roles would also go here.
+
+5. Run the ansible playbook from the management node:
+
+    .. code:: sh
+
+        ansible-playbook playbook.yml
+
+    When this step completes successfully, paperless-ng will be available on the target host at ``http://127.0.0.1:8000`` (or the address you configured).
+
 Migration to paperless-ng
 #########################
 
@@ -395,32 +567,27 @@ Migration to paperless-ng is then performed in a few simple steps:
     paperless.
 
 3.  Download the latest release of paperless-ng. You can either go with the
-    docker-compose files or use the archive to build the image yourself.
+    docker-compose files from `here <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/compose>`_
+    or clone the repository to build the image yourself (see :ref:`above <setup-docker_build>`).
     You can either replace your current paperless folder or put paperless-ng
     in a different location.
 
     .. caution::
 
-        The release include a ``.env`` file. This will set the
+        Paperless includes a ``.env`` file. This will set the
         project name for docker compose to ``paperless`` so that paperless-ng will
         automatically reuse your existing paperless volumes. When you start it, it
         will migrate your existing data. After that, your old paperless installation
         will be incompatible with the migrated volumes.
 
-4.  Copy the ``docker-compose.sqlite.yml`` file to ``docker-compose.yml``.
+4.  Download the ``docker-compose.sqlite.yml`` file to ``docker-compose.yml``.
     If you want to switch to PostgreSQL, do that after you migrated your existing
     SQLite database.
 
-5.  Adjust ``docker-compose.yml`` and
-    ``docker-compose.env`` to your needs.
-    See `docker route`_ for details on which edits are advised.
+5.  Adjust ``docker-compose.yml`` and ``docker-compose.env`` to your needs.
+    See :ref:`setup-docker_hub` for details on which edits are advised.
 
-6.  Since ``docker-compose`` would just use the the old paperless image, we need to
-    manually build a new image:
-
-    .. code:: shell-session
-
-        $ docker-compose build
+6.  :ref:`Update paperless. <administration-updating>`
 
 7.  In order to find your existing documents with the new search feature, you need
     to invoke a one-time operation that will create the search index:
@@ -428,7 +595,7 @@ Migration to paperless-ng is then performed in a few simple steps:
     .. code:: shell-session
 
         $ docker-compose run --rm webserver document_index reindex
-    
+
     This will migrate your database and create the search index. After that,
     paperless will take care of maintaining the index by itself.
 
@@ -441,7 +608,7 @@ Migration to paperless-ng is then performed in a few simple steps:
     This will run paperless in the background and automatically start it on system boot.
 
 9.  Paperless installed a permanent redirect to ``admin/`` in your browser. This
-    redirect is still in place and prevents access to the new UI. Clear
+    redirect is still in place and prevents access to the new UI. Clear your
     browsing cache in order to fix this.
 
 10.  Optionally, follow the instructions below to migrate your existing data to PostgreSQL.
@@ -489,19 +656,17 @@ management commands as below.
 
             $ cd /path/to/paperless
             $ docker-compose run --rm webserver /bin/bash
-        
+
         This will launch the container and initialize the PostgreSQL database.
-    
-    b)  Without docker, open a shell in your virtual environment, switch to
+
+    b)  Without docker, remember to activate any virtual environment, switch to
         the ``src`` directory and create the database schema:
 
         .. code:: shell-session
 
-            $ cd /path/to/paperless
-            $ pipenv shell
-            $ cd src
+            $ cd /path/to/paperless/src
             $ python3 manage.py migrate
-        
+
         This will not copy any data yet.
 
 4.  Dump your data from SQLite:
@@ -509,14 +674,14 @@ management commands as below.
     .. code:: shell-session
 
         $ python3 manage.py dumpdata --database=sqlite --exclude=contenttypes --exclude=auth.Permission > data.json
-    
+
 5.  Load your data into PostgreSQL:
 
     .. code:: shell-session
 
         $ python3 manage.py loaddata data.json
 
-6.  Exit the shell.
+6.  If operating inside Docker, you may exit the shell now.
 
     .. code:: shell-session
 
@@ -560,7 +725,7 @@ as well.
 Considerations for less powerful devices
 ########################################
 
-Paperless runs on Raspberry Pi. However, some things are rather slow on the Pi and 
+Paperless runs on Raspberry Pi. However, some things are rather slow on the Pi and
 configuring some options in paperless can help improve performance immensely:
 
 *   Stick with SQLite to save some resources.
@@ -582,17 +747,15 @@ configuring some options in paperless can help improve performance immensely:
 For details, refer to :ref:`configuration`.
 
 .. note::
-    
+
     Updating the :ref:`automatic matching algorithm <advanced-automatic_matching>`
     takes quite a bit of time. However, the update mechanism checks if your
-    data has changed before doing the heavy lifting. If you experience the 
+    data has changed before doing the heavy lifting. If you experience the
     algorithm taking too much cpu time, consider changing the schedule in the
     admin interface to daily. You can also manually invoke the task
     by changing the date and time of the next run to today/now.
 
-    The actual matching of the algorithm is fast and works on Raspberry Pi as 
+    The actual matching of the algorithm is fast and works on Raspberry Pi as
     well as on any other device.
 
-
-
 .. _redis: https://redis.io/

docs/troubleshooting.rst

@@ -30,9 +30,21 @@ Consumer fails to pickup any new files
 ######################################
 
 If you notice that the consumer will only pickup files in the consumption
-directory at startup, but won't find any other files added later, check out
-the configuration file and enable filesystem polling with the setting
-``PAPERLESS_CONSUMER_POLLING``.
+directory at startup, but won't find any other files added later, you will need to
+enable filesystem polling with the configuration option
+``PAPERLESS_CONSUMER_POLLING``, see :ref:`here <configuration-polling>`.
+
+This will disable listening to filesystem changes with inotify and paperless will
+manually check the consumption directory for changes instead.
+
+
+Paperless always redirects to /admin
+####################################
+
+You probably had the old paperless installed at some point. Paperless installed
+a permanent redirect to /admin in your browser, and you need to clear your
+browsing data / cache to fix that.
+
 
 Operation not permitted
 #######################
@@ -61,6 +73,24 @@ This may have two reasons:
     with Inbox tags. Verify that there are documents in your archive without inbox tags.
     The algorithm will only learn from documents not in your inbox.
 
+UserWarning in sklearn on every single document
+###############################################
+
+You may encounter warnings like this:
+
+.. code::
+    
+    /usr/local/lib/python3.7/site-packages/sklearn/base.py:315:
+    UserWarning: Trying to unpickle estimator CountVectorizer from version 0.23.2 when using version 0.24.0.
+    This might lead to breaking code or invalid results. Use at your own risk.
+
+This happens when certain dependencies of paperless that are responsible for the auto matching algorithm are
+updated. After updating these, your current training data *might* not be compatible anymore. This can be ignored
+in most cases. This warning will disappear automatically when paperless updates the training data.
+
+If you want to get rid of the warning or actually experience issues with automatic matching, delete
+the file ``classification_model.pickle`` in the data directory and let paperless recreate it.
+
 Permission denied errors in the consumption directory
 #####################################################
 
@@ -72,6 +102,50 @@ You might encounter errors such as:
 
 This happens when paperless does not have permission to delete files inside the consumption directory.
 Ensure that ``USERMAP_UID`` and ``USERMAP_GID`` are set to the user id and group id you use on the host operating system, if these are
-different from ``1000``. See :ref:`setup-docker_route`.
+different from ``1000``. See :ref:`setup-docker_hub`.
 
 Also ensure that you are able to read and write to the consumption directory on the host.
+
+Web-UI stuck at "Loading..."
+############################
+
+This might have multiple reasons.
+
+
+1.  If you built the docker image yourself or deployed using the bare metal route,
+    make sure that there are files in ``<paperless-root>/static/frontend/<lang-code>/``.
+    If there are no files, make sure that you executed ``collectstatic`` successfully, either
+    manually or as part of the docker image build.
+
+    If the front end is still missing, make sure that the front end is compiled (files present in
+    ``src/documents/static/frontend``). If it is not, you need to compile the front end yourself
+    or download the release archive instead of cloning the repository.
+
+2.  Check the output of the web server. You might see errors like this:
+
+
+    .. code::
+
+        [2021-01-25 10:08:04 +0000] [40] [ERROR] Socket error processing request.
+        Traceback (most recent call last):
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 134, in handle
+            self.handle_request(listener, req, client, addr)
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 190, in handle_request
+            util.reraise(*sys.exc_info())
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/util.py", line 625, in reraise
+            raise value
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 178, in handle_request
+            resp.write_file(respiter)
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/http/wsgi.py", line 396, in write_file
+            if not self.sendfile(respiter):
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/http/wsgi.py", line 386, in sendfile
+            sent += os.sendfile(sockno, fileno, offset + sent, count)
+        OSError: [Errno 22] Invalid argument
+    
+    To fix this issue, add
+
+    .. code::
+
+        SENDFILE=0
+    
+    to your `docker-compose.env` file.

docs/usage_overview.rst

@@ -110,6 +110,8 @@ The dashboard has a file drop field to upload documents to paperless. Simply dra
 onto this field or select a file with the file dialog. Multiple files are supported.
 
 
+.. _usage-mobile_upload:
+
 Mobile upload
 =============
 
@@ -118,7 +120,7 @@ to share any documents with paperless. This can be combined with any of the mobi
 scanning apps out there, such as Office Lens.
 
 Furthermore, there is the  `Paperless App <https://github.com/bauerj/paperless_app>`_ as well,
-which no only has document upload, but also document editing and browsing.
+which not only has document upload, but also document browsing and download features.
 
 .. _usage-email:
 

paperless.conf.example

@@ -13,6 +13,7 @@
 #PAPERLESS_DBNAME=paperless
 #PAPERLESS_DBUSER=paperless
 #PAPERLESS_DBPASS=paperless
+#PAPERLESS_DBSSLMODE=prefer
 
 # Paths and folders
 
@@ -26,11 +27,12 @@
 
 #PAPERLESS_SECRET_KEY=change-me
 #PAPERLESS_ALLOWED_HOSTS=example.com,www.example.com
-#PAPERLESS_CORS_ALLOWED_HOSTS=localhost:8080,example.com,localhost:8000
+#PAPERLESS_CORS_ALLOWED_HOSTS=http://example.com,http://localhost:8000
 #PAPERLESS_FORCE_SCRIPT_NAME=
 #PAPERLESS_STATIC_URL=/static/
 #PAPERLESS_AUTO_LOGIN_USERNAME=
 #PAPERLESS_COOKIE_PREFIX=
+#PAPERLESS_ENABLE_HTTP_REMOTE_USER=false
 
 # OCR settings
 
@@ -39,7 +41,7 @@
 #PAPERLESS_OCR_OUTPUT_TYPE=pdfa
 #PAPERLESS_OCR_PAGES=1
 #PAPERLESS_OCR_IMAGE_DPI=300
-#PAPERLESS_OCR_USER_ARG={}
+#PAPERLESS_OCR_USER_ARGS={}
 #PAPERLESS_CONVERT_MEMORY_LIMIT=0
 #PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
 
@@ -50,10 +52,20 @@
 #PAPERLESS_TIME_ZONE=UTC
 #PAPERLESS_CONSUMER_POLLING=10
 #PAPERLESS_CONSUMER_DELETE_DUPLICATES=false
+#PAPERLESS_CONSUMER_RECURSIVE=false
+#PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
 #PAPERLESS_OPTIMIZE_THUMBNAILS=true
 #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_FILENAME_DATE_ORDER=YMD
 #PAPERLESS_FILENAME_PARSE_TRANSFORMS=[]
+#PAPERLESS_THUMBNAIL_FONT_NAME=
+#PAPERLESS_IGNORE_DATES=
+
+# Tika settings
+
+#PAPERLESS_TIKA_ENABLED=false
+#PAPERLESS_TIKA_ENDPOINT=http://localhost:9998
+#PAPERLESS_TIKA_GOTENBERG_ENDPOINT=http://localhost:3000
 
 # Binaries
 

requirements.txt

@@ -0,0 +1,74 @@
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
+-i https://pypi.python.org/simple
+--extra-index-url https://www.piwheels.org/simple
+arrow==0.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+asgiref==3.3.1; python_version >= '3.5'
+blessed==1.17.12
+certifi==2020.12.5
+cffi==1.14.4
+chardet==4.0.0; python_version >= '3.1'
+coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+dateparser==0.7.6
+django-cors-headers==3.6.0
+django-extensions==3.1.0
+django-filter==2.4.0
+django-picklefield==3.0.1; python_version >= '3'
+django-q==1.3.4
+django==3.1.5
+djangorestframework==3.12.2
+filelock==3.0.12
+fuzzywuzzy==0.18.0
+gunicorn==20.0.4
+humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+imap-tools==0.34.0
+img2pdf==0.4.0
+importlib-metadata==3.4.0; python_version < '3.8'
+inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+inotifyrecursive==0.3.5
+joblib==1.0.0; python_version >= '3.6'
+langdetect==1.0.8
+lxml==4.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+numpy==1.19.5; python_version >= '3.6'
+ocrmypdf==11.4.5
+pathvalidate==2.3.2
+pdfminer.six==20201018; python_version >= '3.4'
+pdftotext==2.1.5
+pikepdf==2.2.5
+pillow==8.1.0
+pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+psycopg2-binary==2.8.6
+pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.8.1
+python-dotenv==0.15.0
+python-gnupg==0.4.6
+python-levenshtein==0.12.0
+python-magic==0.4.18
+pytz==2020.5
+redis==3.5.3
+regex==2020.11.13
+reportlab==3.5.59
+requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+scikit-learn==0.24.0
+scipy==1.5.4; python_version >= '3.6'
+six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sortedcontainers==2.3.0
+sqlparse==0.4.1; python_version >= '3.5'
+threadpoolctl==2.1.0; python_version >= '3.5'
+tika==1.24
+tqdm==4.56.0
+typing-extensions==3.7.4.3; python_version < '3.8'
+tzlocal==2.1
+urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+watchdog==1.0.2
+wcwidth==0.2.5
+whitenoise==5.2.0
+whoosh==2.7.4
+zipp==3.4.0; python_version >= '3.6'

scripts/make-release.sh

@@ -1,125 +0,0 @@
-#!/bin/bash
-
-# Release checklist
-# - wait for travis build.
-# adjust src/paperless/version.py
-# changelog in the documentation
-# adjust versions in docker/hub/*
-# adjust version in src-ui/src/environments/prod
-# If docker-compose was modified: all compose files are the same.
-
-# Steps:
-# run release script "dev", push
-# if it works: new tag, merge into master
-# on master: make release "lastest", push
-# on master: make release "version-tag", push
-# publish release files
-
-set -e
-
-
-VERSION=$1
-
-if [ -z "$VERSION" ]
-then
-	echo "Need a version string."
-	exit 1
-fi
-
-# source root directory of paperless
-PAPERLESS_ROOT=$(git rev-parse --show-toplevel)
-
-# output directory
-PAPERLESS_DIST="$PAPERLESS_ROOT/dist"
-PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng"
-PAPERLESS_DIST_DOCKERFILES="$PAPERLESS_DIST/paperless-ng-dockerfiles"
-
-if [ -d "$PAPERLESS_DIST" ]
-then
-	echo "Removing $PAPERLESS_DIST"
-	rm "$PAPERLESS_DIST" -r
-fi
-
-mkdir "$PAPERLESS_DIST"
-mkdir "$PAPERLESS_DIST_APP"
-mkdir "$PAPERLESS_DIST_APP/docker"
-mkdir "$PAPERLESS_DIST_APP/scripts"
-mkdir "$PAPERLESS_DIST_DOCKERFILES"
-
-# setup dependencies.
-
-cd "$PAPERLESS_ROOT"
-
-pipenv clean
-pipenv install --dev
-pipenv lock --keep-outdated -r > "$PAPERLESS_DIST_APP/requirements.txt"
-
-# test if the application works.
-
-cd "$PAPERLESS_ROOT/src"
-pipenv run pytest --cov
-pipenv run pycodestyle
-
-# make the documentation.
-
-cd "$PAPERLESS_ROOT/docs"
-make clean html
-
-# copy stuff into place
-
-# the application itself
-
-cp "$PAPERLESS_ROOT/.env" \
-  "$PAPERLESS_ROOT/.dockerignore" \
-	"$PAPERLESS_ROOT/CONTRIBUTING.md" \
-	"$PAPERLESS_ROOT/LICENSE" \
-	"$PAPERLESS_ROOT/Pipfile" \
-	"$PAPERLESS_ROOT/Pipfile.lock" \
-	"$PAPERLESS_ROOT/README.md" "$PAPERLESS_DIST_APP"
-
-cp "$PAPERLESS_ROOT/paperless.conf.example" "$PAPERLESS_DIST_APP/paperless.conf"
-
-# copy python source, templates and static files.
-cd "$PAPERLESS_ROOT"
-find src -wholename '*/templates/*' -o -wholename '*/static/*' -o -name '*.py' | cpio -pdm "$PAPERLESS_DIST_APP"
-
-# build the front end.
-
-cd "$PAPERLESS_ROOT/src-ui"
-ng build --prod --output-hashing none --sourceMap=false --output-path "$PAPERLESS_DIST_APP/src/documents/static/frontend"
-
-# documentation
-cp "$PAPERLESS_ROOT/docs/_build/html/" "$PAPERLESS_DIST_APP/docs" -r
-
-# docker files for building the image yourself
-cp "$PAPERLESS_ROOT/docker/local/"* "$PAPERLESS_DIST_APP"
-cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_APP"
-
-# docker files for pulling from docker hub
-cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST_DOCKERFILES"
-cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST_DOCKERFILES"
-cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_DOCKERFILES"
-
-# auxiliary files required for the docker image
-cp "$PAPERLESS_ROOT/docker/docker-entrypoint.sh" "$PAPERLESS_DIST_APP/docker/"
-cp "$PAPERLESS_ROOT/docker/gunicorn.conf.py" "$PAPERLESS_DIST_APP/docker/"
-cp "$PAPERLESS_ROOT/docker/imagemagick-policy.xml" "$PAPERLESS_DIST_APP/docker/"
-cp "$PAPERLESS_ROOT/docker/supervisord.conf" "$PAPERLESS_DIST_APP/docker/"
-
-# auxiliary files for bare metal installs
-cp "$PAPERLESS_ROOT/scripts/paperless-webserver.service" "$PAPERLESS_DIST_APP/scripts/"
-cp "$PAPERLESS_ROOT/scripts/paperless-consumer.service" "$PAPERLESS_DIST_APP/scripts/"
-cp "$PAPERLESS_ROOT/scripts/paperless-scheduler.service" "$PAPERLESS_DIST_APP/scripts/"
-
-# try to make the docker build.
-
-cd "$PAPERLESS_DIST_APP"
-
-docker build . -t "jonaswinkler/paperless-ng:$VERSION"
-
-# works. package the app!
-
-cd "$PAPERLESS_DIST"
-
-tar -cJf "paperless-ng-$VERSION.tar.xz" paperless-ng/
-tar -cJf "paperless-ng-$VERSION-dockerfiles.tar.xz" paperless-ng-dockerfiles/

scripts/push-release.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -e
-
-
-VERSION=$1
-
-if [ -z "$VERSION" ]
-then
-	echo "Need a version string."
-	exit 1
-fi
-
-# source root directory of paperless
-PAPERLESS_ROOT=$(git rev-parse --show-toplevel)
-
-# output directory
-PAPERLESS_DIST="$PAPERLESS_ROOT/dist"
-PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng"
-
-cd "$PAPERLESS_DIST_APP"
-
-docker push "jonaswinkler/paperless-ng:$VERSION"

scripts/start_services.sh

@@ -1,2 +1,4 @@
 docker run -p 5432:5432 -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
 docker run -d -p 6379:6379 redis:latest
+docker run -p 3000:3000 -d thecodingmachine/gotenberg
+docker run -p 9998:9998 -d apache/tika

src-ui/angular.json

@@ -13,6 +13,14 @@
 			"root": "",
 			"sourceRoot": "src",
 			"prefix": "app",
+			"i18n": {
+				"sourceLocale": "en-US",
+				"locales": {
+					"de": "src/locale/messages.de.xlf",
+					"nl-NL": "src/locale/messages.nl_NL.xlf",
+					"fr": "src/locale/messages.fr.xlf"
+				}
+			},
 			"architect": {
 				"build": {
 					"builder": "@angular-devkit/build-angular:browser",
@@ -23,15 +31,24 @@
 						"main": "src/main.ts",
 						"polyfills": "src/polyfills.ts",
 						"tsConfig": "tsconfig.app.json",
+						"localize": true,
 						"aot": true,
 						"assets": [
 							"src/favicon.ico",
-							"src/assets"
+							"src/assets",
+							"src/manifest.webmanifest", {
+								"glob": "pdf.worker.min.js",
+								"input": "node_modules/pdfjs-dist/build/",
+								"output": "/assets/js/"
+							}
 						],
 						"styles": [
 							"src/styles.scss"
 						],
-						"scripts": []
+						"scripts": [],
+						"allowedCommonJsDependencies": [
+							"ng2-pdf-viewer"
+						]
 					},
 					"configurations": {
 						"production": {
@@ -41,6 +58,7 @@
 									"with": "src/environments/environment.prod.ts"
 								}
 							],
+							"outputPath": "../src/documents/static/frontend/",
 							"optimization": true,
 							"outputHashing": "none",
 							"sourceMap": false,
@@ -61,13 +79,16 @@
 									"maximumError": "10kb"
 								}
 							]
+						},
+						"en-US": {
+							"localize": ["en-US"]
 						}
 					}
 				},
 				"serve": {
 					"builder": "@angular-devkit/build-angular:dev-server",
 					"options": {
-						"browserTarget": "paperless-ui:build"
+						"browserTarget": "paperless-ui:build:en-US"
 					},
 					"configurations": {
 						"production": {
@@ -90,7 +111,8 @@
 						"karmaConfig": "karma.conf.js",
 						"assets": [
 							"src/favicon.ico",
-							"src/assets"
+							"src/assets",
+							"src/manifest.webmanifest"
 						],
 						"styles": [
 							"src/styles.scss"
@@ -127,4 +149,4 @@
 		}
 	},
 	"defaultProject": "paperless-ui"
-}
+}

src-ui/package-lock.json

@@ -331,6 +331,12 @@
             "ms": "^2.1.1"
           }
         },
+        "ini": {
+          "version": "1.3.5",
+          "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+          "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+          "dev": true
+        },
         "uuid": {
           "version": "8.3.0",
           "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.0.tgz",
@@ -2178,6 +2184,14 @@
         "pacote": "9.5.12",
         "semver": "7.3.2",
         "semver-intersect": "1.4.0"
+      },
+      "dependencies": {
+        "ini": {
+          "version": "1.3.5",
+          "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+          "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+          "dev": true
+        }
       }
     },
     "@types/glob": {

src-ui/src/app/app.component.ts

@@ -1,4 +1,5 @@
 import { Component } from '@angular/core';
+import { SettingsService } from './services/settings.service';
 
 @Component({
   selector: 'app-root',
@@ -6,9 +7,11 @@ import { Component } from '@angular/core';
   styleUrls: ['./app.component.scss']
 })
 export class AppComponent {
-  
-  constructor () {
+
+  constructor (private settings: SettingsService) {
+    let anyWindow = (window as any)
+    anyWindow.pdfWorkerSrc = '/assets/js/pdf.worker.min.js';
+    this.settings.updateDarkModeSettings()
   }
 
-
 }

src-ui/src/app/app.module.ts

@@ -13,7 +13,7 @@ import { DocumentTypeListComponent } from './components/manage/document-type-lis
 import { LogsComponent } from './components/manage/logs/logs.component';
 import { SettingsComponent } from './components/manage/settings/settings.component';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
-import { DatePipe } from '@angular/common';
+import { DatePipe, registerLocaleData } from '@angular/common';
 import { NotFoundComponent } from './components/not-found/not-found.component';
 import { CorrespondentListComponent } from './components/manage/correspondent-list/correspondent-list.component';
 import { ConfirmDialogComponent } from './components/common/confirm-dialog/confirm-dialog.component';
@@ -26,12 +26,13 @@ import { ResultHighlightComponent } from './components/search/result-highlight/r
 import { PageHeaderComponent } from './components/common/page-header/page-header.component';
 import { AppFrameComponent } from './components/app-frame/app-frame.component';
 import { ToastsComponent } from './components/common/toasts/toasts.component';
-import { FilterEditorComponent } from './components/filter-editor/filter-editor.component';
-import { FilterDropdownComponent } from './components/filter-editor/filter-dropdown/filter-dropdown.component';
-import { FilterDropdownButtonComponent } from './components/filter-editor/filter-dropdown/filter-dropdown-button/filter-dropdown-button.component';
-import { FilterDropdownDateComponent } from './components/filter-editor/filter-dropdown-date/filter-dropdown-date.component';
+import { FilterEditorComponent } from './components/document-list/filter-editor/filter-editor.component';
+import { FilterableDropdownComponent } from './components/common/filterable-dropdown/filterable-dropdown.component';
+import { ToggleableDropdownButtonComponent } from './components/common/filterable-dropdown/toggleable-dropdown-button/toggleable-dropdown-button.component';
+import { DateDropdownComponent } from './components/common/date-dropdown/date-dropdown.component';
 import { DocumentCardLargeComponent } from './components/document-list/document-card-large/document-card-large.component';
 import { DocumentCardSmallComponent } from './components/document-list/document-card-small/document-card-small.component';
+import { BulkEditorComponent } from './components/document-list/bulk-editor/bulk-editor.component';
 import { NgxFileDropModule } from 'ngx-file-drop';
 import { TextComponent } from './components/common/input/text/text.component';
 import { SelectComponent } from './components/common/input/select/select.component';
@@ -54,8 +55,19 @@ import { FileSizePipe } from './pipes/file-size.pipe';
 import { FilterPipe } from './pipes/filter.pipe';
 import { DocumentTitlePipe } from './pipes/document-title.pipe';
 import { MetadataCollapseComponent } from './components/document-detail/metadata-collapse/metadata-collapse.component';
-import { NgSelectModule } from '@ng-select/ng-select';
 import { SelectDialogComponent } from './components/common/select-dialog/select-dialog.component';
+import { NgSelectModule } from '@ng-select/ng-select';
+import { NumberComponent } from './components/common/input/number/number.component';
+import { SafePipe } from './pipes/safe.pipe';
+import { CustomDatePipe } from './pipes/custom-date.pipe';
+
+import localeFr from '@angular/common/locales/fr';
+import localeNl from '@angular/common/locales/nl';
+import localeDe from '@angular/common/locales/de';
+
+registerLocaleData(localeFr)
+registerLocaleData(localeNl)
+registerLocaleData(localeDe)
 
 @NgModule({
   declarations: [
@@ -80,11 +92,12 @@ import { SelectDialogComponent } from './components/common/select-dialog/select-
     AppFrameComponent,
     ToastsComponent,
     FilterEditorComponent,
-    FilterDropdownComponent,
-    FilterDropdownButtonComponent,
-    FilterDropdownDateComponent,
+    FilterableDropdownComponent,
+    ToggleableDropdownButtonComponent,
+    DateDropdownComponent,
     DocumentCardLargeComponent,
     DocumentCardSmallComponent,
+    BulkEditorComponent,
     TextComponent,
     SelectComponent,
     CheckComponent,
@@ -102,7 +115,10 @@ import { SelectDialogComponent } from './components/common/select-dialog/select-
     FilterPipe,
     DocumentTitlePipe,
     MetadataCollapseComponent,
-    SelectDialogComponent
+    SelectDialogComponent,
+    NumberComponent,
+    SafePipe,
+    CustomDatePipe
   ],
   imports: [
     BrowserModule,

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -1,17 +1,52 @@
 <nav class="navbar navbar-dark sticky-top bg-primary flex-md-nowrap p-0 shadow">
-  <span class="navbar-brand col-md-3 col-lg-2 mr-0 px-3" href="#">
-    <img src="assets/logo-dark-notext.svg" height="18px" class="mr-2">
-    Paperless-ng
-  </span>
-  <button class="navbar-toggler position-absolute d-md-none collapsed" type="button" data-toggle="collapse"
+  <button class="navbar-toggler d-md-none collapsed border-0" type="button" data-toggle="collapse"
     data-target="#sidebarMenu" aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation"
     (click)="isMenuCollapsed = !isMenuCollapsed">
     <span class="navbar-toggler-icon"></span>
   </button>
-  <form (ngSubmit)="search()" class="w-100 m-1">
-    <input class="form-control form-control-dark" type="text" placeholder="Search" aria-label="Search"
-      [formControl]="searchField" [ngbTypeahead]="searchAutoComplete" (selectItem)="itemSelected($event)">
-  </form>
+  <a class="navbar-brand col-auto col-md-3 col-lg-2 mr-0 px-3 py-3 order-sm-0" routerLink="/dashboard">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 198.43 238.91" width="1rem" class="mr-2" fill="currentColor">
+      <path d="M194.7,0C164.22,70.94,17.64,79.74,64.55,194.06c.58,1.47-10.85,17-18.47,29.9-1.76-6.45-3.81-13.48-3.52-14.07,38.11-45.14-27.26-70.65-30.78-107.58C-4.64,131.62-10.5,182.92,39,212.53c.3,0,2.64,11.14,3.81,16.71a58.55,58.55,0,0,0-2.93,6.45c-1.17,2.93,7.62,2.64,7.62,3.22.88-.29,21.7-36.93,22.28-37.23C187.67,174.72,208.48,68.6,194.7,0ZM134.61,74.75C79.5,124,70.12,160.64,71.88,178.53,53.41,134.85,107.64,86.77,134.61,74.75ZM28.2,145.11c10.55,9.67,28.14,39.28,13.19,56.57C44.91,193.77,46.08,175.89,28.2,145.11Z" transform="translate(0 0)"/>
+    </svg>
+    <ng-container i18n="app title">Paperless-ng</ng-container>
+  </a>
+  <div class="search-form-container flex-grow-1 py-2 pb-3 pb-sm-2 px-3 pl-md-4 mr-sm-auto order-3 order-sm-1">
+    <form (ngSubmit)="search()" class="form-inline flex-grow-1">
+      <input class="form-control form-control-sm" type="text" placeholder="Search documents" aria-label="Search"
+        [formControl]="searchField" [ngbTypeahead]="searchAutoComplete" (selectItem)="itemSelected($event)" i18n-placeholder>
+      <svg width="1em" height="1em">
+        <use xlink:href="assets/bootstrap-icons.svg#search"/>
+      </svg>
+    </form>
+  </div>
+  <ul ngbNav class="order-sm-3">
+    <li ngbDropdown class="nav-item dropdown">
+      <button class="btn text-light" id="userDropdown" ngbDropdownToggle>
+        <span *ngIf="displayName" class="navbar-text small mr-2 text-light d-none d-sm-inline">
+          {{displayName}}
+        </span>
+        <svg width="1.3em" height="1.3em">
+          <use xlink:href="assets/bootstrap-icons.svg#person-circle"/>
+        </svg>
+      </button>
+      <div ngbDropdownMenu class="dropdown-menu-right shadow mr-2" aria-labelledby="userDropdown">
+        <div *ngIf="displayName" class="d-sm-none">
+          <p class="small mb-0 px-3" i18n>Logged in as {{displayName}}</p>
+          <div class="dropdown-divider"></div>
+        </div>
+        <a ngbDropdownItem class="nav-link" routerLink="settings" (click)="closeMenu()">
+          <svg class="sidebaricon mr-2" fill="currentColor">
+            <use xlink:href="assets/bootstrap-icons.svg#gear"/>
+          </svg><ng-container i18n>Settings</ng-container>
+        </a>
+        <a ngbDropdownItem class="nav-link" href="accounts/logout/">
+          <svg class="sidebaricon mr-2" fill="currentColor">
+            <use xlink:href="assets/bootstrap-icons.svg#door-open"/>
+          </svg><ng-container i18n>Logout</ng-container>
+        </a>
+      </div>
+    </li>
+  </ul>
 </nav>
 
 <div class="container-fluid">
@@ -28,136 +63,115 @@
             <a class="nav-link" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#house"/>
-              </svg>
-              Dashboard
+              </svg>&nbsp;<ng-container i18n>Dashboard</ng-container>
             </a>
           </li>
           <li class="nav-item">
             <a class="nav-link" routerLink="documents" routerLinkActive="active" [routerLinkActiveOptions]="{exact: true}" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#files"/>
-              </svg>
-              Documents
+              </svg>&nbsp;<ng-container i18n>Documents</ng-container>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted" *ngIf='savedViewService.sidebarViews.length > 0'>
-          <span>Saved views</span>
+          <ng-container i18n>Saved views</ng-container>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item w-100" *ngFor="let view of savedViewService.sidebarViews">
             <a class="nav-link text-truncate" routerLink="view/{{view.id}}" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#funnel"/>
-              </svg>
-              {{view.name}}
+              </svg>&nbsp;{{view.name}}
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted" *ngIf='openDocuments.length > 0'>
-          <span>Open documents</span>
+          <ng-container i18n>Open documents</ng-container>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item w-100" *ngFor='let d of openDocuments'>
             <a class="nav-link text-truncate" routerLink="documents/{{d.id}}" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#file-text"/>
-              </svg>
-              {{d.title | documentTitle}}
+              </svg>&nbsp;{{d.title | documentTitle}}
             </a>
           </li>
           <li class="nav-item w-100" *ngIf="openDocuments.length > 1">
             <a class="nav-link text-truncate" [routerLink]="" (click)="closeAll()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#x"/>
-              </svg>
-              Close all
+              </svg>&nbsp;<ng-container i18n>Close all</ng-container>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
-          <span>Manage</span>
+          <ng-container i18n>Manage</ng-container>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item">
             <a class="nav-link" routerLink="correspondents" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#person"/>
-              </svg>
-              Correspondents
+              </svg>&nbsp;<ng-container i18n>Correspondents</ng-container>
             </a>
           </li>
           <li class="nav-item">
             <a class="nav-link" routerLink="tags" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#tags"/>
-              </svg>
-              Tags
+              </svg>&nbsp;<ng-container i18n>Tags</ng-container>
             </a>
           </li>
           <li class="nav-item">
             <a class="nav-link" routerLink="documenttypes" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#hash"/>
-              </svg>
-              Document types
+              </svg>&nbsp;<ng-container i18n>Document types</ng-container>
             </a>
           </li>
           <li class="nav-item">
             <a class="nav-link" routerLink="logs" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#text-left"/>
-              </svg>
-              Logs
+              </svg>&nbsp;<ng-container i18n>Logs</ng-container>
             </a>
           </li>
           <li class="nav-item">
             <a class="nav-link" routerLink="settings" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#gear"/>
-              </svg>
-              Settings
+              </svg>&nbsp;<ng-container i18n>Settings</ng-container>
             </a>
           </li>
           <li class="nav-item">
             <a class="nav-link" href="admin/">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#toggles"/>
-              </svg>
-              Admin
+              </svg>&nbsp;<ng-container i18n>Admin</ng-container>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
-          <span>Misc</span>
+          <ng-container i18n>Misc</ng-container>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item">
             <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://paperless-ng.readthedocs.io/en/latest/">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#question-circle"/>
-              </svg>
-              Documentation
+              </svg>&nbsp;<ng-container i18n>Documentation</ng-container>
             </a>
           </li>
           <li class="nav-item">
             <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://github.com/jonaswinkler/paperless-ng">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#link"/>
-              </svg>
-              GitHub
-            </a>
-          </li>
-          <li class="nav-item">
-            <a class="nav-link" href="accounts/logout/">
-              <svg class="sidebaricon" fill="currentColor">
-                <use xlink:href="assets/bootstrap-icons.svg#door-open"/>
-              </svg>
-              Logout
+              </svg>&nbsp;<ng-container i18n>GitHub</ng-container>
             </a>
           </li>
         </ul>

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -1,36 +1,30 @@
-
 @import "/src/theme";
-
-  /*
+/*
  * Sidebar
  */
-
- .sidebar {
+.sidebar {
   position: fixed;
   top: 0;
   bottom: 0;
   left: 0;
   z-index: 100; /* Behind the navbar */
-  padding: 48px 0 0; /* Height of navbar */
+  padding: 50px 0 0; /* Height of navbar */
   box-shadow: inset -1px 0 0 rgba(0, 0, 0, .1);
 }
-
 @media (max-width: 767.98px) {
   .sidebar {
-    top: 3rem;
+    top: 3.5rem;
   }
 }
 
 .sidebar-sticky {
   position: relative;
   top: 0;
-  /* height: calc(100vh - 48px); */
   height: 100%;
-  padding-top: .5rem;
+  padding-top: 0.5rem;
   overflow-x: hidden;
   overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */
 }
-
 @supports ((position: -webkit-sticky) or (position: sticky)) {
   .sidebar-sticky {
     position: -webkit-sticky;
@@ -53,36 +47,85 @@
   font-weight: bold;
 }
 
-.sidebar .nav-link:hover .sidebaricon,
-.sidebar .nav-link.active .sidebaricon {
+.sidebar .nav-link.active .sidebaricon,
+.sidebar .nav-link:hover .sidebaricon {
   color: inherit;
 }
 
 .sidebar-heading {
-  font-size: .75rem;
+  font-size: 0.75rem;
   text-transform: uppercase;
 }
 
-
 /*
  * Navbar
  */
 
- .navbar-brand {
-  padding-top: .75rem;
-  padding-bottom: .75rem;
+.navbar-brand {
+  padding-top: 0.75rem;
+  padding-bottom: 0.75rem;
   font-size: 1rem;
-  background-color: rgba(0, 0, 0, .25);
-  box-shadow: inset -1px 0 0 rgba(0, 0, 0, .25);
 }
 
-.navbar .navbar-toggler {
-  top: .25rem;
-  right: 1rem;
+.dropdown.show .dropdown-toggle,
+.dropdown-toggle:hover {
+  opacity: 0.7;
 }
 
-.navbar .form-control {
-  padding: .75rem 1rem;
-  border-width: 0;
-  border-radius: 0;
+.dropdown-toggle::after {
+  margin-left: 0.4em;
+  vertical-align: 0.155em;
+}
+
+.navbar .dropdown-menu {
+  font-size: 0.875rem; // body size
+
+  a svg {
+    opacity: 0.6;
+  }
+}
+
+.navbar .search-form-container {
+  max-width: 550px;
+
+  form {
+    position: relative;
+  }
+
+  svg {
+    position: absolute;
+    left: 0.6rem;
+    color: rgba(255, 255, 255, 0.6);
+  }
+
+  &:focus-within {
+    svg {
+      display: none;
+    }
+
+    .form-control::placeholder {
+      color: rgba(255, 255, 255, 0);
+    }
+  }
+
+  .form-control {
+    color: rgba(255, 255, 255, 0.3);
+    background-color: rgba(0, 0, 0, 0.15);
+    padding-left: 1.8rem;
+    border-color: rgba(255, 255, 255, 0.2);
+    transition: all .3s ease, padding-left 0s ease, background-color 0s ease; // Safari requires all
+    max-width: 600px;
+    min-width: 300px; // 1/2 max
+
+    &::placeholder {
+      color: rgba(255, 255, 255, 0.4);
+    }
+
+    &:focus {
+      background-color: #fff;
+      color: #212529;
+      flex-grow: 1;
+      padding-left: 0.5rem;
+    }
+  }
 }

src-ui/src/app/components/app-frame/app-frame.component.ts

@@ -9,7 +9,8 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service';
 import { SearchService } from 'src/app/services/rest/search.service';
 import { environment } from 'src/environments/environment';
 import { DocumentDetailComponent } from '../document-detail/document-detail.component';
-  
+import { Meta } from '@angular/platform-browser';
+
 @Component({
   selector: 'app-app-frame',
   templateUrl: './app-frame.component.html',
@@ -22,8 +23,10 @@ export class AppFrameComponent implements OnInit, OnDestroy {
     private activatedRoute: ActivatedRoute,
     private openDocumentsService: OpenDocumentsService,
     private searchService: SearchService,
-    public savedViewService: SavedViewService
+    public savedViewService: SavedViewService,
+    private meta: Meta
     ) {
+      
   }
 
   versionString = `${environment.appTitle} ${environment.version}`
@@ -55,7 +58,7 @@ export class AppFrameComponent implements OnInit, OnDestroy {
         term.length < 2 ? from([[]]) : this.searchService.autocomplete(term)
       )
     )
-  
+
   itemSelected(event) {
     event.preventDefault()
     let currentSearch: string = this.searchField.value
@@ -98,4 +101,17 @@ export class AppFrameComponent implements OnInit, OnDestroy {
     }
   }
 
+  get displayName() {
+    // TODO: taken from dashboard component, is this the best way to pass around username?
+    let tagFullName = this.meta.getTag('name=full_name')
+    let tagUsername = this.meta.getTag('name=username')
+    if (tagFullName && tagFullName.content) {
+      return tagFullName.content
+    } else if (tagUsername && tagUsername.content) {
+      return tagUsername.content
+    } else {
+      return null
+    }
+  }
+
 }

src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html

@@ -9,8 +9,8 @@
       <p *ngIf="message">{{message}}</p>
     </div>
     <div class="modal-footer">
-      <button type="button" class="btn btn-outline-dark" (click)="cancelClicked()">Cancel</button>
-      <button type="button" class="btn" [class]="btnClass" (click)="confirmClicked.emit()" [disabled]="!confirmButtonEnabled">
+      <button type="button" class="btn btn-outline-dark" (click)="cancelClicked()" [disabled]="!buttonsEnabled" i18n>Cancel</button>
+      <button type="button" class="btn" [class]="btnClass" (click)="confirmClicked.emit()" [disabled]="!confirmButtonEnabled || !buttonsEnabled">
         {{btnCaption}}
         <span *ngIf="!confirmButtonEnabled"> ({{seconds}})</span>
       </button>

src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts

@@ -14,7 +14,7 @@ export class ConfirmDialogComponent implements OnInit {
   public confirmClicked = new EventEmitter()
 
   @Input()
-  title = "Confirmation"
+  title = $localize`Confirmation`
 
   @Input()
   messageBold
@@ -26,8 +26,11 @@ export class ConfirmDialogComponent implements OnInit {
   btnClass = "btn-primary"
 
   @Input()
-  btnCaption = "Confirm"
+  btnCaption = $localize`Confirm`
 
+  @Input()
+  buttonsEnabled = true
+  
   confirmButtonEnabled = true
   seconds = 0
 

src-ui/src/app/components/common/date-dropdown/date-dropdown.component.html

@@ -1,8 +1,8 @@
-  <div class="btn-group" ngbDropdown role="group">
+  <div class="btn-group w-100" ngbDropdown role="group">
   <button class="btn btn-sm" id="dropdown{{title}}" ngbDropdownToggle [ngClass]="dateBefore || dateAfter ? 'btn-primary' : 'btn-outline-primary'">
     {{title}}
   </button>
-  <div class="dropdown-menu date-filter shadow pt-0" ngbDropdownMenu attr.aria-labelledby="dropdown{{title}}">
+  <div class="dropdown-menu date-dropdown shadow pt-0" ngbDropdownMenu attr.aria-labelledby="dropdown{{title}}">
     <div class="list-group list-group-flush">
         <button *ngFor="let qf of quickFilters" class="list-group-item small list-goup list-group-item-action d-flex p-2 pl-3" role="menuitem" (click)="setDateQuickFilter(qf.id)">
           {{qf.name}}
@@ -10,12 +10,12 @@
         <div class="list-group-item d-flex flex-column align-items-start" role="menuitem">
 
           <div class="mb-2 d-flex flex-row w-100 justify-content-between small">
-            <div>After</div>
+            <div i18n>After</div>
             <a *ngIf="dateAfter" class="btn btn-link p-0 m-0" (click)="clearAfter()">
               <svg width="0.8em" height="0.8em" viewBox="0 0 16 16" class="bi bi-x" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
                 <path fill-rule="evenodd" d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z" />
               </svg>
-              <small>Clear</small>
+              <small i18n>Clear</small>
             </a>
           </div>
 
@@ -26,12 +26,12 @@
         <div class="list-group-item d-flex flex-column align-items-start" role="menuitem">
 
           <div class="mb-2 d-flex flex-row w-100 justify-content-between small">
-            <div>Before</div>
+            <div i18n>Before</div>
             <a *ngIf="dateBefore" class="btn btn-link p-0 m-0" (click)="clearBefore()">
               <svg width="0.8em" height="0.8em" viewBox="0 0 16 16" class="bi bi-x" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
                 <path fill-rule="evenodd" d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z" />
               </svg>
-              <small>Clear</small>
+              <small i18n>Clear</small>
             </a>
           </div>
 

src-ui/src/app/components/common/date-dropdown/date-dropdown.component.scss

@@ -1,4 +1,4 @@
-.date-filter {
+.date-dropdown {
   min-width: 250px;
 
   .btn-link {

src-ui/src/app/components/common/date-dropdown/date-dropdown.component.spec.ts

@@ -1,20 +1,20 @@
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { FilterDropodownComponent } from './filter-dropdown.component';
+import { DateDropdownComponent } from './date-dropdown.component';
 
-describe('FilterDropodownComponent', () => {
-  let component: FilterDropodownComponent;
-  let fixture: ComponentFixture<FilterDropodownComponent>;
+describe('DateDropdownComponent', () => {
+  let component: DateDropdownComponent;
+  let fixture: ComponentFixture<DateDropdownComponent>;
 
   beforeEach(async () => {
     await TestBed.configureTestingModule({
-      declarations: [ FilterDropodownComponent ]
+      declarations: [ DateDropdownComponent ]
     })
     .compileComponents();
   });
 
   beforeEach(() => {
-    fixture = TestBed.createComponent(FilterDropodownComponent);
+    fixture = TestBed.createComponent(DateDropdownComponent);
     component = fixture.componentInstance;
     fixture.detectChanges();
   });

src-ui/src/app/components/common/date-dropdown/date-dropdown.component.ts

@@ -8,31 +8,37 @@ export interface DateSelection {
   after?: string
 }
 
-const FILTER_LAST_7_DAYS = 0
-const FILTER_LAST_MONTH = 1
-const FILTER_LAST_3_MONTHS = 2
-const FILTER_LAST_YEAR = 3
+const LAST_7_DAYS = 0
+const LAST_MONTH = 1
+const LAST_3_MONTHS = 2
+const LAST_YEAR = 3
 
 @Component({
-  selector: 'app-filter-dropdown-date',
-  templateUrl: './filter-dropdown-date.component.html',
-  styleUrls: ['./filter-dropdown-date.component.scss']
+  selector: 'app-date-dropdown',
+  templateUrl: './date-dropdown.component.html',
+  styleUrls: ['./date-dropdown.component.scss']
 })
-export class FilterDropdownDateComponent implements OnInit, OnDestroy {
+export class DateDropdownComponent implements OnInit, OnDestroy {
 
   quickFilters = [
-    {id: FILTER_LAST_7_DAYS, name: "Last 7 days"}, 
-    {id: FILTER_LAST_MONTH, name: "Last month"},
-    {id: FILTER_LAST_3_MONTHS, name: "Last 3 months"},
-    {id: FILTER_LAST_YEAR, name: "Last year"}
+    {id: LAST_7_DAYS, name: $localize`Last 7 days`},
+    {id: LAST_MONTH, name: $localize`Last month`},
+    {id: LAST_3_MONTHS, name: $localize`Last 3 months`},
+    {id: LAST_YEAR, name: $localize`Last year`}
   ]
 
   @Input()
   dateBefore: string
 
+  @Output()
+  dateBeforeChange = new EventEmitter<string>()
+
   @Input()
   dateAfter: string
 
+  @Output()
+  dateAfterChange = new EventEmitter<string>()
+
   @Input()
   title: string
 
@@ -42,7 +48,7 @@ export class FilterDropdownDateComponent implements OnInit, OnDestroy {
   private datesSetDebounce$ = new Subject()
 
   private sub: Subscription
-  
+
   ngOnInit() {
     this.sub = this.datesSetDebounce$.pipe(
       debounceTime(400)
@@ -61,28 +67,30 @@ export class FilterDropdownDateComponent implements OnInit, OnDestroy {
     this.dateBefore = null
     let date = new Date()
     switch (qf) {
-      case FILTER_LAST_7_DAYS:
+      case LAST_7_DAYS:
         date.setDate(date.getDate() - 7)
         break;
 
-      case FILTER_LAST_MONTH:
+      case LAST_MONTH:
         date.setMonth(date.getMonth() - 1)
         break;
 
-      case FILTER_LAST_3_MONTHS:
+      case LAST_3_MONTHS:
         date.setMonth(date.getMonth() - 3)
         break
 
-      case FILTER_LAST_YEAR:
+      case LAST_YEAR:
         date.setFullYear(date.getFullYear() - 1)
         break
-  
+
       }
     this.dateAfter = formatDate(date, 'yyyy-MM-dd', "en-us", "UTC")
     this.onChange()
   }
 
   onChange() {
+    this.dateAfterChange.emit(this.dateAfter)
+    this.dateBeforeChange.emit(this.dateBefore)
     this.datesSet.emit({after: this.dateAfter, before: this.dateBefore})
   }
 
@@ -91,12 +99,12 @@ export class FilterDropdownDateComponent implements OnInit, OnDestroy {
   }
 
   clearBefore() {
-    this.dateBefore = null;
+    this.dateBefore = null
     this.onChange()
   }
 
   clearAfter() {
-    this.dateAfter = null;
+    this.dateAfter = null
     this.onChange()
   }
 

src-ui/src/app/components/common/edit-dialog/edit-dialog.component.ts

@@ -2,10 +2,11 @@ import { Directive, EventEmitter, Input, OnInit, Output } from '@angular/core';
 import { FormGroup } from '@angular/forms';
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
 import { Observable } from 'rxjs';
-import { MATCHING_ALGORITHMS } from 'src/app/data/matching-model';
+import { map } from 'rxjs/operators';
+import { MATCHING_ALGORITHMS, MATCH_AUTO } from 'src/app/data/matching-model';
 import { ObjectWithId } from 'src/app/data/object-with-id';
 import { AbstractPaperlessService } from 'src/app/services/rest/abstract-paperless-service';
-import { Toast, ToastService } from 'src/app/services/toast.service';
+import { ToastService } from 'src/app/services/toast.service';
 
 @Directive()
 export abstract class EditDialogComponent<T extends ObjectWithId> implements OnInit {
@@ -13,8 +14,7 @@ export abstract class EditDialogComponent<T extends ObjectWithId> implements OnI
   constructor(
     private service: AbstractPaperlessService<T>,
     private activeModal: NgbActiveModal,
-    private toastService: ToastService,
-    private entityName: string) { }
+    private toastService: ToastService) { }
 
   @Input()
   dialogMode: string = 'create'
@@ -25,6 +25,12 @@ export abstract class EditDialogComponent<T extends ObjectWithId> implements OnI
   @Output()
   success = new EventEmitter()
 
+  networkActive = false
+
+  closeEnabled = false
+
+  error = null
+
   abstract getForm(): FormGroup
 
   objectForm: FormGroup = this.getForm()
@@ -33,14 +39,31 @@ export abstract class EditDialogComponent<T extends ObjectWithId> implements OnI
     if (this.object != null) {
       this.objectForm.patchValue(this.object)
     }
+
+    // wait to enable close button so it doesnt steal focus from input since its the first clickable element in the DOM
+    setTimeout(() => {
+      this.closeEnabled = true
+    });
+  }
+
+  getCreateTitle() {
+    return $localize`Create new item`
+  }
+
+  getEditTitle() {
+    return $localize`Edit item`
+  }
+
+  getSaveErrorMessage(error: string) {
+    return $localize`Could not save element: ${error}`
   }
 
   getTitle() {
     switch (this.dialogMode) {
       case 'create':
-        return "Create new " + this.entityName
+        return this.getCreateTitle()
       case 'edit':
-        return "Edit " + this.entityName
+        return this.getEditTitle()
       default:
         break;
     }
@@ -50,6 +73,10 @@ export abstract class EditDialogComponent<T extends ObjectWithId> implements OnI
     return MATCHING_ALGORITHMS
   }
 
+  get patternRequired(): boolean {
+    return this.objectForm?.value.matching_algorithm !== MATCH_AUTO
+  }
+
   save() {
     var newObject = Object.assign(Object.assign({}, this.object), this.objectForm.value)
     var serverResponse: Observable<T>
@@ -62,11 +89,13 @@ export abstract class EditDialogComponent<T extends ObjectWithId> implements OnI
       default:
         break;
     }
+    this.networkActive = true
     serverResponse.subscribe(result => {
       this.activeModal.close()
       this.success.emit(result)
     }, error => {
-      this.toastService.showToast(Toast.makeError(`Could not save ${this.entityName}: ${error.error.name}`))
+      this.error = error.error
+      this.networkActive = false
     })
   }
 

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html

@@ -0,0 +1,33 @@
+<div class="btn-group w-100" ngbDropdown role="group" (openChange)="dropdownOpenChange($event)" #dropdown="ngbDropdown">
+  <button class="btn btn-sm" id="dropdown{{title}}" ngbDropdownToggle [ngClass]="!editing && selectionModel.selectionSize() > 0 ? 'btn-primary' : 'btn-outline-primary'">
+    <svg class="toolbaricon" fill="currentColor">
+      <use attr.xlink:href="assets/bootstrap-icons.svg#{{icon}}" />
+    </svg>
+    <div class="d-none d-sm-inline">&nbsp;{{title}}</div>
+    <ng-container *ngIf="!editing && selectionModel.selectionSize() > 0">
+      <div class="badge bg-secondary text-light rounded-pill badge-corner">
+        {{selectionModel.selectionSize()}}
+      </div>
+    </ng-container>
+  </button>
+  <div class="dropdown-menu py-0 shadow" ngbDropdownMenu attr.aria-labelledby="dropdown{{title}}">
+    <div class="list-group list-group-flush">
+      <div class="list-group-item">
+        <div class="input-group input-group-sm">
+          <input class="form-control" type="text" [(ngModel)]="filterText" [placeholder]="filterPlaceholder" (keyup.enter)="listFilterEnter()" #listFilterTextInput>
+        </div>
+      </div>
+      <div *ngIf="selectionModel.items" class="items">
+        <ng-container *ngFor="let item of selectionModel.itemsSorted | filter: filterText">
+          <app-toggleable-dropdown-button *ngIf="allowSelectNone || item.id" [item]="item" [state]="selectionModel.get(item.id)" (toggle)="selectionModel.toggle(item.id)"></app-toggleable-dropdown-button>
+        </ng-container>
+      </div>
+      <button *ngIf="editing" class="list-group-item list-group-item-action bg-light" (click)="applyClicked()" [disabled]="!selectionModel.isDirty()">
+        <small class="ml-1" [ngClass]="{'font-weight-bold': selectionModel.isDirty()}" i18n>Apply</small>
+        <svg width="1.5em" height="1em" viewBox="0 0 16 16" fill="currentColor">
+          <use xlink:href="assets/bootstrap-icons.svg#arrow-right" />
+        </svg>
+      </button>
+    </div>
+  </div>
+</div>

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.spec.ts

@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { FilterableDropodownComponent } from './filterable-dropdown.component';
+
+describe('FilterableDropodownComponent', () => {
+  let component: FilterableDropodownComponent;
+  let fixture: ComponentFixture<FilterableDropodownComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ FilterableDropodownComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(FilterableDropodownComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts

@@ -0,0 +1,272 @@
+import { Component, EventEmitter, Input, Output, ElementRef, ViewChild } from '@angular/core';
+import { FilterPipe } from  'src/app/pipes/filter.pipe';
+import { NgbDropdown } from '@ng-bootstrap/ng-bootstrap'
+import { ToggleableItemState } from './toggleable-dropdown-button/toggleable-dropdown-button.component';
+import { MatchingModel } from 'src/app/data/matching-model';
+import { Subject } from 'rxjs';
+
+export interface ChangedItems {
+  itemsToAdd: MatchingModel[],
+  itemsToRemove: MatchingModel[]
+}
+
+export class FilterableDropdownSelectionModel {
+
+  changed = new Subject<FilterableDropdownSelectionModel>()
+
+  multiple = false
+
+  items: MatchingModel[] = []
+
+  get itemsSorted(): MatchingModel[] {
+    // TODO: this is getting called very often
+    return this.items.sort((a,b) => {
+      if (a.id == null && b.id != null) {
+        return -1
+      } else if (a.id != null && b.id == null) {
+        return 1
+      } else if (this.getNonTemporary(a.id) == ToggleableItemState.NotSelected && this.getNonTemporary(b.id) != ToggleableItemState.NotSelected) {
+        return 1
+      } else if (this.getNonTemporary(a.id) != ToggleableItemState.NotSelected && this.getNonTemporary(b.id) == ToggleableItemState.NotSelected) {
+        return -1
+      } else {
+        return a.name.localeCompare(b.name)
+      }
+    })
+  }
+
+  private selectionStates = new Map<number, ToggleableItemState>()
+
+  private temporarySelectionStates = new Map<number, ToggleableItemState>()
+
+  getSelectedItems() {
+    return this.items.filter(i => this.temporarySelectionStates.get(i.id) == ToggleableItemState.Selected)
+  }
+
+  set(id: number, state: ToggleableItemState, fireEvent = true) {
+    if (state == ToggleableItemState.NotSelected) {
+      this.temporarySelectionStates.delete(id)
+    } else {
+      this.temporarySelectionStates.set(id, state)
+    }
+    if (fireEvent) {
+      this.changed.next(this)
+    }
+  }
+
+  toggle(id: number, fireEvent = true) {
+    let state = this.temporarySelectionStates.get(id)
+    if (state == null || state != ToggleableItemState.Selected) {
+      this.temporarySelectionStates.set(id, ToggleableItemState.Selected)
+    } else if (state == ToggleableItemState.Selected) {
+      this.temporarySelectionStates.delete(id)
+    }
+
+    if (!this.multiple) {
+      for (let key of this.temporarySelectionStates.keys()) {
+        if (key != id) {
+          this.temporarySelectionStates.delete(key)
+        }
+      }
+    }
+
+    if (!id) {
+      for (let key of this.temporarySelectionStates.keys()) {
+        if (key) {
+          this.temporarySelectionStates.delete(key)
+        }
+      }
+    } else {
+      this.temporarySelectionStates.delete(null)
+    }
+
+    if (fireEvent) {
+      this.changed.next(this)
+    }
+    
+  }
+
+  private getNonTemporary(id: number) {
+    return this.selectionStates.get(id) || ToggleableItemState.NotSelected
+  }
+
+  get(id: number) {
+    return this.temporarySelectionStates.get(id) || ToggleableItemState.NotSelected
+  }
+
+  selectionSize() {
+    return this.getSelectedItems().length
+  }
+
+  clear(fireEvent = true) {
+    this.temporarySelectionStates.clear()
+    if (fireEvent) {
+      this.changed.next(this)
+    }
+  }
+
+  isDirty() {
+    if (!Array.from(this.temporarySelectionStates.keys()).every(id => this.temporarySelectionStates.get(id) == this.selectionStates.get(id))) {
+      return true
+    } else if (!Array.from(this.selectionStates.keys()).every(id => this.selectionStates.get(id) == this.temporarySelectionStates.get(id))) {
+      return true
+    } else {
+      return false
+    }
+  }
+
+  isNoneSelected() {
+    return this.selectionSize() == 1 && this.get(null) == ToggleableItemState.Selected
+  }
+
+  init(map) {
+    this.temporarySelectionStates = map
+    this.apply()
+  }
+
+  apply() {
+    this.selectionStates.clear()
+    this.temporarySelectionStates.forEach((value, key) => {
+      this.selectionStates.set(key, value)
+    })
+  }
+
+  reset() {
+    this.temporarySelectionStates.clear()
+    this.selectionStates.forEach((value, key) => {
+      this.temporarySelectionStates.set(key, value)
+    })
+  }
+
+  diff(): ChangedItems {
+    return {
+      itemsToAdd: this.items.filter(item => this.temporarySelectionStates.get(item.id) == ToggleableItemState.Selected && this.selectionStates.get(item.id) != ToggleableItemState.Selected),
+      itemsToRemove: this.items.filter(item => !this.temporarySelectionStates.has(item.id) && this.selectionStates.has(item.id)),
+    }
+  }
+}
+
+@Component({
+  selector: 'app-filterable-dropdown',
+  templateUrl: './filterable-dropdown.component.html',
+  styleUrls: ['./filterable-dropdown.component.scss']
+})
+export class FilterableDropdownComponent {
+
+  @ViewChild('listFilterTextInput') listFilterTextInput: ElementRef
+  @ViewChild('dropdown') dropdown: NgbDropdown
+
+  filterText: string
+
+  @Input()
+  set items(items: MatchingModel[]) {
+    if (items) {
+      this._selectionModel.items = Array.from(items)
+      this._selectionModel.items.unshift({
+        name: $localize`:Filter drop down element to filter for documents with no correspondent/type/tag assigned:Not assigned`,
+        id: null
+      })
+    }
+  }
+
+  get items(): MatchingModel[] {
+    return this._selectionModel.items
+  }
+
+  _selectionModel = new FilterableDropdownSelectionModel()
+
+  @Input()
+  set selectionModel(model: FilterableDropdownSelectionModel) {
+    if (this.selectionModel) {
+      this.selectionModel.changed.complete()
+      model.items = this.selectionModel.items
+      model.multiple = this.selectionModel.multiple
+    }
+    model.changed.subscribe(updatedModel => {
+      this.selectionModelChange.next(updatedModel)
+    })
+    this._selectionModel = model
+  }
+
+  get selectionModel(): FilterableDropdownSelectionModel {
+    return this._selectionModel
+  }
+
+  @Output()
+  selectionModelChange = new EventEmitter<FilterableDropdownSelectionModel>()
+
+  @Input()
+  set multiple(value: boolean) {
+    this.selectionModel.multiple = value
+  }
+
+  get multiple() {
+    return this.selectionModel.multiple
+  }
+
+  @Input()
+  title: string
+
+  @Input()
+  filterPlaceholder: string = ""
+
+  @Input()
+  icon: string
+
+  @Input()
+  allowSelectNone: boolean = false
+
+  @Input()
+  editing = false
+
+  @Input()
+  applyOnClose = false
+
+  @Output()
+  apply = new EventEmitter<ChangedItems>()
+
+  @Output()
+  open = new EventEmitter()
+
+  constructor(private filterPipe: FilterPipe) {
+    this.selectionModel = new FilterableDropdownSelectionModel()
+  }
+
+  applyClicked() {
+    if (this.selectionModel.isDirty()) {
+      this.dropdown.close()
+      if (!this.applyOnClose) {
+        this.apply.emit(this.selectionModel.diff())
+      }
+    }
+  }
+
+  dropdownOpenChange(open: boolean): void {
+    if (open) {
+      setTimeout(() => {
+        this.listFilterTextInput.nativeElement.focus();
+      }, 0)
+      if (this.editing) {
+        this.selectionModel.reset()
+      }
+      this.open.next()
+    } else {
+      this.filterText = ''
+      if (this.applyOnClose && this.selectionModel.isDirty()) {
+        this.apply.emit(this.selectionModel.diff())
+      }
+    }
+  }
+
+  listFilterEnter(): void {
+    let filtered = this.filterPipe.transform(this.items, this.filterText)
+    if (filtered.length == 1) {
+      this.selectionModel.toggle(filtered[0].id)
+      if (this.editing) {
+        this.applyClicked()
+      } else {
+        this.dropdown.close()
+      }
+    }
+  }
+}

src-ui/src/app/components/common/filterable-dropdown/toggleable-dropdown-button/toggleable-dropdown-button.component.html

@@ -0,0 +1,20 @@
+<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-left-0 border-right-0 border-bottom" role="menuitem" (click)="toggleItem()">
+  <div class="selected-icon mr-1">
+    <ng-container *ngIf="isChecked()">
+      <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="currentColor" class="bi bi-check" viewBox="0 0 16 16">
+        <path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
+      </svg>
+    </ng-container>
+    <ng-container *ngIf="isPartiallyChecked()">
+      <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="currentColor" class="bi bi-dash" viewBox="0 0 16 16">
+        <path d="M4 8a.5.5 0 0 1 .5-.5h7a.5.5 0 0 1 0 1h-7A.5.5 0 0 1 4 8z"/>
+      </svg>
+    </ng-container>
+    
+  </div>
+  <div class="mr-1">
+    <app-tag *ngIf="isTag; else displayName" [tag]="item" [clickable]="true" linkTitle="Filter by tag"></app-tag>
+    <ng-template #displayName><small>{{item.name}}</small></ng-template>
+  </div>
+  <div class="badge badge-light rounded-pill ml-auto mr-1">{{item.document_count}}</div>
+</button>

src-ui/src/app/components/common/filterable-dropdown/toggleable-dropdown-button/toggleable-dropdown-button.component.spec.ts

@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ToggleableDropdownButtonComponent } from './toggleable-dropdown-button.component';
+
+describe('ToggleableDropdownButtonComponent', () => {
+  let component: ToggleableDropdownButtonComponent;
+  let fixture: ComponentFixture<ToggleableDropdownButtonComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ ToggleableDropdownButtonComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(ToggleableDropdownButtonComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src-ui/src/app/components/common/filterable-dropdown/toggleable-dropdown-button/toggleable-dropdown-button.component.ts

@@ -0,0 +1,51 @@
+import { Component, EventEmitter, Input, Output, OnInit } from '@angular/core';
+import { MatchingModel } from 'src/app/data/matching-model';
+
+export interface ToggleableItem {
+  item: MatchingModel,
+  state: ToggleableItemState,
+  count: number
+}
+
+export enum ToggleableItemState {
+  NotSelected = 0,
+  Selected = 1,
+  PartiallySelected = 2
+}
+
+@Component({
+  selector: 'app-toggleable-dropdown-button',
+  templateUrl: './toggleable-dropdown-button.component.html',
+  styleUrls: ['./toggleable-dropdown-button.component.scss']
+})
+export class ToggleableDropdownButtonComponent {
+
+  @Input()
+  item: MatchingModel
+
+  @Input()
+  state: ToggleableItemState
+
+  @Input()
+  count: number
+
+  @Output()
+  toggle = new EventEmitter()
+
+  get isTag(): boolean {
+    return 'is_inbox_tag' in this.item
+  }
+
+  toggleItem(): void {
+    this.toggle.emit()
+  }
+
+  isChecked() {
+    return this.state == ToggleableItemState.Selected
+  }
+
+  isPartiallyChecked() {
+    return this.state == ToggleableItemState.PartiallySelected
+  }
+
+}

src-ui/src/app/components/common/input/abstract-input.ts

@@ -1,10 +1,13 @@
-import { Directive, Input, OnInit } from '@angular/core';
+import { Directive, ElementRef, Input, OnInit, ViewChild } from '@angular/core';
 import { ControlValueAccessor } from '@angular/forms';
 import { v4 as uuidv4 } from 'uuid';
 
 @Directive()
 export class AbstractInputComponent<T> implements OnInit, ControlValueAccessor {
 
+  @ViewChild("inputField")
+  inputField: ElementRef
+
   constructor() { }
 
   onChange = (newValue: T) => {};
@@ -24,12 +27,21 @@ export class AbstractInputComponent<T> implements OnInit, ControlValueAccessor {
     this.disabled = isDisabled;
   }
 
+  focus() {
+    if (this.inputField && this.inputField.nativeElement) {
+      this.inputField.nativeElement.focus()
+    }
+  }
+
   @Input()
   title: string
 
   @Input()
   disabled = false;
 
+  @Input()
+  error: string
+
   value: T
 
   ngOnInit(): void {

src-ui/src/app/components/common/input/number/number.component.html

@@ -0,0 +1,14 @@
+<div class="form-group">
+  <label [for]="inputId">{{title}}</label>
+  <div class="input-group" [class.is-invalid]="error">
+    <input type="number" class="form-control" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)" [class.is-invalid]="error">
+    <div class="input-group-append">
+      <button class="btn btn-outline-secondary" type="button" id="button-addon1" (click)="nextAsn()" [disabled]="value">+1</button>
+    </div>
+  </div>
+  <div class="invalid-feedback">
+    {{error}}
+  </div>
+  <small *ngIf="hint" class="form-text text-muted">{{hint}}</small>
+
+</div>

src-ui/src/app/components/common/input/number/number.component.spec.ts

@@ -1,20 +1,20 @@
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { FilterDropdownDateComponent } from './filter-dropdown-date.component';
+import { NumberComponent } from './number.component';
 
-describe('FilterDropdownDateComponent', () => {
-  let component: FilterDropdownDateComponent;
-  let fixture: ComponentFixture<FilterDropdownDateComponent>;
+describe('NumberComponent', () => {
+  let component: NumberComponent;
+  let fixture: ComponentFixture<NumberComponent>;
 
   beforeEach(async () => {
     await TestBed.configureTestingModule({
-      declarations: [ FilterDropdownDateComponent ]
+      declarations: [ NumberComponent ]
     })
     .compileComponents();
   });
 
   beforeEach(() => {
-    fixture = TestBed.createComponent(FilterDropdownDateComponent);
+    fixture = TestBed.createComponent(NumberComponent);
     component = fixture.componentInstance;
     fixture.detectChanges();
   });

src-ui/src/app/components/common/input/number/number.component.ts

@@ -0,0 +1,39 @@
+import { Component, forwardRef } from '@angular/core';
+import { NG_VALUE_ACCESSOR } from '@angular/forms';
+import { FILTER_ASN_ISNULL } from 'src/app/data/filter-rule-type';
+import { DocumentService } from 'src/app/services/rest/document.service';
+import { AbstractInputComponent } from '../abstract-input';
+
+@Component({
+  providers: [{
+    provide: NG_VALUE_ACCESSOR,
+    useExisting: forwardRef(() => NumberComponent),
+    multi: true
+  }],
+  selector: 'app-input-number',
+  templateUrl: './number.component.html',
+  styleUrls: ['./number.component.scss']
+})
+export class NumberComponent extends AbstractInputComponent<number> {
+
+  constructor(private documentService: DocumentService) {
+    super()
+  }
+
+  nextAsn() {
+    if (this.value) {
+      return
+    }
+    this.documentService.listFiltered(1, 1, "archive_serial_number", true, [{rule_type: FILTER_ASN_ISNULL, value: "false"}]).subscribe(
+      results => {
+        if (results.count > 0) {
+          this.value = results.results[0].archive_serial_number + 1
+        } else {
+          this.value = 1
+        }
+        this.onChange(this.value)
+      }
+    )
+  }
+
+}

src-ui/src/app/components/common/input/select/select.component.html

@@ -6,9 +6,11 @@
       [style.color]="textColor"
       [style.background]="backgroundColor"
       [clearable]="allowNull"
+      [items]="items"
+      bindLabel="name"
+      bindValue="id"
       (change)="onChange(value)"
       (blur)="onTouched()">
-      <ng-option *ngFor="let i of items" [value]="i.id">{{i.name}}</ng-option>
     </ng-select>
 
     <div *ngIf="showPlusButton()" class="input-group-append">

src-ui/src/app/components/common/input/tags/tags.component.html

@@ -1,11 +1,13 @@
 <div class="form-group paperless-input-select paperless-input-tags">
-  <label for="tags">Tags</label>
+  <label for="tags" i18n>Tags</label>
 
   <div class="input-group flex-nowrap">
     <ng-select name="tags" [items]="tags" bindLabel="name" bindValue="id" [(ngModel)]="displayValue"
       [multiple]="true"
       [closeOnSelect]="false"
+      [clearSearchOnAdd]="true"
       [disabled]="disabled"
+      [hideSelected]="true"
       (change)="ngSelectChange()">
 
       <ng-template ng-label-tmp let-item="item">

src-ui/src/app/components/common/input/text/text.component.html

@@ -1,5 +1,8 @@
 <div class="form-group">
   <label [for]="inputId">{{title}}</label>
-  <input type="text" class="form-control" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)">
+  <input #inputField type="text" class="form-control" [class.is-invalid]="error" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)">
   <small *ngIf="hint" class="form-text text-muted">{{hint}}</small>
+  <div class="invalid-feedback">
+    {{error}}
+  </div>
 </div>

src-ui/src/app/components/common/page-header/page-header.component.html

@@ -1,9 +1,9 @@
-<div class="row pt-3 pb-1 mb-3 border-bottom align-items-center" >
+<div class="row pt-3 pb-3 pb-md-1 mb-3 border-bottom align-items-center">
   <div class="col-md text-truncate">
     <p class="h2 text-truncate" style="line-height: 1.4">{{title}}</p>
     <p *ngIf="subTitle" class="h5 text-truncate" style="line-height: 1.4">{{subTitle}}</p>
   </div>
-  <div class="btn-toolbar col-auto">
+  <div class="btn-toolbar col col-md-auto">
     <ng-content></ng-content>
   </div>
 </div>

src-ui/src/app/components/common/select-dialog/select-dialog.component.html

@@ -10,6 +10,6 @@
 
 </div>
 <div class="modal-footer">
-  <button type="button" class="btn btn-outline-dark" (click)="cancelClicked()">Cancel</button>
-  <button type="button" class="btn btn-primary" (click)="selectClicked.emit(selected)">Select</button>
+  <button type="button" class="btn btn-outline-dark" (click)="cancelClicked()" i18n>Cancel</button>
+  <button type="button" class="btn btn-primary" (click)="selectClicked.emit(selected)" i18n>Select</button>
 </div>

src-ui/src/app/components/common/select-dialog/select-dialog.component.ts

@@ -15,10 +15,10 @@ export class SelectDialogComponent implements OnInit {
   public selectClicked = new EventEmitter()
 
   @Input()
-  title = "Select"
+  title = $localize`Select`
 
   @Input()
-  message = "Please select an object"
+  message = $localize`Please select an object`
 
   @Input()
   objects: ObjectWithId[] = []

src-ui/src/app/components/dashboard/dashboard.component.html

@@ -1,5 +1,8 @@
-<app-page-header title="Dashboard" [subTitle]="subtitle">
-  <img src="assets/logo.svg" height="80" class="m-2 d-none d-md-block">
+<app-page-header title="Dashboard" [subTitle]="subtitle" i18n-title>
+  <svg class="logo d-none d-md-block" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 2897.4 896.64" height="5rem">
+    <path class="leaf" d="M730.71,0C616.3,266.24,66.21,299.25,242.24,728.31c2.2,5.5-40.71,63.81-69.31,112.22-6.6-24.21-14.3-50.61-13.2-52.81C302.75,618.29,57.41,522.58,44.21,384-17.4,494-39.4,686.5,146.53,797.62c1.1,0,9.9,41.81,14.3,62.71-4.4,8.8-8.8,17.6-11,24.2-4.4,11,28.6,9.91,28.6,12.11,3.3-1.1,81.41-138.63,83.61-139.73C704.31,655.7,782.42,257.44,730.71,0ZM105.82,544.58C145.43,580.89,211.44,692,155.33,756.91,168.53,727.21,172.93,660.1,105.82,544.58ZM269.74,670C200.43,506.08,404,325.65,505.18,280.54,298.35,465.37,263.14,602.89,269.74,670Z" transform="translate(0)" style="fill:#17541f"/>
+    <path class="text" style="fill:#000" d="M1169.27,231.44q.7,17.06.71,38.73t-.71,38a112,112,0,0,1-6.76,38A100.16,100.16,0,0,1,1142.25,380a93.87,93.87,0,0,1-33.76,23.46q-20.25,8.54-46.56,8.53h-36.25V597.5q0,9.94,2.85,14.57c1.89,3.08,3.79,4.86,5.68,5.33a10.54,10.54,0,0,0,9.24,0v18.48h-86V617.4q5.68,2.13,8.53,0,3.56-.71,7.1-5.33t3.56-14.57V166q0-10.66-3.56-15.28t-7.1-6q-2.85-1.42-8.53,0V127.66h104.48q26.31,0,46.56,8.53a93.75,93.75,0,0,1,33.76,23.46,101,101,0,0,1,20.26,33.4A111.05,111.05,0,0,1,1169.27,231.44ZM1118.8,215.8q0-18.48-6-32t-14.22-21.32a74.68,74.68,0,0,0-17.06-12.44q-8.88-4.62-15.28-4.62h-40.52V392.79h40.52q6.39,0,15.28-3.56t17.06-12.44a86,86,0,0,0,14.22-21.67q6-12.8,6-32Zm337.63,389.52a27.35,27.35,0,0,1-2.85,11.72,40.58,40.58,0,0,1-8.17,11.38,54.8,54.8,0,0,1-13.5,9.59,40.07,40.07,0,0,1-18.84,4.27,60,60,0,0,1-30.57-7.82q-13.5-7.82-18.48-20.61a56.12,56.12,0,0,1-6.39-16.35q-9.24,19.18-28.08,32a72.42,72.42,0,0,1-41.58,12.8q-28.44,0-49.76-20.61-12.07-10.67-19.9-27.73t-9.24-39.09q-.71-2.13-.71-8.53v-.71a124.43,124.43,0,0,1,4.26-25.23,113.22,113.22,0,0,1,11-26.3,149.46,149.46,0,0,1,17.77-24.88,128.6,128.6,0,0,1,22.39-20.26,17.39,17.39,0,0,0,4.62-3.2,28.53,28.53,0,0,1,5.33-3.91l2.13-1.06,2.14-1.07q12.8-7.81,22.39-14.22t17.41-12.08a97.25,97.25,0,0,0,8.53-6.4c2.84-2.37,5.93-4.73,9.24-7.1a79.43,79.43,0,0,0,12.09-10,36.51,36.51,0,0,0,8.53-13.5q4.26-7.11,4.26-17.06a67.74,67.74,0,0,0-2.13-17.77,49.09,49.09,0,0,0-14.57-27q-11.73-11.37-26-14.93a38.26,38.26,0,0,0-12.08-2.13q-19.9,0-34.12,7.11-9.24,4.26-13.51,9.59t-4.62,10a10.73,10.73,0,0,0,1.78,7.46,12.91,12.91,0,0,0,5.69,4.26,6,6,0,0,0,1.78.71,6.28,6.28,0,0,1,1.77.72,32.21,32.21,0,0,1,15.28,12.43,34,34,0,0,1,6,19.55,32.78,32.78,0,0,1-32.7,32.7,33.53,33.53,0,0,1-24.17-9.6,30.94,30.94,0,0,1-10-23.1,106.83,106.83,0,0,1,3.2-25.23,84.65,84.65,0,0,1,9.6-23.81q13.5-19.91,32.34-28.79a96.24,96.24,0,0,1,41.58-8.89,93.82,93.82,0,0,1,33.76,6.4,96.18,96.18,0,0,1,29.86,18.13,103.38,103.38,0,0,1,22,28.07q8.88,16.35,8.88,37.68v209q0,12.79.36,16.7t.36,5.34h0q0,9.24,5.68,14.92a18.45,18.45,0,0,0,13.51,5.69q9.24,0,14.92-5.33a20.62,20.62,0,0,0,6.4-13.86Zm-111.6-17.77a52.58,52.58,0,0,0,6.76-21.33q1.07-11.37,1.06-21.32V413.4a63.92,63.92,0,0,1-6,5.33,74,74,0,0,0-6.75,6q-20.62,14.93-35.19,27.37a164.06,164.06,0,0,0-24.52,25.59,113,113,0,0,0-15.64,28.07,121,121,0,0,0-7.11,34.12v10.67q.72,27,14.58,42.29t39.44,15.28a38.56,38.56,0,0,0,20.62-5.69q9.24-6.39,12.79-14.92Zm244.52-304.23q40.51.72,64.68,24.53t24.17,75.7V542.77q0,51.88-24.17,75.34T1589.35,643h-2.13a80.27,80.27,0,0,1-22-2.84h-.71a.63.63,0,0,1-.71-.71,91.74,91.74,0,0,1-28.43-15.64l-6.76-6.76-5.33-7.46v118q0,10,2.85,14.58c1.89,3.08,3.79,5.09,5.68,6a10.54,10.54,0,0,0,9.24,0V766h-86V748.19q5.68,2.13,8.53,0,3.56-1.43,6.75-6t3.2-14.58V326.68q0-9.94-3.2-14.57t-6.75-6q-2.85-1.43-8.53,0V289h68.23v27.72a68.68,68.68,0,0,1,12.09-14.21,75.62,75.62,0,0,1,28.43-15.64.63.63,0,0,1,.71-.71h.71a90.55,90.55,0,0,1,22-2.85Zm35.54,84.59v-.71q-.72-36.26-8.89-51.18t-26.65-16.35a63.12,63.12,0,0,0-24.17,3.2,48,48,0,0,0-19.9,13.15q-8.54,9.24-14.22,24.88t-7.82,37V548.45q2.13,22,7.82,37.32t14.22,24.17a53.23,53.23,0,0,0,19.9,13.15,58.92,58.92,0,0,0,24.17,3.55q18.48-1.42,26.65-16.35t8.89-51.18V367.91Zm179.83-84.59a105.36,105.36,0,0,1,33.41,5.69,86.38,86.38,0,0,1,30.56,17.77,88.06,88.06,0,0,1,21.68,30.57q8.18,18.48,8.18,44.06v90.28H1757.1V544.9q0,44.07,15.28,62.9t38,18.13a113.94,113.94,0,0,0,26.65-4.62,54.83,54.83,0,0,0,22.75-13.15q9.59-9.24,16-26.3t6.4-44.78h16.35V547a153.56,153.56,0,0,1-4.27,36.61A72,72,0,0,1,1852,634.1q-16.71,7.47-40.87,8.18a102.25,102.25,0,0,1-34.83-6.4,96,96,0,0,1-32-19.19,108.91,108.91,0,0,1-23.45-30.21q-9.24-17.41-9.25-39.45v-16.7s-.11-9.12-.35-27.37-.36-29.5-.36-33.76-.11-16-.35-35.19-.36-29.49-.36-30.92v-21q0-25.59,8.89-44.43t22-30.56A90.72,90.72,0,0,1,1771,289.37a98.55,98.55,0,0,1,33.76-6.05Zm46.2,169.89V397.05q0-27-3.55-45.13t-9.6-29.15q-6-11-14.57-15.28a42.36,42.36,0,0,0-19.19-4.26q-22.76,0-34.48,19.9t-11.72,73.92v56.16Zm279.35-113.73a42.31,42.31,0,0,0,.71,5.68,17.92,17.92,0,0,1,0,7.11q-.72,14.93-10.31,24.52a33,33,0,0,1-46.91-.35,33.52,33.52,0,0,1-9.6-24.17q0-17.76,13.51-27a8.33,8.33,0,0,0,1.77-1.06l1.07-1.07a16.22,16.22,0,0,0,6.4-10.31q1.41-6.75-8.53-11.73a26.26,26.26,0,0,0-12.8-3.55q-21.32-1.43-37.67,12.8t-22,49.75V597.5q0,9.94,2.84,14.57t5.69,5.33a10.54,10.54,0,0,0,9.24,0v18.48h-86V617.4q5.68,2.13,8.53,0,3.55-.71,7.11-5.33t3.55-14.57V326.68q0-9.94-3.55-14.57t-7.11-6q-2.85-1.43-8.53,0V288.3h68.24v29.14q5-7.81,9.95-14.57a48.44,48.44,0,0,1,11.72-11.37,59.79,59.79,0,0,1,15.64-7.46q8.89-2.85,22.39-2.85,20.61,0,38.39,14.93t26.3,43.36Zm95.95,258q0,9.94,3.2,14.57c2.13,3.08,4.15,4.86,6,5.33q3.55,2.13,9.24,0v18.48h-86V617.4q5.68,2.13,10,0,2.13-.71,5.69-5.33t3.55-14.57V166.75q0-10.65-3.55-15.28a28.82,28.82,0,0,0-5.69-6q-4.26-2.13-10,0V128.37h67.52V597.5Zm154.25-314.18a105.25,105.25,0,0,1,33.4,5.69,86.33,86.33,0,0,1,30.57,17.77,88.19,88.19,0,0,1,21.68,30.57q8.18,18.48,8.17,44.06v90.28H2332.84V544.9q0,44.07,15.29,62.9t38,18.13a113.93,113.93,0,0,0,Line truncated
+  </svg>
 </app-page-header>
 
 <div class='row'>

src-ui/src/app/components/dashboard/dashboard.component.ts

@@ -30,9 +30,9 @@ export class DashboardComponent implements OnInit {
 
   get subtitle() {
     if (this.displayName) {
-      return `Hello ${this.displayName}, welcome to Paperless-ng!`
+      return $localize`Hello ${this.displayName}, welcome to Paperless-ng!`
     } else {
-      return `Welcome to Paperless-ng!`
+      return $localize`Welcome to Paperless-ng!`
     }
   }
 

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html

@@ -1,18 +1,18 @@
 <app-widget-frame [title]="savedView.name">
 
-  <a header-buttons [routerLink]="" (click)="showAll()">Show all</a>
+  <a header-buttons [routerLink]="" (click)="showAll()" i18n>Show all</a>
 
 
   <table content class="table table-sm table-hover table-borderless">
     <thead>
       <tr>
-        <th>Created</th>
-        <th scope="col">Title</th>
+        <th i18n>Created</th>
+        <th scope="col" i18n>Title</th>
       </tr>
     </thead>
     <tbody>
       <tr *ngFor="let doc of documents" routerLink="/documents/{{doc.id}}">
-        <td>{{doc.created | date}}</td>
+        <td>{{doc.created | customDate}}</td>
         <td>{{doc.title | documentTitle}}<app-tag [tag]="t" *ngFor="let t of doc.tags$ | async" class="ml-1"></app-tag>
       </tr>
     </tbody>

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.scss

@@ -0,0 +1,8 @@
+table {
+  overflow-wrap: anywhere;
+  table-layout: fixed;
+}
+
+th:first-child {
+  width: 25%;
+}

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts

@@ -16,7 +16,7 @@ export class SavedViewWidgetComponent implements OnInit {
     private documentService: DocumentService,
     private router: Router,
     private list: DocumentListViewService) { }
-  
+
   @Input()
   savedView: PaperlessSavedView
 
@@ -34,7 +34,7 @@ export class SavedViewWidgetComponent implements OnInit {
     } else {
       this.list.load(this.savedView)
       this.router.navigate(["documents"])
-      }
+    }
   }
 
 }

src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html

@@ -1,6 +1,6 @@
-<app-widget-frame title="Statistics">
+<app-widget-frame title="Statistics" i18n-title>
   <ng-container content>
-    <p class="card-text">Documents in inbox: {{statistics.documents_inbox}}</p>
-    <p class="card-text">Total documents: {{statistics.documents_total}}</p>
+    <p class="card-text" i18n>Documents in inbox: {{statistics.documents_inbox}}</p>
+    <p class="card-text" i18n>Total documents: {{statistics.documents_total}}</p>
   </ng-container>
 </app-widget-frame>

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html

@@ -1,16 +1,16 @@
-<app-widget-frame title="Upload new documents">
+<app-widget-frame title="Upload new documents" i18n-title>
 
   <div content>
     <form>
-      <ngx-file-drop dropZoneLabel="Drop documents here or" (onFileDrop)="dropped($event)"
+      <ngx-file-drop dropZoneLabel="Drop documents here or" browseBtnLabel="Browse files" (onFileDrop)="dropped($event)"
         (onFileOver)="fileOver($event)" (onFileLeave)="fileLeave($event)" dropZoneClassName="bg-light card"
         multiple="true" contentClassName="justify-content-center d-flex align-items-center p-5" [showBrowseBtn]=true
-        browseBtnClassName="btn btn-sm btn-outline-primary ml-2">
+        browseBtnClassName="btn btn-sm btn-outline-primary ml-2" i18n-dropZoneLabel i18n-browseBtnLabel>
 
       </ngx-file-drop>
     </form>
     <div *ngIf="uploadVisible" class="mt-3">
-      <p>Uploading {{uploadStatus.length}} file(s)</p>
+      <p i18n>{uploadStatus.length, plural, =1 {Uploading file...} =other {Uploading {{uploadStatus.length}} files...}}</p>
       <ngb-progressbar [value]="loadedSum" [max]="totalSum" [striped]="true" [animated]="uploadStatus.length > 0">
       </ngb-progressbar>
     </div>

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts

@@ -2,7 +2,7 @@ import { HttpEventType } from '@angular/common/http';
 import { Component, OnInit } from '@angular/core';
 import { FileSystemFileEntry, NgxFileDropEntry } from 'ngx-file-drop';
 import { DocumentService } from 'src/app/services/rest/document.service';
-import { Toast, ToastService } from 'src/app/services/toast.service';
+import { ToastService } from 'src/app/services/toast.service';
 
 
 interface UploadStatus {
@@ -60,7 +60,7 @@ export class UploadFileWidgetComponent implements OnInit {
             } else if (event.type == HttpEventType.Response) {
               this.uploadStatus.splice(this.uploadStatus.indexOf(uploadStatusObject), 1)
               this.completedFiles += 1
-              this.toastService.showToast(Toast.make("Information", "The document has been uploaded and will be processed by the consumer shortly."))
+              this.toastService.showInfo($localize`The document has been uploaded and will be processed by the consumer shortly.`)
             }
             
           }, error => {
@@ -68,11 +68,11 @@ export class UploadFileWidgetComponent implements OnInit {
             this.completedFiles += 1
             switch (error.status) {
               case 400: {
-                this.toastService.showToast(Toast.makeError(`There was an error while uploading the document: ${error.error.document}`))
+                this.toastService.showInfo($localize`There was an error while uploading the document: ${error.error.document}`)
                 break;
               }
               default: {
-                this.toastService.showToast(Toast.makeError("An error has occurred while uploading the document. Sorry!"))
+                this.toastService.showInfo($localize`An error has occurred while uploading the document. Sorry!`)
                 break;
               }
             }

src-ui/src/app/components/dashboard/widgets/welcome-widget/welcome-widget.component.html

@@ -1,16 +1,16 @@
-<app-widget-frame title="First steps">
+<app-widget-frame title="First steps" i18n-title>
 
   <ng-container content>
     <img src="assets/save-filter.png" class="float-right">
-    <p>Paperless is running! :)</p>
-    <p>You can start uploading documents by dropping them in the file upload box to the right or by dropping them in the configured consumption folder and they'll start showing up in the documents list.
-      After you've added some metadata to your documents, use the filtering mechanisms of paperless to create custom views (such as 'Recently added', 'Tagged TODO') and have them displayed on the dashboard instead of this message.</p>
-    <p>Paperless offers some more features that try to make your life easier, such as:</p>
+    <p i18n>Paperless is running! :)</p>
+    <p i18n>You can start uploading documents by dropping them in the file upload box to the right or by dropping them in the configured consumption folder and they'll start showing up in the documents list.
+      After you've added some metadata to your documents, use the filtering mechanisms of paperless to create custom views (such as 'Recently added', 'Tagged TODO') and they will appear on the dashboard instead of this message.</p>
+    <p i18n>Paperless offers some more features that try to make your life easier:</p>
     <ul>
-      <li>Once you've got a couple documents in paperless and added metadata to them, paperless can assign that metadata to new documents automatically.</li>
-      <li>You can configure paperless to read your mails and add documents from attached files.</li>
+      <li i18n>Once you've got a couple documents in paperless and added metadata to them, paperless can assign that metadata to new documents automatically.</li>
+      <li i18n>You can configure paperless to read your mails and add documents from attached files.</li>
     </ul>
-    <p>Consult the documentation on how to use these features. The section on basic usage also has some information on how to use paperless in general.</p>
+    <p i18n>Consult the documentation on how to use these features. The section on basic usage also has some information on how to use paperless in general.</p>
   </ng-container>
 
 </app-widget-frame>

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -1,19 +1,18 @@
 <app-page-header [(title)]="title">
-    <div class="input-group input-group-sm mr-5" *ngIf="getContentType() == 'application/pdf'">
+    <div class="input-group input-group-sm mr-5 d-none d-md-flex" *ngIf="getContentType() == 'application/pdf' && !useNativePdfViewer">
       <div class="input-group-prepend">
-        <div class="input-group-text">Page </div>
+        <div class="input-group-text" i18n>Page</div>
       </div>
       <input class="form-control flex-grow-0 w-auto" type="number" min="1" [max]="previewNumPages" [(ngModel)]="previewCurrentPage" />
       <div class="input-group-append">
-        <div class="input-group-text">of {{previewNumPages}}</div>
+        <div class="input-group-text" i18n>of {{previewNumPages}}</div>
       </div>
     </div>
 
-    <button type="button" class="btn btn-sm btn-outline-danger mr-2" (click)="delete()">
+    <button type="button" class="btn btn-sm btn-outline-danger mr-2 ml-auto" (click)="delete()">
         <svg class="buttonicon" fill="currentColor">
             <use xlink:href="assets/bootstrap-icons.svg#trash" />
-        </svg>
-        <span class="d-none d-lg-inline"> Delete</span>
+        </svg>&nbsp;<span class="d-none d-lg-inline" i18n>Delete</span>
     </button>
 
     <div class="btn-group mr-2">
@@ -21,14 +20,13 @@
         <a [href]="downloadUrl" class="btn btn-sm btn-outline-primary">
             <svg class="buttonicon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#download" />
-            </svg>
-            <span class="d-none d-lg-inline"> Download</span>
+            </svg>&nbsp;<span class="d-none d-lg-inline" i18n>Download</span>
         </a>
 
         <div class="btn-group" ngbDropdown role="group" *ngIf="metadata?.has_archive_version">
             <button class="btn btn-sm btn-outline-primary dropdown-toggle-split" ngbDropdownToggle></button>
             <div class="dropdown-menu shadow" ngbDropdownMenu>
-                <a ngbDropdownItem [href]="downloadOriginalUrl">Download original</a>
+                <a ngbDropdownItem [href]="downloadOriginalUrl" i18n>Download original</a>
             </div>
         </div>
 
@@ -37,15 +35,13 @@
     <button type="button" class="btn btn-sm btn-outline-primary mr-2" (click)="moreLike()">
         <svg class="buttonicon" fill="currentColor">
             <use xlink:href="assets/bootstrap-icons.svg#three-dots" />
-        </svg>
-        <span class="d-none d-lg-inline"> More like this</span>
+        </svg>&nbsp;<span class="d-none d-lg-inline" i18n>More like this</span>
     </button>
 
     <button type="button" class="btn btn-sm btn-outline-primary" (click)="close()">
         <svg class="buttonicon" fill="currentColor">
             <use xlink:href="assets/bootstrap-icons.svg#x" />
-        </svg>
-        <span class="d-none d-lg-inline"> Close</span>
+        </svg>&nbsp;<span class="d-none d-lg-inline" i18n>Close</span>
     </button>
 </app-page-header>
 
@@ -57,27 +53,23 @@
 
             <ul ngbNav #nav="ngbNav" class="nav-tabs">
                 <li [ngbNavItem]="1">
-                    <a ngbNavLink>Details</a>
+                    <a ngbNavLink i18n>Details</a>
                     <ng-template ngbNavContent>
 
-                        <app-input-text title="Title" formControlName="title"></app-input-text>
-                        <div class="form-group">
-                            <label for="archive_serial_number">Archive Serial Number</label>
-                            <input type="number" class="form-control" id="archive_serial_number"
-                                formControlName='archive_serial_number'>
-                        </div>
-                        <app-input-date-time titleDate="Date created" formControlName="created"></app-input-date-time>
-                        <app-input-select [items]="correspondents" title="Correspondent" formControlName="correspondent" [allowNull]="true"
+                        <app-input-text #inputTitle i18n-title title="Title" formControlName="title" [error]="error?.title"></app-input-text>
+                        <app-input-number i18n-title title="Archive serial number" [error]="error?.archive_serial_number" formControlName='archive_serial_number'></app-input-number>
+                        <app-input-date-time i18n-titleDate titleDate="Date created" formControlName="created"></app-input-date-time>
+                        <app-input-select [items]="correspondents" i18n-title title="Correspondent" formControlName="correspondent" [allowNull]="true"
                             (createNew)="createCorrespondent()"></app-input-select>
-                        <app-input-select [items]="documentTypes" title="Document type" formControlName="document_type" [allowNull]="true"
+                        <app-input-select [items]="documentTypes" i18n-title title="Document type" formControlName="document_type" [allowNull]="true"
                             (createNew)="createDocumentType()"></app-input-select>
-                        <app-input-tags formControlName="tags" title="Tags"></app-input-tags>
+                        <app-input-tags formControlName="tags"></app-input-tags>
 
                     </ng-template>
                 </li>
 
                 <li [ngbNavItem]="2">
-                    <a ngbNavLink>Content</a>
+                    <a ngbNavLink i18n>Content</a>
                     <ng-template ngbNavContent>
                         <div class="form-group">
                             <textarea class="form-control" id="content" rows="20" formControlName='content'></textarea>
@@ -86,48 +78,48 @@
                 </li>
 
                 <li [ngbNavItem]="3">
-                    <a ngbNavLink>Metadata</a>
+                    <a ngbNavLink i18n>Metadata</a>
                     <ng-template ngbNavContent>
 
                         <table class="table table-borderless">
                             <tbody>
                                 <tr>
-                                    <td>Date modified</td>
-                                    <td>{{document.modified | date:'medium'}}</td>
+                                    <td i18n>Date modified</td>
+                                    <td>{{document.modified | customDate}}</td>
                                 </tr>
                                 <tr>
-                                    <td>Date added</td>
-                                    <td>{{document.added | date:'medium'}}</td>
+                                    <td i18n>Date added</td>
+                                    <td>{{document.added | customDate}}</td>
                                 </tr>
                                 <tr>
-                                    <td>Media filename</td>
+                                    <td i18n>Media filename</td>
                                     <td>{{metadata?.media_filename}}</td>
                                 </tr>
                                 <tr>
-                                    <td>Original MD5 Checksum</td>
+                                    <td i18n>Original MD5 checksum</td>
                                     <td>{{metadata?.original_checksum}}</td>
                                 </tr>
                                 <tr>
-                                    <td>Original file size</td>
+                                    <td i18n>Original file size</td>
                                     <td>{{metadata?.original_size | fileSize}}</td>
                                 </tr>
                                 <tr>
-                                    <td>Original mime type</td>
+                                    <td i18n>Original mime type</td>
                                     <td>{{metadata?.original_mime_type}}</td>
                                 </tr>
                                 <tr *ngIf="metadata?.has_archive_version">
-                                    <td>Archive MD5 Checksum</td>
+                                    <td i18n>Archive MD5 checksum</td>
                                     <td>{{metadata?.archive_checksum}}</td>
                                 </tr>
                                 <tr *ngIf="metadata?.has_archive_version">
-                                    <td>Archive file size</td>
+                                    <td i18n>Archive file size</td>
                                     <td>{{metadata?.archive_size | fileSize}}</td>
                                 </tr>
                             </tbody>
                         </table>
 
-                        <app-metadata-collapse title="Original document metadata" [metadata]="metadata.original_metadata" *ngIf="metadata?.original_metadata?.length > 0"></app-metadata-collapse>
-                        <app-metadata-collapse title="Archived document metadata" [metadata]="metadata.archive_metadata" *ngIf="metadata?.archive_metadata?.length > 0"></app-metadata-collapse>
+                        <app-metadata-collapse i18n-title title="Original document metadata" [metadata]="metadata.original_metadata" *ngIf="metadata?.original_metadata?.length > 0"></app-metadata-collapse>
+                        <app-metadata-collapse i18n-title title="Archived document metadata" [metadata]="metadata.archive_metadata" *ngIf="metadata?.archive_metadata?.length > 0"></app-metadata-collapse>
 
                     </ng-template>
                 </li>
@@ -135,16 +127,24 @@
 
             <div [ngbNavOutlet]="nav" class="mt-2"></div>
 
-            <button type="button" class="btn btn-outline-secondary" (click)="discard()">Discard</button>&nbsp;
-            <button type="button" class="btn btn-outline-primary" (click)="saveEditNext()" *ngIf="hasNext()">Save & edit
-                next</button>&nbsp;
-            <button type="submit" class="btn btn-primary">Save</button>&nbsp;
+            <button type="button" class="btn btn-outline-secondary" (click)="discard()" i18n [disabled]="networkActive">Discard</button>&nbsp;
+            <button type="button" class="btn btn-outline-primary" (click)="saveEditNext()" *ngIf="hasNext()" i18n [disabled]="networkActive">Save & next</button>&nbsp;
+            <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive">Save</button>&nbsp;
         </form>
     </div>
 
     <div class="col-md-6 col-xl-8 mb-3">
-      <div class="pdf-viewer-container" *ngIf="getContentType() == 'application/pdf'">
-        <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true" [show-all]="true" [(page)]="previewCurrentPage" (after-load-complete)="pdfPreviewLoaded($event)"></pdf-viewer>
-      </div>
+        <ng-container *ngIf="getContentType() == 'application/pdf'">
+            <div class="preview-sticky pdf-viewer-container" *ngIf="!useNativePdfViewer ; else nativePdfViewer">
+                <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true" [show-all]="true" [(page)]="previewCurrentPage" [render-text-mode]="2" (after-load-complete)="pdfPreviewLoaded($event)"></pdf-viewer>
+            </div>
+            <ng-template #nativePdfViewer>
+                <object [data]="previewUrl | safe" type="application/pdf" class="preview-sticky" width="100%"></object>
+            </ng-template>
+        </ng-container>
+        <ng-container *ngIf="getContentType() == 'text/plain'">
+            <object [data]="previewUrl | safe" type="text/plain" class="preview-sticky" width="100%"></object>
+        </ng-container>
+          
     </div>
 </div>

src-ui/src/app/components/document-detail/document-detail.component.scss

@@ -1,6 +1,9 @@
-.pdf-viewer-container {
+.preview-sticky {
   height: calc(100vh - 160px);
   top: 70px;
   position: sticky;
+}
+
+.pdf-viewer-container {
   background-color: gray;
 }

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit } from '@angular/core';
+import { Component, OnInit, ViewChild } from '@angular/core';
 import { FormControl, FormGroup } from '@angular/forms';
 import { ActivatedRoute, Router } from '@angular/router';
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
@@ -16,6 +16,9 @@ import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.
 import { CorrespondentEditDialogComponent } from '../manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component';
 import { DocumentTypeEditDialogComponent } from '../manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component';
 import { PDFDocumentProxy } from 'ng2-pdf-viewer';
+import { ToastService } from 'src/app/services/toast.service';
+import { TextComponent } from '../common/input/text/text.component';
+import { SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
 
 @Component({
   selector: 'app-document-detail',
@@ -24,8 +27,15 @@ import { PDFDocumentProxy } from 'ng2-pdf-viewer';
 })
 export class DocumentDetailComponent implements OnInit {
 
-  public expandOriginalMetadata = false;
-  public expandArchivedMetadata = false;
+  @ViewChild("inputTitle")
+  titleInput: TextComponent
+
+  expandOriginalMetadata = false
+  expandArchivedMetadata = false
+
+  error: any
+
+  networkActive = false
 
   documentId: number
   document: PaperlessDocument
@@ -60,7 +70,13 @@ export class DocumentDetailComponent implements OnInit {
     private modalService: NgbModal,
     private openDocumentService: OpenDocumentsService,
     private documentListViewService: DocumentListViewService,
-    private documentTitlePipe: DocumentTitlePipe) { }
+    private documentTitlePipe: DocumentTitlePipe,
+    private toastService: ToastService,
+    private settings: SettingsService) { }
+
+  get useNativePdfViewer(): boolean {
+    return this.settings.get(SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER)
+  }
 
   getContentType() {
     return this.metadata?.has_archive_version ? 'application/pdf' : this.metadata?.original_mime_type
@@ -131,19 +147,34 @@ export class DocumentDetailComponent implements OnInit {
   }
 
   save() {
+    this.networkActive = true
     this.documentsService.update(this.document).subscribe(result => {
       this.close()
+      this.networkActive = false
+      this.error = null
+    }, error => {
+      this.networkActive = false
+      this.error = error.error
     })
   }
 
   saveEditNext() {
+    this.networkActive = true
     this.documentsService.update(this.document).subscribe(result => {
+      this.error = null
       this.documentListViewService.getNext(this.document.id).subscribe(nextDocId => {
+        this.networkActive = false
         if (nextDocId) {
           this.openDocumentService.closeDocument(this.document)
           this.router.navigate(['documents', nextDocId])
+          this.titleInput.focus()
         }
+      }, error => {
+        this.networkActive = false
       })
+    }, error => {
+      this.networkActive = false
+      this.error = error.error
     })
   }
 
@@ -158,15 +189,19 @@ export class DocumentDetailComponent implements OnInit {
 
   delete() {
     let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'})
-    modal.componentInstance.title = "Confirm delete"
-    modal.componentInstance.messageBold = `Do you really want to delete document '${this.document.title}'?`
-    modal.componentInstance.message = `The files for this document will be deleted permanently. This operation cannot be undone.`
+    modal.componentInstance.title = $localize`Confirm delete`
+    modal.componentInstance.messageBold = $localize`Do you really want to delete document "${this.document.title}"?`
+    modal.componentInstance.message = $localize`The files for this document will be deleted permanently. This operation cannot be undone.`
     modal.componentInstance.btnClass = "btn-danger"
-    modal.componentInstance.btnCaption = "Delete document"
+    modal.componentInstance.btnCaption = $localize`Delete document`
     modal.componentInstance.confirmClicked.subscribe(() => {
+      modal.componentInstance.buttonsEnabled = false
       this.documentsService.delete(this.document).subscribe(() => {
         modal.close()
         this.close()
+      }, error => {
+        this.toastService.showError($localize`Error deleting document: ${JSON.stringify(error)}`)
+        modal.componentInstance.buttonsEnabled = true
       })
     })