nodes/home.paperless: introduce

PORT_MAP.md

@@ -43,6 +43,7 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports.
 | 22040       | miniflux             | Miniflux Web Interface |
 | 22050       | radicale             | radicale carddav and caldav server |
 | 22060       | pretalx              | gunicorn |
+| 22070       | paperless-ng         | gunicorn |
 | 22999       | nginx                | stub_status |
 
 ## UDP

bundles/nginx/metadata.py

@@ -71,11 +71,12 @@ def letsencrypt(metadata):
     vhosts = {}
 
     for vhost, config in metadata.get('nginx/vhosts', {}).items():
-        domain = config.get('domain', vhost)
+        if config.get('ssl', 'letsencrypt') == 'letsencrypt':
-        domains[domain] = config.get('domain_aliases', set())
+            domain = config.get('domain', vhost)
-        vhosts[vhost] = {
+            domains[domain] = config.get('domain_aliases', set())
-            'ssl': 'letsencrypt',
+            vhosts[vhost] = {
-        }
+                'ssl': 'letsencrypt',
+            }
 
     return {
         'letsencrypt': {

bundles/paperless-ng/files/paperless-consumer.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Paperless consumer
+Requires=redis.service
+
+[Service]
+User=paperless
+Group=paperless
+WorkingDirectory=/opt/paperless/src/src
+ExecStart=/opt/paperless/venv/bin/python manage.py document_consumer
+
+[Install]
+WantedBy=multi-user.target

bundles/paperless-ng/files/paperless-scheduler.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Paperless scheduler
+Requires=redis.service
+
+[Service]
+User=paperless
+Group=paperless
+WorkingDirectory=/opt/paperless/src/src
+ExecStart=/opt/paperless/venv/bin/python manage.py qcluster
+
+[Install]
+WantedBy=multi-user.target

bundles/paperless-ng/files/paperless-webserver.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Paperless webserver
+After=network.target
+Wants=network.target
+Requires=redis.service
+
+[Service]
+User=paperless
+Group=paperless
+WorkingDirectory=/opt/paperless/src/src
+ExecStart=/opt/paperless/venv/bin/gunicorn -c /opt/paperless/src/gunicorn.conf.py -b 127.0.0.1:22070 paperless.asgi:application
+
+[Install]
+WantedBy=multi-user.target

bundles/paperless-ng/files/paperless.conf

@@ -0,0 +1,69 @@
+PAPERLESS_REDIS=redis://localhost:6379
+PAPERLESS_DBHOST=localhost
+PAPERLESS_DBPORT=5432
+PAPERLESS_DBNAME=paperless
+PAPERLESS_DBUSER=paperless
+PAPERLESS_DBPASS=${node.metadata.get('postgresql/roles/paperless/password')}
+PAPERLESS_DBSSLMODE=disable
+
+# Paths and folders
+
+PAPERLESS_CONSUMPTION_DIR=/mnt/paperless/consume
+PAPERLESS_DATA_DIR=/mnt/paperless/data
+PAPERLESS_MEDIA_ROOT=/opt/paperless/media
+PAPERLESS_STATICDIR=/opt/paperless/static
+PAPERLESS_FILENAME_FORMAT={created_year}/{created_month}/{correspondent}/{asn}_{title}
+
+# Security and hosting
+
+PAPERLESS_SECRET_KEY=${repo.vault.random_bytes_as_base64_for(f'{node.name} paperless secret key')}
+PAPERLESS_ALLOWED_HOSTS=${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')}
+PAPERLESS_CORS_ALLOWED_HOSTS=http://${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')},https://${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')}
+#PAPERLESS_FORCE_SCRIPT_NAME=
+#PAPERLESS_STATIC_URL=/static/
+#PAPERLESS_AUTO_LOGIN_USERNAME=
+#PAPERLESS_COOKIE_PREFIX=
+#PAPERLESS_ENABLE_HTTP_REMOTE_USER=false
+
+# OCR settings
+
+PAPERLESS_OCR_LANGUAGE=${'+'.join(sorted(node.metadata.get('paperless/ocr_languages', {'deu', 'eng'})))}
+PAPERLESS_OCR_MODE=skip
+#PAPERLESS_OCR_OUTPUT_TYPE=pdfa
+#PAPERLESS_OCR_PAGES=1
+#PAPERLESS_OCR_IMAGE_DPI=300
+#PAPERLESS_OCR_CLEAN=clean
+#PAPERLESS_OCR_DESKEW=true
+#PAPERLESS_OCR_ROTATE_PAGES=true
+#PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0
+#PAPERLESS_OCR_USER_ARGS={}
+#PAPERLESS_CONVERT_MEMORY_LIMIT=0
+#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
+
+# Software tweaks
+
+PAPERLESS_TASK_WORKERS=${node.metadata.get('vm/cpu', 1)}
+#PAPERLESS_THREADS_PER_WORKER=1
+PAPERLESS_TIME_ZONE=${node.metadata.get('paperless/timezone', 'UTC')}
+#PAPERLESS_CONSUMER_POLLING=10
+#PAPERLESS_CONSUMER_DELETE_DUPLICATES=false
+#PAPERLESS_CONSUMER_RECURSIVE=false
+#PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
+#PAPERLESS_OPTIMIZE_THUMBNAILS=true
+#PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
+#PAPERLESS_FILENAME_DATE_ORDER=YMD
+#PAPERLESS_FILENAME_PARSE_TRANSFORMS=[]
+#PAPERLESS_THUMBNAIL_FONT_NAME=
+#PAPERLESS_IGNORE_DATES=
+
+# Tika settings
+
+#PAPERLESS_TIKA_ENABLED=false
+#PAPERLESS_TIKA_ENDPOINT=http://localhost:9998
+#PAPERLESS_TIKA_GOTENBERG_ENDPOINT=http://localhost:3000
+
+# Binaries
+
+#PAPERLESS_CONVERT_BINARY=/usr/bin/convert
+#PAPERLESS_GS_BINARY=/usr/bin/gs
+#PAPERLESS_OPTIPNG_BINARY=/usr/bin/optipng

bundles/paperless-ng/items.py

@@ -0,0 +1,134 @@
+users = {
+    'paperless': {
+        'home': '/opt/paperless',
+    },
+}
+
+directories = {
+    '/opt/paperless/src': {},
+    '/opt/paperless/media': {
+        'owner': 'paperless',
+    },
+    '/opt/paperless/static': {
+        'owner': 'paperless',
+    },
+}
+
+git_deploy = {
+    '/opt/paperless/src': {
+        'repo': 'https://github.com/jonaswinkler/paperless-ng.git',
+        'rev': node.metadata.get('paperless/version'),
+        'triggers': {
+            'action:paperless_compile_frontend',
+            'action:paperless_install_deps',
+            'action:paperless_migrate_database',
+            'svc_systemd:paperless-consumer:restart',
+            'svc_systemd:paperless-scheduler:restart',
+            'svc_systemd:paperless-webserver:restart',
+        },
+    },
+}
+
+files = {
+    '/etc/systemd/system/paperless-consumer.service': {
+        'triggers': {
+            'action:systemd-reload',
+            'svc_systemd:paperless-consumer:restart',
+        },
+    },
+    '/etc/systemd/system/paperless-scheduler.service': {
+        'triggers': {
+            'action:systemd-reload',
+            'svc_systemd:paperless-scheduler:restart',
+        },
+    },
+    '/etc/systemd/system/paperless-webserver.service': {
+        'triggers': {
+            'action:systemd-reload',
+            'svc_systemd:paperless-webserver:restart',
+        },
+    },
+    '/opt/paperless/src/paperless.conf': {
+        'content_type': 'mako',
+        'needs': {
+            'git_deploy:/opt/paperless/src',
+        },
+        'triggers': {
+            'svc_systemd:paperless-consumer:restart',
+            'svc_systemd:paperless-scheduler:restart',
+            'svc_systemd:paperless-webserver:restart',
+        },
+    },
+}
+
+actions = {
+    'paperless_create_virtualenv': {
+        'command': '/usr/bin/python3 -m virtualenv -p python3 /opt/paperless/venv/',
+        'unless': 'test -d /opt/paperless/venv/',
+        'needs': {
+            # actually /opt/paperless, but we don't create that
+            'directory:/opt/paperless/src',
+        },
+    },
+    'paperless_install_deps': {
+        'command':
+            'cd /opt/paperless/src && '
+            '/opt/paperless/venv/bin/pip install --upgrade pip && '
+            '/opt/paperless/venv/bin/pip install --upgrade -r requirements.txt',
+        'triggered': True,
+        'needs': {
+            'action:paperless_create_virtualenv',
+        },
+    },
+    'paperless_migrate_database': {
+        'command':
+            'cd /opt/paperless/src/src && '
+            'sudo -Hu paperless /opt/paperless/venv/bin/python manage.py migrate',
+        'triggered': True,
+        'needs': {
+            # /mnt/paperless is NOT created by this bundle.
+            'action:paperless_install_deps',
+            'directory:/mnt/paperless',
+            'directory:/opt/paperless/static',
+            'file:/opt/paperless/src/paperless.conf',
+            'user:paperless',
+            'postgres_db:paperless',
+        },
+    },
+    'paperless_compile_frontend': {
+        'command':
+            'cd /opt/paperless/src/src-ui && '
+            'npm install && '
+            'node_modules/.bin/ng build --prod',
+        'triggered': True,
+        'needs': {
+            'file:/opt/paperless/src/paperless.conf',
+            'pkg_apt:nodejs',
+        },
+    },
+}
+
+svc_systemd = {
+    'paperless-consumer': {
+        'needs': {
+            'action:paperless_migrate_database',
+            'file:/etc/systemd/system/paperless-consumer.service',
+            'git_deploy:/opt/paperless/src',
+        },
+    },
+    'paperless-scheduler': {
+        'needs': {
+            'action:paperless_migrate_database',
+            'file:/etc/systemd/system/paperless-scheduler.service',
+            'git_deploy:/opt/paperless/src',
+        },
+    },
+    'paperless-webserver': {
+        'needs': {
+            'action:paperless_compile_frontend',
+            'action:paperless_migrate_database',
+            'file:/etc/systemd/system/paperless-webserver.service',
+            'git_deploy:/opt/paperless/src',
+        },
+    },
+}

bundles/paperless-ng/metadata.py

@@ -0,0 +1,53 @@
+defaults = {
+    'apt': {
+        'packages': {
+            # for paperless itself
+            'fonts-liberation': {},
+            'gnupg': {},
+            'imagemagick': {},
+            'libmagic-dev': {},
+            'libpq-dev': {},
+            'mime-support': {},
+            'optipng': {},
+            'python3-wheel': {},
+
+            # for OCRmyPDF
+            'ghostscript': {},
+            'icc-profiles-free': {},
+            'liblept5': {},
+            'libxml2': {},
+            'pngquant': {},
+            'qpdf': {},
+            'tesseract-ocr': {},
+            'unpaper': {},
+            'zlib1g': {},
+        },
+    },
+    'postgresql': {
+        'roles': {
+            'paperless': {
+                'password': repo.vault.password_for(f'{node.name} postgresql paperless'),
+            },
+        },
+        'databases': {
+            'paperless': {
+                'owner': 'paperless',
+            },
+        },
+    },
+}
+
+@metadata_reactor.provides(
+    'apt/packages',
+)
+def paperless_tesseract_languages(metadata):
+    packages = {}
+
+    for lang in metadata.get('paperless/ocr_languages', {'deu', 'eng'}):
+        packages[f'tesseract-ocr-{lang}'] = {}
+
+    return {
+        'apt': {
+            'packages': packages,
+        },
+    }

data/backup/keys/home.paperless.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABgqlxI4OjgrQfSKqCqA8vSbhNpdPwt56Akk73WMcqTs0nf9tiQsGoneptdO-1x5X_-yEI_YE_SywHo4yZ0ABQjUdNpLDojjDqkT2wZPDeSXgoIQFpnf_JZ84aw89_srH6CBEBGDU6bjljiAarrkAyBWaW21DFSuH1SSwuNiy3Rr1GP6HwTgqMVtNc-W4x6pehViOpkiyvvffgYGTY826YXmV4dCapr3Z8l4acOmSucnnc5YxKXSHl5wk9vTDpyhcT6qQJ7d-_cRCDrZtkPYNWNJRjAVmshIg_QRXwgQU_YPqZRrcQIUGMnaIBNjv0LKcSDPDAD2Rv8GHMLF1Vt6brJ_p3ihY_8KrP6QvwKyvSX1CDVxhwYq9WfCqvlqQOIkVLnn_vS-FzqU98cbef-rZsXLVRe7ODrU-Fg5tOVmKp761VGQSF1l4FIZnkQwF5uj-AmJXgaTfkcvoYhWtFEadnrKYmV92GbymiwPB6EG9SRMgcgpMAYCl9If7oEMjuYBs6bfTq0dfA39xWRJQx9zhAMTAAWHewYwEzME5PuTaQCDSRd2qIYik-DemhRp3suuphvjeuJTL5qyHXIH03yCTRxYf28cw0PVC2B696mI-z0I_-FT5Tc9l4pbh3RZlQ7Z8dNewJX

data/backup/keys/home.paperless.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIbL8JFM5vHyxSf4Ym57K+ssOi7gbk7Ma/+pOoT+1qGy kunsi@kunsi-t470.kunbox.net

nodes/home/nas.py

@@ -123,6 +123,9 @@ nodes['home.nas'] = {
                 '/storage/nas': {
                     '172.19.138.0/24(ro,all_squash,anonuid=65534,anongid=65534,no_subtree_check)',
                 },
+                '/srv/paperless': {
+                    '172.19.138.29/32(rw,all_squash,anonuid=65534,anongid=65534,no_subtree_check)',
+                },
                 '/srv/scansnap': {
                     '172.19.138.0/24(rw,all_squash,anonuid=65534,anongid=65534,no_subtree_check)',
                 },
@@ -208,6 +211,9 @@ nodes['home.nas'] = {
                 'storage/nas': {
                     'mountpoint': '/storage/nas',
                 },
+                'storage/paperless': {
+                    'mountpoint': '/srv/paperless',
+                },
                 'storage/scan': {
                     'mountpoint': '/srv/scansnap',
                 },
@@ -226,6 +232,11 @@ nodes['home.nas'] = {
                         'weekly': 6,
                         'monthly': 12,
                     },
+                    'storage/paperless': {
+                        'daily': 14,
+                        'weekly': 6,
+                        'monthly': 24,
+                    },
                     'storage/scan': {
                         'hourly': 6,
                         'daily': 0,

nodes/home/paperless.py

@@ -0,0 +1,56 @@
+nodes['home.paperless'] = {
+    'hostname': '172.19.138.29',
+    'bundles': {
+        'nfs-client',
+        'nodejs',
+        'redis',
+        'postgresql',
+        'paperless-ng',
+    },
+    'groups': {
+        'debian-buster',
+        'webserver',
+    },
+    'metadata': {
+        'interfaces': {
+            'enp1s0.42': {
+                'ips': {
+                    '172.19.138.29/24',
+                },
+                'gateway4': '172.19.138.1',
+            },
+        },
+        'nfs-client': {
+            'mounts': {
+                'nas_paperless': {
+                    'mountpoint': '/mnt/paperless',
+                    'serverpath': '172.19.138.20:/srv/paperless',
+                    'mount_options': {
+                        'retry=0',
+                        'rw',
+                    },
+                },
+            },
+        },
+        'nginx': {
+            'vhosts': {
+                'paperless': {
+                    'domain': 'paperless.home.kunbox.net',
+                    'ssl': '_.home.kunbox.net',
+                    'proxy': {
+                        '/': {
+                            'target': 'http://127.0.0.1:22070',
+                            'websockets': True,
+                            'proxy_set_header': {
+                                'X-Forwarded-Host': '$server_name',
+                            },
+                        },
+                    },
+                },
+            },
+        },
+        'paperless': {
+            'version': 'ng-1.4.4',
+        },
+    },
+}