diff --git a/PORT_MAP.md b/PORT_MAP.md index 5a85294..e15cfec 100644 --- a/PORT_MAP.md +++ b/PORT_MAP.md @@ -43,6 +43,7 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports. | 22040 | miniflux | Miniflux Web Interface | | 22050 | radicale | radicale carddav and caldav server | | 22060 | pretalx | gunicorn | +| 22070 | paperless-ng | gunicorn | | 22999 | nginx | stub_status | ## UDP diff --git a/bundles/nginx/metadata.py b/bundles/nginx/metadata.py index e7dcf4f..c83ddea 100644 --- a/bundles/nginx/metadata.py +++ b/bundles/nginx/metadata.py @@ -71,11 +71,12 @@ def letsencrypt(metadata): vhosts = {} for vhost, config in metadata.get('nginx/vhosts', {}).items(): - domain = config.get('domain', vhost) - domains[domain] = config.get('domain_aliases', set()) - vhosts[vhost] = { - 'ssl': 'letsencrypt', - } + if config.get('ssl', 'letsencrypt') == 'letsencrypt': + domain = config.get('domain', vhost) + domains[domain] = config.get('domain_aliases', set()) + vhosts[vhost] = { + 'ssl': 'letsencrypt', + } return { 'letsencrypt': { diff --git a/bundles/paperless-ng/files/paperless-consumer.service b/bundles/paperless-ng/files/paperless-consumer.service new file mode 100644 index 0000000..60a95f9 --- /dev/null +++ b/bundles/paperless-ng/files/paperless-consumer.service @@ -0,0 +1,12 @@ +[Unit] +Description=Paperless consumer +Requires=redis.service + +[Service] +User=paperless +Group=paperless +WorkingDirectory=/opt/paperless/src/src +ExecStart=/opt/paperless/venv/bin/python manage.py document_consumer + +[Install] +WantedBy=multi-user.target diff --git a/bundles/paperless-ng/files/paperless-scheduler.service b/bundles/paperless-ng/files/paperless-scheduler.service new file mode 100644 index 0000000..54cfeae --- /dev/null +++ b/bundles/paperless-ng/files/paperless-scheduler.service @@ -0,0 +1,12 @@ +[Unit] +Description=Paperless scheduler +Requires=redis.service + +[Service] +User=paperless +Group=paperless +WorkingDirectory=/opt/paperless/src/src +ExecStart=/opt/paperless/venv/bin/python manage.py qcluster + +[Install] +WantedBy=multi-user.target diff --git a/bundles/paperless-ng/files/paperless-webserver.service b/bundles/paperless-ng/files/paperless-webserver.service new file mode 100644 index 0000000..9bcd926 --- /dev/null +++ b/bundles/paperless-ng/files/paperless-webserver.service @@ -0,0 +1,14 @@ +[Unit] +Description=Paperless webserver +After=network.target +Wants=network.target +Requires=redis.service + +[Service] +User=paperless +Group=paperless +WorkingDirectory=/opt/paperless/src/src +ExecStart=/opt/paperless/venv/bin/gunicorn -c /opt/paperless/src/gunicorn.conf.py -b 127.0.0.1:22070 paperless.asgi:application + +[Install] +WantedBy=multi-user.target diff --git a/bundles/paperless-ng/files/paperless.conf b/bundles/paperless-ng/files/paperless.conf new file mode 100644 index 0000000..4a197a5 --- /dev/null +++ b/bundles/paperless-ng/files/paperless.conf @@ -0,0 +1,69 @@ +PAPERLESS_REDIS=redis://localhost:6379 +PAPERLESS_DBHOST=localhost +PAPERLESS_DBPORT=5432 +PAPERLESS_DBNAME=paperless +PAPERLESS_DBUSER=paperless +PAPERLESS_DBPASS=${node.metadata.get('postgresql/roles/paperless/password')} +PAPERLESS_DBSSLMODE=disable + +# Paths and folders + +PAPERLESS_CONSUMPTION_DIR=/mnt/paperless/consume +PAPERLESS_DATA_DIR=/mnt/paperless/data +PAPERLESS_MEDIA_ROOT=/opt/paperless/media +PAPERLESS_STATICDIR=/opt/paperless/static +PAPERLESS_FILENAME_FORMAT={created_year}/{created_month}/{correspondent}/{asn}_{title} + +# Security and hosting + +PAPERLESS_SECRET_KEY=${repo.vault.random_bytes_as_base64_for(f'{node.name} paperless secret key')} +PAPERLESS_ALLOWED_HOSTS=${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')} +PAPERLESS_CORS_ALLOWED_HOSTS=http://${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')},https://${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')} +#PAPERLESS_FORCE_SCRIPT_NAME= +#PAPERLESS_STATIC_URL=/static/ +#PAPERLESS_AUTO_LOGIN_USERNAME= +#PAPERLESS_COOKIE_PREFIX= +#PAPERLESS_ENABLE_HTTP_REMOTE_USER=false + +# OCR settings + +PAPERLESS_OCR_LANGUAGE=${'+'.join(sorted(node.metadata.get('paperless/ocr_languages', {'deu', 'eng'})))} +PAPERLESS_OCR_MODE=skip +#PAPERLESS_OCR_OUTPUT_TYPE=pdfa +#PAPERLESS_OCR_PAGES=1 +#PAPERLESS_OCR_IMAGE_DPI=300 +#PAPERLESS_OCR_CLEAN=clean +#PAPERLESS_OCR_DESKEW=true +#PAPERLESS_OCR_ROTATE_PAGES=true +#PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0 +#PAPERLESS_OCR_USER_ARGS={} +#PAPERLESS_CONVERT_MEMORY_LIMIT=0 +#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless + +# Software tweaks + +PAPERLESS_TASK_WORKERS=${node.metadata.get('vm/cpu', 1)} +#PAPERLESS_THREADS_PER_WORKER=1 +PAPERLESS_TIME_ZONE=${node.metadata.get('paperless/timezone', 'UTC')} +#PAPERLESS_CONSUMER_POLLING=10 +#PAPERLESS_CONSUMER_DELETE_DUPLICATES=false +#PAPERLESS_CONSUMER_RECURSIVE=false +#PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false +#PAPERLESS_OPTIMIZE_THUMBNAILS=true +#PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh +#PAPERLESS_FILENAME_DATE_ORDER=YMD +#PAPERLESS_FILENAME_PARSE_TRANSFORMS=[] +#PAPERLESS_THUMBNAIL_FONT_NAME= +#PAPERLESS_IGNORE_DATES= + +# Tika settings + +#PAPERLESS_TIKA_ENABLED=false +#PAPERLESS_TIKA_ENDPOINT=http://localhost:9998 +#PAPERLESS_TIKA_GOTENBERG_ENDPOINT=http://localhost:3000 + +# Binaries + +#PAPERLESS_CONVERT_BINARY=/usr/bin/convert +#PAPERLESS_GS_BINARY=/usr/bin/gs +#PAPERLESS_OPTIPNG_BINARY=/usr/bin/optipng diff --git a/bundles/paperless-ng/items.py b/bundles/paperless-ng/items.py new file mode 100644 index 0000000..8b0e605 --- /dev/null +++ b/bundles/paperless-ng/items.py @@ -0,0 +1,134 @@ +users = { + 'paperless': { + 'home': '/opt/paperless', + }, +} + +directories = { + '/opt/paperless/src': {}, + '/opt/paperless/media': { + 'owner': 'paperless', + }, + '/opt/paperless/static': { + 'owner': 'paperless', + }, +} + +git_deploy = { + '/opt/paperless/src': { + 'repo': 'https://github.com/jonaswinkler/paperless-ng.git', + 'rev': node.metadata.get('paperless/version'), + 'triggers': { + 'action:paperless_compile_frontend', + 'action:paperless_install_deps', + 'action:paperless_migrate_database', + 'svc_systemd:paperless-consumer:restart', + 'svc_systemd:paperless-scheduler:restart', + 'svc_systemd:paperless-webserver:restart', + }, + }, +} + +files = { + '/etc/systemd/system/paperless-consumer.service': { + 'triggers': { + 'action:systemd-reload', + 'svc_systemd:paperless-consumer:restart', + }, + }, + '/etc/systemd/system/paperless-scheduler.service': { + 'triggers': { + 'action:systemd-reload', + 'svc_systemd:paperless-scheduler:restart', + }, + }, + '/etc/systemd/system/paperless-webserver.service': { + 'triggers': { + 'action:systemd-reload', + 'svc_systemd:paperless-webserver:restart', + }, + }, + '/opt/paperless/src/paperless.conf': { + 'content_type': 'mako', + 'needs': { + 'git_deploy:/opt/paperless/src', + }, + 'triggers': { + 'svc_systemd:paperless-consumer:restart', + 'svc_systemd:paperless-scheduler:restart', + 'svc_systemd:paperless-webserver:restart', + }, + }, +} + +actions = { + 'paperless_create_virtualenv': { + 'command': '/usr/bin/python3 -m virtualenv -p python3 /opt/paperless/venv/', + 'unless': 'test -d /opt/paperless/venv/', + 'needs': { + # actually /opt/paperless, but we don't create that + 'directory:/opt/paperless/src', + }, + }, + 'paperless_install_deps': { + 'command': + 'cd /opt/paperless/src && ' + '/opt/paperless/venv/bin/pip install --upgrade pip && ' + '/opt/paperless/venv/bin/pip install --upgrade -r requirements.txt', + 'triggered': True, + 'needs': { + 'action:paperless_create_virtualenv', + }, + }, + 'paperless_migrate_database': { + 'command': + 'cd /opt/paperless/src/src && ' + 'sudo -Hu paperless /opt/paperless/venv/bin/python manage.py migrate', + 'triggered': True, + 'needs': { + # /mnt/paperless is NOT created by this bundle. + 'action:paperless_install_deps', + 'directory:/mnt/paperless', + 'directory:/opt/paperless/static', + 'file:/opt/paperless/src/paperless.conf', + 'user:paperless', + 'postgres_db:paperless', + }, + }, + 'paperless_compile_frontend': { + 'command': + 'cd /opt/paperless/src/src-ui && ' + 'npm install && ' + 'node_modules/.bin/ng build --prod', + 'triggered': True, + 'needs': { + 'file:/opt/paperless/src/paperless.conf', + 'pkg_apt:nodejs', + }, + }, +} + +svc_systemd = { + 'paperless-consumer': { + 'needs': { + 'action:paperless_migrate_database', + 'file:/etc/systemd/system/paperless-consumer.service', + 'git_deploy:/opt/paperless/src', + }, + }, + 'paperless-scheduler': { + 'needs': { + 'action:paperless_migrate_database', + 'file:/etc/systemd/system/paperless-scheduler.service', + 'git_deploy:/opt/paperless/src', + }, + }, + 'paperless-webserver': { + 'needs': { + 'action:paperless_compile_frontend', + 'action:paperless_migrate_database', + 'file:/etc/systemd/system/paperless-webserver.service', + 'git_deploy:/opt/paperless/src', + }, + }, +} diff --git a/bundles/paperless-ng/metadata.py b/bundles/paperless-ng/metadata.py new file mode 100644 index 0000000..7669a29 --- /dev/null +++ b/bundles/paperless-ng/metadata.py @@ -0,0 +1,53 @@ +defaults = { + 'apt': { + 'packages': { + # for paperless itself + 'fonts-liberation': {}, + 'gnupg': {}, + 'imagemagick': {}, + 'libmagic-dev': {}, + 'libpq-dev': {}, + 'mime-support': {}, + 'optipng': {}, + 'python3-wheel': {}, + + # for OCRmyPDF + 'ghostscript': {}, + 'icc-profiles-free': {}, + 'liblept5': {}, + 'libxml2': {}, + 'pngquant': {}, + 'qpdf': {}, + 'tesseract-ocr': {}, + 'unpaper': {}, + 'zlib1g': {}, + }, + }, + 'postgresql': { + 'roles': { + 'paperless': { + 'password': repo.vault.password_for(f'{node.name} postgresql paperless'), + }, + }, + 'databases': { + 'paperless': { + 'owner': 'paperless', + }, + }, + }, +} + +@metadata_reactor.provides( + 'apt/packages', +) +def paperless_tesseract_languages(metadata): + packages = {} + + for lang in metadata.get('paperless/ocr_languages', {'deu', 'eng'}): + packages[f'tesseract-ocr-{lang}'] = {} + + return { + 'apt': { + 'packages': packages, + }, + } diff --git a/data/backup/keys/home.paperless.key.vault b/data/backup/keys/home.paperless.key.vault new file mode 100644 index 0000000..364fa26 --- /dev/null +++ b/data/backup/keys/home.paperless.key.vault @@ -0,0 +1 @@ +encrypt$gAAAAABgqlxI4OjgrQfSKqCqA8vSbhNpdPwt56Akk73WMcqTs0nf9tiQsGoneptdO-1x5X_-yEI_YE_SywHo4yZ0ABQjUdNpLDojjDqkT2wZPDeSXgoIQFpnf_JZ84aw89_srH6CBEBGDU6bjljiAarrkAyBWaW21DFSuH1SSwuNiy3Rr1GP6HwTgqMVtNc-W4x6pehViOpkiyvvffgYGTY826YXmV4dCapr3Z8l4acOmSucnnc5YxKXSHl5wk9vTDpyhcT6qQJ7d-_cRCDrZtkPYNWNJRjAVmshIg_QRXwgQU_YPqZRrcQIUGMnaIBNjv0LKcSDPDAD2Rv8GHMLF1Vt6brJ_p3ihY_8KrP6QvwKyvSX1CDVxhwYq9WfCqvlqQOIkVLnn_vS-FzqU98cbef-rZsXLVRe7ODrU-Fg5tOVmKp761VGQSF1l4FIZnkQwF5uj-AmJXgaTfkcvoYhWtFEadnrKYmV92GbymiwPB6EG9SRMgcgpMAYCl9If7oEMjuYBs6bfTq0dfA39xWRJQx9zhAMTAAWHewYwEzME5PuTaQCDSRd2qIYik-DemhRp3suuphvjeuJTL5qyHXIH03yCTRxYf28cw0PVC2B696mI-z0I_-FT5Tc9l4pbh3RZlQ7Z8dNewJX \ No newline at end of file diff --git a/data/backup/keys/home.paperless.pub b/data/backup/keys/home.paperless.pub new file mode 100644 index 0000000..034ca7c --- /dev/null +++ b/data/backup/keys/home.paperless.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIbL8JFM5vHyxSf4Ym57K+ssOi7gbk7Ma/+pOoT+1qGy kunsi@kunsi-t470.kunbox.net diff --git a/nodes/home/nas.py b/nodes/home/nas.py index a5398ec..deae1e9 100644 --- a/nodes/home/nas.py +++ b/nodes/home/nas.py @@ -123,6 +123,9 @@ nodes['home.nas'] = { '/storage/nas': { '172.19.138.0/24(ro,all_squash,anonuid=65534,anongid=65534,no_subtree_check)', }, + '/srv/paperless': { + '172.19.138.29/32(rw,all_squash,anonuid=65534,anongid=65534,no_subtree_check)', + }, '/srv/scansnap': { '172.19.138.0/24(rw,all_squash,anonuid=65534,anongid=65534,no_subtree_check)', }, @@ -208,6 +211,9 @@ nodes['home.nas'] = { 'storage/nas': { 'mountpoint': '/storage/nas', }, + 'storage/paperless': { + 'mountpoint': '/srv/paperless', + }, 'storage/scan': { 'mountpoint': '/srv/scansnap', }, @@ -226,6 +232,11 @@ nodes['home.nas'] = { 'weekly': 6, 'monthly': 12, }, + 'storage/paperless': { + 'daily': 14, + 'weekly': 6, + 'monthly': 24, + }, 'storage/scan': { 'hourly': 6, 'daily': 0, diff --git a/nodes/home/paperless.py b/nodes/home/paperless.py new file mode 100644 index 0000000..ae8fbf8 --- /dev/null +++ b/nodes/home/paperless.py @@ -0,0 +1,56 @@ +nodes['home.paperless'] = { + 'hostname': '172.19.138.29', + 'bundles': { + 'nfs-client', + 'nodejs', + 'redis', + 'postgresql', + 'paperless-ng', + }, + 'groups': { + 'debian-buster', + 'webserver', + }, + 'metadata': { + 'interfaces': { + 'enp1s0.42': { + 'ips': { + '172.19.138.29/24', + }, + 'gateway4': '172.19.138.1', + }, + }, + 'nfs-client': { + 'mounts': { + 'nas_paperless': { + 'mountpoint': '/mnt/paperless', + 'serverpath': '172.19.138.20:/srv/paperless', + 'mount_options': { + 'retry=0', + 'rw', + }, + }, + }, + }, + 'nginx': { + 'vhosts': { + 'paperless': { + 'domain': 'paperless.home.kunbox.net', + 'ssl': '_.home.kunbox.net', + 'proxy': { + '/': { + 'target': 'http://127.0.0.1:22070', + 'websockets': True, + 'proxy_set_header': { + 'X-Forwarded-Host': '$server_name', + }, + }, + }, + }, + }, + }, + 'paperless': { + 'version': 'ng-1.4.4', + }, + }, +}