kunsi/bundlewrap
1
0
Fork 0
Code Issues 1 Pull requests 1 Releases Activity

nodes/home.paperless: introduce
All checks were successful
bundlewrap/pipeline/head This commit looks good
Details

Browse source
This commit is contained in:
Franzi 2021-05-23 17:41:19 +02:00
parent aaf937a89f
commit 22c98a4206
Signed by: kunsi
GPG key ID: 12E3D2136B818350
12 changed files with 370 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
PORT_MAP.md
View file

@ -43,6 +43,7 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports.
| 22040 | miniflux | Miniflux Web Interface |
| 22050 | radicale | radicale carddav and caldav server |
| 22060 | pretalx | gunicorn |
| 22070 | paperless-ng | gunicorn |
| 22999 | nginx | stub_status |
## UDP

1
bundles/nginx/metadata.py
View file

@ -71,6 +71,7 @@ def letsencrypt(metadata):
vhosts = {}
for vhost, config in metadata.get('nginx/vhosts', {}).items():
if config.get('ssl', 'letsencrypt') == 'letsencrypt':
domain = config.get('domain', vhost)
domains[domain] = config.get('domain_aliases', set())
vhosts[vhost] = {

12
bundles/paperless-ng/files/paperless-consumer.service Normal file
View file

@ -0,0 +1,12 @@
[Unit]
Description=Paperless consumer
Requires=redis.service
[Service]
User=paperless
Group=paperless
WorkingDirectory=/opt/paperless/src/src
ExecStart=/opt/paperless/venv/bin/python manage.py document_consumer
[Install]
WantedBy=multi-user.target

12
bundles/paperless-ng/files/paperless-scheduler.service Normal file
View file

@ -0,0 +1,12 @@
[Unit]
Description=Paperless scheduler
Requires=redis.service
[Service]
User=paperless
Group=paperless
WorkingDirectory=/opt/paperless/src/src
ExecStart=/opt/paperless/venv/bin/python manage.py qcluster
[Install]
WantedBy=multi-user.target

14
bundles/paperless-ng/files/paperless-webserver.service Normal file
View file

@ -0,0 +1,14 @@
[Unit]
Description=Paperless webserver
After=network.target
Wants=network.target
Requires=redis.service
[Service]
User=paperless
Group=paperless
WorkingDirectory=/opt/paperless/src/src
ExecStart=/opt/paperless/venv/bin/gunicorn -c /opt/paperless/src/gunicorn.conf.py -b 127.0.0.1:22070 paperless.asgi:application
[Install]
WantedBy=multi-user.target

69
bundles/paperless-ng/files/paperless.conf Normal file
View file

@ -0,0 +1,69 @@
PAPERLESS_REDIS=redis://localhost:6379
PAPERLESS_DBHOST=localhost
PAPERLESS_DBPORT=5432
PAPERLESS_DBNAME=paperless
PAPERLESS_DBUSER=paperless
PAPERLESS_DBPASS=${node.metadata.get('postgresql/roles/paperless/password')}
PAPERLESS_DBSSLMODE=disable
# Paths and folders
PAPERLESS_CONSUMPTION_DIR=/mnt/paperless/consume
PAPERLESS_DATA_DIR=/mnt/paperless/data
PAPERLESS_MEDIA_ROOT=/opt/paperless/media
PAPERLESS_STATICDIR=/opt/paperless/static
PAPERLESS_FILENAME_FORMAT={created_year}/{created_month}/{correspondent}/{asn}_{title}
# Security and hosting
PAPERLESS_SECRET_KEY=${repo.vault.random_bytes_as_base64_for(f'{node.name} paperless secret key')}
PAPERLESS_ALLOWED_HOSTS=${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')}
PAPERLESS_CORS_ALLOWED_HOSTS=http://${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')},https://${node.metadata.get('nginx/vhosts/paperless/domain', '127.0.0.1')}
#PAPERLESS_FORCE_SCRIPT_NAME=
#PAPERLESS_STATIC_URL=/static/
#PAPERLESS_AUTO_LOGIN_USERNAME=
#PAPERLESS_COOKIE_PREFIX=
#PAPERLESS_ENABLE_HTTP_REMOTE_USER=false
# OCR settings
PAPERLESS_OCR_LANGUAGE=${'+'.join(sorted(node.metadata.get('paperless/ocr_languages', {'deu', 'eng'})))}
PAPERLESS_OCR_MODE=skip
#PAPERLESS_OCR_OUTPUT_TYPE=pdfa
#PAPERLESS_OCR_PAGES=1
#PAPERLESS_OCR_IMAGE_DPI=300
#PAPERLESS_OCR_CLEAN=clean
#PAPERLESS_OCR_DESKEW=true
#PAPERLESS_OCR_ROTATE_PAGES=true
#PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0
#PAPERLESS_OCR_USER_ARGS={}
#PAPERLESS_CONVERT_MEMORY_LIMIT=0
#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
# Software tweaks
PAPERLESS_TASK_WORKERS=${node.metadata.get('vm/cpu', 1)}
#PAPERLESS_THREADS_PER_WORKER=1
PAPERLESS_TIME_ZONE=${node.metadata.get('paperless/timezone', 'UTC')}
#PAPERLESS_CONSUMER_POLLING=10
#PAPERLESS_CONSUMER_DELETE_DUPLICATES=false
#PAPERLESS_CONSUMER_RECURSIVE=false
#PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
#PAPERLESS_OPTIMIZE_THUMBNAILS=true
#PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
#PAPERLESS_FILENAME_DATE_ORDER=YMD
#PAPERLESS_FILENAME_PARSE_TRANSFORMS=[]
#PAPERLESS_THUMBNAIL_FONT_NAME=
#PAPERLESS_IGNORE_DATES=
# Tika settings
#PAPERLESS_TIKA_ENABLED=false
#PAPERLESS_TIKA_ENDPOINT=http://localhost:9998
#PAPERLESS_TIKA_GOTENBERG_ENDPOINT=http://localhost:3000
# Binaries
#PAPERLESS_CONVERT_BINARY=/usr/bin/convert
#PAPERLESS_GS_BINARY=/usr/bin/gs
#PAPERLESS_OPTIPNG_BINARY=/usr/bin/optipng

134
bundles/paperless-ng/items.py Normal file
View file

@ -0,0 +1,134 @@
users = {
'paperless': {
'home': '/opt/paperless',
},
}
directories = {
'/opt/paperless/src': {},
'/opt/paperless/media': {
'owner': 'paperless',
},
'/opt/paperless/static': {
'owner': 'paperless',
},
}
git_deploy = {
'/opt/paperless/src': {
'repo': 'https://github.com/jonaswinkler/paperless-ng.git',
'rev': node.metadata.get('paperless/version'),
'triggers': {
'action:paperless_compile_frontend',
'action:paperless_install_deps',
'action:paperless_migrate_database',
'svc_systemd:paperless-consumer:restart',
'svc_systemd:paperless-scheduler:restart',
'svc_systemd:paperless-webserver:restart',
},
},
}
files = {
'/etc/systemd/system/paperless-consumer.service': {
'triggers': {
'action:systemd-reload',
'svc_systemd:paperless-consumer:restart',
},
},
'/etc/systemd/system/paperless-scheduler.service': {
'triggers': {
'action:systemd-reload',
'svc_systemd:paperless-scheduler:restart',
},
},
'/etc/systemd/system/paperless-webserver.service': {
'triggers': {
'action:systemd-reload',
'svc_systemd:paperless-webserver:restart',
},
},
'/opt/paperless/src/paperless.conf': {
'content_type': 'mako',
'needs': {
'git_deploy:/opt/paperless/src',
},
'triggers': {
'svc_systemd:paperless-consumer:restart',
'svc_systemd:paperless-scheduler:restart',
'svc_systemd:paperless-webserver:restart',
},
},
}
actions = {
'paperless_create_virtualenv': {
'command': '/usr/bin/python3 -m virtualenv -p python3 /opt/paperless/venv/',
'unless': 'test -d /opt/paperless/venv/',
'needs': {
# actually /opt/paperless, but we don't create that
'directory:/opt/paperless/src',
},
},
'paperless_install_deps': {
'command':
'cd /opt/paperless/src && '
'/opt/paperless/venv/bin/pip install --upgrade pip && '
'/opt/paperless/venv/bin/pip install --upgrade -r requirements.txt',
'triggered': True,
'needs': {
'action:paperless_create_virtualenv',
},
},
'paperless_migrate_database': {
'command':
'cd /opt/paperless/src/src && '
'sudo -Hu paperless /opt/paperless/venv/bin/python manage.py migrate',
'triggered': True,
'needs': {
# /mnt/paperless is NOT created by this bundle.
'action:paperless_install_deps',
'directory:/mnt/paperless',
'directory:/opt/paperless/static',
'file:/opt/paperless/src/paperless.conf',
'user:paperless',
'postgres_db:paperless',
},
},
'paperless_compile_frontend': {
'command':
'cd /opt/paperless/src/src-ui && '
'npm install && '
'node_modules/.bin/ng build --prod',
'triggered': True,
'needs': {
'file:/opt/paperless/src/paperless.conf',
'pkg_apt:nodejs',
},
},
}
svc_systemd = {
'paperless-consumer': {
'needs': {
'action:paperless_migrate_database',
'file:/etc/systemd/system/paperless-consumer.service',
'git_deploy:/opt/paperless/src',
},
},
'paperless-scheduler': {
'needs': {
'action:paperless_migrate_database',
'file:/etc/systemd/system/paperless-scheduler.service',
'git_deploy:/opt/paperless/src',
},
},
'paperless-webserver': {
'needs': {
'action:paperless_compile_frontend',
'action:paperless_migrate_database',
'file:/etc/systemd/system/paperless-webserver.service',
'git_deploy:/opt/paperless/src',
},
},
}

53
bundles/paperless-ng/metadata.py Normal file
View file

@ -0,0 +1,53 @@
defaults = {
'apt': {
'packages': {
# for paperless itself
'fonts-liberation': {},
'gnupg': {},
'imagemagick': {},
'libmagic-dev': {},
'libpq-dev': {},
'mime-support': {},
'optipng': {},
'python3-wheel': {},
# for OCRmyPDF
'ghostscript': {},
'icc-profiles-free': {},
'liblept5': {},
'libxml2': {},
'pngquant': {},
'qpdf': {},
'tesseract-ocr': {},
'unpaper': {},
'zlib1g': {},
},
},
'postgresql': {
'roles': {
'paperless': {
'password': repo.vault.password_for(f'{node.name} postgresql paperless'),
},
},
'databases': {
'paperless': {
'owner': 'paperless',
},
},
},
}
@metadata_reactor.provides(
'apt/packages',
)
def paperless_tesseract_languages(metadata):
packages = {}
for lang in metadata.get('paperless/ocr_languages', {'deu', 'eng'}):
packages[f'tesseract-ocr-{lang}'] = {}
return {
'apt': {
'packages': packages,
},
}

1
data/backup/keys/home.paperless.key.vault Normal file
View file

@ -0,0 +1 @@
encrypt$gAAAAABgqlxI4OjgrQfSKqCqA8vSbhNpdPwt56Akk73WMcqTs0nf9tiQsGoneptdO-1x5X_-yEI_YE_SywHo4yZ0ABQjUdNpLDojjDqkT2wZPDeSXgoIQFpnf_JZ84aw89_srH6CBEBGDU6bjljiAarrkAyBWaW21DFSuH1SSwuNiy3Rr1GP6HwTgqMVtNc-W4x6pehViOpkiyvvffgYGTY826YXmV4dCapr3Z8l4acOmSucnnc5YxKXSHl5wk9vTDpyhcT6qQJ7d-_cRCDrZtkPYNWNJRjAVmshIg_QRXwgQU_YPqZRrcQIUGMnaIBNjv0LKcSDPDAD2Rv8GHMLF1Vt6brJ_p3ihY_8KrP6QvwKyvSX1CDVxhwYq9WfCqvlqQOIkVLnn_vS-FzqU98cbef-rZsXLVRe7ODrU-Fg5tOVmKp761VGQSF1l4FIZnkQwF5uj-AmJXgaTfkcvoYhWtFEadnrKYmV92GbymiwPB6EG9SRMgcgpMAYCl9If7oEMjuYBs6bfTq0dfA39xWRJQx9zhAMTAAWHewYwEzME5PuTaQCDSRd2qIYik-DemhRp3suuphvjeuJTL5qyHXIH03yCTRxYf28cw0PVC2B696mI-z0I_-FT5Tc9l4pbh3RZlQ7Z8dNewJX

1
data/backup/keys/home.paperless.pub Normal file
View file

@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIbL8JFM5vHyxSf4Ym57K+ssOi7gbk7Ma/+pOoT+1qGy kunsi@kunsi-t470.kunbox.net

11
nodes/home/nas.py
View file

@ -123,6 +123,9 @@ nodes['home.nas'] = {
'/storage/nas': {
'172.19.138.0/24(ro,all_squash,anonuid=65534,anongid=65534,no_subtree_check)',
},
'/srv/paperless': {
'172.19.138.29/32(rw,all_squash,anonuid=65534,anongid=65534,no_subtree_check)',
},
'/srv/scansnap': {
'172.19.138.0/24(rw,all_squash,anonuid=65534,anongid=65534,no_subtree_check)',
},
@ -208,6 +211,9 @@ nodes['home.nas'] = {
'storage/nas': {
'mountpoint': '/storage/nas',
},
'storage/paperless': {
'mountpoint': '/srv/paperless',
},
'storage/scan': {
'mountpoint': '/srv/scansnap',
},
@ -226,6 +232,11 @@ nodes['home.nas'] = {
'weekly': 6,
'monthly': 12,
},
'storage/paperless': {
'daily': 14,
'weekly': 6,
'monthly': 24,
},
'storage/scan': {
'hourly': 6,
'daily': 0,

56
nodes/home/paperless.py Normal file
View file

@ -0,0 +1,56 @@
nodes['home.paperless'] = {
'hostname': '172.19.138.29',
'bundles': {
'nfs-client',
'nodejs',
'redis',
'postgresql',
'paperless-ng',
},
'groups': {
'debian-buster',
'webserver',
},
'metadata': {
'interfaces': {
'enp1s0.42': {
'ips': {
'172.19.138.29/24',
},
'gateway4': '172.19.138.1',
},
},
'nfs-client': {
'mounts': {
'nas_paperless': {
'mountpoint': '/mnt/paperless',
'serverpath': '172.19.138.20:/srv/paperless',
'mount_options': {
'retry=0',
'rw',
},
},
},
},
'nginx': {
'vhosts': {
'paperless': {
'domain': 'paperless.home.kunbox.net',
'ssl': '_.home.kunbox.net',
'proxy': {
'/': {
'target': 'http://127.0.0.1:22070',
'websockets': True,
'proxy_set_header': {
'X-Forwarded-Host': '$server_name',
},
},
},
},
},
},
'paperless': {
'version': 'ng-1.4.4',
},
},
}