From d4b110087fc8ec25eedff24a5227c58b983e5464 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Fri, 12 Feb 2021 16:01:35 +0100 Subject: [PATCH] bundles/matrix-media-repo: introduce, add to htz.ex42-1048908 --- PORT_MAP.md | 1 + bundles/matrix-media-repo/files/config.yaml | 170 ++++++++++++++++++ .../files/matrix-media-repo.service | 15 ++ bundles/matrix-media-repo/items.py | 71 ++++++++ bundles/matrix-media-repo/metadata.py | 74 ++++++++ .../htz.ex42-1048908/matrix.franzi.business | 6 +- nodes/htz/ex42-1048908.py | 14 ++ 7 files changed, 348 insertions(+), 3 deletions(-) create mode 100644 bundles/matrix-media-repo/files/config.yaml create mode 100644 bundles/matrix-media-repo/files/matrix-media-repo.service create mode 100644 bundles/matrix-media-repo/items.py create mode 100644 bundles/matrix-media-repo/metadata.py diff --git a/PORT_MAP.md b/PORT_MAP.md index 903c93a..1111ee5 100644 --- a/PORT_MAP.md +++ b/PORT_MAP.md @@ -37,6 +37,7 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports. | 20020 | mautrix-whatsapp | Bridge | | 20080 | matrix-synapse | client, federation | | 20081 | matrix-synapse | prometheus metrics | +| 20090 | matrix-media-repo | media_repo | | 22000 | gitea | gitea | | 22010 | jenkins-ci | Jenkins CI | | 22020 | travelynx | Travelynx Web | diff --git a/bundles/matrix-media-repo/files/config.yaml b/bundles/matrix-media-repo/files/config.yaml new file mode 100644 index 0000000..52e924a --- /dev/null +++ b/bundles/matrix-media-repo/files/config.yaml @@ -0,0 +1,170 @@ +# General repo configuration +repo: + bindAddress: '${node.metadata['matrix-media-repo'].get('listen-addr', '127.0.0.1')}' + port: ${node.metadata['matrix-media-repo'].get('port', 20090)} + logDirectory: '-' + trustAnyForwardedAddress: false + useForwardedHost: true + +federation: + backoffAt: 20 + +database: + postgres: "postgres://${node.metadata['matrix-media-repo']['database']['user']}:${node.metadata['matrix-media-repo']['database']['password']}@${node.metadata['matrix-media-repo']['database'].get('host', 'localhost')}/${node.metadata['matrix-media-repo']['database']['database']}?sslmode=disable" + + pool: + maxConnections: 25 + maxIdleConnections: 5 + +homeservers: +% for homeserver, config in node.metadata['matrix-media-repo'].get('homeservers', {}).items(): + - name: ${homeserver} + csApi: "${config['domain']}" + backoffAt: ${config.get('backoff_at', 10)} + adminApiKind: "${config.get('api', 'matrix')}" +% endfor + +accessTokens: + maxCacheTimeSeconds: 0 + useLocalAppserviceConfig: false + +admins: +% for user in sorted(node.metadata['matrix-media-repo']['admins']): + - "${user}" +% endfor + +sharedSecretAuth: + enabled: false + token: "${node.metadata['matrix-media-repo']['shared-secret-token']}" + +datastores: + - type: file + enabled: true + forKinds: + - 'thumbnails' + - 'remote_media' + - 'local_media' + - 'archives' + opts: + path: /var/matrix/media + +archiving: + enabled: true + selfService: ${str(node.metadata['matrix-media-repo']['archive']['self-service']).lower()} + targetBytesPerPart: ${node.metadata['matrix-media-repo']['archive'].get('mb_per_part', node.metadata['matrix-media-repo']['upload_max_mb']*2)*1024*1024} + +uploads: + maxBytes: ${node.metadata['matrix-media-repo']['upload_max_mb']*1024*1024} + minBytes: 100 + reportedMaxBytes: 0 + quotas: + enabled: false + +downloads: + maxBytes: ${node.metadata['matrix-media-repo']['download_max_mb']*1024*1024} + numWorkers: ${node.metadata['matrix-media-repo']['workers']} + failureCacheMinutes: 5 + cache: + enabled: true + maxSizeBytes: ${node.metadata['matrix-media-repo']['download_max_mb']*10*1024*1024} + maxFileSizeBytes: ${node.metadata['matrix-media-repo']['upload_max_mb']*1024*1024} + trackedMinutes: 30 + minDownloads: 5 + minCacheTimeSeconds: 300 + minEvictedTimeSeconds: 60 + expireAfterDays: 0 + +urlPreviews: + enabled: true + maxPageSizeBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024} + previewUnsafeCertificates: false + numWords: 50 + maxLength: 200 + numTitleWords: 30 + maxTitleLength: 150 + filePreviewTypes: + - "image/*" + numWorkers: ${node.metadata['matrix-media-repo']['workers']} + disallowedNetworks: + - "127.0.0.1/8" + - "10.0.0.0/8" + - "172.16.0.0/12" + - "192.168.0.0/16" + - "100.64.0.0/10" + - "169.254.0.0/16" + - '::1/128' + - 'fe80::/64' + - 'fc00::/7' + allowedNetworks: + - "0.0.0.0/0" + - "::/0" + expireAfterDays: 0 + defaultLanguage: "en-US,en" + oEmbed: false + +thumbnails: + maxSourceBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024} + numWorkers: ${node.metadata['matrix-media-repo']['workers']} + sizes: + - width: 32 + height: 32 + - width: 96 + height: 96 + - width: 320 + height: 240 + - width: 640 + height: 480 + - width: 768 + height: 240 + - width: 800 + height: 600 + dynamicSizing: false + types: + - "image/jpeg" + - "image/jpg" + - "image/png" + - "image/gif" + - "image/heif" + - "image/webp" + - "image/svg+xml" + - "audio/mpeg" + - "audio/ogg" + - "audio/wav" + - "audio/flac" + - "video/mp4" + allowAnimated: true + defaultAnimated: false + maxAnimateSizeBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024} + stillFrame: 0.5 + expireAfterDays: 0 + +rateLimit: + enabled: true + requestsPerSecond: 2 + burst: 25 + +identicons: + enabled: true + +quarantine: + replaceThumbnails: true + replaceDownloads: false + allowLocalAdmins: true + +timeouts: + urlPreviewTimeoutSeconds: 10 + federationTimeoutSeconds: 120 + clientServerTimeoutSeconds: 30 + +metrics: + enabled: false + bindAddress: "127.0.0.1" + port: 20091 + +featureSupport: + MSC2448: + enabled: false + IPFS: + enabled: false + redis: + enabled: false diff --git a/bundles/matrix-media-repo/files/matrix-media-repo.service b/bundles/matrix-media-repo/files/matrix-media-repo.service new file mode 100644 index 0000000..dc9789d --- /dev/null +++ b/bundles/matrix-media-repo/files/matrix-media-repo.service @@ -0,0 +1,15 @@ +[Unit] +Description=Matrix Media Repo +After=network.target +Requires=postgresql.service + +[Service] +User=matrix-media-repo +Group=matrix-media-repo +ExecStart=/opt/matrix-media-repo/src/bin/media_repo -config /opt/matrix-media-repo/config.yaml +WorkingDirectory=/opt/matrix-media-repo/ +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/bundles/matrix-media-repo/items.py b/bundles/matrix-media-repo/items.py new file mode 100644 index 0000000..b1b6b5f --- /dev/null +++ b/bundles/matrix-media-repo/items.py @@ -0,0 +1,71 @@ +directories = { + '/opt/matrix-media-repo': { + 'owner': 'matrix-media-repo', + 'group': 'matrix-media-repo', + }, + '/opt/matrix-media-repo/src': { + 'owner': 'matrix-media-repo', + 'group': 'matrix-media-repo', + }, + '/var/matrix/media': { + 'owner': 'matrix-media-repo', + 'group': 'matrix-media-repo', + }, +} + +users = { + 'matrix-media-repo': { + 'home': '/opt/matrix-media-repo', + }, +} + +files = { + '/opt/matrix-media-repo/config.yaml': { + 'owner': 'matrix-media-repo', + 'content_type': 'mako', + 'triggers': { + 'svc_systemd:matrix-media-repo:restart', + }, + }, + '/etc/systemd/system/matrix-media-repo.service': { + 'triggers': { + 'action:systemd-reload', + 'svc_systemd:matrix-media-repo:restart', + }, + }, +} + +git_deploy = { + '/opt/matrix-media-repo/src': { + 'repo': 'https://github.com/turt2live/matrix-media-repo.git', + 'rev': node.metadata['matrix-media-repo']['version'], + 'triggers': { + 'action:matrix-media-repo_build', + }, + }, +} + +actions = { + 'matrix-media-repo_build': { + 'command': \ + 'chown -R matrix-media-repo:matrix-media-repo /opt/matrix-media-repo/src && ' + \ + 'sudo -u matrix-media-repo bash -c "cd /opt/matrix-media-repo/src && ' + \ + 'GOBIN=/opt/matrix-media-repo/src/bin go install -v ./cmd/compile_assets && ' + \ + '/opt/matrix-media-repo/src/bin/compile_assets && ' + \ + 'GOBIN=/opt/matrix-media-repo/src/bin go install -ldflags \\\"' + \ + '-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$(cat /opt/matrix-media-repo/src/.bundlewrap_git_deploy) ' + \ + '-X github.com/turt2live/matrix-media-repo/common/version.Version={}\\\" '.format(node.metadata['matrix-media-repo']['version']) + \ + '-v ./cmd/..."', + 'triggered': True, + }, +} + +svc_systemd = { + 'matrix-media-repo': { + 'needs': { + 'action:matrix-media-repo_build', + 'file:/etc/systemd/system/matrix-media-repo.service', + 'file:/opt/matrix-media-repo/config.yaml', + }, + }, +} diff --git a/bundles/matrix-media-repo/metadata.py b/bundles/matrix-media-repo/metadata.py new file mode 100644 index 0000000..d7ef738 --- /dev/null +++ b/bundles/matrix-media-repo/metadata.py @@ -0,0 +1,74 @@ +defaults = { + 'apt': { + 'packages': { + 'golang-go': {}, + 'imagemagick': {}, + 'ffmpeg': {}, + }, + }, + 'icinga2_api': { + 'matrix-media-repo': { + 'services': { + 'MATRIX-MEDIA-REPO PROCESS': { + 'command_on_monitored_host': '/usr/lib/nagios/plugins/check_procs -a media_repo -c 1:', + }, + }, + }, + }, + 'matrix-media-repo': { + 'database': { + 'user': 'matrix-media-repo', + 'password': repo.vault.password_for('{} postgresql matrix-media-repo'.format(node.name)), + 'database': 'matrix-media-repo', + }, + 'archive': { + 'self-service': False, + }, + 'download_max_mb': 100, + 'preview_max_mb': 10, + 'shared-secret-token': repo.vault.password_for('{} matrix-media-repo shared-secret-token'.format(node.name)), + 'upload_max_mb': 100, + }, + 'postgresql': { + 'roles': { + 'matrix-media-repo': { + 'password': repo.vault.password_for('{} postgresql matrix-media-repo'.format(node.name)), + }, + }, + 'databases': { + 'matrix-media-repo': { + 'owner': 'matrix-media-repo', + }, + }, + }, +} + + +@metadata_reactor.provides( + 'matrix-media-repo/workers', +) +def cpu_to_workers(metadata): + return { + 'matrix-media-repo': { + 'workers': max(metadata.get('vm/cpu', 1), 4), + }, + } + + +@metadata_reactor.provides( + 'icinga2_api/matrix-media-repo/services', +) +def icinga_check_for_new_release(metadata): + return { + 'icinga2_api': { + 'matrix-media-repo': { + 'services': { + 'MATRIX-MEDIA-REPO UPDATE': { + 'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_github_for_new_release turt2live/matrix-media-repo {}'.format(metadata.get('matrix-media-repo/version')), + 'vars.notification.mail': True, + 'check_interval': '60m', + }, + }, + }, + }, + } diff --git a/data/nginx/files/extras/htz.ex42-1048908/matrix.franzi.business b/data/nginx/files/extras/htz.ex42-1048908/matrix.franzi.business index 4af9256..c7a4c4c 100644 --- a/data/nginx/files/extras/htz.ex42-1048908/matrix.franzi.business +++ b/data/nginx/files/extras/htz.ex42-1048908/matrix.franzi.business @@ -1,5 +1,3 @@ - client_max_body_size 500M; - location /.well-known/matrix/ { types { } default_type "application/json"; alias /etc/matrix-synapse/wellknown/; @@ -14,9 +12,11 @@ } location /_matrix/media { + client_max_body_size 500M; + proxy_read_timeout 600s; proxy_set_header Host "franzi.business"; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; - proxy_pass http://localhost:8010; + proxy_pass http://localhost:20090; } diff --git a/nodes/htz/ex42-1048908.py b/nodes/htz/ex42-1048908.py index 20be9b5..00ffa7f 100644 --- a/nodes/htz/ex42-1048908.py +++ b/nodes/htz/ex42-1048908.py @@ -3,6 +3,7 @@ nodes['htz.ex42-1048908'] = { 'dovecot', 'gitea', 'jenkins-ci', + 'matrix-media-repo', 'matrix-synapse', 'mautrix-telegram', 'mautrix-whatsapp', @@ -124,6 +125,19 @@ nodes['htz.ex42-1048908'] = { # legacy 'en_DK.UTF-8', }, + 'matrix-media-repo': { + 'version': 'v1.2.2', + 'homeservers': { + 'franzi.business': { + 'domain': 'http://[::1]:20080/', + 'api': 'synapse', + }, + }, + 'admins': { + '@kunsi:franzi.business', + }, + 'upload_max_mb': 500, + }, 'matrix-synapse': { 'server_name': 'franzi.business', 'baseurl': 'matrix.franzi.business',