bundles/matrix-media-repo: introduce, add to htz.ex42-1048908

2021-02-12 16:01:35 +01:00 · 2021-02-12 16:01:35 +01:00 · d4b110087f
commit d4b110087f
parent 638e37c05f
7 changed files with 348 additions and 3 deletions
--- a/PORT_MAP.md
+++ b/PORT_MAP.md
@ -37,6 +37,7 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports.
 | 20020       | mautrix-whatsapp     | Bridge |
 | 20080       | matrix-synapse       | client, federation |
 | 20081       | matrix-synapse       | prometheus metrics |
+| 20090       | matrix-media-repo    | media_repo |
 | 22000       | gitea                | gitea |
 | 22010       | jenkins-ci           | Jenkins CI |
 | 22020       | travelynx            | Travelynx Web |
--- a/bundles/matrix-media-repo/files/config.yaml
+++ b/bundles/matrix-media-repo/files/config.yaml
@ -0,0 +1,170 @@
+# General repo configuration
+repo:
+  bindAddress: '${node.metadata['matrix-media-repo'].get('listen-addr', '127.0.0.1')}'
+  port: ${node.metadata['matrix-media-repo'].get('port', 20090)}
+  logDirectory: '-'
+  trustAnyForwardedAddress: false
+  useForwardedHost: true
+
+federation:
+  backoffAt: 20
+
+database:
+  postgres: "postgres://${node.metadata['matrix-media-repo']['database']['user']}:${node.metadata['matrix-media-repo']['database']['password']}@${node.metadata['matrix-media-repo']['database'].get('host', 'localhost')}/${node.metadata['matrix-media-repo']['database']['database']}?sslmode=disable"
+
+  pool:
+    maxConnections: 25
+    maxIdleConnections: 5
+
+homeservers:
+% for homeserver, config in node.metadata['matrix-media-repo'].get('homeservers', {}).items():
+  - name: ${homeserver}
+    csApi: "${config['domain']}"
+    backoffAt: ${config.get('backoff_at', 10)}
+    adminApiKind: "${config.get('api', 'matrix')}"
+% endfor
+
+accessTokens:
+  maxCacheTimeSeconds: 0
+  useLocalAppserviceConfig: false
+
+admins:
+% for user in sorted(node.metadata['matrix-media-repo']['admins']):
+  - "${user}"
+% endfor
+
+sharedSecretAuth:
+  enabled: false
+  token: "${node.metadata['matrix-media-repo']['shared-secret-token']}"
+
+datastores:
+  - type: file
+    enabled: true
+    forKinds:
+      - 'thumbnails'
+      - 'remote_media'
+      - 'local_media'
+      - 'archives'
+    opts:
+      path: /var/matrix/media
+
+archiving:
+  enabled: true
+  selfService: ${str(node.metadata['matrix-media-repo']['archive']['self-service']).lower()}
+  targetBytesPerPart: ${node.metadata['matrix-media-repo']['archive'].get('mb_per_part', node.metadata['matrix-media-repo']['upload_max_mb']*2)*1024*1024}
+
+uploads:
+  maxBytes: ${node.metadata['matrix-media-repo']['upload_max_mb']*1024*1024}
+  minBytes: 100
+  reportedMaxBytes: 0
+  quotas:
+    enabled: false
+
+downloads:
+  maxBytes: ${node.metadata['matrix-media-repo']['download_max_mb']*1024*1024}
+  numWorkers: ${node.metadata['matrix-media-repo']['workers']}
+  failureCacheMinutes: 5
+  cache:
+    enabled: true
+    maxSizeBytes: ${node.metadata['matrix-media-repo']['download_max_mb']*10*1024*1024}
+    maxFileSizeBytes: ${node.metadata['matrix-media-repo']['upload_max_mb']*1024*1024}
+    trackedMinutes: 30
+    minDownloads: 5
+    minCacheTimeSeconds: 300
+    minEvictedTimeSeconds: 60
+  expireAfterDays: 0
+
+urlPreviews:
+  enabled: true
+  maxPageSizeBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024}
+  previewUnsafeCertificates: false
+  numWords: 50
+  maxLength: 200
+  numTitleWords: 30
+  maxTitleLength: 150
+  filePreviewTypes:
+    - "image/*"
+  numWorkers: ${node.metadata['matrix-media-repo']['workers']}
+  disallowedNetworks:
+    - "127.0.0.1/8"
+    - "10.0.0.0/8"
+    - "172.16.0.0/12"
+    - "192.168.0.0/16"
+    - "100.64.0.0/10"
+    - "169.254.0.0/16"
+    - '::1/128'
+    - 'fe80::/64'
+    - 'fc00::/7'
+  allowedNetworks:
+    - "0.0.0.0/0"
+    - "::/0"
+  expireAfterDays: 0
+  defaultLanguage: "en-US,en"
+  oEmbed: false
+
+thumbnails:
+  maxSourceBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024}
+  numWorkers: ${node.metadata['matrix-media-repo']['workers']}
+  sizes:
+    - width: 32
+      height: 32
+    - width: 96
+      height: 96
+    - width: 320
+      height: 240
+    - width: 640
+      height: 480
+    - width: 768
+      height: 240
+    - width: 800
+      height: 600
+  dynamicSizing: false
+  types:
+    - "image/jpeg"
+    - "image/jpg"
+    - "image/png"
+    - "image/gif"
+    - "image/heif"
+    - "image/webp"
+    - "image/svg+xml"
+    - "audio/mpeg"
+    - "audio/ogg"
+    - "audio/wav"
+    - "audio/flac"
+    - "video/mp4"
+  allowAnimated: true
+  defaultAnimated: false
+  maxAnimateSizeBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024}
+  stillFrame: 0.5
+  expireAfterDays: 0
+
+rateLimit:
+  enabled: true
+  requestsPerSecond: 2
+  burst: 25
+
+identicons:
+  enabled: true
+
+quarantine:
+  replaceThumbnails: true
+  replaceDownloads: false
+  allowLocalAdmins: true
+
+timeouts:
+  urlPreviewTimeoutSeconds: 10
+  federationTimeoutSeconds: 120
+  clientServerTimeoutSeconds: 30
+
+metrics:
+  enabled: false
+  bindAddress: "127.0.0.1"
+  port: 20091
+
+featureSupport:
+  MSC2448:
+    enabled: false
+  IPFS:
+    enabled: false
+  redis:
+    enabled: false
--- a/bundles/matrix-media-repo/files/matrix-media-repo.service
+++ b/bundles/matrix-media-repo/files/matrix-media-repo.service
@ -0,0 +1,15 @@
+[Unit]
+Description=Matrix Media Repo
+After=network.target
+Requires=postgresql.service
+
+[Service]
+User=matrix-media-repo
+Group=matrix-media-repo
+ExecStart=/opt/matrix-media-repo/src/bin/media_repo -config /opt/matrix-media-repo/config.yaml
+WorkingDirectory=/opt/matrix-media-repo/
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target
--- a/bundles/matrix-media-repo/items.py
+++ b/bundles/matrix-media-repo/items.py
@ -0,0 +1,71 @@
+directories = {
+    '/opt/matrix-media-repo': {
+        'owner': 'matrix-media-repo',
+        'group': 'matrix-media-repo',
+    },
+    '/opt/matrix-media-repo/src': {
+        'owner': 'matrix-media-repo',
+        'group': 'matrix-media-repo',
+    },
+    '/var/matrix/media': {
+        'owner': 'matrix-media-repo',
+        'group': 'matrix-media-repo',
+    },
+}
+
+users = {
+    'matrix-media-repo': {
+        'home': '/opt/matrix-media-repo',
+    },
+}
+
+files = {
+    '/opt/matrix-media-repo/config.yaml': {
+        'owner': 'matrix-media-repo',
+        'content_type': 'mako',
+        'triggers': {
+            'svc_systemd:matrix-media-repo:restart',
+        },
+    },
+    '/etc/systemd/system/matrix-media-repo.service': {
+        'triggers': {
+            'action:systemd-reload',
+            'svc_systemd:matrix-media-repo:restart',
+        },
+    },
+}
+
+git_deploy = {
+    '/opt/matrix-media-repo/src': {
+        'repo': 'https://github.com/turt2live/matrix-media-repo.git',
+        'rev': node.metadata['matrix-media-repo']['version'],
+        'triggers': {
+            'action:matrix-media-repo_build',
+        },
+    },
+}
+
+actions = {
+    'matrix-media-repo_build': {
+        'command': \
+            'chown -R matrix-media-repo:matrix-media-repo /opt/matrix-media-repo/src && ' + \
+            'sudo -u matrix-media-repo bash -c "cd /opt/matrix-media-repo/src && ' + \
+            'GOBIN=/opt/matrix-media-repo/src/bin go install -v ./cmd/compile_assets && ' + \
+            '/opt/matrix-media-repo/src/bin/compile_assets && ' + \
+            'GOBIN=/opt/matrix-media-repo/src/bin go install -ldflags \\\"' + \
+            '-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$(cat /opt/matrix-media-repo/src/.bundlewrap_git_deploy) ' + \
+            '-X github.com/turt2live/matrix-media-repo/common/version.Version={}\\\" '.format(node.metadata['matrix-media-repo']['version']) + \
+            '-v ./cmd/..."',
+        'triggered': True,
+    },
+}
+
+svc_systemd = {
+    'matrix-media-repo': {
+        'needs': {
+            'action:matrix-media-repo_build',
+            'file:/etc/systemd/system/matrix-media-repo.service',
+            'file:/opt/matrix-media-repo/config.yaml',
+        },
+    },
+}
--- a/bundles/matrix-media-repo/metadata.py
+++ b/bundles/matrix-media-repo/metadata.py
@ -0,0 +1,74 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'golang-go': {},
+            'imagemagick': {},
+            'ffmpeg': {},
+        },
+    },
+    'icinga2_api': {
+        'matrix-media-repo': {
+            'services': {
+                'MATRIX-MEDIA-REPO PROCESS': {
+                    'command_on_monitored_host': '/usr/lib/nagios/plugins/check_procs -a media_repo -c 1:',
+                },
+            },
+        },
+    },
+    'matrix-media-repo': {
+        'database': {
+            'user': 'matrix-media-repo',
+            'password': repo.vault.password_for('{} postgresql matrix-media-repo'.format(node.name)),
+            'database': 'matrix-media-repo',
+        },
+        'archive': {
+            'self-service': False,
+        },
+        'download_max_mb': 100,
+        'preview_max_mb': 10,
+        'shared-secret-token': repo.vault.password_for('{} matrix-media-repo shared-secret-token'.format(node.name)),
+        'upload_max_mb': 100,
+    },
+    'postgresql': {
+        'roles': {
+            'matrix-media-repo': {
+                'password': repo.vault.password_for('{} postgresql matrix-media-repo'.format(node.name)),
+            },
+        },
+        'databases': {
+            'matrix-media-repo': {
+                'owner': 'matrix-media-repo',
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'matrix-media-repo/workers',
+)
+def cpu_to_workers(metadata):
+    return {
+        'matrix-media-repo': {
+            'workers': max(metadata.get('vm/cpu', 1), 4),
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'icinga2_api/matrix-media-repo/services',
+)
+def icinga_check_for_new_release(metadata):
+    return {
+        'icinga2_api': {
+            'matrix-media-repo': {
+                'services': {
+                    'MATRIX-MEDIA-REPO UPDATE': {
+                        'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_github_for_new_release turt2live/matrix-media-repo {}'.format(metadata.get('matrix-media-repo/version')),
+                        'vars.notification.mail': True,
+                        'check_interval': '60m',
+                    },
+                },
+            },
+        },
+    }
--- a/data/nginx/files/extras/htz.ex42-1048908/matrix.franzi.business
+++ b/data/nginx/files/extras/htz.ex42-1048908/matrix.franzi.business
@ -1,5 +1,3 @@
-    client_max_body_size 500M;
-
    location /.well-known/matrix/ {
        types { } default_type "application/json";
        alias /etc/matrix-synapse/wellknown/;
@ -14,9 +12,11 @@
    }

    location /_matrix/media {
+        client_max_body_size 500M;
+
        proxy_read_timeout 600s;
        proxy_set_header Host "franzi.business";
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_pass http://localhost:8010;
+        proxy_pass http://localhost:20090;
    }
--- a/nodes/htz/ex42-1048908.py
+++ b/nodes/htz/ex42-1048908.py
@ -3,6 +3,7 @@ nodes['htz.ex42-1048908'] = {
        'dovecot',
        'gitea',
        'jenkins-ci',
+        'matrix-media-repo',
        'matrix-synapse',
        'mautrix-telegram',
        'mautrix-whatsapp',
@ -124,6 +125,19 @@ nodes['htz.ex42-1048908'] = {
            # legacy
            'en_DK.UTF-8',
        },
+        'matrix-media-repo': {
+            'version': 'v1.2.2',
+            'homeservers': {
+                'franzi.business': {
+                    'domain': 'http://[::1]:20080/',
+                    'api': 'synapse',
+                },
+            },
+            'admins': {
+                '@kunsi:franzi.business',
+            },
+            'upload_max_mb': 500,
+        },
        'matrix-synapse': {
            'server_name': 'franzi.business',
            'baseurl': 'matrix.franzi.business',