From cc767867cfbc3606ccd4e045b44d07fdf0a51729 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Thu, 22 Dec 2022 19:02:52 +0100 Subject: [PATCH 01/11] add bundle:woodpecker-server --- PORT_MAP.md | 2 + .../files/woodpecker-server.service | 19 ++++ bundles/woodpecker-server/items.py | 35 +++++++ bundles/woodpecker-server/metadata.py | 94 +++++++++++++++++++ nodes/rx300.py | 12 +++ 5 files changed, 162 insertions(+) create mode 100644 bundles/woodpecker-server/files/woodpecker-server.service create mode 100644 bundles/woodpecker-server/items.py create mode 100644 bundles/woodpecker-server/metadata.py diff --git a/PORT_MAP.md b/PORT_MAP.md index a1725cb..40f6d0a 100644 --- a/PORT_MAP.md +++ b/PORT_MAP.md @@ -45,6 +45,8 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports. | 22060 | pretalx | gunicorn | | 22070 | paperless-ng | gunicorn | | 22080 | netbox | gunicorn | +| 22100 | woodpecker-server | http | +| 22101 | woodpecker-server | gRPC | | 22999 | nginx | stub_status | | 22100 | ntfy | http | diff --git a/bundles/woodpecker-server/files/woodpecker-server.service b/bundles/woodpecker-server/files/woodpecker-server.service new file mode 100644 index 0000000..5520b49 --- /dev/null +++ b/bundles/woodpecker-server/files/woodpecker-server.service @@ -0,0 +1,19 @@ +[Unit] +Description=woodpecker ci +After=syslog.target +After=network.target +Requires=postgresql.service + +[Service] +RestartSec=2s +Type=simple +User=woodpecker +Group=woodpecker +ExecStart=/usr/local/bin/woodpecker-server +Restart=always +% for k, v in sorted(env.items()): +Environment=${k}=${v} +% endfor + +[Install] +WantedBy=multi-user.target diff --git a/bundles/woodpecker-server/items.py b/bundles/woodpecker-server/items.py new file mode 100644 index 0000000..cccbb8c --- /dev/null +++ b/bundles/woodpecker-server/items.py @@ -0,0 +1,35 @@ +version = node.metadata.get('woodpecker-server/version') + +actions['install_woodpecker-server'] = { + 'command': ' && '.join([ + f'wget -q -O/tmp/woodpecker-server.deb https://github.com/woodpecker-ci/woodpecker/releases/download/v{version}/woodpecker-server_{version}_amd64.deb', + 'dpkg -i /tmp/woodpecker-server.deb', + ]), + 'unless': f'''bash -c "[[ \"$(woodpecker-server --version | cut -d' ' -f3)\" == "{version}" ]]"''', + 'triggers': { + 'svc_systemd:woodpecker-server:restart', + }, +} + +files['/usr/local/lib/systemd/system/woodpecker-server.service'] = { + 'content_type': 'mako', + 'context': { + 'env': node.metadata.get('woodpecker-server/environment'), + }, + 'triggers': { + 'action:systemd-reload', + 'svc_systemd:woodpecker-server:restart', + }, +} + +svc_systemd['woodpecker-server'] = { + 'needs': { + 'action:install_woodpecker-server', + 'file:/usr/local/lib/systemd/system/woodpecker-server.service', + 'postgres_db:woodpecker', + 'postgres_role:woodpecker', + 'user:woodpecker', + }, +} + +users['woodpecker'] = {} diff --git a/bundles/woodpecker-server/metadata.py b/bundles/woodpecker-server/metadata.py new file mode 100644 index 0000000..b98c89a --- /dev/null +++ b/bundles/woodpecker-server/metadata.py @@ -0,0 +1,94 @@ +from bundlewrap.metadata import atomic + +defaults = { + 'postgresql': { + 'roles': { + 'woodpecker': { + 'password': repo.vault.password_for(f'{node.name} postgresql woodpecker'), + }, + }, + 'databases': { + 'woodpecker': { + 'owner': 'woodpecker', + }, + }, + }, + 'woodpecker-server': { + 'environment': { + 'WOODPECKER_AGENT_SECRET': repo.vault.password_for(f'{node.name} WOODPECKER_AGENT_SECRET'), + 'WOODPECKER_DATABASE_DATASOURCE': repo.vault.password_for(f'{node.name} postgresql woodpecker').format_into( + 'postgres://woodpecker:{}@localhost/woodpecker?sslmode=disable' + ), + 'WOODPECKER_DATABASE_DRIVER': 'postgres', + 'WOODPECKER_GRPC_ADDR': ':22101', + 'WOODPECKER_LOG_LEVEL': 'warn', + 'WOODPECKER_OPEN': 'true', + 'WOODPECKER_SERVER_ADDR': ':22100', + }, + }, +} + + +@metadata_reactor.provides( + 'nginx/vhosts/woodpecker-server', + 'woodpecker-server/environment/WOODPECKER_HOST', +) +def nginx(metadata): + if not node.has_bundle('nginx'): + raise DoNotRunAgain + + ssl = metadata.get('nginx/vhosts/woodpecker-server/ssl', 'letsencrypt') + domain = metadata.get('woodpecker-server/domain') + prefix = 'https' if ssl else 'http' + + return { + 'nginx': { + 'vhosts': { + 'woodpecker-server': { + 'domain': domain, + 'locations': { + '/': { + 'target': 'http://127.0.0.1:22100', + 'additional_config': { + 'proxy_redirect off', + 'chunked_transfer_encoding off', + }, + }, + '/metrics': { + 'return': 403, + }, + '/debug': { + 'return': 403, + }, + }, + 'website_check_path': '/do-login', + 'website_check_string': 'Woodpecker', + }, + }, + }, + 'woodpecker-server': { + 'environment': { + 'WOODPECKER_HOST': f'{prefix}://{domain}', + }, + }, + } + + +@metadata_reactor.provides( + 'firewall/port_rules', +) +def firewall(metadata): + port = metadata.get('woodpecker-server/environment/WOODPECKER_GRPC_ADDR')[1:] + agents = set() + + for node in repo.nodes: + if node.has_bundle('woodpecker-agent'): + agents.add(node.name) + + return { + 'firewall': { + 'port_rules': { + port: atomic(agents), + }, + }, + } diff --git a/nodes/rx300.py b/nodes/rx300.py index f5e1c71..1a6ed40 100644 --- a/nodes/rx300.py +++ b/nodes/rx300.py @@ -35,6 +35,7 @@ nodes['rx300'] = { 'travelynx', 'unbound', 'vmhost', + 'woodpecker-server', 'zfs', }, 'groups': { @@ -334,6 +335,7 @@ nodes['rx300'] = { 'netbox': {'ssl': '_.franzi.business'}, 'radicale': {'ssl': '_.franzi.business'}, 'travelynx': {'ssl': '_.franzi.business'}, + 'woodpecker-server': {'ssl': '_.franzi.business'}, 'daskritzelt-redirect': { 'domain': 'die-brontosaurier-waren-es.org', 'ssl': None, @@ -535,6 +537,16 @@ nodes['rx300'] = { 'enable_linger': True, }, }, + 'woodpecker-server': { + 'domain': 'woodpecker.franzi.business', + 'version': '0.15.5', + 'environment': { + 'WOODPECKER_GITEA': 'true', + 'WOODPECKER_GITEA_URL': 'https://git.franzi.business', + 'WOODPECKER_GITEA_CLIENT': vault.decrypt('encrypt$gAAAAABjpJJQkNyG2B2ThT5yrkGnrPoM33bVYNTyLcuaas4_7ewBRrDb-KO2-JIM895fdI6U6NO8wHQ3gKBxBBYUtt-xgbWW1j4iUrzyt7KhqswSNBIBFfce80UmQ5UuOHsaFPVyyd1W'), + 'WOODPECKER_GITEA_SECRET': vault.decrypt('encrypt$gAAAAABjpJJW95MaCPnK2ngkGf1DLBmV8Y_K6B0Dc8XBM4oN3sPHH54vFbKB1YLODepR-okpXUJGHxqlS7TkTlu4JylRINXiIh7OHRRDaTCkU_bfLSUDnc_VLgDmVULWH09fsveslKw5v1ssl-RBGJg16XXBz1Sq4g=='), + }, + }, 'zfs': { 'module_options': { 'zfs_arc_max_gb': 16, -- 2.39.2 From c2e93c0abb7a1f61811b393a5de28937a2e59c25 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Fri, 23 Dec 2022 16:54:59 +0100 Subject: [PATCH 02/11] bundles/woodpecker: try to get it working --- .../files/woodpecker-server.service | 24 +++++++++++++++++++ bundles/woodpecker-server/items.py | 8 ++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/bundles/woodpecker-server/files/woodpecker-server.service b/bundles/woodpecker-server/files/woodpecker-server.service index 5520b49..3bd7b82 100644 --- a/bundles/woodpecker-server/files/woodpecker-server.service +++ b/bundles/woodpecker-server/files/woodpecker-server.service @@ -9,8 +9,32 @@ RestartSec=2s Type=simple User=woodpecker Group=woodpecker +WorkingDirectory=/var/lib/woodpecker ExecStart=/usr/local/bin/woodpecker-server Restart=always +ReadWritePaths=/var/lib/woodpecker +CapabilityBoundingSet= +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +PrivateUsers=true +ProtectHostname=true +ProtectClock=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +LockPersonality=true +MemoryDenyWriteExecute=true +RestrictRealtime=true +RestrictSUIDSGID=true +PrivateMounts=true +SystemCallArchitectures=native +SystemCallFilter=~@clock @cpu-emulation @debug @keyring @memlock @module @mount @obsolete @raw-io @reboot @setuid @swap + % for k, v in sorted(env.items()): Environment=${k}=${v} % endfor diff --git a/bundles/woodpecker-server/items.py b/bundles/woodpecker-server/items.py index cccbb8c..eb98fe9 100644 --- a/bundles/woodpecker-server/items.py +++ b/bundles/woodpecker-server/items.py @@ -1,5 +1,9 @@ version = node.metadata.get('woodpecker-server/version') +directories['/var/lib/woodpecker'] = { + 'owner': 'woodpecker', +} + actions['install_woodpecker-server'] = { 'command': ' && '.join([ f'wget -q -O/tmp/woodpecker-server.deb https://github.com/woodpecker-ci/woodpecker/releases/download/v{version}/woodpecker-server_{version}_amd64.deb', @@ -32,4 +36,6 @@ svc_systemd['woodpecker-server'] = { }, } -users['woodpecker'] = {} +users['woodpecker'] = { + 'home': '/var/lib/woodpecker', +} -- 2.39.2 From eee786fabf45ff6ffb2f6d32b675860fa84b70cc Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 24 Dec 2022 08:44:40 +0100 Subject: [PATCH 03/11] bundles/woodpecker-server: add GODEBUG=netns=go --- bundles/woodpecker-server/metadata.py | 4 ++++ nodes/rx300.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bundles/woodpecker-server/metadata.py b/bundles/woodpecker-server/metadata.py index b98c89a..257a307 100644 --- a/bundles/woodpecker-server/metadata.py +++ b/bundles/woodpecker-server/metadata.py @@ -24,6 +24,10 @@ defaults = { 'WOODPECKER_LOG_LEVEL': 'warn', 'WOODPECKER_OPEN': 'true', 'WOODPECKER_SERVER_ADDR': ':22100', + + # https://github.com/woodpecker-ci/woodpecker/issues/1497 + # https://github.com/woodpecker-ci/woodpecker/issues/748 + 'GODEBUG': 'netdns=go' }, }, } diff --git a/nodes/rx300.py b/nodes/rx300.py index 1a6ed40..eea38a1 100644 --- a/nodes/rx300.py +++ b/nodes/rx300.py @@ -539,7 +539,7 @@ nodes['rx300'] = { }, 'woodpecker-server': { 'domain': 'woodpecker.franzi.business', - 'version': '0.15.5', + 'version': '0.15.6', 'environment': { 'WOODPECKER_GITEA': 'true', 'WOODPECKER_GITEA_URL': 'https://git.franzi.business', -- 2.39.2 From 019cc693713f5f0fb0e9a8a841ac10baba6365d6 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 24 Dec 2022 17:40:13 +0100 Subject: [PATCH 04/11] add bundle:docker-ce --- bundles/docker-ce/items.py | 11 ++++++ bundles/docker-ce/metadata.py | 15 ++++++++ bundles/nftables/metadata.py | 2 +- data/apt/files/gpg-keys/docker.asc | 62 ++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 bundles/docker-ce/items.py create mode 100644 bundles/docker-ce/metadata.py create mode 100644 data/apt/files/gpg-keys/docker.asc diff --git a/bundles/docker-ce/items.py b/bundles/docker-ce/items.py new file mode 100644 index 0000000..bf56b1c --- /dev/null +++ b/bundles/docker-ce/items.py @@ -0,0 +1,11 @@ +from bundlewrap.metadata import metadata_to_json + +files['/etc/docker/daemon.json'] = { + 'content': metadata_to_json({ + 'iptables': False, + }), + 'before': { + 'pkg_apt:docker-ce', + 'pkg_apt:docker-ce-cli', + } +} diff --git a/bundles/docker-ce/metadata.py b/bundles/docker-ce/metadata.py new file mode 100644 index 0000000..a7d0c98 --- /dev/null +++ b/bundles/docker-ce/metadata.py @@ -0,0 +1,15 @@ +defaults = { + 'apt': { + 'repos': { + 'docker': { + 'items': { + 'deb https://download.docker.com/linux/debian {os_release} stable', + }, + }, + }, + 'packages': { + 'docker-ce': {}, + 'docker-ce-cli': {}, + }, + }, +} diff --git a/bundles/nftables/metadata.py b/bundles/nftables/metadata.py index 08396ce..06faaf0 100644 --- a/bundles/nftables/metadata.py +++ b/bundles/nftables/metadata.py @@ -25,7 +25,7 @@ defaults = { }, } -if not node.has_bundle('vmhost'): +if not node.has_bundle('vmhost') and not node.has_bundle('docker-ce'): # see comment in bundles/vmhost/items.py defaults['apt']['packages']['iptables'] = { 'installed': False, diff --git a/data/apt/files/gpg-keys/docker.asc b/data/apt/files/gpg-keys/docker.asc new file mode 100644 index 0000000..ee7872e --- /dev/null +++ b/data/apt/files/gpg-keys/docker.asc @@ -0,0 +1,62 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth +lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh +38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq +L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 +UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N +cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht +ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo +vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD +G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ +XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj +q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB +tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 +BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO +v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd +tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk +jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m +6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P +XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc +FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 +g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm +ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh +9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 +G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW +FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB +EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF +M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx +Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu +w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk +z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 +eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb +VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa +1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X +zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ +pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 +ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ +BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY +1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp +YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI +mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES +KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 +JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ +cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 +6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 +U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z +VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f +irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk +SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz +QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W +9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw +24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe +dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y +Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR +H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh +/nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ +M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S +xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O +jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG +YT90qFF93M3v01BbxP+EIY2/9tiIPbrd +=0YYh +-----END PGP PUBLIC KEY BLOCK----- -- 2.39.2 From 24f9f87734762a0e619cb021fd5493ed307392ac Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 24 Dec 2022 17:41:27 +0100 Subject: [PATCH 05/11] add bundle:woodpecker-agent --- .../files/woodpecker-agent.service | 42 ++++++++++++++++++ bundles/woodpecker-agent/items.py | 43 +++++++++++++++++++ bundles/woodpecker-agent/metadata.py | 28 ++++++++++++ nodes/woodpecker-agent-1.toml | 24 +++++++++++ 4 files changed, 137 insertions(+) create mode 100644 bundles/woodpecker-agent/files/woodpecker-agent.service create mode 100644 bundles/woodpecker-agent/items.py create mode 100644 bundles/woodpecker-agent/metadata.py create mode 100644 nodes/woodpecker-agent-1.toml diff --git a/bundles/woodpecker-agent/files/woodpecker-agent.service b/bundles/woodpecker-agent/files/woodpecker-agent.service new file mode 100644 index 0000000..096a891 --- /dev/null +++ b/bundles/woodpecker-agent/files/woodpecker-agent.service @@ -0,0 +1,42 @@ +[Unit] +Description=woodpecker ci agent +After=syslog.target +After=network.target + +[Service] +RestartSec=2s +Type=simple +User=woodpecker +Group=woodpecker +WorkingDirectory=/var/lib/woodpecker +ExecStart=/usr/local/bin/woodpecker-agent +Restart=always +ReadWritePaths=/var/lib/woodpecker +CapabilityBoundingSet= +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +PrivateUsers=true +ProtectHostname=true +ProtectClock=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +LockPersonality=true +MemoryDenyWriteExecute=true +RestrictRealtime=true +RestrictSUIDSGID=true +PrivateMounts=true +SystemCallArchitectures=native +SystemCallFilter=~@clock @cpu-emulation @debug @keyring @memlock @module @mount @obsolete @raw-io @reboot @setuid @swap + +% for k, v in sorted(env.items()): +Environment=${k}=${v} +% endfor + +[Install] +WantedBy=multi-user.target diff --git a/bundles/woodpecker-agent/items.py b/bundles/woodpecker-agent/items.py new file mode 100644 index 0000000..d33df40 --- /dev/null +++ b/bundles/woodpecker-agent/items.py @@ -0,0 +1,43 @@ +version = node.metadata.get('woodpecker-agent/version') + +directories['/var/lib/woodpecker'] = { + 'owner': 'woodpecker', +} + +actions['install_woodpecker-agent'] = { + 'command': ' && '.join([ + f'wget -q -O/tmp/woodpecker-agent.deb https://github.com/woodpecker-ci/woodpecker/releases/download/v{version}/woodpecker-agent_{version}_amd64.deb', + 'dpkg -i /tmp/woodpecker-agent.deb', + ]), + 'unless': f'''bash -c "[[ \"$(woodpecker-agent --version | cut -d' ' -f3)\" == "{version}" ]]"''', + 'triggers': {i + 'svc_systemd:woodpecker-agent:restart', + }, +} + +files['/usr/local/lib/systemd/system/woodpecker-agent.service'] = { + 'content_type': 'mako', + 'context': { + 'env': node.metadata.get('woodpecker-agent/environment'), + }, + 'triggers': { + 'action:systemd-reload', + 'svc_systemd:woodpecker-agent:restart', + }, +} + +svc_systemd['woodpecker-agent'] = { + 'after': { + # to make sure we have docker and other eventual dependencies + 'pkg_apt:', + }, + 'needs': { + 'action:install_woodpecker-agent', + 'file:/usr/local/lib/systemd/system/woodpecker-agent.service', + 'user:woodpecker', + }, +} + +users['woodpecker'] = { + 'home': '/var/lib/woodpecker', +} diff --git a/bundles/woodpecker-agent/metadata.py b/bundles/woodpecker-agent/metadata.py new file mode 100644 index 0000000..7a78beb --- /dev/null +++ b/bundles/woodpecker-agent/metadata.py @@ -0,0 +1,28 @@ +@metadata_reactor.provides( + 'woodpecker-agent/environment', + 'woodpecker-agent/version', +) +def nginx(metadata): + env = {} + server = repo.get_node(metadata.get('woodpecker-agent/server')) + + domain = server.metadata.get('woodpecker-server/domain') + port = server.metadata.get('woodpecker-server/environment/WOODPECKER_GRPC_ADDR') + env['WOODPECKER_SERVER'] = f'{domain}{port}' + + env['WOODPECKER_AGENT_SECRET'] = server.metadata.get('woodpecker-server/environment/WOODPECKER_AGENT_SECRET') + + env['WOODPECKER_MAX_PROCS'] = int(int(metadata.get('vm/cpu'))/2) + + env['WOODPECKER_HOSTNAME'] = metadata.get('hostname') + + debug = server.metadata.get('woodpecker-server/environment/GODEBUG', None) + if debug: + env['GODEBUG'] = debug + + return { + 'woodpecker-agent': { + 'environment': env, + 'version': server.metadata.get('woodpecker-server/version'), + }, + } diff --git a/nodes/woodpecker-agent-1.toml b/nodes/woodpecker-agent-1.toml new file mode 100644 index 0000000..d2d6c60 --- /dev/null +++ b/nodes/woodpecker-agent-1.toml @@ -0,0 +1,24 @@ +hostname = "31.47.232.108" +bundles = [ + "docker-ce", + "woodpecker-agent", +] +groups = ["debian-bullseye"] + +[metadata.backups] +exclude_from_backups = true + +[metadata.interfaces.enp1s0] +ips = [ + "31.47.232.108/29", + "2a00:f820:528::5/64", +] +gateway4 = "31.47.232.105" +gateway6 = "2a00:f820:528::1" + +[metadata.woodpecker-agent] +server = "rx300" + +[metadata.vm] +cpu = 8 +ram = 16 -- 2.39.2 From 9b44bcf3a8aefdcd8420b74676b48e4cf7352e03 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 24 Dec 2022 17:58:06 +0100 Subject: [PATCH 06/11] try running the test pipeline in woodpecker --- .woodpecker/bw-test.yml | 23 +++++++++++++++++++++++ .woodpecker/editorconfig.yml | 8 ++++++++ 2 files changed, 31 insertions(+) create mode 100644 .woodpecker/bw-test.yml create mode 100644 .woodpecker/editorconfig.yml diff --git a/.woodpecker/bw-test.yml b/.woodpecker/bw-test.yml new file mode 100644 index 0000000..efa4e3d --- /dev/null +++ b/.woodpecker/bw-test.yml @@ -0,0 +1,23 @@ +pipeline: + install-deps: + image: python:3.10-slim + commands: + - pip install -r requirements.txt + + test-dummymode: + image: python:3.10-slim + commands: + - bw test + environment: + BW_VAULT_DUMMY_MODE: 1 + BW_PASS_DUMMY_MODE: 1 + + test-ignore-missing-faults: + image: python:3.10-slim + commands: + - bw test --ignore-missing-faults + + test-determinism: + image: python:3.10-slim + commands: + - bw test --metadata-determinism 3 --config-determinism 3 diff --git a/.woodpecker/editorconfig.yml b/.woodpecker/editorconfig.yml new file mode 100644 index 0000000..6bac4f8 --- /dev/null +++ b/.woodpecker/editorconfig.yml @@ -0,0 +1,8 @@ +pipeline: + editorconfig: + image: alpine:latest + pipeline: + - wget -O ec-linux-amd64.tar.gz https://github.com/editorconfig-checker/editorconfig-checker/releases/latest/download/ec-linux-amd64.tar.gz + - tar -xzf ec-linux-amd64.tar.gz + - rm ec-linux-amd64.tar.gz + - bin/ec-linux-amd64 -no-color -exclude '^bin/' -- 2.39.2 From d2caadb41b0fbb8c5234146bcf1a3cf89915e551 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 24 Dec 2022 17:59:38 +0100 Subject: [PATCH 07/11] ci: determinism tests need to run using dummy mode --- .woodpecker/bw-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.woodpecker/bw-test.yml b/.woodpecker/bw-test.yml index efa4e3d..5990f05 100644 --- a/.woodpecker/bw-test.yml +++ b/.woodpecker/bw-test.yml @@ -21,3 +21,6 @@ pipeline: image: python:3.10-slim commands: - bw test --metadata-determinism 3 --config-determinism 3 + environment: + BW_VAULT_DUMMY_MODE: 1 + BW_PASS_DUMMY_MODE: 1 -- 2.39.2 From efdff6ef283ac41cfd602cba3e8735aac0c30646 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 24 Dec 2022 18:05:35 +0100 Subject: [PATCH 08/11] ci: fix editorconfig --- .woodpecker/editorconfig.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/editorconfig.yml b/.woodpecker/editorconfig.yml index 6bac4f8..201004f 100644 --- a/.woodpecker/editorconfig.yml +++ b/.woodpecker/editorconfig.yml @@ -1,7 +1,7 @@ pipeline: editorconfig: image: alpine:latest - pipeline: + commands: - wget -O ec-linux-amd64.tar.gz https://github.com/editorconfig-checker/editorconfig-checker/releases/latest/download/ec-linux-amd64.tar.gz - tar -xzf ec-linux-amd64.tar.gz - rm ec-linux-amd64.tar.gz -- 2.39.2 From 071250d798afefc870b487c7e4b4f4dbc5ad1cae Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 24 Dec 2022 18:22:29 +0100 Subject: [PATCH 09/11] bundles/docker-ce: add nftables rules --- bundles/docker-ce/metadata.py | 26 ++++++++++++++++++++++++++ bundles/woodpecker-agent/items.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/bundles/docker-ce/metadata.py b/bundles/docker-ce/metadata.py index a7d0c98..1315d1c 100644 --- a/bundles/docker-ce/metadata.py +++ b/bundles/docker-ce/metadata.py @@ -12,4 +12,30 @@ defaults = { 'docker-ce-cli': {}, }, }, + 'nftables': { + 'rules': { + '00-docker-ce': { + 'inet filter forward ct state { related, established } accept', + 'inet filter forward iifname docker0 accept', + }, + }, + }, } + + +@metadata_reactor.provides( + 'nftables/rules/00-docker-ce', +) +def nftables_nat(metadata): + rules = set() + + for iface in metadata.get('interfaces'): + rules.add(f'nat postrouting oifname {iface} masquerade') + + return { + 'nftables': { + 'rules': { + '00-docker-ce': rules, + }, + }, + } diff --git a/bundles/woodpecker-agent/items.py b/bundles/woodpecker-agent/items.py index d33df40..01e30e4 100644 --- a/bundles/woodpecker-agent/items.py +++ b/bundles/woodpecker-agent/items.py @@ -10,7 +10,7 @@ actions['install_woodpecker-agent'] = { 'dpkg -i /tmp/woodpecker-agent.deb', ]), 'unless': f'''bash -c "[[ \"$(woodpecker-agent --version | cut -d' ' -f3)\" == "{version}" ]]"''', - 'triggers': {i + 'triggers': { 'svc_systemd:woodpecker-agent:restart', }, } -- 2.39.2 From cb4d28c994b8f0813ba23ea23997ee8d3c084d63 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sun, 25 Dec 2022 08:25:00 +0100 Subject: [PATCH 10/11] bundles/woodpecker-agent: fix metadata reactor --- bundles/woodpecker-agent/metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bundles/woodpecker-agent/metadata.py b/bundles/woodpecker-agent/metadata.py index 7a78beb..5d2ff88 100644 --- a/bundles/woodpecker-agent/metadata.py +++ b/bundles/woodpecker-agent/metadata.py @@ -2,7 +2,7 @@ 'woodpecker-agent/environment', 'woodpecker-agent/version', ) -def nginx(metadata): +def environment(metadata): env = {} server = repo.get_node(metadata.get('woodpecker-agent/server')) @@ -16,6 +16,8 @@ def nginx(metadata): env['WOODPECKER_HOSTNAME'] = metadata.get('hostname') + env['WOODPECKER_LOG_LEVEL'] = server.metadata.get('woodpecker-server/environment/WOODPECKER_LOG_LEVEL') + debug = server.metadata.get('woodpecker-server/environment/GODEBUG', None) if debug: env['GODEBUG'] = debug -- 2.39.2 From d282d77a99877e87804eaf2a16dcf11760667e55 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Mon, 26 Dec 2022 12:57:42 +0100 Subject: [PATCH 11/11] bundles/docker-ce: sort nftables rules --- bundles/docker-ce/metadata.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/bundles/docker-ce/metadata.py b/bundles/docker-ce/metadata.py index 1315d1c..cf6e2bb 100644 --- a/bundles/docker-ce/metadata.py +++ b/bundles/docker-ce/metadata.py @@ -12,14 +12,6 @@ defaults = { 'docker-ce-cli': {}, }, }, - 'nftables': { - 'rules': { - '00-docker-ce': { - 'inet filter forward ct state { related, established } accept', - 'inet filter forward iifname docker0 accept', - }, - }, - }, } @@ -27,7 +19,10 @@ defaults = { 'nftables/rules/00-docker-ce', ) def nftables_nat(metadata): - rules = set() + rules = { + 'inet filter forward ct state { related, established } accept', + 'inet filter forward iifname docker0 accept', + } for iface in metadata.get('interfaces'): rules.add(f'nat postrouting oifname {iface} masquerade') @@ -35,7 +30,7 @@ def nftables_nat(metadata): return { 'nftables': { 'rules': { - '00-docker-ce': rules, + '00-docker-ce': sorted(rules), }, }, } -- 2.39.2