kunsi/bundlewrap
1
0
Fork 0
Code Issues 1 Pull requests 1 Releases Activity

update to bw4

Browse source
This commit is contained in:
Franzi 2020-08-18 15:27:55 +02:00
parent d7862918a6
commit 5e2fea8497
Signed by: kunsi
GPG key ID: 12E3D2136B818350
22 changed files with 223 additions and 501 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
bundles/bind/metadata.py
View file

@ -1,52 +1,33 @@
from bundlewrap.metadata import atomic from bundlewrap.metadata import atomic
@metadata_processor defaults = {
def backups(metadata): 'icinga2_api': {
if metadata.get('bind', {}).get('zones_primary_dynamic', {}): 'bind': {
metadata.setdefault('backups', {}).setdefault('paths', set()).add( 'services': {
'/var/lib/bind/primary.dynamic', 'BIND PROCESS': {
) 'command_on_monitored_host': '/usr/lib/nagios/plugins/check_procs -C named -c 1:1',
return metadata, RUN_ME_AGAIN },
@metadata_processor
def monitoring(metadata):
icinga2_api = metadata.setdefault('icinga2_api', {})
node_metadata = icinga2_api.setdefault('bind', {})
services = node_metadata.setdefault('services', {})
services.setdefault('BIND PROCESS', {}).update({
'check_command': 'nrpe',
'vars.nrpe_command': 'check_bind_procs',
})
for interface in metadata.get('bind', {}).get('listen', []):
services.setdefault('BIND PORT {}'.format(interface), {}).update({
'check_command': 'tcp',
'vars.tcp_address': metadata['interfaces'][interface]['ip_addresses'][0],
'vars.tcp_port': 53,
})
nrpe_checks = metadata.setdefault('nrpe', {}).setdefault('custom_nrpe_checks', {})
nrpe_checks['check_bind_procs'] = '/usr/lib/nagios/plugins/check_procs -C named -c 1:1'
return metadata, DONE
@metadata_processor
def sperrfix(metadata):
per_bundle = metadata.get('bind', {}).get('sperrfix', {})
if per_bundle.get('ignore'):
return metadata, DONE
sources = per_bundle.get('sources', {'*'})
return {
'sperrfix': {
'bundle_rules': {
'53': atomic({'sources': sources}),
'53/udp': atomic({'sources': sources}),
}, },
}, },
}, OVERWRITE, RUN_ME_AGAIN },
}
@metadata_reactor
def port_checks(metadata):
services = {}
for interface in metadata.get('bind/listen', set()):
services[f'BIND PORT {interface}'] = {
'check_command': 'tcp',
'vars.tcp_address': metadata.get(f'interfaces/{interface}/ip_addresses')[0],
'vars.tcp_port': 53,
}
return {
'icinga2_api': {
'bind': {
'services': services,
},
},
}

42
bundles/jenkins-ci/metadata.py
View file

@ -1,27 +1,25 @@
@metadata_processor defaults = {
def jenkins_apt_repos(metadata): 'apt': {
return { 'repos': {
'apt': { 'jenkins': {
'repos': { 'key': '150FDE3F7787E7D11EF4E12A9B7D32F2D50582E6',
'jenkins': { 'items': [
'key': '150FDE3F7787E7D11EF4E12A9B7D32F2D50582E6', 'deb https://pkg.jenkins.io/debian-stable binary/',
'items': [ ],
'deb https://pkg.jenkins.io/debian-stable binary/',
],
},
}, },
'unattended-upgrades': { },
'origins': { 'unattended-upgrades': {
'o=jenkins.io,a=binary', 'origins': {
}, 'o=jenkins.io,a=binary',
}, },
'packages': { },
'openjdk-11-jre': {}, 'packages': {
'jenkins': { 'openjdk-11-jre': {},
'needs': { 'jenkins': {
'pkg_apt:openjdk-11-jre', 'needs': {
}, 'pkg_apt:openjdk-11-jre',
}, },
}, },
}, },
}, DEFAULTS, DONE },
}

14
bundles/letsencrypt/metadata.py
View file

@ -1,8 +1,6 @@
@metadata_processor defaults = {
def crontab(metadata): 'cron': {
return { 'letsencrypt_renew': '20 4 * * * root /usr/bin/dehydrated --cron --accept-terms --challenge http-01 > /dev/null',
'cron': { 'letsencrypt_cleanup': '42 23 * * 0 root /usr/bin/dehydrated --cleanup > /dev/null',
'letsencrypt_renew': '20 4 * * * root /usr/bin/dehydrated --cron --accept-terms --challenge http-01 > /dev/null', },
'letsencrypt_cleanup': '42 23 * * 0 root /usr/bin/dehydrated --cleanup > /dev/null', }
},
}, DEFAULTS, DONE

75
bundles/matrix-synapse/metadata.py
View file

@ -1,47 +1,40 @@
@metadata_processor defaults = {
def nodejs_apt_repos(metadata): 'apt': {
return { 'repos': {
'apt': { 'matrix': {
'repos': { 'key': 'AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058',
'matrix': { 'items': [
'key': 'AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058', 'deb https://packages.matrix.org/debian buster main',
'items': [ ],
'deb https://packages.matrix.org/debian buster main',
],
},
},
'unattended-upgrades': {
'origins': {
'o=matrix.org,n=buster,c=main',
},
},
'packages': {
'matrix-synapse-py3': {},
}, },
}, },
}, DEFAULTS, DONE 'unattended-upgrades': {
'origins': {
@metadata_processor 'o=matrix.org,n=buster,c=main',
def synapse_defaults(metadata): },
return { },
'matrix-synapse': { 'packages': {
'registration_shared_secret': repo.vault.human_password_for('{} matrix-synapse registration_shared_secret'.format(node.name)), 'matrix-synapse-py3': {},
'database': { },
'user': 'synapse_user', },
'matrix-synapse': {
'registration_shared_secret': repo.vault.human_password_for('{} matrix-synapse registration_shared_secret'.format(node.name)),
'database': {
'user': 'synapse_user',
'password': repo.vault.password_for('{} postgresql synapse_user'.format(node.name)),
'database': 'synapse',
},
},
'postgresql': {
'users': {
'synapse_user': {
'password': repo.vault.password_for('{} postgresql synapse_user'.format(node.name)), 'password': repo.vault.password_for('{} postgresql synapse_user'.format(node.name)),
'database': 'synapse',
}, },
}, },
'postgresql': { 'databases': {
'users': { 'synapse': {
'synapse_user': { 'owner': 'synapse_user',
'password': repo.vault.password_for('{} postgresql synapse_user'.format(node.name)),
},
}, },
'databases': { },
'synapse': { }
'owner': 'synapse_user', }
},
},
}
}, DEFAULTS, DONE

3
bundles/mx-puppet-discord/items.py
View file

@ -31,9 +31,6 @@ git_deploy = {
'/opt/mx-puppet-discord': { '/opt/mx-puppet-discord': {
'repo': 'https://github.com/matrix-discord/mx-puppet-discord.git', 'repo': 'https://github.com/matrix-discord/mx-puppet-discord.git',
'rev': 'master', 'rev': 'master',
'needs': {
'directory:/opt/mx-puppet-discord',
},
'triggers': { 'triggers': {
'action:mx-puppet-discord_chown', 'action:mx-puppet-discord_chown',
'action:mx-puppet-discord_npm_install', 'action:mx-puppet-discord_npm_install',

65
bundles/mx-puppet-discord/metadata.py
View file

@ -1,46 +1,33 @@
@metadata_processor defaults = {
def mx_puppet_discord_user(metadata): 'users': {
return { 'mx-puppet-discord': {
'home': '/opt/mx-puppet-discord',
'deploy_configs': False,
'home-mode': '0755',
},
},
'matrix-synapse': {
'appservice_configs': {
'/opt/mx-puppet-discord/registration.yaml',
},
},
'mx-puppet-discord': {
'database': {
'user': 'mx-puppet-discord',
'password': repo.vault.password_for('{} postgresql mx-puppet-discord'.format(node.name)),
'database': 'mx-puppet-discord',
},
},
'postgres': {
'users': { 'users': {
'mx-puppet-discord': { 'mx-puppet-discord': {
'home': '/opt/mx-puppet-discord',
'deploy_configs': False,
'home-mode': '0755',
},
},
}, DEFAULTS, DONE
@metadata_processor
def add_mx_puppet_discord_to_synapse(metadata):
return {
'matrix-synapse': {
'appservice_configs': {
'/opt/mx-puppet-discord/registration.yaml',
},
},
}, DEFAULTS, DONE
@metadata_processor
def postgres(metadata):
return {
'mx-puppet-discord': {
'database': {
'user': 'mx-puppet-discord',
'password': repo.vault.password_for('{} postgresql mx-puppet-discord'.format(node.name)), 'password': repo.vault.password_for('{} postgresql mx-puppet-discord'.format(node.name)),
'database': 'mx-puppet-discord',
}, },
}, },
'postgres': { 'databases': {
'users': { 'mx-puppet-discord': {
'mx-puppet-discord': { 'owner': 'mx-puppet-discord',
'password': repo.vault.password_for('{} postgresql mx-puppet-discord'.format(node.name)),
},
},
'databases': {
'mx-puppet-discord': {
'owner': 'mx-puppet-discord',
},
}, },
}, },
}, DEFAULTS, DONE },
}

52
bundles/nginx/metadata.py
View file

@ -1,39 +1,37 @@
@metadata_processor defaults = {
def defaults(metadata): 'apt': {
return { 'repos': {
'apt': { 'nginx': {
'repos': { 'key': '573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62',
'nginx': { 'items': [
'key': '573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62', 'deb http://nginx.org/packages/debian buster nginx',
'items': [ ],
'deb http://nginx.org/packages/debian buster nginx',
],
},
},
'unattended-upgrades': {
'origins': {
'o=nginx,a=stable,n=buster,l=nginx,c=nginx',
},
},
'packages': {
'nginx': {},
}, },
}, },
'nginx': { 'unattended-upgrades': {
'worker_processes': 4, 'origins': {
'worker_connections': 1000, 'o=nginx,a=stable,n=buster,l=nginx,c=nginx',
},
}, },
}, DEFAULTS, DONE 'packages': {
'nginx': {},
},
},
'nginx': {
'worker_processes': 4,
'worker_connections': 1000,
},
}
@metadata_processor @metadata_reactor
def letsencrypt(metadata): def letsencrypt(metadata):
if not node.has_bundle('letsencrypt'): if not node.has_bundle('letsencrypt'):
return metadata, DONE raise DoNotRunAgain
domains = {} domains = {}
for domain in metadata.get('nginx', {}).get('vhosts', {}).keys(): for domain in metadata.get('nginx/vhosts', {}).keys():
domains[domain] = set() domains[domain] = set()
return { return {
@ -43,4 +41,4 @@ def letsencrypt(metadata):
'nginx', 'nginx',
}, },
}, },
}, DEFAULTS, RUN_ME_AGAIN }

54
bundles/nodejs/metadata.py
View file

@ -1,31 +1,29 @@
@metadata_processor defaults = {
def nodejs_apt_repos(metadata): 'apt': {
return { 'repos': {
'apt': { 'yarn': {
'repos': { 'key': '72ECF46A56B4AD39C907BBB71646B01B86E50310',
'yarn': { 'items': [
'key': '72ECF46A56B4AD39C907BBB71646B01B86E50310', 'deb https://dl.yarnpkg.com/debian/ stable main',
'items': [ ],
'deb https://dl.yarnpkg.com/debian/ stable main',
],
},
'node': {
'key': '9FD3B784BC1C6FC31A8A0A1C1655A0AB68576280',
'items': [
'deb https://deb.nodesource.com/node_10.x buster main',
'deb-src https://deb.nodesource.com/node_10.x buster main',
],
},
}, },
'unattended-upgrades': { 'node': {
'origins': { 'key': '9FD3B784BC1C6FC31A8A0A1C1655A0AB68576280',
'o=Node Source,n=buster,l=Node Source,c=main', 'items': [
'o=yarn,a=stable,n=stable,l=yarn-stable,c=main', 'deb https://deb.nodesource.com/node_10.x buster main',
}, 'deb-src https://deb.nodesource.com/node_10.x buster main',
}, ],
'packages': {
'nodejs': {},
'yarn': {},
}, },
}, },
}, DEFAULTS, DONE 'unattended-upgrades': {
'origins': {
'o=Node Source,n=buster,l=Node Source,c=main',
'o=yarn,a=stable,n=stable,l=yarn-stable,c=main',
},
},
'packages': {
'nodejs': {},
'yarn': {},
},
},
}

3
bundles/riot-web/items.py
View file

@ -10,9 +10,6 @@ directories = {
git_deploy = { git_deploy = {
riot_web_root: { riot_web_root: {
'needs': {
'directory:' + riot_web_root,
},
'rev': 'master', 'rev': 'master',
'repo': 'https://github.com/vector-im/riot-web.git', 'repo': 'https://github.com/vector-im/riot-web.git',
'triggers': { 'triggers': {

8
bundles/riot-web/metadata.py
View file

@ -1,12 +1,12 @@
@metadata_processor @metadata_reactor
def nginx_config(metadata): def nginx_config(metadata):
return { return {
'nginx': { 'nginx': {
'vhosts': { 'vhosts': {
metadata['riot-web']['url']: { metadata.get('riot-web/url', None): {
'webroot': '/var/www/chat.franzi.business/webapp/', 'webroot': '/var/www/{}/webapp/'.format(metadata.get('riot-web/url', None)),
'extras': True, 'extras': True,
}, },
}, },
}, },
}, DEFAULTS, DONE }

32
bundles/seafile/metadata.py
View file

@ -1,19 +1,17 @@
@metadata_processor defaults = {
def defaults(metadata): 'apt': {
return { 'packages': {
'apt': { 'mariadb-server': {},
'packages': { 'python3': {},
'mariadb-server': {}, 'python3-setuptools': {},
'python3': {}, 'python3-pip': {},
'python3-setuptools': {},
'python3-pip': {},
},
}, },
'users': { },
'seafile': { 'users': {
'home': '/opt/seafile', 'seafile': {
'deploy_configs': False, 'home': '/opt/seafile',
'home-mode': '0755', 'deploy_configs': False,
}, 'home-mode': '0755',
}, },
}, DEFAULTS, DONE },
}

2
bundles/sudo/files/sudoers
View file

@ -6,6 +6,6 @@ Defaults secure_path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bi
root ALL=(ALL) ALL root ALL=(ALL) ALL
% for user in node.metadata['sudo']: % for user in sorted(node.metadata['sudo']):
${user} ALL=(ALL) NOPASSWD:ALL ${user} ALL=(ALL) NOPASSWD:ALL
% endfor % endfor

12
bundles/sudo/metadata.py
View file

@ -1,11 +1,11 @@
@metadata_processor @metadata_reactor
def sudo_users(metadata): def sudo_users(metadata):
sudoers = [] sudoers = set()
for username, config in metadata.get('users', {}).items(): for username, config in metadata.get('users', {}).items():
if 'sudo' in config and config['sudo']: if 'sudo' in config and config['sudo']:
sudoers.append(username) sudoers.add(username)
metadata['sudo'] = sudoers return {
'sudo': sudoers,
return metadata, RUN_ME_AGAIN }

18
bundles/users/metadata.py
View file

@ -1,11 +1,9 @@
@metadata_processor defaults = {
def apt(metadata): 'apt': {
return { 'packages': {
'apt': { 'fish': {},
'packages': { 'tmux': {},
'fish': {}, 'vim': {},
'tmux': {},
'vim': {},
},
}, },
}, DEFAULTS, DONE },
}

18
bundles/vmhost/metadata.py
View file

@ -1,11 +1,9 @@
@metadata_processor defaults = {
def apt(metadata): 'apt': {
return { 'packages': {
'apt': { 'qemu-kvm': {},
'packages': { 'libvirt-clients': {},
'qemu-kvm': {}, 'libvirt-daemon-system': {},
'libvirt-clients': {},
'libvirt-daemon-system': {},
},
}, },
}, DEFAULTS, DONE },
}

22
bundles/voc-loudness-monitor/metadata.py
View file

@ -1,14 +1,12 @@
@metadata_processor defaults = {
def add_voc_user(metadata): 'apt': {
return { 'packages': {
'apt': { 'ffmpeg': {},
'packages': {
'ffmpeg': {},
},
}, },
'users': { },
'voc': { 'users': {
'home': '/opt/voc-loudness-monitor', 'voc': {
}, 'home': '/opt/voc-loudness-monitor',
}, },
}, DEFAULTS, DONE },
}

208
items/git_deploy.py
View file

@ -1,208 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from atexit import register as at_exit
from os import remove, setpgrp
from os.path import isfile, join
from pipes import quote
from shutil import rmtree
from subprocess import PIPE, Popen
from tempfile import mkdtemp, NamedTemporaryFile
from bundlewrap.exceptions import RepositoryError
from bundlewrap.items import Item
from bundlewrap.utils import cached_property
from bundlewrap.utils.text import mark_for_translation as _, randstr
from bundlewrap.utils.ui import io
REPO_MAP_FILENAME = "git_deploy_repos"
REMOTE_STATE_FILENAME = ".bundlewrap_git_deploy"
def is_ref(rev):
"""
Braindead check to see if our rev is a branch or tag name. False
negatives are OK since this is only used for optimization.
"""
for char in rev:
if char not in "0123456789abcdef":
return True
return False
def clone_to_dir(remote_url, rev):
"""
Clones the given URL to a temporary directory, using a shallow clone
if the given revision is definitely not a commit hash.
Returns the path to the directory.
"""
tmpdir = mkdtemp()
if is_ref(rev):
git_cmdline = ["clone", "--bare", "--depth", "1", "--no-single-branch", remote_url, "."]
else:
git_cmdline = ["clone", "--bare", remote_url, "."]
git_command(git_cmdline, tmpdir)
return tmpdir
def get_local_repo_path(bw_repo_path, repo_name):
"""
From the given BundleWrap repo, get the filesystem path to the git
repo associated with the given internal repo name.
"""
repo_map_path = join(bw_repo_path, REPO_MAP_FILENAME)
if not isfile(repo_map_path):
io.stderr(_("missing repo map for git_deploy at {}").format(repo_map_path))
io.stderr(_("you must create this file with the following format:"))
io.stderr(_(" <value of repo attribute on git_deploy item>: "
"<absolute path to local git repo>"))
io.stderr(_("since the path is local, you should also add the "
"{} file to your gitignore").format(REPO_MAP_FILENAME))
raise RepositoryError(_("missing repo map for git_deploy"))
with open(join(bw_repo_path, REPO_MAP_FILENAME)) as f:
repo_map = f.readlines()
for line in repo_map:
if not line.strip() or line.startswith("#"):
continue
try:
repo, path = line.split(":", 1)
except:
raise RepositoryError(_("unable to parse line from {path}: '{line}'").format(
line=line,
path=repo_map_path,
))
if repo_name == repo:
return path.strip()
raise RepositoryError(_("no path found for repo '{repo}' in {path}").format(
path=repo_map_path,
repo=repo_name,
))
def git_command(cmdline, repo_dir):
"""
Runs the given git command line in the given directory.
Returns stdout of the command.
"""
cmdline = ["git"] + cmdline
io.debug(_("running '{}' in {}").format(
" ".join(cmdline),
repo_dir,
))
git_process = Popen(
cmdline,
cwd=repo_dir,
preexec_fn=setpgrp,
stderr=PIPE,
stdout=PIPE,
)
stdout, stderr = git_process.communicate()
if git_process.returncode != 0:
io.stderr(_("failed command: {}").format(" ".join(cmdline)))
io.stderr(_("stdout:\n{}").format(stdout))
io.stderr(_("stderr:\n{}").format(stderr))
raise RuntimeError(_("`git {command}` failed in {dir}").format(
command=cmdline[1],
dir=repo_dir,
))
return stdout.decode('utf-8').strip()
class GitDeploy(Item):
"""
Facilitates deployment of a given rev from a local git repo to a
node.
"""
BUNDLE_ATTRIBUTE_NAME = "git_deploy"
ITEM_ATTRIBUTES = {
'repo': None,
'rev': None,
'use_xattrs': False,
}
ITEM_TYPE_NAME = "git_deploy"
REQUIRED_ATTRIBUTES = ['repo', 'rev']
def __repr__(self):
return "<GitDeploy path:{} repo:{} rev:{}>".format(
self.name,
self.attributes['repo'],
self.attributes['rev'],
)
@cached_property
def _expanded_rev(self):
git_cmdline = ["rev-parse", self.attributes['rev']]
return git_command(
git_cmdline,
self._repo_dir,
)
@cached_property
def _repo_dir(self):
if "://" in self.attributes['repo']:
repo_dir = clone_to_dir(self.attributes['repo'], self.attributes['rev'])
io.debug(_("registering {} for deletion on exit").format(repo_dir))
at_exit(rmtree, repo_dir)
else:
repo_dir = get_local_repo_path(self.node.repo.path, self.attributes['repo'])
return repo_dir
def cdict(self):
return {'rev': self._expanded_rev}
def fix(self, status):
archive_local = NamedTemporaryFile(delete=False)
try:
archive_local.close()
git_command(
["archive", "-o", archive_local.name, self._expanded_rev],
self._repo_dir,
)
temp_filename = ".bundlewrap_tmp_git_deploy_" + randstr()
try:
self.node.upload(
archive_local.name,
temp_filename,
)
self.node.run("find {} -mindepth 1 -delete".format(quote(self.name)))
self.node.run("tar -xf {} -C {}".format(temp_filename, quote(self.name)))
if self.attributes['use_xattrs']:
self.node.run("attr -q -s bw_git_deploy_rev -V {} {}".format(
self._expanded_rev,
quote(self.name),
))
else:
self.node.run("echo {} > {}".format(
self._expanded_rev,
quote(join(self.name, REMOTE_STATE_FILENAME)),
))
self.node.run("chmod 400 {}".format(
quote(join(self.name, REMOTE_STATE_FILENAME)),
))
finally:
self.node.run("rm -f {}".format(temp_filename))
finally:
remove(archive_local.name)
def sdict(self):
if self.attributes['use_xattrs']:
status_result = self.node.run(
"attr -q -g bw_git_deploy_rev {}".format(quote(self.name)),
may_fail=True,
)
else:
status_result = self.node.run(
"cat {}".format(quote(join(self.name, REMOTE_STATE_FILENAME))),
may_fail=True,
)
if status_result.return_code != 0:
return None
else:
return {'rev': status_result.stdout.decode('utf-8').strip()}

3
nodes/htz-cloud/pirmasens.py
View file

@ -1,6 +1,5 @@
nodes['htz-cloud.pirmasens'] = { nodes['htz-cloud.pirmasens'] = {
'bundles': { 'bundles': set(),
},
'groups': { 'groups': {
'webserver', 'webserver',
}, },

4
nodes/htz-cloud/sewfile.py
View file

@ -2,9 +2,9 @@
# managed by bundlewrap. # managed by bundlewrap.
nodes['htz-cloud.sewfile'] = { nodes['htz-cloud.sewfile'] = {
'bundles': [ 'bundles': {
'seafile', 'seafile',
], },
'groups': { 'groups': {
'webserver', 'webserver',
}, },

4
nodes/htz/ex42-1048908.py
View file

@ -1,5 +1,5 @@
nodes['htz.ex42-1048908'] = { nodes['htz.ex42-1048908'] = {
'bundles': [ 'bundles': {
'jenkins-ci', 'jenkins-ci',
'matrix-synapse', 'matrix-synapse',
'mx-puppet-discord', 'mx-puppet-discord',
@ -8,7 +8,7 @@ nodes['htz.ex42-1048908'] = {
'postgresql', 'postgresql',
'vmhost', 'vmhost',
'voc-loudness-monitor', 'voc-loudness-monitor',
], },
'groups': { 'groups': {
'webserver', 'webserver',
}, },

8
plugins.json
View file

@ -1,8 +0,0 @@
{
"item_git_deploy": {
"files": {
"items/git_deploy.py": "111c81fe88c3e97168251fccf59e1aeb36af3d02"
},
"version": 6
}
}

2
requirements.txt
View file

@ -1 +1 @@
bundlewrap<4.0.0 bundlewrap>=4.0.0