kunsi/bundlewrap
1
0
Fork 0
Code Issues 1 Pull requests 1 Releases Activity

rework netbox-dump script and routeros bundle for better usability

Browse source
This commit is contained in:
Franzi 2024-06-22 20:04:51 +02:00
parent c47b412cf3
commit d1f182607d
Signed by: kunsi
GPG key ID: 12E3D2136B818350
3 changed files with 275 additions and 191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
bundles/routeros/metadata.py
View file

@ -26,7 +26,7 @@ defaults = {
'routeros/vlans', 'routeros/vlans',
) )
def get_ports_from_netbox_dump(metadata): def get_ports_from_netbox_dump(metadata):
with open(join(repo.path, 'configs', f'netbox_device_{node.name}.json')) as f: with open(join(repo.path, 'configs', 'netbox', f'{node.name}.json')) as f:
netbox = load(f) netbox = load(f)
ips = {} ips = {}
@ -45,7 +45,7 @@ def get_ports_from_netbox_dump(metadata):
for ip in conf['ips']: for ip in conf['ips']:
ips[ip] = {'interface': port} ips[ip] = {'interface': port}
if conf['type'] == 'VIRTUAL': if conf['type'].lower() == 'virtual':
# these are VLAN interfaces (for management IPs) # these are VLAN interfaces (for management IPs)
if conf['ips']: if conf['ips']:
# this makes management services available in the VLAN # this makes management services available in the VLAN
@ -77,6 +77,8 @@ def get_ports_from_netbox_dump(metadata):
if conf.get('ips', []): if conf.get('ips', []):
ports[port]['ips'] = set(conf['ips']) ports[port]['ips'] = set(conf['ips'])
if conf['type'] in ( if conf['type'] in (
'1000base-t',
'10gbase-x-sfpp',
'A_1000BASE_T', 'A_1000BASE_T',
'A_10GBASE_X_SFPP', 'A_10GBASE_X_SFPP',
): ):
@ -90,7 +92,7 @@ def get_ports_from_netbox_dump(metadata):
# tagged # tagged
if conf['mode'] == 'TAGGED_ALL': if conf['mode'] in ('TAGGED_ALL', 'tagged-all'):
tagged = set(vlans.keys()) - {conf['untagged_vlan']} tagged = set(vlans.keys()) - {conf['untagged_vlan']}
else: else:
tagged = conf['tagged_vlans'] tagged = conf['tagged_vlans']

108
configs/netbox_device_home.switch-rack.json → configs/netbox/home.switch-rack.json
View file

@ -4,225 +4,225 @@
"description": "home.router (enp1s0)", "description": "home.router (enp1s0)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "TAGGED_ALL", "mode": "tagged-all",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": null "untagged_vlan": null
}, },
"ether10": { "ether10": {
"description": "home.mitel-rfp35 (LAN)", "description": "home.mitel-rfp35 (LAN)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether11": { "ether11": {
"description": "home.usv01 (LAN)", "description": "home.usv01 (LAN)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether12": { "ether12": {
"description": "home.rechenmonster (IPMI)", "description": "home.rechenmonster (IPMI)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether13": { "ether13": {
"description": "", "description": "",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether14": { "ether14": {
"description": "home.rechenmonster (LAN)", "description": "home.rechenmonster (LAN)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether15": { "ether15": {
"description": "", "description": "",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether16": { "ether16": {
"description": "", "description": "",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether17": { "ether17": {
"description": "", "description": "",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether18": { "ether18": {
"description": "", "description": "",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether19": { "ether19": {
"description": "home.lgtv-wohnzimmer", "description": "home.lgtv-wohnzimmer",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether2": { "ether2": {
"description": "Fritz!Box (LAN1)", "description": "Fritz!Box (LAN1)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.wan" "untagged_vlan": "home.wan"
}, },
"ether20": { "ether20": {
"description": "Franzi Laptop", "description": "Franzi Laptop",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether21": { "ether21": {
"description": "Sophie Laptop", "description": "Sophie Laptop",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether22": { "ether22": {
"description": "Sophie Desktop", "description": "Sophie Desktop",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether23": { "ether23": {
"description": "Wohnzimmer Kabel", "description": "Wohnzimmer Kabel",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether24": { "ether24": {
"description": "home.snom-wohnzimmer", "description": "home.snom-wohnzimmer",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether3": { "ether3": {
"description": "home.aruba325-schlafzimmer", "description": "home.aruba325-schlafzimmer",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "TAGGED", "mode": "tagged",
"tagged_vlans": [ "tagged_vlans": [
"ffwi.client", "ffwi.client",
"home.v6only" "home.v6only"
], ],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether4": { "ether4": {
"description": "home.aruba325-wohnzimmer", "description": "home.aruba325-wohnzimmer",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "TAGGED", "mode": "tagged",
"tagged_vlans": [ "tagged_vlans": [
"ffwi.client", "ffwi.client",
"home.v6only" "home.v6only"
], ],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether5": { "ether5": {
"description": "home.nas (eno1)", "description": "home.nas (eno1)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "TAGGED_ALL", "mode": "tagged-all",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": null "untagged_vlan": null
}, },
"ether6": { "ether6": {
"description": "home.aruba325-office", "description": "home.aruba325-office",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "TAGGED", "mode": "tagged",
"tagged_vlans": [ "tagged_vlans": [
"ffwi.client", "ffwi.client",
"home.v6only" "home.v6only"
], ],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether7": { "ether7": {
"description": "RIPE-Probe #28280 (LAN)", "description": "RIPE-Probe #28280 (LAN)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.dmz" "untagged_vlan": "home.dmz"
}, },
"ether8": { "ether8": {
"description": "home.drucker-sophie", "description": "home.drucker-sophie",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"ether9": { "ether9": {
"description": "info-beamer 12199 (LAN)", "description": "info-beamer 12199 (LAN)",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": "ACCESS", "mode": "access",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_1000BASE_T", "type": "1000base-t",
"untagged_vlan": "home.clients" "untagged_vlan": "home.clients"
}, },
"home.clients": { "home.clients": {
@ -231,27 +231,27 @@
"ips": [ "ips": [
"172.19.138.4/24" "172.19.138.4/24"
], ],
"mode": null, "mode": "",
"tagged_vlans": [], "tagged_vlans": [],
"type": "VIRTUAL", "type": "virtual",
"untagged_vlan": null "untagged_vlan": null
}, },
"sfp-sfpplus1": { "sfp-sfpplus1": {
"description": "", "description": "",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": null, "mode": "",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_10GBASE_X_SFPP", "type": "10gbase-x-sfpp",
"untagged_vlan": null "untagged_vlan": null
}, },
"sfp-sfpplus2": { "sfp-sfpplus2": {
"description": "", "description": "",
"enabled": true, "enabled": true,
"ips": [], "ips": [],
"mode": null, "mode": "",
"tagged_vlans": [], "tagged_vlans": [],
"type": "A_10GBASE_X_SFPP", "type": "10gbase-x-sfpp",
"untagged_vlan": null "untagged_vlan": null
} }
}, },

156
scripts/netbox-dump
View file

@ -1,24 +1,48 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from argparse import ArgumentParser
from json import dump from json import dump
from os import environ from os import environ, makedirs, remove, scandir
from os.path import dirname, join from os.path import abspath, dirname, join
from sys import exit from sys import exit
import bwpass import bwpass
from requests import post from requests import post
from bundlewrap.utils.text import validate_name from bundlewrap.utils.text import bold, red, validate_name
from bundlewrap.utils.ui import io
TOKEN = environ.get("NETBOX_AUTH_TOKEN") TOKEN = environ.get("NETBOX_AUTH_TOKEN")
if not TOKEN:
try:
TOKEN = bwpass.attr("netbox.franzi.business/kunsi", "token")
except Exception:
print("NETBOX_AUTH_TOKEN missing")
exit(1)
# editorconfig-checker-disable TARGET_PATH = join(dirname(dirname(abspath(__file__))), "configs", "netbox")
QUERY = """{
device_list(tag: "bundlewrap") { QUERY_SITES = """{
site_list {
name
id
vlans {
name
vid
}
}
}"""
QUERY_DEVICES = """{
device_list(filters: {tag: "bundlewrap", site_id: "SITE_ID"}) {
name name
site {
id id
} }
}"""
QUERY_DEVICE_DETAILS = """{
device(id: DEVICE_ID) {
name
interfaces { interfaces {
id id
name name
@ -59,23 +83,10 @@ QUERY = """{
} }
} }
} }
site_list {
id
vlans {
name
vid
}
}
}""" }"""
# editorconfig-checker-enable
if not TOKEN:
try:
TOKEN = bwpass.attr("netbox.franzi.business/kunsi", "token")
except Exception:
print("NETBOX_AUTH_TOKEN is missing")
exit(1)
def graphql(query):
r = post( r = post(
"https://netbox.franzi.business/graphql/", "https://netbox.franzi.business/graphql/",
headers={ headers={
@ -83,27 +94,68 @@ r = post(
"Authorization": f"Token {TOKEN}", "Authorization": f"Token {TOKEN}",
}, },
json={ json={
"query": QUERY, "query": query,
}, },
) )
r.raise_for_status() r.raise_for_status()
return r.json()["data"]
data = r.json()["data"]
site_vlans = {site["id"]: site["vlans"] for site in data["site_list"]} def filter_results(results, filter_by):
if filter_by is None:
return results
for device in data["device_list"]: out = []
for result in results:
if str(result["id"]) in filter_by or result["name"] in filter_by:
out.append(result)
return out
parser = ArgumentParser()
parser.add_argument("--only-site", nargs="+", type=str)
parser.add_argument("--only-device", nargs="+", type=str)
args = parser.parse_args()
try:
io.activate()
filenames_used = set()
with io.job("getting sites"):
sites = filter_results(
graphql(QUERY_SITES).get("site_list", []), args.only_site
)
io.stdout(f"Processing {len(sites)} sites in total")
for site in sites:
with io.job(f"{bold(site['name'])} getting devices"):
devices = filter_results(
graphql(QUERY_DEVICES.replace("SITE_ID", site["id"])).get(
"device_list", []
),
args.only_device,
)
io.stdout(f"Site {bold(site['name'])} has {len(devices)} devices to process")
for device in devices:
if not device["name"] or not validate_name(device["name"]): if not device["name"] or not validate_name(device["name"]):
# invalid node name, ignore # invalid node name, ignore
continue continue
with io.job(
f"{bold(site['name'])} {bold(device['name'])} getting interfaces"
):
details = graphql(
QUERY_DEVICE_DETAILS.replace("DEVICE_ID", device["id"])
)["device"]
result = { result = {
"interfaces": {}, "interfaces": {},
"vlans": site_vlans[device["site"]["id"]], "vlans": site["vlans"],
} }
for interface in device["interfaces"]: for interface in details["interfaces"]:
description = ""
peers = None peers = None
if interface["connected_endpoints"]: if interface["connected_endpoints"]:
@ -135,19 +187,29 @@ for device in data["device_list"]:
"enabled": interface["enabled"], "enabled": interface["enabled"],
"mode": interface["mode"], "mode": interface["mode"],
"type": interface["type"], "type": interface["type"],
"ips": sorted({i['address'] for i in interface['ip_addresses']}), "ips": sorted(
"untagged_vlan": interface["untagged_vlan"]["name"] {i["address"] for i in interface["ip_addresses"]}
),
"untagged_vlan": (
interface["untagged_vlan"]["name"]
if interface["untagged_vlan"] if interface["untagged_vlan"]
else None, else None
"tagged_vlans": sorted({v["name"] for v in interface["tagged_vlans"]}), ),
"tagged_vlans": sorted(
{v["name"] for v in interface["tagged_vlans"]}
),
} }
if result["interfaces"]:
filename = f"{device['name']}.json"
filenames_used.add(filename)
file_with_path = join(TARGET_PATH, filename)
with io.job(
f"{bold(site['name'])} {bold(device['name'])} writing to {file_with_path}"
):
with open( with open(
join( file_with_path,
dirname(dirname(__file__)),
"configs",
"netbox_device_{}.json".format(device["name"]),
),
"w+", "w+",
) as f: ) as f:
dump( dump(
@ -156,3 +218,23 @@ for device in data["device_list"]:
indent=4, indent=4,
sort_keys=True, sort_keys=True,
) )
else:
io.stdout(
f"device {bold(device['name'])} has no interfaces, {red('not')} dumping!"
)
if not args.only_site and not args.only_device and filenames_used:
with io.job(f"cleaning leftover files from {TARGET_PATH}"):
for direntry in scandir(TARGET_PATH):
filename = direntry.name
if filename.startswith("."):
continue
if not direntry.is_file():
io.stderr(
f"found non-file {filename} in {TARGET_PATH}, please check what's going on!"
)
continue
if filename not in filenames_used:
remove(join(TARGET_PATH, filename))
finally:
io.deactivate()