From 3f41b6358246e0a554c4e8ce8c71576c3bc19cbd Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Sat, 3 Jun 2023 19:07:41 -0600 Subject: [PATCH] Fix the public DNS --- projects/public_dns/BUILD | 9 +- .../src/python/arrdem/updater/__main__.py | 197 ------------------ .../public_dns/src/python/updater/__init__.py | 109 ++++++++++ .../public_dns/src/python/updater/__main__.py | 92 ++++++++ .../src/resources/zonefiles/tirefireind.us.j2 | 2 +- .../public_dns/test/python/test_parsing.py | 60 ++++++ 6 files changed, 265 insertions(+), 204 deletions(-) delete mode 100644 projects/public_dns/src/python/arrdem/updater/__main__.py create mode 100644 projects/public_dns/src/python/updater/__init__.py create mode 100644 projects/public_dns/src/python/updater/__main__.py create mode 100644 projects/public_dns/test/python/test_parsing.py diff --git a/projects/public_dns/BUILD b/projects/public_dns/BUILD index e793e88..989fd2e 100644 --- a/projects/public_dns/BUILD +++ b/projects/public_dns/BUILD @@ -1,11 +1,8 @@ -zapp_binary( +py_project( name = "updater", - main = "src/python/arrdem/updater/__main__.py", + main = "src/python/updater/__main__.py", shebang = "/usr/bin/env python3", - imports = [ - "src/python", - ], - deps = [ + lib_deps = [ "//projects/gandi", py_requirement("jinja2"), py_requirement("pyyaml"), diff --git a/projects/public_dns/src/python/arrdem/updater/__main__.py b/projects/public_dns/src/python/arrdem/updater/__main__.py deleted file mode 100644 index 4631bd7..0000000 --- a/projects/public_dns/src/python/arrdem/updater/__main__.py +++ /dev/null @@ -1,197 +0,0 @@ -""" -A quick and dirty public DNS script, super tightly coupled to my infrastructure. -""" - -import argparse -import os -from pprint import pprint -import re - -from gandi.client import GandiAPI -import jinja2 -import meraki -import yaml - - -RECORD_LINE_PATTERN = re.compile( - r"^(?P\S+)\s+" - r"(?P\S+)\s+" - r"IN\s+" - r"(?P\S+)\s+" - r"(?P[^\s#]+)" - r"(?P\s*#.*?)$" -) - - -def update(m, k, f, *args, **kwargs): - """clojure.core/update for Python's stateful maps.""" - - if k in m: - m[k] = f(m[k], *args, **kwargs) - - return m - - -def parse_zone_record(line): - if match := RECORD_LINE_PATTERN.search(line): - dat = match.groupdict() - dat = update(dat, "rrset_ttl", int) - dat = update(dat, "rrset_values", lambda x: [x]) - return dat - - -def same_record(lr, rr): - """ - A test to see if two records name the same zone entry. - """ - - return lr["rrset_name"] == rr["rrset_name"] and lr["rrset_type"] == rr["rrset_type"] - - -def records_equate(lr, rr): - """ - Equality, ignoring rrset_href which is generated by the API. - """ - - if not same_record(lr, rr): - return False - - elif lr["rrset_ttl"] != rr["rrset_ttl"]: - return False - - elif set(lr["rrset_values"]) != set(rr["rrset_values"]): - return False - - else: - return True - - -def template_and_parse_zone(template_file, template_bindings): - assert template_file is not None - assert template_bindings is not None - - with open(template_file) as fp: - dat = jinja2.Template(fp.read()).render(**template_bindings) - - uncommitted_records = [] - for line in dat.splitlines(): - if line and not line[0] == "#": - record = parse_zone_record(line) - if record: - uncommitted_records.append(record) - - records = [] - - for uncommitted_r in uncommitted_records: - flag = False - for committed_r in records: - if same_record(uncommitted_r, committed_r): - # Join the two records - committed_r["rrset_values"].extend(uncommitted_r["rrset_values"]) - flag = True - - if not flag: - records.append(uncommitted_r) - - sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"])) - - return records - - -def diff_zones(left_zone, right_zone): - """ - Equality between unordered lists of records constituting a zone. - """ - - in_left_not_right = [] - in_right_not_left = [] - for lr in left_zone: - flag = False - for rr in right_zone: - if records_equate(lr, rr): - flag |= True - - if not flag: - in_left_not_right.append(lr) - in_right_not_left.append(rr) - - return in_left_not_right, in_right_not_left - - -parser = argparse.ArgumentParser( - description='"Dynamic" DNS updating for self-hosted services' -) -parser.add_argument("--config", dest="config_file", required=True) -parser.add_argument("--templates", dest="template_dir", required=True) -parser.add_argument("--dry-run", dest="dry", action="store_true", default=False) - - -def main(): - args = parser.parse_args() - - with open(args.config_file, "r") as fp: - config = yaml.safe_load(fp) - - dashboard = meraki.DashboardAPI(config["meraki"]["key"], output_log=False) - org = config["meraki"]["organization"] - device = config["meraki"]["router_serial"] - - uplinks = dashboard.appliance.getOrganizationApplianceUplinkStatuses( - organizationId=org, serials=[device] - )[0]["uplinks"] - - template_bindings = { - "local": { - # One of the two - "public_v4s": [ - link.get("publicIp") for link in uplinks if link.get("publicIp") - ], - }, - # Why isn't there a merge method - **config["bindings"], - } - - print(f"Using config {template_bindings!r}...") - - api = GandiAPI(config["gandi"]["key"]) - - for task in config["tasks"]: - if isinstance(task, str): - task = {"template": task + ".j2", "zones": [task]} - - computed_zone = template_and_parse_zone( - os.path.join(args.template_dir, task["template"]), template_bindings - ) - - print(f"Running task {task!r}...") - - for zone_name in task["zones"] or []: - try: - live_zone = api.domain_records(zone_name) - - lr, rl = diff_zones(computed_zone, live_zone) - if lr or rl: - print(f"Zone {zone_name} differs;") - print("Computed:") - pprint(computed_zone) - pprint("Live:") - pprint(live_zone) - if(rl): - print("Live records not recomputed") - pprint(rl) - if(lr): - print("New records not live") - pprint(lr) - - if not args.dry: - print(api.replace_domain(zone_name, computed_zone)) - else: - print("Zone {} up to date".format(zone_name)) - - except Exception as e: - print("While processing zone {}".format(zone_name)) - raise e - - -if __name__ == "__main__" or 1: - main() diff --git a/projects/public_dns/src/python/updater/__init__.py b/projects/public_dns/src/python/updater/__init__.py new file mode 100644 index 0000000..efd9ba0 --- /dev/null +++ b/projects/public_dns/src/python/updater/__init__.py @@ -0,0 +1,109 @@ +""" +A quick and dirty public DNS script, super tightly coupled to my infrastructure. +""" + +import re + +import jinja2 + + +RECORD_LINE_PATTERN = re.compile( + r"^(?P\S+)\s+(?P\S+)\s+IN\s+(?P\S+)\s+(?P[^\s#]+)(?P\s*#.*?)?$" +) + + +def update(m, k, f, *args, **kwargs): + """clojure.core/update for Python's stateful maps.""" + + if k in m: + m[k] = f(m[k], *args, **kwargs) + + return m + + +def parse_zone_record(line): + if match := RECORD_LINE_PATTERN.match(line): + dat = match.groupdict() + dat = update(dat, "rrset_ttl", int) + dat = update(dat, "rrset_values", lambda x: [x]) + return dat + + +def same_record(lr, rr): + """ + A test to see if two records name the same zone entry. + """ + + return lr["rrset_name"] == rr["rrset_name"] and lr["rrset_type"] == rr["rrset_type"] + + +def records_equate(lr, rr): + """ + Equality, ignoring rrset_href which is generated by the API. + """ + + if not same_record(lr, rr): + return False + + elif lr["rrset_ttl"] != rr["rrset_ttl"]: + return False + + elif set(lr["rrset_values"]) != set(rr["rrset_values"]): + return False + + else: + return True + + +def template_and_parse_zone(template_file, template_bindings): + assert template_file is not None + assert template_bindings is not None + + with open(template_file) as fp: + dat = jinja2.Template(fp.read()).render(**template_bindings) + + uncommitted_records = [] + for line in dat.splitlines(): + if line and not line[0] == "#": + record = parse_zone_record(line) + if record: + uncommitted_records.append(record) + else: + print("ERROR, could not parse line %r" % line) + + records = [] + + for uncommitted_r in uncommitted_records: + flag = False + for committed_r in records: + if same_record(uncommitted_r, committed_r): + # Join the two records + committed_r["rrset_values"].extend(uncommitted_r["rrset_values"]) + flag = True + + if not flag: + records.append(uncommitted_r) + + sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"])) + + return records + + +def diff_zones(left_zone, right_zone): + """ + Equality between unordered lists of records constituting a zone. + """ + + in_left_not_right = [] + in_right_not_left = [] + for lr in left_zone: + flag = False + for rr in right_zone: + if records_equate(lr, rr): + flag |= True + + if not flag: + in_left_not_right.append(lr) + in_right_not_left.append(rr) + + return in_left_not_right, in_right_not_left diff --git a/projects/public_dns/src/python/updater/__main__.py b/projects/public_dns/src/python/updater/__main__.py new file mode 100644 index 0000000..f3109e4 --- /dev/null +++ b/projects/public_dns/src/python/updater/__main__.py @@ -0,0 +1,92 @@ +""" +A quick and dirty public DNS script, super tightly coupled to my infrastructure. +""" + +import argparse +import os +from pprint import pprint + +from gandi.client import GandiAPI +import meraki +import yaml + +from updater import * + + +parser = argparse.ArgumentParser( + description='"Dynamic" DNS updating for self-hosted services' +) +parser.add_argument("--config", dest="config_file", required=True) +parser.add_argument("--templates", dest="template_dir", required=True) +parser.add_argument("--dry-run", dest="dry", action="store_true", default=False) + + +def main(): + args = parser.parse_args() + + with open(args.config_file, "r") as fp: + config = yaml.safe_load(fp) + + dashboard = meraki.DashboardAPI(config["meraki"]["key"], output_log=False) + org = config["meraki"]["organization"] + device = config["meraki"]["router_serial"] + + uplinks = dashboard.appliance.getOrganizationApplianceUplinkStatuses( + organizationId=org, serials=[device] + )[0]["uplinks"] + + template_bindings = { + "local": { + # One of the two + "public_v4s": [ + link.get("publicIp") for link in uplinks if link.get("publicIp") + ], + }, + # Why isn't there a merge method + **config["bindings"], + } + + print(f"Using config {template_bindings!r}...") + + api = GandiAPI(config["gandi"]["key"]) + + for task in config["tasks"]: + if isinstance(task, str): + task = {"template": task + ".j2", "zones": [task]} + + computed_zone = template_and_parse_zone( + os.path.join(args.template_dir, task["template"]), template_bindings + ) + + print(f"Running task {task!r}...") + + for zone_name in task["zones"] or []: + try: + live_zone = api.domain_records(zone_name) + + lr, rl = diff_zones(computed_zone, live_zone) + if lr or rl: + print(f"Zone {zone_name} differs;") + print("Computed:") + pprint(computed_zone) + pprint("Live:") + pprint(live_zone) + if rl: + print("Live records not recomputed") + pprint(rl) + if lr: + print("New records not live") + pprint(lr) + + if not args.dry: + print(api.replace_domain(zone_name, computed_zone)) + else: + print("Zone {} up to date".format(zone_name)) + + except Exception as e: + print("While processing zone {}".format(zone_name)) + raise e + + +if __name__ == "__main__" or 1: + main() diff --git a/projects/public_dns/src/resources/zonefiles/tirefireind.us.j2 b/projects/public_dns/src/resources/zonefiles/tirefireind.us.j2 index 8bf9300..c669c4a 100644 --- a/projects/public_dns/src/resources/zonefiles/tirefireind.us.j2 +++ b/projects/public_dns/src/resources/zonefiles/tirefireind.us.j2 @@ -16,7 +16,7 @@ www {{ ttl }} IN A {{ link }} registry {{ ttl }} IN A {{ link }} mirror {{ ttl }} IN A {{ link }} buildcache {{ ttl }} IN A {{ link }} -feed {{ ttl }} IN A {{ link }} +tentacles {{ ttl }} IN A {{ link }} ton {{ ttl }} IN A {{ link }} relay {{ ttl }} IN A {{ link }} pxe {{ ttl }} IN A {{ link }} diff --git a/projects/public_dns/test/python/test_parsing.py b/projects/public_dns/test/python/test_parsing.py new file mode 100644 index 0000000..68cc790 --- /dev/null +++ b/projects/public_dns/test/python/test_parsing.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import re + +import pytest + +from updater import RECORD_LINE_PATTERN, parse_zone_record, diff_zones + + +def test_record_pattern(): + assert re.match(RECORD_LINE_PATTERN, "foo 300 IN A 1.1.1.1") + assert re.match(RECORD_LINE_PATTERN, "foo\t 300\t IN \tA\t 1.1.1.1") + + +AT_RECORD = { + "comment": None, + "rrset_name": "@", + "rrset_ttl": 300, + "rrset_type": "A", + "rrset_values": ["67.166.27.157"], +} +A_RECORD = { + "comment": None, + "rrset_name": "www", + "rrset_ttl": 300, + "rrset_type": "A", + "rrset_values": ["67.166.27.157"], +} +REGISTRY_RECORD = { + "comment": None, + "rrset_name": "registry", + "rrset_ttl": 300, + "rrset_type": "A", + "rrset_values": ["67.166.27.157"], +} +MIRROR_RECORD = { + "comment": None, + "rrset_name": "mirror", + "rrset_ttl": 300, + "rrset_type": "A", + "rrset_values": ["67.166.27.157"], +} + + +def test_diff_zones(): + z1 = [AT_RECORD, A_RECORD] + z2 = [] + assert diff_zones(z1, z2) == z1, [] + + z1 = [AT_RECORD, A_RECORD] + z2 = [AT_RECORD] + assert diff_zones(z1, z2) == [A_RECORD], [] + + z1 = [AT_RECORD, A_RECORD] + z2 = [A_RECORD] + assert diff_zones(z1, z2) == [AT_RECORD], [] + + z2 = [AT_RECORD, A_RECORD] + z1 = [A_RECORD] + assert diff_zones(z1, z2) == [], [AT_RECORD]