From 0654ae28476b401c4c91ee283639f44c663ee90d Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Sat, 8 May 2021 12:36:29 -0600 Subject: [PATCH] Shovel in the DNS automation machinery --- projects/gandi/BUILD | 10 + projects/gandi/src/python/BUILD | 7 + projects/gandi/src/python/__init__.py | 0 projects/gandi/src/python/client.py | 102 ++++++++++ projects/public-dns/.dockerignore | 8 + projects/public-dns/BUILD | 9 + projects/public-dns/README.md | 0 projects/public-dns/etc/build.sh | 23 +++ .../src/docker/arrdem/updater/Dockerfile | 4 + .../docker/arrdem/updater/docker-compose.yml | 20 ++ .../src/python/arrdem/updater/BUILD | 12 ++ .../src/python/arrdem/updater/__main__.py | 176 ++++++++++++++++++ .../public-dns/src/resources/zonefiles/BUILD | 4 + .../src/resources/zonefiles/arrdem.com.j2 | 20 ++ .../src/resources/zonefiles/paren.party.j2 | 3 + .../src/resources/zonefiles/park.j2 | 9 + .../src/resources/zonefiles/tirefireind.us.j2 | 13 ++ 17 files changed, 420 insertions(+) create mode 100644 projects/gandi/BUILD create mode 100644 projects/gandi/src/python/BUILD create mode 100644 projects/gandi/src/python/__init__.py create mode 100644 projects/gandi/src/python/client.py create mode 100644 projects/public-dns/.dockerignore create mode 100644 projects/public-dns/BUILD create mode 100644 projects/public-dns/README.md create mode 100644 projects/public-dns/etc/build.sh create mode 100644 projects/public-dns/src/docker/arrdem/updater/Dockerfile create mode 100644 projects/public-dns/src/docker/arrdem/updater/docker-compose.yml create mode 100644 projects/public-dns/src/python/arrdem/updater/BUILD create mode 100644 projects/public-dns/src/python/arrdem/updater/__main__.py create mode 100644 projects/public-dns/src/resources/zonefiles/BUILD create mode 100644 projects/public-dns/src/resources/zonefiles/arrdem.com.j2 create mode 100644 projects/public-dns/src/resources/zonefiles/paren.party.j2 create mode 100644 projects/public-dns/src/resources/zonefiles/park.j2 create mode 100644 projects/public-dns/src/resources/zonefiles/tirefireind.us.j2 diff --git a/projects/gandi/BUILD b/projects/gandi/BUILD new file mode 100644 index 0000000..fa5d946 --- /dev/null +++ b/projects/gandi/BUILD @@ -0,0 +1,10 @@ +py_library( + name = "gandi", + src = globs(["src/python/**/*.py"]), + imports = [ + "src/python" + ], + deps = [ + py_requirement("requests"), + ] +) diff --git a/projects/gandi/src/python/BUILD b/projects/gandi/src/python/BUILD new file mode 100644 index 0000000..16c9f4f --- /dev/null +++ b/projects/gandi/src/python/BUILD @@ -0,0 +1,7 @@ +python_library( + name="gandi", + sources=globs("*.py"), + dependencies=[ + "//3rdparty/python:requests", + ] +) diff --git a/projects/gandi/src/python/__init__.py b/projects/gandi/src/python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/projects/gandi/src/python/client.py b/projects/gandi/src/python/client.py new file mode 100644 index 0000000..97d070f --- /dev/null +++ b/projects/gandi/src/python/client.py @@ -0,0 +1,102 @@ +""" +Quick and shitty Gandi REST API driver +""" + +import json +from datetime import datetime + +import requests + + +class GandiAPI(object): + """An extremely incomplete Gandi REST API driver class. + + Exists to close over your API key, and make talking to the API slightly more idiomatic. + + Note: In an effort to be nice, this API maintains a cache of the zones visible with your API + key. The cache is maintained when using this driver, but concurrent modifications of your zone(s) + via the web UI or other processes will obviously undermine it. This cache can be disabled by + setting the `use_cache` kwarg to `False`. + + """ + + def __init__(self, key=None, use_cache=True): + self._base = "https://dns.api.gandi.net/api/v5" + self._key = key + self._use_cache = use_cache + self._zones = None + + # Helpers for making requests with the API key as required by the API. + + def _do_request(self, method, path, headers=None, **kwargs): + headers = headers or {} + headers["X-Api-Key"] = self._key + resp = method(self._base + path, headers=headers, **kwargs) + if resp.status_code > 299: + print(resp.text) + resp.raise_for_status() + + return resp + + def _get(self, path, **kwargs): + return self._do_request(requests.get, path, **kwargs) + + def _post(self, path, **kwargs): + return self._do_request(requests.post, path, **kwargs) + + def _put(self, path, **kwargs): + return self._do_request(requests.put, path, **kwargs) + + # Intentional public API + + def domain_records(self, domain): + return self._get("/domains/{0}/records".format(domain)).json() + + def get_zones(self): + if self._use_cache: + if not self._zones: + self._zones = self._get("/zones").json() + return self._zones + else: + return self._get("/zones").json() + + def get_zone(self, zone_id): + return self._get("/zones/{}".format(zone_id)).json() + + def get_zone_by_name(self, zone_name): + for zone in self.get_zones(): + if zone["name"] == zone_name: + return zone + + def create_zone(self, zone_name): + new_zone_id = self._post("/zones", + headers={"content-type": "application/json"}, + data=json.dumps({"name": zone_name}))\ + .headers["Location"]\ + .split("/")[-1] + new_zone = self.get_zone(new_zone_id) + + # Note: If the cache is active, update the cache. + if self._use_cache and self._zones is not None: + self._zones.append(new_zone) + + return new_zone + + def replace_domain(self, domain, records): + date = datetime.now() + date = "{:%A, %d. %B %Y %I:%M%p}".format(date) + zone_name = f"updater generated domain - {domain} - {date}" + + zone = self.get_zone_by_name(zone_name) + if not zone: + zone = self.create_zone(zone_name) + + print("Using zone", zone["uuid"]) + + for r in records: + self._post("/zones/{0}/records".format(zone["uuid"]), + headers={"content-type": "application/json"}, + data=json.dumps(r)) + + return self._post("/zones/{0}/domains/{1}".format(zone["uuid"], domain), + headers={"content-type": "application/json"}) diff --git a/projects/public-dns/.dockerignore b/projects/public-dns/.dockerignore new file mode 100644 index 0000000..7a54896 --- /dev/null +++ b/projects/public-dns/.dockerignore @@ -0,0 +1,8 @@ +.* +3rdparty +src +test +README.md +pants +pants.ini +etc diff --git a/projects/public-dns/BUILD b/projects/public-dns/BUILD new file mode 100644 index 0000000..1e0cdf4 --- /dev/null +++ b/projects/public-dns/BUILD @@ -0,0 +1,9 @@ +py_binary( + name = "updater", + entry_point = "src/python/arrdem/updater/__main__.py", + deps = [ + "//projects/gandi", + py_requirement("jinja2"), + py_requirement("pyyaml"), + ] +) diff --git a/projects/public-dns/README.md b/projects/public-dns/README.md new file mode 100644 index 0000000..e69de29 diff --git a/projects/public-dns/etc/build.sh b/projects/public-dns/etc/build.sh new file mode 100644 index 0000000..891d190 --- /dev/null +++ b/projects/public-dns/etc/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +function pants2docker() { + # $1 MUST BE a pants target including : final component + ./pants -q binary "$1" + f=$(mktemp) + target=$(echo "$1" | awk -F: '{print $2; exit}') + pex="${target}.pex" + sed "s/\$pex/$pex/g" "${DOCKERFILE:-src/docker/pex.docker}" > "${f}" + + # Build with tags + t="arrdem.${target}:latest" + rt="registry.apartment.arrdem.com:5000/${t}" + + # FIXME (arrdem 2019-09-17): + # Create a tag for the SHA and short SHA of the current ref if clean + docker build -f $f -t "${t}" -t "${rt}" $gt . + docker push "${rt}" +} + +DOCKERFILE=src/docker/arrdem/updater/Dockerfile pants2docker src/python/arrdem/updater:updater + +sudo docker stack deploy --compose-file ./src/docker/arrdem/updater/docker-compose.yml arrdem_updater diff --git a/projects/public-dns/src/docker/arrdem/updater/Dockerfile b/projects/public-dns/src/docker/arrdem/updater/Dockerfile new file mode 100644 index 0000000..34e31a3 --- /dev/null +++ b/projects/public-dns/src/docker/arrdem/updater/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.7 +COPY "./dist/$pex" "/app/$pex" +WORKDIR /app +CMD ["/app/$pex", "--config", "/run/secrets/public-dns.yml"] diff --git a/projects/public-dns/src/docker/arrdem/updater/docker-compose.yml b/projects/public-dns/src/docker/arrdem/updater/docker-compose.yml new file mode 100644 index 0000000..dab62a6 --- /dev/null +++ b/projects/public-dns/src/docker/arrdem/updater/docker-compose.yml @@ -0,0 +1,20 @@ +version: '3.7' + +services: + cron: + image: registry.apartment.arrdem.com:5000/arrdem.updater:latest + deploy: + replicas: 0 + restart_policy: + condition: none + labels: + - swarm.cronjob.enable=true + - "swarm.cronjob.schedule=0 */10 * * * *" + - swarm.cronjob.skip-running=false + - node.platform.arch == x86_64 + secrets: + - public-dns.yml + +secrets: + public-dns.yml: + file: ../../../../config.yml diff --git a/projects/public-dns/src/python/arrdem/updater/BUILD b/projects/public-dns/src/python/arrdem/updater/BUILD new file mode 100644 index 0000000..d844632 --- /dev/null +++ b/projects/public-dns/src/python/arrdem/updater/BUILD @@ -0,0 +1,12 @@ +python_binary( + name='updater', + source='__main__.py', + dependencies=[ + "//src/resources/zonefiles", + "//src/python/gandi", + "//3rdparty/python:requests", + "//3rdparty/python:jinja2", + "//3rdparty/python:PyYAML", + "//3rdparty/python:meraki", + ] +) diff --git a/projects/public-dns/src/python/arrdem/updater/__main__.py b/projects/public-dns/src/python/arrdem/updater/__main__.py new file mode 100644 index 0000000..e00c5d5 --- /dev/null +++ b/projects/public-dns/src/python/arrdem/updater/__main__.py @@ -0,0 +1,176 @@ +""" +A quick and dirty public DNS script, super tightly coupled to my infrastructure. +""" + +import argparse +import re +from pprint import pprint + +import jinja2 +import pkg_resources +import yaml + +from gandi.client import GandiAPI +from meraki import meraki + + +RECORD_LINE_PATTERN = re.compile( + "^(?P\S+)\s+" + "(?P\S+)\s+" + "IN\s+" + "(?P\S+)\s+" + "(?P.+)$") + + +def update(m, k, f, *args, **kwargs): + """clojure.core/update for Python's stateful maps.""" + if k in m: + m[k] = f(m[k], *args, **kwargs) + return m + + +def parse_zone_record(line): + match = RECORD_LINE_PATTERN.search(line) + if match: + dat = match.groupdict() + dat = update(dat, "rrset_ttl", int) + dat = update(dat, "rrset_values", lambda x: [x]) + return dat + + +def same_record(lr, rr): + """ + A test to see if two records name the same zone entry. + """ + + return lr["rrset_name"] == rr["rrset_name"] and \ + lr["rrset_type"] == rr["rrset_type"] + + +def records_equate(lr, rr): + """ + Equality, ignoring rrset_href which is generated by the API. + """ + + if not same_record(lr, rr): + return False + elif lr["rrset_ttl"] != rr["rrset_ttl"]: + return False + elif set(lr["rrset_values"]) != set(rr["rrset_values"]): + return False + else: + return True + + +def template_and_parse_zone(template_name, template_bindings): + assert template_name is not None + assert template_bindings is not None + + dat = pkg_resources.resource_string("zonefiles", template_name).decode("utf-8") + dat = jinja2.Template(dat).render(**template_bindings) + + uncommitted_records = [] + for line in dat.splitlines(): + if line and not line[0] == "#": + record = parse_zone_record(line) + if record: + uncommitted_records.append(record) + + records = [] + + for uncommitted_r in uncommitted_records: + flag = False + for committed_r in records: + if same_record(uncommitted_r, committed_r): + # Join the two records + committed_r["rrset_values"].extend(uncommitted_r["rrset_values"]) + flag = True + + if not flag: + records.append(uncommitted_r) + + sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"])) + + return records + + +def diff_zones(left_zone, right_zone): + """ + Equality between unordered lists of records constituting a zone. + """ + + in_left_not_right = [] + for lr in left_zone: + flag = False + for rr in right_zone: + if same_record(lr, rr) and records_equate(lr, rr): + flag = True + break + + if not flag: + in_left_not_right.append(lr) + + in_right_not_left = [] + for rr in right_zone: + flag = False + for lr in left_zone: + if same_record(lr, rr) and records_equate(lr, rr): + flag = True + break + + if not flag: + in_right_not_left.append(lr) + + return in_left_not_right or in_right_not_left + + +parser = argparse.ArgumentParser(description="\"Dynamic\" DNS updating for self-hosted services") +parser.add_argument("--config", dest="config_file") +parser.add_argument("--dry-run", dest="dry", action="store_true", default=False) + +def main(): + args = parser.parse_args() + config = yaml.load(open(args.config_file, "r")) + + + uplinks = meraki.getdeviceuplink(config["meraki"]["key"], + config["meraki"]["network"], + config["meraki"]["router_serial"], + True) + + template_bindings = { + "local": { + # One of the two + "public_v4s": [link.get("publicIp") for link in uplinks if link.get("publicIp")], + }, + # Why isn't there a merge method + **config["bindings"] + } + + api = GandiAPI(config["gandi"]["key"]) + + for task in config["tasks"]: + if isinstance(task, str): + task = {"template": task + ".j2", + "zones": [task]} + + computed_zone = template_and_parse_zone(task["template"], template_bindings) + + for zone_name in task["zones"]: + try: + live_zone = api.domain_records(zone_name) + + if diff_zones(computed_zone, live_zone): + print("Zone {} differs, computed zone:".format(zone_name)) + pprint(computed_zone) + if not args.dry: + print(api.replace_domain(zone_name, computed_zone)) + else: + print("Zone {} up to date".format(zone_name)) + + except Exception as e: + print("While processing zone {}".format(zone_name)) + raise e + +if __name__ == "__main__" or 1: + main() diff --git a/projects/public-dns/src/resources/zonefiles/BUILD b/projects/public-dns/src/resources/zonefiles/BUILD new file mode 100644 index 0000000..8867bc9 --- /dev/null +++ b/projects/public-dns/src/resources/zonefiles/BUILD @@ -0,0 +1,4 @@ +python_library( + name="zonefiles", + sources=globs("*.j2"), +) diff --git a/projects/public-dns/src/resources/zonefiles/arrdem.com.j2 b/projects/public-dns/src/resources/zonefiles/arrdem.com.j2 new file mode 100644 index 0000000..3849115 --- /dev/null +++ b/projects/public-dns/src/resources/zonefiles/arrdem.com.j2 @@ -0,0 +1,20 @@ +# -*- mode: conf -*- + +# Core records +{% for addr in local.public_v4s %} +@ {{ ttl }} IN A {{ addr }} +www {{ ttl }} IN A {{ addr }} +git {{ ttl }} IN A {{ addr }} +paperbrainz {{ ttl }} IN A {{ addr }} +{% endfor %} + +# Service setup +@ 10800 IN MX 10 in1-smtp.messagingengine.com. +@ 10800 IN MX 20 in2-smtp.messagingengine.com. +mesmtp._domainkey 10800 IN TXT "v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC/arz66E8pNtYK+jWh57KkT+/bIp2iO9JEVb+9xqb9Z/R/l6zvFsUm5voKdhEJKsYkb35xpWJ699HbXWTENzWCLOvFxpPx8+MOh5i5gQDEg7P8vHfs5dDwj+dH5wQHcYTYPOIm5PFdkKemGwzHayFFOGhr5Xgf4PuiohtWZZSxzwIDAQAB" + +email.lists 10800 IN CNAME mailgun.org. +lists 10800 IN MX 10 mxa.mailgun.org. +lists 10800 IN MX 10 mxb.mailgun.org. +lists 10800 IN TXT "v=spf1 include:mailgun.org ~all" +mx._domainkey.lists 10800 IN TXT "k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCcbCDs5bW+LWM9cWLbQTPTCNoiSe7giSFISx9LIU9LBSeE8483wQhx6FMe7jOR3+4njEpIyFtyOrz3ol2JdH+R7d/ZoeUeQekU7/paF90ZS5hvE2wyKCtVFCmBevxI73pcH0JUXZe9/VCTRGtt5Ksny4cBrhMwWQGk5UFyUfBGOQIDAQAB" diff --git a/projects/public-dns/src/resources/zonefiles/paren.party.j2 b/projects/public-dns/src/resources/zonefiles/paren.party.j2 new file mode 100644 index 0000000..5dedfea --- /dev/null +++ b/projects/public-dns/src/resources/zonefiles/paren.party.j2 @@ -0,0 +1,3 @@ +@ {{ ttl }} IN A 217.70.184.38 +www {{ ttl }} IN CNAME paren.party.s3-website-us-west-2.amazonaws.com. +egalitarian {{ ttl }} IN CNAME paren.party.s3-website-us-west-2.amazonaws.com. diff --git a/projects/public-dns/src/resources/zonefiles/park.j2 b/projects/public-dns/src/resources/zonefiles/park.j2 new file mode 100644 index 0000000..ffc7074 --- /dev/null +++ b/projects/public-dns/src/resources/zonefiles/park.j2 @@ -0,0 +1,9 @@ +# -*- mode: conf -*- + +@ 10800 IN A 217.70.184.38 +@ 10800 IN MX 10 spool.mail.gandi.net. +@ 10800 IN MX 50 fb.mail.gandi.net. +@ 10800 IN TXT "v=spf1 include:_mailcust.gandi.net ?all" +blog 10800 IN CNAME blogs.vip.gandi.net. +webmail 10800 IN CNAME webmail.gandi.net. +www 10800 IN CNAME webredir.vip.gandi.net. diff --git a/projects/public-dns/src/resources/zonefiles/tirefireind.us.j2 b/projects/public-dns/src/resources/zonefiles/tirefireind.us.j2 new file mode 100644 index 0000000..6138966 --- /dev/null +++ b/projects/public-dns/src/resources/zonefiles/tirefireind.us.j2 @@ -0,0 +1,13 @@ +# -*- mode: conf -*- + +# Core records +{% for link in local.public_v4s %} +@ {{ ttl }} IN A {{ link }} +www {{ ttl }} IN A {{ link }} +registry {{ ttl }} IN A {{ link }} +mirror {{ ttl }} IN A {{ link }} +buildcache {{ ttl }} IN A {{ link }} +{% endfor %} + +# Gitlab +gitlab {{ ttl }} IN A {{ sroo.public_v4 }}