Shovel in the DNS automation machinery

projects/gandi/BUILD

@@ -0,0 +1,10 @@
+py_library(
+    name = "gandi",
+    src = globs(["src/python/**/*.py"]),
+    imports = [
+        "src/python"
+    ],
+    deps = [
+        py_requirement("requests"),
+    ]
+)

projects/gandi/src/python/BUILD

@@ -0,0 +1,7 @@
+python_library(
+	name="gandi",
+	sources=globs("*.py"),
+	dependencies=[
+	  "//3rdparty/python:requests",
+	]
+)

projects/gandi/src/python/client.py

@@ -0,0 +1,102 @@
+"""
+Quick and shitty Gandi REST API driver
+"""
+
+import json
+from datetime import datetime
+
+import requests
+
+
+class GandiAPI(object):
+  """An extremely incomplete Gandi REST API driver class.
+
+  Exists to close over your API key, and make talking to the API slightly more idiomatic.
+
+  Note: In an effort to be nice, this API maintains a cache of the zones visible with your API
+  key. The cache is maintained when using this driver, but concurrent modifications of your zone(s)
+  via the web UI or other processes will obviously undermine it. This cache can be disabled by
+  setting the `use_cache` kwarg to `False`.
+
+  """
+  
+  def __init__(self, key=None, use_cache=True):
+    self._base = "https://dns.api.gandi.net/api/v5"
+    self._key = key
+    self._use_cache = use_cache
+    self._zones = None
+
+  # Helpers for making requests with the API key as required by the API.
+
+  def _do_request(self, method, path, headers=None, **kwargs):
+    headers = headers or {}
+    headers["X-Api-Key"] = self._key
+    resp = method(self._base + path, headers=headers, **kwargs)
+    if resp.status_code > 299:
+      print(resp.text)
+      resp.raise_for_status()
+
+    return resp
+
+  def _get(self, path, **kwargs):
+    return self._do_request(requests.get, path, **kwargs)
+
+  def _post(self, path, **kwargs):
+    return self._do_request(requests.post, path, **kwargs)
+
+  def _put(self, path, **kwargs):
+    return self._do_request(requests.put, path, **kwargs)
+
+  # Intentional public API
+  
+  def domain_records(self, domain):
+    return self._get("/domains/{0}/records".format(domain)).json()
+
+  def get_zones(self):
+    if self._use_cache:
+      if not self._zones:
+        self._zones = self._get("/zones").json()
+      return self._zones
+    else:
+      return self._get("/zones").json() 
+
+  def get_zone(self, zone_id):
+    return self._get("/zones/{}".format(zone_id)).json()
+
+  def get_zone_by_name(self, zone_name):
+    for zone in self.get_zones():
+      if zone["name"] == zone_name:
+        return zone
+
+  def create_zone(self, zone_name):
+    new_zone_id = self._post("/zones",
+                             headers={"content-type": "application/json"},
+                             data=json.dumps({"name": zone_name}))\
+                      .headers["Location"]\
+                      .split("/")[-1]
+    new_zone = self.get_zone(new_zone_id)
+
+    # Note: If the cache is active, update the cache.
+    if self._use_cache and self._zones is not None:
+      self._zones.append(new_zone)
+
+    return new_zone
+
+  def replace_domain(self, domain, records):
+    date = datetime.now()
+    date = "{:%A, %d. %B %Y %I:%M%p}".format(date)
+    zone_name = f"updater generated domain - {domain} - {date}"
+
+    zone = self.get_zone_by_name(zone_name)
+    if not zone:
+      zone = self.create_zone(zone_name)
+
+    print("Using zone", zone["uuid"])
+
+    for r in records:
+      self._post("/zones/{0}/records".format(zone["uuid"]),
+                 headers={"content-type": "application/json"},
+                 data=json.dumps(r))
+
+    return self._post("/zones/{0}/domains/{1}".format(zone["uuid"], domain),
+                      headers={"content-type": "application/json"})

projects/public-dns/.dockerignore

@@ -0,0 +1,8 @@
+.*
+3rdparty
+src
+test
+README.md
+pants
+pants.ini
+etc

projects/public-dns/BUILD

@@ -0,0 +1,9 @@
+py_binary(
+    name = "updater",
+    entry_point = "src/python/arrdem/updater/__main__.py",
+    deps = [
+        "//projects/gandi",
+        py_requirement("jinja2"),
+        py_requirement("pyyaml"),
+    ]
+)

projects/public-dns/etc/build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+function pants2docker() {
+  # $1 MUST BE a pants target including : final component
+  ./pants -q binary "$1"
+  f=$(mktemp)
+  target=$(echo "$1" | awk -F: '{print $2; exit}')
+  pex="${target}.pex"
+  sed "s/\$pex/$pex/g" "${DOCKERFILE:-src/docker/pex.docker}" > "${f}"
+
+  # Build with tags
+  t="arrdem.${target}:latest"
+  rt="registry.apartment.arrdem.com:5000/${t}"
+
+  # FIXME (arrdem 2019-09-17):
+  #   Create a tag for the SHA and short SHA of the current ref if clean
+  docker build -f $f -t "${t}" -t "${rt}" $gt .
+  docker push "${rt}"
+}
+
+DOCKERFILE=src/docker/arrdem/updater/Dockerfile pants2docker src/python/arrdem/updater:updater
+
+sudo docker stack deploy --compose-file ./src/docker/arrdem/updater/docker-compose.yml arrdem_updater

projects/public-dns/src/docker/arrdem/updater/Dockerfile

@@ -0,0 +1,4 @@
+FROM python:3.7
+COPY "./dist/$pex" "/app/$pex"
+WORKDIR /app
+CMD ["/app/$pex", "--config", "/run/secrets/public-dns.yml"]

projects/public-dns/src/docker/arrdem/updater/docker-compose.yml

@@ -0,0 +1,20 @@
+version: '3.7'
+
+services:
+  cron:
+    image: registry.apartment.arrdem.com:5000/arrdem.updater:latest
+    deploy:
+      replicas: 0
+      restart_policy:
+        condition: none
+      labels:
+        - swarm.cronjob.enable=true
+        - "swarm.cronjob.schedule=0 */10 * * * *"
+        - swarm.cronjob.skip-running=false
+        - node.platform.arch == x86_64
+    secrets:
+      - public-dns.yml
+
+secrets:
+  public-dns.yml:
+    file: ../../../../config.yml

projects/public-dns/src/python/arrdem/updater/BUILD

@@ -0,0 +1,12 @@
+python_binary(
+  name='updater',
+	source='__main__.py',
+	dependencies=[
+	  "//src/resources/zonefiles",
+		"//src/python/gandi",
+		"//3rdparty/python:requests",
+		"//3rdparty/python:jinja2",
+		"//3rdparty/python:PyYAML",
+		"//3rdparty/python:meraki",
+	]
+)

projects/public-dns/src/python/arrdem/updater/__main__.py

@@ -0,0 +1,176 @@
+"""
+A quick and dirty public DNS script, super tightly coupled to my infrastructure.
+"""
+
+import argparse
+import re
+from pprint import pprint
+
+import jinja2
+import pkg_resources
+import yaml
+
+from gandi.client import GandiAPI
+from meraki import meraki
+
+
+RECORD_LINE_PATTERN = re.compile(
+  "^(?P<rrset_name>\S+)\s+"
+  "(?P<rrset_ttl>\S+)\s+"
+  "IN\s+"
+  "(?P<rrset_type>\S+)\s+"
+  "(?P<rrset_values>.+)$")
+
+
+def update(m, k, f, *args, **kwargs):
+  """clojure.core/update for Python's stateful maps."""
+  if k in m:
+    m[k] = f(m[k], *args, **kwargs)
+  return m
+
+
+def parse_zone_record(line):
+  match = RECORD_LINE_PATTERN.search(line)
+  if match:
+    dat = match.groupdict()
+    dat = update(dat, "rrset_ttl", int)
+    dat = update(dat, "rrset_values", lambda x: [x])
+    return dat
+
+
+def same_record(lr, rr):
+  """
+  A test to see if two records name the same zone entry.
+  """
+
+  return lr["rrset_name"] == rr["rrset_name"] and \
+         lr["rrset_type"] == rr["rrset_type"]
+
+
+def records_equate(lr, rr):
+  """
+  Equality, ignoring rrset_href which is generated by the API.
+  """
+
+  if not same_record(lr, rr):
+    return False
+  elif lr["rrset_ttl"] != rr["rrset_ttl"]:
+    return False
+  elif set(lr["rrset_values"]) != set(rr["rrset_values"]):
+    return False
+  else:
+    return True
+
+
+def template_and_parse_zone(template_name, template_bindings):
+  assert template_name is not None
+  assert template_bindings is not None
+
+  dat = pkg_resources.resource_string("zonefiles", template_name).decode("utf-8")
+  dat = jinja2.Template(dat).render(**template_bindings)
+  
+  uncommitted_records = []
+  for line in dat.splitlines():
+    if line and not line[0] == "#":
+      record = parse_zone_record(line)
+      if record:
+        uncommitted_records.append(record)
+
+  records = []
+
+  for uncommitted_r in uncommitted_records:
+    flag = False
+    for committed_r in records:
+      if same_record(uncommitted_r, committed_r):
+        # Join the two records
+        committed_r["rrset_values"].extend(uncommitted_r["rrset_values"])
+        flag = True
+
+    if not flag:
+      records.append(uncommitted_r)
+
+  sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"]))
+
+  return records
+
+
+def diff_zones(left_zone, right_zone):
+  """
+  Equality between unordered lists of records constituting a zone.
+  """
+  
+  in_left_not_right = []
+  for lr in left_zone:
+    flag = False
+    for rr in right_zone:
+      if same_record(lr, rr) and records_equate(lr, rr):
+        flag = True
+        break
+
+    if not flag:
+      in_left_not_right.append(lr)
+
+  in_right_not_left = []
+  for rr in right_zone:
+    flag = False
+    for lr in left_zone:
+      if same_record(lr, rr) and records_equate(lr, rr):
+        flag = True
+        break
+
+    if not flag:
+      in_right_not_left.append(lr)
+
+  return in_left_not_right or in_right_not_left
+
+
+parser = argparse.ArgumentParser(description="\"Dynamic\" DNS updating for self-hosted services")
+parser.add_argument("--config", dest="config_file")
+parser.add_argument("--dry-run", dest="dry", action="store_true", default=False)
+
+def main():
+  args = parser.parse_args()
+  config = yaml.load(open(args.config_file, "r"))
+
+
+  uplinks = meraki.getdeviceuplink(config["meraki"]["key"],
+                                   config["meraki"]["network"],
+                                   config["meraki"]["router_serial"],
+                                   True)
+
+  template_bindings = {
+    "local": {
+      # One of the two
+      "public_v4s": [link.get("publicIp") for link in uplinks if link.get("publicIp")],
+    },
+    # Why isn't there a merge method
+    **config["bindings"]
+  }
+
+  api = GandiAPI(config["gandi"]["key"])
+
+  for task in config["tasks"]:
+    if isinstance(task, str):
+      task = {"template": task + ".j2",
+              "zones": [task]}
+
+    computed_zone = template_and_parse_zone(task["template"], template_bindings)
+
+    for zone_name in task["zones"]:
+      try:
+        live_zone = api.domain_records(zone_name)
+
+        if diff_zones(computed_zone, live_zone):
+          print("Zone {} differs, computed zone:".format(zone_name))
+          pprint(computed_zone)
+          if not args.dry:
+            print(api.replace_domain(zone_name, computed_zone))
+        else:
+          print("Zone {} up to date".format(zone_name))
+
+      except Exception as e:
+        print("While processing zone {}".format(zone_name))
+        raise e
+
+if __name__ == "__main__" or 1:
+  main()

projects/public-dns/src/resources/zonefiles/BUILD

@@ -0,0 +1,4 @@
+python_library(
+  name="zonefiles",
+	sources=globs("*.j2"),
+)

projects/public-dns/src/resources/zonefiles/arrdem.com.j2

@@ -0,0 +1,20 @@
+# -*- mode: conf -*-
+
+# Core records
+{% for addr in local.public_v4s %}
+@           {{ ttl }} IN A {{ addr }}
+www         {{ ttl }} IN A {{ addr }}
+git         {{ ttl }} IN A {{ addr }}
+paperbrainz {{ ttl }} IN A {{ addr }}
+{% endfor %}
+
+# Service setup
+@                   10800 IN MX    10 in1-smtp.messagingengine.com.
+@                   10800 IN MX    20 in2-smtp.messagingengine.com.
+mesmtp._domainkey   10800 IN TXT      "v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC/arz66E8pNtYK+jWh57KkT+/bIp2iO9JEVb+9xqb9Z/R/l6zvFsUm5voKdhEJKsYkb35xpWJ699HbXWTENzWCLOvFxpPx8+MOh5i5gQDEg7P8vHfs5dDwj+dH5wQHcYTYPOIm5PFdkKemGwzHayFFOGhr5Xgf4PuiohtWZZSxzwIDAQAB"
+
+email.lists         10800 IN CNAME    mailgun.org.
+lists               10800 IN MX    10 mxa.mailgun.org.
+lists               10800 IN MX    10 mxb.mailgun.org.
+lists               10800 IN TXT      "v=spf1 include:mailgun.org ~all"
+mx._domainkey.lists 10800 IN TXT      "k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCcbCDs5bW+LWM9cWLbQTPTCNoiSe7giSFISx9LIU9LBSeE8483wQhx6FMe7jOR3+4njEpIyFtyOrz3ol2JdH+R7d/ZoeUeQekU7/paF90ZS5hvE2wyKCtVFCmBevxI73pcH0JUXZe9/VCTRGtt5Ksny4cBrhMwWQGk5UFyUfBGOQIDAQAB"

projects/public-dns/src/resources/zonefiles/paren.party.j2

@@ -0,0 +1,3 @@
+@ {{ ttl }} IN A 217.70.184.38
+www {{ ttl }} IN CNAME paren.party.s3-website-us-west-2.amazonaws.com.
+egalitarian {{ ttl }} IN CNAME paren.party.s3-website-us-west-2.amazonaws.com.

projects/public-dns/src/resources/zonefiles/park.j2

@@ -0,0 +1,9 @@
+# -*- mode: conf -*-
+
+@       10800 IN A        217.70.184.38
+@       10800 IN MX    10 spool.mail.gandi.net.
+@       10800 IN MX    50 fb.mail.gandi.net.
+@       10800 IN TXT      "v=spf1 include:_mailcust.gandi.net ?all"
+blog    10800 IN CNAME    blogs.vip.gandi.net.
+webmail 10800 IN CNAME    webmail.gandi.net.
+www     10800 IN CNAME    webredir.vip.gandi.net.

projects/public-dns/src/resources/zonefiles/tirefireind.us.j2

@@ -0,0 +1,13 @@
+# -*- mode: conf -*-
+
+# Core records
+{% for link in local.public_v4s %}
+@           {{ ttl }} IN A {{ link }}
+www         {{ ttl }} IN A {{ link }}
+registry    {{ ttl }} IN A {{ link }}
+mirror      {{ ttl }} IN A {{ link }}
+buildcache  {{ ttl }} IN A {{ link }}
+{% endfor %}
+
+# Gitlab
+gitlab      {{ ttl }} IN A {{ sroo.public_v4 }}