Fix the public DNS
This commit is contained in:
parent
d6d9a348b1
commit
3f41b63582
6 changed files with 265 additions and 204 deletions
|
|
@ -1,11 +1,8 @@
|
||||||
zapp_binary(
|
py_project(
|
||||||
name = "updater",
|
name = "updater",
|
||||||
main = "src/python/arrdem/updater/__main__.py",
|
main = "src/python/updater/__main__.py",
|
||||||
shebang = "/usr/bin/env python3",
|
shebang = "/usr/bin/env python3",
|
||||||
imports = [
|
lib_deps = [
|
||||||
"src/python",
|
|
||||||
],
|
|
||||||
deps = [
|
|
||||||
"//projects/gandi",
|
"//projects/gandi",
|
||||||
py_requirement("jinja2"),
|
py_requirement("jinja2"),
|
||||||
py_requirement("pyyaml"),
|
py_requirement("pyyaml"),
|
||||||
|
|
|
||||||
|
|
@ -1,197 +0,0 @@
|
||||||
"""
|
|
||||||
A quick and dirty public DNS script, super tightly coupled to my infrastructure.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
from pprint import pprint
|
|
||||||
import re
|
|
||||||
|
|
||||||
from gandi.client import GandiAPI
|
|
||||||
import jinja2
|
|
||||||
import meraki
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
RECORD_LINE_PATTERN = re.compile(
|
|
||||||
r"^(?P<rrset_name>\S+)\s+"
|
|
||||||
r"(?P<rrset_ttl>\S+)\s+"
|
|
||||||
r"IN\s+"
|
|
||||||
r"(?P<rrset_type>\S+)\s+"
|
|
||||||
r"(?P<rrset_values>[^\s#]+)"
|
|
||||||
r"(?P<comment>\s*#.*?)$"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def update(m, k, f, *args, **kwargs):
|
|
||||||
"""clojure.core/update for Python's stateful maps."""
|
|
||||||
|
|
||||||
if k in m:
|
|
||||||
m[k] = f(m[k], *args, **kwargs)
|
|
||||||
|
|
||||||
return m
|
|
||||||
|
|
||||||
|
|
||||||
def parse_zone_record(line):
|
|
||||||
if match := RECORD_LINE_PATTERN.search(line):
|
|
||||||
dat = match.groupdict()
|
|
||||||
dat = update(dat, "rrset_ttl", int)
|
|
||||||
dat = update(dat, "rrset_values", lambda x: [x])
|
|
||||||
return dat
|
|
||||||
|
|
||||||
|
|
||||||
def same_record(lr, rr):
|
|
||||||
"""
|
|
||||||
A test to see if two records name the same zone entry.
|
|
||||||
"""
|
|
||||||
|
|
||||||
return lr["rrset_name"] == rr["rrset_name"] and lr["rrset_type"] == rr["rrset_type"]
|
|
||||||
|
|
||||||
|
|
||||||
def records_equate(lr, rr):
|
|
||||||
"""
|
|
||||||
Equality, ignoring rrset_href which is generated by the API.
|
|
||||||
"""
|
|
||||||
|
|
||||||
if not same_record(lr, rr):
|
|
||||||
return False
|
|
||||||
|
|
||||||
elif lr["rrset_ttl"] != rr["rrset_ttl"]:
|
|
||||||
return False
|
|
||||||
|
|
||||||
elif set(lr["rrset_values"]) != set(rr["rrset_values"]):
|
|
||||||
return False
|
|
||||||
|
|
||||||
else:
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def template_and_parse_zone(template_file, template_bindings):
|
|
||||||
assert template_file is not None
|
|
||||||
assert template_bindings is not None
|
|
||||||
|
|
||||||
with open(template_file) as fp:
|
|
||||||
dat = jinja2.Template(fp.read()).render(**template_bindings)
|
|
||||||
|
|
||||||
uncommitted_records = []
|
|
||||||
for line in dat.splitlines():
|
|
||||||
if line and not line[0] == "#":
|
|
||||||
record = parse_zone_record(line)
|
|
||||||
if record:
|
|
||||||
uncommitted_records.append(record)
|
|
||||||
|
|
||||||
records = []
|
|
||||||
|
|
||||||
for uncommitted_r in uncommitted_records:
|
|
||||||
flag = False
|
|
||||||
for committed_r in records:
|
|
||||||
if same_record(uncommitted_r, committed_r):
|
|
||||||
# Join the two records
|
|
||||||
committed_r["rrset_values"].extend(uncommitted_r["rrset_values"])
|
|
||||||
flag = True
|
|
||||||
|
|
||||||
if not flag:
|
|
||||||
records.append(uncommitted_r)
|
|
||||||
|
|
||||||
sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"]))
|
|
||||||
|
|
||||||
return records
|
|
||||||
|
|
||||||
|
|
||||||
def diff_zones(left_zone, right_zone):
|
|
||||||
"""
|
|
||||||
Equality between unordered lists of records constituting a zone.
|
|
||||||
"""
|
|
||||||
|
|
||||||
in_left_not_right = []
|
|
||||||
in_right_not_left = []
|
|
||||||
for lr in left_zone:
|
|
||||||
flag = False
|
|
||||||
for rr in right_zone:
|
|
||||||
if records_equate(lr, rr):
|
|
||||||
flag |= True
|
|
||||||
|
|
||||||
if not flag:
|
|
||||||
in_left_not_right.append(lr)
|
|
||||||
in_right_not_left.append(rr)
|
|
||||||
|
|
||||||
return in_left_not_right, in_right_not_left
|
|
||||||
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description='"Dynamic" DNS updating for self-hosted services'
|
|
||||||
)
|
|
||||||
parser.add_argument("--config", dest="config_file", required=True)
|
|
||||||
parser.add_argument("--templates", dest="template_dir", required=True)
|
|
||||||
parser.add_argument("--dry-run", dest="dry", action="store_true", default=False)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
with open(args.config_file, "r") as fp:
|
|
||||||
config = yaml.safe_load(fp)
|
|
||||||
|
|
||||||
dashboard = meraki.DashboardAPI(config["meraki"]["key"], output_log=False)
|
|
||||||
org = config["meraki"]["organization"]
|
|
||||||
device = config["meraki"]["router_serial"]
|
|
||||||
|
|
||||||
uplinks = dashboard.appliance.getOrganizationApplianceUplinkStatuses(
|
|
||||||
organizationId=org, serials=[device]
|
|
||||||
)[0]["uplinks"]
|
|
||||||
|
|
||||||
template_bindings = {
|
|
||||||
"local": {
|
|
||||||
# One of the two
|
|
||||||
"public_v4s": [
|
|
||||||
link.get("publicIp") for link in uplinks if link.get("publicIp")
|
|
||||||
],
|
|
||||||
},
|
|
||||||
# Why isn't there a merge method
|
|
||||||
**config["bindings"],
|
|
||||||
}
|
|
||||||
|
|
||||||
print(f"Using config {template_bindings!r}...")
|
|
||||||
|
|
||||||
api = GandiAPI(config["gandi"]["key"])
|
|
||||||
|
|
||||||
for task in config["tasks"]:
|
|
||||||
if isinstance(task, str):
|
|
||||||
task = {"template": task + ".j2", "zones": [task]}
|
|
||||||
|
|
||||||
computed_zone = template_and_parse_zone(
|
|
||||||
os.path.join(args.template_dir, task["template"]), template_bindings
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"Running task {task!r}...")
|
|
||||||
|
|
||||||
for zone_name in task["zones"] or []:
|
|
||||||
try:
|
|
||||||
live_zone = api.domain_records(zone_name)
|
|
||||||
|
|
||||||
lr, rl = diff_zones(computed_zone, live_zone)
|
|
||||||
if lr or rl:
|
|
||||||
print(f"Zone {zone_name} differs;")
|
|
||||||
print("Computed:")
|
|
||||||
pprint(computed_zone)
|
|
||||||
pprint("Live:")
|
|
||||||
pprint(live_zone)
|
|
||||||
if(rl):
|
|
||||||
print("Live records not recomputed")
|
|
||||||
pprint(rl)
|
|
||||||
if(lr):
|
|
||||||
print("New records not live")
|
|
||||||
pprint(lr)
|
|
||||||
|
|
||||||
if not args.dry:
|
|
||||||
print(api.replace_domain(zone_name, computed_zone))
|
|
||||||
else:
|
|
||||||
print("Zone {} up to date".format(zone_name))
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print("While processing zone {}".format(zone_name))
|
|
||||||
raise e
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__" or 1:
|
|
||||||
main()
|
|
||||||
109
projects/public_dns/src/python/updater/__init__.py
Normal file
109
projects/public_dns/src/python/updater/__init__.py
Normal file
|
|
@ -0,0 +1,109 @@
|
||||||
|
"""
|
||||||
|
A quick and dirty public DNS script, super tightly coupled to my infrastructure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
import jinja2
|
||||||
|
|
||||||
|
|
||||||
|
RECORD_LINE_PATTERN = re.compile(
|
||||||
|
r"^(?P<rrset_name>\S+)\s+(?P<rrset_ttl>\S+)\s+IN\s+(?P<rrset_type>\S+)\s+(?P<rrset_values>[^\s#]+)(?P<comment>\s*#.*?)?$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def update(m, k, f, *args, **kwargs):
|
||||||
|
"""clojure.core/update for Python's stateful maps."""
|
||||||
|
|
||||||
|
if k in m:
|
||||||
|
m[k] = f(m[k], *args, **kwargs)
|
||||||
|
|
||||||
|
return m
|
||||||
|
|
||||||
|
|
||||||
|
def parse_zone_record(line):
|
||||||
|
if match := RECORD_LINE_PATTERN.match(line):
|
||||||
|
dat = match.groupdict()
|
||||||
|
dat = update(dat, "rrset_ttl", int)
|
||||||
|
dat = update(dat, "rrset_values", lambda x: [x])
|
||||||
|
return dat
|
||||||
|
|
||||||
|
|
||||||
|
def same_record(lr, rr):
|
||||||
|
"""
|
||||||
|
A test to see if two records name the same zone entry.
|
||||||
|
"""
|
||||||
|
|
||||||
|
return lr["rrset_name"] == rr["rrset_name"] and lr["rrset_type"] == rr["rrset_type"]
|
||||||
|
|
||||||
|
|
||||||
|
def records_equate(lr, rr):
|
||||||
|
"""
|
||||||
|
Equality, ignoring rrset_href which is generated by the API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not same_record(lr, rr):
|
||||||
|
return False
|
||||||
|
|
||||||
|
elif lr["rrset_ttl"] != rr["rrset_ttl"]:
|
||||||
|
return False
|
||||||
|
|
||||||
|
elif set(lr["rrset_values"]) != set(rr["rrset_values"]):
|
||||||
|
return False
|
||||||
|
|
||||||
|
else:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def template_and_parse_zone(template_file, template_bindings):
|
||||||
|
assert template_file is not None
|
||||||
|
assert template_bindings is not None
|
||||||
|
|
||||||
|
with open(template_file) as fp:
|
||||||
|
dat = jinja2.Template(fp.read()).render(**template_bindings)
|
||||||
|
|
||||||
|
uncommitted_records = []
|
||||||
|
for line in dat.splitlines():
|
||||||
|
if line and not line[0] == "#":
|
||||||
|
record = parse_zone_record(line)
|
||||||
|
if record:
|
||||||
|
uncommitted_records.append(record)
|
||||||
|
else:
|
||||||
|
print("ERROR, could not parse line %r" % line)
|
||||||
|
|
||||||
|
records = []
|
||||||
|
|
||||||
|
for uncommitted_r in uncommitted_records:
|
||||||
|
flag = False
|
||||||
|
for committed_r in records:
|
||||||
|
if same_record(uncommitted_r, committed_r):
|
||||||
|
# Join the two records
|
||||||
|
committed_r["rrset_values"].extend(uncommitted_r["rrset_values"])
|
||||||
|
flag = True
|
||||||
|
|
||||||
|
if not flag:
|
||||||
|
records.append(uncommitted_r)
|
||||||
|
|
||||||
|
sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"]))
|
||||||
|
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
def diff_zones(left_zone, right_zone):
|
||||||
|
"""
|
||||||
|
Equality between unordered lists of records constituting a zone.
|
||||||
|
"""
|
||||||
|
|
||||||
|
in_left_not_right = []
|
||||||
|
in_right_not_left = []
|
||||||
|
for lr in left_zone:
|
||||||
|
flag = False
|
||||||
|
for rr in right_zone:
|
||||||
|
if records_equate(lr, rr):
|
||||||
|
flag |= True
|
||||||
|
|
||||||
|
if not flag:
|
||||||
|
in_left_not_right.append(lr)
|
||||||
|
in_right_not_left.append(rr)
|
||||||
|
|
||||||
|
return in_left_not_right, in_right_not_left
|
||||||
92
projects/public_dns/src/python/updater/__main__.py
Normal file
92
projects/public_dns/src/python/updater/__main__.py
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
"""
|
||||||
|
A quick and dirty public DNS script, super tightly coupled to my infrastructure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
from pprint import pprint
|
||||||
|
|
||||||
|
from gandi.client import GandiAPI
|
||||||
|
import meraki
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from updater import *
|
||||||
|
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description='"Dynamic" DNS updating for self-hosted services'
|
||||||
|
)
|
||||||
|
parser.add_argument("--config", dest="config_file", required=True)
|
||||||
|
parser.add_argument("--templates", dest="template_dir", required=True)
|
||||||
|
parser.add_argument("--dry-run", dest="dry", action="store_true", default=False)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
with open(args.config_file, "r") as fp:
|
||||||
|
config = yaml.safe_load(fp)
|
||||||
|
|
||||||
|
dashboard = meraki.DashboardAPI(config["meraki"]["key"], output_log=False)
|
||||||
|
org = config["meraki"]["organization"]
|
||||||
|
device = config["meraki"]["router_serial"]
|
||||||
|
|
||||||
|
uplinks = dashboard.appliance.getOrganizationApplianceUplinkStatuses(
|
||||||
|
organizationId=org, serials=[device]
|
||||||
|
)[0]["uplinks"]
|
||||||
|
|
||||||
|
template_bindings = {
|
||||||
|
"local": {
|
||||||
|
# One of the two
|
||||||
|
"public_v4s": [
|
||||||
|
link.get("publicIp") for link in uplinks if link.get("publicIp")
|
||||||
|
],
|
||||||
|
},
|
||||||
|
# Why isn't there a merge method
|
||||||
|
**config["bindings"],
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"Using config {template_bindings!r}...")
|
||||||
|
|
||||||
|
api = GandiAPI(config["gandi"]["key"])
|
||||||
|
|
||||||
|
for task in config["tasks"]:
|
||||||
|
if isinstance(task, str):
|
||||||
|
task = {"template": task + ".j2", "zones": [task]}
|
||||||
|
|
||||||
|
computed_zone = template_and_parse_zone(
|
||||||
|
os.path.join(args.template_dir, task["template"]), template_bindings
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"Running task {task!r}...")
|
||||||
|
|
||||||
|
for zone_name in task["zones"] or []:
|
||||||
|
try:
|
||||||
|
live_zone = api.domain_records(zone_name)
|
||||||
|
|
||||||
|
lr, rl = diff_zones(computed_zone, live_zone)
|
||||||
|
if lr or rl:
|
||||||
|
print(f"Zone {zone_name} differs;")
|
||||||
|
print("Computed:")
|
||||||
|
pprint(computed_zone)
|
||||||
|
pprint("Live:")
|
||||||
|
pprint(live_zone)
|
||||||
|
if rl:
|
||||||
|
print("Live records not recomputed")
|
||||||
|
pprint(rl)
|
||||||
|
if lr:
|
||||||
|
print("New records not live")
|
||||||
|
pprint(lr)
|
||||||
|
|
||||||
|
if not args.dry:
|
||||||
|
print(api.replace_domain(zone_name, computed_zone))
|
||||||
|
else:
|
||||||
|
print("Zone {} up to date".format(zone_name))
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print("While processing zone {}".format(zone_name))
|
||||||
|
raise e
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__" or 1:
|
||||||
|
main()
|
||||||
|
|
@ -16,7 +16,7 @@ www {{ ttl }} IN A {{ link }}
|
||||||
registry {{ ttl }} IN A {{ link }}
|
registry {{ ttl }} IN A {{ link }}
|
||||||
mirror {{ ttl }} IN A {{ link }}
|
mirror {{ ttl }} IN A {{ link }}
|
||||||
buildcache {{ ttl }} IN A {{ link }}
|
buildcache {{ ttl }} IN A {{ link }}
|
||||||
feed {{ ttl }} IN A {{ link }}
|
tentacles {{ ttl }} IN A {{ link }}
|
||||||
ton {{ ttl }} IN A {{ link }}
|
ton {{ ttl }} IN A {{ link }}
|
||||||
relay {{ ttl }} IN A {{ link }}
|
relay {{ ttl }} IN A {{ link }}
|
||||||
pxe {{ ttl }} IN A {{ link }}
|
pxe {{ ttl }} IN A {{ link }}
|
||||||
|
|
|
||||||
60
projects/public_dns/test/python/test_parsing.py
Normal file
60
projects/public_dns/test/python/test_parsing.py
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from updater import RECORD_LINE_PATTERN, parse_zone_record, diff_zones
|
||||||
|
|
||||||
|
|
||||||
|
def test_record_pattern():
|
||||||
|
assert re.match(RECORD_LINE_PATTERN, "foo 300 IN A 1.1.1.1")
|
||||||
|
assert re.match(RECORD_LINE_PATTERN, "foo\t 300\t IN \tA\t 1.1.1.1")
|
||||||
|
|
||||||
|
|
||||||
|
AT_RECORD = {
|
||||||
|
"comment": None,
|
||||||
|
"rrset_name": "@",
|
||||||
|
"rrset_ttl": 300,
|
||||||
|
"rrset_type": "A",
|
||||||
|
"rrset_values": ["67.166.27.157"],
|
||||||
|
}
|
||||||
|
A_RECORD = {
|
||||||
|
"comment": None,
|
||||||
|
"rrset_name": "www",
|
||||||
|
"rrset_ttl": 300,
|
||||||
|
"rrset_type": "A",
|
||||||
|
"rrset_values": ["67.166.27.157"],
|
||||||
|
}
|
||||||
|
REGISTRY_RECORD = {
|
||||||
|
"comment": None,
|
||||||
|
"rrset_name": "registry",
|
||||||
|
"rrset_ttl": 300,
|
||||||
|
"rrset_type": "A",
|
||||||
|
"rrset_values": ["67.166.27.157"],
|
||||||
|
}
|
||||||
|
MIRROR_RECORD = {
|
||||||
|
"comment": None,
|
||||||
|
"rrset_name": "mirror",
|
||||||
|
"rrset_ttl": 300,
|
||||||
|
"rrset_type": "A",
|
||||||
|
"rrset_values": ["67.166.27.157"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_diff_zones():
|
||||||
|
z1 = [AT_RECORD, A_RECORD]
|
||||||
|
z2 = []
|
||||||
|
assert diff_zones(z1, z2) == z1, []
|
||||||
|
|
||||||
|
z1 = [AT_RECORD, A_RECORD]
|
||||||
|
z2 = [AT_RECORD]
|
||||||
|
assert diff_zones(z1, z2) == [A_RECORD], []
|
||||||
|
|
||||||
|
z1 = [AT_RECORD, A_RECORD]
|
||||||
|
z2 = [A_RECORD]
|
||||||
|
assert diff_zones(z1, z2) == [AT_RECORD], []
|
||||||
|
|
||||||
|
z2 = [AT_RECORD, A_RECORD]
|
||||||
|
z1 = [A_RECORD]
|
||||||
|
assert diff_zones(z1, z2) == [], [AT_RECORD]
|
||||||
Loading…
Reference in a new issue