Compare commits

...

2 commits

13 changed files with 289 additions and 242 deletions

View file

@ -1,11 +1,8 @@
zapp_binary( py_project(
name = "updater", name = "updater",
main = "src/python/arrdem/updater/__main__.py", main = "src/python/updater/__main__.py",
shebang = "/usr/bin/env python3", shebang = "/usr/bin/env python3",
imports = [ lib_deps = [
"src/python",
],
deps = [
"//projects/gandi", "//projects/gandi",
py_requirement("jinja2"), py_requirement("jinja2"),
py_requirement("pyyaml"), py_requirement("pyyaml"),

View file

@ -1,197 +0,0 @@
"""
A quick and dirty public DNS script, super tightly coupled to my infrastructure.
"""
import argparse
import os
from pprint import pprint
import re
from gandi.client import GandiAPI
import jinja2
import meraki
import yaml
RECORD_LINE_PATTERN = re.compile(
r"^(?P<rrset_name>\S+)\s+"
r"(?P<rrset_ttl>\S+)\s+"
r"IN\s+"
r"(?P<rrset_type>\S+)\s+"
r"(?P<rrset_values>[^\s#]+)"
r"(?P<comment>\s*#.*?)$"
)
def update(m, k, f, *args, **kwargs):
"""clojure.core/update for Python's stateful maps."""
if k in m:
m[k] = f(m[k], *args, **kwargs)
return m
def parse_zone_record(line):
if match := RECORD_LINE_PATTERN.search(line):
dat = match.groupdict()
dat = update(dat, "rrset_ttl", int)
dat = update(dat, "rrset_values", lambda x: [x])
return dat
def same_record(lr, rr):
"""
A test to see if two records name the same zone entry.
"""
return lr["rrset_name"] == rr["rrset_name"] and lr["rrset_type"] == rr["rrset_type"]
def records_equate(lr, rr):
"""
Equality, ignoring rrset_href which is generated by the API.
"""
if not same_record(lr, rr):
return False
elif lr["rrset_ttl"] != rr["rrset_ttl"]:
return False
elif set(lr["rrset_values"]) != set(rr["rrset_values"]):
return False
else:
return True
def template_and_parse_zone(template_file, template_bindings):
assert template_file is not None
assert template_bindings is not None
with open(template_file) as fp:
dat = jinja2.Template(fp.read()).render(**template_bindings)
uncommitted_records = []
for line in dat.splitlines():
if line and not line[0] == "#":
record = parse_zone_record(line)
if record:
uncommitted_records.append(record)
records = []
for uncommitted_r in uncommitted_records:
flag = False
for committed_r in records:
if same_record(uncommitted_r, committed_r):
# Join the two records
committed_r["rrset_values"].extend(uncommitted_r["rrset_values"])
flag = True
if not flag:
records.append(uncommitted_r)
sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"]))
return records
def diff_zones(left_zone, right_zone):
"""
Equality between unordered lists of records constituting a zone.
"""
in_left_not_right = []
in_right_not_left = []
for lr in left_zone:
flag = False
for rr in right_zone:
if records_equate(lr, rr):
flag |= True
if not flag:
in_left_not_right.append(lr)
in_right_not_left.append(rr)
return in_left_not_right, in_right_not_left
parser = argparse.ArgumentParser(
description='"Dynamic" DNS updating for self-hosted services'
)
parser.add_argument("--config", dest="config_file", required=True)
parser.add_argument("--templates", dest="template_dir", required=True)
parser.add_argument("--dry-run", dest="dry", action="store_true", default=False)
def main():
args = parser.parse_args()
with open(args.config_file, "r") as fp:
config = yaml.safe_load(fp)
dashboard = meraki.DashboardAPI(config["meraki"]["key"], output_log=False)
org = config["meraki"]["organization"]
device = config["meraki"]["router_serial"]
uplinks = dashboard.appliance.getOrganizationApplianceUplinkStatuses(
organizationId=org, serials=[device]
)[0]["uplinks"]
template_bindings = {
"local": {
# One of the two
"public_v4s": [
link.get("publicIp") for link in uplinks if link.get("publicIp")
],
},
# Why isn't there a merge method
**config["bindings"],
}
print(f"Using config {template_bindings!r}...")
api = GandiAPI(config["gandi"]["key"])
for task in config["tasks"]:
if isinstance(task, str):
task = {"template": task + ".j2", "zones": [task]}
computed_zone = template_and_parse_zone(
os.path.join(args.template_dir, task["template"]), template_bindings
)
print(f"Running task {task!r}...")
for zone_name in task["zones"] or []:
try:
live_zone = api.domain_records(zone_name)
lr, rl = diff_zones(computed_zone, live_zone)
if lr or rl:
print(f"Zone {zone_name} differs;")
print("Computed:")
pprint(computed_zone)
pprint("Live:")
pprint(live_zone)
if(rl):
print("Live records not recomputed")
pprint(rl)
if(lr):
print("New records not live")
pprint(lr)
if not args.dry:
print(api.replace_domain(zone_name, computed_zone))
else:
print("Zone {} up to date".format(zone_name))
except Exception as e:
print("While processing zone {}".format(zone_name))
raise e
if __name__ == "__main__" or 1:
main()

View file

@ -0,0 +1,109 @@
"""
A quick and dirty public DNS script, super tightly coupled to my infrastructure.
"""
import re
import jinja2
RECORD_LINE_PATTERN = re.compile(
r"^(?P<rrset_name>\S+)\s+(?P<rrset_ttl>\S+)\s+IN\s+(?P<rrset_type>\S+)\s+(?P<rrset_values>[^\s#]+)(?P<comment>\s*#.*?)?$"
)
def update(m, k, f, *args, **kwargs):
"""clojure.core/update for Python's stateful maps."""
if k in m:
m[k] = f(m[k], *args, **kwargs)
return m
def parse_zone_record(line):
if match := RECORD_LINE_PATTERN.match(line):
dat = match.groupdict()
dat = update(dat, "rrset_ttl", int)
dat = update(dat, "rrset_values", lambda x: [x])
return dat
def same_record(lr, rr):
"""
A test to see if two records name the same zone entry.
"""
return lr["rrset_name"] == rr["rrset_name"] and lr["rrset_type"] == rr["rrset_type"]
def records_equate(lr, rr):
"""
Equality, ignoring rrset_href which is generated by the API.
"""
if not same_record(lr, rr):
return False
elif lr["rrset_ttl"] != rr["rrset_ttl"]:
return False
elif set(lr["rrset_values"]) != set(rr["rrset_values"]):
return False
else:
return True
def template_and_parse_zone(template_file, template_bindings):
assert template_file is not None
assert template_bindings is not None
with open(template_file) as fp:
dat = jinja2.Template(fp.read()).render(**template_bindings)
uncommitted_records = []
for line in dat.splitlines():
if line and not line[0] == "#":
record = parse_zone_record(line)
if record:
uncommitted_records.append(record)
else:
print("ERROR, could not parse line %r" % line)
records = []
for uncommitted_r in uncommitted_records:
flag = False
for committed_r in records:
if same_record(uncommitted_r, committed_r):
# Join the two records
committed_r["rrset_values"].extend(uncommitted_r["rrset_values"])
flag = True
if not flag:
records.append(uncommitted_r)
sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"]))
return records
def diff_zones(left_zone, right_zone):
"""
Equality between unordered lists of records constituting a zone.
"""
in_left_not_right = []
in_right_not_left = []
for lr in left_zone:
flag = False
for rr in right_zone:
if records_equate(lr, rr):
flag |= True
if not flag:
in_left_not_right.append(lr)
in_right_not_left.append(rr)
return in_left_not_right, in_right_not_left

View file

@ -0,0 +1,92 @@
"""
A quick and dirty public DNS script, super tightly coupled to my infrastructure.
"""
import argparse
import os
from pprint import pprint
from gandi.client import GandiAPI
import meraki
import yaml
from updater import *
parser = argparse.ArgumentParser(
description='"Dynamic" DNS updating for self-hosted services'
)
parser.add_argument("--config", dest="config_file", required=True)
parser.add_argument("--templates", dest="template_dir", required=True)
parser.add_argument("--dry-run", dest="dry", action="store_true", default=False)
def main():
args = parser.parse_args()
with open(args.config_file, "r") as fp:
config = yaml.safe_load(fp)
dashboard = meraki.DashboardAPI(config["meraki"]["key"], output_log=False)
org = config["meraki"]["organization"]
device = config["meraki"]["router_serial"]
uplinks = dashboard.appliance.getOrganizationApplianceUplinkStatuses(
organizationId=org, serials=[device]
)[0]["uplinks"]
template_bindings = {
"local": {
# One of the two
"public_v4s": [
link.get("publicIp") for link in uplinks if link.get("publicIp")
],
},
# Why isn't there a merge method
**config["bindings"],
}
print(f"Using config {template_bindings!r}...")
api = GandiAPI(config["gandi"]["key"])
for task in config["tasks"]:
if isinstance(task, str):
task = {"template": task + ".j2", "zones": [task]}
computed_zone = template_and_parse_zone(
os.path.join(args.template_dir, task["template"]), template_bindings
)
print(f"Running task {task!r}...")
for zone_name in task["zones"] or []:
try:
live_zone = api.domain_records(zone_name)
lr, rl = diff_zones(computed_zone, live_zone)
if lr or rl:
print(f"Zone {zone_name} differs;")
print("Computed:")
pprint(computed_zone)
pprint("Live:")
pprint(live_zone)
if rl:
print("Live records not recomputed")
pprint(rl)
if lr:
print("New records not live")
pprint(lr)
if not args.dry:
print(api.replace_domain(zone_name, computed_zone))
else:
print("Zone {} up to date".format(zone_name))
except Exception as e:
print("While processing zone {}".format(zone_name))
raise e
if __name__ == "__main__" or 1:
main()

View file

@ -16,7 +16,7 @@ www {{ ttl }} IN A {{ link }}
registry {{ ttl }} IN A {{ link }} registry {{ ttl }} IN A {{ link }}
mirror {{ ttl }} IN A {{ link }} mirror {{ ttl }} IN A {{ link }}
buildcache {{ ttl }} IN A {{ link }} buildcache {{ ttl }} IN A {{ link }}
feed {{ ttl }} IN A {{ link }} tentacles {{ ttl }} IN A {{ link }}
ton {{ ttl }} IN A {{ link }} ton {{ ttl }} IN A {{ link }}
relay {{ ttl }} IN A {{ link }} relay {{ ttl }} IN A {{ link }}
pxe {{ ttl }} IN A {{ link }} pxe {{ ttl }} IN A {{ link }}

View file

@ -0,0 +1,60 @@
#!/usr/bin/env python3
import re
import pytest
from updater import RECORD_LINE_PATTERN, parse_zone_record, diff_zones
def test_record_pattern():
assert re.match(RECORD_LINE_PATTERN, "foo 300 IN A 1.1.1.1")
assert re.match(RECORD_LINE_PATTERN, "foo\t 300\t IN \tA\t 1.1.1.1")
AT_RECORD = {
"comment": None,
"rrset_name": "@",
"rrset_ttl": 300,
"rrset_type": "A",
"rrset_values": ["67.166.27.157"],
}
A_RECORD = {
"comment": None,
"rrset_name": "www",
"rrset_ttl": 300,
"rrset_type": "A",
"rrset_values": ["67.166.27.157"],
}
REGISTRY_RECORD = {
"comment": None,
"rrset_name": "registry",
"rrset_ttl": 300,
"rrset_type": "A",
"rrset_values": ["67.166.27.157"],
}
MIRROR_RECORD = {
"comment": None,
"rrset_name": "mirror",
"rrset_ttl": 300,
"rrset_type": "A",
"rrset_values": ["67.166.27.157"],
}
def test_diff_zones():
z1 = [AT_RECORD, A_RECORD]
z2 = []
assert diff_zones(z1, z2) == z1, []
z1 = [AT_RECORD, A_RECORD]
z2 = [AT_RECORD]
assert diff_zones(z1, z2) == [A_RECORD], []
z1 = [AT_RECORD, A_RECORD]
z2 = [A_RECORD]
assert diff_zones(z1, z2) == [AT_RECORD], []
z2 = [AT_RECORD, A_RECORD]
z1 = [A_RECORD]
assert diff_zones(z1, z2) == [], [AT_RECORD]

View file

@ -6,6 +6,7 @@ py_project(
main_deps = [ main_deps = [
"//projects/anosql", "//projects/anosql",
"//projects/anosql-migrations", "//projects/anosql-migrations",
py_requirement("aiosql"),
py_requirement("attrs"), py_requirement("attrs"),
py_requirement("click"), py_requirement("click"),
py_requirement("flask"), py_requirement("flask"),

View file

@ -23,11 +23,6 @@ from tentacles.workers import *
from tentacles.workers import assign_jobs, Worker from tentacles.workers import assign_jobs, Worker
@click.group()
def cli():
pass
def db_factory(app): def db_factory(app):
store = Db(app.config.get("db", {}).get("uri")) store = Db(app.config.get("db", {}).get("uri"))
store.connect() store.connect()
@ -99,7 +94,7 @@ def make_app():
return app return app
@cli.command() @click.command()
@click.option("--hostname", "hostname", type=str, default="0.0.0.0") @click.option("--hostname", "hostname", type=str, default="0.0.0.0")
@click.option("--port", "port", type=int, default=8080) @click.option("--port", "port", type=int, default=8080)
@click.option("--trace/--no-trace", "trace", default=False) @click.option("--trace/--no-trace", "trace", default=False)
@ -154,4 +149,4 @@ def serve(hostname: str, port: int, config: Path, trace: bool):
if __name__ == "__main__": if __name__ == "__main__":
cli() serve()

View file

@ -74,13 +74,7 @@ def get_users():
return redirect("/admin") return redirect("/admin")
@BLUEPRINT.route("/admin/printers") @BLUEPRINT.route("/admin/printers", methods=["GET"])
@requires_admin
def printers():
return render_template("printers.html.j2")
@BLUEPRINT.route("/admin/printers/add", methods=["GET"])
@requires_admin @requires_admin
def add_printer(): def add_printer():
return render_template("add_printer.html.j2") return render_template("add_printer.html.j2")

View file

@ -71,11 +71,14 @@ def post_register():
try: try:
username = request.form["username"] username = request.form["username"]
email = request.form["email"] email = request.form["email"]
has_config = False
group_id = 1 # Normal users group_id = 1 # Normal users
status_id = -3 # Unverified status_id = -3 # Unverified
for user_config in current_app.config.get("users", []): for user_config in current_app.config.get("users", []):
if user_config["email"] == email: if user_config["email"] == email:
has_config = True
if "group_id" in user_config: if "group_id" in user_config:
group_id = user_config["group_id"] group_id = user_config["group_id"]
@ -109,6 +112,11 @@ def post_register():
) )
elif user.status_id == 1: elif user.status_id == 1:
# Do the entire approval dance just to be absolutely sure any statically configured users work
ctx.db.try_verify_user(token=user.verification_token)
ctx.db.approve_user(uid=user.id)
ctx.db.enable_user(uid=user.id)
flash("Welcome, please log in", category="success") flash("Welcome, please log in", category="success")
return render_template("register.html.j2") return render_template("register.html.j2")

View file

@ -21,30 +21,9 @@
</div> </div>
</div> </div>
<div class="row"> <div class="row">
<input type="hidden" name="tested" value="false" /> <input id="tested" type="hidden" name="tested" value="false" />
<input id="test" type="button" value="Test" enabled="false" />
<input id="submit" type="submit" value="Add" onclick="maybeSubmit();" /> <input id="submit" type="submit" value="Add" onclick="maybeSubmit();" />
</div> </div>
</form> </form>
</div> </div>
<script type="text/javascript">
document.getElementById("input").disabled = True;
function testSettings() {
var formData = new FormData(document.getElementById("form"))
var req = new XMLHttpRequest();
req.open("POST", "/printer/test");
req.send(formData);
};
function maybeSubmit() {
if (document.getElementById("tested").value == "true") {
document.getElementById("form").submit();
} else {
console.error("Form values have not been tested!");
}
};
</script>
{% endblock %} {% endblock %}

View file

@ -21,10 +21,10 @@
{% endwith %} {% endwith %}
{% endfor %} {% endfor %}
{% if ctx.is_admin %} {% if ctx.is_admin %}
<a class="button" href="/printers/add">Add a printer</a> <a class="button" href="/admin/printers">Add a printer</a>
{% endif %} {% endif %}
{% else %} {% else %}
No printers available. {% if ctx.is_admin %}<a href="/printers/add">Configure one!</a>{% else %}Ask the admin to configure one!{% endif %} No printers available. {% if ctx.is_admin %}<a href="/admin/printers">Configure one!</a>{% else %}Ask the admin to configure one!{% endif %}
{% endif %} {% endif %}
{% endwith %} {% endwith %}
</div> </div>

View file

@ -38,6 +38,15 @@ class OctoRest(_OR):
response.raise_for_status() response.raise_for_status()
return response return response
def files_info(self, *args, **kwargs):
try:
return super().files_info(*args, **kwargs)
except HTTPError as e:
if e.response.status_code == 404:
return {}
else:
raise e
log = logging.getLogger(__name__) log = logging.getLogger(__name__)