Bugfix: /admin/printers, handle 404 from files_info

projects/public_dns/BUILD

@@ -1,11 +1,8 @@
-zapp_binary(
+py_project(
     name = "updater",
-    main = "src/python/arrdem/updater/__main__.py",
+    main = "src/python/updater/__main__.py",
     shebang = "/usr/bin/env python3",
-    imports = [
+    lib_deps = [
-        "src/python",
-    ],
-    deps = [
         "//projects/gandi",
         py_requirement("jinja2"),
         py_requirement("pyyaml"),

projects/public_dns/src/python/arrdem/updater/__main__.py

@@ -1,197 +0,0 @@
-"""
-A quick and dirty public DNS script, super tightly coupled to my infrastructure.
-"""
-
-import argparse
-import os
-from pprint import pprint
-import re
-
-from gandi.client import GandiAPI
-import jinja2
-import meraki
-import yaml
-
-
-RECORD_LINE_PATTERN = re.compile(
-    r"^(?P<rrset_name>\S+)\s+"
-    r"(?P<rrset_ttl>\S+)\s+"
-    r"IN\s+"
-    r"(?P<rrset_type>\S+)\s+"
-    r"(?P<rrset_values>[^\s#]+)"
-    r"(?P<comment>\s*#.*?)$"
-)
-
-
-def update(m, k, f, *args, **kwargs):
-    """clojure.core/update for Python's stateful maps."""
-
-    if k in m:
-        m[k] = f(m[k], *args, **kwargs)
-
-    return m
-
-
-def parse_zone_record(line):
-    if match := RECORD_LINE_PATTERN.search(line):
-        dat = match.groupdict()
-        dat = update(dat, "rrset_ttl", int)
-        dat = update(dat, "rrset_values", lambda x: [x])
-        return dat
-
-
-def same_record(lr, rr):
-    """
-    A test to see if two records name the same zone entry.
-    """
-
-    return lr["rrset_name"] == rr["rrset_name"] and lr["rrset_type"] == rr["rrset_type"]
-
-
-def records_equate(lr, rr):
-    """
-    Equality, ignoring rrset_href which is generated by the API.
-    """
-
-    if not same_record(lr, rr):
-        return False
-
-    elif lr["rrset_ttl"] != rr["rrset_ttl"]:
-        return False
-
-    elif set(lr["rrset_values"]) != set(rr["rrset_values"]):
-        return False
-
-    else:
-        return True
-
-
-def template_and_parse_zone(template_file, template_bindings):
-    assert template_file is not None
-    assert template_bindings is not None
-
-    with open(template_file) as fp:
-        dat = jinja2.Template(fp.read()).render(**template_bindings)
-
-    uncommitted_records = []
-    for line in dat.splitlines():
-        if line and not line[0] == "#":
-            record = parse_zone_record(line)
-            if record:
-                uncommitted_records.append(record)
-
-    records = []
-
-    for uncommitted_r in uncommitted_records:
-        flag = False
-        for committed_r in records:
-            if same_record(uncommitted_r, committed_r):
-                # Join the two records
-                committed_r["rrset_values"].extend(uncommitted_r["rrset_values"])
-                flag = True
-
-        if not flag:
-            records.append(uncommitted_r)
-
-    sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"]))
-
-    return records
-
-
-def diff_zones(left_zone, right_zone):
-    """
-    Equality between unordered lists of records constituting a zone.
-    """
-
-    in_left_not_right = []
-    in_right_not_left = []
-    for lr in left_zone:
-        flag = False
-        for rr in right_zone:
-            if records_equate(lr, rr):
-                flag |= True
-
-        if not flag:
-            in_left_not_right.append(lr)
-            in_right_not_left.append(rr)
-
-    return in_left_not_right, in_right_not_left
-
-
-parser = argparse.ArgumentParser(
-    description='"Dynamic" DNS updating for self-hosted services'
-)
-parser.add_argument("--config", dest="config_file", required=True)
-parser.add_argument("--templates", dest="template_dir", required=True)
-parser.add_argument("--dry-run", dest="dry", action="store_true", default=False)
-
-
-def main():
-    args = parser.parse_args()
-
-    with open(args.config_file, "r") as fp:
-        config = yaml.safe_load(fp)
-
-    dashboard = meraki.DashboardAPI(config["meraki"]["key"], output_log=False)
-    org = config["meraki"]["organization"]
-    device = config["meraki"]["router_serial"]
-
-    uplinks = dashboard.appliance.getOrganizationApplianceUplinkStatuses(
-        organizationId=org, serials=[device]
-    )[0]["uplinks"]
-
-    template_bindings = {
-        "local": {
-            # One of the two
-            "public_v4s": [
-                link.get("publicIp") for link in uplinks if link.get("publicIp")
-            ],
-        },
-        # Why isn't there a merge method
-        **config["bindings"],
-    }
-
-    print(f"Using config {template_bindings!r}...")
-
-    api = GandiAPI(config["gandi"]["key"])
-
-    for task in config["tasks"]:
-        if isinstance(task, str):
-            task = {"template": task + ".j2", "zones": [task]}
-
-        computed_zone = template_and_parse_zone(
-            os.path.join(args.template_dir, task["template"]), template_bindings
-        )
-
-        print(f"Running task {task!r}...")
-
-        for zone_name in task["zones"] or []:
-            try:
-                live_zone = api.domain_records(zone_name)
-
-                lr, rl = diff_zones(computed_zone, live_zone)
-                if lr or rl:
-                    print(f"Zone {zone_name} differs;")
-                    print("Computed:")
-                    pprint(computed_zone)
-                    pprint("Live:")
-                    pprint(live_zone)
-                    if(rl):
-                        print("Live records not recomputed")
-                        pprint(rl)
-                    if(lr):
-                        print("New records not live")
-                        pprint(lr)
-
-                    if not args.dry:
-                        print(api.replace_domain(zone_name, computed_zone))
-                else:
-                    print("Zone {} up to date".format(zone_name))
-
-            except Exception as e:
-                print("While processing zone {}".format(zone_name))
-                raise e
-
-
-if __name__ == "__main__" or 1:
-    main()

projects/public_dns/src/python/updater/__init__.py

@@ -0,0 +1,109 @@
+"""
+A quick and dirty public DNS script, super tightly coupled to my infrastructure.
+"""
+
+import re
+
+import jinja2
+
+
+RECORD_LINE_PATTERN = re.compile(
+    r"^(?P<rrset_name>\S+)\s+(?P<rrset_ttl>\S+)\s+IN\s+(?P<rrset_type>\S+)\s+(?P<rrset_values>[^\s#]+)(?P<comment>\s*#.*?)?$"
+)
+
+
+def update(m, k, f, *args, **kwargs):
+    """clojure.core/update for Python's stateful maps."""
+
+    if k in m:
+        m[k] = f(m[k], *args, **kwargs)
+
+    return m
+
+
+def parse_zone_record(line):
+    if match := RECORD_LINE_PATTERN.match(line):
+        dat = match.groupdict()
+        dat = update(dat, "rrset_ttl", int)
+        dat = update(dat, "rrset_values", lambda x: [x])
+        return dat
+
+
+def same_record(lr, rr):
+    """
+    A test to see if two records name the same zone entry.
+    """
+
+    return lr["rrset_name"] == rr["rrset_name"] and lr["rrset_type"] == rr["rrset_type"]
+
+
+def records_equate(lr, rr):
+    """
+    Equality, ignoring rrset_href which is generated by the API.
+    """
+
+    if not same_record(lr, rr):
+        return False
+
+    elif lr["rrset_ttl"] != rr["rrset_ttl"]:
+        return False
+
+    elif set(lr["rrset_values"]) != set(rr["rrset_values"]):
+        return False
+
+    else:
+        return True
+
+
+def template_and_parse_zone(template_file, template_bindings):
+    assert template_file is not None
+    assert template_bindings is not None
+
+    with open(template_file) as fp:
+        dat = jinja2.Template(fp.read()).render(**template_bindings)
+
+    uncommitted_records = []
+    for line in dat.splitlines():
+        if line and not line[0] == "#":
+            record = parse_zone_record(line)
+            if record:
+                uncommitted_records.append(record)
+            else:
+                print("ERROR, could not parse line %r" % line)
+
+    records = []
+
+    for uncommitted_r in uncommitted_records:
+        flag = False
+        for committed_r in records:
+            if same_record(uncommitted_r, committed_r):
+                # Join the two records
+                committed_r["rrset_values"].extend(uncommitted_r["rrset_values"])
+                flag = True
+
+        if not flag:
+            records.append(uncommitted_r)
+
+    sorted(records, key=lambda x: (x["rrset_type"], x["rrset_name"]))
+
+    return records
+
+
+def diff_zones(left_zone, right_zone):
+    """
+    Equality between unordered lists of records constituting a zone.
+    """
+
+    in_left_not_right = []
+    in_right_not_left = []
+    for lr in left_zone:
+        flag = False
+        for rr in right_zone:
+            if records_equate(lr, rr):
+                flag |= True
+
+        if not flag:
+            in_left_not_right.append(lr)
+            in_right_not_left.append(rr)
+
+    return in_left_not_right, in_right_not_left

projects/public_dns/src/python/updater/__main__.py

@@ -0,0 +1,92 @@
+"""
+A quick and dirty public DNS script, super tightly coupled to my infrastructure.
+"""
+
+import argparse
+import os
+from pprint import pprint
+
+from gandi.client import GandiAPI
+import meraki
+import yaml
+
+from updater import *
+
+
+parser = argparse.ArgumentParser(
+    description='"Dynamic" DNS updating for self-hosted services'
+)
+parser.add_argument("--config", dest="config_file", required=True)
+parser.add_argument("--templates", dest="template_dir", required=True)
+parser.add_argument("--dry-run", dest="dry", action="store_true", default=False)
+
+
+def main():
+    args = parser.parse_args()
+
+    with open(args.config_file, "r") as fp:
+        config = yaml.safe_load(fp)
+
+    dashboard = meraki.DashboardAPI(config["meraki"]["key"], output_log=False)
+    org = config["meraki"]["organization"]
+    device = config["meraki"]["router_serial"]
+
+    uplinks = dashboard.appliance.getOrganizationApplianceUplinkStatuses(
+        organizationId=org, serials=[device]
+    )[0]["uplinks"]
+
+    template_bindings = {
+        "local": {
+            # One of the two
+            "public_v4s": [
+                link.get("publicIp") for link in uplinks if link.get("publicIp")
+            ],
+        },
+        # Why isn't there a merge method
+        **config["bindings"],
+    }
+
+    print(f"Using config {template_bindings!r}...")
+
+    api = GandiAPI(config["gandi"]["key"])
+
+    for task in config["tasks"]:
+        if isinstance(task, str):
+            task = {"template": task + ".j2", "zones": [task]}
+
+        computed_zone = template_and_parse_zone(
+            os.path.join(args.template_dir, task["template"]), template_bindings
+        )
+
+        print(f"Running task {task!r}...")
+
+        for zone_name in task["zones"] or []:
+            try:
+                live_zone = api.domain_records(zone_name)
+
+                lr, rl = diff_zones(computed_zone, live_zone)
+                if lr or rl:
+                    print(f"Zone {zone_name} differs;")
+                    print("Computed:")
+                    pprint(computed_zone)
+                    pprint("Live:")
+                    pprint(live_zone)
+                    if rl:
+                        print("Live records not recomputed")
+                        pprint(rl)
+                    if lr:
+                        print("New records not live")
+                        pprint(lr)
+
+                    if not args.dry:
+                        print(api.replace_domain(zone_name, computed_zone))
+                else:
+                    print("Zone {} up to date".format(zone_name))
+
+            except Exception as e:
+                print("While processing zone {}".format(zone_name))
+                raise e
+
+
+if __name__ == "__main__" or 1:
+    main()

projects/public_dns/src/resources/zonefiles/tirefireind.us.j2

@@ -16,7 +16,7 @@ www         {{ ttl }} IN A {{ link }}
 registry    {{ ttl }} IN A {{ link }}
 mirror      {{ ttl }} IN A {{ link }}
 buildcache  {{ ttl }} IN A {{ link }}
-feed        {{ ttl }} IN A {{ link }}
+tentacles   {{ ttl }} IN A {{ link }}
 ton         {{ ttl }} IN A {{ link }}
 relay       {{ ttl }} IN A {{ link }}
 pxe         {{ ttl }} IN A {{ link }}

projects/public_dns/test/python/test_parsing.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import re
+
+import pytest
+
+from updater import RECORD_LINE_PATTERN, parse_zone_record, diff_zones
+
+
+def test_record_pattern():
+    assert re.match(RECORD_LINE_PATTERN, "foo 300 IN A 1.1.1.1")
+    assert re.match(RECORD_LINE_PATTERN, "foo\t 300\t   IN   \tA\t   1.1.1.1")
+
+
+AT_RECORD = {
+    "comment": None,
+    "rrset_name": "@",
+    "rrset_ttl": 300,
+    "rrset_type": "A",
+    "rrset_values": ["67.166.27.157"],
+}
+A_RECORD = {
+    "comment": None,
+    "rrset_name": "www",
+    "rrset_ttl": 300,
+    "rrset_type": "A",
+    "rrset_values": ["67.166.27.157"],
+}
+REGISTRY_RECORD = {
+    "comment": None,
+    "rrset_name": "registry",
+    "rrset_ttl": 300,
+    "rrset_type": "A",
+    "rrset_values": ["67.166.27.157"],
+}
+MIRROR_RECORD = {
+    "comment": None,
+    "rrset_name": "mirror",
+    "rrset_ttl": 300,
+    "rrset_type": "A",
+    "rrset_values": ["67.166.27.157"],
+}
+
+
+def test_diff_zones():
+    z1 = [AT_RECORD, A_RECORD]
+    z2 = []
+    assert diff_zones(z1, z2) == z1, []
+
+    z1 = [AT_RECORD, A_RECORD]
+    z2 = [AT_RECORD]
+    assert diff_zones(z1, z2) == [A_RECORD], []
+
+    z1 = [AT_RECORD, A_RECORD]
+    z2 = [A_RECORD]
+    assert diff_zones(z1, z2) == [AT_RECORD], []
+
+    z2 = [AT_RECORD, A_RECORD]
+    z1 = [A_RECORD]
+    assert diff_zones(z1, z2) == [], [AT_RECORD]

projects/tentacles/BUILD

@@ -6,6 +6,7 @@ py_project(
     main_deps = [
         "//projects/anosql",
         "//projects/anosql-migrations",
+        py_requirement("aiosql"),
         py_requirement("attrs"),
         py_requirement("click"),
         py_requirement("flask"),

projects/tentacles/src/python/tentacles/__main__.py

@@ -23,11 +23,6 @@ from tentacles.workers import *
 from tentacles.workers import assign_jobs, Worker
 
 
-@click.group()
-def cli():
-    pass
-
-
 def db_factory(app):
     store = Db(app.config.get("db", {}).get("uri"))
     store.connect()
@@ -99,7 +94,7 @@ def make_app():
     return app
 
 
-@cli.command()
+@click.command()
 @click.option("--hostname", "hostname", type=str, default="0.0.0.0")
 @click.option("--port", "port", type=int, default=8080)
 @click.option("--trace/--no-trace", "trace", default=False)
@@ -154,4 +149,4 @@ def serve(hostname: str, port: int, config: Path, trace: bool):
 
 
 if __name__ == "__main__":
-    cli()
+    serve()

projects/tentacles/src/python/tentacles/blueprints/admin_ui.py

@@ -74,13 +74,7 @@ def get_users():
     return redirect("/admin")
 
 
-@BLUEPRINT.route("/admin/printers")
+@BLUEPRINT.route("/admin/printers", methods=["GET"])
-@requires_admin
-def printers():
-    return render_template("printers.html.j2")
-
-
-@BLUEPRINT.route("/admin/printers/add", methods=["GET"])
 @requires_admin
 def add_printer():
     return render_template("add_printer.html.j2")

projects/tentacles/src/python/tentacles/blueprints/user_ui.py

@@ -71,11 +71,14 @@ def post_register():
     try:
         username = request.form["username"]
         email = request.form["email"]
+        has_config = False
         group_id = 1  # Normal users
         status_id = -3  # Unverified
 
         for user_config in current_app.config.get("users", []):
             if user_config["email"] == email:
+                has_config = True
+
                 if "group_id" in user_config:
                     group_id = user_config["group_id"]
 
@@ -109,6 +112,11 @@ def post_register():
                 )
 
             elif user.status_id == 1:
+                # Do the entire approval dance just to be absolutely sure any statically configured users work
+                ctx.db.try_verify_user(token=user.verification_token)
+                ctx.db.approve_user(uid=user.id)
+                ctx.db.enable_user(uid=user.id)
+
                 flash("Welcome, please log in", category="success")
 
         return render_template("register.html.j2")

projects/tentacles/src/python/tentacles/templates/add_printer.html.j2

@@ -21,30 +21,9 @@
       </div>
     </div>
     <div class="row">
-      <input type="hidden" name="tested" value="false" />
+      <input id="tested" type="hidden" name="tested" value="false" />
-      <input id="test" type="button" value="Test" enabled="false" />
       <input id="submit" type="submit" value="Add" onclick="maybeSubmit();" />
     </div>
   </form>
 </div>
-
-<script type="text/javascript">
-  document.getElementById("input").disabled = True;
-
-  function testSettings() {
-    var formData = new FormData(document.getElementById("form"))
-    var req = new XMLHttpRequest();
-    req.open("POST", "/printer/test");
-    req.send(formData);
-
-  };
-
-  function maybeSubmit() {
-    if (document.getElementById("tested").value == "true") {
-      document.getElementById("form").submit();
-    } else {
-      console.error("Form values have not been tested!");
-    }
-  };
-</script>
 {% endblock %}

projects/tentacles/src/python/tentacles/templates/printers_list.html.j2

@@ -21,10 +21,10 @@
         {% endwith %}
       {% endfor %}
       {% if ctx.is_admin %}
-        <a class="button" href="/printers/add">Add a printer</a>
+        <a class="button" href="/admin/printers">Add a printer</a>
       {% endif %}
     {% else %}
-      No printers available. {% if ctx.is_admin %}<a href="/printers/add">Configure one!</a>{% else %}Ask the admin to configure one!{% endif %}
+      No printers available. {% if ctx.is_admin %}<a href="/admin/printers">Configure one!</a>{% else %}Ask the admin to configure one!{% endif %}
     {% endif %}
   {% endwith %}
 </div>

projects/tentacles/src/python/tentacles/workers.py

@@ -38,6 +38,15 @@ class OctoRest(_OR):
         response.raise_for_status()
         return response
 
+    def files_info(self, *args, **kwargs):
+        try:
+            return super().files_info(*args, **kwargs)
+        except HTTPError as e:
+            if e.response.status_code == 404:
+                return {}
+            else:
+                raise e
+
 
 log = logging.getLogger(__name__)