[NO TESTS] WIP

The latter doesn't support variable substitution

.bazelrc

@@ -0,0 +1,10 @@
+common --curses=auto
+common --show_timestamps=true
+
+test --keep_going
+test --test_output=errors
+test --test_tag_filters=-known-to-fail
+
+# To enable flake8 on all build steps, uncomment this -
+# test --aspects="//tools/flake8:flake8.bzl%flake8_aspect" --output_groups=flake8_checks
+# test --aspects="//tools/black:black.bzl%black_aspect" --output_groups=black_checks

.bazelversion

@@ -0,0 +1 @@
+7.0.0

.envrc

@@ -0,0 +1,2 @@
+export SOURCE=$(dirname $(realpath $0))
+export PATH="${SOURCE}/bin:$PATH"

.gitignore

@@ -19,6 +19,9 @@
 /**/dist
 /**/node_modules
 bazel-*
-projects/public-dns/config.yml
 public/
 tmp/
+/**/*.sqlite*
+/**/config*.toml
+/**/config*.yml
+MODULE.bazel.lock

BUILD.bazel

@@ -0,0 +1,3 @@
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["setup.cfg"])

HACKING.md

@@ -0,0 +1,60 @@
+# Hacking on/in source
+
+## Setup
+
+### Step 1 - Install bazel
+
+As usual, Ubuntu's packaging of the Bazel bootstrap script is ... considerably stale.
+Add the upstream Bazel ppa so we can get reasonably fresh builds.
+
+``` sh
+sudo apt install apt-transport-https curl gnupg
+curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg
+sudo mv bazel.gpg /etc/apt/trusted.gpg.d/
+echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
+```
+
+## Step 2 - Create a canonical `python`
+
+Bazel, sadly, expects that a platform `python` exists for things.
+Due to Bazel's plugins model, changing this is miserable.
+On Archlinux, this isn't a problem. `python` is `python3`. But Ubuntu ... did the other thing.
+
+``` sh
+sudo apt install python-is-python3
+```
+
+### Step 3 - Non-hermetic build deps
+
+The way that Bazel's Python toolchain(s) work is that ultimately they go out to the non-hermetic platform.
+So, in order to be able to use the good tools we have to have some things on the host filesystem.
+
+``` sh
+if [[ "$(sqlite3 --version | awk -F'.' '/^3/ {print $2; exit}')" -lt 35 ]]; then
+  echo "SQLite 3.35.0 or later  (select ... returning support) required"
+  exit 1
+fi
+
+sudo apt install \
+  python3-setuptools \
+  postgresql libpq-dev \
+  sqlite3 libsqlite3-dev
+```
+
+## Working in source
+
+`source activate.sh` is the key.
+It automates a number of tasks -
+1. Building a virtualenv
+2. Synchronizing the virtualenv from the requirements.txt
+3. Updating the virtualenv with all project paths
+4. Activating that virtualenv
+
+`./tools/lint.sh` wraps up various linters into a one-stop shop.
+
+`./tools/fmt.sh` wraps up various formatters into a one-stop shop.
+
+`bazel build ...` will attempt to build everything.
+While with a traditional build system this would be unreasonable, in Bazel which caches build products efficiently it's quite reasonable.
+
+`bazel test ...` likewise will run (or know it doesn't need to run) all the tests.

LICENSE

@@ -1,7 +1,21 @@
+ANTI-CAPITALIST SOFTWARE LICENSE (v 1.4)
+
 Copyright 2017 Reid 'arrdem' McKenzie <me@arrdem.com> 
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+This is anti-capitalist software, released for free use by individuals and organizations that do not operate by capitalist principles.
 
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+Permission is hereby granted, free of charge, to any person or organization (the "User") obtaining a copy of this software and associated documentation files (the "Software"), to use, copy, modify, merge, distribute, and/or sell copies of the Software, subject to the following conditions:
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+1. The above copyright notice and this permission notice shall be included in all copies or modified versions of the Software.
+
+2. The User is one of the following:
+a. An individual person, laboring for themselves
+b. A non-profit organization
+c. An educational institution
+d. An organization that seeks shared profit for all of its members, and allows non-members to set the cost of their labor
+
+3. If the User is an organization with owners, then all owners are workers and all workers are owners with equal equity and/or equal vote.
+
+4. If the User is an organization, then the User is not law enforcement or military, or working for or under either.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MODULE.bazel

@@ -0,0 +1,6 @@
+###############################################################################
+# Bazel now uses Bzlmod by default to manage external dependencies.
+# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
+#
+# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
+###############################################################################

README.md

@@ -1,17 +1,25 @@
 # Reid's monorepo
 
 I've found it inconvenient to develop lots of small Python modules.
-And so I'm going the other way; Bazel in a monorepo with several subprojects so I'm able to reuse a maximum of scaffolding.
+And so I'm going the other way; Bazel in a monorepo with subprojects so I'm able to reuse a maximum of scaffolding.
 
 ## Projects
 
-- [Datalog](projects/datalog)
+- [Datalog](projects/datalog) and the matching [shell](projects/datalog-shell)
+- [YAML Schema](projects/yamlschema) (JSON schema with knowledge of PyYAML's syntax structure & nice errors)
+- [Zapp! (now with a new home and releases)](https://github.com/arrdem/rules_zapp)
+- [Cram (now with a new home)](https://github.com/arrdem/cram)
 - [Flowmetal](projects/flowmetal)
-- [YAML Schema](projects/yamlschema)
+- [Lilith](projects/lilith)
+
+## Hacking (Ubuntu)
+
+See [HACKING.md](HACKING.md)
 
 ## License
 
-Copyright Reid 'arrdem' McKenzie, 4/8/2021.
+All work in this repo unless otherwise noted is copyright © Reid D. McKenzie 2023 or the last commit date whichever is newer and published under the terms of the Anti-Capitalist Software License v1.4 or at your option later.
 
-Unless labeled otherwise, the contents of this repository are distributed under the terms of the MIT license.
+See https://anticapitalist.software/ or the `LICENSE` file.
-See the included `LICENSE` file for more.
+
+Commercial licensing negotiable upon request.

WORKSPACE

@@ -35,27 +35,45 @@ bazel_skylib_workspace()
 ####################################################################################################
 # Python support
 ####################################################################################################
+
 # Using rules_python at a more recent SHA than the last release like a baws
 git_repository(
     name = "rules_python",
     remote = "https://github.com/bazelbuild/rules_python.git",
-    tag = "0.3.0",
+    # tag = "0.4.0",
+    commit = "5f51a4451b478c5fed491614756d16745682fd7c",
 )
 
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
+register_toolchains("//tools/python:python3_toolchain")
+
 # pip package pinnings need to be initialized.
 # this generates a bunch of bzl rules so that each pip dep is a bzl target
-load("@rules_python//python:pip.bzl", "pip_install")
+load("@rules_python//python:pip.bzl", "pip_parse")
 
-pip_install(
+pip_parse(
     name = "arrdem_source_pypi",
-    requirements = "//tools/python:requirements.txt",
+    requirements_lock = "//tools/python:requirements_lock.txt",
-    python_interpreter = "python3",
+    python_interpreter_target = "//tools/python:pythonshim",
 )
 
-####################################################################################################
+# Load the starlark macro which will define your dependencies.
-# Postscript
+load("@arrdem_source_pypi//:requirements.bzl", "install_deps")
-####################################################################################################
+
-# Do toolchain nonsense to use py3
+# Call it to define repos for your requirements.
-register_toolchains(
+install_deps()
-    "//tools/python:toolchain",
+
+git_repository(
+   name = "rules_zapp",
+   remote = "https://git.arrdem.com/arrdem/rules_zapp.git",
+   commit = "961be891e5cff539e14f2050d5cd9e82845ce0f2",
+   # tag = "0.1.2",
 )
+
+# local_repository(
+#     name = "rules_zapp",
+#     path = "/home/arrdem/Documents/hobby/programming/rules_zapp",
+# )

activate.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+mkvirtualenv source
+
+workon source
+
+pip install -r tools/python/requirements.txt
+
+for d in $(find . -type d -path "*/src/python"); do
+    d="$(realpath "${d}")"
+    echo "Adding subproject ${d}"
+    add2virtualenv "${d}"
+done

bin/bazel

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec "${SOURCE}/projects/bazelshim/src/bazelshim/__main__.py" --bazelshim_exclude="$(realpath "${SOURCE}")/bin" "$@"

bin/bazelisk

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec "${SOURCE}/projects/bazelshim/src/bazelshim/__main__.py" --bazelshim_exclude="$(realpath "${SOURCE}")/bin" "$@"

projects/activitypub_relay/BUILD.bazel

@@ -0,0 +1,15 @@
+py_project(
+    name = "activitypub_relay",
+    lib_deps = [
+        py_requirement("aiohttp"),
+        py_requirement("aiohttp_basicauth"),
+        py_requirement("async_lru"),
+        py_requirement("cachetools"),
+        py_requirement("click"),
+        py_requirement("pycryptodome"),
+        py_requirement("pyyaml"),
+        py_requirement("retry"),
+    ],
+    main = "src/relay/__main__.py",
+    shebang = "/usr/bin/env python3"
+)

projects/activitypub_relay/Dockerfile

@@ -0,0 +1,26 @@
+FROM library/python:3.10
+    MAINTAINER Reid 'arrdem' McKenzie <me@arrdem.com>
+
+RUN pip install --upgrade pip
+
+RUN useradd -d /app app
+RUN mkdir -p /app /data
+RUN chown -R app:app /app /data
+USER app
+WORKDIR /app
+VOLUME /data
+ENV DOCKER_RUNNING=true
+
+ENV PYTHONPATH="/app:${PYTHONPATH}"
+ENV PYTHONUNBUFFERED=true
+ENV PATH="/app/.local/bin:${PATH}"
+
+### App specific crap
+# Deps vary least so do them first
+RUN pip3 install --user install aiohttp aiohttp_basicauth async_lru cachetools click pycryptodome pyyaml retry
+
+COPY --chown=app:app docker_relay.sh /app/relay.sh
+COPY --chown=app:app src/python /app/
+
+EXPOSE 8080
+CMD ["/bin/sh", "/app/relay.sh"]

projects/activitypub_relay/README.md

@@ -0,0 +1,80 @@
+# ActivityPub Relay
+
+A generic ActivityPub/LitePub compatible with Pleroma and Mastodon.
+
+### What is a relay?
+
+A relay is a webserver implementing ActivityPub/LitePub.
+Normally when posting content on an ActivityPub server, that content is only listed publicly on the feed of the hosting server, and servers to which your server announces that content.
+
+Relays provide a way for ActivityPub servers to announce posts to and receive posts from a wider audience.
+For instance there are [public lists](https://github.com/brodi1/activitypub-relays) of nodes offering relaying.
+
+### Nuts and bolts of ActivityPub
+
+[ActivityPub](https://www.w3.org/TR/activitypub/) is a protocol by which [Actors](https://www.w3.org/TR/activitypub/#actors) exchange messages.
+An actor consists of two key things - an inbox and an outbox.
+The inbox is a URL to which other actors can `POST` messages.
+The outbox is a URL naming a paginated collection which other actors can `GET` to read messages from.
+
+Any user in an ActivityPub system, for instance `@arrdem@macaw.social` is an actor with such an inbox outbox pair.
+ActivityPub messages for follows of users or messages mentioning users are implemented with messages directly to the user's outbox.
+
+In addition, Mastodon ActivityPub servers themselves have ["instance actors"](https://github.com/mastodon/mastodon/issues/10453).
+These actors communicate using [server to server interactions](https://www.w3.org/TR/activitypub/#server-to-server-interactions).
+Instance actors (f. ex. `https://macaw.social/actor`) emit and receive messages regarding system level activities.
+New posts (`Create`) are the major thing, but plenty of other messages flow through too.
+
+The relay "protocol" is a Follow of the relay's actor by an ActivityPub server that wishes to participate in the relay.
+The relay will acknowledge the follow request, and then itself follow the participating server back.
+These two follow relationships in place, the participating server will announce any new posts to the relay since the relay follows it.
+This is just the normal ActivityPub behavior.
+Likewise, the relay will announce any new posts it gets from elsewhere to the participating server.
+Same deal.
+
+That's it.
+
+### Why relays at all?
+
+In theory, two ActivityPub servers can talk to each other directly.
+Why relays?
+
+Relays are 1 to N bridges.
+Follow one relay, get activity from every other host on that bridge.
+This makes it easier for new, small nodes to participate in a wider network since they only need to follow one host not follow every single other host.
+Traditional relay/clearing house model.
+
+### What is this program?
+
+The relay itself is a webserver providing two major components.
+- A hardcoded webfinger implementation for resolving the user `acct:relay@your.relay.hostname`
+- A message relay, which will perform a _stateless_ relay of new activity to connected nodes
+
+The relay offers three moderation capabilities:
+- An explicit allowlist mode restricting connected nodes
+- A denylist mode restricting connected nodes by name
+- A denylist mode restricting connected nodes by self-identified server software
+
+## Getting Started
+
+Normally, you would direct your LitePub instance software to follow the LitePub actor found on the relay.
+In Pleroma this would be something like:
+
+	$ MIX_ENV=prod mix relay_follow https://your.relay.hostname/actor
+	
+On Mastodon the process is similar, in `Administration > Relays > Add New Relay` one would list the relay's URL
+
+	https://your.relay.hostname/actor
+
+## Status
+
+- Works
+- Poorly packaged
+- Not yet threaded (nor is upstream)
+- Not yet tested (nor is upstream)
+- Missing web-oriented administrative functionality
+- Missing web-oriented configuration/state management
+
+## Copyright
+
+This work is derived from https://git.pleroma.social/pleroma/relay, which is published under the terms of the AGPLv3.

projects/activitypub_relay/deploy.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+cd "$(realpath $(dirname $0))"
+
+IMAGE=registry.tirefireind.us/arrdem/activitypub_relay
+
+docker build -t $IMAGE:latest -t relay .
+docker push $IMAGE:latest
+ssh root@relay.tirefireind.us systemctl restart activitypub-relay
+
+if [ -n "$TAG" ]; then
+    for stem in registry.tirefireind.us/arrdem/activitypub_relay registry.digitalocean.com/macawsocial/relay; do
+        docker tag relay "$stem:$TAG"
+        docker push $stem:"$TAG"
+    done
+fi

projects/activitypub_relay/docker_relay.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env sh
+
+# A launcher script for the dockerized relay
+
+set -e
+
+# First do config init if needed
+if [ ! -f "/data/config.yml" ]; then
+    python3 -m "relay" -c "/data/config.yml" setup
+fi
+
+# Then run the blame thing
+exec python3 -m "relay" -c "/data/config.yml" "${@:-run}"

projects/activitypub_relay/relay.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+cd "$(realpath $(dirname $0))"
+
+bazel build ...
+
+exec ../../bazel-bin/projects/activitypub_relay/activitypub_relay -c $(realpath ./relay_prod.yaml) run

projects/activitypub_relay/relay.yaml.example

@@ -0,0 +1,43 @@
+# this is the path that the object graph will get dumped to (in JSON-LD format),
+# you probably shouldn't change it, but you can if you want.
+db: relay.jsonld
+
+# Listener
+listen: 0.0.0.0
+port: 8080
+
+# Note
+note: "Make a note about your instance here."
+
+# maximum number of inbox posts to do at once
+post_limit: 512
+
+# this section is for ActivityPub
+ap:
+  # this is used for generating activitypub messages, as well as instructions for
+  # linking AP identities.  it should be an SSL-enabled domain reachable by https.
+  host: 'relay.example.com'
+
+  blocked_instances:
+  - 'bad-instance.example.com'
+  - 'another-bad-instance.example.com'
+
+  whitelist_enabled: false
+
+  whitelist:
+  - 'good-instance.example.com'
+  - 'another.good-instance.example.com'
+
+  # uncomment the lines below to prevent certain activitypub software from posting
+  # to the relay (all known relays by default). this uses the software name in nodeinfo
+  #blocked_software:
+  #- 'activityrelay'
+  #- 'aoderelay'
+  #- 'social.seattle.wa.us-relay'
+  #- 'unciarelay'
+
+# cache limits as number of items. only change this if you know what you're doing
+cache:
+  objects: 1024
+  actors: 1024
+  digests: 1024

projects/activitypub_relay/src/relay/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.2"

projects/activitypub_relay/src/relay/__main__.py

@@ -0,0 +1,392 @@
+import asyncio
+import logging
+import os
+from urllib.parse import urlparse
+
+import click
+from relay import __version__, misc
+from relay.application import (
+    Application,
+    request_id_middleware,
+)
+
+
+@click.group(
+    "cli", context_settings={"show_default": True}, invoke_without_command=True
+)
+@click.option("--config", "-c", default="relay.yaml", help="path to the relay's config")
+@click.version_option(version=__version__, prog_name="ActivityRelay")
+@click.pass_context
+def cli(ctx, config):
+    ctx.obj = Application(config, middlewares=[request_id_middleware])
+
+    level = {
+        "debug": logging.DEBUG,
+        "info": logging.INFO,
+        "error": logging.ERROR,
+        "critical": logging.CRITICAL,
+        "fatal": logging.FATAL,
+    }.get(os.getenv("LOG_LEVEL", "INFO").lower(), logging.INFO)
+
+    logging.basicConfig(
+        level=level,
+        format="[%(asctime)s] %(levelname)s: %(message)s",
+    )
+
+
+@cli.group("inbox")
+@click.pass_obj
+def cli_inbox(ctx: Application):
+    "Manage the inboxes in the database"
+
+
+@cli_inbox.command("list")
+@click.pass_obj
+def cli_inbox_list(obj: Application):
+    "List the connected instances or relays"
+
+    click.echo("Connected to the following instances or relays:")
+
+    for inbox in obj.database.inboxes:
+        click.echo(f"- {inbox}")
+
+
+@cli_inbox.command("follow")
+@click.argument("actor")
+@click.pass_obj
+def cli_inbox_follow(obj: Application, actor):
+    "Follow an actor (Relay must be running)"
+
+    if obj.config.is_banned(actor):
+        return click.echo(f"Error: Refusing to follow banned actor: {actor}")
+
+    if not actor.startswith("http"):
+        domain = actor
+        actor = f"https://{actor}/actor"
+
+    else:
+        domain = urlparse(actor).hostname
+
+    try:
+        inbox_data = obj.database["relay-list"][domain]
+        inbox = inbox_data["inbox"]
+
+    except KeyError:
+        actor_data = asyncio.run(misc.request(actor))
+        inbox = actor_data.shared_inbox
+
+    message = misc.Message.new_follow(host=obj.config.host, actor=actor.id)
+
+    asyncio.run(misc.request(inbox, message))
+    click.echo(f"Sent follow message to actor: {actor}")
+
+
+@cli_inbox.command("unfollow")
+@click.argument("actor")
+@click.pass_obj
+def cli_inbox_unfollow(obj: Application, actor):
+    "Unfollow an actor (Relay must be running)"
+
+    if not actor.startswith("http"):
+        domain = actor
+        actor = f"https://{actor}/actor"
+
+    else:
+        domain = urlparse(actor).hostname
+
+    try:
+        inbox_data = obj.database["relay-list"][domain]
+        inbox = inbox_data["inbox"]
+        message = misc.Message.new_unfollow(
+            host=obj.config.host, actor=actor, follow=inbox_data["followid"]
+        )
+
+    except KeyError:
+        actor_data = asyncio.run(misc.request(actor))
+        inbox = actor_data.shared_inbox
+        message = misc.Message.new_unfollow(
+            host=obj.config.host,
+            actor=actor,
+            follow={
+                "type": "Follow",
+                "object": actor,
+                "actor": f"https://{obj.config.host}/actor",
+            },
+        )
+
+    asyncio.run(misc.request(inbox, message))
+    click.echo(f"Sent unfollow message to: {actor}")
+
+
+@cli_inbox.command("add")
+@click.argument("inbox")
+@click.pass_obj
+def cli_inbox_add(obj: Application, inbox):
+    "Add an inbox to the database"
+
+    if not inbox.startswith("http"):
+        inbox = f"https://{inbox}/inbox"
+
+    if obj.config.is_banned(inbox):
+        return click.echo(f"Error: Refusing to add banned inbox: {inbox}")
+
+    if obj.database.add_inbox(inbox):
+        obj.database.save()
+        return click.echo(f"Added inbox to the database: {inbox}")
+
+    click.echo(f"Error: Inbox already in database: {inbox}")
+
+
+@cli_inbox.command("remove")
+@click.argument("inbox")
+@click.pass_obj
+def cli_inbox_remove(obj: Application, inbox):
+    "Remove an inbox from the database"
+
+    try:
+        dbinbox = obj.database.get_inbox(inbox, fail=True)
+
+    except KeyError:
+        click.echo(f"Error: Inbox does not exist: {inbox}")
+        return
+
+    obj.database.del_inbox(dbinbox["domain"])
+    obj.database.save()
+
+    click.echo(f"Removed inbox from the database: {inbox}")
+
+
+@cli.group("instance")
+def cli_instance():
+    "Manage instance bans"
+
+
+@cli_instance.command("list")
+@click.pass_obj
+def cli_instance_list(obj: Application):
+    "List all banned instances"
+
+    click.echo("Banned instances or relays:")
+
+    for domain in obj.config.blocked_instances:
+        click.echo(f"- {domain}")
+
+
+@cli_instance.command("ban")
+@click.argument("target")
+@click.pass_obj
+def cli_instance_ban(obj: Application, target):
+    "Ban an instance and remove the associated inbox if it exists"
+
+    if target.startswith("http"):
+        target = urlparse(target).hostname
+
+    if obj.config.ban_instance(target):
+        obj.config.save()
+
+        if obj.database.del_inbox(target):
+            obj.database.save()
+
+        click.echo(f"Banned instance: {target}")
+        return
+
+    click.echo(f"Instance already banned: {target}")
+
+
+@cli_instance.command("unban")
+@click.argument("target")
+@click.pass_obj
+def cli_instance_unban(obj: Application, target):
+    "Unban an instance"
+
+    if obj.config.unban_instance(target):
+        obj.config.save()
+
+        click.echo(f"Unbanned instance: {target}")
+        return
+
+    click.echo(f"Instance wasn't banned: {target}")
+
+
+@cli.group("software")
+def cli_software():
+    "Manage banned software"
+
+
+@cli_software.command("list")
+@click.pass_obj
+def cli_software_list(obj: Application):
+    "List all banned software"
+
+    click.echo("Banned software:")
+
+    for software in obj.config.blocked_software:
+        click.echo(f"- {software}")
+
+
+@cli_software.command("ban")
+@click.option(
+    "--fetch-nodeinfo/--ignore-nodeinfo",
+    "-f",
+    "fetch_nodeinfo",
+    default=False,
+    help="Treat NAME like a domain and try to fet the software name from nodeinfo",
+)
+@click.argument("name")
+@click.pass_obj
+def cli_software_ban(obj: Application, name, fetch_nodeinfo):
+    "Ban software. Use RELAYS for NAME to ban relays"
+
+    if fetch_nodeinfo:
+        software = asyncio.run(misc.fetch_nodeinfo(name))
+
+        if not software:
+            click.echo(f"Failed to fetch software name from domain: {name}")
+
+        name = software
+
+    if config.ban_software(name):
+        obj.config.save()
+        return click.echo(f"Banned software: {name}")
+
+    click.echo(f"Software already banned: {name}")
+
+
+@cli_software.command("unban")
+@click.option(
+    "--fetch-nodeinfo/--ignore-nodeinfo",
+    "-f",
+    "fetch_nodeinfo",
+    default=False,
+    help="Treat NAME like a domain and try to fet the software name from nodeinfo",
+)
+@click.argument("name")
+@click.pass_obj
+def cli_software_unban(obj: Application, name, fetch_nodeinfo):
+    "Ban software. Use RELAYS for NAME to unban relays"
+
+    if fetch_nodeinfo:
+        software = asyncio.run(misc.fetch_nodeinfo(name))
+
+        if not software:
+            click.echo(f"Failed to fetch software name from domain: {name}")
+
+        name = software
+
+    if obj.config.unban_software(name):
+        obj.config.save()
+        return click.echo(f"Unbanned software: {name}")
+
+    click.echo(f"Software wasn't banned: {name}")
+
+
+@cli.group("whitelist")
+def cli_whitelist():
+    "Manage the instance whitelist"
+
+
+@cli_whitelist.command("list")
+@click.pass_obj
+def cli_whitelist_list(obj: Application):
+    click.echo("Current whitelisted domains")
+
+    for domain in obj.config.whitelist:
+        click.echo(f"- {domain}")
+
+
+@cli_whitelist.command("add")
+@click.argument("instance")
+@click.pass_obj
+def cli_whitelist_add(obj: Application, instance):
+    "Add an instance to the whitelist"
+
+    if not obj.config.add_whitelist(instance):
+        return click.echo(f"Instance already in the whitelist: {instance}")
+
+    obj.config.save()
+    click.echo(f"Instance added to the whitelist: {instance}")
+
+
+@cli_whitelist.command("remove")
+@click.argument("instance")
+@click.pass_obj
+def cli_whitelist_remove(obj: Application, instance):
+    "Remove an instance from the whitelist"
+
+    if not obj.config.del_whitelist(instance):
+        return click.echo(f"Instance not in the whitelist: {instance}")
+
+    obj.config.save()
+
+    if obj.config.whitelist_enabled:
+        if obj.database.del_inbox(inbox):
+            obj.database.save()
+
+    click.echo(f"Removed instance from the whitelist: {instance}")
+
+
+@cli.command("setup")
+@click.pass_obj
+def relay_setup(obj: Application):
+    "Generate a new config"
+
+    if not obj.config.is_docker:
+        while True:
+            obj.config.host = os.getenv("RELAY_HOSTNAME") or click.prompt(
+                "What domain will the relay be hosted on?", default=obj.config.host
+            )
+
+            if not obj.config.host.endswith("example.com"):
+                break
+
+            click.echo("The domain must not be example.com")
+
+        obj.config.listen = os.getenv("LISTEN_ADDRESS") or click.prompt(
+            "Which address should the relay listen on?", default=obj.config.listen
+        )
+
+        while True:
+            obj.config.port = click.prompt(
+                "What TCP port should the relay listen on?",
+                default=obj.config.port,
+                type=int,
+            )
+            break
+
+    else:
+        obj.config.listen = os.getenv("LISTEN_ADDRESS", obj.config.listen)
+        obj.config.port = int(os.getenv("LISTEN_PORT", obj.config.port))
+        obj.config.host = os.getenv("RELAY_HOSTNAME")
+        obj.config.admin_token = os.getenv("RELAY_ADMIN_TOKEN")
+        if not obj.config.host:
+            click.echo("Error: No relay host configured! Set $RELAY_HOSTNAME")
+            exit(1)
+
+    obj.config.save()
+
+    if not obj.config.is_docker and click.confirm(
+        "Relay all setup! Would you like to run it now?"
+    ):
+        relay_run.callback()
+
+
+@cli.command("run")
+@click.pass_obj
+def relay_run(obj: Application):
+    "Run the relay"
+
+    if obj.config.host.endswith("example.com"):
+        return click.echo(
+            'Relay is not set up. Please edit your relay config or run "activityrelay setup".'
+        )
+
+    if not misc.check_open_port(obj.config.listen, obj.config.port):
+        return click.echo(
+            f"Error: A server is already running on port {obj.config.port}"
+        )
+
+    obj.run()
+
+
+if __name__ == "__main__":
+    cli(prog_name="relay")

projects/activitypub_relay/src/relay/application.py

@@ -0,0 +1,134 @@
+import asyncio
+from datetime import datetime, timedelta
+import logging
+import os
+import signal
+from uuid import uuid4
+
+from aiohttp import web
+from cachetools import LRUCache
+from relay.config import RelayConfig
+from relay.database import RelayDatabase
+from relay.misc import (
+    check_open_port,
+    DotDict,
+    set_app,
+)
+from relay.views import routes
+
+
+class Application(web.Application):
+    def __init__(self, cfgpath, middlewares=None):
+        web.Application.__init__(self, middlewares=middlewares)
+
+        self["starttime"] = None
+        self["running"] = False
+        self["is_docker"] = bool(os.environ.get("DOCKER_RUNNING"))
+        self["config"] = RelayConfig(cfgpath)
+
+        if not self["config"].load():
+            self["config"].save()
+
+        self["database"] = RelayDatabase(self["config"])
+        self["database"].load()
+
+        self["cache"] = DotDict(
+            {
+                key: Cache(maxsize=self["config"][key])
+                for key in self["config"].cachekeys
+            }
+        )
+        self["semaphore"] = asyncio.Semaphore(self["config"].push_limit)
+
+        self.set_signal_handler()
+        set_app(self)
+
+    @property
+    def cache(self):
+        return self["cache"]
+
+    @property
+    def config(self):
+        return self["config"]
+
+    @property
+    def database(self):
+        return self["database"]
+
+    @property
+    def is_docker(self):
+        return self["is_docker"]
+
+    @property
+    def semaphore(self):
+        return self["semaphore"]
+
+    @property
+    def uptime(self):
+        if not self["starttime"]:
+            return timedelta(seconds=0)
+
+        uptime = datetime.now() - self["starttime"]
+
+        return timedelta(seconds=uptime.seconds)
+
+    def set_signal_handler(self):
+        signal.signal(signal.SIGHUP, self.stop)
+        signal.signal(signal.SIGINT, self.stop)
+        signal.signal(signal.SIGQUIT, self.stop)
+        signal.signal(signal.SIGTERM, self.stop)
+
+    def run(self):
+        if not check_open_port(self.config.listen, self.config.port):
+            return logging.error(
+                f"A server is already running on port {self.config.port}"
+            )
+
+        for route in routes:
+            self.router.add_route(*route)
+
+        logging.info(
+            f"Starting webserver at {self.config.host} ({self.config.listen}:{self.config.port})"
+        )
+        asyncio.run(self.handle_run())
+
+        # Save off config before exit
+        self.config.save()
+        self.database.save()
+
+    def stop(self, *_):
+        self["running"] = False
+
+    async def handle_run(self):
+        self["running"] = True
+
+        runner = web.AppRunner(
+            self, access_log_format='%{X-Forwarded-For}i "%r" %s %b "%{User-Agent}i"'
+        )
+        await runner.setup()
+
+        site = web.TCPSite(
+            runner, host=self.config.listen, port=self.config.port, reuse_address=True
+        )
+
+        await site.start()
+        self["starttime"] = datetime.now()
+
+        while self["running"]:
+            await asyncio.sleep(0.25)
+
+        await site.stop()
+
+        self["starttime"] = None
+        self["running"] = False
+
+
+class Cache(LRUCache):
+    def set_maxsize(self, value):
+        self.__maxsize = int(value)
+
+
+@web.middleware
+async def request_id_middleware(request, handler):
+    request.id = uuid4()
+    return await handler(request)

projects/activitypub_relay/src/relay/config.py

@@ -0,0 +1,202 @@
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+from relay.misc import DotDict
+import yaml
+
+
+class RelayConfig(DotDict):
+    apkeys = {
+        "host",
+        "whitelist_enabled",
+        "blocked_software",
+        "blocked_instances",
+        "whitelist",
+    }
+
+    cachekeys = {"json", "objects", "digests"}
+
+    def __init__(self, path):
+        DotDict.__init__(self, {})
+
+        self._path = Path(path).expanduser()
+
+        self.reset()
+
+    def __setitem__(self, key, value):
+
+        if key in ["blocked_instances", "blocked_software", "whitelist"]:
+            assert isinstance(value, (list, set, tuple))
+
+        elif key in ["port", "json", "objects", "digests"]:
+            assert isinstance(value, (int))
+
+        elif key == "whitelist_enabled":
+            assert isinstance(value, bool)
+
+        super().__setitem__(key, value)
+
+    @property
+    def is_docker(self):
+        return bool(os.getenv("DOCKER_RUNNING"))
+
+    @property
+    def db(self):
+        return Path(self["db"]).expanduser().resolve()
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def actor(self):
+        return f"https://{self.host}/actor"
+
+    @property
+    def inbox(self):
+        return f"https://{self.host}/inbox"
+
+    @property
+    def keyid(self):
+        return f"{self.actor}#main-key"
+
+    def reset(self):
+        self.clear()
+        self.update(
+            {
+                "db": str(self._path.parent.joinpath(f"{self._path.stem}.jsonld")),
+                "listen": "0.0.0.0",
+                "port": 8080,
+                "note": "Make a note about your instance here.",
+                "push_limit": 512,
+                "host": "relay.example.com",
+                "blocked_software": [],
+                "blocked_instances": [],
+                "whitelist": [],
+                "whitelist_enabled": False,
+                "json": 1024,
+                "objects": 1024,
+                "digests": 1024,
+                "admin_token": None,
+            }
+        )
+
+    def ban_instance(self, instance):
+        if instance.startswith("http"):
+            instance = urlparse(instance).hostname
+
+        if self.is_banned(instance):
+            return False
+
+        self.blocked_instances.append(instance)
+        return True
+
+    def unban_instance(self, instance):
+        if instance.startswith("http"):
+            instance = urlparse(instance).hostname
+
+        try:
+            self.blocked_instances.remove(instance)
+            return True
+
+        except:
+            return False
+
+    def ban_software(self, software):
+        if self.is_banned_software(software):
+            return False
+
+        self.blocked_software.append(software)
+        return True
+
+    def unban_software(self, software):
+        try:
+            self.blocked_software.remove(software)
+            return True
+
+        except:
+            return False
+
+    def add_whitelist(self, instance):
+        if instance.startswith("http"):
+            instance = urlparse(instance).hostname
+
+        if self.is_whitelisted(instance):
+            return False
+
+        self.whitelist.append(instance)
+        return True
+
+    def del_whitelist(self, instance):
+        if instance.startswith("http"):
+            instance = urlparse(instance).hostname
+
+        try:
+            self.whitelist.remove(instance)
+            return True
+
+        except:
+            return False
+
+    def is_banned(self, instance):
+        if instance.startswith("http"):
+            instance = urlparse(instance).hostname
+
+        return instance in self.blocked_instances
+
+    def is_banned_software(self, software):
+        if not software:
+            return False
+
+        return software.lower() in self.blocked_software
+
+    def is_whitelisted(self, instance):
+        if instance.startswith("http"):
+            instance = urlparse(instance).hostname
+
+        return instance in self.whitelist
+
+    def load(self):
+        self.reset()
+
+        options = {}
+
+        try:
+            options["Loader"] = yaml.FullLoader
+
+        except AttributeError:
+            pass
+
+        try:
+            with open(self.path) as fd:
+                config = yaml.load(fd, **options)
+
+        except FileNotFoundError:
+            return False
+
+        if not config:
+            return False
+
+        for key, value in config.items():
+            if key in ["ap", "cache"]:
+                for k, v in value.items():
+                    if k not in self:
+                        continue
+
+                    self[k] = v
+
+            elif key not in self:
+                continue
+
+            self[key] = value
+
+        if self.host.endswith("example.com"):
+            return False
+
+        return True
+
+    def save(self):
+        with open(self._path, "w") as fd:
+            fd.write("---\n")
+            yaml.dump(self.jsonify(), fd, sort_keys=True)

projects/activitypub_relay/src/relay/database.py

@@ -0,0 +1,159 @@
+import json
+import logging
+from urllib.parse import urlparse
+
+from Crypto.PublicKey import RSA
+
+
+class RelayDatabase(dict):
+    def __init__(self, config):
+        dict.__init__(
+            self,
+            {
+                "relay-list": {},
+                "private-key": None,
+                "follow-requests": {},
+                "version": 1,
+            },
+        )
+
+        self.config = config
+        self.PRIVKEY = None
+
+    @property
+    def PUBKEY(self):
+        return self.PRIVKEY.publickey()
+
+    @property
+    def pubkey(self):
+        return self.PUBKEY.exportKey("PEM").decode("utf-8")
+
+    @property
+    def privkey(self):
+        return self["private-key"]
+
+    @property
+    def hostnames(self):
+        return tuple(self["relay-list"].keys())
+
+    @property
+    def inboxes(self):
+        return tuple(data["inbox"] for data in self["relay-list"].values())
+
+    def generate_key(self):
+        self.PRIVKEY = RSA.generate(4096)
+        self["private-key"] = self.PRIVKEY.exportKey("PEM").decode("utf-8")
+
+    def load(self):
+        with self.config.db.open() as fd:
+            data = json.load(fd)
+
+        self["version"] = data.get("version", 1)
+        self["private-key"] = data.get("private-key")
+        self["relay-list"] = data.get("relay-list", {})
+        self["follow-requests"] = data.get("follow-requests")
+
+        if not self.privkey:
+            logging.info("No actor keys present, generating 4096-bit RSA keypair.")
+            self.generate_key()
+            self.save()
+
+        else:
+            self.PRIVKEY = RSA.importKey(self.privkey)
+
+    def save(self):
+        with self.config.db.open("w") as fd:
+            json.dump(self, fd, indent=4)
+
+    def get_inbox(self, domain, fail=False):
+        if domain.startswith("http"):
+            domain = urlparse(domain).hostname
+
+        if domain not in self["relay-list"]:
+            if fail:
+                raise KeyError(domain)
+
+            return
+
+        return self["relay-list"][domain]
+
+    def add_inbox(self, inbox, followid=None, fail=False):
+        assert inbox.startswith("https"), "Inbox must be a url"
+        domain = urlparse(inbox).hostname
+
+        if self.get_inbox(domain):
+            if fail:
+                raise KeyError(domain)
+
+            return False
+
+        self["relay-list"][domain] = {
+            "domain": domain,
+            "inbox": inbox,
+            "followid": followid,
+        }
+
+        logging.debug(f"Added inbox to database: {inbox}")
+        return self["relay-list"][domain]
+
+    def del_inbox(self, domain, followid=None, fail=False):
+        data = self.get_inbox(domain, fail=False)
+
+        if not data:
+            if fail:
+                raise KeyError(domain)
+
+            return False
+
+        if not data["followid"] or not followid or data["followid"] == followid:
+            del self["relay-list"][data["domain"]]
+            logging.debug(f'Removed inbox from database: {data["inbox"]}')
+            return True
+
+        if fail:
+            raise ValueError("Follow IDs do not match")
+
+        logging.debug(
+            f'Follow ID does not match: db = {data["followid"]}, object = {followid}'
+        )
+        return False
+
+    def set_followid(self, domain, followid):
+        if (data := self.get_inbox(domain, fail=True)):
+            data["followid"] = followid
+
+    def get_request(self, domain, fail=True):
+        if domain.startswith("http"):
+            domain = urlparse(domain).hostname
+
+        try:
+            return self["follow-requests"][domain]
+
+        except KeyError as e:
+            if fail:
+                raise e
+
+    def add_request(self, actor, inbox, followid):
+        domain = urlparse(inbox).hostname
+
+        try:
+            if (request := self.get_request(domain)):
+                request["followid"] = followid
+
+        except KeyError:
+            pass
+
+        self["follow-requests"][domain] = {
+            "actor": actor,
+            "inbox": inbox,
+            "followid": followid,
+        }
+
+    def del_request(self, domain):
+        if domain.startswith("http"):
+            domain = urlparse(domain).hostname
+
+        del self["follow-requests"][domain]
+
+    def get_requests(self):
+        return list(self["follow-requests"].items())

projects/activitypub_relay/src/relay/http_debug.py

@@ -0,0 +1,62 @@
+from collections import defaultdict
+import logging
+
+import aiohttp
+
+
+STATS = {
+    "requests": defaultdict(int),
+    "response_codes": defaultdict(int),
+    "response_codes_per_domain": defaultdict(lambda: defaultdict(int)),
+    "delivery_codes": defaultdict(int),
+    "delivery_codes_per_domain": defaultdict(lambda: defaultdict(int)),
+    "exceptions": defaultdict(int),
+    "exceptions_per_domain": defaultdict(lambda: defaultdict(int)),
+    "delivery_exceptions": defaultdict(int),
+    "delivery_exceptions_per_domain": defaultdict(lambda: defaultdict(int)),
+}
+
+
+async def on_request_start(session, trace_config_ctx, params):
+    logging.debug("HTTP START [%r], [%r]", session, params)
+
+    STATS["requests"][params.url.host] += 1
+
+
+async def on_request_end(session, trace_config_ctx, params):
+    logging.debug("HTTP END [%r], [%r]", session, params)
+
+    host = params.url.host
+    status = params.response.status
+
+    STATS["response_codes"][status] += 1
+    STATS["response_codes_per_domain"][host][status] += 1
+
+    if params.method == "POST":
+        STATS["delivery_codes"][status] += 1
+        STATS["delivery_codes_per_domain"][host][status] += 1
+
+
+async def on_request_exception(session, trace_config_ctx, params):
+    logging.debug("HTTP EXCEPTION [%r], [%r]", session, params)
+
+    host = params.url.host
+    exception = repr(params.exception)
+
+    STATS["exceptions"][exception] += 1
+    STATS["exceptions_per_domain"][host][exception] += 1
+
+    if params.method == "POST":
+        STATS["delivery_exceptions"][exception] += 1
+        STATS["delivery_exceptions_per_domain"][host][exception] += 1
+
+
+def http_debug():
+    if logging.DEBUG >= logging.root.level:
+        return
+
+    trace_config = aiohttp.TraceConfig()
+    trace_config.on_request_start.append(on_request_start)
+    trace_config.on_request_end.append(on_request_end)
+    trace_config.on_request_exception.append(on_request_exception)
+    return [trace_config]

projects/activitypub_relay/src/relay/misc.py

@@ -0,0 +1,515 @@
+import base64
+from datetime import datetime
+import json
+from json.decoder import JSONDecodeError
+import logging
+import socket
+from typing import Union
+from urllib.parse import urlparse
+import uuid
+
+from Crypto.Hash import SHA, SHA256, SHA512
+from Crypto.PublicKey import RSA
+from Crypto.Signature import PKCS1_v1_5
+from aiohttp import ClientSession
+from aiohttp.client_exceptions import (
+    ClientConnectorError,
+    ClientResponseError,
+)
+from aiohttp.web import (
+    Response as AiohttpResponse,
+)
+from async_lru import alru_cache
+from relay.http_debug import http_debug
+from retry import retry
+
+
+app = None
+
+HASHES = {"sha1": SHA, "sha256": SHA256, "sha512": SHA512}
+
+MIMETYPES = {
+    "activity": "application/activity+json",
+    "html": "text/html",
+    "json": "application/json",
+    "text": "text/plain",
+}
+
+NODEINFO_NS = {
+    "20": "http://nodeinfo.diaspora.software/ns/schema/2.0",
+    "21": "http://nodeinfo.diaspora.software/ns/schema/2.1",
+}
+
+
+def set_app(new_app):
+    global app
+    app = new_app
+
+
+def build_signing_string(headers, used_headers):
+    return "\n".join(map(lambda x: ": ".join([x.lower(), headers[x]]), used_headers))
+
+
+def check_open_port(host, port):
+    if host == "0.0.0.0":
+        host = "127.0.0.1"
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        try:
+            return s.connect_ex((host, port)) != 0
+
+        except socket.error as e:
+            return False
+
+
+def create_signature_header(headers):
+    headers = {k.lower(): v for k, v in headers.items()}
+    used_headers = headers.keys()
+    sigstring = build_signing_string(headers, used_headers)
+
+    sig = {
+        "keyId": app.config.keyid,
+        "algorithm": "rsa-sha256",
+        "headers": " ".join(used_headers),
+        "signature": sign_signing_string(sigstring, app.database.PRIVKEY),
+    }
+
+    chunks = ['{}="{}"'.format(k, v) for k, v in sig.items()]
+    return ",".join(chunks)
+
+
+def distill_inboxes(object_id):
+    for inbox in app.database.inboxes:
+        if (urlparse(inbox).hostname != urlparse(object_id).hostname):
+            yield inbox
+
+
+def generate_body_digest(body):
+    h = SHA256.new(body.encode("utf-8"))
+    bodyhash = base64.b64encode(h.digest()).decode("utf-8")
+
+    return bodyhash
+
+
+def sign_signing_string(sigstring, key):
+    pkcs = PKCS1_v1_5.new(key)
+    h = SHA256.new()
+    h.update(sigstring.encode("ascii"))
+    sigdata = pkcs.sign(h)
+
+    return base64.b64encode(sigdata).decode("utf-8")
+
+
+def split_signature(sig):
+    default = {"headers": "date"}
+
+    sig = sig.strip().split(",")
+
+    for chunk in sig:
+        k, _, v = chunk.partition("=")
+        v = v.strip('"')
+        default[k] = v
+
+    default["headers"] = default["headers"].split()
+    return default
+
+
+async def fetch_actor_key(actor):
+    actor_data = await request(actor)
+
+    if not actor_data:
+        return None
+
+    try:
+        return RSA.importKey(actor_data["publicKey"]["publicKeyPem"])
+
+    except Exception as e:
+        logging.debug(f"Exception occured while fetching actor key: {e}")
+
+
+@alru_cache
+async def fetch_nodeinfo(domain):
+    nodeinfo_url = None
+    wk_nodeinfo = await request(
+        f"https://{domain}/.well-known/nodeinfo", sign_headers=False, activity=False
+    )
+
+    if not wk_nodeinfo:
+        return
+
+    wk_nodeinfo = WKNodeinfo(wk_nodeinfo)
+
+    for version in ["20", "21"]:
+        try:
+            nodeinfo_url = wk_nodeinfo.get_url(version)
+
+        except KeyError:
+            pass
+
+    if not nodeinfo_url:
+        logging.debug(f"Failed to fetch nodeinfo url for domain: {domain}")
+        return False
+
+    nodeinfo = await request(nodeinfo_url, sign_headers=False, activity=False)
+
+    try:
+        return nodeinfo["software"]["name"]
+
+    except KeyError:
+        return False
+
+
+@retry(exceptions=(ClientConnectorError, ClientResponseError), backoff=1, tries=3)
+async def request(uri, data=None, force=False, sign_headers=True, activity=True):
+    ## If a get request and not force, try to use the cache first
+    url = urlparse(uri)
+    method = "POST" if data else "GET"
+    action = data.get("type") if data else None
+    headers = {
+        "Accept": f'{MIMETYPES["activity"]}, {MIMETYPES["json"]};q=0.9',
+        "User-Agent": "ActivityRelay",
+    }
+
+    if data:
+        headers["Content-Type"] = MIMETYPES["activity" if activity else "json"]
+
+    if sign_headers:
+        signing_headers = {
+            "(request-target)": f"{method.lower()} {url.path}",
+            "Date": datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT"),
+            "Host": url.netloc,
+        }
+
+        if data:
+            assert isinstance(data, dict)
+
+            data = json.dumps(data)
+            signing_headers.update(
+                {
+                    "Digest": f"SHA-256={generate_body_digest(data)}",
+                    "Content-Length": str(len(data.encode("utf-8"))),
+                }
+            )
+
+        signing_headers["Signature"] = create_signature_header(signing_headers)
+
+        del signing_headers["(request-target)"]
+        del signing_headers["Host"]
+
+        headers.update(signing_headers)
+
+    logging.info("%s %s", method, uri)
+    # logging.debug("%r %r %r %r", method, uri, headers, data)
+
+    async with ClientSession(trace_configs=http_debug()) as session:
+        async with session.request(method, uri, headers=headers, data=data) as resp:
+            resp.raise_for_status()
+
+            ## aiohttp has been known to leak if the response hasn't been read,
+            ## so we're just gonna read the request no matter what
+            resp_data = await resp.read()
+
+            ## Not expecting a response, so just return
+            if resp.status == 202:
+                return
+
+            elif resp.status != 200:
+                if not resp_data:
+                    return logging.debug(f"Received error when requesting {uri}: {resp.status} {resp_data}")
+
+                return logging.debug(f"Received error when sending {action} to {uri}: {resp.status} {resp_data}")
+
+            if resp.content_type == MIMETYPES["activity"]:
+                resp_data = await resp.json(loads=Message.new_from_json)
+
+            elif resp.content_type == MIMETYPES["json"]:
+                resp_data = await resp.json(loads=DotDict.new_from_json)
+
+            else:
+                logging.debug(f'Invalid Content-Type for "{url}": {resp.content_type}')
+                return logging.debug(f"Response: {resp_data}")
+
+            logging.debug(f"{uri} >> resp {resp_data}")
+
+            return resp_data
+
+
+async def validate_signature(actor, http_request):
+    pubkey = await fetch_actor_key(actor)
+
+    if not pubkey:
+        return False
+
+    logging.debug(f"actor key: {pubkey}")
+
+    headers = {key.lower(): value for key, value in http_request.headers.items()}
+    headers["(request-target)"] = " ".join(
+        [
+            http_request.method.lower(),
+            http_request.path
+        ]
+    )
+
+    sig = split_signature(headers["signature"])
+    logging.debug(f"sigdata: {sig}")
+
+    sigstring = build_signing_string(headers, sig["headers"])
+    logging.debug(f"sigstring: {sigstring}")
+
+    sign_alg, _, hash_alg = sig["algorithm"].partition("-")
+    logging.debug(f"sign alg: {sign_alg}, hash alg: {hash_alg}")
+
+    sigdata = base64.b64decode(sig["signature"])
+
+    pkcs = PKCS1_v1_5.new(pubkey)
+    h = HASHES[hash_alg].new()
+    h.update(sigstring.encode("ascii"))
+    result = pkcs.verify(h, sigdata)
+
+    http_request["validated"] = result
+
+    logging.debug(f"validates? {result}")
+    return result
+
+
+class DotDict(dict):
+    def __init__(self, _data, **kwargs):
+        dict.__init__(self)
+
+        self.update(_data, **kwargs)
+
+    def __hasattr__(self, k):
+        return k in self
+
+    def __getattr__(self, k):
+        try:
+            return self[k]
+
+        except KeyError:
+            raise AttributeError(
+                f"{self.__class__.__name__} object has no attribute {k}"
+            ) from None
+
+    def __setattr__(self, k, v):
+        if k.startswith("_"):
+            super().__setattr__(k, v)
+
+        else:
+            self[k] = v
+
+    def __setitem__(self, k, v):
+        if type(v) == dict:
+            v = DotDict(v)
+
+        super().__setitem__(k, v)
+
+    def __delattr__(self, k):
+        try:
+            dict.__delitem__(self, k)
+
+        except KeyError:
+            raise AttributeError(
+                f"{self.__class__.__name__} object has no attribute {k}"
+            ) from None
+
+    @classmethod
+    def new_from_json(cls, data):
+        if not data:
+            raise JSONDecodeError("Empty body", data, 1)
+
+        try:
+            return cls(json.loads(data))
+
+        except ValueError:
+            raise JSONDecodeError("Invalid body", data, 1)
+
+    def to_json(self, indent=None):
+        return json.dumps(self, indent=indent)
+
+    def jsonify(self):
+        def _xform(v):
+            if hasattr(v, "jsonify"):
+                return v.jsonify()
+            else:
+                return v
+
+        return {k: _xform(v) for k, v in self.items()}
+
+    def update(self, _data, **kwargs):
+        if isinstance(_data, dict):
+            for key, value in _data.items():
+                self[key] = value
+
+        elif isinstance(_data, (list, tuple, set)):
+            for key, value in _data:
+                self[key] = value
+
+        for key, value in kwargs.items():
+            self[key] = value
+
+    # misc properties
+    @property
+    def domain(self):
+        return urlparse(getattr(self, "id", None) or getattr(self, "actor")).hostname
+
+    # actor properties
+    @property
+    def pubkey(self):
+        return self.publicKey.publicKeyPem
+
+    @property
+    def shared_inbox(self):
+        return self.get("endpoints", {}).get("sharedInbox", self.inbox)
+
+    # activity properties
+    @property
+    def actorid(self):
+        if isinstance(self.actor, dict):
+            return self.actor.id
+
+        return self.actor
+
+    @property
+    def objectid(self):
+        if isinstance(self.object, dict):
+            return self.object.id
+
+        return self.object
+
+
+class Message(DotDict):
+    @classmethod
+    def new_actor(cls, host, pubkey, description=None):
+        return cls(
+            {
+                "@context": "https://www.w3.org/ns/activitystreams",
+                "id": f"https://{host}/actor",
+                "type": "Application",
+                "preferredUsername": "relay",
+                "name": "ActivityRelay",
+                "summary": description or "ActivityRelay bot",
+                "followers": f"https://{host}/followers",
+                "following": f"https://{host}/following",
+                "inbox": f"https://{host}/inbox",
+                "url": f"https://{host}/inbox",
+                "endpoints": {"sharedInbox": f"https://{host}/inbox"},
+                "publicKey": {
+                    "id": f"https://{host}/actor#main-key",
+                    "owner": f"https://{host}/actor",
+                    "publicKeyPem": pubkey,
+                },
+            }
+        )
+
+    @classmethod
+    def new_announce(cls, host, object):
+        return cls(
+            {
+                "@context": "https://www.w3.org/ns/activitystreams",
+                "id": f"https://{host}/activities/{uuid.uuid4()}",
+                "type": "Announce",
+                "to": [f"https://{host}/followers"],
+                "actor": f"https://{host}/actor",
+                "object": object,
+            }
+        )
+
+    @classmethod
+    def new_follow(cls, host, actor):
+        return cls(
+            {
+                "@context": "https://www.w3.org/ns/activitystreams",
+                "type": "Follow",
+                "to": [actor],
+                "object": actor,
+                "id": f"https://{host}/activities/{uuid.uuid4()}",
+                "actor": f"https://{host}/actor",
+            }
+        )
+
+    @classmethod
+    def new_unfollow(cls, host, actor, follow):
+        return cls(
+            {
+                "@context": "https://www.w3.org/ns/activitystreams",
+                "id": f"https://{host}/activities/{uuid.uuid4()}",
+                "type": "Undo",
+                "to": [actor],
+                "actor": f"https://{host}/actor",
+                "object": follow,
+            }
+        )
+
+    @classmethod
+    def new_response(cls, host, actor, followid, accept):
+        return cls(
+            {
+                "@context": "https://www.w3.org/ns/activitystreams",
+                "id": f"https://{host}/activities/{uuid.uuid4()}",
+                "type": "Accept" if accept else "Reject",
+                "to": [actor],
+                "actor": f"https://{host}/actor",
+                "object": {
+                    "id": followid,
+                    "type": "Follow",
+                    "object": f"https://{host}/actor",
+                    "actor": actor,
+                },
+            }
+        )
+
+
+class Response(AiohttpResponse):
+    @classmethod
+    def new(cls, body: Union[dict, str, bytes] = "", status=200, headers=None, ctype="text"):
+        kwargs = {
+            "status": status,
+            "headers": headers,
+            "content_type": MIMETYPES[ctype],
+        }
+
+        if isinstance(body, bytes):
+            kwargs["body"] = body
+
+        elif isinstance(body, dict) and ctype in {"json", "activity"}:
+            kwargs["text"] = json.dumps(body)
+
+        else:
+            kwargs["text"] = body
+
+        return cls(**kwargs)
+
+    @classmethod
+    def new_error(cls, status, body, ctype="text"):
+        if ctype == "json":
+            body = json.dumps({"status": status, "error": body})
+
+        return cls.new(body=body, status=status, ctype=ctype)
+
+    @property
+    def location(self):
+        return self.headers.get("Location")
+
+    @location.setter
+    def location(self, value):
+        self.headers["Location"] = value
+
+
+class WKNodeinfo(DotDict):
+    @classmethod
+    def new(cls, v20, v21):
+        return cls(
+            {
+                "links": [
+                    {"rel": NODEINFO_NS["20"], "href": v20},
+                    {"rel": NODEINFO_NS["21"], "href": v21},
+                ]
+            }
+        )
+
+    def get_url(self, version="20"):
+        for item in self.links:
+            if item["rel"] == NODEINFO_NS[version]:
+                return item["href"]
+
+        raise KeyError(version)

projects/activitypub_relay/src/relay/processors.py

@@ -0,0 +1,125 @@
+import asyncio
+import logging
+
+from relay import misc
+from relay.config import RelayConfig
+from relay.database import RelayDatabase
+
+
+async def handle_relay(request, actor, data, software):
+    if data.objectid in request.app.cache.objects:
+        logging.info(f"already relayed {data.objectid}")
+        return
+
+    logging.info(f"Relaying post from {data.actorid}")
+
+    message = misc.Message.new_announce(
+        host=request.app.config.host, object=data.objectid
+    )
+
+    logging.debug(f">> relay: {message}")
+
+    inboxes = misc.distill_inboxes(data.objectid)
+    futures = [misc.request(inbox, data=message) for inbox in inboxes]
+
+    asyncio.ensure_future(asyncio.gather(*futures))
+    request.app.cache.objects[data.objectid] = message.id
+
+
+async def handle_forward(request, actor, data, software):
+    if data.id in request.app.cache.objects:
+        logging.info(f"already forwarded {data.id}")
+        return
+
+    message = misc.Message.new_announce(host=request.app.config.host, object=data)
+
+    logging.info(f"Forwarding post from {actor.id}")
+    logging.debug(f">> Relay {data}")
+
+    inboxes = misc.distill_inboxes(data.id)
+    futures = [misc.request(inbox, data=message) for inbox in inboxes]
+
+    asyncio.ensure_future(asyncio.gather(*futures))
+    request.app.cache.objects[data.id] = message.id
+
+
+async def handle_follow(request, actor, data, software):
+    config: RelayConfig = request.app.config
+    database: RelayDatabase = request.app.database
+
+    # If the following host is not whitelisted, we want to enqueue the request for review.
+    # This means saving off the two parameters we need later to issue an appropriate acceptance.
+    if config.whitelist_enabled and not config.is_whitelisted(data.domain):
+        database.add_request(actor.id, actor.shared_inbox, data.id)
+        database.save()
+        return
+
+    if not database.add_inbox(actor.shared_inbox, data.id):
+        database.set_followid(actor.id, data.id)
+        database.save()
+
+    await misc.request(
+        actor.shared_inbox,
+        misc.Message.new_response(
+            host=request.app.config.host,
+            actor=actor.id,
+            followid=data.id,
+            accept=True
+        ),
+    )
+
+    await misc.request(
+        actor.shared_inbox,
+        misc.Message.new_follow(
+            host=request.app.config.host,
+            actor=actor.id
+        ),
+    )
+
+
+async def handle_undo(request, actor, data, software):
+    # If the object is not a Follow, forward it
+    if data["object"]["type"] != "Follow":
+        return await handle_forward(request, actor, data, software)
+
+    if not request.app.database.del_inbox(actor.domain, data.id):
+        return
+
+    request.app.database.save()
+
+    message = misc.Message.new_unfollow(
+        host=request.app.config.host, actor=actor.id, follow=data
+    )
+
+    await misc.request(actor.shared_inbox, message)
+
+
+async def handle_dont(request, actor, data, software):
+    """Handle something by ... not handling it."""
+
+    logging.info(f"Disregarding {data!r}")
+
+
+processors = {
+    "Announce": handle_relay,
+    "Create": handle_relay,
+    "Delete": handle_forward,
+    "Follow": handle_follow,
+    "Undo": handle_undo,
+    "Update": handle_forward,
+}
+
+
+async def run_processor(request, actor, data, software):
+    if data.type not in processors:
+        return
+
+    logging.info(f'{request.id}: New "{data.type}" from actor: {actor.id}')
+    logging.debug(f"{request.id}: {data!r}")
+
+    env = dict(data=data, actor=actor, software=software)
+    try:
+        return await processors.get(data.type, handle_dont)(request, actor, data, software)
+
+    except:
+        logging.exception(f"{request.id}] {env!r}")

projects/activitypub_relay/src/relay/views.py

@@ -0,0 +1,299 @@
+import json
+from json.decoder import JSONDecodeError
+import logging
+
+from aiohttp.web import HTTPUnauthorized, Request
+from relay import __version__, misc
+from relay.http_debug import STATS
+from relay.misc import (
+    DotDict,
+    Message,
+    Response,
+    WKNodeinfo,
+)
+from relay.processors import run_processor
+
+
+routes = []
+
+
+def register_route(method, path):
+    def wrapper(func):
+        routes.append([method, path, func])
+        return func
+
+    return wrapper
+
+
+@register_route("GET", "/")
+async def home(request):
+    following = "<ul>" + ("\n".join(f"<li>{it}</li>" for it in request.app.database["relay-list"])) + "</ul>"
+    following_count = len(request.app.database.hostnames)
+    requested = "<ul>" + ("\n".join(f"<li>{it}</li>" for it in request.app.database["follow-requests"])) + "</ul>"
+    requested_count = len(request.app.database["follow-requests"])
+    note = request.app.config.note
+    host = request.app.config.host
+
+    text = f"""\
+<html><head>
+<title>ActivityPub Relay at {host}</title>
+<style>
+body {{ background-color: #000000; color: #FFFFFF; font-family: monospace, arial; font-size: 100%; }}
+a {{ color: #26F; }}
+a:visited {{ color: #46C; }}
+a:hover {{ color: #8AF; }}
+</style>
+</head>
+<body>
+<p>This is an Activity Relay for fediverse instances.</p>
+<p>{note}</p>
+<p>To host your own relay, you may download the code at this address: <a href="https://git.arrdem.com/arrdem/source/src/branch/trunk/projects/activitypub_relay">https://git.arrdem.com/arrdem/source/src/branch/trunk/projects/activitypub_relay</a></p>
+<br><p>This relay is peered with {following_count} registered instances:<br>{following}</p>
+<br><p>Another {requested_count} peers await approval:<br>{requested}</p>
+</body></html>"""
+
+    return Response.new(text, ctype="html")
+
+
+@register_route("GET", "/inbox")
+@register_route("GET", "/actor")
+async def actor(request):
+    data = Message.new_actor(
+        host=request.app.config.host, pubkey=request.app.database.pubkey
+    )
+
+    return Response.new(data, ctype="activity")
+
+
+@register_route("POST", "/inbox")
+@register_route("POST", "/actor")
+async def inbox(request):
+    config = request.app.config
+
+    # reject if missing signature header
+    if "signature" not in request.headers:
+        logging.debug("Actor missing signature header")
+        raise HTTPUnauthorized(body="missing signature")
+
+    # read message and get actor id and domain
+    try:
+        data = await request.json(loads=Message.new_from_json)
+
+        if "actor" not in data:
+            raise KeyError("actor")
+
+    # reject if there is no actor in the message
+    except KeyError:
+        logging.debug("actor not in data")
+        return Response.new_error(400, "no actor in message", "json")
+
+    except:
+        logging.exception("Failed to parse inbox message")
+        return Response.new_error(400, "failed to parse message", "json")
+
+    # FIXME: A lot of this code assumes that we're going to have access to the entire actor descriptor.
+    # This isn't something we actually generally need, and it's not clear how used it is.
+    # The original relay implementation mostly used to determine activity source domain for relaying.
+    # This has been refactored out, since there are other sources of this information.
+    # This PROBABLY means we can do without this data ever ... but here it is.
+
+    # Trying to deal with actors/visibility
+    if isinstance(data.object, dict) and not data.object.get("discoverable", True):
+        actor = DotDict({"id": "dummy-for-undiscoverable-object"})
+
+    # Normal path of looking up the actor...
+    else:
+        try:
+            # FIXME: Needs a cache
+            actor = await misc.request(data.actorid)
+        except:
+            logging.exception(f"{request.id}: {data!r}")
+            return
+
+    logging.debug(f"Inbox >> {data!r}")
+
+    # reject if actor is empty
+    if not actor:
+        logging.debug(f"Failed to fetch actor: {data.actorid}")
+        return Response.new_error(400, "failed to fetch actor", "json")
+
+    # Reject if the actor isn't whitelisted while the whiltelist is enabled
+    # An exception is made for follow requests, which we want to enqueue not reject out of hand
+    elif config.whitelist_enabled and not config.is_whitelisted(data.domain) and data["type"] != "Follow":
+        logging.debug(
+            f"Rejected actor for not being in the whitelist: {data.actorid}"
+        )
+        return Response.new_error(403, "access denied", "json")
+
+    # reject if actor is banned
+    if request.app["config"].is_banned(data.domain):
+        logging.debug(f"Ignored request from banned actor: {data.actorid}")
+        return Response.new_error(403, "access denied", "json")
+
+    # FIXME: Needs a cache
+    software = await misc.fetch_nodeinfo(data.domain)
+
+    # reject if software used by actor is banned
+    if config.blocked_software:
+        if config.is_banned_software(software):
+            logging.debug(f"Rejected actor for using specific software: {software}")
+            return Response.new_error(403, "access denied", "json")
+
+    # reject if the signature is invalid
+    if not (await misc.validate_signature(data.actorid, request)):
+        logging.debug(f"signature validation failed for: {data.actorid}")
+        return Response.new_error(401, "signature check failed", "json")
+
+    logging.debug(f">> payload {data}")
+
+    resp = await run_processor(request, actor, data, software)
+    return resp or Response.new(status=202)
+
+
+@register_route("GET", "/.well-known/webfinger")
+async def webfinger(request):
+    if not (subject := request.query.get("resource")):
+        return Response.new_error(404, "no resource specified", "json")
+
+    if subject != f"acct:relay@{request.app.config.host}":
+        return Response.new_error(404, "user not found", "json")
+
+    data = {
+        "subject": subject,
+        "aliases": [request.app.config.actor],
+        "links": [
+            {
+                "href": request.app.config.actor,
+                "rel": "self",
+                "type": "application/activity+json",
+            },
+            {
+                "href": request.app.config.actor,
+                "rel": "self",
+                "type": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+            },
+        ],
+    }
+
+    return Response.new(data, ctype="json")
+
+
+@register_route("GET", "/nodeinfo/{version:\d.\d\.json}")
+async def nodeinfo_2_0(request):
+    niversion = request.match_info["version"][:3]
+    data = {
+        "openRegistrations": not request.app.config.whitelist_enabled,
+        "protocols": ["activitypub"],
+        "services": {"inbound": [], "outbound": []},
+        "software": {"name": "activityrelay", "version": __version__},
+        "usage": {"localPosts": 0, "users": {"total": 1}},
+        "metadata": {"peers": request.app.database.hostnames},
+        "version": niversion,
+    }
+
+    if niversion == "2.1":
+        data["software"]["repository"] = "https://git.pleroma.social/pleroma/relay"
+
+    return Response.new(data, ctype="json")
+
+
+@register_route("GET", "/.well-known/nodeinfo")
+async def nodeinfo_wellknown(request):
+    data = WKNodeinfo.new(
+        v20=f"https://{request.app.config.host}/nodeinfo/2.0.json",
+        v21=f"https://{request.app.config.host}/nodeinfo/2.1.json",
+    )
+
+    return Response.new(data, ctype="json")
+
+
+@register_route("GET", "/stats")
+async def stats(request):
+    stats = STATS.copy()
+    stats["pending_requests"] = len(request.app.database.get("follow-requests", {}))
+    return Response.new(stats, ctype="json")
+
+
+@register_route("POST", "/admin/config")
+async def set_config(request: Request):
+    if not (auth := request.headers.get("Authorization")):
+        return Response.new_error(403, "access denied", "json")
+
+    if not auth == f"Bearer {request.app.config.admin_token}":
+        return Response.new_error(403, "access denied", "json")
+
+    # FIXME: config doesn't have a way to go from JSON or update, using dict stuff
+    text = await request.text()
+    try:
+        new_config = json.loads(text)
+    except JSONDecodeError as e:
+        logging.exception(f"Unable to load config {text!r}")
+        return Response.new_error(400, "bad request", "json")
+
+    request.app.config.update(new_config)
+
+    if request.app.config.whitelist_enabled:
+        # If there are pending follows which are NOW whitelisted, allow them
+        for domain in request.app.config.whitelist:
+            if (pending_follow := request.app.database.get_request(domain, False)):
+                logging.info(f"Acknowledging queued follow request from {domain}...")
+                await misc.request(
+                    pending_follow["inbox"],
+                    misc.Message.new_response(
+                        host=request.app.config.host,
+                        actor=pending_follow["actor"],
+                        followid=pending_follow["followid"],
+                        accept=True
+                    ),
+                )
+
+                await misc.request(
+                    pending_follow["inbox"],
+                    misc.Message.new_follow(
+                        host=request.app.config.host,
+                        actor=pending_follow["actor"]
+                    ),
+                )
+
+                request.app.database.del_request(domain)
+
+        # FIXME: If there are EXISTING follows which are NO LONGER allowed/are blacklisted, drop them
+
+        request.app.database.save()
+
+    request.app.config.save()
+
+    return Response.new(status=202)
+
+
+@register_route("GET", "/admin/config")
+def get_config(request: Request):
+    if not (auth := request.headers.get("Authorization")):
+        return Response.new_error(403, "access denied", "json")
+
+    if not auth == f"Bearer {request.app.config.admin_token}":
+        return Response.new_error(403, "access denied", "json")
+
+    return Response.new(request.app.config, status=200, ctype="json")
+
+
+@register_route("GET", "/admin/db")
+def get_db(request: Request):
+    if not (auth := request.headers.get("Authorization")):
+        return Response.new_error(403, "access denied", "json")
+
+    if not auth == f"Bearer {request.app.config.admin_token}":
+        return Response.new_error(403, "access denied", "json")
+
+    return Response.new(request.app.database, status=200, ctype="json")
+
+
+@register_route("GET", "/admin/pending")
+def get_pending(request):
+    if not (auth := request.headers.get("Authorization")):
+        return Response.new_error(403, "access denied", "json")
+
+    if not auth == f"Bearer {request.app.config.admin_token}":
+        return Response.new_error(403, "access denied", "json")
+
+    return Response.new(request.app.database["follow-requests"], status=200, ctype="json")

projects/aloe/BUILD.bazel

@@ -0,0 +1,12 @@
+py_project(
+    name = "lib"
+)
+
+zapp_binary(
+    name = "aloe",
+    main = "src/aloe/__main__.py",
+    deps = [
+        ":lib",
+        py_requirement("icmplib"),
+    ],
+)

projects/aloe/README.md

@@ -0,0 +1,122 @@
+# Aloe
+
+> - A [cactus](https://www.cacti.net/)-like plant
+> - Traditionally used for a variety of skin conditions
+
+Aloe is a quick and dirty network weathermapping tool, much like MTR or Cacti.
+Aloe uses multiple threads to first establish a rough network topology via ICMP traceroutes, and then monitor it with ICMP pings.
+
+## Usage
+
+``` sh
+$ bazel build //projects/aloe
+$ sudo ./bazel-bin/projects/aloe/aloe twitter.com google.com
+INFO:__main__:Graph -
+                        ┌─────────────────┐
+                        │    127.0.0.1    │
+                        └─────────────────┘
+                          │
+                          │
+                          ▼
+┌─────────────────┐     ┌─────────────────┐
+│  68.85.107.81   │ ◀── │    10.0.0.1     │
+└─────────────────┘     └─────────────────┘
+  │                       │
+  │                       │
+  │                       ▼
+  │                     ┌─────────────────┐
+  │                     │  68.85.107.85   │
+  │                     └─────────────────┘
+  │                       │
+  │                       │
+  │                       ▼
+  │                     ┌─────────────────┐
+  │                     │   68.86.103.9   │
+  │                     └─────────────────┘
+  │                       │
+  │                       │
+  │                       ▼
+  │                     ┌─────────────────┐
+  └───────────────────▶ │  68.85.89.213   │
+                        └─────────────────┘
+                          │
+                          │
+                          ▼
+                        ┌─────────────────┐
+                        │ 24.124.155.129  │
+                        └─────────────────┘
+                          │
+                          │
+                          ▼
+┌─────────────────┐     ┌─────────────────┐     ┌────────────────┐
+│  96.110.43.241  │ ◀── │  96.216.22.130  │ ──▶ │ 96.110.43.253  │
+└─────────────────┘     └─────────────────┘     └────────────────┘
+  │                       │                       │
+  │                       │                       │
+  ▼                       ▼                       ▼
+┌─────────────────┐     ┌─────────────────┐     ┌────────────────┐
+│  96.110.38.114  │     │  96.110.43.245  │     │ 96.110.38.126  │
+└─────────────────┘     └─────────────────┘     └────────────────┘
+  │                       │                       │
+  │                       │                       │
+  ▼                       ▼                       ▼
+┌─────────────────┐     ┌─────────────────┐     ┌────────────────┐
+│   96.87.8.210   │     │  96.110.38.118  │     │ 23.30.206.218  │
+└─────────────────┘     └─────────────────┘     └────────────────┘
+  │                       │                       │
+  │                       │                       │
+  ▼                       ▼                       ▼
+┌─────────────────┐     ┌─────────────────┐     ┌────────────────┐
+│ 108.170.252.193 │     │ 173.167.57.142  │     │ 172.253.75.177 │
+└─────────────────┘     └─────────────────┘     └────────────────┘
+  │                       │                       │
+  │                       │                       │
+  ▼                       ▼                       ▼
+┌─────────────────┐     ┌─────────────────┐     ┌────────────────┐
+│ 142.250.69.238  │     │ 213.155.133.171 │     │ 142.250.72.46  │
+└─────────────────┘     └─────────────────┘     └────────────────┘
+                          │
+                          │
+                          ▼
+┌─────────────────┐     ┌─────────────────┐
+│  104.244.42.65  │ ◀── │  62.115.49.193  │
+└─────────────────┘     └─────────────────┘
+                          │
+                          │
+                          ▼
+                        ┌─────────────────┐
+                        │ 104.244.42.129  │
+                        └─────────────────┘
+```
+
+If hosts in topology stop responding for 10s or more (the polling interval is ~1s), they are declared to be warning.
+If hosts in topology stop responding for 5s, they are declared down.
+If a host in topology resume responding after 5s or more, they are declared to have recovered.
+If hosts in topology stop responding for 30 min, they are declared dead and monitoring is stopped.
+The topology is reconfigured every 5 min to account to DHCP and upstream changes.
+
+A log of all these events is built in a plain-text format to `incidents.txt`.
+The format of this file is -
+
+```
+UP        <ip>  <date>
+WARN      <ip>  <date>
+DOWN      <ip>  <date>
+RECOVERED <ip>  <date>   <duration>
+DEAD      <ip>  <date>   <duration>
+```
+
+## Future work
+
+- [ ] Log topology
+- [ ] Attempt to identify "root cause" incidents in the route graph which explain downstream failures
+- [ ] Use sqlite3 for aggregation not a plain text file
+- [ ] Use a more sophisticated counters and debounce model of host state in the main thread
+- [ ] Find some way to incorporate rolling counters (mean, median, stddev, deciles, max) into the UI
+- [ ] FFS find some way NOT to depend on that nonsense box-diagram service
+
+## License
+
+Copyright Reid 'arrdem' McKenzie, 11/20/2021.
+
+Published under the terms of the MIT license.

projects/aloe/src/aloe/__main__.py

@@ -0,0 +1,258 @@
+"""Aloe - A shitty weathermapping tool.
+
+Think MTR but with the ability to detect/declare incidents and emit logs.
+
+Periodically traceroutes the egress network, and then walks pings out the egress network recording times and hosts which
+failed to respond. Expects a network in excess of 90% packet delivery, but with variable timings. Intended to probe for
+when packet delivery latencies radically degrade and maintain a report file.
+
+"""
+
+import argparse
+from collections import deque as ringbuffer
+import curses
+from datetime import timedelta
+from itertools import count
+import logging
+import queue
+from queue import Queue
+import sys
+from threading import Event, Lock, Thread
+from time import time
+
+from .cursedlogger import CursesHandler
+from .icmp import *
+from .icmp import _ping
+
+
+log = logging.getLogger(__name__)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("hosts", nargs="+")
+
+
+class HostState(object):
+    """A model of a (bounded) time series of host state.
+
+    """
+
+    def __init__(self,
+                 hostname: str,
+                 history = [],
+                 history_size = 60 * 60 * 24,
+                 is_up: bool = False,
+                 lost: int = 0,
+                 up: float = 0.0):
+        self._lock = Lock()
+        self._state = ringbuffer(maxlen=history_size)
+        self._is_up = is_up
+        self._lost = lost
+        self._up = up
+
+        for resp in history:
+            self.append(resp)
+
+    def append(self, resp):
+        with self._lock:
+            if resp and not self._is_up:
+                # log.debug(f"Host {self._hostname} is up!")
+                self._is_up = self._is_up or resp
+                self._up = resp._time
+
+            elif resp and self._is_up:
+                # log.debug(f"Host {self._hostname} holding up...")
+                pass
+
+            elif not resp and self._is_up:
+                # log.debug(f"Host {self._hostname} is down!")
+                self._is_up = None
+                self._up = None
+
+            elif not resp and not self._is_up:
+                pass
+
+            if not resp:
+                self._lost += 1
+
+            if self._state and not self._state[0]:
+                self._lost -= 1
+
+            self._state.append(resp)
+
+    def last(self):
+        with self._lock:
+            return next(reversed(self._state), None)
+
+    def last_window(self, duration: timedelta = None):
+        with self._lock:
+            l = []
+            t = time() - duration.total_seconds()
+            for i in reversed(self._state):
+                if not i or i._time > t:
+                    l.insert(0, i)
+                else:
+                    break
+            return l
+
+    def loss(self, duration: timedelta):
+        log = self.last_window(duration)
+        if log:
+            return log.count(None) / len(log)
+        else:
+            return 0.0
+
+    def is_up(self, duration: timedelta, threshold = 0.25):
+        return self.loss(duration) <= threshold
+
+    def last_seen(self, now: datetime) -> timedelta:
+        if state := self.last():
+            return now - datetime.fromtimestamp(state._time)
+
+    def up(self, duration: datetime):
+        if self._up:
+            return datetime.fromtimestamp(self._up)
+
+
+class MonitoredHost(object):
+    """A shim (arguably a lambda) for generating a timeline of host state."""
+
+    def __init__(self, hostname: str, timeout: timedelta, id=None):
+        self._hostname = hostname
+        self._timeout = timeout
+        self._sequence = request_sequence(hostname, timeout, id)
+        self._lock = Lock()
+        self._state = HostState(hostname)
+
+    def __call__(self, shutdown: Event, q: Queue):
+        """Monitor a given host by throwing requests into the queue; intended to be a Thread target."""
+
+        while not shutdown.is_set():
+            req = next(self._sequence)
+            resp = _ping(q, req)
+            self._state.append(resp)
+            sleep(shutdown, 1)
+
+    @property
+    def state(self):
+        return self._state
+
+
+def retrace(shutdown, q, opts, hl, hosts):
+    threads = {}
+
+    def create_host(distance, address):
+        with hl:
+            if address not in hosts:
+                log.info(f"Monitoring {address}...")
+                monitor = MonitoredHost(address, timedelta(seconds=4))
+                hosts[address] = (distance, monitor)
+                threads[address] = t = Thread(target=monitor, args=(shutdown, q))
+                t.start()
+
+            else:
+                log.debug(f"Already monitoring {address}...")
+
+    while not shutdown.is_set():
+        for h in opts.hosts:
+            # FIXME: Use a real topology model
+            for hop in traceroute(q, h):
+                if ping(q, hop.address).is_alive:
+                    create_host(hop.distance, hop.address)
+
+        sleep(shutdown, 60 * 5)
+
+
+def render(shutdown, q, stdscr, hl, hosts):
+    dt = timedelta(minutes=30)
+
+    with open("incidents.txt", "a") as fp:
+        incident = False
+        while not shutdown.is_set():
+            rows, cols = stdscr.getmaxyx()
+            down = 0
+            now = datetime.now()
+            i = 0
+            with hl:
+                for i, (name, (distance, host)) in zip(count(1), sorted(hosts.items(), key=lambda x: x[1][0])):
+                    loss = host.state.loss(dt) * 100
+                    state = host.state.last()
+                    if not state:
+                        down += 1
+                        last_seen = "Down"
+                    else:
+                        last_seen = f"{host.state.last_seen(now).total_seconds():.2f}s ago"
+
+                    if up := host.state.up(dt):
+                        up = f" up: {(now - up).total_seconds():.2f}"
+                    else:
+                        up = ""
+
+                    stdscr.addstr(i, 2, f"{distance: <2} {name: <16s}]{up} lost: {loss:.2f}% last: {last_seen}".ljust(cols))
+
+            stdscr.border()
+            stdscr.refresh()
+
+            msg = None
+            if down >= 3 and not incident:
+                incident = True
+                msg = f"{datetime.now()} - {down} hosts down"
+
+            elif down < 3 and incident:
+                incident = False
+                msg = f"{datetime.now()} - network recovered"
+
+            if i != 0 and msg:
+                log.info(msg)
+                fp.write(msg + "\n")
+                fp.flush()
+
+            sleep(shutdown, 1)
+
+
+def main():
+    stdscr = curses.initscr()
+    maxy, maxx = stdscr.getmaxyx()
+
+    begin_x = 2; begin_y = maxy - 12
+    height = 10; width = maxx - 4
+    logscr = curses.newwin(height, width, begin_y, begin_x)
+
+    handler = CursesHandler(logscr)
+    formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
+    handler.setFormatter(formatter)
+    log.addHandler(handler)
+    log.setLevel(logging.DEBUG)
+
+    stdscr = curses.newwin(maxy - height - 2, width, 0, begin_x)
+
+    opts, args = parser.parse_known_args()
+
+    q = queue.Queue()
+    shutdown = Event()
+
+    p = Thread(target=icmp_worker, args=(shutdown, q,))
+    p.start()
+
+    hosts = {}
+    hl = Lock()
+
+    rt = Thread(target=render, args=(shutdown, q, stdscr, hl, hosts))
+    rt.start()
+
+    tt = Thread(target=retrace, args=(shutdown, q, opts, hl, hosts))
+    tt.start()
+
+    try:
+        while True:
+            sleep(shutdown, 1)
+    except (KeyboardInterrupt, SystemExit):
+        pass
+    finally:
+        curses.endwin()
+        sys.stdout.flush()
+        sys.stderr.flush()
+        shutdown.set()
+
+
+if __name__ == "__main__":
+    main()

projects/aloe/src/aloe/cursedlogger.py

@@ -0,0 +1,29 @@
+"""A CURSES screen targeted log record handler."""
+
+from collections import deque as ringbuffer
+from itertools import count
+import logging
+
+
+class CursesHandler(logging.Handler):
+    def __init__(self, screen):
+        logging.Handler.__init__(self)
+        self._screen = screen
+        # FIXME: This should be dynamic not static.
+        self._buff = ringbuffer(maxlen=screen.getmaxyx()[0] - 2)
+
+    def emit(self, record):
+        try:
+            msg = self.format(record) + "\n"
+            self._buff.append(msg)
+            self._screen.clear()
+            for i, msg in zip(count(1), self._buff):
+                self._screen.addstr(i, 2, msg)
+            self._screen.border()
+            self._screen.refresh()
+
+        except (KeyboardInterrupt, SystemExit):
+            raise
+
+        except:
+            self.handleError(record)

projects/aloe/src/aloe/icmp.py

@@ -0,0 +1,251 @@
+"""Tools sitting between aloe and icmplib."""
+
+from datetime import datetime, timedelta
+from itertools import islice as take
+import logging
+import queue
+from random import randint
+from threading import Event, Lock
+from time import sleep as _sleep
+
+from icmplib.exceptions import (
+    ICMPLibError,
+    ICMPSocketError,
+)
+from icmplib.models import (
+    Hop,
+    Host,
+    ICMPReply,
+    ICMPRequest,
+)
+from icmplib.sockets import ICMPv4Socket
+from icmplib.utils import is_hostname, resolve
+
+
+log = logging.getLogger(__name__)
+
+
+def better_repr(self):
+    elems = ", ".join(f"{slot}={getattr(self, slot)}" for slot in self.__slots__)
+    return f"<{type(self).__name__} ({elems})>"
+
+
+ICMPRequest.__repr__ = better_repr
+ICMPReply.__repr__ = better_repr
+
+
+def sleep(event, duration, interval=0.1):
+    total = 0
+    while total < duration:
+        if event.is_set():
+            raise SystemExit()
+        else:
+            _sleep(interval)
+            total += interval
+
+
+class ICMPRequestResponse(object):
+    """A thread-safe request/response structure for pinging a host."""
+
+    PENDING = object()
+    TIMEOUT = object()
+
+    def __init__(self, address, request: ICMPRequest, timeout: timedelta):
+        _timeout = datetime.now() + timeout
+        self._address = address
+        self._request = request
+        self._timeout = _timeout
+        self._lock = Lock()
+        self._response = self.PENDING
+
+    def ready(self):
+        """Determine if this request is still waiting."""
+        return self.get() is not self.PENDING
+
+    def get(self):
+        """Get the response, unless the timeout has passed in which case return a timeout."""
+
+        with self._lock:
+            if self._response is not self.PENDING:
+                return self._response
+            elif self._timeout < datetime.now():
+                return self.TIMEOUT
+            else:
+                return self._response
+
+    def set(self, response):
+        """Set the response, unless the timeout has passed in which case set a timeout."""
+
+        if isinstance(response, ICMPReply):
+            with self._lock:
+                if self._timeout < datetime.now():
+                    self._response = self.TIMEOUT
+                else:
+                    # rtt = (reply.time - request.time) * 1000
+                    self._response = response
+
+
+def icmp_worker(shutdown: Event, q: queue.Queue):
+    """A worker thread which processes ICMP requests; sending packets and listening for matching responses."""
+
+    state = {}
+
+    with ICMPv4Socket(None, True) as sock:
+        while not shutdown.is_set():
+            # Send one
+            try:
+                item = q.get(block=False, timeout=0.001)
+                request = item._request
+                state[(request._id, request._sequence)] = item
+                # log.info(f"Sending request {item._request!r}")
+                sock.send(item._request)
+            except (ICMPLibError, ICMPSocketError, queue.Empty):
+                pass
+
+            # Recieve one
+            try:
+                if response := sock.receive(None, 0.001):
+                    key = (response.id, response.sequence)
+                    if key in state:
+                        # log.info(f"Got response {response!r}")
+                        state[key].set(response)
+                        del state[key]
+                    else:
+                        # log.warning(f"Recieved non-matching response {response!r}")
+                        pass
+            except (ICMPLibError, ICMPSocketError):
+                pass
+
+            # GC one
+            if key := next(iter(state.keys()), None):
+                if state[key].ready():
+                    del state[key]
+
+            # Sleep one
+            sleep(shutdown, 0.001)
+
+
+def traceroute(q: queue.Queue,
+               address: str,
+               first_hop: int = 1,
+               max_hops: int = 32,
+               count: int = 3,
+               id: int = None,
+               family: int = None):
+    if is_hostname(address):
+        address = resolve(address, family)[0]
+
+    mask = ((1<<16) - 1)
+    id = id or randint(1, mask) & 0xFFFF
+    ttl = first_hop
+    host_reached = False
+    hops = []
+
+    while not host_reached and ttl <= max_hops:
+        reply = None
+        packets_sent = 0
+        rtts = []
+
+        for sequence in range(count):
+            request = ICMPRequestResponse(
+                address,
+                ICMPRequest(
+                    destination=address,
+                    id=id,
+                    sequence=sequence,
+                    ttl=ttl
+                ),
+                timedelta(seconds=1),
+            )
+
+            q.put(request)
+            while not request.ready():
+                _sleep(0.1)
+
+            _reply = request.get()
+            if _reply is ICMPRequestResponse.TIMEOUT:
+                _sleep(0.1)
+                continue
+
+            elif _reply:
+                reply = reply or _reply
+                try:
+                    reply.raise_for_status()
+                    host_reached = True
+                except ICMPLibError:
+                    pass
+
+                rtt = (reply.time - request._request.time) * 1000
+                rtts.append(rtt)
+
+        if reply:
+            hops.append(
+                Hop(
+                    address=reply.source,
+                    packets_sent=packets_sent,
+                    rtts=rtts,
+                    distance=ttl
+                )
+            )
+
+        ttl += 1
+
+    return hops
+
+
+def request_sequence(hostname: str,
+                     timeout: timedelta,
+                     id: int = None,
+                     family: int = None):
+    """Generate a sequence of requests monitoring a specific, usable as a request source for a ping."""
+
+    if is_hostname(hostname):
+        destination = resolve(hostname, family)[0]
+    else:
+        destination = hostname
+
+    mask = ((1<<16) - 1)
+    id = id or randint(1, mask) & 0xFFFF
+    sequence = 1
+
+    while True:
+        yield ICMPRequestResponse(
+            hostname,
+            ICMPRequest(
+                destination=destination,
+                id=id,
+                sequence=sequence & mask,
+            ),
+            timeout
+        )
+        sequence += 1
+
+
+def _ping(q: queue.Queue, request: ICMPRequestResponse):
+        q.put(request)
+        while not request.ready():
+            _sleep(0.1)
+
+        _response = request.get()
+        if _response is not ICMPRequestResponse.TIMEOUT:
+            return _response
+
+
+def ping(q: queue.Queue,
+         address: str,
+         count: int = 3,
+         id: int = None,
+         family: int = None) -> Host:
+    """Ping a host N times."""
+
+    rtts = []
+    for request in take(request_sequence(address, timedelta(seconds=1)), count):
+        if reply := _ping(q, request):
+            rtt = (reply.time - request._request.time) * 1000
+            rtts.append(rtt)
+
+    return Host(
+        address=address,
+        packets_sent=count,
+        rtts=rtts,
+    )

projects/anosql-migrations/BUILD.bazel

@@ -0,0 +1,6 @@
+py_project(
+    name = "anosql-migrations",
+    lib_deps = [
+        "//projects/anosql",
+    ],
+)

projects/anosql-migrations/src/anosql_migrations.py

@@ -0,0 +1,189 @@
+"""Quick and dirty migrations for AnoSQL."""
+
+from datetime import datetime
+from hashlib import sha256
+import logging
+import re
+import typing as t
+
+from anosql.core import from_str, Queries
+
+
+log = logging.getLogger(__name__)
+
+
+class MigrationDescriptor(t.NamedTuple):
+    name: str
+    sha256sum: str
+    committed_at: t.Optional[datetime] = None
+
+    def __repr__(self):
+        return f"MigrationDescriptor(name={self.name!r}, sha256sum='{self.sha256sum[:7]}...')"
+
+    def __hash__(self):
+        return hash((self.name, self.sha256sum))
+
+    def __eq__(self, other):
+        return (self.name, self.sha256sum) == (other.name, other.sha256sum)
+
+
+_SQL = """
+-- name: anosql_migrations_create_table#
+-- Create the migrations table for the anosql_migrations plugin.
+CREATE TABLE IF NOT EXISTS `anosql_migration` (
+    `name` TEXT PRIMARY KEY NOT NULL
+,   `committed_at` INT
+,   `sha256sum` TEXT NOT NULL
+,   CONSTRAINT `am_sha256sum_unique` UNIQUE (`sha256sum`)
+);
+
+-- name: anosql_migrations_list
+-- List committed migrations
+SELECT
+    `name`
+,   `committed_at`
+,   `sha256sum`
+FROM `anosql_migration`
+WHERE
+    `committed_at` > 0
+ORDER BY
+    `name` ASC
+;
+
+-- name: anosql_migrations_get
+-- Get a given migration by name
+SELECT
+    `name`
+,   `committed_at`,
+,   `sha256sum`
+FROM `anosql_migration`
+WHERE
+    `name` = :name
+ORDER BY
+    `rowid` ASC
+;
+
+-- name: anosql_migrations_create<!
+-- Insert a migration, marking it as committed
+INSERT OR REPLACE INTO `anosql_migration` (
+    `name`
+,   `committed_at`
+,   `sha256sum`
+) VALUES (
+    :name
+,   :date
+,   :sha256sum
+);
+"""
+
+
+def with_migrations(driver_adapter, queries: Queries, conn) -> Queries:
+    """Initialize the migrations plugin."""
+
+    # Compile SQL as needed from the _SQL constant
+    _q = from_str(_SQL, driver_adapter)
+
+    # Merge. Sigh.
+    for _qname in _q.available_queries:
+        queries.add_query(_qname, getattr(_q, _qname))
+
+    # Create the migrations table
+    create_tables(queries, conn)
+
+    return queries
+
+
+def create_tables(queries: Queries, conn) -> None:
+    """Create the migrations table (if it doesn't exist)."""
+
+    if queries.anosql_migrations_create_table(conn):
+        log.info("Created migrations table")
+
+    # Insert the bootstrap 'fixup' record
+    execute_migration(
+        queries,
+        conn,
+        MigrationDescriptor(
+            name="anosql_migrations_create_table",
+            sha256sum=sha256(
+                queries.anosql_migrations_create_table.sql.encode("utf-8")
+            ).hexdigest(),
+        ),
+    )
+
+
+def committed_migrations(queries: Queries, conn) -> t.Iterable[MigrationDescriptor]:
+    """Enumerate migrations committed to the database."""
+
+    for name, committed_at, sha256sum in queries.anosql_migrations_list(conn):
+        yield MigrationDescriptor(
+            name=name,
+            committed_at=datetime.fromtimestamp(committed_at),
+            sha256sum=sha256sum,
+        )
+
+
+def available_migrations(queries: Queries, conn) -> t.Iterable[MigrationDescriptor]:
+    """Enumerate all available migrations, executed or no."""
+
+    for query_name in sorted(queries.available_queries):
+        if not re.match("^migration", query_name):
+            continue
+
+        if query_name.endswith("_cursor"):
+            continue
+
+        # query_name: str
+        # query_fn: t.Callable + {.__name__, .__doc__, .sql}
+        query_fn = getattr(queries, query_name)
+        yield MigrationDescriptor(
+            name=query_name,
+            committed_at=None,
+            sha256sum=sha256(query_fn.sql.encode("utf-8")).hexdigest(),
+        )
+
+
+def execute_migration(queries: Queries, conn, migration: MigrationDescriptor):
+    """Execute a given migration singularly."""
+
+    with conn:
+        # Mark the migration as in flight
+        queries.anosql_migrations_create(
+            conn,
+            # Args
+            name=migration.name,
+            date=-1,
+            sha256sum=migration.sha256sum,
+        )
+
+        # Run the migration function
+        getattr(queries, migration.name)(conn)
+
+        # Mark the migration as committed
+        queries.anosql_migrations_create(
+            conn,
+            # Args
+            name=migration.name,
+            date=int(datetime.utcnow().timestamp()),
+            sha256sum=migration.sha256sum,
+        )
+
+
+def run_migrations(queries, conn):
+    """Run all remaining migrations."""
+
+    avail = set(available_migrations(queries, conn))
+    committed = set(committed_migrations(queries, conn))
+
+    for migration in sorted(avail, key=lambda m: m.name):
+        if migration in committed:
+            log.info(f"Skipping committed migration {migration.name}")
+
+        else:
+            log.info(f"Beginning migration {migration.name}")
+
+            try:
+                execute_migration(queries, conn, migration)
+            except Exception as e:
+                log.exception(f"Migration {migration.name} failed!", e)
+                raise e

projects/anosql-migrations/test/test_migrations.py

@@ -0,0 +1,109 @@
+"""Tests covering the migrations framework."""
+
+import sqlite3
+
+import anosql
+from anosql.core import Queries
+import anosql_migrations
+import pytest
+
+
+_SQL = """\
+-- name: migration_0000_create_kv
+CREATE TABLE kv (`id` INT, `key` TEXT, `value` TEXT);
+"""
+
+
+def table_exists(conn, table_name):
+    return list(
+        conn.execute(
+            f"""\
+    SELECT (
+        `name`
+    )
+    FROM `sqlite_master`
+    WHERE
+        `type` = 'table'
+    AND `name` = '{table_name}'
+    ;"""
+        )
+    )
+
+
+@pytest.fixture
+def conn() -> sqlite3.Connection:
+    """Return an (empty) SQLite instance."""
+
+    return sqlite3.connect(":memory:")
+
+
+def test_connect(conn: sqlite3.Connection):
+    """Assert that the connection works and we can execute against it."""
+
+    assert list(conn.execute("SELECT 1;")) == [
+        (1,),
+    ]
+
+
+@pytest.fixture
+def queries(conn) -> Queries:
+    """A fixture for building a (migrations capable) anosql queries object."""
+
+    q = anosql.from_str(_SQL, "sqlite3")
+    return anosql_migrations.with_migrations("sqlite3", q, conn)
+
+
+def test_queries(queries):
+    """Assert that we can construct a queries instance with migrations features."""
+
+    assert isinstance(queries, Queries)
+    assert hasattr(queries, "anosql_migrations_create_table")
+    assert hasattr(queries, "anosql_migrations_list")
+    assert hasattr(queries, "anosql_migrations_create")
+
+
+def test_migrations_create_table(conn, queries):
+    """Assert that the migrations system will (automagically) create the table."""
+
+    assert table_exists(conn, "anosql_migration"), "Migrations table did not create"
+
+
+def test_migrations_list(conn, queries):
+    """Test that we can list out available migrations."""
+
+    ms = list(anosql_migrations.available_migrations(queries, conn))
+    assert any(
+        m.name == "migration_0000_create_kv" for m in ms
+    ), f"Didn't find in {ms!r}"
+
+
+def test_committed_migrations(conn, queries):
+    """Assert that only the bootstrap migration is committed to the empty connection."""
+
+    ms = list(anosql_migrations.committed_migrations(queries, conn))
+    assert len(ms) == 1
+    assert ms[0].name == "anosql_migrations_create_table"
+
+
+def test_apply_migrations(conn, queries):
+    """Assert that if we apply migrations, the requisite table is created."""
+
+    anosql_migrations.run_migrations(queries, conn)
+    assert table_exists(conn, "kv")
+
+
+@pytest.fixture
+def migrated_conn(conn, queries):
+    """Generate a connection whithin which the `kv` migration has already been run."""
+
+    anosql_migrations.run_migrations(queries, conn)
+    return conn
+
+
+def test_post_committed_migrations(migrated_conn, queries):
+    """Assert that the create_kv migration has been committed."""
+
+    ms = list(anosql_migrations.committed_migrations(queries, migrated_conn))
+    assert any(m.name == "migration_0000_create_kv" for m in ms), "\n".join(
+        migrated_conn.iterdump()
+    )

projects/anosql/BUILD.bazel

@@ -0,0 +1,10 @@
+py_project(
+    name="anosql",
+    test_deps = [
+        py_requirement("pytest-postgresql"),
+        py_requirement("psycopg2"),
+    ],
+    test_tags = [
+        "known-to-fail",
+    ],
+)

projects/anosql/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2014-2017, Honza Pokorny
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those
+of the authors and should not be interpreted as representing official policies,
+either expressed or implied, of the FreeBSD Project.

projects/anosql/README.rst

@@ -0,0 +1,16 @@
+anosql
+======
+
+A Python library for using SQL
+
+Inspired by the excellent Yesql library by Kris Jenkins. In my mother tongue, ano means yes.
+
+This is a vendored copy of `anosql`_, which exists to capture a few pre-deprecation patches that never made it into the last release, and bazel-ify everything for my use.
+
+License
+-------
+
+BSD, short and sweet
+
+.. _anosql: https://github.com/honza/anosql
+.. _Yesql: https://github.com/krisajenkins/yesql/

projects/anosql/doc/Makefile

@@ -0,0 +1,225 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  applehelp  to make an Apple Help Book"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  epub3      to make an epub3"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+	@echo "  dummy      to check syntax errors of document sources"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*
+
+.PHONY: html
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: dirhtml
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+.PHONY: singlehtml
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+.PHONY: pickle
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+.PHONY: json
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+.PHONY: htmlhelp
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+.PHONY: qthelp
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/anosql.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/anosql.qhc"
+
+.PHONY: applehelp
+applehelp:
+	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+	@echo
+	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+	@echo "N.B. You won't be able to view it unless you put it in" \
+	      "~/Library/Documentation/Help or install it in your application" \
+	      "bundle."
+
+.PHONY: devhelp
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/anosql"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/anosql"
+	@echo "# devhelp"
+
+.PHONY: epub
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+.PHONY: epub3
+epub3:
+	$(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3
+	@echo
+	@echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3."
+
+.PHONY: latex
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+.PHONY: latexpdf
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: latexpdfja
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: text
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+.PHONY: man
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+.PHONY: texinfo
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+.PHONY: info
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+.PHONY: gettext
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+.PHONY: changes
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+.PHONY: linkcheck
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."
+
+.PHONY: xml
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+.PHONY: pseudoxml
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+.PHONY: dummy
+dummy:
+	$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
+	@echo
+	@echo "Build finished. Dummy builder generates no files."

projects/anosql/doc/conf.py

@@ -0,0 +1,339 @@
+# -*- coding: utf-8 -*-
+#
+# anosql documentation build configuration file, created by
+# sphinx-quickstart on Mon Jul 25 09:16:20 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+import pkg_resources
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon"]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = ".rst"
+
+# The encoding of source files.
+#
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = "index"
+
+# General information about the project.
+project = u"anosql"
+copyright = u"2014-2017, Honza Pokorny"
+author = u"Honza Pokorny"
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = pkg_resources.get_distribution("anosql").version
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#
+# today = ''
+#
+# Else, today_fmt is used as the format for a strftime call.
+#
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "alabaster"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.
+# "<project> v<release> documentation" by default.
+#
+# html_title = u'anosql v0.1.2'
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#
+# html_logo = None
+
+# The name of an image file (relative to this directory) to use as a favicon of
+# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#
+# html_extra_path = []
+
+# If not None, a 'Last updated on:' timestamp is inserted at every page
+# bottom, using the given strftime format.
+# The empty string is equivalent to '%b %d, %Y'.
+#
+# html_last_updated_fmt = None
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+#
+# html_domain_indices = True
+
+# If false, no index is generated.
+#
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'
+#
+# html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# 'ja' uses this config value.
+# 'zh' user can custom change `jieba` dictionary path.
+#
+# html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#
+# html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "anosqldoc"
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, "anosql.tex", u"anosql Documentation", u"Honza Pokorny", "manual"),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+#
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#
+# latex_appendices = []
+
+# It false, will not define \strong, \code, 	itleref, \crossref ... but only
+# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
+# packages.
+#
+# latex_keep_old_macro_names = True
+
+# If false, no module index is generated.
+#
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(master_doc, "anosql", u"anosql Documentation", [author], 1)]
+
+# If true, show URL addresses after external links.
+#
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc,
+        "anosql",
+        u"anosql Documentation",
+        author,
+        "anosql",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
+]
+
+# Documents to append as an appendix to all manuals.
+#
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+#
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#
+# texinfo_no_detailmenu = False

projects/anosql/doc/defining_queries.rst

@@ -0,0 +1,185 @@
+####################
+Defining SQL Queries
+####################
+
+Query Names & Comments
+======================
+
+Name definitions are how ``anosql`` determines how to name the SQL code blocks which are loaded.
+A query name definition is a normal SQL comment starting with "\-\- name:" and is followed by the
+name of the query. You can use ``-`` or ``_`` in your query names, but the methods in Python
+will always be valid Python names using underscores.
+
+.. code-block:: sql
+
+    -- name: get-all-blogs
+    select * from blogs;
+
+The above example when loaded by ``anosql.from_path`` will return an object with a
+``.get_all_blogs(conn)`` method.
+
+Your SQL comments will be added to your methods as Python docstrings, and accessible by calling
+``help()`` on them.
+
+.. code-block:: sql
+
+    -- name: get-all-blogs
+    -- Fetch all fields for every blog in the database.
+    select * from blogs;
+
+
+.. code-block:: python
+
+    queries = anosql.from_path("blogs.sql", "sqlite3")
+    help(anosql.get_all_blogs)
+
+returns
+
+.. code-block:: text
+
+    Help on function get_user_blogs in module anosql.anosql:
+
+    get_all_blogs(conn, *args, **kwargs)
+        Fetch all fields for every blog in the database.
+
+.. _query-operations:
+
+Query Operations
+================
+
+Adding query operator symbols to the end of query names will inform ``anosql`` of how to
+execute and return results. In the above section the ``get-all-blogs`` name has no special operator
+characters trailing it. This lack of operator is actually the most basic operator which performs
+SQL ``select`` statements and returns a list of rows. When writing an application you will often
+need to perform other operations besides selects, like inserts, deletes, and bulk opearations. The
+operators detailed in this section let you declare in your SQL how your code should be executed
+by the database driver.
+
+Insert/Update/Delete with ``!``
+-------------------------------
+
+The ``!`` operator will execute SQL without returning any results. It is meant for use with ``insert``,
+``update``, and ``delete`` statements for which returned data is not required.
+
+.. code-block:: sql
+
+    -- name: publish-blog!
+    insert into blogs(userid, title, content) values (:userid, :title, :content);
+
+    -- name: remove-blog!
+    -- Remove a blog from the database
+    delete from blogs where blogid = :blogid;
+
+
+The methods generated are:
+
+    - ``publish_blog(conn, *args, **kwargs)``
+    - ``remove_blog(conn, *args, **kwargs)``
+
+Each of them can be run to alter the database, but both will return ``None``.
+
+Insert Returning with ``<!``
+----------------------------
+
+Sometimes when performing an insert it is necessary to receive some information back about the
+newly created database row. The ``<!`` operator tells anosql to perform execute the insert query, but to also expect and
+return some data.
+
+In SQLite this means the ``cur.lastrowid`` will be returned.
+
+.. code-block:: sql
+
+    -- name: publish-blog<!
+    insert into blogs(userid, title, content) values (:userid, :title, :content);
+
+Will return the ``blogid`` of the inserted row.
+
+PostgreSQL however allows returning multiple values via the ``returning`` clause of insert
+queries.
+
+.. code-block:: sql
+
+    -- name: publish-blog<!
+    insert into blogs (
+        userid,
+        title,
+        content
+    )
+    values (
+        :userid,
+        :title,
+        :content
+    )
+    returning blogid, title;
+
+This will insert the new blog row and return both it's ``blogid`` and ``title`` value as follows::
+
+    queries = anosql.from_path("blogs.sql", "psycopg2")
+    blogid, title = queries.publish_blog(conn, userid=1, title="Hi", content="word.")
+
+Insert/Update/Delete Many with ``*!``
+-------------------------------------
+
+The DB-API 2.0 drivers like ``sqlite3`` and ``psycopg2`` have an ``executemany`` method which
+execute a SQL command against all parameter sequences or mappings found in a sequence. This
+is useful for bulk updates to the database. The below example is a PostgreSQL statement to insert
+many blog rows.
+
+.. code-block:: sql
+
+    -- name: bulk-publish*!
+    -- Insert many blogs at once
+    insert into blogs (
+        userid,
+        title,
+        content,
+        published
+    )
+    values (
+        :userid,
+        :title,
+        :content,
+        :published
+    )
+
+Applying this to a list of blogs in Python::
+
+    queries = anosql.from_path("blogs.sql", "psycopg2")
+    blogs = [
+        {"userid": 1, "title": "First Blog", "content": "...", published: datetime(2018, 1, 1)},
+        {"userid": 1, "title": "Next Blog", "content": "...", published: datetime(2018, 1, 2)},
+        {"userid": 2, "title": "Hey, Hey!", "content": "...", published: datetime(2018, 7, 28)},
+    ]
+    queries.bulk_publish(conn, blogs)
+
+Execute SQL script statements with ``#``
+---------------------------------------------
+
+Executes some SQL statements as a script. These methods don't do variable substitution, or return
+any rows. An example use case is using data definition statements like `create` table in order to
+setup your database.
+
+.. code-block:: sql
+
+    -- name: create-schema#
+    create table users (
+        userid integer not null primary key,
+        username text not null,
+        firstname integer not null,
+        lastname text not null
+    );
+
+    create table blogs (
+        blogid integer not null primary key,
+        userid integer not null,
+        title text not null,
+        content text not null,
+        published date not null default CURRENT_DATE,
+        foreign key(userid) references users(userid)
+    );
+
+From code::
+
+    queries = anosql.from_path("create_schema.sql", "sqlite3")
+    queries.create_schema(conn)
+

projects/anosql/doc/extending.rst

@@ -0,0 +1,50 @@
+.. _extending-anosql:
+
+################
+Extending anosql
+################
+
+.. _driver-adapters:
+
+Driver Adapters
+---------------
+
+Database driver adapters in ``anosql`` are duck-typed classes which follow the below interface.::
+
+    class MyDbAdapter():
+        def process_sql(self, name, op_type, sql):
+            pass
+
+        def select(self, conn, sql, parameters):
+            pass
+
+        @contextmanager
+        def select_cursor(self, conn, sql, parameters):
+            pass
+
+        def insert_update_delete(self, conn, sql, parameters):
+            pass
+
+        def insert_update_delete_many(self, conn, sql, parameters):
+            pass
+
+        def insert_returning(self, conn, sql, parameters):
+            pass
+
+        def execute_script(self, conn, sql):
+            pass
+
+
+    anosql.core.register_driver_adapter("mydb", MyDbAdapter)
+
+If your adapter constructor takes arguments you can register a function which can build
+your adapter instance::
+
+    def adapter_factory():
+        return MyDbAdapter("foo", 42)
+
+    anosql.core.register_driver_adapter("mydb", adapter_factory)
+
+Looking at the source of the builtin
+`adapters/ <https://github.com/honza/anosql/tree/master/anosql/adapters>`_ is a great place
+to start seeing how you may write your own database driver adapter.

projects/anosql/doc/getting_started.rst

@@ -0,0 +1,56 @@
+###############
+Getting Started
+###############
+
+Below is an example of a program which can print ``"{greeting}, {world_name}!"`` from data held in a minimal SQLite
+database containing greetings and worlds.
+
+The SQL is in a ``greetings.sql`` file with ``-- name:`` definitions on each query to tell ``anosql`` under which name
+we would like to be able to execute them. For example, the query under the name ``get-all-greetings`` in the example
+below will be available to us after loading via ``anosql.from_path`` as a method ``get_all_greetings(conn)``.
+Each method on an ``anosql.Queries`` object accepts a database connection to use in communicating with the database.
+
+.. code-block:: sql
+
+    -- name: get-all-greetings
+    -- Get all the greetings in the database
+    select greeting_id, greeting from greetings;
+
+    -- name: get-worlds-by-name
+    -- Get all the world record from the database.
+    select world_id,
+           world_name,
+           location
+      from worlds
+     where world_name = :world_name;
+
+By specifying ``db_driver="sqlite3"`` we can use the Python stdlib ``sqlite3`` driver to execute these SQL queries and
+get the results. We're also using the ``sqlite3.Row`` type for our records to make it easy to access our data via
+their column names rather than as tuple indices.
+
+.. code-block:: python
+
+    import sqlite3
+    import anosql
+
+    queries = anosql.from_path("greetings.sql", db_driver="sqlite3")
+    conn = sqlite3.connect("greetings.db")
+    conn.row_factory = sqlite3.Row
+
+    greetings = queries.get_greetings(conn)
+    worlds = queries.get_worlds_by_name(conn, world_name="Earth")
+    # greetings = [
+    #     <Row greeting_id=1, greeting="Hi">,
+    #     <Row greeting_id=2, greeting="Aloha">,
+    #     <Row greeting_id=3, greeting="Hola">
+    # ]
+    # worlds = [<Row world_id=1, world_name="Earth">]
+
+    for world_row in worlds:
+        for greeting_row in greetings:
+            print(f"{greeting_row['greeting']}, {world_row['world_name']}!")
+    # Hi, Earth!
+    # Aloha, Earth!
+    # Hola, Earth!
+
+    conn.close()

projects/anosql/doc/index.rst

@@ -0,0 +1,142 @@
+.. anosql documentation master file, created by
+   sphinx-quickstart on Mon Jul 25 09:16:20 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to anosql's documentation!
+==================================
+
+.. image:: https://badge.fury.io/py/anosql.svg
+    :target: https://badge.fury.io/py/anosql
+    :alt: pypi package version
+
+.. image:: http://readthedocs.org/projects/anosql/badge/?version=latest
+    :target: http://anosql.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation status
+
+.. image:: https://travis-ci.org/honza/anosql.svg?branch=master
+    :target: https://travis-ci.org/honza/anosql
+    :alt: Travis build status
+
+A Python library for using SQL
+
+Inspired by the excellent `Yesql`_ library by Kris Jenkins.  In my mother
+tongue, *ano* means *yes*.
+
+If you are on python3.6+ or need ``anosql`` to work with ``asyncio`` based database drivers, see the related project `aiosql <https://github.com/nackjicholson/aiosql>`_.
+
+Installation
+------------
+
+::
+
+    $ pip install anosql
+
+Usage
+-----
+
+Basics
+******
+
+Given a ``queries.sql`` file:
+
+.. code-block:: sql
+
+    -- name: get-all-greetings
+    -- Get all the greetings in the database
+    SELECT * FROM greetings;
+
+We can issue SQL queries, like so:
+
+.. code-block:: python
+
+    import anosql
+    import psycopg2
+    import sqlite3
+
+    # PostgreSQL
+    conn = psycopg2.connect('...')
+    queries = anosql.from_path('queries.sql', 'psycopg2')
+
+    # Or, Sqlite3...
+    conn = sqlite3.connect('cool.db')
+    queries = anosql.from_path('queries.sql', 'sqlite3')
+
+    queries.get_all_greetings(conn)
+    # => [(1, 'Hi')]
+
+    queries.get_all_greetings.__doc__
+    # => Get all the greetings in the database
+
+    queries.get_all_greetings.sql
+    # => SELECT * FROM greetings;
+
+    queries.available_queries
+    # => ['get_all_greetings']
+
+
+Parameters
+**********
+
+Often, you want to change parts of the query dynamically, particularly values in the ``WHERE`` clause.
+You can use parameters to do this:
+
+.. code-block:: sql
+
+  -- name: get-greetings-for-language
+  -- Get all the greetings in the database for a given language
+  SELECT *
+  FROM greetings
+  WHERE lang = %s;
+
+And they become positional parameters:
+
+.. code-block:: python
+
+  visitor_language = "en"
+  queries.get_all_greetings_for_language(conn, visitor_language)
+
+
+
+Named Parameters
+****************
+
+To make queries with many parameters more understandable and maintainable, you can give the parameters names:
+
+.. code-block:: sql
+
+  -- name: get-greetings-for-language
+  -- Get all the greetings in the database for given language and length
+  SELECT *
+  FROM greetings
+  WHERE lang = :lang
+  AND len(greeting) <= :length_limit;
+
+If you were writing a Postgresql query, you could also format the parameters as ``%s(lang)`` and ``%s(length_limit)``.
+
+Then, call your queries like you would any Python function with named parameters:
+
+.. code-block:: python
+
+  visitor_language = "en"
+
+  greetings_for_texting = queries.get_all_greetings(conn, lang=visitor_language, length_limit=140)
+
+
+Contents
+--------
+.. toctree::
+   :maxdepth: 2
+
+   Getting Started <getting_started>
+   Defining Queries <defining_queries>
+   Extending anosql <extending>
+   Upgrading <upgrading>
+   API <source/modules>
+
+License
+-------
+
+BSD, short and sweet
+
+.. _Yesql: https://github.com/krisajenkins/yesql/

projects/anosql/doc/make.bat

@@ -0,0 +1,281 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+	:help
+	echo.Please use `make ^<target^>` where ^<target^> is one of
+	echo.  html       to make standalone HTML files
+	echo.  dirhtml    to make HTML files named index.html in directories
+	echo.  singlehtml to make a single large HTML file
+	echo.  pickle     to make pickle files
+	echo.  json       to make JSON files
+	echo.  htmlhelp   to make HTML files and a HTML help project
+	echo.  qthelp     to make HTML files and a qthelp project
+	echo.  devhelp    to make HTML files and a Devhelp project
+	echo.  epub       to make an epub
+	echo.  epub3      to make an epub3
+	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+	echo.  text       to make text files
+	echo.  man        to make manual pages
+	echo.  texinfo    to make Texinfo files
+	echo.  gettext    to make PO message catalogs
+	echo.  changes    to make an overview over all changed/added/deprecated items
+	echo.  xml        to make Docutils-native XML files
+	echo.  pseudoxml  to make pseudoxml-XML files for display purposes
+	echo.  linkcheck  to check all external links for integrity
+	echo.  doctest    to run all doctests embedded in the documentation if enabled
+	echo.  coverage   to run coverage check of the documentation if enabled
+	echo.  dummy      to check syntax errors of document sources
+	goto end
+)
+
+if "%1" == "clean" (
+	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+	del /q /s %BUILDDIR%\*
+	goto end
+)
+
+
+REM Check if sphinx-build is available and fallback to Python version if any
+%SPHINXBUILD% 1>NUL 2>NUL
+if errorlevel 9009 goto sphinx_python
+goto sphinx_ok
+
+:sphinx_python
+
+set SPHINXBUILD=python -m sphinx.__init__
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+:sphinx_ok
+
+
+if "%1" == "html" (
+	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+	goto end
+)
+
+if "%1" == "dirhtml" (
+	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+	goto end
+)
+
+if "%1" == "singlehtml" (
+	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+	goto end
+)
+
+if "%1" == "pickle" (
+	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the pickle files.
+	goto end
+)
+
+if "%1" == "json" (
+	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the JSON files.
+	goto end
+)
+
+if "%1" == "htmlhelp" (
+	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+	goto end
+)
+
+if "%1" == "qthelp" (
+	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\anosql.qhcp
+	echo.To view the help file:
+	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\anosql.ghc
+	goto end
+)
+
+if "%1" == "devhelp" (
+	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished.
+	goto end
+)
+
+if "%1" == "epub" (
+	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The epub file is in %BUILDDIR%/epub.
+	goto end
+)
+
+if "%1" == "epub3" (
+	%SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The epub3 file is in %BUILDDIR%/epub3.
+	goto end
+)
+
+if "%1" == "latex" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "latexpdf" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	cd %BUILDDIR%/latex
+	make all-pdf
+	cd %~dp0
+	echo.
+	echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "latexpdfja" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	cd %BUILDDIR%/latex
+	make all-pdf-ja
+	cd %~dp0
+	echo.
+	echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "text" (
+	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The text files are in %BUILDDIR%/text.
+	goto end
+)
+
+if "%1" == "man" (
+	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The manual pages are in %BUILDDIR%/man.
+	goto end
+)
+
+if "%1" == "texinfo" (
+	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+	goto end
+)
+
+if "%1" == "gettext" (
+	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+	goto end
+)
+
+if "%1" == "changes" (
+	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.The overview file is in %BUILDDIR%/changes.
+	goto end
+)
+
+if "%1" == "linkcheck" (
+	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+	goto end
+)
+
+if "%1" == "doctest" (
+	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+	goto end
+)
+
+if "%1" == "coverage" (
+	%SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Testing of coverage in the sources finished, look at the ^
+results in %BUILDDIR%/coverage/python.txt.
+	goto end
+)
+
+if "%1" == "xml" (
+	%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The XML files are in %BUILDDIR%/xml.
+	goto end
+)
+
+if "%1" == "pseudoxml" (
+	%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
+	goto end
+)
+
+if "%1" == "dummy" (
+	%SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. Dummy builder generates no files.
+	goto end
+)
+
+:end

projects/anosql/doc/source/anosql.adapters.psycopg2.rst

@@ -0,0 +1,7 @@
+anosql.adapters.psycopg2 module
+===============================
+
+.. automodule:: anosql.adapters.psycopg2
+    :members:
+    :undoc-members:
+    :show-inheritance:

projects/anosql/doc/source/anosql.adapters.rst

@@ -0,0 +1,18 @@
+anosql.adapters package
+=======================
+
+Submodules
+----------
+
+.. toctree::
+
+   anosql.adapters.psycopg2
+   anosql.adapters.sqlite3
+
+Module contents
+---------------
+
+.. automodule:: anosql.adapters
+    :members:
+    :undoc-members:
+    :show-inheritance:

projects/anosql/doc/source/anosql.adapters.sqlite3.rst

@@ -0,0 +1,7 @@
+anosql.adapters.sqlite3 module
+==============================
+
+.. automodule:: anosql.adapters.sqlite3
+    :members:
+    :undoc-members:
+    :show-inheritance:

projects/anosql/doc/source/anosql.core.rst

@@ -0,0 +1,7 @@
+anosql.core module
+==================
+
+.. automodule:: anosql.core
+    :members:
+    :undoc-members:
+    :show-inheritance:

projects/anosql/doc/source/anosql.exceptions.rst

@@ -0,0 +1,7 @@
+anosql.exceptions module
+========================
+
+.. automodule:: anosql.exceptions
+    :members:
+    :undoc-members:
+    :show-inheritance:

projects/anosql/doc/source/anosql.patterns.rst

@@ -0,0 +1,7 @@
+anosql.patterns module
+======================
+
+.. automodule:: anosql.patterns
+    :members:
+    :undoc-members:
+    :show-inheritance:

projects/anosql/doc/source/anosql.rst

@@ -0,0 +1,26 @@
+anosql package
+==============
+
+Subpackages
+-----------
+
+.. toctree::
+
+    anosql.adapters
+
+Submodules
+----------
+
+.. toctree::
+
+   anosql.core
+   anosql.exceptions
+   anosql.patterns
+
+Module contents
+---------------
+
+.. automodule:: anosql
+    :members:
+    :undoc-members:
+    :show-inheritance:

projects/anosql/doc/source/modules.rst

@@ -0,0 +1,7 @@
+anosql
+======
+
+.. toctree::
+   :maxdepth: 4
+
+   anosql

projects/anosql/doc/upgrading.rst

@@ -0,0 +1,83 @@
+#########
+Upgrading
+#########
+
+Upgrading from 0.x to 1.x
+=========================
+
+Changed ``load_queries`` and ``load_queries_from_string``
+---------------------------------------------------------
+
+These methods were changed, mostly for brevity. To load ``anosql`` queries, you should now use
+the ``anosql.from_str`` to load queries from a SQL string, and ``anosql.from_path`` to load queries
+from a SQL file, or directory of SQL files.
+
+Removed the ``$`` "record" operator
+-----------------------------------
+
+Because most database drivers have more efficient, robust, and featureful ways of controlling the
+rows and records output, this feature was removed.
+
+See:
+
+* `sqlite.Row <https://docs.python.org/2/library/sqlite3.html#sqlite3.Row>`_
+* `psycopg2 - Connection and Cursor subclasses <http://initd.org/psycopg/docs/extras.html#connection-and-cursor-subclasses>`_
+
+
+SQLite example::
+
+    conn = sqlite3.connect("...")
+    conn.row_factory = sqlite3.Row
+    actual = queries.get_all_users(conn)
+
+    assert actual[0]["userid"] == 1
+    assert actual[0]["username"] == "bobsmith"
+    assert actual[0][2] == "Bob"
+    assert actual[0]["lastname" == "Smith"
+
+PostgreSQL example::
+
+    with psycopg2.connect("...", cursor_factory=psycopg2.extras.RealDictCursor) as conn:
+        actual = queries.get_all_users(conn)
+
+    assert actual[0] == {
+        "userid": 1,
+        "username": "bobsmith",
+        "firstname": "Bob",
+        "lastname": "Smith",
+    }
+
+Driver Adapter classes instead of QueryLoader
+---------------------------------------------
+
+I'm not aware of anyone who actually has made or distributed an extension for ``anosql``, as it was
+only available in its current form for a few weeks. So this notice is really just for completeness.
+
+For ``0.3.x`` versions of ``anosql`` in order to add a new database extensions you had to build a
+subclass of ``anosql.QueryLoader``. This base class is no longer available, and driver adapters no
+longer have to extend from any class at all. They are duck-typed classes which are expected to
+adhere to a standard interface. For more information about this see :ref:`Extending anosql <extending-anosql>`.
+
+New Things
+==========
+
+Use the database driver ``cursor`` directly
+-------------------------------------------
+
+All the queries with a `SELECT` type have a duplicate method suffixed by `_cursor` which is a context manager to the database cursor. So `get_all_blogs(conn)` can also be used as:
+
+::
+
+    rows = queries.get_all_blogs(conn)
+    # [(1, "My Blog", "yadayada"), ...]
+
+    with queries.get_all_blogs_cursor(conn) as cur:
+        # All the power of the underlying cursor object! Not limited to just a list of rows.
+        for row in cur:
+            print(row)
+
+
+New operator types for runnings scripts ``#`` and bulk-inserts ``*!``
+---------------------------------------------------------------------
+
+See :ref:`Query Operations <query-operations>`

projects/anosql/src/anosql/__init__.py

@@ -0,0 +1,18 @@
+from .core import (
+    from_path,
+    from_str,
+    SQLOperationType,
+)
+from .exceptions import (
+    SQLLoadException,
+    SQLParseException,
+)
+
+
+__all__ = [
+    "from_path",
+    "from_str",
+    "SQLOperationType",
+    "SQLLoadException",
+    "SQLParseException",
+]

projects/anosql/src/anosql/adapters/psycopg2.py

@@ -0,0 +1,61 @@
+from contextlib import contextmanager
+
+from ..patterns import var_pattern
+
+
+def replacer(match):
+    gd = match.groupdict()
+    if gd["dblquote"] is not None:
+        return gd["dblquote"]
+    elif gd["quote"] is not None:
+        return gd["quote"]
+    else:
+        return "{lead}%({var_name})s{trail}".format(
+            lead=gd["lead"],
+            var_name=gd["var_name"],
+            trail=gd["trail"],
+        )
+
+
+class PsycoPG2Adapter(object):
+    @staticmethod
+    def process_sql(_query_name, _op_type, sql):
+        return var_pattern.sub(replacer, sql)
+
+    @staticmethod
+    def select(conn, _query_name, sql, parameters):
+        with conn.cursor() as cur:
+            cur.execute(sql, parameters)
+            return cur.fetchall()
+
+    @staticmethod
+    @contextmanager
+    def select_cursor(conn, _query_name, sql, parameters):
+        with conn.cursor() as cur:
+            cur.execute(sql, parameters)
+            yield cur
+
+    @staticmethod
+    def insert_update_delete(conn, _query_name, sql, parameters):
+        with conn.cursor() as cur:
+            cur.execute(sql, parameters)
+
+    @staticmethod
+    def insert_update_delete_many(conn, _query_name, sql, parameters):
+        with conn.cursor() as cur:
+            cur.executemany(sql, parameters)
+
+    @staticmethod
+    def insert_returning(conn, _query_name, sql, parameters):
+        with conn.cursor() as cur:
+            cur.execute(sql, parameters)
+            res = cur.fetchone()
+            if res:
+                return res[0] if len(res) == 1 else res
+            else:
+                return None
+
+    @staticmethod
+    def execute_script(conn, sql):
+        with conn.cursor() as cur:
+            cur.execute(sql)

projects/anosql/src/anosql/adapters/sqlite3.py

@@ -0,0 +1,96 @@
+"""
+A driver object implementing support for SQLite3
+"""
+
+from contextlib import contextmanager
+import logging
+import re
+import sqlite3
+
+
+log = logging.getLogger(__name__)
+
+
+class SQLite3DriverAdapter(object):
+    @staticmethod
+    def process_sql(_query_name, _op_type, sql):
+        """Munge queries.
+
+        Args:
+            _query_name (str): The name of the sql query.
+            _op_type (anosql.SQLOperationType): The type of SQL operation performed by the sql.
+            sql (str): The sql as written before processing.
+
+        Returns:
+            str: A normalized form of the query suitable to logging or copy/paste.
+
+        """
+
+        # Normalize out comments
+        sql = re.sub(r"-{2,}.*?\n", "", sql)
+
+        # Normalize out a variety of syntactically irrelevant whitespace
+        #
+        # FIXME: This is technically invalid, because what if you had `foo ` as
+        # a table name. Shit idea, but this won't handle it correctly.
+        sql = re.sub(r"\s+", " ", sql)
+        sql = re.sub(r"\(\s+", "(", sql)
+        sql = re.sub(r"\s+\)", ")", sql)
+        sql = re.sub(r"\s+,", ",", sql)
+        sql = re.sub(r"\s+;", ";", sql)
+
+        return sql
+
+    @staticmethod
+    def select(conn, _query_name, sql, parameters):
+        cur = conn.cursor()
+        log.debug({"sql": sql, "parameters": parameters})
+        cur.execute(sql, parameters)
+        results = cur.fetchall()
+        cur.close()
+        return results
+
+    @staticmethod
+    @contextmanager
+    def select_cursor(conn: sqlite3.Connection, _query_name, sql, parameters):
+        cur = conn.cursor()
+        log.debug({"sql": sql, "parameters": parameters})
+        cur.execute(sql, parameters)
+        try:
+            yield cur
+        finally:
+            cur.close()
+
+    @staticmethod
+    def insert_update_delete(conn: sqlite3.Connection, _query_name, sql, parameters):
+        log.debug({"sql": sql, "parameters": parameters})
+        conn.execute(sql, parameters)
+
+    @staticmethod
+    def insert_update_delete_many(
+        conn: sqlite3.Connection, _query_name, sql, parameters
+    ):
+        log.debug({"sql": sql, "parameters": parameters})
+        conn.executemany(sql, parameters)
+
+    @staticmethod
+    def insert_returning(conn: sqlite3.Connection, _query_name, sql, parameters):
+        cur = conn.cursor()
+        log.debug({"sql": sql, "parameters": parameters})
+        cur.execute(sql, parameters)
+
+        if "returning" not in sql.lower():
+            # Original behavior - return the last row ID
+            results = cur.lastrowid
+        else:
+            # New behavior - honor a `RETURNING` clause
+            results = cur.fetchall()
+
+        log.debug({"results": results})
+        cur.close()
+        return results
+
+    @staticmethod
+    def execute_script(conn: sqlite3.Connection, sql):
+        log.debug({"sql": sql, "parameters": None})
+        conn.executescript(sql)

projects/anosql/src/anosql/core.py

@@ -0,0 +1,362 @@
+import os
+
+from .adapters.psycopg2 import PsycoPG2Adapter
+from .adapters.sqlite3 import SQLite3DriverAdapter
+from .exceptions import (
+    SQLLoadException,
+    SQLParseException,
+)
+from .patterns import (
+    doc_comment_pattern,
+    empty_pattern,
+    query_name_definition_pattern,
+    valid_query_name_pattern,
+)
+
+
+_ADAPTERS = {
+    "psycopg2": PsycoPG2Adapter,
+    "sqlite3": SQLite3DriverAdapter,
+}
+
+
+def register_driver_adapter(driver_name, driver_adapter):
+    """Registers custom driver adapter classes to extend ``anosql`` to to handle additional drivers.
+
+    For details on how to create a new driver adapter see :ref:`driver-adapters` documentation.
+
+    Args:
+        driver_name (str): The driver type name.
+        driver_adapter (callable): Either n class or function which creates an instance of a
+                                   driver adapter.
+
+    Returns:
+        None
+
+    Examples:
+        To register a new loader::
+
+            class MyDbAdapter():
+                def process_sql(self, name, op_type, sql):
+                    pass
+
+                def select(self, conn, sql, parameters):
+                    pass
+
+                @contextmanager
+                def select_cursor(self, conn, sql, parameters):
+                    pass
+
+                def insert_update_delete(self, conn, sql, parameters):
+                    pass
+
+                def insert_update_delete_many(self, conn, sql, parameters):
+                    pass
+
+                def insert_returning(self, conn, sql, parameters):
+                    pass
+
+                def execute_script(self, conn, sql):
+                    pass
+
+
+            anosql.register_driver_adapter("mydb", MyDbAdapter)
+
+        If your adapter constructor takes arguments you can register a function which can build
+        your adapter instance::
+
+            def adapter_factory():
+                return MyDbAdapter("foo", 42)
+
+            anosql.register_driver_adapter("mydb", adapter_factory)
+
+    """
+    _ADAPTERS[driver_name] = driver_adapter
+
+
+def get_driver_adapter(driver_name):
+    """Get the driver adapter instance registered by the ``driver_name``.
+
+    Args:
+        driver_name (str): The database driver name.
+
+    Returns:
+        object: A driver adapter class.
+    """
+    try:
+        driver_adapter = _ADAPTERS[driver_name]
+    except KeyError:
+        raise ValueError("Encountered unregistered driver_name: {}".format(driver_name))
+
+    return driver_adapter()
+
+
+class SQLOperationType(object):
+    """Enumeration (kind of) of anosql operation types"""
+
+    INSERT_RETURNING = 0
+    INSERT_UPDATE_DELETE = 1
+    INSERT_UPDATE_DELETE_MANY = 2
+    SCRIPT = 3
+    SELECT = 4
+    SELECT_ONE_ROW = 5
+
+
+class Queries:
+    """Container object with dynamic methods built from SQL queries.
+
+    The ``-- name`` definition comments in the SQL content determine what the dynamic
+    methods of this class will be named.
+
+    @DynamicAttrs
+    """
+
+    def __init__(self, queries=None):
+        """Queries constructor.
+
+        Args:
+            queries (list(tuple)):
+        """
+        if queries is None:
+            queries = []
+        self._available_queries = set()
+
+        for query_name, fn in queries:
+            self.add_query(query_name, fn)
+
+    @property
+    def available_queries(self):
+        """Returns listing of all the available query methods loaded in this class.
+
+        Returns:
+            list(str): List of dot-separated method accessor names.
+        """
+        return sorted(self._available_queries)
+
+    def __repr__(self):
+        return "Queries(" + self.available_queries.__repr__() + ")"
+
+    def add_query(self, query_name, fn):
+        """Adds a new dynamic method to this class.
+
+        Args:
+            query_name (str): The method name as found in the SQL content.
+            fn (function): The loaded query function.
+
+        Returns:
+
+        """
+        setattr(self, query_name, fn)
+        self._available_queries.add(query_name)
+
+    def add_child_queries(self, child_name, child_queries):
+        """Adds a Queries object as a property.
+
+        Args:
+            child_name (str): The property name to group the child queries under.
+            child_queries (Queries): Queries instance to add as sub-queries.
+
+        Returns:
+            None
+
+        """
+        setattr(self, child_name, child_queries)
+        for child_query_name in child_queries.available_queries:
+            self._available_queries.add("{}.{}".format(child_name, child_query_name))
+
+
+def _create_fns(query_name, docs, op_type, sql, driver_adapter):
+    def fn(conn, *args, **kwargs):
+        parameters = kwargs if len(kwargs) > 0 else args
+        if op_type == SQLOperationType.INSERT_RETURNING:
+            return driver_adapter.insert_returning(conn, query_name, sql, parameters)
+        elif op_type == SQLOperationType.INSERT_UPDATE_DELETE:
+            return driver_adapter.insert_update_delete(
+                conn, query_name, sql, parameters
+            )
+        elif op_type == SQLOperationType.INSERT_UPDATE_DELETE_MANY:
+            return driver_adapter.insert_update_delete_many(
+                conn, query_name, sql, *parameters
+            )
+        elif op_type == SQLOperationType.SCRIPT:
+            return driver_adapter.execute_script(conn, sql)
+        elif op_type == SQLOperationType.SELECT_ONE_ROW:
+            res = driver_adapter.select(conn, query_name, sql, parameters)
+            return res[0] if len(res) == 1 else None
+        elif op_type == SQLOperationType.SELECT:
+            return driver_adapter.select(conn, query_name, sql, parameters)
+        else:
+            raise ValueError("Unknown op_type: {}".format(op_type))
+
+    fn.__name__ = query_name
+    fn.__doc__ = docs
+    fn.sql = sql
+
+    ctx_mgr_method_name = "{}_cursor".format(query_name)
+
+    def ctx_mgr(conn, *args, **kwargs):
+        parameters = kwargs if len(kwargs) > 0 else args
+        return driver_adapter.select_cursor(conn, query_name, sql, parameters)
+
+    ctx_mgr.__name__ = ctx_mgr_method_name
+    ctx_mgr.__doc__ = docs
+    ctx_mgr.sql = sql
+
+    if op_type == SQLOperationType.SELECT:
+        return [(query_name, fn), (ctx_mgr_method_name, ctx_mgr)]
+
+    return [(query_name, fn)]
+
+
+def load_methods(sql_text, driver_adapter):
+    lines = sql_text.strip().splitlines()
+    query_name = lines[0].replace("-", "_")
+
+    if query_name.endswith("<!"):
+        op_type = SQLOperationType.INSERT_RETURNING
+        query_name = query_name[:-2]
+    elif query_name.endswith("*!"):
+        op_type = SQLOperationType.INSERT_UPDATE_DELETE_MANY
+        query_name = query_name[:-2]
+    elif query_name.endswith("!"):
+        op_type = SQLOperationType.INSERT_UPDATE_DELETE
+        query_name = query_name[:-1]
+    elif query_name.endswith("#"):
+        op_type = SQLOperationType.SCRIPT
+        query_name = query_name[:-1]
+    elif query_name.endswith("?"):
+        op_type = SQLOperationType.SELECT_ONE_ROW
+        query_name = query_name[:-1]
+    else:
+        op_type = SQLOperationType.SELECT
+
+    if not valid_query_name_pattern.match(query_name):
+        raise SQLParseException(
+            'name must convert to valid python variable, got "{}".'.format(query_name)
+        )
+
+    docs = ""
+    sql = ""
+    for line in lines[1:]:
+        match = doc_comment_pattern.match(line)
+        if match:
+            docs += match.group(1) + "\n"
+        else:
+            sql += line + "\n"
+
+    docs = docs.strip()
+    sql = driver_adapter.process_sql(query_name, op_type, sql.strip())
+
+    return _create_fns(query_name, docs, op_type, sql, driver_adapter)
+
+
+def load_queries_from_sql(sql, driver_adapter):
+    queries = []
+    for query_text in query_name_definition_pattern.split(sql):
+        if not empty_pattern.match(query_text):
+            for method_pair in load_methods(query_text, driver_adapter):
+                queries.append(method_pair)
+    return queries
+
+
+def load_queries_from_file(file_path, driver_adapter):
+    with open(file_path) as fp:
+        return load_queries_from_sql(fp.read(), driver_adapter)
+
+
+def load_queries_from_dir_path(dir_path, query_loader):
+    if not os.path.isdir(dir_path):
+        raise ValueError("The path {} must be a directory".format(dir_path))
+
+    def _recurse_load_queries(path):
+        queries = Queries()
+        for item in os.listdir(path):
+            item_path = os.path.join(path, item)
+            if os.path.isfile(item_path) and not item.endswith(".sql"):
+                continue
+            elif os.path.isfile(item_path) and item.endswith(".sql"):
+                for name, fn in load_queries_from_file(item_path, query_loader):
+                    queries.add_query(name, fn)
+            elif os.path.isdir(item_path):
+                child_queries = _recurse_load_queries(item_path)
+                queries.add_child_queries(item, child_queries)
+            else:
+                # This should be practically unreachable.
+                raise SQLLoadException(
+                    "The path must be a directory or file, got {}".format(item_path)
+                )
+        return queries
+
+    return _recurse_load_queries(dir_path)
+
+
+def from_str(sql, driver_name):
+    """Load queries from a SQL string.
+
+    Args:
+        sql (str) A string containing SQL statements and anosql name:
+        driver_name (str): The database driver to use to load and execute queries.
+
+    Returns:
+        Queries
+
+    Example:
+        Loading queries from a SQL string::
+
+            import sqlite3
+            import anosql
+
+            sql_text = \"""
+            -- name: get-all-greetings
+            -- Get all the greetings in the database
+            select * from greetings;
+
+            -- name: get-users-by-username
+            -- Get all the users from the database,
+            -- and return it as a dict
+            select * from users where username =:username;
+            \"""
+
+            queries = anosql.from_str(sql_text, db_driver="sqlite3")
+            queries.get_all_greetings(conn)
+            queries.get_users_by_username(conn, username="willvaughn")
+
+    """
+    driver_adapter = get_driver_adapter(driver_name)
+    return Queries(load_queries_from_sql(sql, driver_adapter))
+
+
+def from_path(sql_path, driver_name):
+    """Load queries from a sql file, or a directory of sql files.
+
+    Args:
+        sql_path (str): Path to a ``.sql`` file or directory containing ``.sql`` files.
+        driver_name (str): The database driver to use to load and execute queries.
+
+    Returns:
+        Queries
+
+    Example:
+        Loading queries paths::
+
+            import sqlite3
+            import anosql
+
+            queries = anosql.from_path("./greetings.sql", driver_name="sqlite3")
+            queries2 = anosql.from_path("./sql_dir", driver_name="sqlite3")
+
+    """
+    if not os.path.exists(sql_path):
+        raise SQLLoadException("File does not exist: {}.".format(sql_path), sql_path)
+
+    driver_adapter = get_driver_adapter(driver_name)
+
+    if os.path.isdir(sql_path):
+        return load_queries_from_dir_path(sql_path, driver_adapter)
+    elif os.path.isfile(sql_path):
+        return Queries(load_queries_from_file(sql_path, driver_adapter))
+    else:
+        raise SQLLoadException(
+            "The sql_path must be a directory or file, got {}".format(sql_path),
+            sql_path,
+        )

projects/anosql/src/anosql/exceptions.py

@@ -0,0 +1,6 @@
+class SQLLoadException(Exception):
+    pass
+
+
+class SQLParseException(Exception):
+    pass

projects/anosql/src/anosql/patterns.py

@@ -0,0 +1,31 @@
+import re
+
+
+query_name_definition_pattern = re.compile(r"--\s*name\s*:\s*")
+"""
+Pattern: Identifies name definition comments.
+"""
+
+empty_pattern = re.compile(r"^\s*$")
+"""
+Pattern: Identifies empty lines.
+"""
+
+valid_query_name_pattern = re.compile(r"\w+")
+"""
+Pattern: Enforces names are valid python variable names.
+"""
+
+doc_comment_pattern = re.compile(r"\s*--\s*(.*)$")
+"""
+Pattern: Identifies SQL comments.
+"""
+
+var_pattern = re.compile(
+    r'(?P<dblquote>"[^"]+")|'
+    r"(?P<quote>\'[^\']+\')|"
+    r"(?P<lead>[^:]):(?P<var_name>[\w-]+)(?P<trail>[^:]?)"
+)
+"""
+Pattern: Identifies variable definitions in SQL code.
+"""

projects/anosql/test/blogdb/data/blogs_data.csv

@@ -0,0 +1,3 @@
+1,What I did Today,"I mowed the lawn - washed some clothes - ate a burger.",2017-07-28
+3,Testing,Is this thing on?,2018-01-01
+1,How to make a pie.,"1. Make crust\n2. Fill\n3. Bake\n4.Eat",2018-11-23

projects/anosql/test/blogdb/data/users_data.csv

@@ -0,0 +1,3 @@
+bobsmith,Bob,Smith
+johndoe,John,Doe
+janedoe,Jane,Doe

projects/anosql/test/blogdb/sql/blogs/blogs.sql

@@ -0,0 +1,26 @@
+-- name: publish-blog<!
+insert into blogs (
+  userid,
+  title,
+  content,
+  published
+)
+values (
+  :userid,
+  :title,
+  :content,
+  :published
+)
+
+-- name: remove-blog!
+-- Remove a blog from the database
+delete from blogs where blogid = :blogid;
+
+
+-- name: get-user-blogs
+-- Get blogs authored by a user.
+  select title,
+         published
+    from blogs
+   where userid = :userid
+order by published desc;

projects/anosql/test/blogdb/sql/blogs/blogs_pg.sql

@@ -0,0 +1,41 @@
+-- name: pg-get-blogs-published-after
+-- Get all blogs by all authors published after the given date.
+  select title,
+         username,
+         to_char(published, 'YYYY-MM-DD HH24:MI') as published
+    from blogs
+    join users using(userid)
+   where published >= :published
+order by published desc;
+
+
+-- name: pg-publish-blog<!
+insert into blogs (
+  userid,
+  title,
+  content,
+  published
+)
+values (
+  :userid,
+  :title,
+  :content,
+  :published
+)
+returning blogid, title;
+
+
+-- name: pg-bulk-publish*!
+-- Insert many blogs at once
+insert into blogs (
+  userid,
+  title,
+  content,
+  published
+)
+values (
+  :userid,
+  :title,
+  :content,
+  :published
+)

projects/anosql/test/blogdb/sql/blogs/blogs_sqlite.sql

@@ -0,0 +1,20 @@
+-- name: sqlite-get-blogs-published-after
+-- Get all blogs by all authors published after the given date.
+    select b.title,
+           u.username,
+           strftime('%Y-%m-%d %H:%M', b.published) as published
+      from blogs b
+inner join users u on b.userid = u.userid
+     where b.published >= :published
+  order by b.published desc;
+
+
+-- name: sqlite-bulk-publish*!
+-- Insert many blogs at once
+insert into blogs (
+  userid,
+  title,
+  content,
+  published
+)
+values (?, ?, ? , ?);

projects/anosql/test/blogdb/sql/users/users.sql

@@ -0,0 +1,11 @@
+-- name: get-all
+-- Get all user records
+select * from users;
+
+-- name: get-all-sorted
+-- Get all user records sorted by username
+select * from users order by username asc;
+
+-- name: get-one?
+-- Get one user based on its id
+select username, firstname, lastname from users where userid = %s;

projects/anosql/test/conftest.py

@@ -0,0 +1,122 @@
+import csv
+import os
+import sqlite3
+
+import pytest
+
+
+BLOGDB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "blogdb")
+USERS_DATA_PATH = os.path.join(BLOGDB_PATH, "data", "users_data.csv")
+BLOGS_DATA_PATH = os.path.join(BLOGDB_PATH, "data", "blogs_data.csv")
+
+
+def populate_sqlite3_db(db_path):
+    conn = sqlite3.connect(db_path)
+    cur = conn.cursor()
+    cur.executescript(
+        """
+            create table users (
+                userid integer not null primary key,
+                username text not null,
+                firstname integer not null,
+                lastname text not null
+            );
+
+            create table blogs (
+                blogid integer not null primary key,
+                userid integer not null,
+                title text not null,
+                content text not null,
+                published date not null default CURRENT_DATE,
+                foreign key(userid) references users(userid)
+            );
+            """
+    )
+
+    with open(USERS_DATA_PATH) as fp:
+        users = list(csv.reader(fp))
+        cur.executemany(
+            """
+               insert into users (
+                    username,
+                    firstname,
+                    lastname
+               ) values (?, ?, ?);""",
+            users,
+        )
+    with open(BLOGS_DATA_PATH) as fp:
+        blogs = list(csv.reader(fp))
+        cur.executemany(
+            """
+                insert into blogs (
+                    userid,
+                    title,
+                    content,
+                    published
+                ) values (?, ?, ?, ?);""",
+            blogs,
+        )
+
+    conn.commit()
+    conn.close()
+
+
+@pytest.fixture()
+def sqlite3_db_path(tmpdir):
+    db_path = os.path.join(tmpdir.strpath, "blogdb.db")
+    populate_sqlite3_db(db_path)
+    return db_path
+
+
+@pytest.fixture()
+def sqlite3_conn(sqlite3_db_path):
+    conn = sqlite3.connect(sqlite3_db_path)
+    yield conn
+    conn.close()
+
+
+@pytest.fixture
+def pg_conn(postgresql):
+    with postgresql:
+        # Loads data from blogdb fixture data
+        with postgresql.cursor() as cur:
+            cur.execute(
+                """
+                create table users (
+                    userid serial not null primary key,
+                    username varchar(32) not null,
+                    firstname varchar(255) not null,
+                    lastname varchar(255) not null
+                );"""
+            )
+            cur.execute(
+                """
+                create table blogs (
+                    blogid serial not null primary key,
+                    userid integer not null references users(userid),
+                    title varchar(255) not null,
+                    content text not null,
+                    published date not null default CURRENT_DATE
+                );"""
+            )
+
+        with postgresql.cursor() as cur:
+            with open(USERS_DATA_PATH) as fp:
+                cur.copy_from(
+                    fp, "users", sep=",", columns=["username", "firstname", "lastname"]
+                )
+            with open(BLOGS_DATA_PATH) as fp:
+                cur.copy_from(
+                    fp,
+                    "blogs",
+                    sep=",",
+                    columns=["userid", "title", "content", "published"],
+                )
+
+    return postgresql
+
+
+@pytest.fixture()
+def pg_dsn(pg_conn):
+    p = pg_conn.get_dsn_parameters()
+    return "postgres://{user}@{host}:{port}/{dbname}".format(**p)

projects/anosql/test/test_psycopg2.py

@@ -0,0 +1,132 @@
+from datetime import date
+import os
+
+import anosql
+import psycopg2
+import psycopg2.extras
+import pytest
+
+
+@pytest.fixture()
+def queries():
+    dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "blogdb", "sql")
+    return anosql.from_path(dir_path, "psycopg2")
+
+
+def test_record_query(pg_conn, queries):
+    dsn = pg_conn.get_dsn_parameters()
+    with psycopg2.connect(cursor_factory=psycopg2.extras.RealDictCursor, **dsn) as conn:
+        actual = queries.users.get_all(conn)
+
+    assert len(actual) == 3
+    assert actual[0] == {
+        "userid": 1,
+        "username": "bobsmith",
+        "firstname": "Bob",
+        "lastname": "Smith",
+    }
+
+
+def test_parameterized_query(pg_conn, queries):
+    actual = queries.blogs.get_user_blogs(pg_conn, userid=1)
+    expected = [
+        ("How to make a pie.", date(2018, 11, 23)),
+        ("What I did Today", date(2017, 7, 28)),
+    ]
+    assert actual == expected
+
+
+def test_parameterized_record_query(pg_conn, queries):
+    dsn = pg_conn.get_dsn_parameters()
+    with psycopg2.connect(cursor_factory=psycopg2.extras.RealDictCursor, **dsn) as conn:
+        actual = queries.blogs.pg_get_blogs_published_after(
+            conn, published=date(2018, 1, 1)
+        )
+
+    expected = [
+        {
+            "title": "How to make a pie.",
+            "username": "bobsmith",
+            "published": "2018-11-23 00:00",
+        },
+        {"title": "Testing", "username": "janedoe", "published": "2018-01-01 00:00"},
+    ]
+
+    assert actual == expected
+
+
+def test_select_cursor_context_manager(pg_conn, queries):
+    with queries.blogs.get_user_blogs_cursor(pg_conn, userid=1) as cursor:
+        actual = cursor.fetchall()
+        expected = [
+            ("How to make a pie.", date(2018, 11, 23)),
+            ("What I did Today", date(2017, 7, 28)),
+        ]
+        assert actual == expected
+
+
+def test_insert_returning(pg_conn, queries):
+    with pg_conn:
+        blogid, title = queries.blogs.pg_publish_blog(
+            pg_conn,
+            userid=2,
+            title="My first blog",
+            content="Hello, World!",
+            published=date(2018, 12, 4),
+        )
+        with pg_conn.cursor() as cur:
+            cur.execute(
+                """\
+                select blogid,
+                       title
+                  from blogs
+                 where blogid = %s;
+                """,
+                (blogid,),
+            )
+            expected = cur.fetchone()
+
+    assert (blogid, title) == expected
+
+
+def test_delete(pg_conn, queries):
+    # Removing the "janedoe" blog titled "Testing"
+    actual = queries.blogs.remove_blog(pg_conn, blogid=2)
+    assert actual is None
+
+    janes_blogs = queries.blogs.get_user_blogs(pg_conn, userid=3)
+    assert len(janes_blogs) == 0
+
+
+def test_insert_many(pg_conn, queries):
+    blogs = [
+        {
+            "userid": 2,
+            "title": "Blog Part 1",
+            "content": "content - 1",
+            "published": date(2018, 12, 4),
+        },
+        {
+            "userid": 2,
+            "title": "Blog Part 2",
+            "content": "content - 2",
+            "published": date(2018, 12, 5),
+        },
+        {
+            "userid": 2,
+            "title": "Blog Part 3",
+            "content": "content - 3",
+            "published": date(2018, 12, 6),
+        },
+    ]
+
+    with pg_conn:
+        actual = queries.blogs.pg_bulk_publish(pg_conn, blogs)
+        assert actual is None
+
+        johns_blogs = queries.blogs.get_user_blogs(pg_conn, userid=2)
+        assert johns_blogs == [
+            ("Blog Part 3", date(2018, 12, 6)),
+            ("Blog Part 2", date(2018, 12, 5)),
+            ("Blog Part 1", date(2018, 12, 4)),
+        ]

projects/anosql/test/test_simple.py

@@ -0,0 +1,238 @@
+import anosql
+import pytest
+
+
+@pytest.fixture
+def sqlite(request):
+    import sqlite3
+
+    sqlconnection = sqlite3.connect(":memory:")
+
+    def fin():
+        "teardown"
+        print("teardown")
+        sqlconnection.close()
+
+    request.addfinalizer(fin)
+
+    return sqlconnection
+
+
+def test_simple_query(sqlite):
+    _test_create_insert = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (a, b, c);\n\n"
+        "-- name: insert-some-value!\n"
+        "INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n"
+    )
+
+    q = anosql.from_str(_test_create_insert, "sqlite3")
+    q.create_some_table(sqlite)
+    q.insert_some_value(sqlite)
+
+
+def test_auto_insert_query(sqlite):
+    _test_create_insert = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (a, b, c);\n\n"
+        "-- name: insert-some-value<!\n"
+        "INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n"
+    )
+
+    q = anosql.from_str(_test_create_insert, "sqlite3")
+    q.create_some_table(sqlite)
+    assert q.insert_some_value(sqlite) == 1
+    assert q.insert_some_value(sqlite) == 2
+    assert q.insert_some_value(sqlite) == 3
+
+
+def test_parametrized_insert(sqlite):
+    _test_create_insert = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (a, b, c);\n\n"
+        "-- name: insert-some-value!\n"
+        "INSERT INTO foo (a, b, c) VALUES (?, ?, ?);\n\n"
+        "-- name: get-all-values\n"
+        "SELECT * FROM foo;\n"
+    )
+
+    q = anosql.from_str(_test_create_insert, "sqlite3")
+    q.create_some_table(sqlite)
+    q.insert_some_value(sqlite, 10, 11, 12)
+    assert q.get_all_values(sqlite) == [(10, 11, 12)]
+
+
+def test_parametrized_insert_named(sqlite):
+    _test_create_insert = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (a, b, c);\n\n"
+        "-- name: insert-some-value!\n"
+        "INSERT INTO foo (a, b, c) VALUES (:a, :b, :c);\n\n"
+        "-- name: get-all-values\n"
+        "SELECT * FROM foo;\n"
+    )
+
+    q = anosql.from_str(_test_create_insert, "sqlite3")
+    q.create_some_table(sqlite)
+    q.insert_some_value(sqlite, c=12, b=11, a=10)
+    assert q.get_all_values(sqlite) == [(10, 11, 12)]
+
+
+def test_one_row(sqlite):
+    _test_one_row = (
+        "-- name: one-row?\n"
+        "SELECT 1, 'hello';\n\n"
+        "-- name: two-rows?\n"
+        "SELECT 1 UNION SELECT 2;\n"
+    )
+    q = anosql.from_str(_test_one_row, "sqlite3")
+    assert q.one_row(sqlite) == (1, "hello")
+    assert q.two_rows(sqlite) is None
+
+
+def test_simple_query_pg(postgresql):
+    _queries = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
+        "-- name: insert-some-value!\n"
+        "INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n\n"
+        "-- name: get-all-values\n"
+        "SELECT a, b, c FROM foo;\n"
+    )
+
+    q = anosql.from_str(_queries, "psycopg2")
+
+    q.create_some_table(postgresql)
+    q.insert_some_value(postgresql)
+
+    assert q.get_all_values(postgresql) == [(1, 2, 3)]
+
+
+def test_auto_insert_query_pg(postgresql):
+    _queries = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
+        "-- name: insert-some-value<!\n"
+        "INSERT INTO foo (a, b, c) VALUES (1, 2, 3) returning id;\n\n"
+        "-- name: get-all-values\n"
+        "SELECT a, b, c FROM foo;\n"
+    )
+
+    q = anosql.from_str(_queries, "psycopg2")
+
+    q.create_some_table(postgresql)
+
+    assert q.insert_some_value(postgresql) == 1
+    assert q.insert_some_value(postgresql) == 2
+
+
+def test_parameterized_insert_pg(postgresql):
+    _queries = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
+        "-- name: insert-some-value!\n"
+        "INSERT INTO foo (a, b, c) VALUES (%s, %s, %s);\n\n"
+        "-- name: get-all-values\n"
+        "SELECT a, b, c FROM foo;\n"
+    )
+
+    q = anosql.from_str(_queries, "psycopg2")
+
+    q.create_some_table(postgresql)
+    q.insert_some_value(postgresql, 1, 2, 3)
+
+    assert q.get_all_values(postgresql) == [(1, 2, 3)]
+
+
+def test_auto_parameterized_insert_query_pg(postgresql):
+    _queries = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
+        "-- name: insert-some-value<!\n"
+        "INSERT INTO foo (a, b, c) VALUES (%s, %s, %s) returning id;\n\n"
+        "-- name: get-all-values\n"
+        "SELECT a, b, c FROM foo;\n"
+    )
+
+    q = anosql.from_str(_queries, "psycopg2")
+
+    q.create_some_table(postgresql)
+
+    assert q.insert_some_value(postgresql, 1, 2, 3) == 1
+    assert q.get_all_values(postgresql) == [(1, 2, 3)]
+
+    assert q.insert_some_value(postgresql, 1, 2, 3) == 2
+
+
+def test_parameterized_select_pg(postgresql):
+    _queries = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
+        "-- name: insert-some-value!\n"
+        "INSERT INTO foo (a, b, c) VALUES (1, 2, 3)\n\n"
+        "-- name: get-all-values\n"
+        "SELECT a, b, c FROM foo WHERE a = %s;\n"
+    )
+
+    q = anosql.from_str(_queries, "psycopg2")
+
+    q.create_some_table(postgresql)
+    q.insert_some_value(postgresql)
+
+    assert q.get_all_values(postgresql, 1) == [(1, 2, 3)]
+
+
+def test_parameterized_insert_named_pg(postgresql):
+    _queries = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
+        "-- name: insert-some-value!\n"
+        "INSERT INTO foo (a, b, c) VALUES (%(a)s, %(b)s, %(c)s)\n\n"
+        "-- name: get-all-values\n"
+        "SELECT a, b, c FROM foo;\n"
+    )
+
+    q = anosql.from_str(_queries, "psycopg2")
+
+    q.create_some_table(postgresql)
+    q.insert_some_value(postgresql, a=1, b=2, c=3)
+
+    assert q.get_all_values(postgresql) == [(1, 2, 3)]
+
+
+def test_parameterized_select_named_pg(postgresql):
+    _queries = (
+        "-- name: create-some-table#\n"
+        "-- testing insertion\n"
+        "CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
+        "-- name: insert-some-value!\n"
+        "INSERT INTO foo (a, b, c) VALUES (1, 2, 3)\n\n"
+        "-- name: get-all-values\n"
+        "SELECT a, b, c FROM foo WHERE a = %(a)s;\n"
+    )
+
+    q = anosql.from_str(_queries, "psycopg2")
+
+    q.create_some_table(postgresql)
+    q.insert_some_value(postgresql)
+
+    assert q.get_all_values(postgresql, a=1) == [(1, 2, 3)]
+
+
+def test_without_trailing_semi_colon_pg():
+    """Make sure keywords ending queries are recognized even without
+    semi-colons.
+    """
+    _queries = "-- name: get-by-a\n" "SELECT a, b, c FROM foo WHERE a = :a\n"
+    q = anosql.from_str(_queries, "psycopg2")
+    assert q.get_by_a.sql == "SELECT a, b, c FROM foo WHERE a = %(a)s"

projects/anosql/test/test_sqlite3.py

@@ -0,0 +1,114 @@
+import os
+
+import anosql
+import pytest
+
+
+def dict_factory(cursor, row):
+    d = {}
+    for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+    return d
+
+
+@pytest.fixture()
+def queries():
+    dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "blogdb", "sql")
+    return anosql.from_path(dir_path, "sqlite3")
+
+
+def test_record_query(sqlite3_conn, queries):
+    sqlite3_conn.row_factory = dict_factory
+    actual = queries.users.get_all(sqlite3_conn)
+
+    assert len(actual) == 3
+    assert actual[0] == {
+        "userid": 1,
+        "username": "bobsmith",
+        "firstname": "Bob",
+        "lastname": "Smith",
+    }
+
+
+def test_parameterized_query(sqlite3_conn, queries):
+    actual = queries.blogs.get_user_blogs(sqlite3_conn, userid=1)
+    expected = [
+        ("How to make a pie.", "2018-11-23"),
+        ("What I did Today", "2017-07-28"),
+    ]
+    assert actual == expected
+
+
+def test_parameterized_record_query(sqlite3_conn, queries):
+    sqlite3_conn.row_factory = dict_factory
+    actual = queries.blogs.sqlite_get_blogs_published_after(
+        sqlite3_conn, published="2018-01-01"
+    )
+
+    expected = [
+        {
+            "title": "How to make a pie.",
+            "username": "bobsmith",
+            "published": "2018-11-23 00:00",
+        },
+        {"title": "Testing", "username": "janedoe", "published": "2018-01-01 00:00"},
+    ]
+
+    assert actual == expected
+
+
+def test_select_cursor_context_manager(sqlite3_conn, queries):
+    with queries.blogs.get_user_blogs_cursor(sqlite3_conn, userid=1) as cursor:
+        actual = cursor.fetchall()
+        expected = [
+            ("How to make a pie.", "2018-11-23"),
+            ("What I did Today", "2017-07-28"),
+        ]
+        assert actual == expected
+
+
+def test_insert_returning(sqlite3_conn, queries):
+    with sqlite3_conn:
+        blogid = queries.blogs.publish_blog(
+            sqlite3_conn,
+            userid=2,
+            title="My first blog",
+            content="Hello, World!",
+            published="2018-12-04",
+        )
+    print(blogid, type(blogid))
+    cur = sqlite3_conn.cursor()
+    cur.execute("SELECT `title` FROM `blogs` WHERE `blogid` = ?;", (blogid,))
+    actual = cur.fetchone()
+    cur.close()
+    expected = ("My first blog",)
+
+    assert actual == expected
+
+
+def test_delete(sqlite3_conn, queries):
+    # Removing the "janedoe" blog titled "Testing"
+    actual = queries.blogs.remove_blog(sqlite3_conn, blogid=2)
+    assert actual is None
+
+    janes_blogs = queries.blogs.get_user_blogs(sqlite3_conn, userid=3)
+    assert len(janes_blogs) == 0
+
+
+def test_insert_many(sqlite3_conn, queries):
+    blogs = [
+        (2, "Blog Part 1", "content - 1", "2018-12-04"),
+        (2, "Blog Part 2", "content - 2", "2018-12-05"),
+        (2, "Blog Part 3", "content - 3", "2018-12-06"),
+    ]
+
+    with sqlite3_conn:
+        actual = queries.blogs.sqlite_bulk_publish(sqlite3_conn, blogs)
+        assert actual is None
+
+        johns_blogs = queries.blogs.get_user_blogs(sqlite3_conn, userid=2)
+        assert johns_blogs == [
+            ("Blog Part 3", "2018-12-06"),
+            ("Blog Part 2", "2018-12-05"),
+            ("Blog Part 1", "2018-12-04"),
+        ]

projects/archiver/BUILD.bazel

@@ -0,0 +1,20 @@
+zapp_binary(
+    name = "hash_copy",
+    main = "hash_copy.py",
+    srcs = [
+        "util.py"
+    ],
+)
+
+zapp_binary(
+    name = "org_photos",
+    main = "org_photos.py",
+    srcs = [
+        "util.py"
+    ],
+    deps = [
+        py_requirement("ExifRead"),
+        py_requirement("yaspin"),
+    ],
+    shebang = "/usr/bin/env python3"
+)

projects/archiver/hash_copy.py

@@ -0,0 +1,56 @@
+"""
+A tree deduplicator and archiver tool.
+"""
+
+import argparse
+from pathlib import Path
+from shutil import copy2 as copyfile
+
+from .util import *
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("from_dir", type=Path)
+parser.add_argument("to_dir", type=Path)
+
+
+def main():
+    opts, args = parser.parse_known_args()
+
+    for abs_src_path in opts.from_dir.glob("**/*"):
+        rel_src_path = abs_src_path.relative_to(opts.from_dir)
+        abs_dest_path = opts.to_dir / rel_src_path
+
+        if abs_src_path.is_dir():
+            print("dir", abs_src_path, "->", abs_dest_path)
+            abs_dest_path.mkdir(exist_ok=True)
+
+        elif abs_src_path.is_file():
+            print("file", abs_src_path, "->", abs_dest_path)
+
+            if not abs_dest_path.exists():
+                copyfile(abs_src_path, abs_dest_path)
+
+            else:
+                # If you trust mtime, this can go a lot faster
+                trust_mtime = False
+
+                if (
+                    trust_mtime
+                    and abs_dest_path.stat().st_mtime < abs_src_path.stat().st_mtime
+                ):
+                    pass
+
+                elif (src_checksum := checksum_path(abs_src_path)) != (
+                    dest_checksum := checksum_path(abs_dest_path)
+                ):
+                    print(
+                        f"file conflict (src {src_checksum}, dest {dest_checksum}), correcting..."
+                    )
+                    copyfile(abs_src_path, abs_dest_path)
+
+            abs_src_path.unlink()
+
+
+if __name__ == "__main__":
+    main()

projects/archiver/notes.md

@@ -0,0 +1,288 @@
+# EXIF tags dataset (exifread edition)
+
+- 'EXIF ApertureValue'
+- 'EXIF BodySerialNumber'
+- 'EXIF BrightnessValue'
+- 'EXIF CVAPattern'
+- 'EXIF CameraOwnerName'
+- 'EXIF ColorSpace'
+- 'EXIF ComponentsConfiguration'
+- 'EXIF CompressedBitsPerPixel'
+- 'EXIF Contrast'
+- 'EXIF CustomRendered'
+- 'EXIF DateTimeDigitized'
+- 'EXIF DateTimeOriginal'
+- 'EXIF DeviceSettingDescription'
+- 'EXIF DigitalZoomRatio'
+- 'EXIF ExifImageLength'
+- 'EXIF ExifImageWidth'
+- 'EXIF ExifVersion'
+- 'EXIF ExposureBiasValue'
+- 'EXIF ExposureIndex'
+- 'EXIF ExposureMode'
+- 'EXIF ExposureProgram'
+- 'EXIF ExposureTime'
+- 'EXIF FNumber'
+- 'EXIF FileSource'
+- 'EXIF Flash'
+- 'EXIF FlashEnergy'
+- 'EXIF FlashPixVersion'
+- 'EXIF FocalLength'
+- 'EXIF FocalLengthIn35mmFilm'
+- 'EXIF FocalPlaneResolutionUnit'
+- 'EXIF FocalPlaneXResolution'
+- 'EXIF FocalPlaneYResolution'
+- 'EXIF GainControl'
+- 'EXIF ISOSpeedRatings'
+- 'EXIF ImageUniqueID'
+- 'EXIF InteroperabilityOffset'
+- 'EXIF JPEGInterchangeFormat'
+- 'EXIF JPEGInterchangeFormatLength'
+- 'EXIF LensMake'
+- 'EXIF LensModel'
+- 'EXIF LensSerialNumber'
+- 'EXIF LensSpecification'
+- 'EXIF LightSource'
+- 'EXIF MakerNote'
+- 'EXIF MaxApertureValue'
+- 'EXIF MeteringMode'
+- 'EXIF OffsetSchema'
+- 'EXIF OffsetTime'
+- 'EXIF OffsetTimeDigitized'
+- 'EXIF OffsetTimeOriginal'
+- 'EXIF Padding'
+- 'EXIF RecommendedExposureIndex'
+- 'EXIF Saturation'
+- 'EXIF SceneCaptureType'
+- 'EXIF SceneType'
+- 'EXIF SensingMethod'
+- 'EXIF SensitivityType'
+- 'EXIF Sharpness'
+- 'EXIF ShutterSpeedValue'
+- 'EXIF SubSecTime'
+- 'EXIF SubSecTimeDigitized'
+- 'EXIF SubSecTimeOriginal'
+- 'EXIF SubjectArea'
+- 'EXIF SubjectDistance'
+- 'EXIF SubjectDistanceRange'
+- 'EXIF UserComment'
+- 'EXIF WhiteBalance'
+- 'GPS GPSAltitude'
+- 'GPS GPSAltitudeRef'
+- 'GPS GPSDOP'
+- 'GPS GPSDate'
+- 'GPS GPSImgDirection'
+- 'GPS GPSImgDirectionRef'
+- 'GPS GPSLatitude'
+- 'GPS GPSLatitudeRef'
+- 'GPS GPSLongitude'
+- 'GPS GPSLongitudeRef'
+- 'GPS GPSMapDatum'
+- 'GPS GPSMeasureMode'
+- 'GPS GPSProcessingMethod'
+- 'GPS GPSTimeStamp'
+- 'GPS GPSVersionID'
+- 'GPS Tag 0xEA1C'
+- 'Image Artist'
+- 'Image BitsPerSample'
+- 'Image Copyright'
+- 'Image DateTime'
+- 'Image DateTimeDigitized'
+- 'Image ExifOffset'
+- 'Image ExposureMode'
+- 'Image ExposureProgram'
+- 'Image ExposureTime'
+- 'Image FNumber'
+- 'Image Flash'
+- 'Image FocalLength'
+- 'Image GPSInfo'
+- 'Image ISOSpeedRatings'
+- 'Image ImageDescription'
+- 'Image ImageLength'
+- 'Image ImageWidth'
+- 'Image JPEGInterchangeFormat'
+- 'Image JPEGInterchangeFormatLength'
+- 'Image LightSource'
+- 'Image Make'
+- 'Image MeteringMode'
+- 'Image Model'
+- 'Image Orientation'
+- 'Image Padding'
+- 'Image PhotometricInterpretation'
+- 'Image PrintIM'
+- 'Image ResolutionUnit'
+- 'Image SamplesPerPixel'
+- 'Image Software'
+- 'Image UserComment'
+- 'Image WhiteBalance'
+- 'Image XPComment'
+- 'Image XPKeywords'
+- 'Image XPTitle'
+- 'Image XResolution'
+- 'Image YCbCrPositioning'
+- 'Image YResolution'
+- 'Interoperability InteroperabilityIndex'
+- 'Interoperability InteroperabilityVersion'
+- 'JPEGThumbnail'
+- 'MakerNote AEBracketCompensationApplied'
+- 'MakerNote AESetting'
+- 'MakerNote AFAreaMode'
+- 'MakerNote AFInfo2'
+- 'MakerNote AFPointSelected'
+- 'MakerNote AFPointUsed'
+- 'MakerNote ActiveDLighting'
+- 'MakerNote AspectInfo'
+- 'MakerNote AutoBracketRelease'
+- 'MakerNote AutoFlashMode'
+- 'MakerNote BracketMode'
+- 'MakerNote BracketShotNumber'
+- 'MakerNote BracketValue'
+- 'MakerNote BracketingMode'
+- 'MakerNote CanonImageWidth'
+- 'MakerNote ColorBalance'
+- 'MakerNote ColorSpace'
+- 'MakerNote ContinuousDriveMode'
+- 'MakerNote Contrast'
+- 'MakerNote CropHiSpeed'
+- 'MakerNote CropInfo'
+- 'MakerNote DigitalVariProgram'
+- 'MakerNote DigitalZoom'
+- 'MakerNote DustRemovalData'
+- 'MakerNote EasyShootingMode'
+- 'MakerNote ExposureDifference'
+- 'MakerNote ExposureMode'
+- 'MakerNote ExposureTuning'
+- 'MakerNote ExternalFlashExposureComp'
+- 'MakerNote FileInfo'
+- 'MakerNote FileNumber'
+- 'MakerNote FilterEffect'
+- 'MakerNote FirmwareVersion'
+- 'MakerNote FlashActivity'
+- 'MakerNote FlashBias'
+- 'MakerNote FlashBracketCompensationApplied'
+- 'MakerNote FlashCompensation'
+- 'MakerNote FlashDetails'
+- 'MakerNote FlashExposureLock'
+- 'MakerNote FlashInfo'
+- 'MakerNote FlashMode'
+- 'MakerNote FlashSetting'
+- 'MakerNote FocalLength'
+- 'MakerNote FocalType'
+- 'MakerNote FocalUnitsPerMM'
+- 'MakerNote FocusMode'
+- 'MakerNote FocusType'
+- 'MakerNote HDRImageType'
+- 'MakerNote HighISONoiseReduction'
+- 'MakerNote ISO'
+- 'MakerNote ISOInfo'
+- 'MakerNote ISOSetting'
+- 'MakerNote ISOSpeedRequested'
+- 'MakerNote ImageDataSize'
+- 'MakerNote ImageSize'
+- 'MakerNote ImageStabilization'
+- 'MakerNote ImageType'
+- 'MakerNote InternalSerialNumber'
+- 'MakerNote LensData'
+- 'MakerNote LensFStops'
+- 'MakerNote LensMinMaxFocalMaxAperture'
+- 'MakerNote LensModel'
+- 'MakerNote LensType'
+- 'MakerNote LiveViewShooting'
+- 'MakerNote LongExposureNoiseReduction2'
+- 'MakerNote LongFocalLengthOfLensInFocalUnits'
+- 'MakerNote MacroMagnification'
+- 'MakerNote Macromode'
+- 'MakerNote MakernoteVersion'
+- 'MakerNote ManualFlashOutput'
+- 'MakerNote MeteringMode'
+- 'MakerNote ModelID'
+- 'MakerNote MultiExposure'
+- 'MakerNote NikonPreview'
+- 'MakerNote NoiseReduction'
+- 'MakerNote NumAFPoints'
+- 'MakerNote OwnerName'
+- 'MakerNote PhotoCornerCoordinates'
+- 'MakerNote PictureControl'
+- 'MakerNote PowerUpTime'
+- 'MakerNote ProgramShift'
+- 'MakerNote Quality'
+- 'MakerNote RawJpgQuality'
+- 'MakerNote RawJpgSize'
+- 'MakerNote RecordMode'
+- 'MakerNote RetouchHistory'
+- 'MakerNote Saturation'
+- 'MakerNote SelfTimer'
+- 'MakerNote SequenceNumber'
+- 'MakerNote SerialNumber'
+- 'MakerNote Sharpness'
+- 'MakerNote ShortFocalLengthOfLensInFocalUnits'
+- 'MakerNote ShotInfo'
+- 'MakerNote SlowShutter'
+- 'MakerNote SpotMeteringMode'
+- 'MakerNote SubjectDistance'
+- 'MakerNote Tag 0x0001'
+- 'MakerNote Tag 0x0002'
+- 'MakerNote Tag 0x0003'
+- 'MakerNote Tag 0x0004'
+- 'MakerNote Tag 0x0005'
+- 'MakerNote Tag 0x0006'
+- 'MakerNote Tag 0x0007'
+- 'MakerNote Tag 0x0008'
+- 'MakerNote Tag 0x0009'
+- 'MakerNote Tag 0x000E'
+- 'MakerNote Tag 0x0014'
+- 'MakerNote Tag 0x0015'
+- 'MakerNote Tag 0x0019'
+- 'MakerNote Tag 0x002B'
+- 'MakerNote Tag 0x002C'
+- 'MakerNote Tag 0x002D'
+- 'MakerNote Tag 0x0083'
+- 'MakerNote Tag 0x0099'
+- 'MakerNote Tag 0x009D'
+- 'MakerNote Tag 0x00A0'
+- 'MakerNote Tag 0x00A3'
+- 'MakerNote Tag 0x00AA'
+- 'MakerNote Tag 0x00BB'
+- 'MakerNote Tag 0x00D0'
+- 'MakerNote Tag 0x00E0'
+- 'MakerNote Tag 0x4001'
+- 'MakerNote Tag 0x4008'
+- 'MakerNote Tag 0x4009'
+- 'MakerNote Tag 0x4010'
+- 'MakerNote Tag 0x4011'
+- 'MakerNote Tag 0x4012'
+- 'MakerNote Tag 0x4015'
+- 'MakerNote Tag 0x4016'
+- 'MakerNote Tag 0x4017'
+- 'MakerNote Tag 0x4018'
+- 'MakerNote Tag 0x4019'
+- 'MakerNote Tag 0x4020'
+- 'MakerNote ThumbnailImageValidArea'
+- 'MakerNote ToningEffect'
+- 'MakerNote TotalShutterReleases'
+- 'MakerNote Unknown'
+- 'MakerNote VRInfo'
+- 'MakerNote ValidAFPoints'
+- 'MakerNote WBBracketMode'
+- 'MakerNote WBBracketValueAB'
+- 'MakerNote WBBracketValueGM'
+- 'MakerNote WhiteBalance'
+- 'MakerNote WhiteBalanceBias'
+- 'MakerNote WhiteBalanceRBCoeff'
+- 'MakerNote Whitebalance'
+- 'MakerNote WorldTime'
+- 'Thumbnail Compression'
+- 'Thumbnail DateTime'
+- 'Thumbnail ImageDescription'
+- 'Thumbnail ImageLength'
+- 'Thumbnail ImageWidth'
+- 'Thumbnail JPEGInterchangeFormat'
+- 'Thumbnail JPEGInterchangeFormatLength'
+- 'Thumbnail Make'
+- 'Thumbnail Model'
+- 'Thumbnail Orientation'
+- 'Thumbnail ResolutionUnit'
+- 'Thumbnail Software'
+- 'Thumbnail XResolution'
+- 'Thumbnail YCbCrPositioning'
+- 'Thumbnail YResolution'

projects/archiver/org_photos.py

@@ -0,0 +1,454 @@
+"""Organize (and annotate) photos predictably.
+
+This script is designed to eat a Dropbox style "camera uploads" directory containing many years of photos with various
+metadata (or data only in the file name) and stomp that all down to something consistent by doing EXIF parsing and
+falling back to filename parsing if need be.
+
+Note that this script does NOT do EXIF rewriting, although it probably should.
+
+Generates an output tree of the format -
+  {year}/v1_{iso_date}_{camera_make}_{camera_model}_{device_fingerprint}.{ext}
+
+The idea being that chronological-ish order is the default way users will want to view (or process) photos.
+Note that this means if you do interleaved shooting between devices, it should Just Work (TM).
+
+Inspired by https://github.com/herval/org_photos/blob/main/org_photos.rb
+
+"""
+
+import argparse
+from datetime import datetime, timedelta
+from hashlib import sha256, sha512
+import os
+from pathlib import Path
+import re
+from shutil import copy2 as copyfile
+import stat
+import sys
+import typing as t
+
+from .util import *
+
+# FIXME: use piexif, which supports writeback not exifread.
+import exifread
+from yaspin import Spinner, yaspin
+
+
+_print = print
+
+
+def print(*strs, **kwargs):
+    _print("\r", *strs, **kwargs)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("src_dir", type=Path)
+parser.add_argument("dest_dir", type=Path)
+parser.add_argument("destructive", action="store_true", default=False)
+
+
+MODIFIED_ISO_DATE = "%Y:%m:%dT%H:%M:%SF%f"
+SPINNER = Spinner(["|", "/", "-", "\\"], 200)
+KNOWN_IMG_TYPES = {
+    ".jpg": ".jpeg",
+    ".jpeg": ".jpeg",
+    ".png": ".png",
+    ".mov": ".mov",
+    ".gif": ".gif",
+    ".mp4": ".mp4",
+    ".m4a": ".m4a",
+    ".oga": ".oga",  # How the hell do I have ogg files kicking around
+}
+
+
+def exif_tags(p: Path) -> object:
+    """Return the EXIF tags on an image."""
+    with open(p, "rb") as fp:
+        return exifread.process_file(fp)
+
+
+def sanitize(s: str) -> str:
+    """Something like b64encode; sanitize a string to a path-friendly version."""
+
+    to_del = [" ", ";", ":", "_", "-", "/", "\\", "."]
+    s = s.lower()
+    for c in to_del:
+        s = s.replace(c, "")
+    return s
+
+
+def safe_strptime(date, format):
+    try:
+        return datetime.strptime(date, format)
+    except ValueError:
+        pass
+
+
+def safe_ymdhmms(date):
+    fmt = (
+        r"(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})"
+        r" "
+        r"(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2})(?P<millisecond>\d{3})"
+    )
+    m = re.match(fmt, date)
+    if m:
+        return datetime(
+            year=int(m.group("year")),
+            month=int(m.group("month")),
+            day=int(m.group("day")),
+            hour=int(m.group("hour")),
+            minute=int(m.group("minute")),
+            second=int(m.group("second")),
+            microsecond=int(m.group("millisecond")) * 1000,
+        )
+
+
+def date_from_name(fname: str):
+    # Discard common prefixes
+    fname = fname.replace("IMG_", "")
+    fname = fname.replace("PXL_", "")
+    fname = fname.replace("VID_", "")
+
+    if fname.startswith("v1_"):
+        # The date segment of the v1 file name
+        fname = fname.split("_")[1]
+
+    # Guess if we have a v0 file name, which neglected the prefix
+    elif fname.count("_") == 3:
+        # A bug
+        fname = fname.split("_")[-1].replace("rc", "")[:25]
+
+    # We have some other garbage, make our date parsing life easier.
+    else:
+        # A couple of date formats use _ as field separators, consistently choice " " instead so that we can write fewer
+        # date patterns and be more correct.
+        fname = fname.replace("_", " ")
+        fname = re.sub(
+            r"(-\d+)(-\d+)*$", r"\1", fname
+        )  # deal with -1-2 etc. crap from Dropbox
+
+    # Try to guess the date
+    # File date formats:
+    for unfmt in [
+        # Our date format
+        lambda d: safe_strptime(d, MODIFIED_ISO_DATE),
+        # A bug
+        # 2014:08:21T19:4640F1408672000
+        # 2015:12:14T23:0933F1450159773
+        lambda d: safe_strptime(d, "%Y:%m:%dT%H:%M%SF%f"),
+        # 2020-12-21 17.15.09.0
+        lambda d: safe_strptime(d, "%Y-%m-%d %H.%M.%S.%f"),
+        # 2020-12-21 17.15.09
+        lambda d: safe_strptime(d, "%Y-%m-%d %H.%M.%S"),
+        # 2019-02-09 12.45.32-6
+        # 2019-01-13 13.43.45-16
+        lambda d: safe_strptime(d, "%Y-%m-%d %H.%M.%S-%f"),
+        # Note the _1 or such may not be millis, but we assume it is.
+        # 20171113_130826_1
+        # 20171113 130826 1
+        lambda d: safe_strptime(d, "%Y%m%d %H%M%S %f"),
+        # 20180404_114639
+        # 20180404 114639
+        lambda d: safe_strptime(d, "%Y%m%d %H%M%S"),
+        # 2017-11-05_15:15:55
+        # 2017-11-05 15:15:55
+        lambda d: safe_strptime(d, "%Y-%m-%d %H:%M:%S"),
+        lambda d: safe_strptime(d, "%Y%m%d %h%m%s%f"),
+        # From the dashcam(s)
+        # 20220427_211616_00023F
+        # 20220510_213347_04187R
+        lambda d: safe_strptime(d, "%Y%m%d %h%m%s %fF"),
+        lambda d: safe_strptime(d, "%Y%m%d %h%m%s %fR"),
+        # HACK:
+        #   Python doesn't support %s as milliseconds; these don't quite work.
+        #   So use a custom matcher.
+        # 20210526 002327780
+        # 20210417_220753284
+        # 20210417 220753284
+        # 20210304 204755545
+        # 20211111 224304117
+        safe_ymdhmms,
+    ]:
+        val = unfmt(fname)
+        if val is not None:
+            return val
+
+
+def date_from_path(p: Path):
+    """Try to munge a datestamp out of a path."""
+
+    fname = ".".join(p.name.split(".")[:-1])
+
+    date = date_from_name(fname)
+    if not date:
+        print(f"Warning: Unable to infer datetime from {fname!r}", file=sys.stderr)
+    return date
+
+
+def normalize_ext(p: Path):
+    renaming = KNOWN_IMG_TYPES
+    exts = [e.lower() for e in p.suffixes]
+    # Guess an ext out of potentially many, allowing only for folding of effective dupes
+    exts = set(renaming[e] for e in exts if e in renaming)
+    assert len(exts) == 1
+    return list(exts)[0]
+
+
+class ImgInfo(t.NamedTuple):
+    file_path: Path
+    tags: dict
+
+    camera_make: str
+    camera_model: str
+    camera_sn: str
+
+    lens_make: str
+    lens_model: str
+    lens_sn: str
+
+    software: str
+
+    date: datetime
+
+    # A dirty bit, indicating whether the info needs to be written back
+    dirty: bool = False
+
+    shasum_prefix: int = 9
+
+    def device_fingerprint(self):
+        """Compute a stable 'fingerprint' for the device that took the shot."""
+
+        return checksum_list(
+            [
+                self.camera_make,
+                self.camera_model,
+                self.camera_sn,
+                self.lens_make,
+                self.lens_model,
+                self.lens_sn,
+                self.software,
+            ]
+        )[: self.shasum_prefix]
+
+    def file_fingerprint(self):
+        """Compute a 'fingerprint' for the file itself.
+
+        Note that this hash DOES include EXIF data, and is not stable.
+
+        """
+        return self.file_sha256sum()[: self.shasum_prefix]
+
+    def file_sha256sum(self):
+        return checksum_path(self.file_path, sha256)
+
+    def file_sha512sum(self):
+        return checksum_path(self.file_path, sha512)
+
+    def incr(self, offset: int) -> "ImgInfo":
+        return ImgInfo(
+            self.file_path,
+            self.tags,
+            self.camera_make,
+            self.camera_model,
+            self.camera_sn,
+            self.lens_make,
+            self.lens_model,
+            self.lens_sn,
+            self.software,
+            self.date + timedelta(microseconds=offset),
+            True,
+        )
+
+def img_info(p: Path) -> ImgInfo:
+    """Figure out everything we know from the image info."""
+
+    tags = exif_tags(p)
+
+    def get_tag(tag, default=None):
+        if v := tags.get(tag):
+            if isinstance(v.values, list):
+                return v.values[0]
+            elif isinstance(v.values, str):
+                return v.values
+            else:
+                raise ValueError(f"Don't know how to simplify {v!r}")
+        else:
+            return default
+
+    ## Camera data
+    camera_make = get_tag("Image Make", "Unknown")
+    camera_model = get_tag("Image Model", "Unknown")
+    camera_sn = get_tag("MakerNote SerialNumber", "Unknown")
+    lens_make = get_tag("EXIF LensMake", "Unknown")
+    lens_model = get_tag("EXIF LensModel", "Unknown")
+    lens_sn = get_tag("EXIF LensSerialNumber", "Unknown")
+    software = get_tag("Image Software", "Unknown")
+    dirty = False
+
+    # Fixup magic for "correcting" metadata where possible
+    # FIXME: This could be a postprocessing pass
+    if camera_make == "Unknown" and p.name.lower().startswith("dji"):
+        # Magic constants from found examples
+        camera_make = "DJI"
+        camera_model = "FC220"
+        software = "v02.09.514"
+        dirty |= True
+
+    elif camera_make == "Unknown" and p.name.startswith("PXL_"):
+        camera_make = "Google"
+        camera_model = "Pixel"
+        dirty |= True
+
+    ## Timestamp data
+    stat = p.stat()
+
+    # 2019:03:31 15:59:26
+    date = (
+        get_tag("Image DateTime")
+        or get_tag("EXIF DateTimeOriginal")
+        or get_tag("EXIF DateTimeDigitized")
+    )
+    if date and (date := safe_strptime(date, "%Y:%m:%d %H:%M:%S")):
+        pass
+    elif date := date_from_path(p):
+        dirty |= True
+    else:
+        # The oldest of the mtime and the ctime
+        date = datetime.fromtimestamp(min([stat.st_mtime, stat.st_ctime]))
+        dirty |= True
+
+    # 944404
+    subsec = int(
+        get_tag("EXIF SubSecTime")
+        or get_tag("EXIF SubSecTimeOriginal")
+        or get_tag("EXIF SubSecTimeDigitized")
+        or "0"
+    )
+
+    # GoPro burst format is G%f.JPG or something close to it
+    if subsec == 0 and (m := re.match(r"g.*(\d{6}).jpe?g", p.name.lower())):
+        subsec = int(m.group(1))
+
+    date = date.replace(microsecond=subsec)
+
+    if not (2015 <= date.year <= datetime.now().year):
+        raise ValueError(f"{p}'s inferred date ({date!r}) is beyond the sanity-check range!")
+
+    return ImgInfo(
+        p,
+        tags,
+        camera_make,
+        camera_model,
+        camera_sn,
+        lens_make,
+        lens_model,
+        lens_sn,
+        software,
+        date,
+        dirty,
+    )
+
+
+def is_readonly(src: Path) -> bool:
+    statinfo = os.stat(src, dir_fd=None, follow_symlinks=True)
+    if getattr(statinfo, "st_flags", 0) & stat.UF_IMMUTABLE:
+        return True
+
+    return False
+
+
+def main():
+    opts, args = parser.parse_known_args()
+
+    def _delete(src):
+        if opts.destructive:
+            if is_readonly(src):
+                print(f"  warning: {src} is read-only, unable to remove")
+            else:
+                try:
+                    src.unlink()
+                    print("  unlink: ok")
+                except PermissionError:
+                    print("  unlink: fail")
+
+    def _copy(src, target):
+        print(f"  rename: {target}")
+        try:
+            if not opts.destructive:
+                raise OSError()
+
+            src.rename(target)  # Execute the rename
+
+        except OSError:  # cross-device move
+            with yaspin(SPINNER):
+                copyfile(src, target)
+
+                _delete(src)
+
+    print("---")
+
+    sequence_name = None
+    sequence = 0
+
+    for src in opts.src_dir.glob("**/*"):
+        print(f"{src}:")
+        ext = "." + src.name.lower().split(".")[-1]
+        print(f"  msg: ext inferred as {ext}")
+
+        if src.is_dir():
+            continue
+
+        elif src.name.lower().startswith("."):
+            continue
+
+        elif ext in ["thm", "lrv", "ico", "sav"] or src.name.startswith("._"):
+            _delete(src)
+            continue
+
+        elif ext in KNOWN_IMG_TYPES:
+            info = img_info(src)
+            year_dir = Path(opts.dest_dir / str(info.date.year))
+            year_dir.mkdir(exist_ok=True)  # Ignore existing and continue
+            # Figure out a stable file name
+            stable_name = f"v1_{info.date.strftime(MODIFIED_ISO_DATE)}_{sanitize(info.camera_make)}_{sanitize(info.camera_model)}_{info.device_fingerprint()}"
+
+            # De-conflict using a sequence number added to the sub-seconds field
+            if sequence_name == stable_name:
+                sequence += 1
+                info = info.incr(sequence)
+                print(f"  warning: de-conflicting filenames with sequence {sequence}")
+                stable_name = f"v1_{info.date.strftime(MODIFIED_ISO_DATE)}_{sanitize(info.camera_make)}_{sanitize(info.camera_model)}_{info.device_fingerprint()}"
+
+            else:
+                sequence = 0
+                sequence_name = stable_name
+
+            try:
+                ext = normalize_ext(src)
+            except AssertionError:
+                continue  # Just skip fucked up files
+            target = Path(year_dir / f"{stable_name}{ext}")
+
+            if not target.exists():
+                # src & !target => copy
+                _copy(src, target)
+
+            elif src == target:
+                # src == target; skip DO NOT DELETE SRC
+                pass
+
+            elif checksum_path_blocks(src) == checksum_path_blocks(target):
+                print(f"  ok: {target}")
+                # src != target && id(src) == id(target); delete src
+                _delete(src)
+
+            else:
+                # src != target && id(src) != id(target); replace target with src?
+                print(f"  warning: {target} is a content-id collision with a different checksum; skipping")
+
+        else:
+            print(f"  msg: unknown filetype {ext}")
+
+
+if __name__ == "__main__":
+    main()

projects/archiver/util.py

@@ -0,0 +1,68 @@
+from hashlib import sha256
+from pathlib import Path
+import typing as t
+
+
+def iter_chunks(fp):
+    yield from iter(lambda: fp.read(4096), b"")
+
+
+def take(n, iter):
+    """Take the first N items lazily off of an iterable."""
+
+    for _ in range(0, n):
+        try:
+            yield next(iter)
+        except StopIteration:
+            break
+
+
+def checksum_list(iter, sum=sha256, salt=b";"):
+    """Compute the checksum of a bunch of stuff from an iterable."""
+
+    sum = sum()
+    for i in iter:
+        if salt:
+            sum.update(salt)  # Merkle tree salting.
+        if isinstance(i, str):
+            i = str.encode(i, "utf-8")
+        try:
+            sum.update(i)
+        except Exception as e:
+            print(i, type(i))
+            raise e
+
+    return sum.hexdigest()
+
+
+def checksum_path_blocks(p: Path, sum=sha256) -> t.Iterable[str]:
+    """Compute block-wise checksums of a file.
+
+    Inspired by the Dropbox content-hashing interface -
+
+    https://www.dropbox.com/developers/reference/content-hash
+
+    """
+
+    def _helper():
+        with open(p, "rb") as fp:
+            for chunk in iter_chunks(fp):
+                digest = sum()
+                digest.update(chunk)
+                yield digest.hexdigest()
+
+    return list(_helper())
+
+
+def checksum_path(p: Path, sum=sha256) -> str:
+    """Compute 'the' checksum of an entire file.
+
+    Note that this does semi-streaming I/O.
+
+    """
+
+    sum = sum()
+    with open(p, "rb") as fp:
+        for chunk in iter_chunks(fp):
+            sum.update(chunk)
+    return sum.hexdigest()

projects/archlinux/rollback.py

@@ -0,0 +1,94 @@
+import os
+import re
+import sys
+
+
+def main(opts, args):
+    """Usage: python rollback.py date
+
+    Parse /var/log/pacman.log, enumerating package transactions since the
+    specified date and building a plan for restoring the state of your system to
+    what it was at the specified date.
+
+    Assumes:
+    - /var/log/pacman.log has not been truncated
+
+    - /var/cache/pacman/pkg has not been flushed and still contains all required
+      packages
+
+    - The above paths are Arch default and have not been customized
+
+    - That it is not necessary to remove any "installed" packages
+
+    Note: no attempt is made to inspect the dependency graph of packages to be
+    downgraded to detect when a package is already transitively listed for
+    downgrading. This can create some annoying errors where eg. systemd will be
+    downgraded, meaning libsystemd will also be downgraded, but pacman considers
+    explicitly listing the downgrade of libsystemd when it will already be
+    transitively downgraded an error.
+
+    """
+
+    date, = args
+
+    print("Attempting to roll back package state to that of {0}...\n"
+          .format(date),
+          file=sys.stderr)
+
+    # These patterns can't be collapsed because we want to select different
+    # version identifying strings depending on which case we're in. Not ideal,
+    # but it works.
+
+    # Ex. [2017-04-01 09:51] [ALPM] upgraded filesystem (2016.12-2 -> 2017.03-2)
+    upgraded_pattern = re.compile(
+        ".*? upgraded (?P<name>\w+) \((?P<from>[^ ]+) -> (?P<to>[^\)]+)\)")
+
+    # Ex: [2018-02-23 21:18] [ALPM] downgraded emacs (25.3-3 -> 25.3-2)
+    downgraded_pattern = re.compile(
+        ".*? downgraded (?P<name>\w+) \((?P<to>[^ ]+) -> (?P<from>[^\)]+)\)")
+
+    # Ex: [2017-03-31 07:05] [ALPM] removed gdm (3.22.3-1)
+    removed_pattern = re.compile(
+        ".*? removed (?P<name>\w+) \((?P<from>[^ ]+)\)")
+
+    checkpoint = {}
+    flag = False
+
+    with open("/var/log/pacman.log") as logfile:
+        for line in logfile:
+            if date in line:
+                flag = True
+            elif not flag:
+                continue
+
+            match = re.match(upgraded_pattern, line)\
+                or re.match(downgraded_pattern, line)\
+                or re.match(removed_pattern, line)
+
+            if match:
+                package = match.group("name")
+                from_rev = match.group("from")
+                if package not in checkpoint:
+                    checkpoint[package] = from_rev
+                continue
+
+    print("Checkpoint state:")
+    for k in checkpoint.keys():
+        print("{0} -> {1}".format(k, checkpoint[k]), file=sys.stderr)
+
+    pkgcache = "/var/cache/pacman/pkg"
+    pkgs = os.listdir(pkgcache)
+    pkgnames = ["{0}-{1}".format(k, v) for k, v in checkpoint.items()]
+
+    selected_pkgs = set([os.path.join(pkgcache, p)
+                         for n in pkgnames
+                         for p in pkgs
+                         if n in p])
+
+    print("Suggested incantation:\n", file=sys.stderr)
+    print("sudo pacman --noconfirm -U {}"
+          .format("\\\n  ".join(selected_pkgs)))
+
+
+if __name__ == "__main__":
+  main(None, sys.argv[1:])

projects/async_cache/BUILD.bazel

@@ -0,0 +1,3 @@
+py_project(
+    name = "async_cache"
+)

projects/async_cache/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Rajat Singh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

projects/async_cache/README.md

@@ -0,0 +1,13 @@
+# Async cache
+
+An LRU and TTL cache for async functions in Python.
+
+- `alru_cache` provides an LRU cache decorator with configurable size.
+- `attl_cache` provides a TTL+LRU cache decorator with configurable size.
+
+Neither cache proactively expires keys.
+Maintenance occurs only when requesting keys out of the cache.
+
+## License
+
+Derived from https://github.com/iamsinghrajat/async-cache, published under the MIT license.

projects/async_cache/src/async_cache/__init__.py

@@ -0,0 +1,4 @@
+"""The interface to the package. Just re-exports implemented caches."""
+
+from .lru import ALRU, alru_cache  # noqa
+from .ttl import ATTL, attl_cache  # noqa

projects/async_cache/src/async_cache/key.py

@@ -0,0 +1,23 @@
+from typing import Any
+
+
+class KEY:
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+
+    def __eq__(self, obj):
+        return hash(self) == hash(obj)
+
+    def __hash__(self):
+        def _hash(param: Any):
+            if isinstance(param, tuple):
+                return tuple(map(_hash, param))
+            if isinstance(param, dict):
+                return tuple(map(_hash, param.items()))
+            elif hasattr(param, "__dict__"):
+                return str(vars(param))
+            else:
+                return str(param)
+
+        return hash(_hash(self.args) + _hash(self.kwargs))

projects/async_cache/src/async_cache/lru.py

@@ -0,0 +1,44 @@
+from collections import OrderedDict
+
+from .key import KEY
+
+
+class LRU(OrderedDict):
+    def __init__(self, maxsize, *args, **kwargs):
+        self.maxsize = maxsize
+        super().__init__(*args, **kwargs)
+
+    def __getitem__(self, key):
+        value = super().__getitem__(key)
+        self.move_to_end(key)
+        return value
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        if self.maxsize and len(self) > self.maxsize:
+            oldest = next(iter(self))
+            del self[oldest]
+
+
+class ALRU(object):
+    def __init__(self, maxsize=128):
+        """
+        :param maxsize: Use maxsize as None for unlimited size cache
+        """
+        self.lru = LRU(maxsize=maxsize)
+
+    def __call__(self, func):
+        async def wrapper(*args, **kwargs):
+            key = KEY(args, kwargs)
+            if key in self.lru:
+                return self.lru[key]
+            else:
+                self.lru[key] = await func(*args, **kwargs)
+                return self.lru[key]
+
+        wrapper.__name__ += func.__name__
+
+        return wrapper
+
+
+alru_cache = ALRU

projects/async_cache/src/async_cache/ttl.py

@@ -0,0 +1,77 @@
+import datetime
+from typing import Optional, Union
+
+from .key import KEY
+from .lru import LRU
+
+
+class ATTL:
+    class _TTL(LRU):
+        def __init__(self, ttl: Optional[Union[datetime.timedelta, int, float]], maxsize: int):
+            super().__init__(maxsize=maxsize)
+
+            if isinstance(ttl, datetime.timedelta):
+                self.ttl = ttl
+            elif isinstance(ttl, (int, float)):
+                self.ttl = datetime.timedelta(seconds=ttl)
+            elif ttl is None:
+                self.ttl = None
+            else:
+                raise ValueError("TTL must be int or timedelta")
+
+            self.maxsize = maxsize
+
+        def __contains__(self, key):
+            if key not in self.keys():
+                return False
+            else:
+                key_expiration = super().__getitem__(key)[1]
+                if key_expiration and key_expiration < datetime.datetime.now():
+                    del self[key]
+                    return False
+                else:
+                    return True
+
+        def __getitem__(self, key):
+            if key in self:
+                value = super().__getitem__(key)[0]
+                return value
+            raise KeyError
+
+        def __setitem__(self, key, value):
+            ttl_value = (
+                    datetime.datetime.now() + self.ttl
+            ) if self.ttl else None
+            super().__setitem__(key, (value, ttl_value))
+
+    def __init__(
+            self,
+            ttl: Optional[Union[datetime.timedelta, int]] = datetime.timedelta(seconds=60),
+            maxsize: int = 1024,
+            skip_args: int = 0
+    ):
+        """
+        :param ttl: Use ttl as None for non expiring cache
+        :param maxsize: Use maxsize as None for unlimited size cache
+        :param skip_args: Use `1` to skip first arg of func in determining cache key
+        """
+        self.ttl = self._TTL(ttl=ttl, maxsize=maxsize)
+        self.skip_args = skip_args
+
+    def __call__(self, func):
+        async def wrapper(*args, **kwargs):
+            key = KEY(args[self.skip_args:], kwargs)
+            if key in self.ttl:
+                val = self.ttl[key]
+            else:
+                self.ttl[key] = await func(*args, **kwargs)
+                val = self.ttl[key]
+
+            return val
+
+        wrapper.__name__ += func.__name__
+
+        return wrapper
+
+
+attl_cache = ATTL

projects/async_cache/test/test_lru.py

@@ -0,0 +1,82 @@
+import asyncio
+import time
+
+from async_cache import ALRU, ATTL
+
+
+@ALRU(maxsize=128)
+async def func(wait: int):
+    await asyncio.sleep(wait)
+
+
+class TestClassFunc:
+    @ALRU(maxsize=128)
+    async def obj_func(self, wait: int):
+        await asyncio.sleep(wait)
+
+    @staticmethod
+    @ATTL(maxsize=128, ttl=None, skip_args=1)
+    async def skip_arg_func(arg: int, wait: int):
+        await asyncio.sleep(wait)
+
+    @classmethod
+    @ALRU(maxsize=128)
+    async def class_func(cls, wait: int):
+        await asyncio.sleep(wait)
+
+
+def test():
+    t1 = time.time()
+    asyncio.get_event_loop().run_until_complete(func(4))
+    t2 = time.time()
+    asyncio.get_event_loop().run_until_complete(func(4))
+    t3 = time.time()
+    t_first_exec = (t2 - t1) * 1000
+    t_second_exec = (t3 - t2) * 1000
+    print(t_first_exec)
+    print(t_second_exec)
+    assert t_first_exec > 4000
+    assert t_second_exec < 4000
+
+
+def test_obj_fn():
+    t1 = time.time()
+    obj = TestClassFunc()
+    asyncio.get_event_loop().run_until_complete(obj.obj_func(4))
+    t2 = time.time()
+    asyncio.get_event_loop().run_until_complete(obj.obj_func(4))
+    t3 = time.time()
+    t_first_exec = (t2 - t1) * 1000
+    t_second_exec = (t3 - t2) * 1000
+    print(t_first_exec)
+    print(t_second_exec)
+    assert t_first_exec > 4000
+    assert t_second_exec < 4000
+
+
+def test_class_fn():
+    t1 = time.time()
+    asyncio.get_event_loop().run_until_complete(TestClassFunc.class_func(4))
+    t2 = time.time()
+    asyncio.get_event_loop().run_until_complete(TestClassFunc.class_func(4))
+    t3 = time.time()
+    t_first_exec = (t2 - t1) * 1000
+    t_second_exec = (t3 - t2) * 1000
+    print(t_first_exec)
+    print(t_second_exec)
+    assert t_first_exec > 4000
+    assert t_second_exec < 4000
+
+
+def test_skip_args():
+    t1 = time.time()
+    asyncio.get_event_loop().run_until_complete(TestClassFunc.skip_arg_func(5, 4))
+    t2 = time.time()
+    asyncio.get_event_loop().run_until_complete(TestClassFunc.skip_arg_func(6, 4))
+    t3 = time.time()
+    t_first_exec = (t2 - t1) * 1000
+    t_second_exec = (t3 - t2) * 1000
+    print(t_first_exec)
+    print(t_second_exec)
+    assert t_first_exec > 4000
+    assert t_second_exec < 4000

projects/async_cache/test/test_ttl.py

@@ -0,0 +1,57 @@
+import asyncio
+import time
+
+from async_cache import ATTL
+
+
+@ATTL(ttl=60)
+async def long_expiration_fn(wait: int):
+    await asyncio.sleep(wait)
+    return wait
+
+
+@ATTL(ttl=5)
+async def short_expiration_fn(wait: int):
+    await asyncio.sleep(wait)
+    return wait
+
+
+@ATTL(ttl=3)
+async def short_cleanup_fn(wait: int):
+    await asyncio.sleep(wait)
+    return wait
+
+
+def test_cache_hit():
+    t1 = time.time()
+    asyncio.get_event_loop().run_until_complete(long_expiration_fn(4))
+    t2 = time.time()
+    asyncio.get_event_loop().run_until_complete(long_expiration_fn(4))
+    t3 = time.time()
+    t_first_exec = (t2 - t1) * 1000
+    t_second_exec = (t3 - t2) * 1000
+    print(t_first_exec)
+    print(t_second_exec)
+    assert t_first_exec > 4000
+    assert t_second_exec < 4000
+
+
+def test_cache_expiration():
+    t1 = time.time()
+    asyncio.get_event_loop().run_until_complete(short_expiration_fn(1))
+    t2 = time.time()
+    asyncio.get_event_loop().run_until_complete(short_expiration_fn(1))
+    t3 = time.time()
+    time.sleep(5)
+    t4 = time.time()
+    asyncio.get_event_loop().run_until_complete(short_expiration_fn(1))
+    t5 = time.time()
+    t_first_exec = (t2 - t1) * 1000
+    t_second_exec = (t3 - t2) * 1000
+    t_third_exec = (t5 - t4) * 1000
+    print(t_first_exec)
+    print(t_second_exec)
+    print(t_third_exec)
+    assert t_first_exec > 1000
+    assert t_second_exec < 1000
+    assert t_third_exec > 1000

projects/bazelshim/BUILD.bazel

@@ -0,0 +1,4 @@
+py_project(
+    name = "bazelshim",
+    main = "src/bazelshim/__main__.py",
+)

projects/bazelshim/src/bazelshim/__main__.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+
+# Since this can't run under Bazel, we have to set up the sys.path ourselves
+import os
+import sys
+from pathlib import Path
+
+if (p := str(Path(sys.argv[0]).absolute().parent.parent)) not in sys.path:
+    sys.path.pop(0)  # Remove '.' / ''
+    sys.path.insert(0, p)  # Insert the bazelshim root
+
+# Now that's out of the way...
+from dataclasses import dataclass
+from shlex import quote, split as shlex
+import sys
+import tomllib
+from pathlib import Path
+from typing import List, Optional
+from itertools import chain
+
+
+# FIXME: Allow user-defined extensions here
+VERBS = [
+    "aquery",
+    "build",
+    "clean",
+    "coverage",
+    "cquery",
+    "dump",
+    "fetch",
+    "help",
+    "info",
+    "mod",
+    "query",
+    "run",
+    "sync",
+    "test",
+    "watch",
+]
+
+
+def path():
+    for it in os.getenv("PATH").split(":"):
+        yield Path(it)
+
+
+def which(cmd):
+    for it in path():
+        f: Path = (it / cmd).absolute()
+        if f.exists() and f.stat().st_mode & 0x700:
+            yield f
+
+
+def normalize_opts(args: List[str]) -> List[str]:
+    acc = []
+
+    if args[0].endswith("bazel") or args[0].endswith("bazelis"):
+        acc.append(args.pop(0))
+
+    while len(args) >= 2:
+        if args[0] == "--":
+            # Break
+            acc.extend(args)
+            break
+
+        elif "=" in args[0]:
+            # If it's a k/v form pass it through
+            acc.append(args.pop(0))
+
+        elif args[0].startswith("--no"):
+            # Convert --no<foo> args to --<foo>=no
+            acc.append("--" + args.pop(0).lstrip("--no") + "=false")
+
+        elif args[0] == "--isatty=0":
+            acc.append("--isatty=false")
+            args.pop(0)
+
+        elif (
+            args[0].startswith("--")
+            and not args[1].startswith("--")
+            and args[1] not in VERBS
+        ):
+            # If the next thing isn't an opt, assume it's a '--a b' form
+            acc.append(args[0] + "=" + args[1])
+            args.pop(0)
+            args.pop(0)
+
+        elif args[0].startswith("--"):
+            # Assume it's a boolean true flag
+            acc.append(args.pop(0) + "=true")
+
+        else:
+            acc.append(args.pop(0))
+
+    else:
+        if args:
+            acc.extend(args)
+
+    return acc
+
+
+@dataclass
+class BazelCli:
+    binary: str
+    startup_opts: List[str]
+    command: Optional[str]
+    command_opts: List[str]
+    subprocess_opts: List[str]
+
+    @classmethod
+    def parse_cli(cls, args: List[str]) -> "BazelCli":
+        args = normalize_opts(args)
+        binary = args.pop(0)
+
+        startup_opts = []
+        while args and args[0].startswith("--"):
+            startup_opts.append(args.pop(0))
+
+        command = None
+        if args and args[0] in VERBS:
+            command = args.pop(0)
+
+        command_opts = []
+        while args and args[0] != "--":
+            command_opts.append(args.pop(0))
+
+        subprocess_opts = []
+        if args:
+            if args[0] == "--":
+                args.pop(0)
+            subprocess_opts.extend(args)
+
+        return cls(
+            binary=binary,
+            startup_opts=startup_opts,
+            command=command,
+            command_opts=command_opts,
+            subprocess_opts=subprocess_opts,
+        )
+
+    def render_cli(self):
+        acc = [
+            self.binary,
+            *self.startup_opts,
+        ]
+        if self.command:
+            if self.command == "watch":
+                acc.extend(self.command_opts)
+            else:
+                acc.append(self.command)
+                acc.extend(self.command_opts)
+
+            if self.command == "test":
+                acc.extend(["--test_arg=" + it for it in self.subprocess_opts])
+            elif self.command == "run":
+                acc.append("--")
+                acc.extend(self.subprocess_opts)
+            elif self.command and not self.subprocess_opts:
+                pass
+            else:
+                print(
+                    f"Warning: {self.command} does not support -- args! {self.subprocess_opts!r}",
+                    file=sys.stderr,
+                )
+
+        return acc
+
+    def executable(self, exclude: List[Path]):
+        """Try to resolve as via which() an executable to delegate to."""
+
+        if self.command == "watch":
+            for p in chain(which("ibazel")):
+                if p.parent not in exclude:
+                    return str(p)
+
+        else:
+            for p in chain(which("bazelisk"), which("bazel")):
+                if p.parent not in exclude:
+                    return str(p)
+
+    def render_text(self, next):
+        lines = []
+        lines.append("  " + next)
+        base_prefix = "      "
+        for arg in self.render_cli()[1:]:
+            prefix = base_prefix
+            if arg in VERBS or arg == self.command:
+                prefix = "    "
+            elif arg == "--":
+                base_prefix += "  "
+            lines.append(prefix + arg)
+        return "\\\n".join(lines)
+
+
+# FIXME: Use some sort of plugin model here to implement interceptors
+def middleware(cli):
+    return cli
+
+
+if __name__ == "__main__":
+    # This script has a magical flag to help with resolving bazel
+    exclude = []
+    while len(sys.argv) > 1 and sys.argv[1].startswith("--bazelshim_exclude"):
+        exclude.append(Path(sys.argv.pop(1).split("=")[1]).absolute())
+
+    us = Path(sys.argv[0]).absolute()
+    exclude.append(us.parent)
+
+    cli = BazelCli.parse_cli(["bazel"] + sys.argv[1:])
+    cli = middleware(cli)
+    next = cli.executable(exclude=exclude)
+    if sys.stderr.isatty() and not "--isatty=false" in cli.command_opts:
+        print(
+            "\u001b[33mInfo\u001b[0m: Executing\n" + cli.render_text(next),
+            file=sys.stderr,
+        )
+    os.execv(next, cli.render_cli())

projects/bazelshim/test/test_normalizer.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+from shlex import split as shlex
+
+from bazelshim.__main__ import normalize_opts
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "a, b",
+    [
+        (
+            "bazel clean",
+            [
+                "bazel",
+                "clean",
+            ],
+        ),
+        (
+            "bazel --client_debug clean",
+            [
+                "bazel",
+                "--client_debug=true",
+                "clean",
+            ],
+        ),
+        (
+            "bazel build //foo:bar //baz:*",
+            [
+                "bazel",
+                "build",
+                "//foo:bar",
+                "//baz:*",
+            ],
+        ),
+        (
+            "bazel test //foo:bar //baz:* -- -vvv",
+            [
+                "bazel",
+                "test",
+                "//foo:bar",
+                "//baz:*",
+                "--",
+                "-vvv",
+            ],
+        ),
+        (
+            "bazel test --shell_executable /bin/bish //foo:bar //baz:* -- -vvv",
+            [
+                "bazel",
+                "test",
+                "--shell_executable=/bin/bish",
+                "//foo:bar",
+                "//baz:*",
+                "--",
+                "-vvv",
+            ],
+        ),
+        (
+            "bazel run //foo:bar -- --foo=bar --baz=qux",
+            [
+                "bazel",
+                "run",
+                "//foo:bar",
+                "--",
+                "--foo=bar",
+                "--baz=qux",
+            ],
+        ),
+    ],
+)
+def test_normalize_opts(a, b):
+    assert normalize_opts(shlex(a)) == b

projects/bazelshim/test/test_parser.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+from shlex import split as shlex
+
+from bazelshim.__main__ import BazelCli
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "a, b",
+    [
+        (
+            "bazel clean",
+            BazelCli("bazel", [], "clean", [], []),
+        ),
+        (
+            "bazel --client_debug clean",
+            BazelCli("bazel", ["--client_debug=true"], "clean", [], []),
+        ),
+        (
+            "bazel build //foo:bar //baz:*",
+            BazelCli("bazel", [], "build", ["//foo:bar", "//baz:*"], []),
+        ),
+        (
+            "bazel test //foo:bar //baz:* -- -vvv",
+            BazelCli("bazel", [], "test", ["//foo:bar", "//baz:*"], ["-vvv"]),
+        ),
+        (
+            "bazel test --shell_executable /bin/bish //foo:bar //baz:* -- -vvv",
+            BazelCli(
+                "bazel",
+                [],
+                "test",
+                ["--shell_executable=/bin/bish", "//foo:bar", "//baz:*"],
+                ["-vvv"],
+            ),
+        ),
+        (
+            "bazel run //foo:bar -- --foo=bar --baz=qux",
+            BazelCli(
+                "bazel",
+                [],
+                "run",
+                ["//foo:bar"],
+                ["--foo=bar", "--baz=qux"],
+            ),
+        ),
+    ],
+)
+def test_normalize_opts(a, b):
+    assert BazelCli.parse_cli(shlex(a)) == b

projects/bussard/src/bussard/reader.py

@@ -9,7 +9,10 @@ zonefiles through the parser.
 
 from types import LambdaType
 
-from bussard.gen.parser import parse as _parse, Parser  # noqa
+from bussard.gen.parser import (  # noqa
+    parse as _parse,
+    Parser,
+)
 from bussard.gen.types import *  # noqa
 
 

projects/bussard/test/bussard/test_reader.py

@@ -3,7 +3,12 @@ Tests of the Bussard reader.
 """
 
 import bussard.reader as t
-from bussard.reader import Actions, Parser, read, read1
+from bussard.reader import (
+    Actions,
+    Parser,
+    read,
+    read1,
+)
 
 
 def parse_word(input):