Initial state

tools/python/BUILD

@@ -0,0 +1,136 @@
+load("@rules_python//python:defs.bzl",
+     "py_runtime_pair"
+)
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+exports_files([
+    "defs.bzl",
+    "bzl_pytest_shim.py",
+    "bzl_unittest_shim.py",
+])
+
+py_runtime(
+    name = "python_runtime",
+    interpreter_path = "/usr/bin/python3",
+    python_version = "PY3",
+    visibility = ["//visibility:public"],
+)
+
+py_runtime_pair(
+    name = "runtime",
+    py2_runtime = None,
+    py3_runtime = ":python_runtime",
+)
+
+toolchain(
+    name = "toolchain",
+    toolchain = ":runtime",
+    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+)
+
+py_binary(
+    name = "autoflake",
+    main = "autoflake_shim.py",
+    deps = [
+        py_requirement("autoflake"),
+    ]
+)
+
+py_binary(
+    name = "black",
+    main = "black_shim.py",
+    deps = [
+        py_requirement("black"),
+    ]
+)
+
+py_binary(
+    name = "isort",
+    main = "isort_shim.py",
+    deps = [
+        py_requirement("isort"),
+    ]
+)
+
+py_binary(
+    name = "sphinx",
+    main = "sphinx_shim.py",
+    deps = [
+        py_requirement("click"),
+        py_requirement("recommonmark"),
+        py_requirement("sphinx"),
+        py_requirement("sphinxcontrib-openapi"),
+        py_requirement("sphinxcontrib-programoutput"),
+        py_requirement("livereload"),
+    ]
+)
+
+py_pytest(
+    name = "test_licenses",
+    srcs = [
+        "test_licenses.py",
+    ],
+    data = [
+        "requirements.txt",
+    ],
+    deps = [
+        py_requirement("requests"),
+    ]
+)
+
+py_binary(
+    name = "unify",
+    main = "unify_shim.py",
+    deps = [
+        py_requirement("unify"),
+    ]
+)
+
+py_binary(
+    name = "openapi",
+    main = "openapi_shim.py",
+    deps = [
+        py_requirement("openapi-spec-validator"),
+    ]
+)
+
+# WARNING: YAMLLINT is GLP3'd code. Do not extend, modify or depend on this as a lib.
+py_binary(
+    name = "yamllint",
+    main = "yamllint_shim.py",
+    deps = [
+        py_requirement("yamllint"),
+    ]
+)
+
+py_binary(
+    name = "reqsort",
+    main = "reqsort.py",
+    deps = [
+        py_requirement("click"),
+    ]
+)
+
+
+py_binary(
+    name = "templater",
+    main = "templater.py",
+    deps = [
+        py_requirement("click"),
+        py_requirement("jinja2"),
+        py_requirement("PyYAML"),
+    ]
+)
+
+py_binary(
+    name = "xfmt",
+    main = "xfmt.py",
+    deps = [
+        py_requirement("beautifulsoup4"),
+        py_requirement("click"),
+        py_requirement("lxml"),
+    ]
+)

tools/python/autoflake_shim.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+"""
+Shim for executing autoflake.
+"""
+
+import re
+import sys
+
+from autoflake import main
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

tools/python/black_shim.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+"""
+Shim for executing black.
+"""
+
+import re
+import sys
+
+from black import patched_main
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(patched_main())

tools/python/bzl_pytest_shim.py

@@ -0,0 +1,12 @@
+import sys
+
+import pytest
+
+if __name__ == "__main__":
+    print(sys.version_info, file=sys.stderr)
+    print(sys.argv, file=sys.stderr)
+
+    cmdline = ["--ignore=external"] + sys.argv[1:]
+    print(cmdline, file=sys.stderr)
+
+    sys.exit(pytest.main(cmdline))

tools/python/bzl_unittest_shim.py

@@ -0,0 +1,66 @@
+"""Universal launcher for unit tests"""
+
+import argparse
+import logging
+import os
+import sys
+import unittest
+
+
+def main():
+    """Parse args, collect tests and run them"""
+    # Disable *.pyc files
+    sys.dont_write_bytecode = True
+
+    # Add ".." to module search path
+    cur_dir = os.path.dirname(os.path.realpath(__file__))
+    top_dir = os.path.abspath(os.path.join(cur_dir, os.pardir))
+    sys.path.append(top_dir)
+
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="verbosity level, use: [-v | -vv | -vvv]",
+    )
+    parser.add_argument(
+        "-s", "--start-directory", default=None, help="directory to start discovery"
+    )
+    parser.add_argument(
+        "-p",
+        "--pattern",
+        default="test*.py",
+        help="pattern to match test files ('test*.py' default)",
+    )
+    parser.add_argument(
+        "test", nargs="*", help="test specs (e.g. module.TestCase.test_func)"
+    )
+    args = parser.parse_args()
+
+    if not args.start_directory:
+        args.start_directory = cur_dir
+
+    if args.verbose > 2:
+        logging.basicConfig(level=logging.DEBUG, format="DEBUG: %(message)s")
+
+    loader = unittest.TestLoader()
+    if args.test:
+        # Add particular tests
+        for test in args.test:
+            suite = unittest.TestSuite()
+            suite.addTests(loader.loadTestsFromName(test))
+    else:
+        # Find all tests
+        suite = loader.discover(args.start_directory, args.pattern)
+
+    runner = unittest.TextTestRunner(verbosity=args.verbose)
+    result = runner.run(suite)
+    return result.wasSuccessful()
+
+
+if __name__ == "__main__":
+    # NOTE: True(success) -> 0, False(fail) -> 1
+    exit(not main())

tools/python/defs.bzl

@@ -0,0 +1,146 @@
+load("@arrdem_source_pypi//:requirements.bzl",
+     _py_requirement = "requirement"
+)
+
+load("@rules_python//python:defs.bzl",
+     _py_binary = "py_binary",
+     _py_test = "py_test",
+     _py_library = "py_library",
+)
+
+
+def py_requirement(*args, **kwargs):
+  """A re-export of requirement()"""
+  return _py_requirement(*args, **kwargs)
+
+
+def py_test(python_version=None, **kwargs):
+  """A re-export of py_test()"""
+
+  if python_version and python_version != "PY3":
+    fail("py3k only!")
+
+  return _py_test(
+    python_version="PY3",
+    **kwargs,
+  )
+
+
+def py_pytest(name, srcs, deps, main=None, python_version=None, args=None, **kwargs):
+  """A py_test target which uses pytest."""
+
+  if python_version and python_version != "PY3":
+    fail("py3k only!")
+
+  f = "//tools/python:bzl_pytest_shim.py"
+
+  deps = [
+      py_requirement("pytest"),
+      py_requirement("jedi"),
+      py_requirement("pytest-pudb"),
+  ] + deps
+
+  srcs = [f] + srcs
+
+  t = py_test(
+    name = name,
+    srcs = srcs,
+    main = f,
+    args = args,
+    python_version="PY3",
+    deps = deps,
+    **kwargs,
+  )
+
+  # FIXME (arrdem 2020-09-27):
+  #   This really needs to be a py_image_test.
+  #   Not clear how to achieve that.
+  # py_image(
+  #   name = name + ".containerized",
+  #   main = f,
+  #   args = args,
+  #   srcs = srcs,
+  #   deps = deps,
+  #   **kwargs,
+  # )
+
+  return t
+
+
+def py_unittest(srcs=[], **kwargs):
+  """A helper for running unittest tests"""
+
+  f = "//tools/python:bzl_unittest_shim.py"
+  return py_test(
+      main = f,
+      srcs = [f] + srcs,
+      **kwargs
+  )
+
+
+def py_binary(python_version=None, main=None, srcs=None, **kwargs):
+  """A re-export of py_binary()"""
+
+  if python_version and python_version != "PY3":
+    fail("py3k only!")
+
+  srcs = srcs or []
+  if main not in srcs:
+    srcs = [main] + srcs
+
+  return _py_binary(
+    python_version = "PY3",
+    main = main,
+    srcs = srcs,
+    **kwargs,
+  )
+
+
+def py_library(srcs_version=None, **kwargs):
+  """A re-export of py_library()"""
+
+  if srcs_version and srcs_version != "PY3":
+    fail("py3k only!")
+
+  return _py_library(
+    srcs_version="PY3",
+    **kwargs
+  )
+
+
+ResourceGroupInfo = provider(
+    fields = {
+        "srcs": "files to use from Python",
+    },
+)
+
+
+def _resource_impl(ctx):
+    srcs = []
+    for target in ctx.attr.srcs:
+        srcs.extend(target.files.to_list())
+    transitive_srcs = depset(direct = srcs)
+
+    return [
+        ResourceGroupInfo(
+            srcs = ctx.attr.srcs,
+        ),
+        PyInfo(
+            has_py2_only_sources = False,
+            has_py3_only_sources = True,
+            uses_shared_libraries = False,
+            transitive_sources = transitive_srcs,
+        ),
+    ]
+
+py_resources = rule(
+    implementation = _resource_impl,
+    attrs = {
+        "srcs": attr.label_list(
+            allow_empty = True,
+            mandatory = True,
+            allow_files = True,
+            doc = "Files to hand through to Python",
+        ),
+    },
+)

tools/python/isort_shim.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+"""
+Shim for executing isort.
+"""
+
+import re
+import sys
+
+from isort.main import main
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

tools/python/openapi_shim.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+"""Shim for executing the openapi spec validator."""
+
+import re
+import sys
+
+from openapi_spec_validator.__main__ import main
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

tools/python/reqsort.py

@@ -0,0 +1,72 @@
+"""
+Platform independent sorting/formatting for requirements.txt
+"""
+
+import re
+
+import click
+
+REQ_PATTERN = re.compile(
+    r"(?P<pkgname>[a-zA-Z0-9_-]+)(?P<features>\[.*?\])?==(?P<version>[^\s;#]+)|(.*?#egg=(?P<eggname>[a-zA-Z0-9_-]+))"
+)
+
+
+SHITLIST = [
+    "pip",
+    "pkg-resources",
+    "setuptools",
+]
+
+
+def sort_key(requirement: str) -> str:
+    requirement = requirement.lower()
+    match = re.match(REQ_PATTERN, requirement)
+    sort_key = (
+        (match.group("pkgname") or match.group("eggname"))  # Get the match group
+        .replace("-", "")  # We ignore -
+        .replace("_", "")  # And _
+    )
+    return sort_key
+
+
+@click.command()
+@click.option("--execute/--dryrun", "execute", default=False)
+@click.argument("requirements")
+def main(requirements, execute):
+    """Given the path of a requirements.txt, format it.
+
+    If running in --execute, rewrite the source file with sorted contents and exit 0.
+
+    If running in --dryrun, exit 0 if --execute would produce no changes otherwise exit 1.
+
+    """
+
+    with open(requirements) as f:
+        lines = f.readlines()
+        f.seek(0)
+        # Preserve an initial "buffer" for equality testing
+        initial_buff = f.read()
+
+    # Trim whitespace
+    lines = [l.strip() for l in lines]
+
+    # Discard comments and shitlisted packages
+    lines = [l for l in lines if not l.startswith("#") and not sort_key(l) in SHITLIST]
+
+    # And sort, ignoring case explicitly
+    lines = sorted(lines, key=sort_key)
+
+    # And generate a new "buffer"
+    new_buff = "\n".join(lines) + "\n"
+
+    if new_buff != initial_buff and not execute:
+        exit(1)
+
+    else:
+        with open(requirements, "w") as f:
+            f.write(new_buff)
+        exit(0)
+
+
+if __name__ == "__main__":
+    main()

tools/python/requirements.txt

@@ -0,0 +1,67 @@
+alabaster==0.7.12
+appdirs==1.4.4
+attrs==20.3.0
+autoflake==1.4
+Babel==2.9.0
+beautifulsoup4==4.9.3
+black==20.8b1
+certifi==2020.12.5
+chardet==4.0.0
+click==7.1.2
+commonmark==0.9.1
+docutils==0.17
+idna==2.10
+imagesize==1.2.0
+iniconfig==1.1.1
+isodate==0.6.0
+isort==5.8.0
+jedi==0.18.0
+Jinja2==2.11.3
+jsonschema==3.2.0
+livereload==2.6.3
+lxml==4.6.3
+m2r==0.2.1
+MarkupSafe==1.1.1
+mistune==0.8.4
+mypy-extensions==0.4.3
+openapi-schema-validator==0.1.5
+openapi-spec-validator==0.3.0
+packaging==20.9
+parso==0.8.2
+pathspec==0.8.1
+pluggy==0.13.1
+pudb==2020.1
+py==1.10.0
+pyflakes==2.3.1
+Pygments==2.8.1
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pytest==6.2.3
+pytest-pudb==0.7.0
+pytz==2021.1
+PyYAML==5.4.1
+recommonmark==0.7.1
+regex==2021.4.4
+requests==2.25.1
+six==1.15.0
+snowballstemmer==2.1.0
+soupsieve==2.2.1
+Sphinx==3.5.3
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-httpdomain==1.7.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-openapi==0.7.0
+sphinxcontrib-programoutput==0.17
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+toml==0.10.2
+tornado==6.1
+typed-ast==1.4.2
+typing-extensions==3.7.4.3
+unify==0.5
+untokenize==0.1.1
+urllib3==1.26.4
+urwid==2.1.2
+yamllint==1.26.1

tools/python/sphinx_shim.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+
+"""A documentation generator.
+
+This is a shim tool which wraps up a whole bunch of Sphinx internals in a single "convenient"
+entrypoint. Former tweeps may recognize some parallels to the `docbird` tool developed by Twitter's
+techdocs team.
+
+"""
+
+import builtins
+from functools import wraps
+import io
+import os
+import sys
+
+import click
+import livereload
+from sphinx.application import Sphinx
+from sphinx.cmd.build import main as build
+from sphinx.cmd.quickstart import main as new
+from sphinx.ext.apidoc import main as apidoc
+from sphinx.ext.autosummary.generate import main as autosummary
+from sphinx.util.docutils import docutils_namespace, patch_docutils
+
+
+@click.group()
+def cli():
+    """A documentation generator.
+
+    Just a shim to a variety of upstream Sphinx commands typically distributed as separate binaries
+    for some dang reason.
+
+    Note that due to subcommand argument parsing '-- --help' is likely required.
+
+    Subcommands have not been renamed (or customized, yet) from their Sphinx equivalents.
+
+    """
+
+
+@cli.group()
+def generate():
+    """Subcommands for doing RST header generation."""
+
+
+@generate.command(
+    "apidoc",
+    context_settings=dict(
+        ignore_unknown_options=True,
+    ),
+)
+@click.argument("argv", nargs=-1, type=click.UNPROCESSED)
+def do_apidoc(argv):
+    """Use sphinx.ext.apidoc to generate API documentation."""
+    return apidoc(argv)
+
+
+@generate.command(
+    "summary",
+    context_settings=dict(
+        ignore_unknown_options=True,
+    ),
+)
+@click.argument("argv", nargs=-1, type=click.UNPROCESSED)
+def do_summary(argv):
+    """Use sphinx.ext.autosummary to generate module summaries."""
+    return autosummary(argv)
+
+
+@cli.command(
+    "new",
+    context_settings=dict(
+        ignore_unknown_options=True,
+    ),
+)
+@click.argument("argv", nargs=-1, type=click.UNPROCESSED)
+def do_new(argv):
+    """Create a new Sphinx in the current directory."""
+    return new(argv)
+
+
+@cli.command(
+    "build",
+    context_settings=dict(
+        ignore_unknown_options=True,
+    ),
+)
+@click.argument("sourcedir")
+@click.argument("outputdir")
+@click.option("-c", "--confdir")
+@click.option("-d", "--doctreedir")
+@click.option("-b", "--builder", default="html")
+@click.option("--freshenv/--no-freshenv", default=False)
+@click.option("-W", "--warning-is-error", "werror", is_flag=True, flag_value=True)
+@click.option("-t", "--tag", "tags", multiple=True)
+def do_build(
+    sourcedir, outputdir, confdir, doctreedir, builder, freshenv, werror, tags
+):
+    """Build a single Sphinx project."""
+
+    if not confdir:
+        confdir = sourcedir
+
+    if not doctreedir:
+        doctreedir = os.path.join(outputdir, ".doctrees")
+
+    status = sys.stdout
+    warning = sys.stderr
+    error = sys.stderr
+
+    confdir = confdir or sourcedir
+    confoverrides = {}  # FIXME: support these
+    with patch_docutils(confdir), docutils_namespace():
+        app = Sphinx(
+            sourcedir,
+            confdir,
+            outputdir,
+            doctreedir,
+            builder,
+            confoverrides,
+            status,
+            warning,
+            freshenv,
+            werror,
+            tags,
+            1,
+            4,
+            False,
+        )
+        app.build(True, [])
+        return app.statuscode
+
+
+@cli.command(
+    "serve",
+    context_settings=dict(
+        ignore_unknown_options=True,
+    ),
+)
+@click.option("-h", "--host", default="localhost")
+@click.option("-p", "--port", type=int, default=8080)
+@click.argument("sourcedir")
+@click.argument("outputdir")
+def do_serve(host, port, sourcedir, outputdir):
+    """Build and then serve a Sphinx tree."""
+
+    sourcedir = os.path.realpath(sourcedir)
+    outputdir = os.path.realpath(outputdir)
+
+    server = livereload.Server()
+
+    # HACK (arrdem 2020-10-31):
+    #   Okay. This is an elder hack, and I'm proud of it.
+    #
+    #   The naive implementation of the watching server is to watch the input files, which is
+    #   obviously correct. However, Sphinx has a BUNCH of operators like include and mdinclude and
+    #   soforth which can cause a Sphinx doctree to have file dependencies OUTSIDE of the "trivial"
+    #   source path dependency set.
+    #
+    #   In order to make sure that rebuilding does what the user intends, we trace calls to the
+    #   open() function and attempt to dynamically discover the dependency set of the site. This
+    #   allows us to trigger strictly correct rebuilds unlike other Sphinx implementations which
+    #   need to be restarted under some circumstances.
+    def opener(old_open):
+        @wraps(old_open)
+        def tracking_open(path, mode="r", *args, **kw):
+            file = old_open(path, mode, *args, **kw)
+            if isinstance(path, int):
+                # If you're doing something weird with file pointers, ignore it.
+                pass
+            else:
+                path = os.path.realpath(path)
+
+                if "w" in mode:
+                    # If we're writing a file, it's an output for sure. Ignore it.
+                    ignorelist.add(path)
+                elif (
+                    not path.startswith(outputdir)
+                    and path not in ignorelist
+                    and not path in watchlist
+                ):
+                    # Watch any source file (file we open for reading)
+                    server.watch(path, build)
+                    watchlist.add(path)
+            return file
+
+        return tracking_open
+
+    ignorelist = set()
+    watchlist = set()
+
+    def build():
+        try:
+            old_open = open
+            builtins.open = opener(old_open)
+            io.open = opener(old_open)
+            do_build([sourcedir, outputdir])
+        except SystemExit:
+            pass
+        finally:
+            builtins.open = old_open
+            io.open = old_open
+
+    build()
+    server.watch(
+        "conf.py", build
+    )  # Not sure why this isn't picked up, but it doesn't seem to be.
+
+    server.serve(port=port, host=host, root=outputdir)
+
+
+if __name__ == "__main__":
+    # Hack in a -- delimeter to bypass click arg parsing
+    if not (sys.argv + [""])[1].startswith("-"):
+        sys.argv = [sys.argv[0], "--"] + sys.argv[1:]
+
+    # Use click subcommands for everything else
+    exit(cli())

tools/python/templater.py

@@ -0,0 +1,75 @@
+"""A tiny template(s) tool.
+
+Processes Jekyll/Hyde/Hugo/... style 'fontmatter' headers, applying Jinja2/Liquid templating from an
+optional templates and includes directory.
+
+"""
+
+import os
+import re
+
+import click
+import jinja2
+import yaml
+
+FONTMATTER_PATTERN = re.compile(
+    r"^(---\n\r?(?P<fontmatter>.*?)\n\r?---\n\r?)?(?P<content>.+)$", re.DOTALL
+)
+
+
+@click.command()
+@click.option("-i", "--include", "include_dir", multiple=True)
+@click.option("-t", "--template", "template_dir", multiple=True)
+@click.option("-c", "--config", "config_file")
+@click.argument("infile")
+@click.argument("outfile")
+def main(include_dir, template_dir, config_file, infile, outfile):
+    """Apply templating.
+
+    Consume infile, processing it with templating and write the results to outfile.
+
+    """
+
+    loaders = []
+    for d in include_dir:
+        loaders.append(jinja2.FileSystemLoader(os.path.realpath(d)))
+
+    for d in template_dir:
+        loaders.append(jinja2.FileSystemLoader(os.path.realpath(d)))
+
+    # Build a j2 environment using the potentially various loaders..
+    environment = jinja2.Environment(loader=jinja2.ChoiceLoader(loaders))
+
+    # Load a site config
+    if config_file:
+        with open(config_file) as f:
+            site = yaml.safe_load(f.read())
+    else:
+        site = {}
+
+    # Figure out doing the fontmatter nonsense...
+    with open(infile, "r") as f:
+        buff = f.read()
+
+    match = re.match(FONTMATTER_PATTERN, buff)
+    if fontmatter := match.group("fontmatter"):
+        fontmatter = yaml.safe_load(fontmatter)
+    else:
+        fontmatter = {}
+
+    # Render the file contents
+    template = environment.from_string(match.group("content"))
+    content = template.render(site=site, page=fontmatter)
+
+    # If there's a configured `layout:` stick the content in the layout.
+    if "layout" in fontmatter:
+        template = environment.get_template(fontmatter.get("layout"))
+        content = template.render(content=content, site=site, page=fontmatter)
+
+    # And dump the results
+    with open(outfile, "w") as f:
+        f.write(content)
+
+
+if __name__ == "__main__":
+    main()

tools/python/test_licenses.py

@@ -0,0 +1,142 @@
+"""
+Validate 3rdparty library licenses as approved.
+"""
+
+import re
+
+import pytest
+import requests
+
+# Licenses approved as representing non-copyleft and not precluding commercial usage.
+# This is all easy, there's a good schema here.
+APPROVED_LICENSES = [
+    "License :: OSI Approved :: MIT License",
+    "License :: OSI Approved :: Apache Software License",
+    "License :: OSI Approved :: BSD License",
+    "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
+    "License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)",
+    "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    "License :: OSI Approved :: Python Software Foundation License",
+    "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)",
+    "License :: OSI Approved :: ISC License (ISCL)",
+]
+
+# This data is GARBO.
+LICENSES_BY_LOWERNAME = {
+    "apache 2.0": "License :: OSI Approved :: Apache Software License",
+    "apache": "License :: OSI Approved :: Apache Software License",
+    "bsd 3 clause": "License :: OSI Approved :: BSD License",
+    "bsd": "License :: OSI Approved :: BSD License",
+    "gplv3": "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "http://www.apache.org/licenses/license-2.0": "License :: OSI Approved :: Apache Software License",
+    "isc": "License :: OSI Approved :: ISC License (ISCL)",
+    "mit": "License :: OSI Approved :: MIT License",
+    "mpl 2.0": "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    "mpl": "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
+    "psf": "License :: OSI Approved :: Python Software Foundation License",
+}
+
+# Mash in some cases.
+LICENSES_BY_LOWERNAME.update(
+    {l.split(" :: ")[-1].lower(): l for l in APPROVED_LICENSES}
+)
+
+# As a workaround for packages which don"t have correct meadata on PyPi, hand-verified packages
+APPROVED_PACKAGES = [
+    "yamllint",  # WARNING: YAMLLINT IS GLP3"d.
+    "Flask_Log_Request_ID",  # MIT, currently depended on as a git dep.
+]
+
+REQ_PATTERN = re.compile(
+    r"(?P<pkgname>[a-zA-Z0-9_-]+)(?P<features>\[.*?\])?==(?P<version>[^\s;#]+)|(.*?#egg=(?P<eggname>[a-zA-Z0-9_-]+))"
+)
+
+
+def parse_requirement(line):
+    """Given a requirement return the requirement name and version as a tuple.
+
+    Only the strict `==` version pinning subset is supported.
+    Features are supported.
+    """
+
+    if m := re.match(REQ_PATTERN, line):
+        return (m.group("pkgname") or m.group("eggname")), m.group("version")
+
+
+@pytest.mark.parametrize(
+    "line,t",
+    [
+        ("foo==1.2.3", ("foo", "1.2.3")),
+        ("foo[bar]==1.2.3", ("foo", "1.2.3")),
+        ("foo[bar, baz, qux]==1.2.3", ("foo", "1.2.3")),
+        # Various stuff we should ignore
+        ("# comment line", None),
+        ("    # garbage whitespace", None),
+        ("     \t", None),
+    ],
+)
+def test_parse_requirement(line, t):
+    """The irony of testing one"s tests is not lost."""
+
+    assert parse_requirement(line) == t
+
+
+with open("tools/python/requirements.txt") as f:
+    PACKAGES = [parse_requirement(l) for l in f.readlines()]
+
+
+def bash_license(ln):
+    if ln:
+        ln = re.sub("[(),]|( version)|( license)", "", ln.lower())
+        ln = LICENSES_BY_LOWERNAME.get(ln, ln)
+    return ln
+
+
+def licenses(package, version):
+    """Get package metadata (the licenses list) from PyPi.
+
+    pip and other tools use the local package metadata to introspect licenses which requires that
+    packages be installed. Going to PyPi isn't strictly reproducible both because the PyPi database
+    could be updated and we could see network failures but there really isn't a good way to solve
+    this problem.
+
+    """
+    l = []
+
+    # If we don't have a version (eg. forked git dep) assume we've got the same license constraints
+    # as the latest upstream release. After all we can't re-license stuff.
+    if not version:
+        blob = requests.get(f"https://pypi.python.org/pypi/{package}/json").json()
+        if ln := bash_license(blob.get("license")):
+            l.append(ln)
+        else:
+            try:
+                version = list(blob.get("releases", {}).keys())[-1]
+            except IndexError:
+                pass
+
+    # If we have a version, try to pull that release's metadata since it may have more/better.
+    if version:
+        blob = requests.get(
+            f"https://pypi.python.org/pypi/{package}/{version}/json"
+        ).json()
+        l = [
+            c
+            for c in blob.get("info", {}).get("classifiers", [])
+            if c.startswith("License")
+        ]
+        ln = blob.get("info", {}).get("license")
+        if ln and not l:
+            l.append(bash_license(ln))
+
+    return l
+
+
+@pytest.mark.parametrize("package,version", PACKAGES)
+def test_approved_license(package, version):
+    """Ensure that a given package is either allowed by name or uses an approved license."""
+
+    _licenses = licenses(package, version)
+    assert package in APPROVED_PACKAGES or any(
+        l in APPROVED_LICENSES for l in _licenses
+    ), f"{package} was not approved and its license(s) were unknown {_licenses!r}"

tools/python/unify_shim.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+"""
+Shim for executing isort.
+"""
+
+
+from unify import main
+
+if __name__ == "__main__":
+    exit(main())

tools/python/xfmt.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+"""
+A quick and dirty XML formatter.
+"""
+
+from bs4 import BeautifulSoup
+import click
+
+
+@click.command()
+@click.argument("filename")
+def main(filename):
+    with open(filename) as f:
+        bs = BeautifulSoup(f, "xml")
+
+    with open(filename, "w") as of:
+        of.write(bs.prettify())
+        of.write("\n")
+
+    print(f"Formatted {filename}!")
+
+
+if __name__ == "__main__":
+    main()

tools/python/yamllint_shim.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+
+# WARNING: Yamllint is GPL3'd code.
+
+"""A shim for executing yamllint."""
+
+import re
+import sys
+
+from yamllint.cli import run
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0])
+    sys.exit(run())