Import a bunch of my Bazel infrastructure

MODULE.bazel

@@ -0,0 +1,24 @@
+bazel_dep(name = "rules_python", version = "0.19.0")
+
+pip = use_extension("@rules_python//python:extensions.bzl", "pip")
+
+pip.parse(
+    name = "pypa",
+    requirements_lock = "//tools/python:requirements_lock.txt",
+)
+
+use_repo(pip, "pypa")
+
+# (Optional) Register a specific python toolchain instead of using the host version
+python = use_extension("@rules_python//python:extensions.bzl", "python")
+
+python.toolchain(
+    name = "python3_10",
+    python_version = "3.10",
+)
+
+use_repo(python, "python3_10_toolchains")
+
+register_toolchains(
+    "@python3_10_toolchains//:all",
+)

WORKSPACE

@@ -0,0 +1,16 @@
+workspace(
+    name = "arrdem_flowmetal",
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_python",
+    sha256 = "ffc7b877c95413c82bfd5482c017edcf759a6250d8b24e82f41f3c8b8d9e287e",
+    strip_prefix = "rules_python-0.19.0",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.19.0/rules_python-0.19.0.tar.gz",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()

tools/build_rules/BUILD

@@ -0,0 +1,3 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])

tools/build_rules/cp.bzl

@@ -0,0 +1,47 @@
+load("@bazel_skylib//rules:copy_file.bzl",
+     "copy_file",
+)
+
+def cp(name, src, **kwargs):
+    """A slightly more convenient cp() rule. Name and out should always be the same."""
+
+    rule_name = name.replace(".", "_").replace(":", "/").replace("//", "").replace("/", "_")
+    copy_file(
+        name = rule_name,
+        src = src,
+        out = name,
+        **kwargs
+    )
+    return rule_name
+
+
+def _copy_filegroup_impl(ctx):
+    all_outputs = []
+    for t in ctx.attr.deps:
+        t_prefix = t.label.package
+        for f in t.files.to_list():
+            # Strip out the source prefix...
+            path = f.short_path.replace(t_prefix + "/", "")
+            out = ctx.actions.declare_file(path)
+            print(ctx.attr.name, t.label, f, " => ", path)
+            all_outputs += [out]
+            ctx.actions.run_shell(
+                outputs=[out],
+                inputs=depset([f]),
+                arguments=[f.path, out.path],
+                command="cp $1 $2"
+            )
+
+    return [
+        DefaultInfo(
+            files=depset(all_outputs),
+            runfiles=ctx.runfiles(files=all_outputs))
+    ]
+
+
+copy_filegroups = rule(
+    implementation=_copy_filegroup_impl,
+    attrs={
+        "deps": attr.label_list(),
+    },
+)

tools/build_rules/prelude_bazel

@@ -0,0 +1,24 @@
+# -*- mode: bazel -*-
+# A global prelude for all BUILD[.bazel] files
+
+load("//tools/python:defs.bzl",
+     "py_library",
+     "py_binary",
+     "py_unittest",
+     "py_pytest",
+     "py_resources",
+     "py_project",
+)
+
+load("@pypa//:requirements.bzl",
+     py_requirement="requirement"
+)
+
+load("@bazel_skylib//rules:copy_file.bzl",
+    "copy_file",
+)
+
+load("//tools/build_rules:cp.bzl",
+     "cp",
+     "copy_filegroups"
+)

tools/python/BUILD

@@ -0,0 +1,47 @@
+load("@rules_python//python:defs.bzl",
+     "py_runtime_pair",
+)
+
+load("@arrdem_source_pypi//:requirements.bzl", "all_requirements")
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+exports_files([
+    "defs.bzl",
+    "bzl_pytest_shim.py",
+    "bzl_unittest_shim.py",
+    "pythonshim",
+])
+
+py_runtime(
+    name = "python3_runtime",
+    files = [],
+    interpreter = ":pythonshim",
+    python_version = "PY3",
+    visibility = ["//visibility:public"],
+)
+
+py_runtime_pair(
+    name = "python_runtime",
+    py2_runtime = None,
+    py3_runtime = ":python3_runtime",
+)
+
+toolchain(
+    name = "python3_toolchain",
+    toolchain = ":python_runtime",
+    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+)
+
+py_pytest(
+    name = "test_licenses",
+    srcs = [
+        "test_licenses.py",
+    ],
+    data = [
+        "requirements.txt",
+    ],
+    deps = all_requirements,
+)

tools/python/bzl_pytest_shim.py

@@ -0,0 +1,11 @@
+"""A shim for executing pytest."""
+
+import sys
+
+import pytest
+
+
+if __name__ == "__main__":
+    cmdline = ["--ignore=external"] + sys.argv[1:]
+    print(cmdline, file=sys.stderr)
+    sys.exit(pytest.main(cmdline))

tools/python/bzl_unittest_shim.py

@@ -0,0 +1,66 @@
+"""Universal launcher for unit tests"""
+
+import argparse
+import logging
+import os
+import sys
+import unittest
+
+
+def main():
+    """Parse args, collect tests and run them"""
+    # Disable *.pyc files
+    sys.dont_write_bytecode = True
+
+    # Add ".." to module search path
+    cur_dir = os.path.dirname(os.path.realpath(__file__))
+    top_dir = os.path.abspath(os.path.join(cur_dir, os.pardir))
+    sys.path.append(top_dir)
+
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="verbosity level, use: [-v | -vv | -vvv]",
+    )
+    parser.add_argument(
+        "-s", "--start-directory", default=None, help="directory to start discovery"
+    )
+    parser.add_argument(
+        "-p",
+        "--pattern",
+        default="test*.py",
+        help="pattern to match test files ('test*.py' default)",
+    )
+    parser.add_argument(
+        "test", nargs="*", help="test specs (e.g. module.TestCase.test_func)"
+    )
+    args = parser.parse_args()
+
+    if not args.start_directory:
+        args.start_directory = cur_dir
+
+    if args.verbose > 2:
+        logging.basicConfig(level=logging.DEBUG, format="DEBUG: %(message)s")
+
+    loader = unittest.TestLoader()
+    if args.test:
+        # Add particular tests
+        for test in args.test:
+            suite = unittest.TestSuite()
+            suite.addTests(loader.loadTestsFromName(test))
+    else:
+        # Find all tests
+        suite = loader.discover(args.start_directory, args.pattern)
+
+    runner = unittest.TextTestRunner(verbosity=args.verbose)
+    result = runner.run(suite)
+    return result.wasSuccessful()
+
+
+if __name__ == "__main__":
+    # NOTE: True(success) -> 0, False(fail) -> 1
+    exit(not main())

tools/python/defs.bzl

@@ -0,0 +1,237 @@
+load("@arrdem_source_pypi//:requirements.bzl",
+     _py_requirement = "requirement"
+)
+
+load("@rules_python//python:defs.bzl",
+     "py_runtime",
+     "py_runtime_pair",
+     _py_binary = "py_binary",
+     _py_test = "py_test",
+     _py_library = "py_library",
+)
+
+load("@bazel_skylib//lib:sets.bzl", "sets")
+
+
+def py_requirement(*args, **kwargs):
+    """A re-export of requirement()"""
+    return _py_requirement(*args, **kwargs)
+
+
+def py_test(python_version=None, **kwargs):
+    """A re-export of py_test()"""
+
+    if python_version and python_version != "PY3":
+        fail("py3k only!")
+
+    return _py_test(
+        python_version="PY3",
+        **kwargs,
+    )
+
+
+def py_pytest(name, srcs, deps, main=None, python_version=None, args=None, **kwargs):
+    """A py_test target which uses pytest."""
+
+    if python_version and python_version != "PY3":
+        fail("py3k only!")
+
+    f = "//tools/python:bzl_pytest_shim.py"
+
+    deps = sets.to_list(sets.make([
+        py_requirement("pytest"),
+        py_requirement("pytest-pudb"),
+        py_requirement("pytest-cov"),
+        py_requirement("pytest-timeout"),
+    ] + deps))
+
+    srcs = [f] + srcs
+
+    py_test(
+      name = name,
+      srcs = srcs,
+      main = f,
+      args = args,
+      python_version="PY3",
+      deps = deps,
+      **kwargs,
+    )
+
+    # zapp_test(
+    #   name = name + ".zapp",
+    #   main = f,
+    #   args = args,
+    #   srcs = srcs,
+    #   deps = deps,
+    #   test = True,
+    #   zip_safe = False,
+    #   **kwargs,
+    # )
+
+    # FIXME (arrdem 2020-09-27):
+    #   Generate a py_image_test.
+    #   Not clear how to achieve that.
+
+
+def py_unittest(srcs=[], **kwargs):
+    """A helper for running unittest tests"""
+
+    f = "//tools/python:bzl_unittest_shim.py"
+    return py_test(
+        main = f,
+        srcs = [f] + srcs,
+        **kwargs
+    )
+
+
+def py_binary(python_version=None, main=None, srcs=None, **kwargs):
+    """A re-export of py_binary()"""
+
+    if python_version and python_version != "PY3":
+        fail("py3k only!")
+
+    srcs = srcs or []
+    if main not in srcs:
+        srcs = [main] + srcs
+
+    return _py_binary(
+        python_version = "PY3",
+        main = main,
+        srcs = srcs,
+        **kwargs,
+    )
+
+
+def py_library(srcs_version=None, **kwargs):
+    """A re-export of py_library()"""
+
+    if srcs_version and srcs_version != "PY3":
+        fail("py3k only!")
+
+    return _py_library(
+        srcs_version="PY3",
+        **kwargs
+    )
+
+
+ResourceGroupInfo = provider(
+    fields = {
+        "srcs": "files to use from Python",
+    },
+)
+
+
+def _resource_impl(ctx):
+    srcs = []
+    for target in ctx.attr.srcs:
+        srcs.extend(target.files.to_list())
+    transitive_srcs = depset(direct = srcs)
+
+    return [
+        ResourceGroupInfo(
+            srcs = ctx.attr.srcs,
+        ),
+        PyInfo(
+            has_py2_only_sources = False,
+            has_py3_only_sources = True,
+            uses_shared_libraries = False,
+            transitive_sources = transitive_srcs,
+        ),
+    ]
+
+py_resources = rule(
+    implementation = _resource_impl,
+    attrs = {
+        "srcs": attr.label_list(
+            allow_empty = True,
+            mandatory = True,
+            allow_files = True,
+            doc = "Files to hand through to Python",
+        ),
+    },
+)
+
+def py_project(name=None,
+               main=None,
+               main_deps=None,
+               shebang=None,
+               lib_srcs=None,
+               lib_deps=None,
+               lib_data=None,
+               test_srcs=None,
+               test_deps=None,
+               test_data=None):
+    """
+    A helper for defining conventionally-formatted python project.
+
+    Assumes that there's a {src,test}/{resources,python} where src/ is a library and test/ is local tests only.
+
+    Each test_*.py source generates its own implicit test target. This allows for automatic test parallelism. Non
+    test_*.py files are implicitly srcs for the generated test targets. This is the same as making them implicitly a
+    testonly lib.
+
+    """
+
+    lib_srcs = lib_srcs or native.glob(["src/python/**/*.py"],
+                                       exclude=[
+                                           "**/*.pyc",
+                                       ])
+    lib_data = lib_data or native.glob(["src/resources/**/*",
+                                        "src/python/**/*"],
+                                       exclude=[
+                                           "**/*.py",
+                                           "**/*.pyc",
+                                       ])
+    test_srcs = test_srcs or native.glob(["test/python/**/*.py"],
+                                         exclude=[
+                                             "**/*.pyc",
+                                         ])
+    test_data = test_data or native.glob(["test/resources/**/*",
+                                          "test/python/**/*"],
+                                         exclude=[
+                                             "**/*.py",
+                                             "**/*.pyc",
+                                         ])
+
+    lib_name = name if not main else "lib"
+
+    py_library(
+        name=lib_name,
+        srcs=lib_srcs,
+        deps=lib_deps,
+        data=lib_data,
+        imports=[
+            "src/python",
+            "src/resources",
+        ],
+        visibility = [
+            "//visibility:public",
+        ],
+    )
+
+    if main:
+        py_binary(
+            name=name,
+            main=main,
+            deps=(main_deps or []) + [lib_name],
+            imports=[
+                "src/python",
+                "src/resources",
+            ],
+            visibility = [
+                "//visibility:public",
+            ],
+        )
+
+    for src in test_srcs:
+        if "test_" in src:
+            py_pytest(
+                name=src.split("/")[-1],
+                srcs=[src] + [f for f in test_srcs if "test_" not in f],
+                deps=[lib_name] + (test_deps or []),
+                data=test_data,
+                imports=[
+                    "test/python",
+                    "test/resources",
+                ],
+            )

tools/python/pythonshim

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Bazel STRONGLY disapproves of linking dynamically to a Python interpreter.
+# But ... that's exactly what we want to do.
+# So this script exists to find a 'compliant' Python install and use that.
+
+PYTHONREV="3.10"
+CMD="python${PYTHONREV}"
+
+if [ -x "$(command -v "$CMD")" ]; then
+    exec "$(which "$CMD")" "$@"
+else
+    case "$(uname)" in
+        Darwin)
+            # FIXME: What if it isn't there?
+            exec /opt/homebrew/bin/"$CMD" "$@"
+            ;;
+    esac
+    echo "Error: Unable to find a viable Python executable" >&2
+    exit 1
+fi

tools/python/requirements.in

@@ -0,0 +1,3 @@
+attrs
+cattrs
+black

tools/python/test_licenses.py

@@ -0,0 +1,144 @@
+"""
+Validate 3rdparty library licenses as approved.
+"""
+
+import re
+
+from pkg_resources import (
+    DistInfoDistribution,
+    working_set,
+)
+import pytest
+
+
+# Licenses approved as representing non-copyleft and not precluding commercial usage.
+# This is all easy, there's a good schema here.
+APPROVED_LICENSES = [
+    MIT := "License :: OSI Approved :: MIT License",
+    APACHE := "License :: OSI Approved :: Apache Software License",
+    BSD := "License :: OSI Approved :: BSD License",
+    MPL10 := "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
+    MPL11 := "License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)",
+    MPL20 := "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    PSFL := "License :: OSI Approved :: Python Software Foundation License",
+    LGPL := "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)",
+    LGPL3 := "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)",
+    ISCL := "License :: OSI Approved :: ISC License (ISCL)",
+]
+
+UNAPPROVED_LICENSES = [
+    GPL1 := "License :: OSI Approved :: GNU General Public License",
+    GPL2 := "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
+    GPL3 := "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+]
+
+# This data is GARBO.
+LICENSES_BY_LOWERNAME = {
+    "apache 2.0": APACHE,
+    "apache": APACHE,
+    "http://www.apache.org/licenses/license-2.0": APACHE,
+    "bsd 3": BSD,
+    "bsd": BSD,
+    "gpl": GPL1,
+    "gpl2": GPL2,
+    "gpl3": GPL3,
+    "lgpl": LGPL,
+    "lgpl3": LGPL3,
+    "isc": ISCL,
+    "mit": MIT,
+    "mpl": MPL10,
+    "mpl 2.0": MPL20,
+    "psf": PSFL,
+}
+
+# Mash in some cases.
+LICENSES_BY_LOWERNAME.update(
+    {lic.split(" :: ")[-1].lower(): lic for lic in APPROVED_LICENSES}
+)
+
+# As a workaround for packages which don"t have correct meadata on PyPi, hand-verified packages
+APPROVED_PACKAGES = [
+    "yamllint",  # WARNING: YAMLLINT IS GLP3"d.
+    "Flask_Log_Request_ID",  # MIT, currently depended on as a git dep.
+    "anosql",  # BSD
+]
+
+
+def bash_license(ln):
+    while True:
+        lnn = re.sub(
+            r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)|(or later)",
+            "",
+            ln.lower(),
+        )
+        if ln != lnn:
+            ln = lnn
+        else:
+            break
+
+    ln = LICENSES_BY_LOWERNAME.get(ln, ln)
+    return ln
+
+
+@pytest.mark.parametrize(
+    "a,b",
+    [
+        ("MIT", MIT),
+        ("mit", MIT),
+        ("BSD", BSD),
+        ("BSD 3-clause", BSD),
+        ("BSD 3 clause", BSD),
+        ("GPL3", GPL3),
+        ("GPL v3", GPL3),
+        ("GPLv3", GPL3),
+    ],
+)
+def test_bash_license(a, b):
+    assert bash_license(a) == b
+
+
+def licenses(dist: DistInfoDistribution):
+    """Get dist metadata (the licenses list) from PyPi.
+
+    pip and other tools use the local dist metadata to introspect licenses which requires that
+    packages be installed. Going to PyPi isn't strictly reproducible both because the PyPi database
+    could be updated and we could see network failures but there really isn't a good way to solve
+    this problem.
+
+    """
+
+    lics = []
+    name = dist.project_name
+    version = dist.version
+    print(name, version, type(dist))
+
+    meta = dist.get_metadata(dist.PKG_INFO).split("\n")
+    classifiers = [
+        l.replace("Classifier: ", "", 1) for l in meta if l.startswith("Classifier: ")
+    ]
+    license = bash_license(
+        next((l for l in meta if l.startswith("License:")), "License: UNKNOWN").replace(
+            "License: ", "", 1
+        )
+    )
+    lics.extend(l for l in classifiers if l.startswith("License ::"))
+
+    if not lics:
+        lics.append(license)
+
+    return lics
+
+
+@pytest.mark.parametrize(
+    "dist",
+    (w for w in working_set if w.location.find("arrdem_source_pypi") != -1),
+    ids=lambda dist: dist.project_name,
+)
+def test_approved_license(dist: DistInfoDistribution):
+    """Ensure that a given package is either allowed by name or uses an approved license."""
+
+    _licenses = licenses(dist)
+    print(dist.location)
+    assert dist.project_name in APPROVED_PACKAGES or any(
+        lic in APPROVED_LICENSES for lic in _licenses
+    ), f"{dist.project_name} ({dist.location}) was not approved and its license(s) were unknown {_licenses!r}"