From 471af02d9bdb5319da6c5f3f1e25c42db8d23480 Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Wed, 8 Mar 2023 15:11:09 -0700 Subject: [PATCH] Import a bunch of my Bazel infrastructure --- MODULE.bazel | 24 +++ WORKSPACE | 16 ++ tools/build_rules/BUILD | 3 + tools/build_rules/cp.bzl | 47 ++++++ tools/build_rules/prelude_bazel | 24 +++ tools/python/BUILD | 47 ++++++ tools/python/bzl_pytest_shim.py | 11 ++ tools/python/bzl_unittest_shim.py | 66 ++++++++ tools/python/defs.bzl | 237 +++++++++++++++++++++++++++++ tools/python/pythonshim | 21 +++ tools/python/requirements.in | 3 + tools/python/requirements_lock.txt | 0 tools/python/test_licenses.py | 144 ++++++++++++++++++ 13 files changed, 643 insertions(+) create mode 100644 MODULE.bazel create mode 100644 WORKSPACE create mode 100644 tools/build_rules/BUILD create mode 100644 tools/build_rules/cp.bzl create mode 100644 tools/build_rules/prelude_bazel create mode 100644 tools/python/BUILD create mode 100644 tools/python/bzl_pytest_shim.py create mode 100644 tools/python/bzl_unittest_shim.py create mode 100644 tools/python/defs.bzl create mode 100755 tools/python/pythonshim create mode 100644 tools/python/requirements.in create mode 100644 tools/python/requirements_lock.txt create mode 100644 tools/python/test_licenses.py diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000..57257d5 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,24 @@ +bazel_dep(name = "rules_python", version = "0.19.0") + +pip = use_extension("@rules_python//python:extensions.bzl", "pip") + +pip.parse( + name = "pypa", + requirements_lock = "//tools/python:requirements_lock.txt", +) + +use_repo(pip, "pypa") + +# (Optional) Register a specific python toolchain instead of using the host version +python = use_extension("@rules_python//python:extensions.bzl", "python") + +python.toolchain( + name = "python3_10", + python_version = "3.10", +) + +use_repo(python, "python3_10_toolchains") + +register_toolchains( + "@python3_10_toolchains//:all", +) diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..85830e1 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,16 @@ +workspace( + name = "arrdem_flowmetal", +) + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_python", + sha256 = "ffc7b877c95413c82bfd5482c017edcf759a6250d8b24e82f41f3c8b8d9e287e", + strip_prefix = "rules_python-0.19.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.19.0/rules_python-0.19.0.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() diff --git a/tools/build_rules/BUILD b/tools/build_rules/BUILD new file mode 100644 index 0000000..e6d05e7 --- /dev/null +++ b/tools/build_rules/BUILD @@ -0,0 +1,3 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) diff --git a/tools/build_rules/cp.bzl b/tools/build_rules/cp.bzl new file mode 100644 index 0000000..f006f69 --- /dev/null +++ b/tools/build_rules/cp.bzl @@ -0,0 +1,47 @@ +load("@bazel_skylib//rules:copy_file.bzl", + "copy_file", +) + +def cp(name, src, **kwargs): + """A slightly more convenient cp() rule. Name and out should always be the same.""" + + rule_name = name.replace(".", "_").replace(":", "/").replace("//", "").replace("/", "_") + copy_file( + name = rule_name, + src = src, + out = name, + **kwargs + ) + return rule_name + + +def _copy_filegroup_impl(ctx): + all_outputs = [] + for t in ctx.attr.deps: + t_prefix = t.label.package + for f in t.files.to_list(): + # Strip out the source prefix... + path = f.short_path.replace(t_prefix + "/", "") + out = ctx.actions.declare_file(path) + print(ctx.attr.name, t.label, f, " => ", path) + all_outputs += [out] + ctx.actions.run_shell( + outputs=[out], + inputs=depset([f]), + arguments=[f.path, out.path], + command="cp $1 $2" + ) + + return [ + DefaultInfo( + files=depset(all_outputs), + runfiles=ctx.runfiles(files=all_outputs)) + ] + + +copy_filegroups = rule( + implementation=_copy_filegroup_impl, + attrs={ + "deps": attr.label_list(), + }, +) diff --git a/tools/build_rules/prelude_bazel b/tools/build_rules/prelude_bazel new file mode 100644 index 0000000..4947b73 --- /dev/null +++ b/tools/build_rules/prelude_bazel @@ -0,0 +1,24 @@ +# -*- mode: bazel -*- +# A global prelude for all BUILD[.bazel] files + +load("//tools/python:defs.bzl", + "py_library", + "py_binary", + "py_unittest", + "py_pytest", + "py_resources", + "py_project", +) + +load("@pypa//:requirements.bzl", + py_requirement="requirement" +) + +load("@bazel_skylib//rules:copy_file.bzl", + "copy_file", +) + +load("//tools/build_rules:cp.bzl", + "cp", + "copy_filegroups" +) diff --git a/tools/python/BUILD b/tools/python/BUILD new file mode 100644 index 0000000..181f52e --- /dev/null +++ b/tools/python/BUILD @@ -0,0 +1,47 @@ +load("@rules_python//python:defs.bzl", + "py_runtime_pair", +) + +load("@arrdem_source_pypi//:requirements.bzl", "all_requirements") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +exports_files([ + "defs.bzl", + "bzl_pytest_shim.py", + "bzl_unittest_shim.py", + "pythonshim", +]) + +py_runtime( + name = "python3_runtime", + files = [], + interpreter = ":pythonshim", + python_version = "PY3", + visibility = ["//visibility:public"], +) + +py_runtime_pair( + name = "python_runtime", + py2_runtime = None, + py3_runtime = ":python3_runtime", +) + +toolchain( + name = "python3_toolchain", + toolchain = ":python_runtime", + toolchain_type = "@bazel_tools//tools/python:toolchain_type", +) + +py_pytest( + name = "test_licenses", + srcs = [ + "test_licenses.py", + ], + data = [ + "requirements.txt", + ], + deps = all_requirements, +) diff --git a/tools/python/bzl_pytest_shim.py b/tools/python/bzl_pytest_shim.py new file mode 100644 index 0000000..5a65b98 --- /dev/null +++ b/tools/python/bzl_pytest_shim.py @@ -0,0 +1,11 @@ +"""A shim for executing pytest.""" + +import sys + +import pytest + + +if __name__ == "__main__": + cmdline = ["--ignore=external"] + sys.argv[1:] + print(cmdline, file=sys.stderr) + sys.exit(pytest.main(cmdline)) diff --git a/tools/python/bzl_unittest_shim.py b/tools/python/bzl_unittest_shim.py new file mode 100644 index 0000000..c21ea4e --- /dev/null +++ b/tools/python/bzl_unittest_shim.py @@ -0,0 +1,66 @@ +"""Universal launcher for unit tests""" + +import argparse +import logging +import os +import sys +import unittest + + +def main(): + """Parse args, collect tests and run them""" + # Disable *.pyc files + sys.dont_write_bytecode = True + + # Add ".." to module search path + cur_dir = os.path.dirname(os.path.realpath(__file__)) + top_dir = os.path.abspath(os.path.join(cur_dir, os.pardir)) + sys.path.append(top_dir) + + # Parse command line arguments + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="verbosity level, use: [-v | -vv | -vvv]", + ) + parser.add_argument( + "-s", "--start-directory", default=None, help="directory to start discovery" + ) + parser.add_argument( + "-p", + "--pattern", + default="test*.py", + help="pattern to match test files ('test*.py' default)", + ) + parser.add_argument( + "test", nargs="*", help="test specs (e.g. module.TestCase.test_func)" + ) + args = parser.parse_args() + + if not args.start_directory: + args.start_directory = cur_dir + + if args.verbose > 2: + logging.basicConfig(level=logging.DEBUG, format="DEBUG: %(message)s") + + loader = unittest.TestLoader() + if args.test: + # Add particular tests + for test in args.test: + suite = unittest.TestSuite() + suite.addTests(loader.loadTestsFromName(test)) + else: + # Find all tests + suite = loader.discover(args.start_directory, args.pattern) + + runner = unittest.TextTestRunner(verbosity=args.verbose) + result = runner.run(suite) + return result.wasSuccessful() + + +if __name__ == "__main__": + # NOTE: True(success) -> 0, False(fail) -> 1 + exit(not main()) diff --git a/tools/python/defs.bzl b/tools/python/defs.bzl new file mode 100644 index 0000000..70bcf57 --- /dev/null +++ b/tools/python/defs.bzl @@ -0,0 +1,237 @@ +load("@arrdem_source_pypi//:requirements.bzl", + _py_requirement = "requirement" +) + +load("@rules_python//python:defs.bzl", + "py_runtime", + "py_runtime_pair", + _py_binary = "py_binary", + _py_test = "py_test", + _py_library = "py_library", +) + +load("@bazel_skylib//lib:sets.bzl", "sets") + + +def py_requirement(*args, **kwargs): + """A re-export of requirement()""" + return _py_requirement(*args, **kwargs) + + +def py_test(python_version=None, **kwargs): + """A re-export of py_test()""" + + if python_version and python_version != "PY3": + fail("py3k only!") + + return _py_test( + python_version="PY3", + **kwargs, + ) + + +def py_pytest(name, srcs, deps, main=None, python_version=None, args=None, **kwargs): + """A py_test target which uses pytest.""" + + if python_version and python_version != "PY3": + fail("py3k only!") + + f = "//tools/python:bzl_pytest_shim.py" + + deps = sets.to_list(sets.make([ + py_requirement("pytest"), + py_requirement("pytest-pudb"), + py_requirement("pytest-cov"), + py_requirement("pytest-timeout"), + ] + deps)) + + srcs = [f] + srcs + + py_test( + name = name, + srcs = srcs, + main = f, + args = args, + python_version="PY3", + deps = deps, + **kwargs, + ) + + # zapp_test( + # name = name + ".zapp", + # main = f, + # args = args, + # srcs = srcs, + # deps = deps, + # test = True, + # zip_safe = False, + # **kwargs, + # ) + + # FIXME (arrdem 2020-09-27): + # Generate a py_image_test. + # Not clear how to achieve that. + + +def py_unittest(srcs=[], **kwargs): + """A helper for running unittest tests""" + + f = "//tools/python:bzl_unittest_shim.py" + return py_test( + main = f, + srcs = [f] + srcs, + **kwargs + ) + + +def py_binary(python_version=None, main=None, srcs=None, **kwargs): + """A re-export of py_binary()""" + + if python_version and python_version != "PY3": + fail("py3k only!") + + srcs = srcs or [] + if main not in srcs: + srcs = [main] + srcs + + return _py_binary( + python_version = "PY3", + main = main, + srcs = srcs, + **kwargs, + ) + + +def py_library(srcs_version=None, **kwargs): + """A re-export of py_library()""" + + if srcs_version and srcs_version != "PY3": + fail("py3k only!") + + return _py_library( + srcs_version="PY3", + **kwargs + ) + + +ResourceGroupInfo = provider( + fields = { + "srcs": "files to use from Python", + }, +) + + +def _resource_impl(ctx): + srcs = [] + for target in ctx.attr.srcs: + srcs.extend(target.files.to_list()) + transitive_srcs = depset(direct = srcs) + + return [ + ResourceGroupInfo( + srcs = ctx.attr.srcs, + ), + PyInfo( + has_py2_only_sources = False, + has_py3_only_sources = True, + uses_shared_libraries = False, + transitive_sources = transitive_srcs, + ), + ] + +py_resources = rule( + implementation = _resource_impl, + attrs = { + "srcs": attr.label_list( + allow_empty = True, + mandatory = True, + allow_files = True, + doc = "Files to hand through to Python", + ), + }, +) + +def py_project(name=None, + main=None, + main_deps=None, + shebang=None, + lib_srcs=None, + lib_deps=None, + lib_data=None, + test_srcs=None, + test_deps=None, + test_data=None): + """ + A helper for defining conventionally-formatted python project. + + Assumes that there's a {src,test}/{resources,python} where src/ is a library and test/ is local tests only. + + Each test_*.py source generates its own implicit test target. This allows for automatic test parallelism. Non + test_*.py files are implicitly srcs for the generated test targets. This is the same as making them implicitly a + testonly lib. + + """ + + lib_srcs = lib_srcs or native.glob(["src/python/**/*.py"], + exclude=[ + "**/*.pyc", + ]) + lib_data = lib_data or native.glob(["src/resources/**/*", + "src/python/**/*"], + exclude=[ + "**/*.py", + "**/*.pyc", + ]) + test_srcs = test_srcs or native.glob(["test/python/**/*.py"], + exclude=[ + "**/*.pyc", + ]) + test_data = test_data or native.glob(["test/resources/**/*", + "test/python/**/*"], + exclude=[ + "**/*.py", + "**/*.pyc", + ]) + + lib_name = name if not main else "lib" + + py_library( + name=lib_name, + srcs=lib_srcs, + deps=lib_deps, + data=lib_data, + imports=[ + "src/python", + "src/resources", + ], + visibility = [ + "//visibility:public", + ], + ) + + if main: + py_binary( + name=name, + main=main, + deps=(main_deps or []) + [lib_name], + imports=[ + "src/python", + "src/resources", + ], + visibility = [ + "//visibility:public", + ], + ) + + for src in test_srcs: + if "test_" in src: + py_pytest( + name=src.split("/")[-1], + srcs=[src] + [f for f in test_srcs if "test_" not in f], + deps=[lib_name] + (test_deps or []), + data=test_data, + imports=[ + "test/python", + "test/resources", + ], + ) diff --git a/tools/python/pythonshim b/tools/python/pythonshim new file mode 100755 index 0000000..a3a0a7e --- /dev/null +++ b/tools/python/pythonshim @@ -0,0 +1,21 @@ +#!/bin/sh + +# Bazel STRONGLY disapproves of linking dynamically to a Python interpreter. +# But ... that's exactly what we want to do. +# So this script exists to find a 'compliant' Python install and use that. + +PYTHONREV="3.10" +CMD="python${PYTHONREV}" + +if [ -x "$(command -v "$CMD")" ]; then + exec "$(which "$CMD")" "$@" +else + case "$(uname)" in + Darwin) + # FIXME: What if it isn't there? + exec /opt/homebrew/bin/"$CMD" "$@" + ;; + esac + echo "Error: Unable to find a viable Python executable" >&2 + exit 1 +fi diff --git a/tools/python/requirements.in b/tools/python/requirements.in new file mode 100644 index 0000000..ade5670 --- /dev/null +++ b/tools/python/requirements.in @@ -0,0 +1,3 @@ +attrs +cattrs +black diff --git a/tools/python/requirements_lock.txt b/tools/python/requirements_lock.txt new file mode 100644 index 0000000..e69de29 diff --git a/tools/python/test_licenses.py b/tools/python/test_licenses.py new file mode 100644 index 0000000..32e7246 --- /dev/null +++ b/tools/python/test_licenses.py @@ -0,0 +1,144 @@ +""" +Validate 3rdparty library licenses as approved. +""" + +import re + +from pkg_resources import ( + DistInfoDistribution, + working_set, +) +import pytest + + +# Licenses approved as representing non-copyleft and not precluding commercial usage. +# This is all easy, there's a good schema here. +APPROVED_LICENSES = [ + MIT := "License :: OSI Approved :: MIT License", + APACHE := "License :: OSI Approved :: Apache Software License", + BSD := "License :: OSI Approved :: BSD License", + MPL10 := "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)", + MPL11 := "License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)", + MPL20 := "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + PSFL := "License :: OSI Approved :: Python Software Foundation License", + LGPL := "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)", + LGPL3 := "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + ISCL := "License :: OSI Approved :: ISC License (ISCL)", +] + +UNAPPROVED_LICENSES = [ + GPL1 := "License :: OSI Approved :: GNU General Public License", + GPL2 := "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", + GPL3 := "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", +] + +# This data is GARBO. +LICENSES_BY_LOWERNAME = { + "apache 2.0": APACHE, + "apache": APACHE, + "http://www.apache.org/licenses/license-2.0": APACHE, + "bsd 3": BSD, + "bsd": BSD, + "gpl": GPL1, + "gpl2": GPL2, + "gpl3": GPL3, + "lgpl": LGPL, + "lgpl3": LGPL3, + "isc": ISCL, + "mit": MIT, + "mpl": MPL10, + "mpl 2.0": MPL20, + "psf": PSFL, +} + +# Mash in some cases. +LICENSES_BY_LOWERNAME.update( + {lic.split(" :: ")[-1].lower(): lic for lic in APPROVED_LICENSES} +) + +# As a workaround for packages which don"t have correct meadata on PyPi, hand-verified packages +APPROVED_PACKAGES = [ + "yamllint", # WARNING: YAMLLINT IS GLP3"d. + "Flask_Log_Request_ID", # MIT, currently depended on as a git dep. + "anosql", # BSD +] + + +def bash_license(ln): + while True: + lnn = re.sub( + r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)|(or later)", + "", + ln.lower(), + ) + if ln != lnn: + ln = lnn + else: + break + + ln = LICENSES_BY_LOWERNAME.get(ln, ln) + return ln + + +@pytest.mark.parametrize( + "a,b", + [ + ("MIT", MIT), + ("mit", MIT), + ("BSD", BSD), + ("BSD 3-clause", BSD), + ("BSD 3 clause", BSD), + ("GPL3", GPL3), + ("GPL v3", GPL3), + ("GPLv3", GPL3), + ], +) +def test_bash_license(a, b): + assert bash_license(a) == b + + +def licenses(dist: DistInfoDistribution): + """Get dist metadata (the licenses list) from PyPi. + + pip and other tools use the local dist metadata to introspect licenses which requires that + packages be installed. Going to PyPi isn't strictly reproducible both because the PyPi database + could be updated and we could see network failures but there really isn't a good way to solve + this problem. + + """ + + lics = [] + name = dist.project_name + version = dist.version + print(name, version, type(dist)) + + meta = dist.get_metadata(dist.PKG_INFO).split("\n") + classifiers = [ + l.replace("Classifier: ", "", 1) for l in meta if l.startswith("Classifier: ") + ] + license = bash_license( + next((l for l in meta if l.startswith("License:")), "License: UNKNOWN").replace( + "License: ", "", 1 + ) + ) + lics.extend(l for l in classifiers if l.startswith("License ::")) + + if not lics: + lics.append(license) + + return lics + + +@pytest.mark.parametrize( + "dist", + (w for w in working_set if w.location.find("arrdem_source_pypi") != -1), + ids=lambda dist: dist.project_name, +) +def test_approved_license(dist: DistInfoDistribution): + """Ensure that a given package is either allowed by name or uses an approved license.""" + + _licenses = licenses(dist) + print(dist.location) + assert dist.project_name in APPROVED_PACKAGES or any( + lic in APPROVED_LICENSES for lic in _licenses + ), f"{dist.project_name} ({dist.location}) was not approved and its license(s) were unknown {_licenses!r}"