From dea064ff13043918e4ca4e8eb962911b84d1aa41 Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Sun, 19 Sep 2021 17:59:18 -0600 Subject: [PATCH] Get test_licenses using pkg_info ala liccheck Fixes #2 --- WORKSPACE | 14 +++++-- tools/black/__main__.py | 2 +- tools/python/BUILD | 9 ++--- tools/python/defs.bzl | 13 +++++-- tools/python/requirements.txt | 4 +- tools/python/test_licenses.py | 69 +++++++++++------------------------ 6 files changed, 48 insertions(+), 63 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 23bfb4c..1f73504 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -40,7 +40,7 @@ bazel_skylib_workspace() git_repository( name = "rules_python", remote = "https://github.com/bazelbuild/rules_python.git", - tag = "0.3.0", + tag = "0.4.0", # commit = "...", ) @@ -48,14 +48,20 @@ register_toolchains("//tools/python:python3_toolchain") # pip package pinnings need to be initialized. # this generates a bunch of bzl rules so that each pip dep is a bzl target -load("@rules_python//python:pip.bzl", "pip_install") +load("@rules_python//python:pip.bzl", "pip_parse") -pip_install( +pip_parse( name = "arrdem_source_pypi", - requirements = "//tools/python:requirements.txt", + requirements_lock = "//tools/python:requirements.txt", python_interpreter = "/usr/bin/python3.9", ) +# Load the starlark macro which will define your dependencies. +load("@arrdem_source_pypi//:requirements.bzl", "install_deps") + +# Call it to define repos for your requirements. +install_deps() + # git_repository( # name = "rules_zapp", # remote = "https://github.com/arrdem/rules_zapp.git", diff --git a/tools/black/__main__.py b/tools/black/__main__.py index cff5a1c..238de21 100644 --- a/tools/black/__main__.py +++ b/tools/black/__main__.py @@ -3,7 +3,7 @@ import argparse import sys -from black import patched_main, nullcontext +from black import nullcontext, patched_main parser = argparse.ArgumentParser() diff --git a/tools/python/BUILD b/tools/python/BUILD index 024e500..29ea757 100644 --- a/tools/python/BUILD +++ b/tools/python/BUILD @@ -1,7 +1,9 @@ load("@rules_python//python:defs.bzl", - "py_runtime_pair" + "py_runtime_pair", ) +load("@arrdem_source_pypi//:requirements.bzl", "all_requirements") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) @@ -40,8 +42,5 @@ py_pytest( data = [ "requirements.txt", ], - deps = [ - py_requirement("requests"), - py_requirement("requirements-parser"), - ] + deps = all_requirements, ) diff --git a/tools/python/defs.bzl b/tools/python/defs.bzl index 862685e..eb42cdd 100644 --- a/tools/python/defs.bzl +++ b/tools/python/defs.bzl @@ -14,6 +14,9 @@ load("@rules_zapp//zapp:zapp.bzl", "zapp_binary", ) +load("@bazel_skylib//lib:sets.bzl", "sets") + + def py_requirement(*args, **kwargs): """A re-export of requirement()""" return _py_requirement(*args, **kwargs) @@ -39,11 +42,11 @@ def py_pytest(name, srcs, deps, main=None, python_version=None, args=None, **kwa f = "//tools/python:bzl_pytest_shim.py" - deps = [ + deps = sets.to_list(sets.make([ py_requirement("pytest"), py_requirement("jedi"), py_requirement("pytest-pudb"), - ] + deps + ] + deps)) srcs = [f] + srcs @@ -60,12 +63,14 @@ def py_pytest(name, srcs, deps, main=None, python_version=None, args=None, **kwa # FIXME (arrdem 2020-09-27): # This really needs to be a py_image_test. # Not clear how to achieve that. - # py_image( - # name = name + ".containerized", + # zapp_binary( + # name = name + ".hermetic", # main = f, # args = args, # srcs = srcs, # deps = deps, + # test = True, + # zip_safe = False, # **kwargs, # ) diff --git a/tools/python/requirements.txt b/tools/python/requirements.txt index 698e580..4d601dc 100644 --- a/tools/python/requirements.txt +++ b/tools/python/requirements.txt @@ -45,7 +45,7 @@ packaging==20.9 parso==0.8.2 pathspec==0.9.0 pep517==0.11.0 -piexif==1.1.3 +pip==20.3.3 pip-tools==6.2.0 platformdirs==2.3.0 pluggy==0.13.1 @@ -73,6 +73,7 @@ regex==2021.8.28 requests==2.25.1 requests-toolbelt==0.9.1 requirements-parser==0.2.0 +setuptools==51.0.0 six==1.15.0 snowballstemmer==2.1.0 sortedcontainers==2.3.0 @@ -99,6 +100,7 @@ urwid==2.1.2 wcwidth==0.2.5 webencodings==0.5.1 Werkzeug==2.0.1 +wheel==0.36.2 yamllint==1.26.1 yarl==1.6.3 yaspin==1.5.0 diff --git a/tools/python/test_licenses.py b/tools/python/test_licenses.py index 804b8f0..22f43f2 100644 --- a/tools/python/test_licenses.py +++ b/tools/python/test_licenses.py @@ -4,10 +4,8 @@ Validate 3rdparty library licenses as approved. import re +from pkg_resources import DistInfoDistribution, working_set import pytest -import requests -import requirements -from requirements.requirement import Requirement # Licenses approved as representing non-copyleft and not precluding commercial usage. @@ -63,14 +61,10 @@ APPROVED_PACKAGES = [ ] -with open("tools/python/requirements.txt") as fd: - PACKAGES = list(requirements.parse(fd)) - - def bash_license(ln): while True: lnn = re.sub( - r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)", "", ln.lower() + r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)|(or later)", "", ln.lower() ) if ln != lnn: ln = lnn @@ -98,59 +92,38 @@ def test_bash_license(a, b): assert bash_license(a) == b -def licenses(package: Requirement): - """Get package metadata (the licenses list) from PyPi. +def licenses(dist: DistInfoDistribution): + """Get dist metadata (the licenses list) from PyPi. - pip and other tools use the local package metadata to introspect licenses which requires that + pip and other tools use the local dist metadata to introspect licenses which requires that packages be installed. Going to PyPi isn't strictly reproducible both because the PyPi database could be updated and we could see network failures but there really isn't a good way to solve this problem. """ + lics = [] - version = next((v for op, v in package.specs if op == "=="), None) - print(package.name, version) + name = dist.project_name + version = dist.version + print(name, version, type(dist)) - # If we don't have a version (eg. forked git dep) assume we've got the same license constraints - # as the latest upstream release. After all we can't re-license stuff. - if not version: - blob = requests.get( - f"https://pypi.org/pypi/{package.name}/json", - headers={"Accept": "application/json"}, - ).json() - if ln := bash_license(blob.get("license")): - lics.append(ln) - else: - try: - version = list(blob.get("releases", {}).keys())[-1] - except IndexError: - pass + meta = dist.get_metadata(dist.PKG_INFO).split("\n") + classifiers = [l.replace("Classifier: ", "", 1) for l in meta if l.startswith("Classifier: ")] + license = bash_license(next((l for l in meta if l.startswith("License:")), "License: UNKNOWN").replace("License: ", "", 1)) + lics.extend(l for l in classifiers if l.startswith("License ::")) - # If we have a version, try to pull that release's metadata since it may have more/better. - if version: - blob = requests.get( - f"https://pypi.org/pypi/{package.name}/{version}/json", - headers={"Accept": "application/json"}, - ).json() - lics.extend( - [ - c - for c in blob.get("info", {}).get("classifiers", []) - if c.startswith("License") - ] - ) - ln = blob.get("info", {}).get("license") - if ln and not lics: - lics.append(bash_license(ln)) + if not lics: + lics.append(license) return lics -@pytest.mark.parametrize("package", PACKAGES) -def test_approved_license(package): +@pytest.mark.parametrize("dist", (w for w in working_set if w.location.find("arrdem_source_pypi") != -1), ids=lambda dist: dist.project_name) +def test_approved_license(dist: DistInfoDistribution): """Ensure that a given package is either allowed by name or uses an approved license.""" - _licenses = licenses(package) - assert package.name in APPROVED_PACKAGES or any( + _licenses = licenses(dist) + print(dist.location) + assert dist.project_name in APPROVED_PACKAGES or any( lic in APPROVED_LICENSES for lic in _licenses - ), f"{package} was not approved and its license(s) were unknown {_licenses!r}" + ), f"{dist.project_name} ({dist.location}) was not approved and its license(s) were unknown {_licenses!r}"