From fce09a6e7a7bca56871f68504504a4d83b9a6bc0 Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Wed, 4 Aug 2021 00:16:55 -0600 Subject: [PATCH] Chuck out reqsort for a stub of something more general --- projects/reqman/BUILD | 7 + projects/reqman/README.md | 6 + projects/reqman/src/python/reqman/__main__.py | 167 ++++++++++++++++++ projects/reqsort/BUILD | 10 -- projects/reqsort/README.md | 9 - .../reqsort/src/python/reqsort/__main__.py | 73 -------- tools/fmt.sh | 4 +- tools/lint.sh | 5 +- 8 files changed, 184 insertions(+), 97 deletions(-) create mode 100644 projects/reqman/BUILD create mode 100644 projects/reqman/README.md create mode 100644 projects/reqman/src/python/reqman/__main__.py delete mode 100644 projects/reqsort/BUILD delete mode 100644 projects/reqsort/README.md delete mode 100644 projects/reqsort/src/python/reqsort/__main__.py diff --git a/projects/reqman/BUILD b/projects/reqman/BUILD new file mode 100644 index 0000000..b356df7 --- /dev/null +++ b/projects/reqman/BUILD @@ -0,0 +1,7 @@ +py_binary( + name = "reqman", + main = "src/python/reqman/__main__.py", + deps = [ + py_requirement("click"), + ], +) diff --git a/projects/reqman/README.md b/projects/reqman/README.md new file mode 100644 index 0000000..6e47be2 --- /dev/null +++ b/projects/reqman/README.md @@ -0,0 +1,6 @@ +# Req[uirements] Man[ager] + +A tool designed to be a one-stop-shop interface for managing `requirements.txt` in a monorepo. +Provides a convenient way to check formatting, apply formatting, expunge unused requirements and add new requirements. + +Inspired in some part by the `pip-tools` project's `pip-compile`; but designed to not make you do any intermediate file dancing. diff --git a/projects/reqman/src/python/reqman/__main__.py b/projects/reqman/src/python/reqman/__main__.py new file mode 100644 index 0000000..b97cf84 --- /dev/null +++ b/projects/reqman/src/python/reqman/__main__.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +"""ReqMan; the Requirements Manager. + +Reqman is a quick and dirty tool that knows about Bazel, rules_python and requirements.txt files. It +exists to extend the upstream (and very good) pip-tools / pip-compile task with some helpers +providing better ergonomics for manipulating a monorepo. + +Usage: + reqman deps + Print out a summary of all available dependencies. + + reqman install + Install a new (pinned) dependency (and any transitives) into the locked requirements.txt + + reqman install --upgrade + Upgrade a given (pinned) dependency (and any transitives) + + reqman lint + Check for now-unused dependencies + + reqman clean + Rewrite the requirements to emove unused dependencies + +""" + +import re +import subprocess + +import click + + +REQ_PATTERN = re.compile( + r"(?P[a-zA-Z0-9_-]+)(?P\[.*?\])?(?P((==|>=?|<=?|~=|!=)(?P[^\s;#]+),)+)?|(.*?#egg=(?P[a-zA-Z0-9_-]+))" +) + + +SHITLIST = [ + "pip", + "pkg-resources", + "setuptools", +] + +def req_name(requirement: str) -> str: + requirement = requirement.lower() + match = re.match(REQ_PATTERN, requirement) + return match.group("pkgname") or match.group("eggname") + + +def sort_key(requirement: str) -> str: + return ( + req_name(requirement) # Get the match group + .lower() # We ignore case + .replace("-", "") # We ignore - + .replace("_", "") # And _ + ) + + +def _bq(query): + """Enumerate the PyPi package names from a Bazel query.""" + + unused = subprocess.check_output(["bazel", "query", query, "--output=package"]).decode("utf-8") + for l in unused.split("\n"): + if l: + yield l.replace("@arrdem_source_pypi//pypi__", "") + + +def _unused(): + """Find unused requirements.""" + + return set(_bq("@arrdem_source_pypi//...")) - set(_bq("filter('//pypi__', deps(//...))")) + + +def _load(fname): + """Slurp requirements from a file.""" + + with open(fname, "r") as reqfile: + # FIXME (arrdem 2021-08-03): + # Return a parse, not just lines. + return list(l.strip() for l in reqfile) + + +def _write(fname, reqs): + """Dump requirements back to a file in sorted format.""" + + reqs = sorted(reqs, key=sort_key) + with open(fname, "w") as f: + for r in reqs: + f.write(f"{r}\n") + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.argument("requirements") +def deps(requirements): + """Enumerate the installed dependencies.""" + + for r in _load(requirements): + print(f"- {r}") + + +@cli.command() +@click.option("--no-upgrade/--upgrade", name="upgrade", default=False) +@click.argument("requirements") +def install(requirements, upgrade): + """Install (or upgrade) a dependency. + + This is somewhat tricky because we need to come up with, format and persist a new dependency + solution. We aren't just doing formatting here. + + """ + + + + +@cli.command() +@click.option("--shout/--quiet", "-q/-S", default=True) +@click.argument("requirements") +def lint(shout, requirements): + """Check for unused dependencies.""" + + unused = list(_unused()) + if shout: + if unused: + print("Unused deps:") + for d in unused: + print(f" - {d}") + + if unused: + exit(1) + + reqs = _load(requirements) + if reqs != sorted(reqs, key=sort_key): + exit(1) + + +@cli.command() +@click.argument("requirements") +def clean(requirements): + """Expunge unused dependencies.""" + + unused = set(_unused()) + reqs = _load(requirements) + usedreqs = [r for r in reqs if sort_key(r) not in unused] + _write(requirements, usedreqs) + if usedreqs != reqs: + exit(1) + + +@cli.command() +@click.argument("requirements") +def sort(requirements): + """Just format the requirements file.""" + + reqs = _load(requirements) + sortedreqs = sorted(reqs, key=sort_key) + _write(requirements, sortedreqs) + if reqs != sortedreqs: + exit(1) + + +if __name__ == "__main__": + cli() diff --git a/projects/reqsort/BUILD b/projects/reqsort/BUILD deleted file mode 100644 index 70e4478..0000000 --- a/projects/reqsort/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -py_binary( - name = "reqsort", - main = "src/python/reqsort/__main__.py", - deps = [ - py_requirement("click"), - ], - imports = [ - "src/python", - ], -) diff --git a/projects/reqsort/README.md b/projects/reqsort/README.md deleted file mode 100644 index 8deacf1..0000000 --- a/projects/reqsort/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# reqsort - -A `requirements.txt` formatter and sorter. - -## LICENSE - -Copyright Reid 'arrdem' McKenzie August 2021. - -Published under the terms of the MIT license. diff --git a/projects/reqsort/src/python/reqsort/__main__.py b/projects/reqsort/src/python/reqsort/__main__.py deleted file mode 100644 index 6d86da9..0000000 --- a/projects/reqsort/src/python/reqsort/__main__.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -Platform independent sorting/formatting for requirements.txt -""" - -import re - -import click - - -REQ_PATTERN = re.compile( - r"(?P[a-zA-Z0-9_-]+)(?P\[.*?\])?==(?P[^\s;#]+)|(.*?#egg=(?P[a-zA-Z0-9_-]+))" -) - - -SHITLIST = [ - "pip", - "pkg-resources", - "setuptools", -] - - -def sort_key(requirement: str) -> str: - requirement = requirement.lower() - match = re.match(REQ_PATTERN, requirement) - sort_key = ( - (match.group("pkgname") or match.group("eggname")) # Get the match group - .replace("-", "") # We ignore - - .replace("_", "") # And _ - ) - return sort_key - - -@click.command() -@click.option("--execute/--dryrun", "execute", default=False) -@click.argument("requirements") -def main(requirements, execute): - """Given the path of a requirements.txt, format it. - - If running in --execute, rewrite the source file with sorted contents and exit 0. - - If running in --dryrun, exit 0 if --execute would produce no changes otherwise exit 1. - - """ - - with open(requirements) as f: - lines = f.readlines() - f.seek(0) - # Preserve an initial "buffer" for equality testing - initial_buff = f.read() - - # Trim whitespace - lines = [l.strip() for l in lines] - - # Discard comments and shitlisted packages - lines = [l for l in lines if not l.startswith("#") and not sort_key(l) in SHITLIST] - - # And sort, ignoring case explicitly - lines = sorted(lines, key=sort_key) - - # And generate a new "buffer" - new_buff = "\n".join(lines) + "\n" - - if new_buff != initial_buff and not execute: - exit(1) - - else: - with open(requirements, "w") as f: - f.write(new_buff) - exit(0) - - -if __name__ == "__main__": - main() diff --git a/tools/fmt.sh b/tools/fmt.sh index 35e6415..4f714b7 100755 --- a/tools/fmt.sh +++ b/tools/fmt.sh @@ -2,11 +2,11 @@ set -euox pipefail cd "$(git rev-parse --show-toplevel)" -bazel build //tools/python/... //projects/reqsort +bazel build //tools/python/... //projects/reqman DIRS=(projects tools) bazel-bin/tools/python/autoflake -ir "${DIRS[@]}" bazel-bin/tools/python/isort "${DIRS[@]}" bazel-bin/tools/python/unify --quote '"' -ir "${DIRS[@]}" -bazel-bin/projects/reqsort/reqsort --execute tools/python/requirements.txt +bazel-bin/projects/reqman/reqman sort tools/python/requirements.txt diff --git a/tools/lint.sh b/tools/lint.sh index 7970892..038073b 100755 --- a/tools/lint.sh +++ b/tools/lint.sh @@ -3,13 +3,12 @@ set -euox pipefail cd "$(git rev-parse --show-toplevel)" bazel build //tools/python/... -DIRS=(*) +DIRS=(tools projects) bazel-bin/tools/python/autoflake -r "${DIRS[@]}" -bazel-bin/tools/python/black --check "${DIRS[@]}" bazel-bin/tools/python/isort --check "${DIRS[@]}" bazel-bin/tools/python/unify --quote '"' -cr "${DIRS[@]}" -bazel-bin/tools/python/reqsort --dryrun tools/python/requirements.txt +bazel-bin/projects/reqman/reqman lint tools/python/requirements.txt for f in $(find . -type f -name "openapi.yaml"); do bazel-bin/tools/python/openapi "${f}" && echo "Schema $f OK"