Chuck out reqsort for a stub of something more general

projects/reqman/BUILD

@@ -0,0 +1,7 @@
+py_binary(
+    name = "reqman",
+    main = "src/python/reqman/__main__.py",
+    deps = [
+        py_requirement("click"),
+    ],
+)

projects/reqman/README.md

@@ -0,0 +1,6 @@
+# Req[uirements] Man[ager]
+
+A tool designed to be a one-stop-shop interface for managing `requirements.txt` in a monorepo.
+Provides a convenient way to check formatting, apply formatting, expunge unused requirements and add new requirements.
+
+Inspired in some part by the `pip-tools` project's `pip-compile`; but designed to not make you do any intermediate file dancing.

projects/reqman/src/python/reqman/__main__.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+
+"""ReqMan; the Requirements Manager.
+
+Reqman is a quick and dirty tool that knows about Bazel, rules_python and requirements.txt files. It
+exists to extend the upstream (and very good) pip-tools / pip-compile task with some helpers
+providing better ergonomics for manipulating a monorepo.
+
+Usage:
+  reqman deps
+    Print out a summary of all available dependencies.
+
+  reqman install <requirement>
+    Install a new (pinned) dependency (and any transitives) into the locked requirements.txt
+
+  reqman install --upgrade <requirement>
+    Upgrade a given (pinned) dependency (and any transitives)
+
+  reqman lint
+    Check for now-unused dependencies
+
+  reqman clean
+    Rewrite the requirements to emove unused dependencies
+
+"""
+
+import re
+import subprocess
+
+import click
+
+
+REQ_PATTERN = re.compile(
+    r"(?P<pkgname>[a-zA-Z0-9_-]+)(?P<features>\[.*?\])?(?P<specifiers>((==|>=?|<=?|~=|!=)(?P<version>[^\s;#]+),)+)?|(.*?#egg=(?P<eggname>[a-zA-Z0-9_-]+))"
+)
+
+
+SHITLIST = [
+    "pip",
+    "pkg-resources",
+    "setuptools",
+]
+
+def req_name(requirement: str) -> str:
+    requirement = requirement.lower()
+    match = re.match(REQ_PATTERN, requirement)
+    return match.group("pkgname") or match.group("eggname")
+
+
+def sort_key(requirement: str) -> str:
+    return (
+        req_name(requirement)  # Get the match group
+        .lower()           # We ignore case
+        .replace("-", "")  # We ignore -
+        .replace("_", "")  # And _
+    )
+
+
+def _bq(query):
+    """Enumerate the PyPi package names from a Bazel query."""
+
+    unused = subprocess.check_output(["bazel", "query", query, "--output=package"]).decode("utf-8")
+    for l in unused.split("\n"):
+        if l:
+            yield l.replace("@arrdem_source_pypi//pypi__", "")
+
+
+def _unused():
+    """Find unused requirements."""
+
+    return set(_bq("@arrdem_source_pypi//...")) - set(_bq("filter('//pypi__', deps(//...))"))
+
+
+def _load(fname):
+    """Slurp requirements from a file."""
+
+    with open(fname, "r") as reqfile:
+        # FIXME (arrdem 2021-08-03):
+        #   Return a parse, not just lines.
+        return list(l.strip() for l in reqfile)
+
+
+def _write(fname, reqs):
+    """Dump requirements back to a file in sorted format."""
+
+    reqs = sorted(reqs, key=sort_key)
+    with open(fname, "w") as f:
+        for r in reqs:
+            f.write(f"{r}\n")
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.argument("requirements")
+def deps(requirements):
+    """Enumerate the installed dependencies."""
+
+    for r in _load(requirements):
+        print(f"- {r}")
+
+
+@cli.command()
+@click.option("--no-upgrade/--upgrade", name="upgrade", default=False)
+@click.argument("requirements")
+def install(requirements, upgrade):
+    """Install (or upgrade) a dependency.
+
+    This is somewhat tricky because we need to come up with, format and persist a new dependency
+    solution. We aren't just doing formatting here.
+
+    """
+
+
+
+
+@cli.command()
+@click.option("--shout/--quiet", "-q/-S", default=True)
+@click.argument("requirements")
+def lint(shout, requirements):
+    """Check for unused dependencies."""
+
+    unused = list(_unused())
+    if shout:
+        if unused:
+            print("Unused deps:")
+        for d in unused:
+            print(f" - {d}")
+
+    if unused:
+        exit(1)
+
+    reqs = _load(requirements)
+    if reqs != sorted(reqs, key=sort_key):
+        exit(1)
+
+
+@cli.command()
+@click.argument("requirements")
+def clean(requirements):
+    """Expunge unused dependencies."""
+
+    unused = set(_unused())
+    reqs = _load(requirements)
+    usedreqs = [r for r in reqs if sort_key(r) not in unused]
+    _write(requirements, usedreqs)
+    if usedreqs != reqs:
+        exit(1)
+
+
+@cli.command()
+@click.argument("requirements")
+def sort(requirements):
+    """Just format the requirements file."""
+
+    reqs = _load(requirements)
+    sortedreqs = sorted(reqs, key=sort_key)
+    _write(requirements, sortedreqs)
+    if reqs != sortedreqs:
+        exit(1)
+
+
+if __name__ == "__main__":
+    cli()

projects/reqsort/BUILD

@@ -1,10 +0,0 @@
-py_binary(
-    name = "reqsort",
-    main = "src/python/reqsort/__main__.py",
-    deps = [
-        py_requirement("click"),
-    ],
-    imports = [
-        "src/python",
-    ],
-)

projects/reqsort/README.md

@@ -1,9 +0,0 @@
-# reqsort
-
-A `requirements.txt` formatter and sorter.
-
-## LICENSE
-
-Copyright Reid 'arrdem' McKenzie August 2021.
-
-Published under the terms of the MIT license.

projects/reqsort/src/python/reqsort/__main__.py

@@ -1,73 +0,0 @@
-"""
-Platform independent sorting/formatting for requirements.txt
-"""
-
-import re
-
-import click
-
-
-REQ_PATTERN = re.compile(
-    r"(?P<pkgname>[a-zA-Z0-9_-]+)(?P<features>\[.*?\])?==(?P<version>[^\s;#]+)|(.*?#egg=(?P<eggname>[a-zA-Z0-9_-]+))"
-)
-
-
-SHITLIST = [
-    "pip",
-    "pkg-resources",
-    "setuptools",
-]
-
-
-def sort_key(requirement: str) -> str:
-    requirement = requirement.lower()
-    match = re.match(REQ_PATTERN, requirement)
-    sort_key = (
-        (match.group("pkgname") or match.group("eggname"))  # Get the match group
-        .replace("-", "")  # We ignore -
-        .replace("_", "")  # And _
-    )
-    return sort_key
-
-
-@click.command()
-@click.option("--execute/--dryrun", "execute", default=False)
-@click.argument("requirements")
-def main(requirements, execute):
-    """Given the path of a requirements.txt, format it.
-
-    If running in --execute, rewrite the source file with sorted contents and exit 0.
-
-    If running in --dryrun, exit 0 if --execute would produce no changes otherwise exit 1.
-
-    """
-
-    with open(requirements) as f:
-        lines = f.readlines()
-        f.seek(0)
-        # Preserve an initial "buffer" for equality testing
-        initial_buff = f.read()
-
-    # Trim whitespace
-    lines = [l.strip() for l in lines]
-
-    # Discard comments and shitlisted packages
-    lines = [l for l in lines if not l.startswith("#") and not sort_key(l) in SHITLIST]
-
-    # And sort, ignoring case explicitly
-    lines = sorted(lines, key=sort_key)
-
-    # And generate a new "buffer"
-    new_buff = "\n".join(lines) + "\n"
-
-    if new_buff != initial_buff and not execute:
-        exit(1)
-
-    else:
-        with open(requirements, "w") as f:
-            f.write(new_buff)
-        exit(0)
-
-
-if __name__ == "__main__":
-    main()

tools/fmt.sh

@@ -2,11 +2,11 @@
 set -euox pipefail
 cd "$(git rev-parse --show-toplevel)"
 
-bazel build //tools/python/... //projects/reqsort
+bazel build //tools/python/... //projects/reqman
 
 DIRS=(projects tools)
 
 bazel-bin/tools/python/autoflake -ir "${DIRS[@]}"
 bazel-bin/tools/python/isort "${DIRS[@]}"
 bazel-bin/tools/python/unify --quote '"' -ir "${DIRS[@]}"
-bazel-bin/projects/reqsort/reqsort --execute tools/python/requirements.txt
+bazel-bin/projects/reqman/reqman sort tools/python/requirements.txt

tools/lint.sh

@@ -3,13 +3,12 @@ set -euox pipefail
 cd "$(git rev-parse --show-toplevel)"
 bazel build //tools/python/...
 
-DIRS=(*)
+DIRS=(tools projects)
 
 bazel-bin/tools/python/autoflake -r "${DIRS[@]}"
-bazel-bin/tools/python/black --check "${DIRS[@]}"
 bazel-bin/tools/python/isort --check "${DIRS[@]}"
 bazel-bin/tools/python/unify --quote '"' -cr "${DIRS[@]}"
-bazel-bin/tools/python/reqsort --dryrun tools/python/requirements.txt
+bazel-bin/projects/reqman/reqman lint tools/python/requirements.txt
 
 for f in $(find . -type f -name "openapi.yaml"); do
   bazel-bin/tools/python/openapi "${f}" && echo "Schema $f OK"