Chuck out reqsort for a stub of something more general

This commit is contained in:
Reid 'arrdem' McKenzie 2021-08-04 00:16:55 -06:00
parent 023cfd7ac0
commit fce09a6e7a
8 changed files with 184 additions and 97 deletions

7
projects/reqman/BUILD Normal file
View file

@ -0,0 +1,7 @@
py_binary(
name = "reqman",
main = "src/python/reqman/__main__.py",
deps = [
py_requirement("click"),
],
)

View file

@ -0,0 +1,6 @@
# Req[uirements] Man[ager]
A tool designed to be a one-stop-shop interface for managing `requirements.txt` in a monorepo.
Provides a convenient way to check formatting, apply formatting, expunge unused requirements and add new requirements.
Inspired in some part by the `pip-tools` project's `pip-compile`; but designed to not make you do any intermediate file dancing.

View file

@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""ReqMan; the Requirements Manager.
Reqman is a quick and dirty tool that knows about Bazel, rules_python and requirements.txt files. It
exists to extend the upstream (and very good) pip-tools / pip-compile task with some helpers
providing better ergonomics for manipulating a monorepo.
Usage:
reqman deps
Print out a summary of all available dependencies.
reqman install <requirement>
Install a new (pinned) dependency (and any transitives) into the locked requirements.txt
reqman install --upgrade <requirement>
Upgrade a given (pinned) dependency (and any transitives)
reqman lint
Check for now-unused dependencies
reqman clean
Rewrite the requirements to emove unused dependencies
"""
import re
import subprocess
import click
REQ_PATTERN = re.compile(
r"(?P<pkgname>[a-zA-Z0-9_-]+)(?P<features>\[.*?\])?(?P<specifiers>((==|>=?|<=?|~=|!=)(?P<version>[^\s;#]+),)+)?|(.*?#egg=(?P<eggname>[a-zA-Z0-9_-]+))"
)
SHITLIST = [
"pip",
"pkg-resources",
"setuptools",
]
def req_name(requirement: str) -> str:
requirement = requirement.lower()
match = re.match(REQ_PATTERN, requirement)
return match.group("pkgname") or match.group("eggname")
def sort_key(requirement: str) -> str:
return (
req_name(requirement) # Get the match group
.lower() # We ignore case
.replace("-", "") # We ignore -
.replace("_", "") # And _
)
def _bq(query):
"""Enumerate the PyPi package names from a Bazel query."""
unused = subprocess.check_output(["bazel", "query", query, "--output=package"]).decode("utf-8")
for l in unused.split("\n"):
if l:
yield l.replace("@arrdem_source_pypi//pypi__", "")
def _unused():
"""Find unused requirements."""
return set(_bq("@arrdem_source_pypi//...")) - set(_bq("filter('//pypi__', deps(//...))"))
def _load(fname):
"""Slurp requirements from a file."""
with open(fname, "r") as reqfile:
# FIXME (arrdem 2021-08-03):
# Return a parse, not just lines.
return list(l.strip() for l in reqfile)
def _write(fname, reqs):
"""Dump requirements back to a file in sorted format."""
reqs = sorted(reqs, key=sort_key)
with open(fname, "w") as f:
for r in reqs:
f.write(f"{r}\n")
@click.group()
def cli():
pass
@cli.command()
@click.argument("requirements")
def deps(requirements):
"""Enumerate the installed dependencies."""
for r in _load(requirements):
print(f"- {r}")
@cli.command()
@click.option("--no-upgrade/--upgrade", name="upgrade", default=False)
@click.argument("requirements")
def install(requirements, upgrade):
"""Install (or upgrade) a dependency.
This is somewhat tricky because we need to come up with, format and persist a new dependency
solution. We aren't just doing formatting here.
"""
@cli.command()
@click.option("--shout/--quiet", "-q/-S", default=True)
@click.argument("requirements")
def lint(shout, requirements):
"""Check for unused dependencies."""
unused = list(_unused())
if shout:
if unused:
print("Unused deps:")
for d in unused:
print(f" - {d}")
if unused:
exit(1)
reqs = _load(requirements)
if reqs != sorted(reqs, key=sort_key):
exit(1)
@cli.command()
@click.argument("requirements")
def clean(requirements):
"""Expunge unused dependencies."""
unused = set(_unused())
reqs = _load(requirements)
usedreqs = [r for r in reqs if sort_key(r) not in unused]
_write(requirements, usedreqs)
if usedreqs != reqs:
exit(1)
@cli.command()
@click.argument("requirements")
def sort(requirements):
"""Just format the requirements file."""
reqs = _load(requirements)
sortedreqs = sorted(reqs, key=sort_key)
_write(requirements, sortedreqs)
if reqs != sortedreqs:
exit(1)
if __name__ == "__main__":
cli()

View file

@ -1,10 +0,0 @@
py_binary(
name = "reqsort",
main = "src/python/reqsort/__main__.py",
deps = [
py_requirement("click"),
],
imports = [
"src/python",
],
)

View file

@ -1,9 +0,0 @@
# reqsort
A `requirements.txt` formatter and sorter.
## LICENSE
Copyright Reid 'arrdem' McKenzie August 2021.
Published under the terms of the MIT license.

View file

@ -1,73 +0,0 @@
"""
Platform independent sorting/formatting for requirements.txt
"""
import re
import click
REQ_PATTERN = re.compile(
r"(?P<pkgname>[a-zA-Z0-9_-]+)(?P<features>\[.*?\])?==(?P<version>[^\s;#]+)|(.*?#egg=(?P<eggname>[a-zA-Z0-9_-]+))"
)
SHITLIST = [
"pip",
"pkg-resources",
"setuptools",
]
def sort_key(requirement: str) -> str:
requirement = requirement.lower()
match = re.match(REQ_PATTERN, requirement)
sort_key = (
(match.group("pkgname") or match.group("eggname")) # Get the match group
.replace("-", "") # We ignore -
.replace("_", "") # And _
)
return sort_key
@click.command()
@click.option("--execute/--dryrun", "execute", default=False)
@click.argument("requirements")
def main(requirements, execute):
"""Given the path of a requirements.txt, format it.
If running in --execute, rewrite the source file with sorted contents and exit 0.
If running in --dryrun, exit 0 if --execute would produce no changes otherwise exit 1.
"""
with open(requirements) as f:
lines = f.readlines()
f.seek(0)
# Preserve an initial "buffer" for equality testing
initial_buff = f.read()
# Trim whitespace
lines = [l.strip() for l in lines]
# Discard comments and shitlisted packages
lines = [l for l in lines if not l.startswith("#") and not sort_key(l) in SHITLIST]
# And sort, ignoring case explicitly
lines = sorted(lines, key=sort_key)
# And generate a new "buffer"
new_buff = "\n".join(lines) + "\n"
if new_buff != initial_buff and not execute:
exit(1)
else:
with open(requirements, "w") as f:
f.write(new_buff)
exit(0)
if __name__ == "__main__":
main()

View file

@ -2,11 +2,11 @@
set -euox pipefail set -euox pipefail
cd "$(git rev-parse --show-toplevel)" cd "$(git rev-parse --show-toplevel)"
bazel build //tools/python/... //projects/reqsort bazel build //tools/python/... //projects/reqman
DIRS=(projects tools) DIRS=(projects tools)
bazel-bin/tools/python/autoflake -ir "${DIRS[@]}" bazel-bin/tools/python/autoflake -ir "${DIRS[@]}"
bazel-bin/tools/python/isort "${DIRS[@]}" bazel-bin/tools/python/isort "${DIRS[@]}"
bazel-bin/tools/python/unify --quote '"' -ir "${DIRS[@]}" bazel-bin/tools/python/unify --quote '"' -ir "${DIRS[@]}"
bazel-bin/projects/reqsort/reqsort --execute tools/python/requirements.txt bazel-bin/projects/reqman/reqman sort tools/python/requirements.txt

View file

@ -3,13 +3,12 @@ set -euox pipefail
cd "$(git rev-parse --show-toplevel)" cd "$(git rev-parse --show-toplevel)"
bazel build //tools/python/... bazel build //tools/python/...
DIRS=(*) DIRS=(tools projects)
bazel-bin/tools/python/autoflake -r "${DIRS[@]}" bazel-bin/tools/python/autoflake -r "${DIRS[@]}"
bazel-bin/tools/python/black --check "${DIRS[@]}"
bazel-bin/tools/python/isort --check "${DIRS[@]}" bazel-bin/tools/python/isort --check "${DIRS[@]}"
bazel-bin/tools/python/unify --quote '"' -cr "${DIRS[@]}" bazel-bin/tools/python/unify --quote '"' -cr "${DIRS[@]}"
bazel-bin/tools/python/reqsort --dryrun tools/python/requirements.txt bazel-bin/projects/reqman/reqman lint tools/python/requirements.txt
for f in $(find . -type f -name "openapi.yaml"); do for f in $(find . -type f -name "openapi.yaml"); do
bazel-bin/tools/python/openapi "${f}" && echo "Schema $f OK" bazel-bin/tools/python/openapi "${f}" && echo "Schema $f OK"