Fmt.

projects/calf/setup.py

@@ -2,7 +2,7 @@
 
 from os import path
 
-from setuptools import setup, find_namespace_packages
+from setuptools import find_namespace_packages, setup
 
 
 # Fetch the README contents

projects/calf/src/python/calf/cursedrepl.py

@@ -3,7 +3,7 @@ Some shared scaffolding for building terminal "REPL" drivers.
 """
 
 import curses
-from curses.textpad import Textbox, rectangle
+from curses.textpad import rectangle, Textbox
 
 
 def curse_repl(handle_buffer):

projects/calf/src/python/calf/lexer.py

@@ -9,9 +9,9 @@ parsing, linting or other use.
 import io
 import re
 
-from calf.token import CalfToken
-from calf.io.reader import PeekPosReader
 from calf.grammar import TOKENS
+from calf.io.reader import PeekPosReader
+from calf.token import CalfToken
 from calf.util import *
 
 

projects/calf/src/python/calf/parser.py

@@ -5,8 +5,8 @@ The Calf parser.
 from itertools import tee
 import logging
 
-from calf.lexer import CalfLexer, lex_buffer, lex_file
 from calf.grammar import MATCHING, WHITESPACE_TYPES
+from calf.lexer import CalfLexer, lex_buffer, lex_file
 from calf.token import *
 
 

projects/calf/tests/python/conftest.py

@@ -4,4 +4,5 @@ Fixtures for testing Calf.
 
 import pytest
 
+
 parametrize = pytest.mark.parametrize

projects/calf/tests/python/test_lexer.py

@@ -7,7 +7,6 @@ trip through the lexer.
 
 import calf.lexer as cl
 from conftest import parametrize
-
 import pytest
 
 

projects/calf/tests/python/test_parser.py

@@ -4,7 +4,6 @@ Tests of calf.parser
 
 import calf.parser as cp
 from conftest import parametrize
-
 import pytest
 
 

projects/calf/tests/python/test_reader.py

@@ -1,9 +1,8 @@
 """
 """
 
-from conftest import parametrize
-
 from calf.reader import read_buffer
+from conftest import parametrize
 
 
 @parametrize(

projects/datalog-shell/__main__.py

@@ -58,13 +58,13 @@ from datalog.debris import Timing
 from datalog.evaluator import select
 from datalog.reader import pr_str, read_command, read_dataset
 from datalog.types import (
-    CachedDataset,
+  CachedDataset,
-    Constant,
+  Constant,
-    Dataset,
+  Dataset,
-    LVar,
+  LVar,
-    PartlyIndexedDataset,
+  PartlyIndexedDataset,
-    Rule,
+  Rule,
-    TableIndexedDataset,
+  TableIndexedDataset
 )
 
 from prompt_toolkit import print_formatted_text, prompt, PromptSession

projects/datalog/src/python/datalog/evaluator.py

@@ -7,14 +7,7 @@ from itertools import chain
 
 from datalog.parser import parse
 from datalog.reader import pr_str, read
-from datalog.types import (
+from datalog.types import CachedDataset, Constant, Dataset, LVar, Rule, TableIndexedDataset
-    CachedDataset,
-    Constant,
-    Dataset,
-    LVar,
-    Rule,
-    TableIndexedDataset,
-)
 
 
 def match(tuple, expr, bindings=None):

projects/datalog/test/python/test_datalog_evaluator.py

@@ -2,13 +2,13 @@
 
 from datalog.easy import read, select
 from datalog.types import (
-    CachedDataset,
+  CachedDataset,
-    Constant,
+  Constant,
-    Dataset,
+  Dataset,
-    LVar,
+  LVar,
-    PartlyIndexedDataset,
+  PartlyIndexedDataset,
-    Rule,
+  Rule,
-    TableIndexedDataset,
+  TableIndexedDataset
 )
 
 import pytest

projects/datalog/test/python/test_datalog_reader.py

@@ -2,9 +2,10 @@
 Reader tests.
 """
 
+from datalog.reader import read
+
 import pytest
 
-from datalog.reader import read
 
 EXS = [
     "%foo\n",

projects/flowmetal/setup.py

@@ -1,5 +1,6 @@
 from setuptools import setup
 
+
 setup(
     name="arrdem.flowmetal",
     # Package metadata

projects/flowmetal/src/python/flowmetal/__main__.py

@@ -2,9 +2,8 @@
 The Flowmetal server entry point.
 """
 
-from flowmetal import frontend, interpreter, scheduler, reaper
-
 import click
+from flowmetal import frontend, interpreter, reaper, scheduler
 
 
 @click.group()

projects/gandi/src/python/gandi/client.py

@@ -2,8 +2,8 @@
 Quick and shitty Gandi REST API driver
 """
 
-import json
 from datetime import datetime
+import json
 
 import requests
 

projects/public-dns/src/python/arrdem/updater/__main__.py

@@ -2,21 +2,21 @@
 A quick and dirty public DNS script, super tightly coupled to my infrastructure.
 """
 
-import sys
-import os
 import argparse
-import re
+import os
 from pprint import pprint
+import re
+import sys
+
 
 for e in sys.path:
     print(e)
 
 from gandi.client import GandiAPI
-
 import jinja2
+import meraki
 import pkg_resources
 import yaml
-import meraki
 
 
 RECORD_LINE_PATTERN = re.compile(

projects/ratchet/setup.py

@@ -1,5 +1,6 @@
 from setuptools import setup
 
+
 setup(
     name="arrdem.ratchet",
     # Package metadata

projects/ratchet/src/python/ratchet/__init__.py

@@ -8,7 +8,7 @@ from abc import ABC, abstractmethod
 class Message:
     """Messages can be sent. That's it.
 
-    Messages have headers, which may 
+    Messages have headers, which may
 
     Other things can filter the stream of inbound messages and do log processing, but that's the whole basis of the
     thing.
@@ -67,57 +67,51 @@ class Driver(ABC):
     """Shared interface for Ratchet backend drivers."""
 
     @abstractmethod
-    def __init__(message_ttl=60000,
+    def __init__(message_ttl=60000, message_space="_", message_author=""):
-                 message_space="_",
-                 message_author=""):
         """Initialize the driver."""
 
     @abstractmethod
-    def create_message(self,
+    def create_message(
-                       message: str,
+        self, message: str, ttl: int = None, space: str = None, author: str = None
-                       ttl: int = None,
+    ) -> Message:
-                       space: str = None,
-                       author: str = None) -> Message:
         """Create a single message."""
 
     @abstractmethod
-    def create_event(self,
+    def create_event(
-                     timeout: int,
+        self, timeout: int, ttl: int = None, space: str = None, author: str = None
-                     ttl: int = None,
+    ):
-                     space: str = None,
-                     author: str = None):
         """Create a (pending) event."""
 
     @abstractmethod
-    def set_event(self,
+    def set_event(
-                  timeout: int,
+        self, timeout: int, ttl: int = None, space: str = None, author: str = None
-                  ttl: int = None,
+    ):
-                  space: str = None,
-                  author: str = None):
         """Attempt to mark an event as set."""
 
     @abstractmethod
-    def create_request(self,
+    def create_request(
-                       body: str,
+        self,
-                       timeout: int,
+        body: str,
-                       ttl: int = None,
+        timeout: int,
-                       space: str = None,
+        ttl: int = None,
-                       author: str = None):
+        space: str = None,
+        author: str = None,
+    ):
         """Create a (pending) request."""
 
     @abstractmethod
-    def deliver_request(self,
+    def deliver_request(
-                        request_id,
+        self,
-                        response: str,
+        request_id,
-                        ttl: int = None,
+        response: str,
-                        space: str = None,
+        ttl: int = None,
-                        author: str = None):
+        space: str = None,
+        author: str = None,
+    ):
         """Deliver a response to a (pending) request."""
 
     @abstractmethod
-    def revoke_request(self,
+    def revoke_request(
-                       request_id,
+        self, request_id, ttl: int = None, space: str = None, author: str = None
-                       ttl: int = None,
+    ):
-                       space: str = None,
-                       author: str = None):
         """Revoke a (pending) request."""

projects/ratchet/src/python/ratchet/backend/sqlite.py

@@ -2,12 +2,12 @@
 An implementation of the ratchet model against SQLite.
 """
 
-import os
-import sqlite3 as sql
 from contextlib import closing
+import os
 import socket
+import sqlite3 as sql
 
-from ratchet import Message, Event, Request
+from ratchet import Event, Message, Request
 
 
 SCHEMA_SCRIPT = """

projects/yamlschema/setup.py

@@ -1,5 +1,6 @@
 from setuptools import setup
 
+
 setup(
     name="arrdem.yamlschema",
     # Package metadata

projects/yamlschema/src/python/yamlschema/__init__.py

@@ -2,11 +2,11 @@
 JSONSchema linting for YAML documents.
 """
 
-import logging
-import typing as t
 from enum import Enum
 from io import StringIO
+import logging
 import re
+import typing as t
 
 import yaml
 from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode
@@ -58,9 +58,13 @@ class YamlLinter(object):
             schema = self._schema
             for e in path:
                 if not e:
-                    raise ValueError(f"Unable to dereference {ref}; contains empty segment!")
+                    raise ValueError(
+                        f"Unable to dereference {ref}; contains empty segment!"
+                    )
                 if not (schema := schema.get(e)):
-                    raise ValueError(f"Unable to dereference {ref}; references missing sub-document!")
+                    raise ValueError(
+                        f"Unable to dereference {ref}; references missing sub-document!"
+                    )
 
         return schema
 
@@ -175,7 +179,10 @@ class YamlLinter(object):
 
         else:
             yield LintRecord(
-                LintLevel.MISSMATCH, node, schema, f"Expected an integer, got a {node.tag}"
+                LintLevel.MISSMATCH,
+                node,
+                schema,
+                f"Expected an integer, got a {node.tag}",
             )
 
     def lint_number(self, schema, node: Node) -> t.Iterable[LintRecord]:
@@ -185,7 +192,10 @@ class YamlLinter(object):
 
         else:
             yield LintRecord(
-                LintLevel.MISSMATCH, node, schema, f"Expected an integer, got a {node.tag}"
+                LintLevel.MISSMATCH,
+                node,
+                schema,
+                f"Expected an integer, got a {node.tag}",
             )
 
     def _lint_num_range(self, schema, node: Node, value) -> t.Iterable[LintRecord]:

projects/yamlschema/test/python/test_yamlschema.py

@@ -2,9 +2,8 @@
 Tests covering the YAML linter.
 """
 
-from yamlschema import lint_buffer
-
 import pytest
+from yamlschema import lint_buffer
 
 
 @pytest.mark.parametrize(
@@ -100,20 +99,31 @@ def test_lint_document_fails(msg, schema, obj):
     assert list(lint_buffer(schema, obj)), msg
 
 
-@pytest.mark.parametrize("msg, schema, obj", [
+@pytest.mark.parametrize(
-    ("Basic usage of $ref",
+    "msg, schema, obj",
-     {"$ref": "#/definitions/Foo",
+    [
-      "definitions": {
+        (
-          "Foo": {"type": "string"},
+            "Basic usage of $ref",
-      }},
+            {
-     "---\nfoo"),
+                "$ref": "#/definitions/Foo",
-    ("Use of nested references",
+                "definitions": {
-     {"$ref": "#/definitions/Foos",
+                    "Foo": {"type": "string"},
-      "definitions": {
+                },
-          "Foos": {"type": "array", "items": {"$ref": "#/definitions/Foo"}},
+            },
-          "Foo": {"type": "string"},
+            "---\nfoo",
-      }},
+        ),
-     "---\n- foo\n- bar\n- baz"),
+        (
-])
+            "Use of nested references",
+            {
+                "$ref": "#/definitions/Foos",
+                "definitions": {
+                    "Foos": {"type": "array", "items": {"$ref": "#/definitions/Foo"}},
+                    "Foo": {"type": "string"},
+                },
+            },
+            "---\n- foo\n- bar\n- baz",
+        ),
+    ],
+)
 def test_ref_references(msg, schema, obj):
     assert not list(lint_buffer(schema, obj)), msg

tools/python/BUILD

@@ -70,6 +70,7 @@ py_pytest(
     ],
     deps = [
         py_requirement("requests"),
+        py_requirement("requirements-parser"),
     ]
 )
 

tools/python/autoflake_shim.py

@@ -9,6 +9,7 @@ import sys
 
 from autoflake import main
 
+
 if __name__ == "__main__":
     sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

tools/python/bzl_pytest_shim.py

@@ -1,10 +1,10 @@
 """A shim for executing pytest."""
 
-import os
 import sys
 
 import pytest
 
+
 if __name__ == "__main__":
     cmdline = ["--ignore=external"] + sys.argv[1:]
     sys.exit(pytest.main(cmdline))

tools/python/isort_shim.py

@@ -9,6 +9,7 @@ import sys
 
 from isort.main import main
 
+
 if __name__ == "__main__":
     sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

tools/python/openapi_shim.py

@@ -7,6 +7,7 @@ import sys
 
 from openapi_spec_validator.__main__ import main
 
+
 if __name__ == "__main__":
     sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

tools/python/requirements.txt

@@ -64,6 +64,7 @@ redis==3.5.3
 regex==2021.4.4
 requests==2.25.1
 requests-toolbelt==0.9.1
+requirements-parser==0.2.0
 rfc3986==1.5.0
 SecretStorage==3.3.1
 six==1.15.0

tools/python/templater.py

@@ -12,6 +12,7 @@ import click
 import jinja2
 import yaml
 
+
 FONTMATTER_PATTERN = re.compile(
     r"^(---\n\r?(?P<fontmatter>.*?)\n\r?---\n\r?)?(?P<content>.+)$", re.DOTALL
 )

tools/python/test_licenses.py

@@ -6,35 +6,51 @@ import re
 
 import pytest
 import requests
+import requirements
+from requirements.requirement import Requirement
+
 
 # Licenses approved as representing non-copyleft and not precluding commercial usage.
 # This is all easy, there's a good schema here.
 APPROVED_LICENSES = [
-    "License :: OSI Approved :: MIT License",
+    MIT    := "License :: OSI Approved :: MIT License",
-    "License :: OSI Approved :: Apache Software License",
+    APACHE := "License :: OSI Approved :: Apache Software License",
-    "License :: OSI Approved :: BSD License",
+    BSD    := "License :: OSI Approved :: BSD License",
-    "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
+    MPL10  := "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
-    "License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)",
+    MPL11  := "License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)",
-    "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    MPL20  := "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
-    "License :: OSI Approved :: Python Software Foundation License",
+    PSFL   := "License :: OSI Approved :: Python Software Foundation License",
-    "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)",
+    LGPL   := "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)",
-    "License :: OSI Approved :: ISC License (ISCL)",
+    ISCL   := "License :: OSI Approved :: ISC License (ISCL)",
+]
+
+UNAPPROVED_LICENSES = [
+    GPL1 := "License :: OSI Approved :: GNU General Public License",
+    GPL2 := "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
+    GPL3 := "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
 ]
 
 # This data is GARBO.
 LICENSES_BY_LOWERNAME = {
-    "apache 2.0": "License :: OSI Approved :: Apache Software License",
+    "apache 2.0": APACHE,
-    "apache": "License :: OSI Approved :: Apache Software License",
+    "apache": APACHE,
-    "bsd 3 clause": "License :: OSI Approved :: BSD License",
+    "http://www.apache.org/licenses/license-2.0": APACHE,
-    "bsd 3-clause": "License :: OSI Approved :: BSD License",
+
-    "bsd": "License :: OSI Approved :: BSD License",
+    "bsd 3": BSD,
-    "gplv3": "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "bsd": BSD,
-    "http://www.apache.org/licenses/license-2.0": "License :: OSI Approved :: Apache Software License",
+
-    "isc": "License :: OSI Approved :: ISC License (ISCL)",
+    "gpl": GPL1,
-    "mit": "License :: OSI Approved :: MIT License",
+    "gpl2": GPL2,
-    "mpl 2.0": "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    "gpl3": GPL3,
-    "mpl": "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
+
-    "psf": "License :: OSI Approved :: Python Software Foundation License",
+    "isc": ISCL,
+
+    "mit": MIT,
+
+    "mpl": MPL10,
+    "mpl 2.0": MPL20,
+
+    "psf": PSFL,
 }
 
 # Mash in some cases.
@@ -49,52 +65,38 @@ APPROVED_PACKAGES = [
     "anosql",  # BSD
 ]
 
-REQ_PATTERN = re.compile(
-    r"(?P<pkgname>[a-zA-Z0-9_-]+)(?P<features>\[.*?\])?==(?P<version>[^\s;#]+)|(.*?#egg=(?P<eggname>[a-zA-Z0-9_-]+))"
-)
 
-
+with open("tools/python/requirements.txt") as fd:
-def parse_requirement(line):
+    PACKAGES = list(requirements.parse(fd))
-    """Given a requirement return the requirement name and version as a tuple.
-
-    Only the strict `==` version pinning subset is supported.
-    Features are supported.
-    """
-
-    if m := re.match(REQ_PATTERN, line):
-        return (m.group("pkgname") or m.group("eggname")), m.group("version")
-
-
-@pytest.mark.parametrize(
-    "line,t",
-    [
-        ("foo==1.2.3", ("foo", "1.2.3")),
-        ("foo[bar]==1.2.3", ("foo", "1.2.3")),
-        ("foo[bar, baz, qux]==1.2.3", ("foo", "1.2.3")),
-        # Various stuff we should ignore
-        ("# comment line", None),
-        ("    # garbage whitespace", None),
-        ("     \t", None),
-    ],
-)
-def test_parse_requirement(line, t):
-    """The irony of testing one"s tests is not lost."""
-
-    assert parse_requirement(line) == t
-
-
-with open("tools/python/requirements.txt") as f:
-    PACKAGES = [parse_requirement(l) for l in f.readlines()]
 
 
 def bash_license(ln):
-    if ln:
+    while True:
-        ln = re.sub("[(),]|( version)|( license)", "", ln.lower())
+        lnn = re.sub(r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)", "", ln.lower())
-        ln = LICENSES_BY_LOWERNAME.get(ln, ln)
+        if ln != lnn:
+            ln = lnn
+        else:
+            break
+
+    ln = LICENSES_BY_LOWERNAME.get(ln, ln)
     return ln
 
 
-def licenses(package, version):
+@pytest.mark.parametrize("a,b", [
+    ("MIT", MIT),
+    ("mit", MIT),
+    ("BSD", BSD),
+    ("BSD 3-clause", BSD),
+    ("BSD 3 clause", BSD),
+    ("GPL3", GPL3),
+    ("GPL v3", GPL3),
+    ("GPLv3", GPL3),
+])
+def test_bash_license(a, b):
+    assert bash_license(a) == b
+
+
+def licenses(package: Requirement):
     """Get package metadata (the licenses list) from PyPi.
 
     pip and other tools use the local package metadata to introspect licenses which requires that
@@ -104,11 +106,16 @@ def licenses(package, version):
 
     """
     l = []
+    version = next((v for op, v in package.specs if op == "=="), None)
+    print(package.name, version)
 
     # If we don't have a version (eg. forked git dep) assume we've got the same license constraints
     # as the latest upstream release. After all we can't re-license stuff.
     if not version:
-        blob = requests.get(f"https://pypi.python.org/pypi/{package}/json").json()
+        blob = requests.get(
+            f"https://pypi.org/pypi/{package.name}/json",
+            headers={"Accept": "application/json"}
+        ).json()
         if ln := bash_license(blob.get("license")):
             l.append(ln)
         else:
@@ -120,7 +127,8 @@ def licenses(package, version):
     # If we have a version, try to pull that release's metadata since it may have more/better.
     if version:
         blob = requests.get(
-            f"https://pypi.python.org/pypi/{package}/{version}/json"
+            f"https://pypi.org/pypi/{package.name}/{version}/json",
+            headers={"Accept": "application/json"}
         ).json()
         l = [
             c
@@ -134,11 +142,11 @@ def licenses(package, version):
     return l
 
 
-@pytest.mark.parametrize("package,version", PACKAGES)
+@pytest.mark.parametrize("package", PACKAGES)
-def test_approved_license(package, version):
+def test_approved_license(package):
     """Ensure that a given package is either allowed by name or uses an approved license."""
 
-    _licenses = licenses(package, version)
+    _licenses = licenses(package)
-    assert package in APPROVED_PACKAGES or any(
+    assert package.name in APPROVED_PACKAGES or any(
         l in APPROVED_LICENSES for l in _licenses
     ), f"{package} was not approved and its license(s) were unknown {_licenses!r}"

tools/python/unify_shim.py

@@ -7,5 +7,6 @@ Shim for executing isort.
 
 from unify import main
 
+
 if __name__ == "__main__":
     exit(main())

tools/python/yamllint_shim.py

@@ -9,6 +9,7 @@ import sys
 
 from yamllint.cli import run
 
+
 if __name__ == "__main__":
     sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0])
     sys.exit(run())