This commit is contained in:
Reid 'arrdem' McKenzie 2021-05-31 12:28:46 -06:00
parent 4c5d2aaed2
commit 03b37675b5
32 changed files with 193 additions and 169 deletions

View file

@ -2,7 +2,7 @@
from os import path
from setuptools import setup, find_namespace_packages
from setuptools import find_namespace_packages, setup
# Fetch the README contents

View file

@ -3,7 +3,7 @@ Some shared scaffolding for building terminal "REPL" drivers.
"""
import curses
from curses.textpad import Textbox, rectangle
from curses.textpad import rectangle, Textbox
def curse_repl(handle_buffer):

View file

@ -9,9 +9,9 @@ parsing, linting or other use.
import io
import re
from calf.token import CalfToken
from calf.io.reader import PeekPosReader
from calf.grammar import TOKENS
from calf.io.reader import PeekPosReader
from calf.token import CalfToken
from calf.util import *

View file

@ -5,8 +5,8 @@ The Calf parser.
from itertools import tee
import logging
from calf.lexer import CalfLexer, lex_buffer, lex_file
from calf.grammar import MATCHING, WHITESPACE_TYPES
from calf.lexer import CalfLexer, lex_buffer, lex_file
from calf.token import *

View file

@ -4,4 +4,5 @@ Fixtures for testing Calf.
import pytest
parametrize = pytest.mark.parametrize

View file

@ -7,7 +7,6 @@ trip through the lexer.
import calf.lexer as cl
from conftest import parametrize
import pytest

View file

@ -4,7 +4,6 @@ Tests of calf.parser
import calf.parser as cp
from conftest import parametrize
import pytest

View file

@ -1,9 +1,8 @@
"""
"""
from conftest import parametrize
from calf.reader import read_buffer
from conftest import parametrize
@parametrize(

View file

@ -58,13 +58,13 @@ from datalog.debris import Timing
from datalog.evaluator import select
from datalog.reader import pr_str, read_command, read_dataset
from datalog.types import (
CachedDataset,
Constant,
Dataset,
LVar,
PartlyIndexedDataset,
Rule,
TableIndexedDataset,
CachedDataset,
Constant,
Dataset,
LVar,
PartlyIndexedDataset,
Rule,
TableIndexedDataset
)
from prompt_toolkit import print_formatted_text, prompt, PromptSession

View file

@ -7,14 +7,7 @@ from itertools import chain
from datalog.parser import parse
from datalog.reader import pr_str, read
from datalog.types import (
CachedDataset,
Constant,
Dataset,
LVar,
Rule,
TableIndexedDataset,
)
from datalog.types import CachedDataset, Constant, Dataset, LVar, Rule, TableIndexedDataset
def match(tuple, expr, bindings=None):

View file

@ -2,13 +2,13 @@
from datalog.easy import read, select
from datalog.types import (
CachedDataset,
Constant,
Dataset,
LVar,
PartlyIndexedDataset,
Rule,
TableIndexedDataset,
CachedDataset,
Constant,
Dataset,
LVar,
PartlyIndexedDataset,
Rule,
TableIndexedDataset
)
import pytest

View file

@ -2,9 +2,10 @@
Reader tests.
"""
from datalog.reader import read
import pytest
from datalog.reader import read
EXS = [
"%foo\n",

View file

@ -1,5 +1,6 @@
from setuptools import setup
setup(
name="arrdem.flowmetal",
# Package metadata

View file

@ -2,9 +2,8 @@
The Flowmetal server entry point.
"""
from flowmetal import frontend, interpreter, scheduler, reaper
import click
from flowmetal import frontend, interpreter, reaper, scheduler
@click.group()

View file

@ -2,8 +2,8 @@
Quick and shitty Gandi REST API driver
"""
import json
from datetime import datetime
import json
import requests

View file

@ -2,21 +2,21 @@
A quick and dirty public DNS script, super tightly coupled to my infrastructure.
"""
import sys
import os
import argparse
import re
import os
from pprint import pprint
import re
import sys
for e in sys.path:
print(e)
from gandi.client import GandiAPI
import jinja2
import meraki
import pkg_resources
import yaml
import meraki
RECORD_LINE_PATTERN = re.compile(

View file

@ -1,5 +1,6 @@
from setuptools import setup
setup(
name="arrdem.ratchet",
# Package metadata

View file

@ -8,7 +8,7 @@ from abc import ABC, abstractmethod
class Message:
"""Messages can be sent. That's it.
Messages have headers, which may
Messages have headers, which may
Other things can filter the stream of inbound messages and do log processing, but that's the whole basis of the
thing.
@ -67,57 +67,51 @@ class Driver(ABC):
"""Shared interface for Ratchet backend drivers."""
@abstractmethod
def __init__(message_ttl=60000,
message_space="_",
message_author=""):
def __init__(message_ttl=60000, message_space="_", message_author=""):
"""Initialize the driver."""
@abstractmethod
def create_message(self,
message: str,
ttl: int = None,
space: str = None,
author: str = None) -> Message:
def create_message(
self, message: str, ttl: int = None, space: str = None, author: str = None
) -> Message:
"""Create a single message."""
@abstractmethod
def create_event(self,
timeout: int,
ttl: int = None,
space: str = None,
author: str = None):
def create_event(
self, timeout: int, ttl: int = None, space: str = None, author: str = None
):
"""Create a (pending) event."""
@abstractmethod
def set_event(self,
timeout: int,
ttl: int = None,
space: str = None,
author: str = None):
def set_event(
self, timeout: int, ttl: int = None, space: str = None, author: str = None
):
"""Attempt to mark an event as set."""
@abstractmethod
def create_request(self,
body: str,
timeout: int,
ttl: int = None,
space: str = None,
author: str = None):
def create_request(
self,
body: str,
timeout: int,
ttl: int = None,
space: str = None,
author: str = None,
):
"""Create a (pending) request."""
@abstractmethod
def deliver_request(self,
request_id,
response: str,
ttl: int = None,
space: str = None,
author: str = None):
def deliver_request(
self,
request_id,
response: str,
ttl: int = None,
space: str = None,
author: str = None,
):
"""Deliver a response to a (pending) request."""
@abstractmethod
def revoke_request(self,
request_id,
ttl: int = None,
space: str = None,
author: str = None):
def revoke_request(
self, request_id, ttl: int = None, space: str = None, author: str = None
):
"""Revoke a (pending) request."""

View file

@ -2,12 +2,12 @@
An implementation of the ratchet model against SQLite.
"""
import os
import sqlite3 as sql
from contextlib import closing
import os
import socket
import sqlite3 as sql
from ratchet import Message, Event, Request
from ratchet import Event, Message, Request
SCHEMA_SCRIPT = """

View file

@ -1,5 +1,6 @@
from setuptools import setup
setup(
name="arrdem.yamlschema",
# Package metadata

View file

@ -2,11 +2,11 @@
JSONSchema linting for YAML documents.
"""
import logging
import typing as t
from enum import Enum
from io import StringIO
import logging
import re
import typing as t
import yaml
from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode
@ -58,9 +58,13 @@ class YamlLinter(object):
schema = self._schema
for e in path:
if not e:
raise ValueError(f"Unable to dereference {ref}; contains empty segment!")
raise ValueError(
f"Unable to dereference {ref}; contains empty segment!"
)
if not (schema := schema.get(e)):
raise ValueError(f"Unable to dereference {ref}; references missing sub-document!")
raise ValueError(
f"Unable to dereference {ref}; references missing sub-document!"
)
return schema
@ -175,7 +179,10 @@ class YamlLinter(object):
else:
yield LintRecord(
LintLevel.MISSMATCH, node, schema, f"Expected an integer, got a {node.tag}"
LintLevel.MISSMATCH,
node,
schema,
f"Expected an integer, got a {node.tag}",
)
def lint_number(self, schema, node: Node) -> t.Iterable[LintRecord]:
@ -185,7 +192,10 @@ class YamlLinter(object):
else:
yield LintRecord(
LintLevel.MISSMATCH, node, schema, f"Expected an integer, got a {node.tag}"
LintLevel.MISSMATCH,
node,
schema,
f"Expected an integer, got a {node.tag}",
)
def _lint_num_range(self, schema, node: Node, value) -> t.Iterable[LintRecord]:

View file

@ -2,9 +2,8 @@
Tests covering the YAML linter.
"""
from yamlschema import lint_buffer
import pytest
from yamlschema import lint_buffer
@pytest.mark.parametrize(
@ -100,20 +99,31 @@ def test_lint_document_fails(msg, schema, obj):
assert list(lint_buffer(schema, obj)), msg
@pytest.mark.parametrize("msg, schema, obj", [
("Basic usage of $ref",
{"$ref": "#/definitions/Foo",
"definitions": {
"Foo": {"type": "string"},
}},
"---\nfoo"),
("Use of nested references",
{"$ref": "#/definitions/Foos",
"definitions": {
"Foos": {"type": "array", "items": {"$ref": "#/definitions/Foo"}},
"Foo": {"type": "string"},
}},
"---\n- foo\n- bar\n- baz"),
])
@pytest.mark.parametrize(
"msg, schema, obj",
[
(
"Basic usage of $ref",
{
"$ref": "#/definitions/Foo",
"definitions": {
"Foo": {"type": "string"},
},
},
"---\nfoo",
),
(
"Use of nested references",
{
"$ref": "#/definitions/Foos",
"definitions": {
"Foos": {"type": "array", "items": {"$ref": "#/definitions/Foo"}},
"Foo": {"type": "string"},
},
},
"---\n- foo\n- bar\n- baz",
),
],
)
def test_ref_references(msg, schema, obj):
assert not list(lint_buffer(schema, obj)), msg

View file

@ -70,6 +70,7 @@ py_pytest(
],
deps = [
py_requirement("requests"),
py_requirement("requirements-parser"),
]
)

View file

@ -9,6 +9,7 @@ import sys
from autoflake import main
if __name__ == "__main__":
sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
sys.exit(main())

View file

@ -1,10 +1,10 @@
"""A shim for executing pytest."""
import os
import sys
import pytest
if __name__ == "__main__":
cmdline = ["--ignore=external"] + sys.argv[1:]
sys.exit(pytest.main(cmdline))

View file

@ -9,6 +9,7 @@ import sys
from isort.main import main
if __name__ == "__main__":
sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0])
sys.exit(main())

View file

@ -7,6 +7,7 @@ import sys
from openapi_spec_validator.__main__ import main
if __name__ == "__main__":
sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
sys.exit(main())

View file

@ -64,6 +64,7 @@ redis==3.5.3
regex==2021.4.4
requests==2.25.1
requests-toolbelt==0.9.1
requirements-parser==0.2.0
rfc3986==1.5.0
SecretStorage==3.3.1
six==1.15.0

View file

@ -12,6 +12,7 @@ import click
import jinja2
import yaml
FONTMATTER_PATTERN = re.compile(
r"^(---\n\r?(?P<fontmatter>.*?)\n\r?---\n\r?)?(?P<content>.+)$", re.DOTALL
)

View file

@ -6,35 +6,51 @@ import re
import pytest
import requests
import requirements
from requirements.requirement import Requirement
# Licenses approved as representing non-copyleft and not precluding commercial usage.
# This is all easy, there's a good schema here.
APPROVED_LICENSES = [
"License :: OSI Approved :: MIT License",
"License :: OSI Approved :: Apache Software License",
"License :: OSI Approved :: BSD License",
"License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
"License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)",
"License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
"License :: OSI Approved :: Python Software Foundation License",
"License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)",
"License :: OSI Approved :: ISC License (ISCL)",
MIT := "License :: OSI Approved :: MIT License",
APACHE := "License :: OSI Approved :: Apache Software License",
BSD := "License :: OSI Approved :: BSD License",
MPL10 := "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
MPL11 := "License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)",
MPL20 := "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
PSFL := "License :: OSI Approved :: Python Software Foundation License",
LGPL := "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)",
ISCL := "License :: OSI Approved :: ISC License (ISCL)",
]
UNAPPROVED_LICENSES = [
GPL1 := "License :: OSI Approved :: GNU General Public License",
GPL2 := "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
GPL3 := "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
]
# This data is GARBO.
LICENSES_BY_LOWERNAME = {
"apache 2.0": "License :: OSI Approved :: Apache Software License",
"apache": "License :: OSI Approved :: Apache Software License",
"bsd 3 clause": "License :: OSI Approved :: BSD License",
"bsd 3-clause": "License :: OSI Approved :: BSD License",
"bsd": "License :: OSI Approved :: BSD License",
"gplv3": "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"http://www.apache.org/licenses/license-2.0": "License :: OSI Approved :: Apache Software License",
"isc": "License :: OSI Approved :: ISC License (ISCL)",
"mit": "License :: OSI Approved :: MIT License",
"mpl 2.0": "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
"mpl": "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
"psf": "License :: OSI Approved :: Python Software Foundation License",
"apache 2.0": APACHE,
"apache": APACHE,
"http://www.apache.org/licenses/license-2.0": APACHE,
"bsd 3": BSD,
"bsd": BSD,
"gpl": GPL1,
"gpl2": GPL2,
"gpl3": GPL3,
"isc": ISCL,
"mit": MIT,
"mpl": MPL10,
"mpl 2.0": MPL20,
"psf": PSFL,
}
# Mash in some cases.
@ -49,52 +65,38 @@ APPROVED_PACKAGES = [
"anosql", # BSD
]
REQ_PATTERN = re.compile(
r"(?P<pkgname>[a-zA-Z0-9_-]+)(?P<features>\[.*?\])?==(?P<version>[^\s;#]+)|(.*?#egg=(?P<eggname>[a-zA-Z0-9_-]+))"
)
def parse_requirement(line):
"""Given a requirement return the requirement name and version as a tuple.
Only the strict `==` version pinning subset is supported.
Features are supported.
"""
if m := re.match(REQ_PATTERN, line):
return (m.group("pkgname") or m.group("eggname")), m.group("version")
@pytest.mark.parametrize(
"line,t",
[
("foo==1.2.3", ("foo", "1.2.3")),
("foo[bar]==1.2.3", ("foo", "1.2.3")),
("foo[bar, baz, qux]==1.2.3", ("foo", "1.2.3")),
# Various stuff we should ignore
("# comment line", None),
(" # garbage whitespace", None),
(" \t", None),
],
)
def test_parse_requirement(line, t):
"""The irony of testing one"s tests is not lost."""
assert parse_requirement(line) == t
with open("tools/python/requirements.txt") as f:
PACKAGES = [parse_requirement(l) for l in f.readlines()]
with open("tools/python/requirements.txt") as fd:
PACKAGES = list(requirements.parse(fd))
def bash_license(ln):
if ln:
ln = re.sub("[(),]|( version)|( license)", "", ln.lower())
ln = LICENSES_BY_LOWERNAME.get(ln, ln)
while True:
lnn = re.sub(r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)", "", ln.lower())
if ln != lnn:
ln = lnn
else:
break
ln = LICENSES_BY_LOWERNAME.get(ln, ln)
return ln
def licenses(package, version):
@pytest.mark.parametrize("a,b", [
("MIT", MIT),
("mit", MIT),
("BSD", BSD),
("BSD 3-clause", BSD),
("BSD 3 clause", BSD),
("GPL3", GPL3),
("GPL v3", GPL3),
("GPLv3", GPL3),
])
def test_bash_license(a, b):
assert bash_license(a) == b
def licenses(package: Requirement):
"""Get package metadata (the licenses list) from PyPi.
pip and other tools use the local package metadata to introspect licenses which requires that
@ -104,11 +106,16 @@ def licenses(package, version):
"""
l = []
version = next((v for op, v in package.specs if op == "=="), None)
print(package.name, version)
# If we don't have a version (eg. forked git dep) assume we've got the same license constraints
# as the latest upstream release. After all we can't re-license stuff.
if not version:
blob = requests.get(f"https://pypi.python.org/pypi/{package}/json").json()
blob = requests.get(
f"https://pypi.org/pypi/{package.name}/json",
headers={"Accept": "application/json"}
).json()
if ln := bash_license(blob.get("license")):
l.append(ln)
else:
@ -120,7 +127,8 @@ def licenses(package, version):
# If we have a version, try to pull that release's metadata since it may have more/better.
if version:
blob = requests.get(
f"https://pypi.python.org/pypi/{package}/{version}/json"
f"https://pypi.org/pypi/{package.name}/{version}/json",
headers={"Accept": "application/json"}
).json()
l = [
c
@ -134,11 +142,11 @@ def licenses(package, version):
return l
@pytest.mark.parametrize("package,version", PACKAGES)
def test_approved_license(package, version):
@pytest.mark.parametrize("package", PACKAGES)
def test_approved_license(package):
"""Ensure that a given package is either allowed by name or uses an approved license."""
_licenses = licenses(package, version)
assert package in APPROVED_PACKAGES or any(
_licenses = licenses(package)
assert package.name in APPROVED_PACKAGES or any(
l in APPROVED_LICENSES for l in _licenses
), f"{package} was not approved and its license(s) were unknown {_licenses!r}"

View file

@ -7,5 +7,6 @@ Shim for executing isort.
from unify import main
if __name__ == "__main__":
exit(main())

View file

@ -9,6 +9,7 @@ import sys
from yamllint.cli import run
if __name__ == "__main__":
sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0])
sys.exit(run())