Black all the things
This commit is contained in:
parent
7170fd40a8
commit
2494211ef2
31 changed files with 395 additions and 294 deletions
|
@ -100,10 +100,16 @@ def create_tables(queries: Queries, conn) -> None:
|
||||||
log.info("Created migrations table")
|
log.info("Created migrations table")
|
||||||
|
|
||||||
# Insert the bootstrap 'fixup' record
|
# Insert the bootstrap 'fixup' record
|
||||||
execute_migration(queries, conn,
|
execute_migration(
|
||||||
MigrationDescriptor(
|
queries,
|
||||||
name="anosql_migrations_create_table",
|
conn,
|
||||||
sha256sum=sha256(queries.anosql_migrations_create_table.sql.encode("utf-8")).hexdigest()))
|
MigrationDescriptor(
|
||||||
|
name="anosql_migrations_create_table",
|
||||||
|
sha256sum=sha256(
|
||||||
|
queries.anosql_migrations_create_table.sql.encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def committed_migrations(queries: Queries, conn) -> t.Iterable[MigrationDescriptor]:
|
def committed_migrations(queries: Queries, conn) -> t.Iterable[MigrationDescriptor]:
|
||||||
|
@ -133,7 +139,8 @@ def available_migrations(queries: Queries, conn) -> t.Iterable[MigrationDescript
|
||||||
yield MigrationDescriptor(
|
yield MigrationDescriptor(
|
||||||
name=query_name,
|
name=query_name,
|
||||||
committed_at=None,
|
committed_at=None,
|
||||||
sha256sum=sha256(query_fn.sql.encode("utf-8")).hexdigest())
|
sha256sum=sha256(query_fn.sql.encode("utf-8")).hexdigest(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def execute_migration(queries: Queries, conn, migration: MigrationDescriptor):
|
def execute_migration(queries: Queries, conn, migration: MigrationDescriptor):
|
||||||
|
|
|
@ -15,7 +15,9 @@ CREATE TABLE kv (`id` INT, `key` TEXT, `value` TEXT);
|
||||||
|
|
||||||
|
|
||||||
def table_exists(conn, table_name):
|
def table_exists(conn, table_name):
|
||||||
return list(conn.execute(f"""\
|
return list(
|
||||||
|
conn.execute(
|
||||||
|
f"""\
|
||||||
SELECT (
|
SELECT (
|
||||||
`name`
|
`name`
|
||||||
)
|
)
|
||||||
|
@ -23,7 +25,9 @@ def table_exists(conn, table_name):
|
||||||
WHERE
|
WHERE
|
||||||
`type` = 'table'
|
`type` = 'table'
|
||||||
AND `name` = '{table_name}'
|
AND `name` = '{table_name}'
|
||||||
;"""))
|
;"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
@ -36,7 +40,9 @@ def conn() -> sqlite3.Connection:
|
||||||
def test_connect(conn: sqlite3.Connection):
|
def test_connect(conn: sqlite3.Connection):
|
||||||
"""Assert that the connection works and we can execute against it."""
|
"""Assert that the connection works and we can execute against it."""
|
||||||
|
|
||||||
assert list(conn.execute("SELECT 1;")) == [(1, ), ]
|
assert list(conn.execute("SELECT 1;")) == [
|
||||||
|
(1,),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
@ -66,7 +72,9 @@ def test_migrations_list(conn, queries):
|
||||||
"""Test that we can list out available migrations."""
|
"""Test that we can list out available migrations."""
|
||||||
|
|
||||||
ms = list(anosql_migrations.available_migrations(queries, conn))
|
ms = list(anosql_migrations.available_migrations(queries, conn))
|
||||||
assert any(m.name == "migration_0000_create_kv" for m in ms), f"Didn't find in {ms!r}"
|
assert any(
|
||||||
|
m.name == "migration_0000_create_kv" for m in ms
|
||||||
|
), f"Didn't find in {ms!r}"
|
||||||
|
|
||||||
|
|
||||||
def test_committed_migrations(conn, queries):
|
def test_committed_migrations(conn, queries):
|
||||||
|
@ -96,4 +104,6 @@ def test_post_committed_migrations(migrated_conn, queries):
|
||||||
"""Assert that the create_kv migration has been committed."""
|
"""Assert that the create_kv migration has been committed."""
|
||||||
|
|
||||||
ms = list(anosql_migrations.committed_migrations(queries, migrated_conn))
|
ms = list(anosql_migrations.committed_migrations(queries, migrated_conn))
|
||||||
assert any(m.name == "migration_0000_create_kv" for m in ms), "\n".join(migrated_conn.iterdump())
|
assert any(m.name == "migration_0000_create_kv" for m in ms), "\n".join(
|
||||||
|
migrated_conn.iterdump()
|
||||||
|
)
|
||||||
|
|
|
@ -240,29 +240,25 @@ htmlhelp_basename = "anosqldoc"
|
||||||
# -- Options for LaTeX output ---------------------------------------------
|
# -- Options for LaTeX output ---------------------------------------------
|
||||||
|
|
||||||
latex_elements = {
|
latex_elements = {
|
||||||
# The paper size ('letterpaper' or 'a4paper').
|
# The paper size ('letterpaper' or 'a4paper').
|
||||||
#
|
#
|
||||||
# 'papersize': 'letterpaper',
|
# 'papersize': 'letterpaper',
|
||||||
|
# The font size ('10pt', '11pt' or '12pt').
|
||||||
# The font size ('10pt', '11pt' or '12pt').
|
#
|
||||||
#
|
# 'pointsize': '10pt',
|
||||||
# 'pointsize': '10pt',
|
# Additional stuff for the LaTeX preamble.
|
||||||
|
#
|
||||||
# Additional stuff for the LaTeX preamble.
|
# 'preamble': '',
|
||||||
#
|
# Latex figure (float) alignment
|
||||||
# 'preamble': '',
|
#
|
||||||
|
# 'figure_align': 'htbp',
|
||||||
# Latex figure (float) alignment
|
|
||||||
#
|
|
||||||
# 'figure_align': 'htbp',
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Grouping the document tree into LaTeX files. List of tuples
|
# Grouping the document tree into LaTeX files. List of tuples
|
||||||
# (source start file, target name, title,
|
# (source start file, target name, title,
|
||||||
# author, documentclass [howto, manual, or own class]).
|
# author, documentclass [howto, manual, or own class]).
|
||||||
latex_documents = [
|
latex_documents = [
|
||||||
(master_doc, "anosql.tex", u"anosql Documentation",
|
(master_doc, "anosql.tex", u"anosql Documentation", u"Honza Pokorny", "manual"),
|
||||||
u"Honza Pokorny", "manual"),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
# The name of an image file (relative to this directory) to place at the top of
|
# The name of an image file (relative to this directory) to place at the top of
|
||||||
|
@ -302,10 +298,7 @@ latex_documents = [
|
||||||
|
|
||||||
# One entry per manual page. List of tuples
|
# One entry per manual page. List of tuples
|
||||||
# (source start file, name, description, authors, manual section).
|
# (source start file, name, description, authors, manual section).
|
||||||
man_pages = [
|
man_pages = [(master_doc, "anosql", u"anosql Documentation", [author], 1)]
|
||||||
(master_doc, "anosql", u"anosql Documentation",
|
|
||||||
[author], 1)
|
|
||||||
]
|
|
||||||
|
|
||||||
# If true, show URL addresses after external links.
|
# If true, show URL addresses after external links.
|
||||||
#
|
#
|
||||||
|
@ -318,9 +311,15 @@ man_pages = [
|
||||||
# (source start file, target name, title, author,
|
# (source start file, target name, title, author,
|
||||||
# dir menu entry, description, category)
|
# dir menu entry, description, category)
|
||||||
texinfo_documents = [
|
texinfo_documents = [
|
||||||
(master_doc, "anosql", u"anosql Documentation",
|
(
|
||||||
author, "anosql", "One line description of project.",
|
master_doc,
|
||||||
"Miscellaneous"),
|
"anosql",
|
||||||
|
u"anosql Documentation",
|
||||||
|
author,
|
||||||
|
"anosql",
|
||||||
|
"One line description of project.",
|
||||||
|
"Miscellaneous",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
# Documents to append as an appendix to all manuals.
|
# Documents to append as an appendix to all manuals.
|
||||||
|
|
|
@ -2,4 +2,10 @@ from .core import from_path, from_str, SQLOperationType
|
||||||
from .exceptions import SQLLoadException, SQLParseException
|
from .exceptions import SQLLoadException, SQLParseException
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["from_path", "from_str", "SQLOperationType", "SQLLoadException", "SQLParseException"]
|
__all__ = [
|
||||||
|
"from_path",
|
||||||
|
"from_str",
|
||||||
|
"SQLOperationType",
|
||||||
|
"SQLLoadException",
|
||||||
|
"SQLParseException",
|
||||||
|
]
|
||||||
|
|
|
@ -53,7 +53,9 @@ class SQLite3DriverAdapter(object):
|
||||||
conn.execute(sql, parameters)
|
conn.execute(sql, parameters)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def insert_update_delete_many(conn: sqlite3.Connection, _query_name, sql, parameters):
|
def insert_update_delete_many(
|
||||||
|
conn: sqlite3.Connection, _query_name, sql, parameters
|
||||||
|
):
|
||||||
log.debug({"sql": sql, "parameters": parameters})
|
log.debug({"sql": sql, "parameters": parameters})
|
||||||
conn.executemany(sql, parameters)
|
conn.executemany(sql, parameters)
|
||||||
|
|
||||||
|
|
|
@ -4,10 +4,10 @@ from .adapters.psycopg2 import PsycoPG2Adapter
|
||||||
from .adapters.sqlite3 import SQLite3DriverAdapter
|
from .adapters.sqlite3 import SQLite3DriverAdapter
|
||||||
from .exceptions import SQLLoadException, SQLParseException
|
from .exceptions import SQLLoadException, SQLParseException
|
||||||
from .patterns import (
|
from .patterns import (
|
||||||
doc_comment_pattern,
|
doc_comment_pattern,
|
||||||
empty_pattern,
|
empty_pattern,
|
||||||
query_name_definition_pattern,
|
query_name_definition_pattern,
|
||||||
valid_query_name_pattern
|
valid_query_name_pattern,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -89,8 +89,8 @@ def get_driver_adapter(driver_name):
|
||||||
|
|
||||||
|
|
||||||
class SQLOperationType(object):
|
class SQLOperationType(object):
|
||||||
"""Enumeration (kind of) of anosql operation types
|
"""Enumeration (kind of) of anosql operation types"""
|
||||||
"""
|
|
||||||
INSERT_RETURNING = 0
|
INSERT_RETURNING = 0
|
||||||
INSERT_UPDATE_DELETE = 1
|
INSERT_UPDATE_DELETE = 1
|
||||||
INSERT_UPDATE_DELETE_MANY = 2
|
INSERT_UPDATE_DELETE_MANY = 2
|
||||||
|
@ -168,9 +168,13 @@ def _create_fns(query_name, docs, op_type, sql, driver_adapter):
|
||||||
if op_type == SQLOperationType.INSERT_RETURNING:
|
if op_type == SQLOperationType.INSERT_RETURNING:
|
||||||
return driver_adapter.insert_returning(conn, query_name, sql, parameters)
|
return driver_adapter.insert_returning(conn, query_name, sql, parameters)
|
||||||
elif op_type == SQLOperationType.INSERT_UPDATE_DELETE:
|
elif op_type == SQLOperationType.INSERT_UPDATE_DELETE:
|
||||||
return driver_adapter.insert_update_delete(conn, query_name, sql, parameters)
|
return driver_adapter.insert_update_delete(
|
||||||
|
conn, query_name, sql, parameters
|
||||||
|
)
|
||||||
elif op_type == SQLOperationType.INSERT_UPDATE_DELETE_MANY:
|
elif op_type == SQLOperationType.INSERT_UPDATE_DELETE_MANY:
|
||||||
return driver_adapter.insert_update_delete_many(conn, query_name, sql, *parameters)
|
return driver_adapter.insert_update_delete_many(
|
||||||
|
conn, query_name, sql, *parameters
|
||||||
|
)
|
||||||
elif op_type == SQLOperationType.SCRIPT:
|
elif op_type == SQLOperationType.SCRIPT:
|
||||||
return driver_adapter.execute_script(conn, sql)
|
return driver_adapter.execute_script(conn, sql)
|
||||||
elif op_type == SQLOperationType.SELECT_ONE_ROW:
|
elif op_type == SQLOperationType.SELECT_ONE_ROW:
|
||||||
|
@ -351,5 +355,5 @@ def from_path(sql_path, driver_name):
|
||||||
else:
|
else:
|
||||||
raise SQLLoadException(
|
raise SQLLoadException(
|
||||||
"The sql_path must be a directory or file, got {}".format(sql_path),
|
"The sql_path must be a directory or file, got {}".format(sql_path),
|
||||||
sql_path
|
sql_path,
|
||||||
)
|
)
|
||||||
|
|
|
@ -102,10 +102,15 @@ def pg_conn(postgresql):
|
||||||
|
|
||||||
with postgresql.cursor() as cur:
|
with postgresql.cursor() as cur:
|
||||||
with open(USERS_DATA_PATH) as fp:
|
with open(USERS_DATA_PATH) as fp:
|
||||||
cur.copy_from(fp, "users", sep=",", columns=["username", "firstname", "lastname"])
|
cur.copy_from(
|
||||||
|
fp, "users", sep=",", columns=["username", "firstname", "lastname"]
|
||||||
|
)
|
||||||
with open(BLOGS_DATA_PATH) as fp:
|
with open(BLOGS_DATA_PATH) as fp:
|
||||||
cur.copy_from(
|
cur.copy_from(
|
||||||
fp, "blogs", sep=",", columns=["userid", "title", "content", "published"]
|
fp,
|
||||||
|
"blogs",
|
||||||
|
sep=",",
|
||||||
|
columns=["userid", "title", "content", "published"],
|
||||||
)
|
)
|
||||||
|
|
||||||
return postgresql
|
return postgresql
|
||||||
|
|
|
@ -29,17 +29,26 @@ def test_record_query(pg_conn, queries):
|
||||||
|
|
||||||
def test_parameterized_query(pg_conn, queries):
|
def test_parameterized_query(pg_conn, queries):
|
||||||
actual = queries.blogs.get_user_blogs(pg_conn, userid=1)
|
actual = queries.blogs.get_user_blogs(pg_conn, userid=1)
|
||||||
expected = [("How to make a pie.", date(2018, 11, 23)), ("What I did Today", date(2017, 7, 28))]
|
expected = [
|
||||||
|
("How to make a pie.", date(2018, 11, 23)),
|
||||||
|
("What I did Today", date(2017, 7, 28)),
|
||||||
|
]
|
||||||
assert actual == expected
|
assert actual == expected
|
||||||
|
|
||||||
|
|
||||||
def test_parameterized_record_query(pg_conn, queries):
|
def test_parameterized_record_query(pg_conn, queries):
|
||||||
dsn = pg_conn.get_dsn_parameters()
|
dsn = pg_conn.get_dsn_parameters()
|
||||||
with psycopg2.connect(cursor_factory=psycopg2.extras.RealDictCursor, **dsn) as conn:
|
with psycopg2.connect(cursor_factory=psycopg2.extras.RealDictCursor, **dsn) as conn:
|
||||||
actual = queries.blogs.pg_get_blogs_published_after(conn, published=date(2018, 1, 1))
|
actual = queries.blogs.pg_get_blogs_published_after(
|
||||||
|
conn, published=date(2018, 1, 1)
|
||||||
|
)
|
||||||
|
|
||||||
expected = [
|
expected = [
|
||||||
{"title": "How to make a pie.", "username": "bobsmith", "published": "2018-11-23 00:00"},
|
{
|
||||||
|
"title": "How to make a pie.",
|
||||||
|
"username": "bobsmith",
|
||||||
|
"published": "2018-11-23 00:00",
|
||||||
|
},
|
||||||
{"title": "Testing", "username": "janedoe", "published": "2018-01-01 00:00"},
|
{"title": "Testing", "username": "janedoe", "published": "2018-01-01 00:00"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ import pytest
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sqlite(request):
|
def sqlite(request):
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
|
||||||
sqlconnection = sqlite3.connect(":memory:")
|
sqlconnection = sqlite3.connect(":memory:")
|
||||||
|
|
||||||
def fin():
|
def fin():
|
||||||
|
@ -18,11 +19,13 @@ def sqlite(request):
|
||||||
|
|
||||||
|
|
||||||
def test_simple_query(sqlite):
|
def test_simple_query(sqlite):
|
||||||
_test_create_insert = ("-- name: create-some-table#\n"
|
_test_create_insert = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (a, b, c);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value!\n"
|
"CREATE TABLE foo (a, b, c);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n")
|
"-- name: insert-some-value!\n"
|
||||||
|
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_test_create_insert, "sqlite3")
|
q = anosql.from_str(_test_create_insert, "sqlite3")
|
||||||
q.create_some_table(sqlite)
|
q.create_some_table(sqlite)
|
||||||
|
@ -30,11 +33,13 @@ def test_simple_query(sqlite):
|
||||||
|
|
||||||
|
|
||||||
def test_auto_insert_query(sqlite):
|
def test_auto_insert_query(sqlite):
|
||||||
_test_create_insert = ("-- name: create-some-table#\n"
|
_test_create_insert = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (a, b, c);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value<!\n"
|
"CREATE TABLE foo (a, b, c);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n")
|
"-- name: insert-some-value<!\n"
|
||||||
|
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_test_create_insert, "sqlite3")
|
q = anosql.from_str(_test_create_insert, "sqlite3")
|
||||||
q.create_some_table(sqlite)
|
q.create_some_table(sqlite)
|
||||||
|
@ -44,13 +49,15 @@ def test_auto_insert_query(sqlite):
|
||||||
|
|
||||||
|
|
||||||
def test_parametrized_insert(sqlite):
|
def test_parametrized_insert(sqlite):
|
||||||
_test_create_insert = ("-- name: create-some-table#\n"
|
_test_create_insert = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (a, b, c);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value!\n"
|
"CREATE TABLE foo (a, b, c);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (?, ?, ?);\n\n"
|
"-- name: insert-some-value!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (?, ?, ?);\n\n"
|
||||||
"SELECT * FROM foo;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT * FROM foo;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_test_create_insert, "sqlite3")
|
q = anosql.from_str(_test_create_insert, "sqlite3")
|
||||||
q.create_some_table(sqlite)
|
q.create_some_table(sqlite)
|
||||||
|
@ -59,13 +66,15 @@ def test_parametrized_insert(sqlite):
|
||||||
|
|
||||||
|
|
||||||
def test_parametrized_insert_named(sqlite):
|
def test_parametrized_insert_named(sqlite):
|
||||||
_test_create_insert = ("-- name: create-some-table#\n"
|
_test_create_insert = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (a, b, c);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value!\n"
|
"CREATE TABLE foo (a, b, c);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (:a, :b, :c);\n\n"
|
"-- name: insert-some-value!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (:a, :b, :c);\n\n"
|
||||||
"SELECT * FROM foo;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT * FROM foo;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_test_create_insert, "sqlite3")
|
q = anosql.from_str(_test_create_insert, "sqlite3")
|
||||||
q.create_some_table(sqlite)
|
q.create_some_table(sqlite)
|
||||||
|
@ -74,23 +83,27 @@ def test_parametrized_insert_named(sqlite):
|
||||||
|
|
||||||
|
|
||||||
def test_one_row(sqlite):
|
def test_one_row(sqlite):
|
||||||
_test_one_row = ("-- name: one-row?\n"
|
_test_one_row = (
|
||||||
"SELECT 1, 'hello';\n\n"
|
"-- name: one-row?\n"
|
||||||
"-- name: two-rows?\n"
|
"SELECT 1, 'hello';\n\n"
|
||||||
"SELECT 1 UNION SELECT 2;\n")
|
"-- name: two-rows?\n"
|
||||||
|
"SELECT 1 UNION SELECT 2;\n"
|
||||||
|
)
|
||||||
q = anosql.from_str(_test_one_row, "sqlite3")
|
q = anosql.from_str(_test_one_row, "sqlite3")
|
||||||
assert q.one_row(sqlite) == (1, "hello")
|
assert q.one_row(sqlite) == (1, "hello")
|
||||||
assert q.two_rows(sqlite) is None
|
assert q.two_rows(sqlite) is None
|
||||||
|
|
||||||
|
|
||||||
def test_simple_query_pg(postgresql):
|
def test_simple_query_pg(postgresql):
|
||||||
_queries = ("-- name: create-some-table#\n"
|
_queries = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value!\n"
|
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n\n"
|
"-- name: insert-some-value!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n\n"
|
||||||
"SELECT a, b, c FROM foo;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT a, b, c FROM foo;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_queries, "psycopg2")
|
q = anosql.from_str(_queries, "psycopg2")
|
||||||
|
|
||||||
|
@ -101,13 +114,15 @@ def test_simple_query_pg(postgresql):
|
||||||
|
|
||||||
|
|
||||||
def test_auto_insert_query_pg(postgresql):
|
def test_auto_insert_query_pg(postgresql):
|
||||||
_queries = ("-- name: create-some-table#\n"
|
_queries = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value<!\n"
|
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3) returning id;\n\n"
|
"-- name: insert-some-value<!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3) returning id;\n\n"
|
||||||
"SELECT a, b, c FROM foo;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT a, b, c FROM foo;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_queries, "psycopg2")
|
q = anosql.from_str(_queries, "psycopg2")
|
||||||
|
|
||||||
|
@ -118,13 +133,15 @@ def test_auto_insert_query_pg(postgresql):
|
||||||
|
|
||||||
|
|
||||||
def test_parameterized_insert_pg(postgresql):
|
def test_parameterized_insert_pg(postgresql):
|
||||||
_queries = ("-- name: create-some-table#\n"
|
_queries = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value!\n"
|
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (%s, %s, %s);\n\n"
|
"-- name: insert-some-value!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (%s, %s, %s);\n\n"
|
||||||
"SELECT a, b, c FROM foo;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT a, b, c FROM foo;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_queries, "psycopg2")
|
q = anosql.from_str(_queries, "psycopg2")
|
||||||
|
|
||||||
|
@ -135,13 +152,15 @@ def test_parameterized_insert_pg(postgresql):
|
||||||
|
|
||||||
|
|
||||||
def test_auto_parameterized_insert_query_pg(postgresql):
|
def test_auto_parameterized_insert_query_pg(postgresql):
|
||||||
_queries = ("-- name: create-some-table#\n"
|
_queries = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value<!\n"
|
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (%s, %s, %s) returning id;\n\n"
|
"-- name: insert-some-value<!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (%s, %s, %s) returning id;\n\n"
|
||||||
"SELECT a, b, c FROM foo;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT a, b, c FROM foo;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_queries, "psycopg2")
|
q = anosql.from_str(_queries, "psycopg2")
|
||||||
|
|
||||||
|
@ -154,13 +173,15 @@ def test_auto_parameterized_insert_query_pg(postgresql):
|
||||||
|
|
||||||
|
|
||||||
def test_parameterized_select_pg(postgresql):
|
def test_parameterized_select_pg(postgresql):
|
||||||
_queries = ("-- name: create-some-table#\n"
|
_queries = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value!\n"
|
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3)\n\n"
|
"-- name: insert-some-value!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3)\n\n"
|
||||||
"SELECT a, b, c FROM foo WHERE a = %s;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT a, b, c FROM foo WHERE a = %s;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_queries, "psycopg2")
|
q = anosql.from_str(_queries, "psycopg2")
|
||||||
|
|
||||||
|
@ -171,13 +192,15 @@ def test_parameterized_select_pg(postgresql):
|
||||||
|
|
||||||
|
|
||||||
def test_parameterized_insert_named_pg(postgresql):
|
def test_parameterized_insert_named_pg(postgresql):
|
||||||
_queries = ("-- name: create-some-table#\n"
|
_queries = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value!\n"
|
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (%(a)s, %(b)s, %(c)s)\n\n"
|
"-- name: insert-some-value!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (%(a)s, %(b)s, %(c)s)\n\n"
|
||||||
"SELECT a, b, c FROM foo;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT a, b, c FROM foo;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_queries, "psycopg2")
|
q = anosql.from_str(_queries, "psycopg2")
|
||||||
|
|
||||||
|
@ -188,13 +211,15 @@ def test_parameterized_insert_named_pg(postgresql):
|
||||||
|
|
||||||
|
|
||||||
def test_parameterized_select_named_pg(postgresql):
|
def test_parameterized_select_named_pg(postgresql):
|
||||||
_queries = ("-- name: create-some-table#\n"
|
_queries = (
|
||||||
"-- testing insertion\n"
|
"-- name: create-some-table#\n"
|
||||||
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
"-- testing insertion\n"
|
||||||
"-- name: insert-some-value!\n"
|
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
|
||||||
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3)\n\n"
|
"-- name: insert-some-value!\n"
|
||||||
"-- name: get-all-values\n"
|
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3)\n\n"
|
||||||
"SELECT a, b, c FROM foo WHERE a = %(a)s;\n")
|
"-- name: get-all-values\n"
|
||||||
|
"SELECT a, b, c FROM foo WHERE a = %(a)s;\n"
|
||||||
|
)
|
||||||
|
|
||||||
q = anosql.from_str(_queries, "psycopg2")
|
q = anosql.from_str(_queries, "psycopg2")
|
||||||
|
|
||||||
|
@ -208,7 +233,6 @@ def test_without_trailing_semi_colon_pg():
|
||||||
"""Make sure keywords ending queries are recognized even without
|
"""Make sure keywords ending queries are recognized even without
|
||||||
semi-colons.
|
semi-colons.
|
||||||
"""
|
"""
|
||||||
_queries = ("-- name: get-by-a\n"
|
_queries = "-- name: get-by-a\n" "SELECT a, b, c FROM foo WHERE a = :a\n"
|
||||||
"SELECT a, b, c FROM foo WHERE a = :a\n")
|
|
||||||
q = anosql.from_str(_queries, "psycopg2")
|
q = anosql.from_str(_queries, "psycopg2")
|
||||||
assert q.get_by_a.sql == "SELECT a, b, c FROM foo WHERE a = %(a)s"
|
assert q.get_by_a.sql == "SELECT a, b, c FROM foo WHERE a = %(a)s"
|
||||||
|
|
|
@ -32,16 +32,25 @@ def test_record_query(sqlite3_conn, queries):
|
||||||
|
|
||||||
def test_parameterized_query(sqlite3_conn, queries):
|
def test_parameterized_query(sqlite3_conn, queries):
|
||||||
actual = queries.blogs.get_user_blogs(sqlite3_conn, userid=1)
|
actual = queries.blogs.get_user_blogs(sqlite3_conn, userid=1)
|
||||||
expected = [("How to make a pie.", "2018-11-23"), ("What I did Today", "2017-07-28")]
|
expected = [
|
||||||
|
("How to make a pie.", "2018-11-23"),
|
||||||
|
("What I did Today", "2017-07-28"),
|
||||||
|
]
|
||||||
assert actual == expected
|
assert actual == expected
|
||||||
|
|
||||||
|
|
||||||
def test_parameterized_record_query(sqlite3_conn, queries):
|
def test_parameterized_record_query(sqlite3_conn, queries):
|
||||||
sqlite3_conn.row_factory = dict_factory
|
sqlite3_conn.row_factory = dict_factory
|
||||||
actual = queries.blogs.sqlite_get_blogs_published_after(sqlite3_conn, published="2018-01-01")
|
actual = queries.blogs.sqlite_get_blogs_published_after(
|
||||||
|
sqlite3_conn, published="2018-01-01"
|
||||||
|
)
|
||||||
|
|
||||||
expected = [
|
expected = [
|
||||||
{"title": "How to make a pie.", "username": "bobsmith", "published": "2018-11-23 00:00"},
|
{
|
||||||
|
"title": "How to make a pie.",
|
||||||
|
"username": "bobsmith",
|
||||||
|
"published": "2018-11-23 00:00",
|
||||||
|
},
|
||||||
{"title": "Testing", "username": "janedoe", "published": "2018-01-01 00:00"},
|
{"title": "Testing", "username": "janedoe", "published": "2018-01-01 00:00"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -51,7 +60,10 @@ def test_parameterized_record_query(sqlite3_conn, queries):
|
||||||
def test_select_cursor_context_manager(sqlite3_conn, queries):
|
def test_select_cursor_context_manager(sqlite3_conn, queries):
|
||||||
with queries.blogs.get_user_blogs_cursor(sqlite3_conn, userid=1) as cursor:
|
with queries.blogs.get_user_blogs_cursor(sqlite3_conn, userid=1) as cursor:
|
||||||
actual = cursor.fetchall()
|
actual = cursor.fetchall()
|
||||||
expected = [("How to make a pie.", "2018-11-23"), ("What I did Today", "2017-07-28")]
|
expected = [
|
||||||
|
("How to make a pie.", "2018-11-23"),
|
||||||
|
("What I did Today", "2017-07-28"),
|
||||||
|
]
|
||||||
assert actual == expected
|
assert actual == expected
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,12 +2,15 @@ from damm import Damm
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("num", [
|
@pytest.mark.parametrize(
|
||||||
"0", # 0 itself is the start Damm state
|
"num",
|
||||||
"37", # [0, 3] => 7
|
[
|
||||||
"92", # [0, 9] => 2
|
"0", # 0 itself is the start Damm state
|
||||||
"1234", # Amusingly, this is a 0-product.
|
"37", # [0, 3] => 7
|
||||||
])
|
"92", # [0, 9] => 2
|
||||||
|
"1234", # Amusingly, this is a 0-product.
|
||||||
|
],
|
||||||
|
)
|
||||||
def test_num_verifies(num):
|
def test_num_verifies(num):
|
||||||
"""Assert that known-good Damm checks pass."""
|
"""Assert that known-good Damm checks pass."""
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@ from hypothesis import given
|
||||||
from hypothesis.strategies import integers
|
from hypothesis.strategies import integers
|
||||||
|
|
||||||
|
|
||||||
@given(integers(0, 1<<512))
|
@given(integers(0, 1 << 512))
|
||||||
def test_num_checks_verify(num):
|
def test_num_checks_verify(num):
|
||||||
"""Assert the generated Damm check for number verifies."""
|
"""Assert the generated Damm check for number verifies."""
|
||||||
|
|
||||||
|
|
|
@ -56,12 +56,12 @@ from datalog.debris import Timing
|
||||||
from datalog.evaluator import select
|
from datalog.evaluator import select
|
||||||
from datalog.reader import pr_str, read_command, read_dataset
|
from datalog.reader import pr_str, read_command, read_dataset
|
||||||
from datalog.types import (
|
from datalog.types import (
|
||||||
CachedDataset,
|
CachedDataset,
|
||||||
Dataset,
|
Dataset,
|
||||||
LVar,
|
LVar,
|
||||||
PartlyIndexedDataset,
|
PartlyIndexedDataset,
|
||||||
Rule,
|
Rule,
|
||||||
TableIndexedDataset
|
TableIndexedDataset,
|
||||||
)
|
)
|
||||||
|
|
||||||
from prompt_toolkit import print_formatted_text, PromptSession
|
from prompt_toolkit import print_formatted_text, PromptSession
|
||||||
|
|
|
@ -2,11 +2,11 @@
|
||||||
|
|
||||||
from datalog.easy import read, select
|
from datalog.easy import read, select
|
||||||
from datalog.types import (
|
from datalog.types import (
|
||||||
CachedDataset,
|
CachedDataset,
|
||||||
Constant,
|
Constant,
|
||||||
Dataset,
|
Dataset,
|
||||||
PartlyIndexedDataset,
|
PartlyIndexedDataset,
|
||||||
TableIndexedDataset
|
TableIndexedDataset,
|
||||||
)
|
)
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
@ -194,7 +194,9 @@ def test_alternate_rule_lrec(db_cls):
|
||||||
"""Testing that both recursion and alternation work."""
|
"""Testing that both recursion and alternation work."""
|
||||||
|
|
||||||
if db_cls == Dataset:
|
if db_cls == Dataset:
|
||||||
pytest.xfail("left-recursive rules aren't supported with a trivial store and no planner")
|
pytest.xfail(
|
||||||
|
"left-recursive rules aren't supported with a trivial store and no planner"
|
||||||
|
)
|
||||||
|
|
||||||
d = read(
|
d = read(
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -91,4 +91,8 @@ if __name__ == "__main__":
|
||||||
with open(filename) as f:
|
with open(filename) as f:
|
||||||
root = ast.parse(f.read(), filename)
|
root = ast.parse(f.read(), filename)
|
||||||
|
|
||||||
print(yaml.dump(YAMLTreeDumper().visit(root), default_flow_style=False, sort_keys=False))
|
print(
|
||||||
|
yaml.dump(
|
||||||
|
YAMLTreeDumper().visit(root), default_flow_style=False, sort_keys=False
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
|
@ -314,7 +314,7 @@ class ModuleInterpreter(StrictNodeVisitor):
|
||||||
#
|
#
|
||||||
# Note that we have to do the .posonlyargs dance
|
# Note that we have to do the .posonlyargs dance
|
||||||
if argspec.vararg:
|
if argspec.vararg:
|
||||||
self.ns[argspec.vararg.arg] = args[len(argspec.args):]
|
self.ns[argspec.vararg.arg] = args[len(argspec.args) :]
|
||||||
else:
|
else:
|
||||||
if len(args) > len(argspec.args or getattr(argspec, "posonlyargs", ())):
|
if len(args) > len(argspec.args or getattr(argspec, "posonlyargs", ())):
|
||||||
arg_num_mismatch()
|
arg_num_mismatch()
|
||||||
|
@ -791,8 +791,9 @@ class ModuleInterpreter(StrictNodeVisitor):
|
||||||
for n in node.names:
|
for n in node.names:
|
||||||
if n in self.ns and self.ns[n] is not GLOBAL:
|
if n in self.ns and self.ns[n] is not GLOBAL:
|
||||||
raise SyntaxError(
|
raise SyntaxError(
|
||||||
"SyntaxError: name '{}' is assigned to before global declaration"
|
"SyntaxError: name '{}' is assigned to before global declaration".format(
|
||||||
.format(n)
|
n
|
||||||
|
)
|
||||||
)
|
)
|
||||||
# Don't store GLOBAL in the top-level namespace
|
# Don't store GLOBAL in the top-level namespace
|
||||||
if self.ns.parent:
|
if self.ns.parent:
|
||||||
|
@ -895,6 +896,7 @@ class ModuleInterpreter(StrictNodeVisitor):
|
||||||
def visit_Ellipsis(self, node):
|
def visit_Ellipsis(self, node):
|
||||||
# In Py3k only
|
# In Py3k only
|
||||||
from ast import Ellipsis
|
from ast import Ellipsis
|
||||||
|
|
||||||
return Ellipsis
|
return Ellipsis
|
||||||
|
|
||||||
def visit_Print(self, node):
|
def visit_Print(self, node):
|
||||||
|
@ -929,8 +931,8 @@ class InterpreterSystem(object):
|
||||||
name = name.replace(".", os.path.sep)
|
name = name.replace(".", os.path.sep)
|
||||||
for e in self.path:
|
for e in self.path:
|
||||||
for ext in [
|
for ext in [
|
||||||
# ".flow",
|
# ".flow",
|
||||||
".py",
|
".py",
|
||||||
]:
|
]:
|
||||||
if os.path.isdir(e):
|
if os.path.isdir(e):
|
||||||
f = os.path.join(e, name + ext)
|
f = os.path.join(e, name + ext)
|
||||||
|
@ -942,10 +944,14 @@ class InterpreterSystem(object):
|
||||||
|
|
||||||
elif os.path.isfile(e):
|
elif os.path.isfile(e):
|
||||||
# FIXME (arrdem 2021-05-31)
|
# FIXME (arrdem 2021-05-31)
|
||||||
raise RuntimeError("Import from .zip/.whl/.egg archives aren't supported yet")
|
raise RuntimeError(
|
||||||
|
"Import from .zip/.whl/.egg archives aren't supported yet"
|
||||||
|
)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
self.modules[name] = __import__(name, globals, locals, fromlist, level)
|
self.modules[name] = __import__(
|
||||||
|
name, globals, locals, fromlist, level
|
||||||
|
)
|
||||||
|
|
||||||
return self.modules[name]
|
return self.modules[name]
|
||||||
|
|
||||||
|
|
|
@ -27,6 +27,7 @@ for _ in range(10):
|
||||||
def bar(a, b, **bs):
|
def bar(a, b, **bs):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -43,10 +43,10 @@ def timer(val: float) -> str:
|
||||||
"""Given a time in NS, convert it to integral NS/MS/S such that the non-decimal part is integral."""
|
"""Given a time in NS, convert it to integral NS/MS/S such that the non-decimal part is integral."""
|
||||||
|
|
||||||
for factor, unit in [
|
for factor, unit in [
|
||||||
(1e9, "s"),
|
(1e9, "s"),
|
||||||
(1e6, "ms"),
|
(1e6, "ms"),
|
||||||
(1e3, "us"),
|
(1e3, "us"),
|
||||||
(1, "ns"),
|
(1, "ns"),
|
||||||
]:
|
]:
|
||||||
scaled_val = val / factor
|
scaled_val = val / factor
|
||||||
if 1e4 > scaled_val > 1.0:
|
if 1e4 > scaled_val > 1.0:
|
||||||
|
@ -60,21 +60,20 @@ def bench(callable, reps):
|
||||||
with timing() as t:
|
with timing() as t:
|
||||||
callable()
|
callable()
|
||||||
timings.append(t.duration)
|
timings.append(t.duration)
|
||||||
print(f"""Ran {callable.__name__!r} {reps} times, total time {timer(run_t.duration)}
|
print(
|
||||||
|
f"""Ran {callable.__name__!r} {reps} times, total time {timer(run_t.duration)}
|
||||||
mean: {timer(mean(timings))}
|
mean: {timer(mean(timings))}
|
||||||
median: {timer(median(timings))}
|
median: {timer(median(timings))}
|
||||||
stddev: {timer(stdev(timings))}
|
stddev: {timer(stdev(timings))}
|
||||||
test overhead: {timer((run_t.duration - sum(timings)) / reps)}
|
test overhead: {timer((run_t.duration - sum(timings)) / reps)}
|
||||||
""")
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_reference_json(reps):
|
def test_reference_json(reps):
|
||||||
"""As a reference benchmark, test just appending to a file."""
|
"""As a reference benchmark, test just appending to a file."""
|
||||||
|
|
||||||
jobs = [
|
jobs = [{"user_id": randint(0, 1 << 32), "msg": randstr(256)} for _ in range(reps)]
|
||||||
{"user_id": randint(0, 1<<32), "msg": randstr(256)}
|
|
||||||
for _ in range(reps)
|
|
||||||
]
|
|
||||||
jobs_i = iter(jobs)
|
jobs_i = iter(jobs)
|
||||||
|
|
||||||
def naive_serialize():
|
def naive_serialize():
|
||||||
|
@ -86,15 +85,13 @@ def test_reference_json(reps):
|
||||||
def test_reference_fsync(reps):
|
def test_reference_fsync(reps):
|
||||||
"""As a reference benchmark, test just appending to a file."""
|
"""As a reference benchmark, test just appending to a file."""
|
||||||
|
|
||||||
jobs = [
|
jobs = [{"user_id": randint(0, 1 << 32), "msg": randstr(256)} for _ in range(reps)]
|
||||||
{"user_id": randint(0, 1<<32), "msg": randstr(256)}
|
|
||||||
for _ in range(reps)
|
|
||||||
]
|
|
||||||
jobs_i = iter(jobs)
|
jobs_i = iter(jobs)
|
||||||
|
|
||||||
handle, path = tempfile.mkstemp()
|
handle, path = tempfile.mkstemp()
|
||||||
os.close(handle)
|
os.close(handle)
|
||||||
with open(path, "w") as fd:
|
with open(path, "w") as fd:
|
||||||
|
|
||||||
def naive_fsync():
|
def naive_fsync():
|
||||||
fd.write(json.dumps([next(jobs_i), ["CREATED"]]))
|
fd.write(json.dumps([next(jobs_i), ["CREATED"]]))
|
||||||
fd.flush()
|
fd.flush()
|
||||||
|
@ -106,10 +103,7 @@ def test_reference_fsync(reps):
|
||||||
def test_insert(q, reps):
|
def test_insert(q, reps):
|
||||||
"""Benchmark insertion time to a given SQLite DB."""
|
"""Benchmark insertion time to a given SQLite DB."""
|
||||||
|
|
||||||
jobs = [
|
jobs = [{"user_id": randint(0, 1 << 32), "msg": randstr(256)} for _ in range(reps)]
|
||||||
{"user_id": randint(0, 1<<32), "msg": randstr(256)}
|
|
||||||
for _ in range(reps)
|
|
||||||
]
|
|
||||||
jobs_i = iter(jobs)
|
jobs_i = iter(jobs)
|
||||||
|
|
||||||
def insert():
|
def insert():
|
||||||
|
|
|
@ -160,7 +160,9 @@ def compile_query(query):
|
||||||
elif isinstance(query, str):
|
elif isinstance(query, str):
|
||||||
terms = [query]
|
terms = [query]
|
||||||
|
|
||||||
assert not any(keyword in query.lower() for keyword in ["select", "update", "delete", ";"])
|
assert not any(
|
||||||
|
keyword in query.lower() for keyword in ["select", "update", "delete", ";"]
|
||||||
|
)
|
||||||
return " AND ".join(terms)
|
return " AND ".join(terms)
|
||||||
|
|
||||||
|
|
||||||
|
@ -173,7 +175,6 @@ class Job(NamedTuple):
|
||||||
|
|
||||||
|
|
||||||
class JobQueue(object):
|
class JobQueue(object):
|
||||||
|
|
||||||
def __init__(self, path):
|
def __init__(self, path):
|
||||||
self._db = sqlite3.connect(path)
|
self._db = sqlite3.connect(path)
|
||||||
self._queries = anosql.from_str(_SQL, "sqlite3")
|
self._queries = anosql.from_str(_SQL, "sqlite3")
|
||||||
|
@ -196,7 +197,7 @@ class JobQueue(object):
|
||||||
json.loads(payload),
|
json.loads(payload),
|
||||||
json.loads(events),
|
json.loads(events),
|
||||||
json.loads(state),
|
json.loads(state),
|
||||||
datetime.fromtimestamp(int(modified))
|
datetime.fromtimestamp(int(modified)),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _from_result(self, result) -> Job:
|
def _from_result(self, result) -> Job:
|
||||||
|
@ -227,6 +228,7 @@ class JobQueue(object):
|
||||||
|
|
||||||
if limit:
|
if limit:
|
||||||
limit = int(limit)
|
limit = int(limit)
|
||||||
|
|
||||||
def lf(iterable):
|
def lf(iterable):
|
||||||
iterable = iter(iterable)
|
iterable = iter(iterable)
|
||||||
for i in range(limit):
|
for i in range(limit):
|
||||||
|
@ -234,6 +236,7 @@ class JobQueue(object):
|
||||||
yield next(iterable)
|
yield next(iterable)
|
||||||
except StopIteration:
|
except StopIteration:
|
||||||
break
|
break
|
||||||
|
|
||||||
jobs = lf(jobs)
|
jobs = lf(jobs)
|
||||||
|
|
||||||
return self._from_results(jobs)
|
return self._from_results(jobs)
|
||||||
|
@ -265,9 +268,7 @@ class JobQueue(object):
|
||||||
"""Fetch all available data about a given job by ID."""
|
"""Fetch all available data about a given job by ID."""
|
||||||
|
|
||||||
with self._db as db:
|
with self._db as db:
|
||||||
return self._from_result(
|
return self._from_result(self._queries.job_get(db, id=job_id))
|
||||||
self._queries.job_get(db, id=job_id)
|
|
||||||
)
|
|
||||||
|
|
||||||
def cas_state(self, job_id, old_state, new_state):
|
def cas_state(self, job_id, old_state, new_state):
|
||||||
"""CAS update a job's state, returning the updated job or indicating a conflict."""
|
"""CAS update a job's state, returning the updated job or indicating a conflict."""
|
||||||
|
@ -287,11 +288,7 @@ class JobQueue(object):
|
||||||
|
|
||||||
with self._db as db:
|
with self._db as db:
|
||||||
return self._from_result(
|
return self._from_result(
|
||||||
self._queries.job_append_event(
|
self._queries.job_append_event(db, id=job_id, event=json.dumps(event))
|
||||||
db,
|
|
||||||
id=job_id,
|
|
||||||
event=json.dumps(event)
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def delete_job(self, job_id):
|
def delete_job(self, job_id):
|
||||||
|
|
|
@ -52,9 +52,7 @@ def get_jobs():
|
||||||
|
|
||||||
query = blob.get("query", "true")
|
query = blob.get("query", "true")
|
||||||
|
|
||||||
return jsonify({
|
return jsonify({"jobs": [job_as_json(j) for j in request.q.query(query)]}), 200
|
||||||
"jobs": [job_as_json(j) for j in request.q.query(query)]
|
|
||||||
}), 200
|
|
||||||
|
|
||||||
|
|
||||||
@app.route("/api/v0/job/create", methods=["POST"])
|
@app.route("/api/v0/job/create", methods=["POST"])
|
||||||
|
@ -64,9 +62,7 @@ def create_job():
|
||||||
blob = request.get_json(force=True)
|
blob = request.get_json(force=True)
|
||||||
payload = blob["payload"]
|
payload = blob["payload"]
|
||||||
state = blob.get("state", None)
|
state = blob.get("state", None)
|
||||||
job = request.q.create(
|
job = request.q.create(payload, state)
|
||||||
payload, state
|
|
||||||
)
|
|
||||||
return jsonify(job_as_json(job)), 200
|
return jsonify(job_as_json(job)), 200
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -20,7 +20,7 @@ class Job(t.NamedTuple):
|
||||||
payload=obj["payload"],
|
payload=obj["payload"],
|
||||||
events=obj["events"],
|
events=obj["events"],
|
||||||
state=obj["state"],
|
state=obj["state"],
|
||||||
modified=datetime.fromtimestamp(obj["modified"])
|
modified=datetime.fromtimestamp(obj["modified"]),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -32,63 +32,62 @@ class JobqClient(object):
|
||||||
def jobs(self, query=None, limit=10) -> t.Iterable[Job]:
|
def jobs(self, query=None, limit=10) -> t.Iterable[Job]:
|
||||||
"""Enumerate jobs on the queue."""
|
"""Enumerate jobs on the queue."""
|
||||||
|
|
||||||
for job in self._session.post(self._url + "/api/v0/job",
|
for job in (
|
||||||
json={"query": query or [],
|
self._session.post(
|
||||||
"limit": limit})\
|
self._url + "/api/v0/job", json={"query": query or [], "limit": limit}
|
||||||
.json()\
|
)
|
||||||
.get("jobs"):
|
.json()
|
||||||
|
.get("jobs")
|
||||||
|
):
|
||||||
yield Job.from_json(job)
|
yield Job.from_json(job)
|
||||||
|
|
||||||
def poll(self, query, state) -> Job:
|
def poll(self, query, state) -> Job:
|
||||||
"""Poll the job queue for the first job matching the given query, atomically advancing it to the given state and returning the advanced Job."""
|
"""Poll the job queue for the first job matching the given query, atomically advancing it to the given state and returning the advanced Job."""
|
||||||
|
|
||||||
return Job.from_json(
|
return Job.from_json(
|
||||||
self._session
|
self._session.post(
|
||||||
.post(self._url + "/api/v0/job/poll",
|
self._url + "/api/v0/job/poll", json={"query": query, "state": state}
|
||||||
json={"query": query,
|
).json()
|
||||||
"state": state})
|
)
|
||||||
.json())
|
|
||||||
|
|
||||||
def create(self, payload: object, state=None) -> Job:
|
def create(self, payload: object, state=None) -> Job:
|
||||||
"""Create a new job in the system."""
|
"""Create a new job in the system."""
|
||||||
|
|
||||||
return Job.from_json(
|
return Job.from_json(
|
||||||
self._session
|
self._session.post(
|
||||||
.post(self._url + "/api/v0/job/create",
|
self._url + "/api/v0/job/create",
|
||||||
json={"payload": payload,
|
json={"payload": payload, "state": state},
|
||||||
"state": state})
|
).json()
|
||||||
.json())
|
)
|
||||||
|
|
||||||
def fetch(self, job: Job) -> Job:
|
def fetch(self, job: Job) -> Job:
|
||||||
"""Fetch the current state of a job."""
|
"""Fetch the current state of a job."""
|
||||||
|
|
||||||
return Job.from_json(
|
return Job.from_json(
|
||||||
self._session
|
self._session.get(self._url + "/api/v0/job/" + job.id).json()
|
||||||
.get(self._url + "/api/v0/job/" + job.id)
|
)
|
||||||
.json())
|
|
||||||
|
|
||||||
def advance(self, job: Job, state: object) -> Job:
|
def advance(self, job: Job, state: object) -> Job:
|
||||||
"""Attempt to advance a job to a subsequent state."""
|
"""Attempt to advance a job to a subsequent state."""
|
||||||
|
|
||||||
return Job.from_json(
|
return Job.from_json(
|
||||||
self._session
|
self._session.post(
|
||||||
.post(job.url + "/state",
|
job.url + "/state", json={"old": job.state, "new": state}
|
||||||
json={"old": job.state,
|
).json()
|
||||||
"new": state})
|
)
|
||||||
.json())
|
|
||||||
|
|
||||||
def event(self, job: Job, event: object) -> Job:
|
def event(self, job: Job, event: object) -> Job:
|
||||||
"""Attempt to record an event against a job."""
|
"""Attempt to record an event against a job."""
|
||||||
|
|
||||||
return Job.from_json(
|
return Job.from_json(
|
||||||
self._session
|
self._session.post(
|
||||||
.post(self._url + f"/api/v0/job/{job.id}/event",
|
self._url + f"/api/v0/job/{job.id}/event", json=event
|
||||||
json=event)
|
).json()
|
||||||
.json())
|
)
|
||||||
|
|
||||||
def delete(self, job: Job) -> None:
|
def delete(self, job: Job) -> None:
|
||||||
"""Delete a remote job."""
|
"""Delete a remote job."""
|
||||||
|
|
||||||
return (self._session
|
return self._session.delete(
|
||||||
.delete(self._url + f"/api/v0/job/{job.id}")
|
self._url + f"/api/v0/job/{job.id}"
|
||||||
.raise_for_status())
|
).raise_for_status()
|
||||||
|
|
|
@ -12,7 +12,12 @@ from kazoo.exceptions import NodeExistsError
|
||||||
from kazoo.protocol.states import ZnodeStat
|
from kazoo.protocol.states import ZnodeStat
|
||||||
from kazoo.recipe.lock import Lock, ReadLock, WriteLock
|
from kazoo.recipe.lock import Lock, ReadLock, WriteLock
|
||||||
from kook.config import current_config, KookConfig
|
from kook.config import current_config, KookConfig
|
||||||
from toolz.dicttoolz import assoc as _assoc, dissoc as _dissoc, merge as _merge, update_in
|
from toolz.dicttoolz import (
|
||||||
|
assoc as _assoc,
|
||||||
|
dissoc as _dissoc,
|
||||||
|
merge as _merge,
|
||||||
|
update_in,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def assoc(m, k, v):
|
def assoc(m, k, v):
|
||||||
|
|
|
@ -14,10 +14,12 @@ setup(
|
||||||
packages=[
|
packages=[
|
||||||
"lilith",
|
"lilith",
|
||||||
],
|
],
|
||||||
package_data={"": [
|
package_data={
|
||||||
"src/python/lilith/*.lark",
|
"": [
|
||||||
"src/python/lilith/*.lil",
|
"src/python/lilith/*.lark",
|
||||||
]},
|
"src/python/lilith/*.lil",
|
||||||
|
]
|
||||||
|
},
|
||||||
include_package_data=True,
|
include_package_data=True,
|
||||||
install_requires=requirements,
|
install_requires=requirements,
|
||||||
entry_points={
|
entry_points={
|
||||||
|
|
|
@ -89,16 +89,18 @@ class Proquint(object):
|
||||||
val = n << 8 | m
|
val = n << 8 | m
|
||||||
|
|
||||||
# This is slightly un-idiomatic, but it precisely captures the coding definition
|
# This is slightly un-idiomatic, but it precisely captures the coding definition
|
||||||
yield "".join([
|
yield "".join(
|
||||||
dict[val >> shift & mask]
|
[
|
||||||
for dict, shift, mask in [
|
dict[val >> shift & mask]
|
||||||
(cls.CONSONANTS, 0xC, 0xf),
|
for dict, shift, mask in [
|
||||||
(cls.VOWELS, 0xA, 0x3),
|
(cls.CONSONANTS, 0xC, 0xF),
|
||||||
(cls.CONSONANTS, 0x6, 0xf),
|
(cls.VOWELS, 0xA, 0x3),
|
||||||
(cls.VOWELS, 0x4, 0x3),
|
(cls.CONSONANTS, 0x6, 0xF),
|
||||||
(cls.CONSONANTS, 0x0, 0xf)
|
(cls.VOWELS, 0x4, 0x3),
|
||||||
|
(cls.CONSONANTS, 0x0, 0xF),
|
||||||
|
]
|
||||||
]
|
]
|
||||||
])
|
)
|
||||||
|
|
||||||
# Core methods
|
# Core methods
|
||||||
################################################################################################
|
################################################################################################
|
||||||
|
|
|
@ -10,7 +10,9 @@ from proquint import Proquint
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
g = parser.add_mutually_exclusive_group()
|
g = parser.add_mutually_exclusive_group()
|
||||||
g.add_argument("-g", "--generate", dest="generate", default=False, action="store_true")
|
g.add_argument("-g", "--generate", dest="generate", default=False, action="store_true")
|
||||||
g.add_argument("-p", "--predictable", dest="predictable", default=False, action="store_true")
|
g.add_argument(
|
||||||
|
"-p", "--predictable", dest="predictable", default=False, action="store_true"
|
||||||
|
)
|
||||||
g.add_argument("-d", "--decode", dest="decode", default=False, action="store_true")
|
g.add_argument("-d", "--decode", dest="decode", default=False, action="store_true")
|
||||||
g.add_argument("-e", "--encode", dest="encode", default=False, action="store_true")
|
g.add_argument("-e", "--encode", dest="encode", default=False, action="store_true")
|
||||||
parser.add_argument("-w", "--width", dest="width", type=int, default=32)
|
parser.add_argument("-w", "--width", dest="width", type=int, default=32)
|
||||||
|
|
|
@ -38,7 +38,6 @@ examples = [
|
||||||
(536870912, 32, "fabab-babab"),
|
(536870912, 32, "fabab-babab"),
|
||||||
(1073741824, 32, "habab-babab"),
|
(1073741824, 32, "habab-babab"),
|
||||||
(2147483648, 32, "mabab-babab"),
|
(2147483648, 32, "mabab-babab"),
|
||||||
|
|
||||||
# A random value
|
# A random value
|
||||||
(3232235536, 32, "safom-babib"),
|
(3232235536, 32, "safom-babib"),
|
||||||
]
|
]
|
||||||
|
@ -53,4 +52,6 @@ def test_decode_examples(val, width, qint):
|
||||||
def test_encode_examples(val, width, qint):
|
def test_encode_examples(val, width, qint):
|
||||||
encoded_qint = proquint.Proquint.encode(val, width)
|
encoded_qint = proquint.Proquint.encode(val, width)
|
||||||
decoded_val = proquint.Proquint.decode(encoded_qint)
|
decoded_val = proquint.Proquint.decode(encoded_qint)
|
||||||
assert encoded_qint == qint, f"did not encode {val} to {qint}; got {encoded_qint} ({decoded_val})"
|
assert (
|
||||||
|
encoded_qint == qint
|
||||||
|
), f"did not encode {val} to {qint}; got {encoded_qint} ({decoded_val})"
|
||||||
|
|
|
@ -5,25 +5,21 @@ from hypothesis.strategies import integers
|
||||||
import proquint
|
import proquint
|
||||||
|
|
||||||
|
|
||||||
@given(integers(min_value=0, max_value=1<<16))
|
@given(integers(min_value=0, max_value=1 << 16))
|
||||||
def test_round_trip_16(val):
|
def test_round_trip_16(val):
|
||||||
assert proquint.Proquint.decode(
|
assert proquint.Proquint.decode(proquint.Proquint.encode(val, 16)) == val
|
||||||
proquint.Proquint.encode(val, 16)) == val
|
|
||||||
|
|
||||||
|
|
||||||
@given(integers(min_value=0, max_value=1<<32))
|
@given(integers(min_value=0, max_value=1 << 32))
|
||||||
def test_round_trip_32(val):
|
def test_round_trip_32(val):
|
||||||
assert proquint.Proquint.decode(
|
assert proquint.Proquint.decode(proquint.Proquint.encode(val, 32)) == val
|
||||||
proquint.Proquint.encode(val, 32)) == val
|
|
||||||
|
|
||||||
|
|
||||||
@given(integers(min_value=0, max_value=1<<64))
|
@given(integers(min_value=0, max_value=1 << 64))
|
||||||
def test_round_trip_64(val):
|
def test_round_trip_64(val):
|
||||||
assert proquint.Proquint.decode(
|
assert proquint.Proquint.decode(proquint.Proquint.encode(val, 64)) == val
|
||||||
proquint.Proquint.encode(val, 64)) == val
|
|
||||||
|
|
||||||
|
|
||||||
@given(integers(min_value=0, max_value=1<<512))
|
@given(integers(min_value=0, max_value=1 << 512))
|
||||||
def test_round_trip_512(val):
|
def test_round_trip_512(val):
|
||||||
assert proquint.Proquint.decode(
|
assert proquint.Proquint.decode(proquint.Proquint.encode(val, 512)) == val
|
||||||
proquint.Proquint.encode(val, 512)) == val
|
|
||||||
|
|
|
@ -49,7 +49,7 @@ def req_name(requirement: str) -> str:
|
||||||
def sort_key(requirement: str) -> str:
|
def sort_key(requirement: str) -> str:
|
||||||
return (
|
return (
|
||||||
req_name(requirement) # Get the match group
|
req_name(requirement) # Get the match group
|
||||||
.lower() # We ignore case
|
.lower() # We ignore case
|
||||||
.replace("-", "") # We ignore -
|
.replace("-", "") # We ignore -
|
||||||
.replace("_", "") # And _
|
.replace("_", "") # And _
|
||||||
)
|
)
|
||||||
|
@ -58,7 +58,9 @@ def sort_key(requirement: str) -> str:
|
||||||
def _bq(query):
|
def _bq(query):
|
||||||
"""Enumerate the PyPi package names from a Bazel query."""
|
"""Enumerate the PyPi package names from a Bazel query."""
|
||||||
|
|
||||||
unused = subprocess.check_output(["bazel", "query", query, "--output=package"]).decode("utf-8")
|
unused = subprocess.check_output(
|
||||||
|
["bazel", "query", query, "--output=package"]
|
||||||
|
).decode("utf-8")
|
||||||
for line in unused.split("\n"):
|
for line in unused.split("\n"):
|
||||||
if line:
|
if line:
|
||||||
yield line.replace("@arrdem_source_pypi//pypi__", "")
|
yield line.replace("@arrdem_source_pypi//pypi__", "")
|
||||||
|
@ -67,7 +69,9 @@ def _bq(query):
|
||||||
def _unused():
|
def _unused():
|
||||||
"""Find unused requirements."""
|
"""Find unused requirements."""
|
||||||
|
|
||||||
return set(_bq("@arrdem_source_pypi//...")) - set(_bq("filter('//pypi__', deps(//...))"))
|
return set(_bq("@arrdem_source_pypi//...")) - set(
|
||||||
|
_bq("filter('//pypi__', deps(//...))")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _load(fname):
|
def _load(fname):
|
||||||
|
|
|
@ -199,7 +199,7 @@ class YamlLinter(object):
|
||||||
)
|
)
|
||||||
|
|
||||||
def _lint_num_range(self, schema, node: Node, value) -> t.Iterable[LintRecord]:
|
def _lint_num_range(self, schema, node: Node, value) -> t.Iterable[LintRecord]:
|
||||||
""""FIXME."""
|
""" "FIXME."""
|
||||||
|
|
||||||
if (base := schema.get("multipleOf")) is not None:
|
if (base := schema.get("multipleOf")) is not None:
|
||||||
if value % base != 0:
|
if value % base != 0:
|
||||||
|
|
|
@ -5,6 +5,8 @@ attrs==20.3.0
|
||||||
autoflake==1.4
|
autoflake==1.4
|
||||||
Babel==2.9.0
|
Babel==2.9.0
|
||||||
beautifulsoup4==4.9.3
|
beautifulsoup4==4.9.3
|
||||||
|
black==21.8b0
|
||||||
|
bleach==4.0.0
|
||||||
certifi==2020.12.5
|
certifi==2020.12.5
|
||||||
chardet==4.0.0
|
chardet==4.0.0
|
||||||
click==7.1.2
|
click==7.1.2
|
||||||
|
@ -40,8 +42,10 @@ openapi-schema-validator==0.1.5
|
||||||
openapi-spec-validator==0.3.0
|
openapi-spec-validator==0.3.0
|
||||||
packaging==20.9
|
packaging==20.9
|
||||||
parso==0.8.2
|
parso==0.8.2
|
||||||
pathspec==0.8.1
|
pathspec==0.9.0
|
||||||
|
pep517==0.11.0
|
||||||
pip-tools==6.2.0
|
pip-tools==6.2.0
|
||||||
|
platformdirs==2.3.0
|
||||||
pluggy==0.13.1
|
pluggy==0.13.1
|
||||||
port-for==0.6.1
|
port-for==0.6.1
|
||||||
prompt-toolkit==3.0.18
|
prompt-toolkit==3.0.18
|
||||||
|
@ -63,6 +67,7 @@ PyYAML==5.4.1
|
||||||
readme-renderer==29.0
|
readme-renderer==29.0
|
||||||
recommonmark==0.7.1
|
recommonmark==0.7.1
|
||||||
redis==3.5.3
|
redis==3.5.3
|
||||||
|
regex==2021.8.28
|
||||||
requests==2.25.1
|
requests==2.25.1
|
||||||
requests-toolbelt==0.9.1
|
requests-toolbelt==0.9.1
|
||||||
requirements-parser==0.2.0
|
requirements-parser==0.2.0
|
||||||
|
@ -81,15 +86,18 @@ sphinxcontrib-programoutput==0.17
|
||||||
sphinxcontrib-qthelp==1.0.3
|
sphinxcontrib-qthelp==1.0.3
|
||||||
sphinxcontrib-serializinghtml==1.1.4
|
sphinxcontrib-serializinghtml==1.1.4
|
||||||
toml==0.10.2
|
toml==0.10.2
|
||||||
|
tomli==1.2.1
|
||||||
tornado==6.1
|
tornado==6.1
|
||||||
typed-ast==1.4.2
|
typed-ast==1.4.2
|
||||||
typing-extensions==3.7.4.3
|
typing-extensions==3.10.0.2
|
||||||
unify==0.5
|
unify==0.5
|
||||||
untokenize==0.1.1
|
untokenize==0.1.1
|
||||||
urllib3==1.26.4
|
urllib3==1.26.4
|
||||||
urwid==2.1.2
|
urwid==2.1.2
|
||||||
wcwidth==0.2.5
|
wcwidth==0.2.5
|
||||||
|
webencodings==0.5.1
|
||||||
Werkzeug==2.0.1
|
Werkzeug==2.0.1
|
||||||
yamllint==1.26.1
|
yamllint==1.26.1
|
||||||
yarl==1.6.3
|
yarl==1.6.3
|
||||||
yaspin==1.5.0
|
yaspin==1.5.0
|
||||||
|
zipp==3.5.0
|
||||||
|
|
|
@ -13,16 +13,16 @@ from requirements.requirement import Requirement
|
||||||
# Licenses approved as representing non-copyleft and not precluding commercial usage.
|
# Licenses approved as representing non-copyleft and not precluding commercial usage.
|
||||||
# This is all easy, there's a good schema here.
|
# This is all easy, there's a good schema here.
|
||||||
APPROVED_LICENSES = [
|
APPROVED_LICENSES = [
|
||||||
MIT := "License :: OSI Approved :: MIT License",
|
MIT := "License :: OSI Approved :: MIT License",
|
||||||
APACHE := "License :: OSI Approved :: Apache Software License",
|
APACHE := "License :: OSI Approved :: Apache Software License",
|
||||||
BSD := "License :: OSI Approved :: BSD License",
|
BSD := "License :: OSI Approved :: BSD License",
|
||||||
MPL10 := "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
|
MPL10 := "License :: OSI Approved :: Mozilla Public License 1.0 (MPL)",
|
||||||
MPL11 := "License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)",
|
MPL11 := "License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)",
|
||||||
MPL20 := "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
|
MPL20 := "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
|
||||||
PSFL := "License :: OSI Approved :: Python Software Foundation License",
|
PSFL := "License :: OSI Approved :: Python Software Foundation License",
|
||||||
LGPL := "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)",
|
LGPL := "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)",
|
||||||
LGPL3 := "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)",
|
LGPL3 := "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)",
|
||||||
ISCL := "License :: OSI Approved :: ISC License (ISCL)",
|
ISCL := "License :: OSI Approved :: ISC License (ISCL)",
|
||||||
]
|
]
|
||||||
|
|
||||||
UNAPPROVED_LICENSES = [
|
UNAPPROVED_LICENSES = [
|
||||||
|
@ -36,23 +36,17 @@ LICENSES_BY_LOWERNAME = {
|
||||||
"apache 2.0": APACHE,
|
"apache 2.0": APACHE,
|
||||||
"apache": APACHE,
|
"apache": APACHE,
|
||||||
"http://www.apache.org/licenses/license-2.0": APACHE,
|
"http://www.apache.org/licenses/license-2.0": APACHE,
|
||||||
|
|
||||||
"bsd 3": BSD,
|
"bsd 3": BSD,
|
||||||
"bsd": BSD,
|
"bsd": BSD,
|
||||||
|
|
||||||
"gpl": GPL1,
|
"gpl": GPL1,
|
||||||
"gpl2": GPL2,
|
"gpl2": GPL2,
|
||||||
"gpl3": GPL3,
|
"gpl3": GPL3,
|
||||||
"lgpl": LGPL,
|
"lgpl": LGPL,
|
||||||
"lgpl3": LGPL3,
|
"lgpl3": LGPL3,
|
||||||
|
|
||||||
"isc": ISCL,
|
"isc": ISCL,
|
||||||
|
|
||||||
"mit": MIT,
|
"mit": MIT,
|
||||||
|
|
||||||
"mpl": MPL10,
|
"mpl": MPL10,
|
||||||
"mpl 2.0": MPL20,
|
"mpl 2.0": MPL20,
|
||||||
|
|
||||||
"psf": PSFL,
|
"psf": PSFL,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -75,7 +69,9 @@ with open("tools/python/requirements.txt") as fd:
|
||||||
|
|
||||||
def bash_license(ln):
|
def bash_license(ln):
|
||||||
while True:
|
while True:
|
||||||
lnn = re.sub(r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)", "", ln.lower())
|
lnn = re.sub(
|
||||||
|
r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)", "", ln.lower()
|
||||||
|
)
|
||||||
if ln != lnn:
|
if ln != lnn:
|
||||||
ln = lnn
|
ln = lnn
|
||||||
else:
|
else:
|
||||||
|
@ -85,16 +81,19 @@ def bash_license(ln):
|
||||||
return ln
|
return ln
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("a,b", [
|
@pytest.mark.parametrize(
|
||||||
("MIT", MIT),
|
"a,b",
|
||||||
("mit", MIT),
|
[
|
||||||
("BSD", BSD),
|
("MIT", MIT),
|
||||||
("BSD 3-clause", BSD),
|
("mit", MIT),
|
||||||
("BSD 3 clause", BSD),
|
("BSD", BSD),
|
||||||
("GPL3", GPL3),
|
("BSD 3-clause", BSD),
|
||||||
("GPL v3", GPL3),
|
("BSD 3 clause", BSD),
|
||||||
("GPLv3", GPL3),
|
("GPL3", GPL3),
|
||||||
])
|
("GPL v3", GPL3),
|
||||||
|
("GPLv3", GPL3),
|
||||||
|
],
|
||||||
|
)
|
||||||
def test_bash_license(a, b):
|
def test_bash_license(a, b):
|
||||||
assert bash_license(a) == b
|
assert bash_license(a) == b
|
||||||
|
|
||||||
|
@ -117,7 +116,7 @@ def licenses(package: Requirement):
|
||||||
if not version:
|
if not version:
|
||||||
blob = requests.get(
|
blob = requests.get(
|
||||||
f"https://pypi.org/pypi/{package.name}/json",
|
f"https://pypi.org/pypi/{package.name}/json",
|
||||||
headers={"Accept": "application/json"}
|
headers={"Accept": "application/json"},
|
||||||
).json()
|
).json()
|
||||||
if ln := bash_license(blob.get("license")):
|
if ln := bash_license(blob.get("license")):
|
||||||
lics.append(ln)
|
lics.append(ln)
|
||||||
|
@ -131,13 +130,15 @@ def licenses(package: Requirement):
|
||||||
if version:
|
if version:
|
||||||
blob = requests.get(
|
blob = requests.get(
|
||||||
f"https://pypi.org/pypi/{package.name}/{version}/json",
|
f"https://pypi.org/pypi/{package.name}/{version}/json",
|
||||||
headers={"Accept": "application/json"}
|
headers={"Accept": "application/json"},
|
||||||
).json()
|
).json()
|
||||||
lics.extend([
|
lics.extend(
|
||||||
c
|
[
|
||||||
for c in blob.get("info", {}).get("classifiers", [])
|
c
|
||||||
if c.startswith("License")
|
for c in blob.get("info", {}).get("classifiers", [])
|
||||||
])
|
if c.startswith("License")
|
||||||
|
]
|
||||||
|
)
|
||||||
ln = blob.get("info", {}).get("license")
|
ln = blob.get("info", {}).get("license")
|
||||||
if ln and not lics:
|
if ln and not lics:
|
||||||
lics.append(bash_license(ln))
|
lics.append(bash_license(ln))
|
||||||
|
|
Loading…
Reference in a new issue