From 5adee17f15b8d38f8c3c11d4c43a44d35239d6af Mon Sep 17 00:00:00 2001
From: Reid 'arrdem' McKenzie <me@arrdem.com>
Date: Sat, 13 May 2023 16:56:01 -0600
Subject: [PATCH 1/2] Infra tweaks
---
WORKSPACE | 2 +-
tools/python/requirements.in | 5 ++++-
tools/python/{requirements.txt => requirements_lock.txt} | 8 ++++++++
3 files changed, 13 insertions(+), 2 deletions(-)
rename tools/python/{requirements.txt => requirements_lock.txt} (93%)
diff --git a/WORKSPACE b/WORKSPACE
index 58d3a24..ed39724 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -52,7 +52,7 @@ load("@rules_python//python:pip.bzl", "pip_parse")
pip_parse(
name = "arrdem_source_pypi",
- requirements_lock = "//tools/python:requirements.txt",
+ requirements_lock = "//tools/python:requirements_lock.txt",
python_interpreter_target = "//tools/python:pythonshim",
)
diff --git a/tools/python/requirements.in b/tools/python/requirements.in
index da7413a..45ce9b1 100644
--- a/tools/python/requirements.in
+++ b/tools/python/requirements.in
@@ -2,6 +2,7 @@ ExifRead
aiohttp
aiohttp_basicauth
async_lru
+attrs
autoflake
beautifulsoup4
black
@@ -30,6 +31,8 @@ pycryptodome
pyrsistent
pytest-cov
pytest-postgresql
+pytest-pudb
+pytest-timeout
pyyaml
recommonmark
redis
@@ -40,7 +43,7 @@ smbus2
sphinx
sphinxcontrib-openapi
sphinxcontrib-programoutput
+toml
unify
yamllint
yaspin
-toml
diff --git a/tools/python/requirements.txt b/tools/python/requirements_lock.txt
similarity index 93%
rename from tools/python/requirements.txt
rename to tools/python/requirements_lock.txt
index e51fccd..8a4eb52 100644
--- a/tools/python/requirements.txt
+++ b/tools/python/requirements_lock.txt
@@ -9,6 +9,7 @@ autoflake==2.0.2
Babel==2.12.1
beautifulsoup4==4.12.0
black==23.1.0
+blinker==1.6.2
cachetools==5.3.0
certifi==2022.12.7
charset-normalizer==3.1.0
@@ -32,6 +33,7 @@ imagesize==1.4.1
iniconfig==2.0.0
isort==5.12.0
itsdangerous==2.1.2
+jedi==0.18.2
Jinja2==3.1.2
jsonschema==4.17.3
jsonschema-spec==0.1.4
@@ -51,6 +53,7 @@ octorest==0.4
openapi-schema-validator==0.4.4
openapi-spec-validator==0.5.6
packaging==23.0
+parso==0.8.3
pathable==0.4.3
pathspec==0.11.1
picobox==2.2.0
@@ -61,6 +64,7 @@ prompt-toolkit==3.0.38
proquint==0.2.1
psutil==5.9.4
psycopg2==2.9.5
+pudb==2022.1.3
py==1.11.0
pycodestyle==2.10.0
pycryptodome==3.17
@@ -70,6 +74,8 @@ pyrsistent==0.19.3
pytest==7.2.2
pytest-cov==4.0.0
pytest-postgresql==4.1.1
+pytest-pudb==0.7.0
+pytest-timeout==2.1.0
PyYAML==6.0
recommonmark==0.7.1
redis==4.5.3
@@ -100,6 +106,8 @@ typing_extensions==4.5.0
unify==0.5
untokenize==0.1.1
urllib3==1.26.15
+urwid==2.1.2
+urwid-readline==0.13
wcwidth==0.2.6
websocket-client==1.5.1
Werkzeug==2.2.3
From b069959ee701333b6294982f20b697bfd14e85a1 Mon Sep 17 00:00:00 2001
From: Reid 'arrdem' McKenzie <me@arrdem.com>
Date: Sat, 13 May 2023 16:56:17 -0600
Subject: [PATCH 2/2] Tapping on tentacles
---
projects/tentacles/BUILD | 11 +
projects/tentacles/README.md | 46 +++
.../src/python/tentacles/__init__.py | 0
.../src/python/tentacles/__main__.py | 24 ++
.../src/python/tentacles/blueprints.py | 7 +
.../tentacles/src/python/tentacles/store.py | 265 ++++++++++++++++++
projects/tentacles/test/python/conftest.py | 39 +++
projects/tentacles/test/python/test_store.py | 40 +++
8 files changed, 432 insertions(+)
create mode 100644 projects/tentacles/BUILD
create mode 100644 projects/tentacles/README.md
create mode 100644 projects/tentacles/src/python/tentacles/__init__.py
create mode 100644 projects/tentacles/src/python/tentacles/__main__.py
create mode 100644 projects/tentacles/src/python/tentacles/blueprints.py
create mode 100644 projects/tentacles/src/python/tentacles/store.py
create mode 100644 projects/tentacles/test/python/conftest.py
create mode 100644 projects/tentacles/test/python/test_store.py
diff --git a/projects/tentacles/BUILD b/projects/tentacles/BUILD
new file mode 100644
index 0000000..66ccda3
--- /dev/null
+++ b/projects/tentacles/BUILD
@@ -0,0 +1,11 @@
+py_project(
+ name = "tentacles",
+ main = "src/python/tentacles/__main__.py",
+ main_deps = [
+ "//projects/anosql",
+ "//projects/anosql-migrations",
+ py_requirement("click"),
+ py_requirement("flask"),
+ py_requirement("jinja2"),
+ ]
+)
diff --git a/projects/tentacles/README.md b/projects/tentacles/README.md
new file mode 100644
index 0000000..5a585a0
--- /dev/null
+++ b/projects/tentacles/README.md
@@ -0,0 +1,46 @@
+# Tentacles
+
+A simple queue system for OctoPrint, designed to receive jobs and forward them to connected OctoPrint instances as they are available and ready.
+
+## Workflow
+
+Username+Password users
+
+API keys mapped to users
+
+Users mapped to job priority
+
+API keys mapped to user sub-priority
+
+User-facing API can upload gcode files to storage and create a file entry in the DB
+
+Jobs can be created from new or existing file entries in the DB
+
+Jobs are mapped to the priority of the API key (and user) who created them
+
+Jobs get Runs - which track assigning a pending Job to a Printer
+
+Highest priority job schedules first/next non-Running Job to the first available printer.
+This requires copying the gcode file to the target device, and starting it running.
+
+Running Runs are polled to detect loss/failure. Lost or failed Runs are deleted, and the Job returns to the queue.
+A priority penalty may be called for.
+
+## Notes
+
+Checking bed status on a printer
+
+ $ curl 'http://10.0.0.6/api/plugin/bedready' \
+ -X POST \
+ -H 'Accept: application/json' \
+ -H 'Content-Type: application/json; charset=UTF-8' \
+ -H 'Authorization: Bearer 8295AA7DDED645629C5006F2CC2042C2' \
+ --data-raw '{"command":"check_bed","reference":"reference_2023-05-11T05:22:40.212Z.jpg"}' \
+ | jq .
+
+Uploading a file and starting to print it
+- https://github.com/prusa3d/PrusaSlicer/blob/0384d631d6ef1aaadcb68da031eba9a586b102ed/src/slic3r/Utils/OctoPrint.cpp#L936
+- https://docs.octoprint.org/en/master/api/files.html#upload-file-or-create-folder
+- &print=true
+
+Doesn't appear possible to execute request/response g-code through the API? Can't detect when the printer's offsets aren't calibrated?
diff --git a/projects/tentacles/src/python/tentacles/__init__.py b/projects/tentacles/src/python/tentacles/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/projects/tentacles/src/python/tentacles/__main__.py b/projects/tentacles/src/python/tentacles/__main__.py
new file mode 100644
index 0000000..6f447a5
--- /dev/null
+++ b/projects/tentacles/src/python/tentacles/__main__.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+"""
+
+"""
+
+import click
+import flask
+
+
+@click.group()
+def cli():
+ pass
+
+
+@cli.command()
+def serve():
+ app = flask.Flask()
+ app.register_blueprint()
+ app.run()
+
+
+if __name__ == "__main__":
+ cli()
diff --git a/projects/tentacles/src/python/tentacles/blueprints.py b/projects/tentacles/src/python/tentacles/blueprints.py
new file mode 100644
index 0000000..e0eeab7
--- /dev/null
+++ b/projects/tentacles/src/python/tentacles/blueprints.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+"""Blueprints for the Tentacles app."""
+
+from flask import Blueprint
+
+BLUEPRINT = Blueprint(__name__, __qualname__, template_folder=__package__)
diff --git a/projects/tentacles/src/python/tentacles/store.py b/projects/tentacles/src/python/tentacles/store.py
new file mode 100644
index 0000000..a5f7a85
--- /dev/null
+++ b/projects/tentacles/src/python/tentacles/store.py
@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+
+from datetime import timedelta, datetime
+from hashlib import sha3_256
+from pathlib import Path
+import sqlite3
+from textwrap import indent
+
+from collections import namedtuple
+from typing import Optional
+
+
+PRELUDE = """
+CREATE TABLE IF NOT EXISTS groups (
+ id INTEGER PRIMARY KEY AUTOINCREMENT
+ , name TEXT
+ , priority INTEGER CHECK(priority IS NOT NULL AND priority > 0)
+ DEFAULT 100
+);
+
+CREATE TABLE IF NOT EXISTS users (
+ id INTEGER PRIMARY KEY AUTOINCREMENT
+ , group_id INTEGER
+ , name TEXT
+ , hash TEXT
+ , FOREIGN KEY(group_id) REFERENCES groups(id)
+ , UNIQUE(name)
+);
+
+CREATE TABLE IF NOT EXISTS user_keys (
+ id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(48))))
+ , user_id INTEGER
+ , expiration TEXT
+);
+
+CREATE TABLE IF NOT EXISTS files (
+ id INTEGER PRIMARY KEY AUTOINCREMENT
+ , user_id INTEGER
+ , filename TEXT
+ , upload_date TEXT
+ , FOREIGN KEY(user_id) REFERENCES user(id)
+);
+
+CREATE TABLE IF NOT EXISTS jobs (
+ id INTEGER PRIMARY KEY AUTOINCREMENT
+ , user_id INTEGER
+ , FOREIGN KEY(user_id) REFERENCES user(id)
+);
+
+CREATE TABLE IF NOT EXISTs printer_statuses (
+ id INTEGER PRIMARY KEY AUTOINCREMENT
+ , name TEXT
+ , UNIQUE(name)
+);
+
+INSERT OR IGNORE INTO printer_statuses (name) values ('disconnected');
+INSERT OR IGNORE INTO printer_statuses (name) values ('connected');
+INSERT OR IGNORE INTO printer_statuses (name) values ('idle');
+INSERT OR IGNORE INTO printer_statuses (name) values ('running');
+INSERT OR IGNORE INTO printer_statuses (name) values ('error');
+
+CREATE TABLE IF NOT EXISTS printers (
+ id INTEGER PRIMARY KEY AUTOINCREMENT
+ , url TEXT
+ , api_key TEXT
+ , status_id INTEGER
+ , last_poll_date TEXT
+ , FOREIGN KEY(status_id) REFERENCES printer_statuses(id)
+);
+
+CREATE TABLE IF NOT EXISTS runs (
+ id INTEGER PRIMARY KEY AUTOINCREMENT
+ , job_id INTEGER
+ , started_at TEXT
+ , finished_at TEXT
+ , state TEXT
+ , printer_id INTEGER
+ , FOREIGN KEY(job_id) REFERENCES job(id)
+ , FOREIGN KEY(printer_id) REFERENCES printer(id)
+);
+"""
+
+
+def requires_conn(f):
+ def _helper(self, *args, **kwargs):
+ if self._conn is None:
+ raise ConnectionError(f"A connection is required for {f.__name__}")
+ return f(self, *args, **kwargs)
+
+ return _helper
+
+
+class Store(object):
+ def __init__(self, path):
+ self._path = path
+ self._conn: sqlite3.Connection = None
+
+ def connect(self):
+ if not self._conn:
+ self._conn = sqlite3.connect(self._path, isolation_level="IMMEDIATE")
+ for hunk in PRELUDE.split("\n\n"):
+ try:
+ self._conn.executescript(hunk).fetchall()
+ except sqlite3.OperationalError as e:
+ raise RuntimeError(
+ f"Unable to execute startup script:\n{indent(hunk, ' > ')}"
+ ) from e
+
+ def close(self):
+ if self._conn:
+ self._conn.commit()
+ self._conn.close()
+ self._conn = None
+
+ @requires_conn
+ def try_create_user(self, username, password):
+ """Attempt to create a new user."""
+
+ digest = sha3_256()
+ digest.update(password.encode("utf-8"))
+ return self._conn.execute(
+ "INSERT INTO users (name, hash) VALUES (?, ?) RETURNING (id)",
+ [username, digest.hexdigest()],
+ ).fetchone()[0]
+
+ @requires_conn
+ def list_users(self):
+ return self._conn.execute("SELECT id, name FROM users").fetchall()
+
+ ################################################################################
+ # Sessions / 'keys'
+
+ @requires_conn
+ def _create_session(self, uid: int, ttl: Optional[timedelta]):
+ return self._conn.execute(
+ "INSERT INTO user_keys (user_id, expiration) VALUES (?, ?) RETURNING (id)",
+ [uid, (datetime.now() + ttl).isoformat() if ttl else None],
+ ).fetchone()[0]
+
+ @requires_conn
+ def try_login(self, username: str, password: str, ttl: timedelta) -> Optional[str]:
+ """Given a username and an (unsecured) password, attempt to authenticate the named user.
+
+ If successful, return the ID of a new session/key for that user.
+
+ """
+
+ digest = sha3_256()
+ digest.update(password.encode("utf-8"))
+ res = self._conn.execute(
+ "SELECT id FROM users WHERE name=? AND hash=? LIMIT 1",
+ [username, digest.hexdigest()],
+ ).fetchone()
+ if not res:
+ return
+ uid = res[0]
+ return self._create_session(uid, ttl)
+
+ @requires_conn
+ def create_key(self, kid: str, ttl) -> Optional[str]:
+ """Given an _existing_ login session, create a new key.
+
+ This allows the user to create more or less permanent API keys associated with their identity.
+
+ """
+
+ if uid := self.try_key(kid):
+ return self._create_session(uid, ttl)
+
+ @requires_conn
+ def list_keys(self):
+ return self._conn.execute("SELECT id, user_id FROM user_keys").fetchall()
+
+ @requires_conn
+ def fetch_key(self, kid) -> tuple:
+ return self._conn.execute(
+ "SELECT * FROM user_keys WHERE id = ?", [kid]
+ ).fetchone()
+
+ @requires_conn
+ def try_key(self, kid: str):
+ """Try to find the mapped user for a session."""
+
+ res = self._conn.execute(
+ "SELECT user_id FROM user_keys WHERE expiration IS NULL OR unixepoch(expiration) > unixepoch('now') and id = ?",
+ [kid],
+ ).fetchone()
+ if res:
+ return res[0]
+
+ @requires_conn
+ def refresh_key(self, kid: str, ttl: timedelta):
+ """Automagically renew an API key which is still in use.
+
+ Mostly intended for dealing with web sessions which should implicitly extend, but which use the same mechanism as API keys.
+
+ """
+
+ self._conn.execute(
+ "UPDATE user_keys SET expiration = ? WHERE id = ?",
+ [(datetime.now() + ttl).isoformat(), kid],
+ )
+
+ @requires_conn
+ def delete_key(self, kid: str):
+ """Remove a session/key; equivalent to logout."""
+
+ self._conn.execute("DELETE FROM user_keys WHERE id = ?", [kid])
+
+ ################################################################################
+ # Printers
+ #
+ # Printers represent connections to OctoPrint instances controlling physical machines.
+ @requires_conn
+ def create_printer(self):
+ pass
+
+ @requires_conn
+ def list_printers(self):
+ pass
+
+ @requires_conn
+ def update_printer_status(self):
+ pass
+
+ ################################################################################
+ # Files
+ #
+ # A record of local files on disk, and the users who own then.
+ @requires_conn
+ def create_file(self, uid: int, path: Path):
+ pass
+
+ @requires_conn
+ def list_files(self, uid: int):
+ pass
+
+ @requires_conn
+ def delete_file(self, uid: int, fid: int):
+ pass
+
+ ################################################################################
+ # Job
+ #
+ # A request by a user for a given file to be printed.
+ @requires_conn
+ def create_job(self, uid: int):
+ pass
+
+ @requires_conn
+ def list_jobs(self, uid: int):
+ pass
+
+ @requires_conn
+ def delete_job(self, uid: int, jid: int):
+ pass
+
+ ################################################################################
+ # Run
+ #
+ # A record tha that a Job has been assigned to a Printer and is running.
+ # Could perhaps be eliminated in favor of rolling state into the Job.
+ @requires_conn
+ def list_runs(self, uid: int):
+ pass
diff --git a/projects/tentacles/test/python/conftest.py b/projects/tentacles/test/python/conftest.py
new file mode 100644
index 0000000..ace7acd
--- /dev/null
+++ b/projects/tentacles/test/python/conftest.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+from datetime import timedelta
+import tentacles.store as s
+
+import pytest
+
+
+@pytest.yield_fixture
+def store():
+ conn = s.Store(":memory:")
+ conn.connect()
+ yield conn
+ conn.close()
+
+
+@pytest.fixture
+def username_testy():
+ return "testy@test.com"
+
+
+@pytest.fixture
+def password_testy():
+ return "testpw"
+
+
+@pytest.fixture
+def uid_testy(store, username_testy, password_testy):
+ return store.try_create_user(username_testy, password_testy)
+
+
+@pytest.fixture
+def login_ttl():
+ return timedelta(hours=12)
+
+
+@pytest.fixture
+def sid_testy(store, uid_testy, username_testy, password_testy, login_ttl):
+ return store.try_login(username_testy, password_testy, login_ttl)
diff --git a/projects/tentacles/test/python/test_store.py b/projects/tentacles/test/python/test_store.py
new file mode 100644
index 0000000..d79d390
--- /dev/null
+++ b/projects/tentacles/test/python/test_store.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+from tentacles.store import Store
+
+import pytest
+
+
+def test_store_initializes(store: Store):
+ assert isinstance(store, Store)
+
+
+def test_mkuser(store: Store, uid_testy, username_testy):
+ assert store.list_users() == [(uid_testy, username_testy)]
+
+
+def test_mksession(store: Store, uid_testy, username_testy, password_testy, login_ttl):
+ sid = store.try_login(username_testy, password_testy, login_ttl)
+ assert sid is not None
+ assert store.list_keys() == [(sid, uid_testy)]
+ assert store.try_key(sid) == uid_testy
+
+
+def test_refresh_key(store: Store, sid_testy, login_ttl):
+ before = store.fetch_key(sid_testy)
+ store.refresh_key(sid_testy, login_ttl * 2)
+ after = store.fetch_key(sid_testy)
+ assert before != after
+
+
+def tets_mkkey(store: Store, sid_testy, uid_testy):
+ assert store.try_key(sid_testy) == uid_testy
+ new_key = store.create_key(sid_testy, None)
+ assert new_key is not None
+ assert store.try_key(new_key) == uid_testy
+
+
+def test_logout(store: Store, sid_testy):
+ assert store.try_key(sid_testy)
+ store.delete_key(sid_testy)
+ assert not store.try_key(sid_testy)