Tapping on tentacles

WORKSPACE

@@ -52,7 +52,7 @@ load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
     name = "arrdem_source_pypi",
-    requirements_lock = "//tools/python:requirements.txt",
+    requirements_lock = "//tools/python:requirements_lock.txt",
     python_interpreter_target = "//tools/python:pythonshim",
 )
 

projects/tentacles/BUILD

@@ -0,0 +1,11 @@
+py_project(
+    name = "tentacles",
+    main = "src/python/tentacles/__main__.py",
+    main_deps = [
+        "//projects/anosql",
+        "//projects/anosql-migrations",
+        py_requirement("click"),
+        py_requirement("flask"),
+        py_requirement("jinja2"),
+    ]
+)

projects/tentacles/README.md

@@ -0,0 +1,46 @@
+# Tentacles
+
+A simple queue system for OctoPrint, designed to receive jobs and forward them to connected OctoPrint instances as they are available and ready.
+
+## Workflow
+
+Username+Password users
+
+API keys mapped to users
+
+Users mapped to job priority
+
+API keys mapped to user sub-priority
+
+User-facing API can upload gcode files to storage and create a file entry in the DB
+
+Jobs can be created from new or existing file entries in the DB
+
+Jobs are mapped to the priority of the API key (and user) who created them
+
+Jobs get Runs - which track assigning a pending Job to a Printer
+
+Highest priority job schedules first/next non-Running Job to the first available printer.
+This requires copying the gcode file to the target device, and starting it running.
+
+Running Runs are polled to detect loss/failure. Lost or failed Runs are deleted, and the Job returns to the queue.
+A priority penalty may be called for.
+
+## Notes
+
+Checking bed status on a printer
+
+    $ curl 'http://10.0.0.6/api/plugin/bedready' \
+        -X POST \
+        -H 'Accept: application/json' \
+        -H 'Content-Type: application/json; charset=UTF-8' \
+        -H 'Authorization: Bearer 8295AA7DDED645629C5006F2CC2042C2' \
+        --data-raw '{"command":"check_bed","reference":"reference_2023-05-11T05:22:40.212Z.jpg"}' \
+        | jq .
+
+Uploading a file and starting to print it
+- https://github.com/prusa3d/PrusaSlicer/blob/0384d631d6ef1aaadcb68da031eba9a586b102ed/src/slic3r/Utils/OctoPrint.cpp#L936
+- https://docs.octoprint.org/en/master/api/files.html#upload-file-or-create-folder
+- &print=true
+
+Doesn't appear possible to execute request/response g-code through the API? Can't detect when the printer's offsets aren't calibrated?

projects/tentacles/src/python/tentacles/__main__.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+"""
+
+"""
+
+import click
+import flask
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+def serve():
+    app = flask.Flask()
+    app.register_blueprint()
+    app.run()
+
+
+if __name__ == "__main__":
+    cli()

projects/tentacles/src/python/tentacles/blueprints.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+"""Blueprints for the Tentacles app."""
+
+from flask import Blueprint
+
+BLUEPRINT = Blueprint(__name__, __qualname__, template_folder=__package__)

projects/tentacles/src/python/tentacles/store.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+
+from datetime import timedelta, datetime
+from hashlib import sha3_256
+from pathlib import Path
+import sqlite3
+from textwrap import indent
+
+from collections import namedtuple
+from typing import Optional
+
+
+PRELUDE = """
+CREATE TABLE IF NOT EXISTS groups (
+   id INTEGER PRIMARY KEY AUTOINCREMENT
+ , name TEXT
+ , priority INTEGER CHECK(priority IS NOT NULL AND priority > 0)
+            DEFAULT 100
+);
+
+CREATE TABLE IF NOT EXISTS users (
+   id INTEGER PRIMARY KEY AUTOINCREMENT
+ , group_id INTEGER
+ , name TEXT
+ , hash TEXT
+ , FOREIGN KEY(group_id) REFERENCES groups(id)
+ , UNIQUE(name)
+);
+
+CREATE TABLE IF NOT EXISTS user_keys (
+   id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(48))))
+ , user_id INTEGER
+ , expiration TEXT
+);
+
+CREATE TABLE IF NOT EXISTS files (
+   id INTEGER PRIMARY KEY AUTOINCREMENT
+ , user_id INTEGER
+ , filename TEXT
+ , upload_date TEXT
+ , FOREIGN KEY(user_id) REFERENCES user(id)
+);
+
+CREATE TABLE IF NOT EXISTS jobs (
+   id INTEGER PRIMARY KEY AUTOINCREMENT
+ , user_id INTEGER
+ , FOREIGN KEY(user_id) REFERENCES user(id)
+);
+
+CREATE TABLE IF NOT EXISTs printer_statuses (
+   id INTEGER PRIMARY KEY AUTOINCREMENT
+ , name TEXT
+ , UNIQUE(name)
+);
+
+INSERT OR IGNORE INTO printer_statuses (name) values ('disconnected');
+INSERT OR IGNORE INTO printer_statuses (name) values ('connected');
+INSERT OR IGNORE INTO printer_statuses (name) values ('idle');
+INSERT OR IGNORE INTO printer_statuses (name) values ('running');
+INSERT OR IGNORE INTO printer_statuses (name) values ('error');
+
+CREATE TABLE IF NOT EXISTS printers (
+   id INTEGER PRIMARY KEY AUTOINCREMENT
+ , url TEXT
+ , api_key TEXT
+ , status_id INTEGER
+ , last_poll_date TEXT
+ , FOREIGN KEY(status_id) REFERENCES printer_statuses(id)
+);
+
+CREATE TABLE IF NOT EXISTS runs (
+   id INTEGER PRIMARY KEY AUTOINCREMENT
+ , job_id INTEGER
+ , started_at TEXT
+ , finished_at TEXT
+ , state TEXT
+ , printer_id INTEGER
+ , FOREIGN KEY(job_id) REFERENCES job(id)
+ , FOREIGN KEY(printer_id) REFERENCES printer(id)
+);
+"""
+
+
+def requires_conn(f):
+    def _helper(self, *args, **kwargs):
+        if self._conn is None:
+            raise ConnectionError(f"A connection is required for {f.__name__}")
+        return f(self, *args, **kwargs)
+
+    return _helper
+
+
+class Store(object):
+    def __init__(self, path):
+        self._path = path
+        self._conn: sqlite3.Connection = None
+
+    def connect(self):
+        if not self._conn:
+            self._conn = sqlite3.connect(self._path, isolation_level="IMMEDIATE")
+            for hunk in PRELUDE.split("\n\n"):
+                try:
+                    self._conn.executescript(hunk).fetchall()
+                except sqlite3.OperationalError as e:
+                    raise RuntimeError(
+                        f"Unable to execute startup script:\n{indent(hunk, ' > ')}"
+                    ) from e
+
+    def close(self):
+        if self._conn:
+            self._conn.commit()
+            self._conn.close()
+            self._conn = None
+
+    @requires_conn
+    def try_create_user(self, username, password):
+        """Attempt to create a new user."""
+
+        digest = sha3_256()
+        digest.update(password.encode("utf-8"))
+        return self._conn.execute(
+            "INSERT INTO users (name, hash) VALUES (?, ?) RETURNING (id)",
+            [username, digest.hexdigest()],
+        ).fetchone()[0]
+
+    @requires_conn
+    def list_users(self):
+        return self._conn.execute("SELECT id, name FROM users").fetchall()
+
+    ################################################################################
+    # Sessions / 'keys'
+
+    @requires_conn
+    def _create_session(self, uid: int, ttl: Optional[timedelta]):
+        return self._conn.execute(
+            "INSERT INTO user_keys (user_id, expiration) VALUES (?, ?) RETURNING (id)",
+            [uid, (datetime.now() + ttl).isoformat() if ttl else None],
+        ).fetchone()[0]
+
+    @requires_conn
+    def try_login(self, username: str, password: str, ttl: timedelta) -> Optional[str]:
+        """Given a username and an (unsecured) password, attempt to authenticate the named user.
+
+        If successful, return the ID of a new session/key for that user.
+
+        """
+
+        digest = sha3_256()
+        digest.update(password.encode("utf-8"))
+        res = self._conn.execute(
+            "SELECT id FROM users WHERE name=? AND hash=? LIMIT 1",
+            [username, digest.hexdigest()],
+        ).fetchone()
+        if not res:
+            return
+        uid = res[0]
+        return self._create_session(uid, ttl)
+
+    @requires_conn
+    def create_key(self, kid: str, ttl) -> Optional[str]:
+        """Given an _existing_ login session, create a new key.
+
+        This allows the user to create more or less permanent API keys associated with their identity.
+
+        """
+
+        if uid := self.try_key(kid):
+            return self._create_session(uid, ttl)
+
+    @requires_conn
+    def list_keys(self):
+        return self._conn.execute("SELECT id, user_id FROM user_keys").fetchall()
+
+    @requires_conn
+    def fetch_key(self, kid) -> tuple:
+        return self._conn.execute(
+            "SELECT * FROM user_keys WHERE id = ?", [kid]
+        ).fetchone()
+
+    @requires_conn
+    def try_key(self, kid: str):
+        """Try to find the mapped user for a session."""
+
+        res = self._conn.execute(
+            "SELECT user_id FROM user_keys WHERE expiration IS NULL OR unixepoch(expiration) > unixepoch('now') and id = ?",
+            [kid],
+        ).fetchone()
+        if res:
+            return res[0]
+
+    @requires_conn
+    def refresh_key(self, kid: str, ttl: timedelta):
+        """Automagically renew an API key which is still in use.
+
+        Mostly intended for dealing with web sessions which should implicitly extend, but which use the same mechanism as API keys.
+
+        """
+
+        self._conn.execute(
+            "UPDATE user_keys SET expiration = ? WHERE id = ?",
+            [(datetime.now() + ttl).isoformat(), kid],
+        )
+
+    @requires_conn
+    def delete_key(self, kid: str):
+        """Remove a session/key; equivalent to logout."""
+
+        self._conn.execute("DELETE FROM user_keys WHERE id = ?", [kid])
+
+    ################################################################################
+    # Printers
+    #
+    # Printers represent connections to OctoPrint instances controlling physical machines.
+    @requires_conn
+    def create_printer(self):
+        pass
+
+    @requires_conn
+    def list_printers(self):
+        pass
+
+    @requires_conn
+    def update_printer_status(self):
+        pass
+
+    ################################################################################
+    # Files
+    #
+    # A record of local files on disk, and the users who own then.
+    @requires_conn
+    def create_file(self, uid: int, path: Path):
+        pass
+
+    @requires_conn
+    def list_files(self, uid: int):
+        pass
+
+    @requires_conn
+    def delete_file(self, uid: int, fid: int):
+        pass
+
+    ################################################################################
+    # Job
+    #
+    # A request by a user for a given file to be printed.
+    @requires_conn
+    def create_job(self, uid: int):
+        pass
+
+    @requires_conn
+    def list_jobs(self, uid: int):
+        pass
+
+    @requires_conn
+    def delete_job(self, uid: int, jid: int):
+        pass
+
+    ################################################################################
+    # Run
+    #
+    # A record tha that a Job has been assigned to a Printer and is running.
+    # Could perhaps be eliminated in favor of rolling state into the Job.
+    @requires_conn
+    def list_runs(self, uid: int):
+        pass

projects/tentacles/test/python/conftest.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+from datetime import timedelta
+import tentacles.store as s
+
+import pytest
+
+
+@pytest.yield_fixture
+def store():
+    conn = s.Store(":memory:")
+    conn.connect()
+    yield conn
+    conn.close()
+
+
+@pytest.fixture
+def username_testy():
+    return "testy@test.com"
+
+
+@pytest.fixture
+def password_testy():
+    return "testpw"
+
+
+@pytest.fixture
+def uid_testy(store, username_testy, password_testy):
+    return store.try_create_user(username_testy, password_testy)
+
+
+@pytest.fixture
+def login_ttl():
+    return timedelta(hours=12)
+
+
+@pytest.fixture
+def sid_testy(store, uid_testy, username_testy, password_testy, login_ttl):
+    return store.try_login(username_testy, password_testy, login_ttl)

projects/tentacles/test/python/test_store.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+from tentacles.store import Store
+
+import pytest
+
+
+def test_store_initializes(store: Store):
+    assert isinstance(store, Store)
+
+
+def test_mkuser(store: Store, uid_testy, username_testy):
+    assert store.list_users() == [(uid_testy, username_testy)]
+
+
+def test_mksession(store: Store, uid_testy, username_testy, password_testy, login_ttl):
+    sid = store.try_login(username_testy, password_testy, login_ttl)
+    assert sid is not None
+    assert store.list_keys() == [(sid, uid_testy)]
+    assert store.try_key(sid) == uid_testy
+
+
+def test_refresh_key(store: Store, sid_testy, login_ttl):
+    before = store.fetch_key(sid_testy)
+    store.refresh_key(sid_testy, login_ttl * 2)
+    after = store.fetch_key(sid_testy)
+    assert before != after
+
+
+def tets_mkkey(store: Store, sid_testy, uid_testy):
+    assert store.try_key(sid_testy) == uid_testy
+    new_key = store.create_key(sid_testy, None)
+    assert new_key is not None
+    assert store.try_key(new_key) == uid_testy
+
+
+def test_logout(store: Store, sid_testy):
+    assert store.try_key(sid_testy)
+    store.delete_key(sid_testy)
+    assert not store.try_key(sid_testy)

tools/python/requirements.in

@@ -2,6 +2,7 @@ ExifRead
 aiohttp
 aiohttp_basicauth
 async_lru
+attrs
 autoflake
 beautifulsoup4
 black
@@ -30,6 +31,8 @@ pycryptodome
 pyrsistent
 pytest-cov
 pytest-postgresql
+pytest-pudb
+pytest-timeout
 pyyaml
 recommonmark
 redis
@@ -40,7 +43,7 @@ smbus2
 sphinx
 sphinxcontrib-openapi
 sphinxcontrib-programoutput
+toml
 unify
 yamllint
 yaspin
-toml

tools/python/requirements_lock.txt

@@ -9,6 +9,7 @@ autoflake==2.0.2
 Babel==2.12.1
 beautifulsoup4==4.12.0
 black==23.1.0
+blinker==1.6.2
 cachetools==5.3.0
 certifi==2022.12.7
 charset-normalizer==3.1.0
@@ -32,6 +33,7 @@ imagesize==1.4.1
 iniconfig==2.0.0
 isort==5.12.0
 itsdangerous==2.1.2
+jedi==0.18.2
 Jinja2==3.1.2
 jsonschema==4.17.3
 jsonschema-spec==0.1.4
@@ -51,6 +53,7 @@ octorest==0.4
 openapi-schema-validator==0.4.4
 openapi-spec-validator==0.5.6
 packaging==23.0
+parso==0.8.3
 pathable==0.4.3
 pathspec==0.11.1
 picobox==2.2.0
@@ -61,6 +64,7 @@ prompt-toolkit==3.0.38
 proquint==0.2.1
 psutil==5.9.4
 psycopg2==2.9.5
+pudb==2022.1.3
 py==1.11.0
 pycodestyle==2.10.0
 pycryptodome==3.17
@@ -70,6 +74,8 @@ pyrsistent==0.19.3
 pytest==7.2.2
 pytest-cov==4.0.0
 pytest-postgresql==4.1.1
+pytest-pudb==0.7.0
+pytest-timeout==2.1.0
 PyYAML==6.0
 recommonmark==0.7.1
 redis==4.5.3
@@ -100,6 +106,8 @@ typing_extensions==4.5.0
 unify==0.5
 untokenize==0.1.1
 urllib3==1.26.15
+urwid==2.1.2
+urwid-readline==0.13
 wcwidth==0.2.6
 websocket-client==1.5.1
 Werkzeug==2.2.3