Compare commits

...

403 commits

Author SHA1 Message Date
1ad35f95f1 [NO TESTS] WIP 2024-05-13 22:31:23 -06:00
0ce7e07264 Get the tests passing again 2024-04-07 01:30:45 -06:00
1ffccfae5b M0ar 2024-02-12 18:57:45 -07:00
ffa19de66b Add the MODULE.bazel file 2024-02-12 18:57:34 -07:00
4f346d6ac0 Bump deps 2024-02-12 18:54:41 -07:00
027335bd5b Leave some notes 2024-02-06 21:28:40 -07:00
d09b6bb2c6 Sort 2024-02-06 20:55:46 -07:00
2766c4bc6e Strip the .py suffix 2024-02-06 20:55:36 -07:00
fec681d41b Finishing touches 2024-02-06 20:55:24 -07:00
77912e9b65 [NO TESTS] WIP 2024-02-06 12:50:59 -07:00
e82259f756 KTF support 2024-02-06 12:38:04 -07:00
ab3357df3f Bazelshim 2024-02-06 12:37:41 -07:00
27af0d2dff Fix imports 2024-02-06 11:20:44 -07:00
89ac85693d m0ar 2024-02-06 11:19:46 -07:00
514e90d55d fix 2024-02-06 11:18:33 -07:00
4f7249e476 setuptools, not Maven 2024-02-06 11:17:32 -07:00
45da5b603d BUILD.bazel 2024-02-06 10:01:19 -07:00
6dc4b5b270 More records 2023-12-24 20:31:54 -07:00
13bbf2ce57 Not sure why this broke, but fixing DNS sync 2023-11-29 22:41:10 -07:00
2a499c2d0f Fix where we pull the login name from 2023-10-03 14:39:58 -06:00
9e85f28008 Enable ignoring issues by author ID 2023-10-03 00:07:23 -06:00
8ffb3a1fc1 Enable ignoring PRs by author 2023-10-03 00:04:34 -06:00
8db16e8cb1 More tuning of the unnotifier 2023-10-02 22:51:38 -06:00
a26d1d8b40 Use a patched textlib; lolbazel 2023-10-02 22:19:46 -06:00
39fb155824 Minor tweaks 2023-10-02 22:19:28 -06:00
9043cbe92b [NO TESTS] WIP 2023-08-16 19:07:11 -06:00
d21b1b2ddc Change LICENSE 2023-07-27 17:43:20 -06:00
15816ef524 Tighten readme; license 2023-07-27 17:33:08 -06:00
ada34c8ac9 More notes 2023-07-27 17:26:03 -06:00
5895e7f404 Blah 2023-07-27 17:23:39 -06:00
ad9816f55d Create the unnotifier 2023-07-27 16:34:26 -06:00
e9efc1dc84 Update requirements 2023-07-27 16:34:13 -06:00
8d9702dd32 Feat: cancel icon 2023-07-20 00:23:55 -06:00
0ed2b90d99 Fix: do not try to re-map assigned and pending jobs 2023-07-20 00:18:03 -06:00
ec10fa359d Fix: Need to return user_id here for the jobs list 2023-07-20 00:13:38 -06:00
72a8753639 Feat: Improving breaks; show other users' jobs & files to admins 2023-07-08 22:30:12 -06:00
f5bb2765a7 Feat: improve responsive behavior w/ icons 2023-07-08 22:10:40 -06:00
f3cd8b317a Fix: don't print hashes 2023-07-08 22:10:13 -06:00
6bc0710479 Stomping down the dot regressions 2023-07-08 21:00:25 -06:00
b3d3f9269f Cleanup: Rely on top-level exception handling 2023-07-08 19:55:54 -06:00
4b36a3cd19 Fix: Make job starting EVEN MORE idempotent 2023-07-08 19:54:44 -06:00
9b52c546e4 Fix: Make starting jobs more idempotent 2023-07-08 19:36:45 -06:00
f05dad1a18 Fix: need to specify the kwarg 2023-07-08 19:32:47 -06:00
5b39d9e653 Fix: recover filament IDs correctly; edit nozzle size; nozzle as a schedule constraint 2023-07-08 19:21:32 -06:00
5ceb2d1488 Delete & notify on bad files 2023-07-08 18:51:54 -06:00
db9e8c1105 Fix: use labels for the success and failure counts 2023-07-08 18:09:46 -06:00
253a4e5b59 Fix: use executemany not executescript
The latter doesn't support variable substitution
2023-07-08 18:05:23 -06:00
37cfcb190e Fix: user_id not uid 2023-07-08 17:54:46 -06:00
dcf2de3463 Fix: Need to specify table name for subquery reference 2023-07-08 17:52:28 -06:00
94058435a3 Cascade when deleting files 2023-07-08 17:48:24 -06:00
1d6eb61b69 Fix: SELECT WHERE syntax 2023-07-08 17:48:07 -06:00
1916e06ea2 Fix: need to flow through the nozzle size 2023-07-08 17:40:15 -06:00
32c5f5f597 More limits refinements 2023-07-08 17:35:17 -06:00
f38ee4ad97 Fix tests, create gcode analysis machinery 2023-07-08 16:47:12 -06:00
c1375cb179 Get printer editing working end to end 2023-07-08 13:45:31 -06:00
0fa65cfbe2 Header. 2023-07-08 13:44:57 -06:00
cfcfb755b9 Rework the login page using skeleton 2023-07-08 13:44:33 -06:00
b39a0b1bf5 Full width password inputs 2023-07-08 13:44:17 -06:00
d3e19bddfe Break up the schema file 2023-07-08 13:14:04 -06:00
76996ccd84 Start setting up for filament; sizes as scheduling constraints 2023-07-06 23:55:03 -06:00
c218bade5c Fmt. 2023-07-06 22:55:26 -06:00
0813c6825a Bugfix: incorrect login URL 2023-07-06 22:54:54 -06:00
10a0e54c33 Fix: if the login attempt failed, return None 2023-06-22 15:01:21 -06:00
3345cee21b Fix: Only if it would be non-null 2023-06-22 15:01:04 -06:00
c9da93f00c Limit returned job history 2023-06-19 23:29:30 -06:00
01b64c34ab Use the bedready plugin to check if the bed is clear or occupied 2023-06-19 23:25:24 -06:00
4628d1a28b Create an 'occupied' state for unready beds 2023-06-19 23:25:24 -06:00
7b0cf463fb Cache API clients to reduce 500 risk(s) 2023-06-19 23:24:10 -06:00
01c2ad121b fmt 2023-06-05 01:07:54 -06:00
2201a5384b fix: need to get row id to start jobs automatically 2023-06-05 01:07:47 -06:00
e2f3e18dff Bugstomping in the tests 2023-06-04 23:53:00 -06:00
058800b5dd feat: Decorate the streams with a border 2023-06-03 22:32:24 -06:00
d79a2a578f feat: Printer edit flow; dashboard webcams 2023-06-03 22:19:26 -06:00
de7f91947f Fix: display job runtimes in history 2023-06-03 20:04:06 -06:00
1274ab8ec2 Fix: Hide the controls when logged out 2023-06-03 19:59:33 -06:00
afe4f26ff7 More of same 2023-06-03 19:56:54 -06:00
e6cdb3666b Tweak: show the whole history to logged out users 2023-06-03 19:54:59 -06:00
78e456dfd0 Bugfix: don't mark successes as failures by accident 2023-06-03 19:53:29 -06:00
1807eb75dd Bugfix: Harden against lost jobs 2023-06-03 19:46:26 -06:00
dd6f6ac59f Show print runtime 2023-06-03 19:17:35 -06:00
74f635fcf6 Bugfix: don't show the file controls when logged out 2023-06-03 19:13:43 -06:00
c749adfaf6 Bugfix: /admin/printers, handle 404 from files_info 2023-06-03 19:08:11 -06:00
3f41b63582 Fix the public DNS 2023-06-03 19:07:41 -06:00
d6d9a348b1 Get a user approval flow working 2023-06-03 17:44:34 -06:00
42ad82d96e Add the interpreter 2023-06-03 15:39:34 -06:00
95d67b4aeb Create a trace mode for SQL inspection 2023-06-03 15:14:02 -06:00
48c30fdcf8 A whoooooole lot of refactoring 2023-06-03 15:09:50 -06:00
bda009482b More splitting 2023-06-03 13:22:49 -06:00
59f6096945 Shuffle 2023-06-03 13:20:43 -06:00
413825d8c7 Overhauling atop aiosql 2023-06-03 13:20:05 -06:00
53cdd5ca14 Embed cherrypy directly 2023-06-03 10:34:25 -06:00
7857cc3ffb Pull in aiosql; the anosql heir 2023-06-03 10:31:11 -06:00
657b0fe820 Make data and srcs additive 2023-06-03 01:34:57 -06:00
0bf20224d1 More config ignores 2023-06-03 01:34:43 -06:00
764b723254 Use sid 2023-06-03 01:34:28 -06:00
aa017aa8c5 Working on dockerization 2023-06-03 01:24:13 -06:00
d9db59b1db Simplify emails with a spool 2023-06-02 23:58:29 -06:00
2a59c41903 And get the shim working to my taste 2023-06-02 23:24:03 -06:00
b981e1255e Bolt up a WSGI server shim; tapping on verification 2023-06-02 23:07:59 -06:00
883c74bf62 Raise the connect timeout to 1s from 1/10s
OctoPrint isn't fast and this should mitigate some error flakes.
2023-05-30 22:24:08 -06:00
acaf3d0645 Fmt. 2023-05-30 22:22:30 -06:00
b17bafa077 Enable selective formatting 2023-05-30 22:22:13 -06:00
9b5ed98b16 Tapping towards a real user flow 2023-05-30 22:21:51 -06:00
e44390a37f Ignore config files generally 2023-05-30 22:21:08 -06:00
198115860e Get PrusaSlicer working; fix some auth problems 2023-05-29 10:27:52 -06:00
f003351290 Add tooltips 2023-05-29 00:05:43 -06:00
02b40b3cc4 copy API keys; CSS refinements 2023-05-28 23:54:13 -06:00
2ccdd48d63 Lotta state sync improvements 2023-05-28 22:57:11 -06:00
e082a4aa11 Fmt. 2023-05-28 17:37:47 -06:00
984d976c8c CSS tweaking; disable printing hashes 2023-05-28 17:31:34 -06:00
5546934fae UI overhaul; job cancellation 2023-05-28 17:21:05 -06:00
2fde40d1ab Tolerate timeouts 2023-05-28 00:21:31 -06:00
7e9588c11a Pushing jobs to printers works now 2023-05-28 00:15:36 -06:00
77143fb0d6 And deal with job termination 2023-05-27 22:13:19 -06:00
304d72cd83 Wiring up the printer control machinery 2023-05-27 22:03:25 -06:00
f1ea2cb645 Make import paths work right 2023-05-27 18:55:38 -06:00
2dfb2ca574 Style flailings 2023-05-27 18:55:18 -06:00
ae7891605a autoflake 2023-05-27 10:42:30 -06:00
d959a13aa2 Some bare minimum of styling the API key page 2023-05-27 00:45:34 -06:00
3adbdeb254 autoflake 2023-05-27 00:31:42 -06:00
409d04a648 Create a ctx global to replace request abuse 2023-05-27 00:30:39 -06:00
bedad7d86b Get API key management wired up 2023-05-26 23:54:36 -06:00
aacc9f5c1c Update requirements 2023-05-26 22:39:54 -06:00
3ad716837c Adding printers sorta works now 2023-05-23 00:15:16 -06:00
66c08101c7 Ignore sqlite binary dbs 2023-05-22 22:31:08 -06:00
f18d738462 Update requirements 2023-05-22 22:30:33 -06:00
b6b0f87a84 More tentacles progress 2023-05-22 22:21:53 -06:00
4413df7132 [NO TESTS] WIP 2023-05-21 23:16:53 -06:00
ae6044c76e [NO TESTS] WIP 2023-05-19 21:32:25 -06:00
603caca9bb A quick logo 2023-05-19 01:25:12 -06:00
5a284aa980 Register/Login/Logout somewhat working 2023-05-19 00:52:07 -06:00
63b2c9042b Bazelisk ftw 2023-05-19 00:51:32 -06:00
4c75d4694a Update some Python machinery, create sass tools 2023-05-19 00:51:18 -06:00
61dc694078 Tapping on tentacles 2023-05-13 16:58:17 -06:00
c23aa10f7a Infra tweaks 2023-05-13 16:56:01 -06:00
b44f00bb4f [NO TESTS] WIP 2023-05-10 22:46:01 -06:00
3a5f0cdef8 Need this import 2023-04-06 01:33:13 -06:00
b49408c6c4 Tweaking the packaging for a PKGBUILD 2023-04-06 01:26:36 -06:00
52177d3392 [NO TESTS] WIP 2023-03-20 18:50:05 -06:00
e4b15ac7e5 [NO TESTS] WIP 2023-03-20 18:50:00 -06:00
4324d14f02 Create a view for dumping the DB 2022-12-10 14:12:11 -07:00
1540ff1e2b Paranoia: save before exit 2022-11-26 23:08:25 -07:00
7b65a10bd8 Remove dead migration code that was fucking me over 2022-11-26 23:08:13 -07:00
ca10da03b3 Manually index the key 2022-11-26 22:52:13 -07:00
b1a78da719 Bugfix: need to use the pending's inbox 2022-11-26 22:51:57 -07:00
27fea36e48 jsonify the config object 2022-11-26 22:51:31 -07:00
21a524743c Don't select keys when saving 2022-11-26 22:17:24 -07:00
9f8c79a5ee Handle missing bodies more gracefully 2022-11-26 21:57:10 -07:00
33b1b8e39f Fix duplicate method name 2022-11-26 21:56:49 -07:00
c8388e0856 Enumerate pending follows, fix CSS 2022-11-26 21:56:33 -07:00
8cf17b942b Automate re-tagging and pushing 2022-11-26 21:56:06 -07:00
e90ce6b8ef Tweaks to working state 2022-11-26 20:05:35 -07:00
e5b9b133fc Create some quick and dirty admin endpoints 2022-11-26 19:26:39 -07:00
35a37aab8a Rework so that dockerized deployments can self-configure 2022-11-26 18:13:56 -07:00
2ade04ae7b Working towards a queue of follow requests 2022-11-26 17:24:53 -07:00
ad69d0bc44 Add a deploy script 2022-11-26 16:04:44 -07:00
6522c72b54 Whyyy docker-specific behavior 2022-11-26 15:46:42 -07:00
88bcd0fcf1 don't need that 2022-11-26 14:20:38 -07:00
4cce85710c public-dns -> public_dns 2022-11-26 14:20:22 -07:00
110ffc6ee1 Update zapp 2022-11-26 14:18:51 -07:00
7b7f20c4c3 Remove dead migration code 2022-11-21 01:51:42 -07:00
b2b7363797 Import the async caches 2022-11-21 00:04:31 -07:00
ea7acd915c Unused code. 2022-11-20 22:41:51 -07:00
70587ac360 Fmt 2022-11-20 22:39:44 -07:00
01c24f64ab Start trimming some of the __main__ crud 2022-11-20 22:34:07 -07:00
c21ee1fe48 Setting up to do some minimal admin auth for the relay 2022-11-20 22:00:12 -07:00
eb71f33e28 DNS for the relay 2022-11-19 23:55:01 -07:00
4faa29c124 Update some of the build infra 2022-11-19 23:54:45 -07:00
30f1d488b4 Sigh. 2022-11-19 23:51:09 -07:00
9d7ee93f8c Intern the relay 2022-11-19 23:45:47 -07:00
8737b7456b WIP 2022-10-26 22:23:41 -06:00
d102083a64 Bump Meraki API client 2022-10-19 09:52:37 -06:00
b26515dbc0 Tweaking the diff and equality logic 2022-10-19 09:52:18 -06:00
4498f350aa Cram has graduated 2022-10-19 09:00:14 -06:00
fbe837797a Tweaking the stack analyzer 2022-08-19 00:43:42 -06:00
71a60ae079 Get and2/3 working 2022-08-17 00:18:08 -06:00
77340aeb3b Get or3 working 2022-08-17 00:11:01 -06:00
2c92447947 Get or2 working again via the assembler 2022-08-17 00:07:09 -06:00
5a3a6bc96f Make the handle_* methods user-visible 2022-08-13 00:09:55 -06:00
510ee6abce [NO TESTS] WIP 2022-08-13 00:07:38 -06:00
93f5c7bc24 [NO TESTS] WIP 2022-08-13 00:03:49 -06:00
b241e6e0b4 Notes for later. 2022-08-12 23:53:48 -06:00
559e334e85 Create a generic interpreter API 2022-08-12 23:46:34 -06:00
66d90b1d44 Fmt. 2022-08-12 23:26:56 -06:00
a484137be2 Rewrite using handlers, singledispatch, hooks 2022-08-12 23:26:42 -06:00
3ee2f8ece9 Integrate marking labels/slots 2022-08-09 10:05:26 -06:00
2557ff1e4f Renaming variant operations 2022-08-09 09:39:33 -06:00
a7b236fa5f Promote cram out 2022-07-28 23:45:00 -06:00
5a34f095d7 Junk table dispatch for execute 2022-07-28 20:46:56 -06:00
b9617079ab return will abort the loop, use continue 2022-07-28 19:25:45 -06:00
0514e4ebb2 More cases of neading to clean during execution 2022-07-28 19:25:30 -06:00
20f3f01b6a Sigh 'stable' releases 2022-07-28 19:24:38 -06:00
b471e2ab82 Add a way to bust the log cache 2022-07-28 19:24:17 -06:00
bbb4f83071 Deal with the node being a bad link 2022-07-28 18:24:03 -06:00
31a0019546 ton.tirefire 2022-07-28 15:16:44 -06:00
1df60b873b Be better at removing dead links 2022-07-27 23:04:25 -06:00
3a8677c630 Update docstring 2022-07-26 00:24:30 -06:00
7a40392253 Implement auto-migrate (aka fmt) 2022-07-26 00:23:26 -06:00
f151e57a12 Hardening cram, stratify uninstalls 2022-07-25 22:44:10 -06:00
176d8129d6 Ignore .gitkeep files 2022-07-21 22:48:59 -06:00
58e63924d9 Short hostname only 2022-07-21 22:48:49 -06:00
353cf72b65 Fmt. 2022-07-15 19:37:34 -06:00
eb128d8b54 Refactor to make Opcode the base class 2022-07-15 19:33:32 -06:00
7e7c557dea Ditch the .impl pattern for now 2022-07-02 01:47:11 -06:00
21b4b4432a Slam out a two-pass assembler 2022-07-02 00:35:03 -06:00
13b9afb195 [NO TESTS] WIP 2022-06-27 22:35:21 -06:00
25234f8f00 Get VARAINT/VTEST implemented 2022-06-16 10:55:18 -06:00
7fe052e7fc [broken] Throw out builtin bool 2022-06-15 23:10:54 -06:00
d2d6840bad Rewrite bootstrap to sorta support type signatures 2022-06-15 23:01:55 -06:00
54ff11f13c Wire up define_type 2022-06-15 21:56:52 -06:00
d567f95c1f Better module printing 2022-06-15 09:46:03 -06:00
7f3efc882a Code better in the morning 2022-06-15 09:21:43 -06:00
366440a3b9 [broken] Almost to a reworked Function model 2022-06-15 01:46:32 -06:00
8469d11825 Better parser tests 2022-06-15 01:12:22 -06:00
6f1145cb5e Don't inject JEDI everywhere. Argh. 2022-06-15 01:11:44 -06:00
496dfb7026 Calling it a night 2022-06-14 01:19:30 -06:00
9b6e11818e Typing tweaks 2022-06-13 21:11:15 -06:00
8feb3f3989 Shuffling a bunch of stuff around 2022-06-13 20:23:43 -06:00
bc7cb3e909 [NO TESTS] WIP 2022-06-13 11:44:28 -06:00
0075c4c9f8 [NO TESTS] WIP 2022-06-11 15:04:55 -06:00
6916715123 Eliminate CALLS, add SLOT & DEBUG, rewrite bootstrap 2022-06-11 00:28:24 -06:00
f382afd9f1 Inject pytest-timeout 2022-06-11 00:27:52 -06:00
50f4a4cdb1 Tapping towards structs and variants 2022-06-07 10:32:15 -06:00
dc71833082 [NO TESTS] WIP 2022-06-01 00:22:53 -06:00
d5123351d5 Fmt. 2022-06-01 00:11:14 -06:00
af65fbb6f6 Get CLOSUREC working 2022-06-01 00:09:59 -06:00
3debddf0b2 Get CLOSUREF/CALLC working 2022-05-31 23:54:11 -06:00
d1fab97c85 Test stubs need returns now 2022-05-31 23:17:32 -06:00
824547821c An evening of hacking 2022-05-31 23:08:29 -06:00
c1498c98eb Split out Ichor 2022-05-31 19:25:18 -06:00
0ffc6c6bb7 Shogoth -> Shoggoth 2022-05-31 19:04:14 -06:00
b4005b3b6d Rename 2022-05-31 18:24:39 -06:00
c2e9b73946 A single-stepping GCODE tool 2022-05-31 09:41:32 -06:00
88b4e7da1f Fmt. 2022-05-31 09:41:10 -06:00
56b6ddd3ea [NO TESTS] WIP 2022-05-31 09:39:53 -06:00
13355aeef4 Tapping 2022-05-16 20:31:36 -06:00
68313e8db9 Tapping towards a slightly functional reader 2022-05-11 23:21:22 -07:00
8a70d2b64d Minor crimes to deal with platform-variable pythons 2022-05-11 23:20:59 -07:00
06eda8167f Be smart about the READONLY flag; consolidate delete 2022-05-11 22:58:35 -07:00
3c80d598be Errant whitespace 2022-05-11 11:01:23 -07:00
29cb977688 Turn that off; didn't work 2022-04-21 00:18:22 -06:00
352f7b8184 Function references; stubs for closures 2022-04-21 00:17:59 -06:00
d82a662b39 Happy hacking; setting up for structs 2022-04-15 00:31:58 -06:00
e7464c843d Turn this shit off 2022-04-12 01:49:43 -06:00
e04324884f Fmt. 2022-04-12 01:49:29 -06:00
298699c1d4 Get the Shogoth VM to a bit better state; py3.10 Ba'azel 2022-04-12 01:49:12 -06:00
c21eacf107 A bytecode VM sketch 2022-03-29 01:29:18 -06:00
69f4578817 Add a licensing note 2022-03-17 11:03:45 -06:00
3c70b29ad1 Import the garage project 2022-02-21 16:35:36 -07:00
d7ecd0cf7a Reuse the old feed static IP 2022-02-21 16:35:01 -07:00
bfb5a8ffc6 Realpath 2022-02-21 16:35:01 -07:00
3ea17b5d26 TOML format 2022-02-15 10:24:06 -07:00
cd9530ad66 config.toml -> pkg.toml, get cleanup working 2022-02-15 10:22:53 -07:00
00f2ba3bd7 A second cut at the cram tool, now with TOML and tests 2022-02-15 02:17:54 -07:00
a731f5e6cb Add a record for feed 2022-02-02 00:45:14 -07:00
5986396f18 Reader's 'done' 2022-01-09 16:32:11 -07:00
7cc79e5986 Initial parse/read driver REPL 2022-01-09 14:46:21 -07:00
3dbb810074 More deps fuckery 2022-01-09 11:23:06 -07:00
025c9c5671 Bump most of the deps 2022-01-09 00:22:56 -07:00
60d40bd956 cram fixes 2022-01-09 00:07:00 -07:00
bb50fa2c20 Normalize SQL so its nicer when logged 2022-01-08 23:53:24 -07:00
89cedc120a Fmt. 2021-12-27 00:49:28 -07:00
b5bb534664 Roll over API exceptions 2021-12-27 00:49:17 -07:00
355ec6974f [NO TESTS] WIP 2021-12-27 00:31:36 -07:00
df653a1819 Forgot some locks. Sigh. 2021-12-12 21:51:23 -07:00
37435f6423 Fairly viable aloe state 2021-12-12 21:36:21 -07:00
3e4d02f2f4 WIP - refactoring towards a better state 2021-12-12 20:55:39 -07:00
c589cbd6c5 WIP - somewhat working v2 state 2021-12-12 19:44:12 -07:00
39eff4e53a Spinner while copying; handle dirt files better 2021-12-05 11:35:19 -07:00
af63cd201f A quick and dirty octoprint CURSES dashboard 2021-12-05 11:35:09 -07:00
0dd2da7fd4 Fix obviously colliding ICMP stream IDs
But it looks like the underlying ICMPLIB model of receive() is broken for what I'm doing
2021-11-22 01:32:11 -07:00
fb8717e535 Don't need those 2021-11-20 23:11:47 -07:00
1273beea3b Blah. 2021-11-20 23:10:00 -07:00
74d543824b And done 2021-11-20 23:06:26 -07:00
a9eecc6998 And buffing 2021-11-20 22:37:57 -07:00
a0766fc42e Overhaul to a considerably more useful multiprocess state 2021-11-20 22:05:45 -07:00
a4cd4568cf WIP on Aloe 2021-11-20 14:39:14 -07:00
b6b1f23188 Fmt. 2021-11-09 12:31:31 -07:00
0d81f6540d Fixing list as a global and local fn 2021-11-09 12:31:18 -07:00
028873aa49 Bump click 2021-11-09 12:30:53 -07:00
2249302b06 More docs tweaks 2021-11-05 13:20:00 -06:00
ed633c9ab0 Documenting for publication 2021-11-05 12:18:45 -06:00
db08760964 Add pytest-cov by default when pytesting 2021-11-05 11:47:33 -06:00
ac5f5e1637 Finishing and testing the driver 2021-11-05 11:47:17 -06:00
3a58156427 Tapping on the revamped clusterctrl 2021-11-02 01:12:06 -06:00
a4b720158f Factor out HACKING 2021-11-01 09:47:59 -06:00
7d53405e71 Documentation tweaks 2021-10-31 11:53:28 -06:00
6cba9c3ad1 New host records 2021-10-31 10:59:45 -06:00
6f05542142 Teach cram about profile and host subpackages 2021-10-31 10:59:27 -06:00
9ac3f56b5b Blah 2021-10-27 22:47:15 -06:00
ef9e54e647 Rename to match the upstream naming 2021-10-27 22:44:46 -06:00
92e5605f8c exists() is the wrong predicate; false-negatives on broken links 2021-10-20 23:54:17 -06:00
3290a8aefc Final-ish tested driver 2021-10-13 20:28:54 -06:00
a17f7e497b Styleguide 2021-10-13 01:07:32 -06:00
684409cd16 A driver from scratch 2021-10-13 01:07:06 -06:00
73bddf2a52 Black. 2021-10-11 22:38:55 -06:00
7bb78e0497 Squashme: command whitespace 2021-10-11 22:33:34 -06:00
522f1b6628 Hoist out helper fns
Note that since __main__ is in global scope, no closure conversion is required
2021-10-11 22:26:44 -06:00
b18edf1c4a Invalid config should be a nonzero exit 2021-10-11 22:23:53 -06:00
17034424c2 Format the commands so they'll be easier to pull apart 2021-10-11 22:00:53 -06:00
6640f54407 Unify to double quotes 2021-10-11 21:55:35 -06:00
7713d6d9a8 Fix bananas use of commas 2021-10-11 21:54:38 -06:00
0d2acb676f Fix just bananas usage of parens 2021-10-11 21:53:28 -06:00
9eed6ef803 Refactor into a single print() call 2021-10-11 21:42:19 -06:00
de4a8bdaa8 Refactor out constants 2021-10-11 21:41:48 -06:00
b0130c27a6 Refactor into __main__ block 2021-10-11 21:26:06 -06:00
e02af611ad Refactor xra1200 import 2021-10-11 21:24:43 -06:00
c4283e4be1 Initial hatctl state 2021-10-11 21:22:49 -06:00
aab2ec1c33 More improvements and an execution optimizer 2021-10-11 00:09:51 -06:00
b9e40b5bb1 Add profile.d/default to the default requirements 2021-10-10 22:57:53 -06:00
f89459294d Factor out the VFS since it's kinda neat 2021-10-10 22:40:05 -06:00
4d9a777fc8 Get fmt working again 2021-10-10 21:42:55 -06:00
2ab0339e49 Document working in here 2021-10-10 21:42:38 -06:00
8deb1bed25 First cut at a cram tool 2021-10-10 21:41:01 -06:00
376d032808 Fmt. 2021-10-10 21:40:27 -06:00
6636b75457 What do you want from me 2021-10-07 10:52:08 -06:00
19c306d3df Hail ba'azel 2021-10-07 10:49:41 -06:00
bbfee1d3ff Create bazel-test.yml 2021-10-07 09:58:35 -06:00
bb5b7aa9b4 So long yakshave.club, I never used you 2021-10-07 09:51:02 -06:00
b37d999796 De-conflict sequential source files that map to the same path 2021-10-04 13:31:00 -06:00
9aae534971 [NO TESTS] WIP 2021-09-25 00:43:50 -06:00
840e764b91 Fmt. 2021-09-24 22:37:38 -06:00
a6e59b2e0c Fmt. 2021-09-19 18:05:22 -06:00
223fd5688b Get test_licenses using pkg_info ala liccheck
Fixes #2
2021-09-19 17:59:18 -06:00
13d7f07b03 [NO TESTS] WIP 2021-09-07 02:21:34 -06:00
95e546ac55 Oh that's a delight 2021-09-07 01:00:37 -06:00
544e1cd151 [NO TESTS] WIP 2021-09-06 21:54:12 -06:00
74af14847e The archiver tools 2021-09-06 21:15:03 -06:00
23fc856791 Push mail forwarding for tirefireind 2021-09-06 21:12:10 -06:00
4ce35091eb Document the black shim a touch 2021-09-02 22:24:50 -06:00
c20fdde98c Syntax errors 2021-09-02 22:10:48 -06:00
9ed2165072 Black all the things 2021-09-02 22:10:35 -06:00
289e028e5b Turn on black as a linter 2021-09-02 22:10:24 -06:00
6eeb89cba4 Get black working as a linter 2021-09-02 22:10:12 -06:00
d501a4730d Update projects list 2021-08-30 01:18:55 -06:00
fae63eab11 Turn on flake8 for good 2021-08-30 01:07:13 -06:00
2b9d3ad927 Done with flake8 2021-08-30 01:06:21 -06:00
29aaea1a45 Lint headway 2021-08-30 00:43:58 -06:00
32035a005e The binary operator rules are silly 2021-08-30 00:43:49 -06:00
ae16691738 More tweaks 2021-08-30 00:40:02 -06:00
3612d2b9ee Msc. lint stomping 2021-08-30 00:30:44 -06:00
bf59df5ad7 The incantation for flake8 2021-08-30 00:30:23 -06:00
81386229c6 Get flake8 working as an aspect 2021-08-30 00:29:43 -06:00
3a1ffe6d6a Fmt. 2021-08-29 22:35:10 -06:00
01743bfbdd More tooling 2021-08-29 22:35:00 -06:00
fbb843a36c Set better test defaults 2021-08-29 22:19:09 -06:00
54ab71f19c And lint 2021-08-29 22:18:57 -06:00
bc06fc01ff More breaking out 2021-08-29 22:17:57 -06:00
3b9e4076a5 Break tools out into their own dirs 2021-08-29 22:13:59 -06:00
b56030804c Futzing with toolchains 2021-08-29 21:59:32 -06:00
8f72fc8d62 [NO TESTS] WIP 2021-08-29 19:23:39 -06:00
fe03f023e7 Re-freeze 2021-08-29 16:04:21 -06:00
19303931bc [NO TESTS] WIP 2021-08-23 10:51:41 -06:00
7bff95fe32 Last Lilith commit I promise 2021-08-23 10:10:03 -06:00
db9c0f7bf8 Add a Def() read-eval cache 2021-08-22 11:15:28 -06:00
2fbb0f7e1c linting 2021-08-21 22:25:47 -06:00
6c2c9190c9 Updating the docs and somewhat packaging for submission 2021-08-21 22:25:31 -06:00
3fc5bff69c Dirty awful hacks to get exec working 2021-08-21 21:38:42 -06:00
b8862bdeaf Main deps. 2021-08-21 21:38:07 -06:00
a6ce37de82 Working lexical/module scope & hello world 2021-08-21 21:17:48 -06:00
4a5262c95c Making headway towards a runnable state 2021-08-21 20:02:54 -06:00
d02f53dc52 Somewhat working launcher with prelude! 2021-08-21 18:58:33 -06:00
15c7417516 Blah. 2021-08-21 17:25:19 -06:00
f69b243c4f Convert lilith to a runnable 2021-08-21 17:20:13 -06:00
385cd9d83e Add support for runnable projects 2021-08-21 17:18:46 -06:00
d6a811118d Get arrays and mappings working 2021-08-21 17:14:57 -06:00
8db42aa17d Get symbols working, styleguide 2021-08-21 16:58:59 -06:00
7e1e8b2ad4 Adding Symbols 2021-08-21 16:44:49 -06:00
20ed127bf7 Deal with string escaping 2021-08-21 14:23:39 -06:00
02c5f61bb8 Hello, world! 2021-08-21 14:07:57 -06:00
5531b80331 Ready to try interpreting 2021-08-21 13:13:56 -06:00
e59adb1621 Tooling work 2021-08-21 11:49:56 -06:00
014ce0b21d Working Lilith block header parser 2021-08-21 11:49:46 -06:00
43bbcda050 Initial deeply busted Lilith state for LangJam 2021-08-20 23:16:59 -06:00
2b101bf02c Update benchmarks 2021-08-20 10:10:04 -06:00
01a5a1b67d Overhaul client and server 2021-08-20 01:37:20 -06:00
99590ae534 Tap out test coverage of the jobq 2021-08-20 01:12:50 -06:00
a3a800ab07 Make the jobq closeable and document it. 2021-08-19 23:54:08 -06:00
7608b6f004 Document benchmark results 2021-08-19 23:45:15 -06:00
23840cba8e Ignore .pyc object files 2021-08-19 23:29:06 -06:00
fe0fd3fdb1 Split the jobq into a library and a daemon 2021-08-19 23:28:33 -06:00
8c98338782 Truncate 2021-08-15 00:02:56 -06:00
961710961d Update public DNS from the first time through Ba'azel 2021-08-15 00:02:09 -06:00
5e9878742c Truncate. 2021-08-14 11:55:58 -06:00
692327e276 Be consistent about 'payload' 2021-08-14 11:23:27 -06:00
518648b238 Make the initial state user-definable 2021-08-14 11:18:19 -06:00
7436b65210 Fix exprs 2021-08-14 11:09:27 -06:00
4c4dab5913 Document CAS on POST /state 2021-08-14 11:05:44 -06:00
ba0ce2eb28 Allow LGPL deps 2021-08-14 09:25:47 -06:00
8f88a4da45 Get the anosql tests clean 2021-08-14 09:20:58 -06:00
d242f73dc4 An initial crack at a jobq 2021-08-14 00:31:07 -06:00
477d3f1055 Expunge the pypi reference 2021-08-13 19:57:17 -06:00
9d759fa187 Anosql is abandoned upstream. Vendor it. 2021-08-13 19:56:00 -06:00
0b9c68bce0 Add support for resources 2021-08-13 19:55:41 -06:00
668d58ba8c Zapp reqman 2021-08-13 17:17:15 -06:00
2e0026eadc Knock out a migrations system for AnoSQL 2021-08-13 16:45:53 -06:00
bdb4832ff7 Add a qint tool 2021-08-12 14:50:09 -06:00
767574cb44 Get the updater zapping nicely 2021-08-12 14:28:57 -06:00
f04740bde5 Lang spec. for emacs 2021-08-12 14:28:41 -06:00
60b629348d Bump rules_zapp for bugfixes 2021-08-12 14:28:26 -06:00
f0aa9a6579 Relocate zapp to its own repo 2021-08-12 12:51:50 -06:00
95ea44ab4d More README tweaks 2021-08-08 10:05:26 -06:00
37266cdcd9 Initial zapp state (#1)
This commit implements zapp! and rules_zapp, as a proof of concept of
lighter-weight easily hacked on self-extracting zipapps. Think Pex or
Shiv but with less behavior and more user control. Or at least
hackability.
2021-08-08 00:16:37 -06:00
403 changed files with 23817 additions and 1014 deletions

10
.bazelrc Normal file
View file

@ -0,0 +1,10 @@
common --curses=auto
common --show_timestamps=true
test --keep_going
test --test_output=errors
test --test_tag_filters=-known-to-fail
# To enable flake8 on all build steps, uncomment this -
# test --aspects="//tools/flake8:flake8.bzl%flake8_aspect" --output_groups=flake8_checks
# test --aspects="//tools/black:black.bzl%black_aspect" --output_groups=black_checks

1
.bazelversion Normal file
View file

@ -0,0 +1 @@
7.0.0

2
.envrc Normal file
View file

@ -0,0 +1,2 @@
export SOURCE=$(dirname $(realpath $0))
export PATH="${SOURCE}/bin:$PATH"

5
.gitignore vendored
View file

@ -19,6 +19,9 @@
/**/dist /**/dist
/**/node_modules /**/node_modules
bazel-* bazel-*
projects/public-dns/config.yml
public/ public/
tmp/ tmp/
/**/*.sqlite*
/**/config*.toml
/**/config*.yml
MODULE.bazel.lock

3
BUILD.bazel Normal file
View file

@ -0,0 +1,3 @@
package(default_visibility = ["//visibility:public"])
exports_files(["setup.cfg"])

60
HACKING.md Normal file
View file

@ -0,0 +1,60 @@
# Hacking on/in source
## Setup
### Step 1 - Install bazel
As usual, Ubuntu's packaging of the Bazel bootstrap script is ... considerably stale.
Add the upstream Bazel ppa so we can get reasonably fresh builds.
``` sh
sudo apt install apt-transport-https curl gnupg
curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg
sudo mv bazel.gpg /etc/apt/trusted.gpg.d/
echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
```
## Step 2 - Create a canonical `python`
Bazel, sadly, expects that a platform `python` exists for things.
Due to Bazel's plugins model, changing this is miserable.
On Archlinux, this isn't a problem. `python` is `python3`. But Ubuntu ... did the other thing.
``` sh
sudo apt install python-is-python3
```
### Step 3 - Non-hermetic build deps
The way that Bazel's Python toolchain(s) work is that ultimately they go out to the non-hermetic platform.
So, in order to be able to use the good tools we have to have some things on the host filesystem.
``` sh
if [[ "$(sqlite3 --version | awk -F'.' '/^3/ {print $2; exit}')" -lt 35 ]]; then
echo "SQLite 3.35.0 or later (select ... returning support) required"
exit 1
fi
sudo apt install \
python3-setuptools \
postgresql libpq-dev \
sqlite3 libsqlite3-dev
```
## Working in source
`source activate.sh` is the key.
It automates a number of tasks -
1. Building a virtualenv
2. Synchronizing the virtualenv from the requirements.txt
3. Updating the virtualenv with all project paths
4. Activating that virtualenv
`./tools/lint.sh` wraps up various linters into a one-stop shop.
`./tools/fmt.sh` wraps up various formatters into a one-stop shop.
`bazel build ...` will attempt to build everything.
While with a traditional build system this would be unreasonable, in Bazel which caches build products efficiently it's quite reasonable.
`bazel test ...` likewise will run (or know it doesn't need to run) all the tests.

20
LICENSE
View file

@ -1,7 +1,21 @@
ANTI-CAPITALIST SOFTWARE LICENSE (v 1.4)
Copyright 2017 Reid 'arrdem' McKenzie <me@arrdem.com> Copyright 2017 Reid 'arrdem' McKenzie <me@arrdem.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: This is anti-capitalist software, released for free use by individuals and organizations that do not operate by capitalist principles.
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. Permission is hereby granted, free of charge, to any person or organization (the "User") obtaining a copy of this software and associated documentation files (the "Software"), to use, copy, modify, merge, distribute, and/or sell copies of the Software, subject to the following conditions:
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1. The above copyright notice and this permission notice shall be included in all copies or modified versions of the Software.
2. The User is one of the following:
a. An individual person, laboring for themselves
b. A non-profit organization
c. An educational institution
d. An organization that seeks shared profit for all of its members, and allows non-members to set the cost of their labor
3. If the User is an organization with owners, then all owners are workers and all workers are owners with equal equity and/or equal vote.
4. If the User is an organization, then the User is not law enforcement or military, or working for or under either.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

6
MODULE.bazel Normal file
View file

@ -0,0 +1,6 @@
###############################################################################
# Bazel now uses Bzlmod by default to manage external dependencies.
# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
#
# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
###############################################################################

View file

@ -1,17 +1,25 @@
# Reid's monorepo # Reid's monorepo
I've found it inconvenient to develop lots of small Python modules. I've found it inconvenient to develop lots of small Python modules.
And so I'm going the other way; Bazel in a monorepo with several subprojects so I'm able to reuse a maximum of scaffolding. And so I'm going the other way; Bazel in a monorepo with subprojects so I'm able to reuse a maximum of scaffolding.
## Projects ## Projects
- [Datalog](projects/datalog) - [Datalog](projects/datalog) and the matching [shell](projects/datalog-shell)
- [YAML Schema](projects/yamlschema) (JSON schema with knowledge of PyYAML's syntax structure & nice errors)
- [Zapp! (now with a new home and releases)](https://github.com/arrdem/rules_zapp)
- [Cram (now with a new home)](https://github.com/arrdem/cram)
- [Flowmetal](projects/flowmetal) - [Flowmetal](projects/flowmetal)
- [YAML Schema](projects/yamlschema) - [Lilith](projects/lilith)
## Hacking (Ubuntu)
See [HACKING.md](HACKING.md)
## License ## License
Copyright Reid 'arrdem' McKenzie, 4/8/2021. All work in this repo unless otherwise noted is copyright © Reid D. McKenzie 2023 or the last commit date whichever is newer and published under the terms of the Anti-Capitalist Software License v1.4 or at your option later.
Unless labeled otherwise, the contents of this repository are distributed under the terms of the MIT license. See https://anticapitalist.software/ or the `LICENSE` file.
See the included `LICENSE` file for more.
Commercial licensing negotiable upon request.

View file

@ -35,27 +35,45 @@ bazel_skylib_workspace()
#################################################################################################### ####################################################################################################
# Python support # Python support
#################################################################################################### ####################################################################################################
# Using rules_python at a more recent SHA than the last release like a baws # Using rules_python at a more recent SHA than the last release like a baws
git_repository( git_repository(
name = "rules_python", name = "rules_python",
remote = "https://github.com/bazelbuild/rules_python.git", remote = "https://github.com/bazelbuild/rules_python.git",
tag = "0.3.0", # tag = "0.4.0",
commit = "5f51a4451b478c5fed491614756d16745682fd7c",
) )
load("@rules_python//python:repositories.bzl", "py_repositories")
py_repositories()
register_toolchains("//tools/python:python3_toolchain")
# pip package pinnings need to be initialized. # pip package pinnings need to be initialized.
# this generates a bunch of bzl rules so that each pip dep is a bzl target # this generates a bunch of bzl rules so that each pip dep is a bzl target
load("@rules_python//python:pip.bzl", "pip_install") load("@rules_python//python:pip.bzl", "pip_parse")
pip_install( pip_parse(
name = "arrdem_source_pypi", name = "arrdem_source_pypi",
requirements = "//tools/python:requirements.txt", requirements_lock = "//tools/python:requirements_lock.txt",
python_interpreter = "python3", python_interpreter_target = "//tools/python:pythonshim",
) )
#################################################################################################### # Load the starlark macro which will define your dependencies.
# Postscript load("@arrdem_source_pypi//:requirements.bzl", "install_deps")
####################################################################################################
# Do toolchain nonsense to use py3 # Call it to define repos for your requirements.
register_toolchains( install_deps()
"//tools/python:toolchain",
git_repository(
name = "rules_zapp",
remote = "https://git.arrdem.com/arrdem/rules_zapp.git",
commit = "961be891e5cff539e14f2050d5cd9e82845ce0f2",
# tag = "0.1.2",
) )
# local_repository(
# name = "rules_zapp",
# path = "/home/arrdem/Documents/hobby/programming/rules_zapp",
# )

13
activate.sh Normal file
View file

@ -0,0 +1,13 @@
#!/bin/bash
mkvirtualenv source
workon source
pip install -r tools/python/requirements.txt
for d in $(find . -type d -path "*/src/python"); do
d="$(realpath "${d}")"
echo "Adding subproject ${d}"
add2virtualenv "${d}"
done

2
bin/bazel Executable file
View file

@ -0,0 +1,2 @@
#!/bin/sh
exec "${SOURCE}/projects/bazelshim/src/bazelshim/__main__.py" --bazelshim_exclude="$(realpath "${SOURCE}")/bin" "$@"

2
bin/bazelisk Executable file
View file

@ -0,0 +1,2 @@
#!/bin/sh
exec "${SOURCE}/projects/bazelshim/src/bazelshim/__main__.py" --bazelshim_exclude="$(realpath "${SOURCE}")/bin" "$@"

View file

@ -0,0 +1,15 @@
py_project(
name = "activitypub_relay",
lib_deps = [
py_requirement("aiohttp"),
py_requirement("aiohttp_basicauth"),
py_requirement("async_lru"),
py_requirement("cachetools"),
py_requirement("click"),
py_requirement("pycryptodome"),
py_requirement("pyyaml"),
py_requirement("retry"),
],
main = "src/relay/__main__.py",
shebang = "/usr/bin/env python3"
)

View file

@ -0,0 +1,26 @@
FROM library/python:3.10
MAINTAINER Reid 'arrdem' McKenzie <me@arrdem.com>
RUN pip install --upgrade pip
RUN useradd -d /app app
RUN mkdir -p /app /data
RUN chown -R app:app /app /data
USER app
WORKDIR /app
VOLUME /data
ENV DOCKER_RUNNING=true
ENV PYTHONPATH="/app:${PYTHONPATH}"
ENV PYTHONUNBUFFERED=true
ENV PATH="/app/.local/bin:${PATH}"
### App specific crap
# Deps vary least so do them first
RUN pip3 install --user install aiohttp aiohttp_basicauth async_lru cachetools click pycryptodome pyyaml retry
COPY --chown=app:app docker_relay.sh /app/relay.sh
COPY --chown=app:app src/python /app/
EXPOSE 8080
CMD ["/bin/sh", "/app/relay.sh"]

View file

@ -0,0 +1,80 @@
# ActivityPub Relay
A generic ActivityPub/LitePub compatible with Pleroma and Mastodon.
### What is a relay?
A relay is a webserver implementing ActivityPub/LitePub.
Normally when posting content on an ActivityPub server, that content is only listed publicly on the feed of the hosting server, and servers to which your server announces that content.
Relays provide a way for ActivityPub servers to announce posts to and receive posts from a wider audience.
For instance there are [public lists](https://github.com/brodi1/activitypub-relays) of nodes offering relaying.
### Nuts and bolts of ActivityPub
[ActivityPub](https://www.w3.org/TR/activitypub/) is a protocol by which [Actors](https://www.w3.org/TR/activitypub/#actors) exchange messages.
An actor consists of two key things - an inbox and an outbox.
The inbox is a URL to which other actors can `POST` messages.
The outbox is a URL naming a paginated collection which other actors can `GET` to read messages from.
Any user in an ActivityPub system, for instance `@arrdem@macaw.social` is an actor with such an inbox outbox pair.
ActivityPub messages for follows of users or messages mentioning users are implemented with messages directly to the user's outbox.
In addition, Mastodon ActivityPub servers themselves have ["instance actors"](https://github.com/mastodon/mastodon/issues/10453).
These actors communicate using [server to server interactions](https://www.w3.org/TR/activitypub/#server-to-server-interactions).
Instance actors (f. ex. `https://macaw.social/actor`) emit and receive messages regarding system level activities.
New posts (`Create`) are the major thing, but plenty of other messages flow through too.
The relay "protocol" is a Follow of the relay's actor by an ActivityPub server that wishes to participate in the relay.
The relay will acknowledge the follow request, and then itself follow the participating server back.
These two follow relationships in place, the participating server will announce any new posts to the relay since the relay follows it.
This is just the normal ActivityPub behavior.
Likewise, the relay will announce any new posts it gets from elsewhere to the participating server.
Same deal.
That's it.
### Why relays at all?
In theory, two ActivityPub servers can talk to each other directly.
Why relays?
Relays are 1 to N bridges.
Follow one relay, get activity from every other host on that bridge.
This makes it easier for new, small nodes to participate in a wider network since they only need to follow one host not follow every single other host.
Traditional relay/clearing house model.
### What is this program?
The relay itself is a webserver providing two major components.
- A hardcoded webfinger implementation for resolving the user `acct:relay@your.relay.hostname`
- A message relay, which will perform a _stateless_ relay of new activity to connected nodes
The relay offers three moderation capabilities:
- An explicit allowlist mode restricting connected nodes
- A denylist mode restricting connected nodes by name
- A denylist mode restricting connected nodes by self-identified server software
## Getting Started
Normally, you would direct your LitePub instance software to follow the LitePub actor found on the relay.
In Pleroma this would be something like:
$ MIX_ENV=prod mix relay_follow https://your.relay.hostname/actor
On Mastodon the process is similar, in `Administration > Relays > Add New Relay` one would list the relay's URL
https://your.relay.hostname/actor
## Status
- Works
- Poorly packaged
- Not yet threaded (nor is upstream)
- Not yet tested (nor is upstream)
- Missing web-oriented administrative functionality
- Missing web-oriented configuration/state management
## Copyright
This work is derived from https://git.pleroma.social/pleroma/relay, which is published under the terms of the AGPLv3.

View file

@ -0,0 +1,16 @@
#!/usr/bin/env sh
cd "$(realpath $(dirname $0))"
IMAGE=registry.tirefireind.us/arrdem/activitypub_relay
docker build -t $IMAGE:latest -t relay .
docker push $IMAGE:latest
ssh root@relay.tirefireind.us systemctl restart activitypub-relay
if [ -n "$TAG" ]; then
for stem in registry.tirefireind.us/arrdem/activitypub_relay registry.digitalocean.com/macawsocial/relay; do
docker tag relay "$stem:$TAG"
docker push $stem:"$TAG"
done
fi

View file

@ -0,0 +1,13 @@
#!/usr/bin/env sh
# A launcher script for the dockerized relay
set -e
# First do config init if needed
if [ ! -f "/data/config.yml" ]; then
python3 -m "relay" -c "/data/config.yml" setup
fi
# Then run the blame thing
exec python3 -m "relay" -c "/data/config.yml" "${@:-run}"

View file

@ -0,0 +1,7 @@
#!/usr/bin/env sh
cd "$(realpath $(dirname $0))"
bazel build ...
exec ../../bazel-bin/projects/activitypub_relay/activitypub_relay -c $(realpath ./relay_prod.yaml) run

View file

@ -0,0 +1,43 @@
# this is the path that the object graph will get dumped to (in JSON-LD format),
# you probably shouldn't change it, but you can if you want.
db: relay.jsonld
# Listener
listen: 0.0.0.0
port: 8080
# Note
note: "Make a note about your instance here."
# maximum number of inbox posts to do at once
post_limit: 512
# this section is for ActivityPub
ap:
# this is used for generating activitypub messages, as well as instructions for
# linking AP identities. it should be an SSL-enabled domain reachable by https.
host: 'relay.example.com'
blocked_instances:
- 'bad-instance.example.com'
- 'another-bad-instance.example.com'
whitelist_enabled: false
whitelist:
- 'good-instance.example.com'
- 'another.good-instance.example.com'
# uncomment the lines below to prevent certain activitypub software from posting
# to the relay (all known relays by default). this uses the software name in nodeinfo
#blocked_software:
#- 'activityrelay'
#- 'aoderelay'
#- 'social.seattle.wa.us-relay'
#- 'unciarelay'
# cache limits as number of items. only change this if you know what you're doing
cache:
objects: 1024
actors: 1024
digests: 1024

View file

@ -0,0 +1 @@
__version__ = "0.2.2"

View file

@ -0,0 +1,392 @@
import asyncio
import logging
import os
from urllib.parse import urlparse
import click
from relay import __version__, misc
from relay.application import (
Application,
request_id_middleware,
)
@click.group(
"cli", context_settings={"show_default": True}, invoke_without_command=True
)
@click.option("--config", "-c", default="relay.yaml", help="path to the relay's config")
@click.version_option(version=__version__, prog_name="ActivityRelay")
@click.pass_context
def cli(ctx, config):
ctx.obj = Application(config, middlewares=[request_id_middleware])
level = {
"debug": logging.DEBUG,
"info": logging.INFO,
"error": logging.ERROR,
"critical": logging.CRITICAL,
"fatal": logging.FATAL,
}.get(os.getenv("LOG_LEVEL", "INFO").lower(), logging.INFO)
logging.basicConfig(
level=level,
format="[%(asctime)s] %(levelname)s: %(message)s",
)
@cli.group("inbox")
@click.pass_obj
def cli_inbox(ctx: Application):
"Manage the inboxes in the database"
@cli_inbox.command("list")
@click.pass_obj
def cli_inbox_list(obj: Application):
"List the connected instances or relays"
click.echo("Connected to the following instances or relays:")
for inbox in obj.database.inboxes:
click.echo(f"- {inbox}")
@cli_inbox.command("follow")
@click.argument("actor")
@click.pass_obj
def cli_inbox_follow(obj: Application, actor):
"Follow an actor (Relay must be running)"
if obj.config.is_banned(actor):
return click.echo(f"Error: Refusing to follow banned actor: {actor}")
if not actor.startswith("http"):
domain = actor
actor = f"https://{actor}/actor"
else:
domain = urlparse(actor).hostname
try:
inbox_data = obj.database["relay-list"][domain]
inbox = inbox_data["inbox"]
except KeyError:
actor_data = asyncio.run(misc.request(actor))
inbox = actor_data.shared_inbox
message = misc.Message.new_follow(host=obj.config.host, actor=actor.id)
asyncio.run(misc.request(inbox, message))
click.echo(f"Sent follow message to actor: {actor}")
@cli_inbox.command("unfollow")
@click.argument("actor")
@click.pass_obj
def cli_inbox_unfollow(obj: Application, actor):
"Unfollow an actor (Relay must be running)"
if not actor.startswith("http"):
domain = actor
actor = f"https://{actor}/actor"
else:
domain = urlparse(actor).hostname
try:
inbox_data = obj.database["relay-list"][domain]
inbox = inbox_data["inbox"]
message = misc.Message.new_unfollow(
host=obj.config.host, actor=actor, follow=inbox_data["followid"]
)
except KeyError:
actor_data = asyncio.run(misc.request(actor))
inbox = actor_data.shared_inbox
message = misc.Message.new_unfollow(
host=obj.config.host,
actor=actor,
follow={
"type": "Follow",
"object": actor,
"actor": f"https://{obj.config.host}/actor",
},
)
asyncio.run(misc.request(inbox, message))
click.echo(f"Sent unfollow message to: {actor}")
@cli_inbox.command("add")
@click.argument("inbox")
@click.pass_obj
def cli_inbox_add(obj: Application, inbox):
"Add an inbox to the database"
if not inbox.startswith("http"):
inbox = f"https://{inbox}/inbox"
if obj.config.is_banned(inbox):
return click.echo(f"Error: Refusing to add banned inbox: {inbox}")
if obj.database.add_inbox(inbox):
obj.database.save()
return click.echo(f"Added inbox to the database: {inbox}")
click.echo(f"Error: Inbox already in database: {inbox}")
@cli_inbox.command("remove")
@click.argument("inbox")
@click.pass_obj
def cli_inbox_remove(obj: Application, inbox):
"Remove an inbox from the database"
try:
dbinbox = obj.database.get_inbox(inbox, fail=True)
except KeyError:
click.echo(f"Error: Inbox does not exist: {inbox}")
return
obj.database.del_inbox(dbinbox["domain"])
obj.database.save()
click.echo(f"Removed inbox from the database: {inbox}")
@cli.group("instance")
def cli_instance():
"Manage instance bans"
@cli_instance.command("list")
@click.pass_obj
def cli_instance_list(obj: Application):
"List all banned instances"
click.echo("Banned instances or relays:")
for domain in obj.config.blocked_instances:
click.echo(f"- {domain}")
@cli_instance.command("ban")
@click.argument("target")
@click.pass_obj
def cli_instance_ban(obj: Application, target):
"Ban an instance and remove the associated inbox if it exists"
if target.startswith("http"):
target = urlparse(target).hostname
if obj.config.ban_instance(target):
obj.config.save()
if obj.database.del_inbox(target):
obj.database.save()
click.echo(f"Banned instance: {target}")
return
click.echo(f"Instance already banned: {target}")
@cli_instance.command("unban")
@click.argument("target")
@click.pass_obj
def cli_instance_unban(obj: Application, target):
"Unban an instance"
if obj.config.unban_instance(target):
obj.config.save()
click.echo(f"Unbanned instance: {target}")
return
click.echo(f"Instance wasn't banned: {target}")
@cli.group("software")
def cli_software():
"Manage banned software"
@cli_software.command("list")
@click.pass_obj
def cli_software_list(obj: Application):
"List all banned software"
click.echo("Banned software:")
for software in obj.config.blocked_software:
click.echo(f"- {software}")
@cli_software.command("ban")
@click.option(
"--fetch-nodeinfo/--ignore-nodeinfo",
"-f",
"fetch_nodeinfo",
default=False,
help="Treat NAME like a domain and try to fet the software name from nodeinfo",
)
@click.argument("name")
@click.pass_obj
def cli_software_ban(obj: Application, name, fetch_nodeinfo):
"Ban software. Use RELAYS for NAME to ban relays"
if fetch_nodeinfo:
software = asyncio.run(misc.fetch_nodeinfo(name))
if not software:
click.echo(f"Failed to fetch software name from domain: {name}")
name = software
if config.ban_software(name):
obj.config.save()
return click.echo(f"Banned software: {name}")
click.echo(f"Software already banned: {name}")
@cli_software.command("unban")
@click.option(
"--fetch-nodeinfo/--ignore-nodeinfo",
"-f",
"fetch_nodeinfo",
default=False,
help="Treat NAME like a domain and try to fet the software name from nodeinfo",
)
@click.argument("name")
@click.pass_obj
def cli_software_unban(obj: Application, name, fetch_nodeinfo):
"Ban software. Use RELAYS for NAME to unban relays"
if fetch_nodeinfo:
software = asyncio.run(misc.fetch_nodeinfo(name))
if not software:
click.echo(f"Failed to fetch software name from domain: {name}")
name = software
if obj.config.unban_software(name):
obj.config.save()
return click.echo(f"Unbanned software: {name}")
click.echo(f"Software wasn't banned: {name}")
@cli.group("whitelist")
def cli_whitelist():
"Manage the instance whitelist"
@cli_whitelist.command("list")
@click.pass_obj
def cli_whitelist_list(obj: Application):
click.echo("Current whitelisted domains")
for domain in obj.config.whitelist:
click.echo(f"- {domain}")
@cli_whitelist.command("add")
@click.argument("instance")
@click.pass_obj
def cli_whitelist_add(obj: Application, instance):
"Add an instance to the whitelist"
if not obj.config.add_whitelist(instance):
return click.echo(f"Instance already in the whitelist: {instance}")
obj.config.save()
click.echo(f"Instance added to the whitelist: {instance}")
@cli_whitelist.command("remove")
@click.argument("instance")
@click.pass_obj
def cli_whitelist_remove(obj: Application, instance):
"Remove an instance from the whitelist"
if not obj.config.del_whitelist(instance):
return click.echo(f"Instance not in the whitelist: {instance}")
obj.config.save()
if obj.config.whitelist_enabled:
if obj.database.del_inbox(inbox):
obj.database.save()
click.echo(f"Removed instance from the whitelist: {instance}")
@cli.command("setup")
@click.pass_obj
def relay_setup(obj: Application):
"Generate a new config"
if not obj.config.is_docker:
while True:
obj.config.host = os.getenv("RELAY_HOSTNAME") or click.prompt(
"What domain will the relay be hosted on?", default=obj.config.host
)
if not obj.config.host.endswith("example.com"):
break
click.echo("The domain must not be example.com")
obj.config.listen = os.getenv("LISTEN_ADDRESS") or click.prompt(
"Which address should the relay listen on?", default=obj.config.listen
)
while True:
obj.config.port = click.prompt(
"What TCP port should the relay listen on?",
default=obj.config.port,
type=int,
)
break
else:
obj.config.listen = os.getenv("LISTEN_ADDRESS", obj.config.listen)
obj.config.port = int(os.getenv("LISTEN_PORT", obj.config.port))
obj.config.host = os.getenv("RELAY_HOSTNAME")
obj.config.admin_token = os.getenv("RELAY_ADMIN_TOKEN")
if not obj.config.host:
click.echo("Error: No relay host configured! Set $RELAY_HOSTNAME")
exit(1)
obj.config.save()
if not obj.config.is_docker and click.confirm(
"Relay all setup! Would you like to run it now?"
):
relay_run.callback()
@cli.command("run")
@click.pass_obj
def relay_run(obj: Application):
"Run the relay"
if obj.config.host.endswith("example.com"):
return click.echo(
'Relay is not set up. Please edit your relay config or run "activityrelay setup".'
)
if not misc.check_open_port(obj.config.listen, obj.config.port):
return click.echo(
f"Error: A server is already running on port {obj.config.port}"
)
obj.run()
if __name__ == "__main__":
cli(prog_name="relay")

View file

@ -0,0 +1,134 @@
import asyncio
from datetime import datetime, timedelta
import logging
import os
import signal
from uuid import uuid4
from aiohttp import web
from cachetools import LRUCache
from relay.config import RelayConfig
from relay.database import RelayDatabase
from relay.misc import (
check_open_port,
DotDict,
set_app,
)
from relay.views import routes
class Application(web.Application):
def __init__(self, cfgpath, middlewares=None):
web.Application.__init__(self, middlewares=middlewares)
self["starttime"] = None
self["running"] = False
self["is_docker"] = bool(os.environ.get("DOCKER_RUNNING"))
self["config"] = RelayConfig(cfgpath)
if not self["config"].load():
self["config"].save()
self["database"] = RelayDatabase(self["config"])
self["database"].load()
self["cache"] = DotDict(
{
key: Cache(maxsize=self["config"][key])
for key in self["config"].cachekeys
}
)
self["semaphore"] = asyncio.Semaphore(self["config"].push_limit)
self.set_signal_handler()
set_app(self)
@property
def cache(self):
return self["cache"]
@property
def config(self):
return self["config"]
@property
def database(self):
return self["database"]
@property
def is_docker(self):
return self["is_docker"]
@property
def semaphore(self):
return self["semaphore"]
@property
def uptime(self):
if not self["starttime"]:
return timedelta(seconds=0)
uptime = datetime.now() - self["starttime"]
return timedelta(seconds=uptime.seconds)
def set_signal_handler(self):
signal.signal(signal.SIGHUP, self.stop)
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGQUIT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
def run(self):
if not check_open_port(self.config.listen, self.config.port):
return logging.error(
f"A server is already running on port {self.config.port}"
)
for route in routes:
self.router.add_route(*route)
logging.info(
f"Starting webserver at {self.config.host} ({self.config.listen}:{self.config.port})"
)
asyncio.run(self.handle_run())
# Save off config before exit
self.config.save()
self.database.save()
def stop(self, *_):
self["running"] = False
async def handle_run(self):
self["running"] = True
runner = web.AppRunner(
self, access_log_format='%{X-Forwarded-For}i "%r" %s %b "%{User-Agent}i"'
)
await runner.setup()
site = web.TCPSite(
runner, host=self.config.listen, port=self.config.port, reuse_address=True
)
await site.start()
self["starttime"] = datetime.now()
while self["running"]:
await asyncio.sleep(0.25)
await site.stop()
self["starttime"] = None
self["running"] = False
class Cache(LRUCache):
def set_maxsize(self, value):
self.__maxsize = int(value)
@web.middleware
async def request_id_middleware(request, handler):
request.id = uuid4()
return await handler(request)

View file

@ -0,0 +1,202 @@
import os
from pathlib import Path
from urllib.parse import urlparse
from relay.misc import DotDict
import yaml
class RelayConfig(DotDict):
apkeys = {
"host",
"whitelist_enabled",
"blocked_software",
"blocked_instances",
"whitelist",
}
cachekeys = {"json", "objects", "digests"}
def __init__(self, path):
DotDict.__init__(self, {})
self._path = Path(path).expanduser()
self.reset()
def __setitem__(self, key, value):
if key in ["blocked_instances", "blocked_software", "whitelist"]:
assert isinstance(value, (list, set, tuple))
elif key in ["port", "json", "objects", "digests"]:
assert isinstance(value, (int))
elif key == "whitelist_enabled":
assert isinstance(value, bool)
super().__setitem__(key, value)
@property
def is_docker(self):
return bool(os.getenv("DOCKER_RUNNING"))
@property
def db(self):
return Path(self["db"]).expanduser().resolve()
@property
def path(self):
return self._path
@property
def actor(self):
return f"https://{self.host}/actor"
@property
def inbox(self):
return f"https://{self.host}/inbox"
@property
def keyid(self):
return f"{self.actor}#main-key"
def reset(self):
self.clear()
self.update(
{
"db": str(self._path.parent.joinpath(f"{self._path.stem}.jsonld")),
"listen": "0.0.0.0",
"port": 8080,
"note": "Make a note about your instance here.",
"push_limit": 512,
"host": "relay.example.com",
"blocked_software": [],
"blocked_instances": [],
"whitelist": [],
"whitelist_enabled": False,
"json": 1024,
"objects": 1024,
"digests": 1024,
"admin_token": None,
}
)
def ban_instance(self, instance):
if instance.startswith("http"):
instance = urlparse(instance).hostname
if self.is_banned(instance):
return False
self.blocked_instances.append(instance)
return True
def unban_instance(self, instance):
if instance.startswith("http"):
instance = urlparse(instance).hostname
try:
self.blocked_instances.remove(instance)
return True
except:
return False
def ban_software(self, software):
if self.is_banned_software(software):
return False
self.blocked_software.append(software)
return True
def unban_software(self, software):
try:
self.blocked_software.remove(software)
return True
except:
return False
def add_whitelist(self, instance):
if instance.startswith("http"):
instance = urlparse(instance).hostname
if self.is_whitelisted(instance):
return False
self.whitelist.append(instance)
return True
def del_whitelist(self, instance):
if instance.startswith("http"):
instance = urlparse(instance).hostname
try:
self.whitelist.remove(instance)
return True
except:
return False
def is_banned(self, instance):
if instance.startswith("http"):
instance = urlparse(instance).hostname
return instance in self.blocked_instances
def is_banned_software(self, software):
if not software:
return False
return software.lower() in self.blocked_software
def is_whitelisted(self, instance):
if instance.startswith("http"):
instance = urlparse(instance).hostname
return instance in self.whitelist
def load(self):
self.reset()
options = {}
try:
options["Loader"] = yaml.FullLoader
except AttributeError:
pass
try:
with open(self.path) as fd:
config = yaml.load(fd, **options)
except FileNotFoundError:
return False
if not config:
return False
for key, value in config.items():
if key in ["ap", "cache"]:
for k, v in value.items():
if k not in self:
continue
self[k] = v
elif key not in self:
continue
self[key] = value
if self.host.endswith("example.com"):
return False
return True
def save(self):
with open(self._path, "w") as fd:
fd.write("---\n")
yaml.dump(self.jsonify(), fd, sort_keys=True)

View file

@ -0,0 +1,159 @@
import json
import logging
from urllib.parse import urlparse
from Crypto.PublicKey import RSA
class RelayDatabase(dict):
def __init__(self, config):
dict.__init__(
self,
{
"relay-list": {},
"private-key": None,
"follow-requests": {},
"version": 1,
},
)
self.config = config
self.PRIVKEY = None
@property
def PUBKEY(self):
return self.PRIVKEY.publickey()
@property
def pubkey(self):
return self.PUBKEY.exportKey("PEM").decode("utf-8")
@property
def privkey(self):
return self["private-key"]
@property
def hostnames(self):
return tuple(self["relay-list"].keys())
@property
def inboxes(self):
return tuple(data["inbox"] for data in self["relay-list"].values())
def generate_key(self):
self.PRIVKEY = RSA.generate(4096)
self["private-key"] = self.PRIVKEY.exportKey("PEM").decode("utf-8")
def load(self):
with self.config.db.open() as fd:
data = json.load(fd)
self["version"] = data.get("version", 1)
self["private-key"] = data.get("private-key")
self["relay-list"] = data.get("relay-list", {})
self["follow-requests"] = data.get("follow-requests")
if not self.privkey:
logging.info("No actor keys present, generating 4096-bit RSA keypair.")
self.generate_key()
self.save()
else:
self.PRIVKEY = RSA.importKey(self.privkey)
def save(self):
with self.config.db.open("w") as fd:
json.dump(self, fd, indent=4)
def get_inbox(self, domain, fail=False):
if domain.startswith("http"):
domain = urlparse(domain).hostname
if domain not in self["relay-list"]:
if fail:
raise KeyError(domain)
return
return self["relay-list"][domain]
def add_inbox(self, inbox, followid=None, fail=False):
assert inbox.startswith("https"), "Inbox must be a url"
domain = urlparse(inbox).hostname
if self.get_inbox(domain):
if fail:
raise KeyError(domain)
return False
self["relay-list"][domain] = {
"domain": domain,
"inbox": inbox,
"followid": followid,
}
logging.debug(f"Added inbox to database: {inbox}")
return self["relay-list"][domain]
def del_inbox(self, domain, followid=None, fail=False):
data = self.get_inbox(domain, fail=False)
if not data:
if fail:
raise KeyError(domain)
return False
if not data["followid"] or not followid or data["followid"] == followid:
del self["relay-list"][data["domain"]]
logging.debug(f'Removed inbox from database: {data["inbox"]}')
return True
if fail:
raise ValueError("Follow IDs do not match")
logging.debug(
f'Follow ID does not match: db = {data["followid"]}, object = {followid}'
)
return False
def set_followid(self, domain, followid):
if (data := self.get_inbox(domain, fail=True)):
data["followid"] = followid
def get_request(self, domain, fail=True):
if domain.startswith("http"):
domain = urlparse(domain).hostname
try:
return self["follow-requests"][domain]
except KeyError as e:
if fail:
raise e
def add_request(self, actor, inbox, followid):
domain = urlparse(inbox).hostname
try:
if (request := self.get_request(domain)):
request["followid"] = followid
except KeyError:
pass
self["follow-requests"][domain] = {
"actor": actor,
"inbox": inbox,
"followid": followid,
}
def del_request(self, domain):
if domain.startswith("http"):
domain = urlparse(domain).hostname
del self["follow-requests"][domain]
def get_requests(self):
return list(self["follow-requests"].items())

View file

@ -0,0 +1,62 @@
from collections import defaultdict
import logging
import aiohttp
STATS = {
"requests": defaultdict(int),
"response_codes": defaultdict(int),
"response_codes_per_domain": defaultdict(lambda: defaultdict(int)),
"delivery_codes": defaultdict(int),
"delivery_codes_per_domain": defaultdict(lambda: defaultdict(int)),
"exceptions": defaultdict(int),
"exceptions_per_domain": defaultdict(lambda: defaultdict(int)),
"delivery_exceptions": defaultdict(int),
"delivery_exceptions_per_domain": defaultdict(lambda: defaultdict(int)),
}
async def on_request_start(session, trace_config_ctx, params):
logging.debug("HTTP START [%r], [%r]", session, params)
STATS["requests"][params.url.host] += 1
async def on_request_end(session, trace_config_ctx, params):
logging.debug("HTTP END [%r], [%r]", session, params)
host = params.url.host
status = params.response.status
STATS["response_codes"][status] += 1
STATS["response_codes_per_domain"][host][status] += 1
if params.method == "POST":
STATS["delivery_codes"][status] += 1
STATS["delivery_codes_per_domain"][host][status] += 1
async def on_request_exception(session, trace_config_ctx, params):
logging.debug("HTTP EXCEPTION [%r], [%r]", session, params)
host = params.url.host
exception = repr(params.exception)
STATS["exceptions"][exception] += 1
STATS["exceptions_per_domain"][host][exception] += 1
if params.method == "POST":
STATS["delivery_exceptions"][exception] += 1
STATS["delivery_exceptions_per_domain"][host][exception] += 1
def http_debug():
if logging.DEBUG >= logging.root.level:
return
trace_config = aiohttp.TraceConfig()
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
trace_config.on_request_exception.append(on_request_exception)
return [trace_config]

View file

@ -0,0 +1,515 @@
import base64
from datetime import datetime
import json
from json.decoder import JSONDecodeError
import logging
import socket
from typing import Union
from urllib.parse import urlparse
import uuid
from Crypto.Hash import SHA, SHA256, SHA512
from Crypto.PublicKey import RSA
from Crypto.Signature import PKCS1_v1_5
from aiohttp import ClientSession
from aiohttp.client_exceptions import (
ClientConnectorError,
ClientResponseError,
)
from aiohttp.web import (
Response as AiohttpResponse,
)
from async_lru import alru_cache
from relay.http_debug import http_debug
from retry import retry
app = None
HASHES = {"sha1": SHA, "sha256": SHA256, "sha512": SHA512}
MIMETYPES = {
"activity": "application/activity+json",
"html": "text/html",
"json": "application/json",
"text": "text/plain",
}
NODEINFO_NS = {
"20": "http://nodeinfo.diaspora.software/ns/schema/2.0",
"21": "http://nodeinfo.diaspora.software/ns/schema/2.1",
}
def set_app(new_app):
global app
app = new_app
def build_signing_string(headers, used_headers):
return "\n".join(map(lambda x: ": ".join([x.lower(), headers[x]]), used_headers))
def check_open_port(host, port):
if host == "0.0.0.0":
host = "127.0.0.1"
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
try:
return s.connect_ex((host, port)) != 0
except socket.error as e:
return False
def create_signature_header(headers):
headers = {k.lower(): v for k, v in headers.items()}
used_headers = headers.keys()
sigstring = build_signing_string(headers, used_headers)
sig = {
"keyId": app.config.keyid,
"algorithm": "rsa-sha256",
"headers": " ".join(used_headers),
"signature": sign_signing_string(sigstring, app.database.PRIVKEY),
}
chunks = ['{}="{}"'.format(k, v) for k, v in sig.items()]
return ",".join(chunks)
def distill_inboxes(object_id):
for inbox in app.database.inboxes:
if (urlparse(inbox).hostname != urlparse(object_id).hostname):
yield inbox
def generate_body_digest(body):
h = SHA256.new(body.encode("utf-8"))
bodyhash = base64.b64encode(h.digest()).decode("utf-8")
return bodyhash
def sign_signing_string(sigstring, key):
pkcs = PKCS1_v1_5.new(key)
h = SHA256.new()
h.update(sigstring.encode("ascii"))
sigdata = pkcs.sign(h)
return base64.b64encode(sigdata).decode("utf-8")
def split_signature(sig):
default = {"headers": "date"}
sig = sig.strip().split(",")
for chunk in sig:
k, _, v = chunk.partition("=")
v = v.strip('"')
default[k] = v
default["headers"] = default["headers"].split()
return default
async def fetch_actor_key(actor):
actor_data = await request(actor)
if not actor_data:
return None
try:
return RSA.importKey(actor_data["publicKey"]["publicKeyPem"])
except Exception as e:
logging.debug(f"Exception occured while fetching actor key: {e}")
@alru_cache
async def fetch_nodeinfo(domain):
nodeinfo_url = None
wk_nodeinfo = await request(
f"https://{domain}/.well-known/nodeinfo", sign_headers=False, activity=False
)
if not wk_nodeinfo:
return
wk_nodeinfo = WKNodeinfo(wk_nodeinfo)
for version in ["20", "21"]:
try:
nodeinfo_url = wk_nodeinfo.get_url(version)
except KeyError:
pass
if not nodeinfo_url:
logging.debug(f"Failed to fetch nodeinfo url for domain: {domain}")
return False
nodeinfo = await request(nodeinfo_url, sign_headers=False, activity=False)
try:
return nodeinfo["software"]["name"]
except KeyError:
return False
@retry(exceptions=(ClientConnectorError, ClientResponseError), backoff=1, tries=3)
async def request(uri, data=None, force=False, sign_headers=True, activity=True):
## If a get request and not force, try to use the cache first
url = urlparse(uri)
method = "POST" if data else "GET"
action = data.get("type") if data else None
headers = {
"Accept": f'{MIMETYPES["activity"]}, {MIMETYPES["json"]};q=0.9',
"User-Agent": "ActivityRelay",
}
if data:
headers["Content-Type"] = MIMETYPES["activity" if activity else "json"]
if sign_headers:
signing_headers = {
"(request-target)": f"{method.lower()} {url.path}",
"Date": datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT"),
"Host": url.netloc,
}
if data:
assert isinstance(data, dict)
data = json.dumps(data)
signing_headers.update(
{
"Digest": f"SHA-256={generate_body_digest(data)}",
"Content-Length": str(len(data.encode("utf-8"))),
}
)
signing_headers["Signature"] = create_signature_header(signing_headers)
del signing_headers["(request-target)"]
del signing_headers["Host"]
headers.update(signing_headers)
logging.info("%s %s", method, uri)
# logging.debug("%r %r %r %r", method, uri, headers, data)
async with ClientSession(trace_configs=http_debug()) as session:
async with session.request(method, uri, headers=headers, data=data) as resp:
resp.raise_for_status()
## aiohttp has been known to leak if the response hasn't been read,
## so we're just gonna read the request no matter what
resp_data = await resp.read()
## Not expecting a response, so just return
if resp.status == 202:
return
elif resp.status != 200:
if not resp_data:
return logging.debug(f"Received error when requesting {uri}: {resp.status} {resp_data}")
return logging.debug(f"Received error when sending {action} to {uri}: {resp.status} {resp_data}")
if resp.content_type == MIMETYPES["activity"]:
resp_data = await resp.json(loads=Message.new_from_json)
elif resp.content_type == MIMETYPES["json"]:
resp_data = await resp.json(loads=DotDict.new_from_json)
else:
logging.debug(f'Invalid Content-Type for "{url}": {resp.content_type}')
return logging.debug(f"Response: {resp_data}")
logging.debug(f"{uri} >> resp {resp_data}")
return resp_data
async def validate_signature(actor, http_request):
pubkey = await fetch_actor_key(actor)
if not pubkey:
return False
logging.debug(f"actor key: {pubkey}")
headers = {key.lower(): value for key, value in http_request.headers.items()}
headers["(request-target)"] = " ".join(
[
http_request.method.lower(),
http_request.path
]
)
sig = split_signature(headers["signature"])
logging.debug(f"sigdata: {sig}")
sigstring = build_signing_string(headers, sig["headers"])
logging.debug(f"sigstring: {sigstring}")
sign_alg, _, hash_alg = sig["algorithm"].partition("-")
logging.debug(f"sign alg: {sign_alg}, hash alg: {hash_alg}")
sigdata = base64.b64decode(sig["signature"])
pkcs = PKCS1_v1_5.new(pubkey)
h = HASHES[hash_alg].new()
h.update(sigstring.encode("ascii"))
result = pkcs.verify(h, sigdata)
http_request["validated"] = result
logging.debug(f"validates? {result}")
return result
class DotDict(dict):
def __init__(self, _data, **kwargs):
dict.__init__(self)
self.update(_data, **kwargs)
def __hasattr__(self, k):
return k in self
def __getattr__(self, k):
try:
return self[k]
except KeyError:
raise AttributeError(
f"{self.__class__.__name__} object has no attribute {k}"
) from None
def __setattr__(self, k, v):
if k.startswith("_"):
super().__setattr__(k, v)
else:
self[k] = v
def __setitem__(self, k, v):
if type(v) == dict:
v = DotDict(v)
super().__setitem__(k, v)
def __delattr__(self, k):
try:
dict.__delitem__(self, k)
except KeyError:
raise AttributeError(
f"{self.__class__.__name__} object has no attribute {k}"
) from None
@classmethod
def new_from_json(cls, data):
if not data:
raise JSONDecodeError("Empty body", data, 1)
try:
return cls(json.loads(data))
except ValueError:
raise JSONDecodeError("Invalid body", data, 1)
def to_json(self, indent=None):
return json.dumps(self, indent=indent)
def jsonify(self):
def _xform(v):
if hasattr(v, "jsonify"):
return v.jsonify()
else:
return v
return {k: _xform(v) for k, v in self.items()}
def update(self, _data, **kwargs):
if isinstance(_data, dict):
for key, value in _data.items():
self[key] = value
elif isinstance(_data, (list, tuple, set)):
for key, value in _data:
self[key] = value
for key, value in kwargs.items():
self[key] = value
# misc properties
@property
def domain(self):
return urlparse(getattr(self, "id", None) or getattr(self, "actor")).hostname
# actor properties
@property
def pubkey(self):
return self.publicKey.publicKeyPem
@property
def shared_inbox(self):
return self.get("endpoints", {}).get("sharedInbox", self.inbox)
# activity properties
@property
def actorid(self):
if isinstance(self.actor, dict):
return self.actor.id
return self.actor
@property
def objectid(self):
if isinstance(self.object, dict):
return self.object.id
return self.object
class Message(DotDict):
@classmethod
def new_actor(cls, host, pubkey, description=None):
return cls(
{
"@context": "https://www.w3.org/ns/activitystreams",
"id": f"https://{host}/actor",
"type": "Application",
"preferredUsername": "relay",
"name": "ActivityRelay",
"summary": description or "ActivityRelay bot",
"followers": f"https://{host}/followers",
"following": f"https://{host}/following",
"inbox": f"https://{host}/inbox",
"url": f"https://{host}/inbox",
"endpoints": {"sharedInbox": f"https://{host}/inbox"},
"publicKey": {
"id": f"https://{host}/actor#main-key",
"owner": f"https://{host}/actor",
"publicKeyPem": pubkey,
},
}
)
@classmethod
def new_announce(cls, host, object):
return cls(
{
"@context": "https://www.w3.org/ns/activitystreams",
"id": f"https://{host}/activities/{uuid.uuid4()}",
"type": "Announce",
"to": [f"https://{host}/followers"],
"actor": f"https://{host}/actor",
"object": object,
}
)
@classmethod
def new_follow(cls, host, actor):
return cls(
{
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Follow",
"to": [actor],
"object": actor,
"id": f"https://{host}/activities/{uuid.uuid4()}",
"actor": f"https://{host}/actor",
}
)
@classmethod
def new_unfollow(cls, host, actor, follow):
return cls(
{
"@context": "https://www.w3.org/ns/activitystreams",
"id": f"https://{host}/activities/{uuid.uuid4()}",
"type": "Undo",
"to": [actor],
"actor": f"https://{host}/actor",
"object": follow,
}
)
@classmethod
def new_response(cls, host, actor, followid, accept):
return cls(
{
"@context": "https://www.w3.org/ns/activitystreams",
"id": f"https://{host}/activities/{uuid.uuid4()}",
"type": "Accept" if accept else "Reject",
"to": [actor],
"actor": f"https://{host}/actor",
"object": {
"id": followid,
"type": "Follow",
"object": f"https://{host}/actor",
"actor": actor,
},
}
)
class Response(AiohttpResponse):
@classmethod
def new(cls, body: Union[dict, str, bytes] = "", status=200, headers=None, ctype="text"):
kwargs = {
"status": status,
"headers": headers,
"content_type": MIMETYPES[ctype],
}
if isinstance(body, bytes):
kwargs["body"] = body
elif isinstance(body, dict) and ctype in {"json", "activity"}:
kwargs["text"] = json.dumps(body)
else:
kwargs["text"] = body
return cls(**kwargs)
@classmethod
def new_error(cls, status, body, ctype="text"):
if ctype == "json":
body = json.dumps({"status": status, "error": body})
return cls.new(body=body, status=status, ctype=ctype)
@property
def location(self):
return self.headers.get("Location")
@location.setter
def location(self, value):
self.headers["Location"] = value
class WKNodeinfo(DotDict):
@classmethod
def new(cls, v20, v21):
return cls(
{
"links": [
{"rel": NODEINFO_NS["20"], "href": v20},
{"rel": NODEINFO_NS["21"], "href": v21},
]
}
)
def get_url(self, version="20"):
for item in self.links:
if item["rel"] == NODEINFO_NS[version]:
return item["href"]
raise KeyError(version)

View file

@ -0,0 +1,125 @@
import asyncio
import logging
from relay import misc
from relay.config import RelayConfig
from relay.database import RelayDatabase
async def handle_relay(request, actor, data, software):
if data.objectid in request.app.cache.objects:
logging.info(f"already relayed {data.objectid}")
return
logging.info(f"Relaying post from {data.actorid}")
message = misc.Message.new_announce(
host=request.app.config.host, object=data.objectid
)
logging.debug(f">> relay: {message}")
inboxes = misc.distill_inboxes(data.objectid)
futures = [misc.request(inbox, data=message) for inbox in inboxes]
asyncio.ensure_future(asyncio.gather(*futures))
request.app.cache.objects[data.objectid] = message.id
async def handle_forward(request, actor, data, software):
if data.id in request.app.cache.objects:
logging.info(f"already forwarded {data.id}")
return
message = misc.Message.new_announce(host=request.app.config.host, object=data)
logging.info(f"Forwarding post from {actor.id}")
logging.debug(f">> Relay {data}")
inboxes = misc.distill_inboxes(data.id)
futures = [misc.request(inbox, data=message) for inbox in inboxes]
asyncio.ensure_future(asyncio.gather(*futures))
request.app.cache.objects[data.id] = message.id
async def handle_follow(request, actor, data, software):
config: RelayConfig = request.app.config
database: RelayDatabase = request.app.database
# If the following host is not whitelisted, we want to enqueue the request for review.
# This means saving off the two parameters we need later to issue an appropriate acceptance.
if config.whitelist_enabled and not config.is_whitelisted(data.domain):
database.add_request(actor.id, actor.shared_inbox, data.id)
database.save()
return
if not database.add_inbox(actor.shared_inbox, data.id):
database.set_followid(actor.id, data.id)
database.save()
await misc.request(
actor.shared_inbox,
misc.Message.new_response(
host=request.app.config.host,
actor=actor.id,
followid=data.id,
accept=True
),
)
await misc.request(
actor.shared_inbox,
misc.Message.new_follow(
host=request.app.config.host,
actor=actor.id
),
)
async def handle_undo(request, actor, data, software):
# If the object is not a Follow, forward it
if data["object"]["type"] != "Follow":
return await handle_forward(request, actor, data, software)
if not request.app.database.del_inbox(actor.domain, data.id):
return
request.app.database.save()
message = misc.Message.new_unfollow(
host=request.app.config.host, actor=actor.id, follow=data
)
await misc.request(actor.shared_inbox, message)
async def handle_dont(request, actor, data, software):
"""Handle something by ... not handling it."""
logging.info(f"Disregarding {data!r}")
processors = {
"Announce": handle_relay,
"Create": handle_relay,
"Delete": handle_forward,
"Follow": handle_follow,
"Undo": handle_undo,
"Update": handle_forward,
}
async def run_processor(request, actor, data, software):
if data.type not in processors:
return
logging.info(f'{request.id}: New "{data.type}" from actor: {actor.id}')
logging.debug(f"{request.id}: {data!r}")
env = dict(data=data, actor=actor, software=software)
try:
return await processors.get(data.type, handle_dont)(request, actor, data, software)
except:
logging.exception(f"{request.id}] {env!r}")

View file

@ -0,0 +1,299 @@
import json
from json.decoder import JSONDecodeError
import logging
from aiohttp.web import HTTPUnauthorized, Request
from relay import __version__, misc
from relay.http_debug import STATS
from relay.misc import (
DotDict,
Message,
Response,
WKNodeinfo,
)
from relay.processors import run_processor
routes = []
def register_route(method, path):
def wrapper(func):
routes.append([method, path, func])
return func
return wrapper
@register_route("GET", "/")
async def home(request):
following = "<ul>" + ("\n".join(f"<li>{it}</li>" for it in request.app.database["relay-list"])) + "</ul>"
following_count = len(request.app.database.hostnames)
requested = "<ul>" + ("\n".join(f"<li>{it}</li>" for it in request.app.database["follow-requests"])) + "</ul>"
requested_count = len(request.app.database["follow-requests"])
note = request.app.config.note
host = request.app.config.host
text = f"""\
<html><head>
<title>ActivityPub Relay at {host}</title>
<style>
body {{ background-color: #000000; color: #FFFFFF; font-family: monospace, arial; font-size: 100%; }}
a {{ color: #26F; }}
a:visited {{ color: #46C; }}
a:hover {{ color: #8AF; }}
</style>
</head>
<body>
<p>This is an Activity Relay for fediverse instances.</p>
<p>{note}</p>
<p>To host your own relay, you may download the code at this address: <a href="https://git.arrdem.com/arrdem/source/src/branch/trunk/projects/activitypub_relay">https://git.arrdem.com/arrdem/source/src/branch/trunk/projects/activitypub_relay</a></p>
<br><p>This relay is peered with {following_count} registered instances:<br>{following}</p>
<br><p>Another {requested_count} peers await approval:<br>{requested}</p>
</body></html>"""
return Response.new(text, ctype="html")
@register_route("GET", "/inbox")
@register_route("GET", "/actor")
async def actor(request):
data = Message.new_actor(
host=request.app.config.host, pubkey=request.app.database.pubkey
)
return Response.new(data, ctype="activity")
@register_route("POST", "/inbox")
@register_route("POST", "/actor")
async def inbox(request):
config = request.app.config
# reject if missing signature header
if "signature" not in request.headers:
logging.debug("Actor missing signature header")
raise HTTPUnauthorized(body="missing signature")
# read message and get actor id and domain
try:
data = await request.json(loads=Message.new_from_json)
if "actor" not in data:
raise KeyError("actor")
# reject if there is no actor in the message
except KeyError:
logging.debug("actor not in data")
return Response.new_error(400, "no actor in message", "json")
except:
logging.exception("Failed to parse inbox message")
return Response.new_error(400, "failed to parse message", "json")
# FIXME: A lot of this code assumes that we're going to have access to the entire actor descriptor.
# This isn't something we actually generally need, and it's not clear how used it is.
# The original relay implementation mostly used to determine activity source domain for relaying.
# This has been refactored out, since there are other sources of this information.
# This PROBABLY means we can do without this data ever ... but here it is.
# Trying to deal with actors/visibility
if isinstance(data.object, dict) and not data.object.get("discoverable", True):
actor = DotDict({"id": "dummy-for-undiscoverable-object"})
# Normal path of looking up the actor...
else:
try:
# FIXME: Needs a cache
actor = await misc.request(data.actorid)
except:
logging.exception(f"{request.id}: {data!r}")
return
logging.debug(f"Inbox >> {data!r}")
# reject if actor is empty
if not actor:
logging.debug(f"Failed to fetch actor: {data.actorid}")
return Response.new_error(400, "failed to fetch actor", "json")
# Reject if the actor isn't whitelisted while the whiltelist is enabled
# An exception is made for follow requests, which we want to enqueue not reject out of hand
elif config.whitelist_enabled and not config.is_whitelisted(data.domain) and data["type"] != "Follow":
logging.debug(
f"Rejected actor for not being in the whitelist: {data.actorid}"
)
return Response.new_error(403, "access denied", "json")
# reject if actor is banned
if request.app["config"].is_banned(data.domain):
logging.debug(f"Ignored request from banned actor: {data.actorid}")
return Response.new_error(403, "access denied", "json")
# FIXME: Needs a cache
software = await misc.fetch_nodeinfo(data.domain)
# reject if software used by actor is banned
if config.blocked_software:
if config.is_banned_software(software):
logging.debug(f"Rejected actor for using specific software: {software}")
return Response.new_error(403, "access denied", "json")
# reject if the signature is invalid
if not (await misc.validate_signature(data.actorid, request)):
logging.debug(f"signature validation failed for: {data.actorid}")
return Response.new_error(401, "signature check failed", "json")
logging.debug(f">> payload {data}")
resp = await run_processor(request, actor, data, software)
return resp or Response.new(status=202)
@register_route("GET", "/.well-known/webfinger")
async def webfinger(request):
if not (subject := request.query.get("resource")):
return Response.new_error(404, "no resource specified", "json")
if subject != f"acct:relay@{request.app.config.host}":
return Response.new_error(404, "user not found", "json")
data = {
"subject": subject,
"aliases": [request.app.config.actor],
"links": [
{
"href": request.app.config.actor,
"rel": "self",
"type": "application/activity+json",
},
{
"href": request.app.config.actor,
"rel": "self",
"type": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
},
],
}
return Response.new(data, ctype="json")
@register_route("GET", "/nodeinfo/{version:\d.\d\.json}")
async def nodeinfo_2_0(request):
niversion = request.match_info["version"][:3]
data = {
"openRegistrations": not request.app.config.whitelist_enabled,
"protocols": ["activitypub"],
"services": {"inbound": [], "outbound": []},
"software": {"name": "activityrelay", "version": __version__},
"usage": {"localPosts": 0, "users": {"total": 1}},
"metadata": {"peers": request.app.database.hostnames},
"version": niversion,
}
if niversion == "2.1":
data["software"]["repository"] = "https://git.pleroma.social/pleroma/relay"
return Response.new(data, ctype="json")
@register_route("GET", "/.well-known/nodeinfo")
async def nodeinfo_wellknown(request):
data = WKNodeinfo.new(
v20=f"https://{request.app.config.host}/nodeinfo/2.0.json",
v21=f"https://{request.app.config.host}/nodeinfo/2.1.json",
)
return Response.new(data, ctype="json")
@register_route("GET", "/stats")
async def stats(request):
stats = STATS.copy()
stats["pending_requests"] = len(request.app.database.get("follow-requests", {}))
return Response.new(stats, ctype="json")
@register_route("POST", "/admin/config")
async def set_config(request: Request):
if not (auth := request.headers.get("Authorization")):
return Response.new_error(403, "access denied", "json")
if not auth == f"Bearer {request.app.config.admin_token}":
return Response.new_error(403, "access denied", "json")
# FIXME: config doesn't have a way to go from JSON or update, using dict stuff
text = await request.text()
try:
new_config = json.loads(text)
except JSONDecodeError as e:
logging.exception(f"Unable to load config {text!r}")
return Response.new_error(400, "bad request", "json")
request.app.config.update(new_config)
if request.app.config.whitelist_enabled:
# If there are pending follows which are NOW whitelisted, allow them
for domain in request.app.config.whitelist:
if (pending_follow := request.app.database.get_request(domain, False)):
logging.info(f"Acknowledging queued follow request from {domain}...")
await misc.request(
pending_follow["inbox"],
misc.Message.new_response(
host=request.app.config.host,
actor=pending_follow["actor"],
followid=pending_follow["followid"],
accept=True
),
)
await misc.request(
pending_follow["inbox"],
misc.Message.new_follow(
host=request.app.config.host,
actor=pending_follow["actor"]
),
)
request.app.database.del_request(domain)
# FIXME: If there are EXISTING follows which are NO LONGER allowed/are blacklisted, drop them
request.app.database.save()
request.app.config.save()
return Response.new(status=202)
@register_route("GET", "/admin/config")
def get_config(request: Request):
if not (auth := request.headers.get("Authorization")):
return Response.new_error(403, "access denied", "json")
if not auth == f"Bearer {request.app.config.admin_token}":
return Response.new_error(403, "access denied", "json")
return Response.new(request.app.config, status=200, ctype="json")
@register_route("GET", "/admin/db")
def get_db(request: Request):
if not (auth := request.headers.get("Authorization")):
return Response.new_error(403, "access denied", "json")
if not auth == f"Bearer {request.app.config.admin_token}":
return Response.new_error(403, "access denied", "json")
return Response.new(request.app.database, status=200, ctype="json")
@register_route("GET", "/admin/pending")
def get_pending(request):
if not (auth := request.headers.get("Authorization")):
return Response.new_error(403, "access denied", "json")
if not auth == f"Bearer {request.app.config.admin_token}":
return Response.new_error(403, "access denied", "json")
return Response.new(request.app.database["follow-requests"], status=200, ctype="json")

12
projects/aloe/BUILD.bazel Normal file
View file

@ -0,0 +1,12 @@
py_project(
name = "lib"
)
zapp_binary(
name = "aloe",
main = "src/aloe/__main__.py",
deps = [
":lib",
py_requirement("icmplib"),
],
)

122
projects/aloe/README.md Normal file
View file

@ -0,0 +1,122 @@
# Aloe
> - A [cactus](https://www.cacti.net/)-like plant
> - Traditionally used for a variety of skin conditions
Aloe is a quick and dirty network weathermapping tool, much like MTR or Cacti.
Aloe uses multiple threads to first establish a rough network topology via ICMP traceroutes, and then monitor it with ICMP pings.
## Usage
``` sh
$ bazel build //projects/aloe
$ sudo ./bazel-bin/projects/aloe/aloe twitter.com google.com
INFO:__main__:Graph -
┌─────────────────┐
│ 127.0.0.1 │
└─────────────────┘
┌─────────────────┐ ┌─────────────────┐
│ 68.85.107.81 │ ◀── │ 10.0.0.1 │
└─────────────────┘ └─────────────────┘
│ │
│ │
│ ▼
│ ┌─────────────────┐
│ │ 68.85.107.85 │
│ └─────────────────┘
│ │
│ │
│ ▼
│ ┌─────────────────┐
│ │ 68.86.103.9 │
│ └─────────────────┘
│ │
│ │
│ ▼
│ ┌─────────────────┐
└───────────────────▶ │ 68.85.89.213 │
└─────────────────┘
┌─────────────────┐
│ 24.124.155.129 │
└─────────────────┘
┌─────────────────┐ ┌─────────────────┐ ┌────────────────┐
│ 96.110.43.241 │ ◀── │ 96.216.22.130 │ ──▶ │ 96.110.43.253 │
└─────────────────┘ └─────────────────┘ └────────────────┘
│ │ │
│ │ │
▼ ▼ ▼
┌─────────────────┐ ┌─────────────────┐ ┌────────────────┐
│ 96.110.38.114 │ │ 96.110.43.245 │ │ 96.110.38.126 │
└─────────────────┘ └─────────────────┘ └────────────────┘
│ │ │
│ │ │
▼ ▼ ▼
┌─────────────────┐ ┌─────────────────┐ ┌────────────────┐
│ 96.87.8.210 │ │ 96.110.38.118 │ │ 23.30.206.218 │
└─────────────────┘ └─────────────────┘ └────────────────┘
│ │ │
│ │ │
▼ ▼ ▼
┌─────────────────┐ ┌─────────────────┐ ┌────────────────┐
│ 108.170.252.193 │ │ 173.167.57.142 │ │ 172.253.75.177 │
└─────────────────┘ └─────────────────┘ └────────────────┘
│ │ │
│ │ │
▼ ▼ ▼
┌─────────────────┐ ┌─────────────────┐ ┌────────────────┐
│ 142.250.69.238 │ │ 213.155.133.171 │ │ 142.250.72.46 │
└─────────────────┘ └─────────────────┘ └────────────────┘
┌─────────────────┐ ┌─────────────────┐
│ 104.244.42.65 │ ◀── │ 62.115.49.193 │
└─────────────────┘ └─────────────────┘
┌─────────────────┐
│ 104.244.42.129 │
└─────────────────┘
```
If hosts in topology stop responding for 10s or more (the polling interval is ~1s), they are declared to be warning.
If hosts in topology stop responding for 5s, they are declared down.
If a host in topology resume responding after 5s or more, they are declared to have recovered.
If hosts in topology stop responding for 30 min, they are declared dead and monitoring is stopped.
The topology is reconfigured every 5 min to account to DHCP and upstream changes.
A log of all these events is built in a plain-text format to `incidents.txt`.
The format of this file is -
```
UP <ip> <date>
WARN <ip> <date>
DOWN <ip> <date>
RECOVERED <ip> <date> <duration>
DEAD <ip> <date> <duration>
```
## Future work
- [ ] Log topology
- [ ] Attempt to identify "root cause" incidents in the route graph which explain downstream failures
- [ ] Use sqlite3 for aggregation not a plain text file
- [ ] Use a more sophisticated counters and debounce model of host state in the main thread
- [ ] Find some way to incorporate rolling counters (mean, median, stddev, deciles, max) into the UI
- [ ] FFS find some way NOT to depend on that nonsense box-diagram service
## License
Copyright Reid 'arrdem' McKenzie, 11/20/2021.
Published under the terms of the MIT license.

View file

@ -0,0 +1,258 @@
"""Aloe - A shitty weathermapping tool.
Think MTR but with the ability to detect/declare incidents and emit logs.
Periodically traceroutes the egress network, and then walks pings out the egress network recording times and hosts which
failed to respond. Expects a network in excess of 90% packet delivery, but with variable timings. Intended to probe for
when packet delivery latencies radically degrade and maintain a report file.
"""
import argparse
from collections import deque as ringbuffer
import curses
from datetime import timedelta
from itertools import count
import logging
import queue
from queue import Queue
import sys
from threading import Event, Lock, Thread
from time import time
from .cursedlogger import CursesHandler
from .icmp import *
from .icmp import _ping
log = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
parser.add_argument("hosts", nargs="+")
class HostState(object):
"""A model of a (bounded) time series of host state.
"""
def __init__(self,
hostname: str,
history = [],
history_size = 60 * 60 * 24,
is_up: bool = False,
lost: int = 0,
up: float = 0.0):
self._lock = Lock()
self._state = ringbuffer(maxlen=history_size)
self._is_up = is_up
self._lost = lost
self._up = up
for resp in history:
self.append(resp)
def append(self, resp):
with self._lock:
if resp and not self._is_up:
# log.debug(f"Host {self._hostname} is up!")
self._is_up = self._is_up or resp
self._up = resp._time
elif resp and self._is_up:
# log.debug(f"Host {self._hostname} holding up...")
pass
elif not resp and self._is_up:
# log.debug(f"Host {self._hostname} is down!")
self._is_up = None
self._up = None
elif not resp and not self._is_up:
pass
if not resp:
self._lost += 1
if self._state and not self._state[0]:
self._lost -= 1
self._state.append(resp)
def last(self):
with self._lock:
return next(reversed(self._state), None)
def last_window(self, duration: timedelta = None):
with self._lock:
l = []
t = time() - duration.total_seconds()
for i in reversed(self._state):
if not i or i._time > t:
l.insert(0, i)
else:
break
return l
def loss(self, duration: timedelta):
log = self.last_window(duration)
if log:
return log.count(None) / len(log)
else:
return 0.0
def is_up(self, duration: timedelta, threshold = 0.25):
return self.loss(duration) <= threshold
def last_seen(self, now: datetime) -> timedelta:
if state := self.last():
return now - datetime.fromtimestamp(state._time)
def up(self, duration: datetime):
if self._up:
return datetime.fromtimestamp(self._up)
class MonitoredHost(object):
"""A shim (arguably a lambda) for generating a timeline of host state."""
def __init__(self, hostname: str, timeout: timedelta, id=None):
self._hostname = hostname
self._timeout = timeout
self._sequence = request_sequence(hostname, timeout, id)
self._lock = Lock()
self._state = HostState(hostname)
def __call__(self, shutdown: Event, q: Queue):
"""Monitor a given host by throwing requests into the queue; intended to be a Thread target."""
while not shutdown.is_set():
req = next(self._sequence)
resp = _ping(q, req)
self._state.append(resp)
sleep(shutdown, 1)
@property
def state(self):
return self._state
def retrace(shutdown, q, opts, hl, hosts):
threads = {}
def create_host(distance, address):
with hl:
if address not in hosts:
log.info(f"Monitoring {address}...")
monitor = MonitoredHost(address, timedelta(seconds=4))
hosts[address] = (distance, monitor)
threads[address] = t = Thread(target=monitor, args=(shutdown, q))
t.start()
else:
log.debug(f"Already monitoring {address}...")
while not shutdown.is_set():
for h in opts.hosts:
# FIXME: Use a real topology model
for hop in traceroute(q, h):
if ping(q, hop.address).is_alive:
create_host(hop.distance, hop.address)
sleep(shutdown, 60 * 5)
def render(shutdown, q, stdscr, hl, hosts):
dt = timedelta(minutes=30)
with open("incidents.txt", "a") as fp:
incident = False
while not shutdown.is_set():
rows, cols = stdscr.getmaxyx()
down = 0
now = datetime.now()
i = 0
with hl:
for i, (name, (distance, host)) in zip(count(1), sorted(hosts.items(), key=lambda x: x[1][0])):
loss = host.state.loss(dt) * 100
state = host.state.last()
if not state:
down += 1
last_seen = "Down"
else:
last_seen = f"{host.state.last_seen(now).total_seconds():.2f}s ago"
if up := host.state.up(dt):
up = f" up: {(now - up).total_seconds():.2f}"
else:
up = ""
stdscr.addstr(i, 2, f"{distance: <2} {name: <16s}]{up} lost: {loss:.2f}% last: {last_seen}".ljust(cols))
stdscr.border()
stdscr.refresh()
msg = None
if down >= 3 and not incident:
incident = True
msg = f"{datetime.now()} - {down} hosts down"
elif down < 3 and incident:
incident = False
msg = f"{datetime.now()} - network recovered"
if i != 0 and msg:
log.info(msg)
fp.write(msg + "\n")
fp.flush()
sleep(shutdown, 1)
def main():
stdscr = curses.initscr()
maxy, maxx = stdscr.getmaxyx()
begin_x = 2; begin_y = maxy - 12
height = 10; width = maxx - 4
logscr = curses.newwin(height, width, begin_y, begin_x)
handler = CursesHandler(logscr)
formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
handler.setFormatter(formatter)
log.addHandler(handler)
log.setLevel(logging.DEBUG)
stdscr = curses.newwin(maxy - height - 2, width, 0, begin_x)
opts, args = parser.parse_known_args()
q = queue.Queue()
shutdown = Event()
p = Thread(target=icmp_worker, args=(shutdown, q,))
p.start()
hosts = {}
hl = Lock()
rt = Thread(target=render, args=(shutdown, q, stdscr, hl, hosts))
rt.start()
tt = Thread(target=retrace, args=(shutdown, q, opts, hl, hosts))
tt.start()
try:
while True:
sleep(shutdown, 1)
except (KeyboardInterrupt, SystemExit):
pass
finally:
curses.endwin()
sys.stdout.flush()
sys.stderr.flush()
shutdown.set()
if __name__ == "__main__":
main()

View file

@ -0,0 +1,29 @@
"""A CURSES screen targeted log record handler."""
from collections import deque as ringbuffer
from itertools import count
import logging
class CursesHandler(logging.Handler):
def __init__(self, screen):
logging.Handler.__init__(self)
self._screen = screen
# FIXME: This should be dynamic not static.
self._buff = ringbuffer(maxlen=screen.getmaxyx()[0] - 2)
def emit(self, record):
try:
msg = self.format(record) + "\n"
self._buff.append(msg)
self._screen.clear()
for i, msg in zip(count(1), self._buff):
self._screen.addstr(i, 2, msg)
self._screen.border()
self._screen.refresh()
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)

View file

@ -0,0 +1,251 @@
"""Tools sitting between aloe and icmplib."""
from datetime import datetime, timedelta
from itertools import islice as take
import logging
import queue
from random import randint
from threading import Event, Lock
from time import sleep as _sleep
from icmplib.exceptions import (
ICMPLibError,
ICMPSocketError,
)
from icmplib.models import (
Hop,
Host,
ICMPReply,
ICMPRequest,
)
from icmplib.sockets import ICMPv4Socket
from icmplib.utils import is_hostname, resolve
log = logging.getLogger(__name__)
def better_repr(self):
elems = ", ".join(f"{slot}={getattr(self, slot)}" for slot in self.__slots__)
return f"<{type(self).__name__} ({elems})>"
ICMPRequest.__repr__ = better_repr
ICMPReply.__repr__ = better_repr
def sleep(event, duration, interval=0.1):
total = 0
while total < duration:
if event.is_set():
raise SystemExit()
else:
_sleep(interval)
total += interval
class ICMPRequestResponse(object):
"""A thread-safe request/response structure for pinging a host."""
PENDING = object()
TIMEOUT = object()
def __init__(self, address, request: ICMPRequest, timeout: timedelta):
_timeout = datetime.now() + timeout
self._address = address
self._request = request
self._timeout = _timeout
self._lock = Lock()
self._response = self.PENDING
def ready(self):
"""Determine if this request is still waiting."""
return self.get() is not self.PENDING
def get(self):
"""Get the response, unless the timeout has passed in which case return a timeout."""
with self._lock:
if self._response is not self.PENDING:
return self._response
elif self._timeout < datetime.now():
return self.TIMEOUT
else:
return self._response
def set(self, response):
"""Set the response, unless the timeout has passed in which case set a timeout."""
if isinstance(response, ICMPReply):
with self._lock:
if self._timeout < datetime.now():
self._response = self.TIMEOUT
else:
# rtt = (reply.time - request.time) * 1000
self._response = response
def icmp_worker(shutdown: Event, q: queue.Queue):
"""A worker thread which processes ICMP requests; sending packets and listening for matching responses."""
state = {}
with ICMPv4Socket(None, True) as sock:
while not shutdown.is_set():
# Send one
try:
item = q.get(block=False, timeout=0.001)
request = item._request
state[(request._id, request._sequence)] = item
# log.info(f"Sending request {item._request!r}")
sock.send(item._request)
except (ICMPLibError, ICMPSocketError, queue.Empty):
pass
# Recieve one
try:
if response := sock.receive(None, 0.001):
key = (response.id, response.sequence)
if key in state:
# log.info(f"Got response {response!r}")
state[key].set(response)
del state[key]
else:
# log.warning(f"Recieved non-matching response {response!r}")
pass
except (ICMPLibError, ICMPSocketError):
pass
# GC one
if key := next(iter(state.keys()), None):
if state[key].ready():
del state[key]
# Sleep one
sleep(shutdown, 0.001)
def traceroute(q: queue.Queue,
address: str,
first_hop: int = 1,
max_hops: int = 32,
count: int = 3,
id: int = None,
family: int = None):
if is_hostname(address):
address = resolve(address, family)[0]
mask = ((1<<16) - 1)
id = id or randint(1, mask) & 0xFFFF
ttl = first_hop
host_reached = False
hops = []
while not host_reached and ttl <= max_hops:
reply = None
packets_sent = 0
rtts = []
for sequence in range(count):
request = ICMPRequestResponse(
address,
ICMPRequest(
destination=address,
id=id,
sequence=sequence,
ttl=ttl
),
timedelta(seconds=1),
)
q.put(request)
while not request.ready():
_sleep(0.1)
_reply = request.get()
if _reply is ICMPRequestResponse.TIMEOUT:
_sleep(0.1)
continue
elif _reply:
reply = reply or _reply
try:
reply.raise_for_status()
host_reached = True
except ICMPLibError:
pass
rtt = (reply.time - request._request.time) * 1000
rtts.append(rtt)
if reply:
hops.append(
Hop(
address=reply.source,
packets_sent=packets_sent,
rtts=rtts,
distance=ttl
)
)
ttl += 1
return hops
def request_sequence(hostname: str,
timeout: timedelta,
id: int = None,
family: int = None):
"""Generate a sequence of requests monitoring a specific, usable as a request source for a ping."""
if is_hostname(hostname):
destination = resolve(hostname, family)[0]
else:
destination = hostname
mask = ((1<<16) - 1)
id = id or randint(1, mask) & 0xFFFF
sequence = 1
while True:
yield ICMPRequestResponse(
hostname,
ICMPRequest(
destination=destination,
id=id,
sequence=sequence & mask,
),
timeout
)
sequence += 1
def _ping(q: queue.Queue, request: ICMPRequestResponse):
q.put(request)
while not request.ready():
_sleep(0.1)
_response = request.get()
if _response is not ICMPRequestResponse.TIMEOUT:
return _response
def ping(q: queue.Queue,
address: str,
count: int = 3,
id: int = None,
family: int = None) -> Host:
"""Ping a host N times."""
rtts = []
for request in take(request_sequence(address, timedelta(seconds=1)), count):
if reply := _ping(q, request):
rtt = (reply.time - request._request.time) * 1000
rtts.append(rtt)
return Host(
address=address,
packets_sent=count,
rtts=rtts,
)

View file

@ -0,0 +1,6 @@
py_project(
name = "anosql-migrations",
lib_deps = [
"//projects/anosql",
],
)

View file

@ -0,0 +1,189 @@
"""Quick and dirty migrations for AnoSQL."""
from datetime import datetime
from hashlib import sha256
import logging
import re
import typing as t
from anosql.core import from_str, Queries
log = logging.getLogger(__name__)
class MigrationDescriptor(t.NamedTuple):
name: str
sha256sum: str
committed_at: t.Optional[datetime] = None
def __repr__(self):
return f"MigrationDescriptor(name={self.name!r}, sha256sum='{self.sha256sum[:7]}...')"
def __hash__(self):
return hash((self.name, self.sha256sum))
def __eq__(self, other):
return (self.name, self.sha256sum) == (other.name, other.sha256sum)
_SQL = """
-- name: anosql_migrations_create_table#
-- Create the migrations table for the anosql_migrations plugin.
CREATE TABLE IF NOT EXISTS `anosql_migration` (
`name` TEXT PRIMARY KEY NOT NULL
, `committed_at` INT
, `sha256sum` TEXT NOT NULL
, CONSTRAINT `am_sha256sum_unique` UNIQUE (`sha256sum`)
);
-- name: anosql_migrations_list
-- List committed migrations
SELECT
`name`
, `committed_at`
, `sha256sum`
FROM `anosql_migration`
WHERE
`committed_at` > 0
ORDER BY
`name` ASC
;
-- name: anosql_migrations_get
-- Get a given migration by name
SELECT
`name`
, `committed_at`,
, `sha256sum`
FROM `anosql_migration`
WHERE
`name` = :name
ORDER BY
`rowid` ASC
;
-- name: anosql_migrations_create<!
-- Insert a migration, marking it as committed
INSERT OR REPLACE INTO `anosql_migration` (
`name`
, `committed_at`
, `sha256sum`
) VALUES (
:name
, :date
, :sha256sum
);
"""
def with_migrations(driver_adapter, queries: Queries, conn) -> Queries:
"""Initialize the migrations plugin."""
# Compile SQL as needed from the _SQL constant
_q = from_str(_SQL, driver_adapter)
# Merge. Sigh.
for _qname in _q.available_queries:
queries.add_query(_qname, getattr(_q, _qname))
# Create the migrations table
create_tables(queries, conn)
return queries
def create_tables(queries: Queries, conn) -> None:
"""Create the migrations table (if it doesn't exist)."""
if queries.anosql_migrations_create_table(conn):
log.info("Created migrations table")
# Insert the bootstrap 'fixup' record
execute_migration(
queries,
conn,
MigrationDescriptor(
name="anosql_migrations_create_table",
sha256sum=sha256(
queries.anosql_migrations_create_table.sql.encode("utf-8")
).hexdigest(),
),
)
def committed_migrations(queries: Queries, conn) -> t.Iterable[MigrationDescriptor]:
"""Enumerate migrations committed to the database."""
for name, committed_at, sha256sum in queries.anosql_migrations_list(conn):
yield MigrationDescriptor(
name=name,
committed_at=datetime.fromtimestamp(committed_at),
sha256sum=sha256sum,
)
def available_migrations(queries: Queries, conn) -> t.Iterable[MigrationDescriptor]:
"""Enumerate all available migrations, executed or no."""
for query_name in sorted(queries.available_queries):
if not re.match("^migration", query_name):
continue
if query_name.endswith("_cursor"):
continue
# query_name: str
# query_fn: t.Callable + {.__name__, .__doc__, .sql}
query_fn = getattr(queries, query_name)
yield MigrationDescriptor(
name=query_name,
committed_at=None,
sha256sum=sha256(query_fn.sql.encode("utf-8")).hexdigest(),
)
def execute_migration(queries: Queries, conn, migration: MigrationDescriptor):
"""Execute a given migration singularly."""
with conn:
# Mark the migration as in flight
queries.anosql_migrations_create(
conn,
# Args
name=migration.name,
date=-1,
sha256sum=migration.sha256sum,
)
# Run the migration function
getattr(queries, migration.name)(conn)
# Mark the migration as committed
queries.anosql_migrations_create(
conn,
# Args
name=migration.name,
date=int(datetime.utcnow().timestamp()),
sha256sum=migration.sha256sum,
)
def run_migrations(queries, conn):
"""Run all remaining migrations."""
avail = set(available_migrations(queries, conn))
committed = set(committed_migrations(queries, conn))
for migration in sorted(avail, key=lambda m: m.name):
if migration in committed:
log.info(f"Skipping committed migration {migration.name}")
else:
log.info(f"Beginning migration {migration.name}")
try:
execute_migration(queries, conn, migration)
except Exception as e:
log.exception(f"Migration {migration.name} failed!", e)
raise e

View file

@ -0,0 +1,109 @@
"""Tests covering the migrations framework."""
import sqlite3
import anosql
from anosql.core import Queries
import anosql_migrations
import pytest
_SQL = """\
-- name: migration_0000_create_kv
CREATE TABLE kv (`id` INT, `key` TEXT, `value` TEXT);
"""
def table_exists(conn, table_name):
return list(
conn.execute(
f"""\
SELECT (
`name`
)
FROM `sqlite_master`
WHERE
`type` = 'table'
AND `name` = '{table_name}'
;"""
)
)
@pytest.fixture
def conn() -> sqlite3.Connection:
"""Return an (empty) SQLite instance."""
return sqlite3.connect(":memory:")
def test_connect(conn: sqlite3.Connection):
"""Assert that the connection works and we can execute against it."""
assert list(conn.execute("SELECT 1;")) == [
(1,),
]
@pytest.fixture
def queries(conn) -> Queries:
"""A fixture for building a (migrations capable) anosql queries object."""
q = anosql.from_str(_SQL, "sqlite3")
return anosql_migrations.with_migrations("sqlite3", q, conn)
def test_queries(queries):
"""Assert that we can construct a queries instance with migrations features."""
assert isinstance(queries, Queries)
assert hasattr(queries, "anosql_migrations_create_table")
assert hasattr(queries, "anosql_migrations_list")
assert hasattr(queries, "anosql_migrations_create")
def test_migrations_create_table(conn, queries):
"""Assert that the migrations system will (automagically) create the table."""
assert table_exists(conn, "anosql_migration"), "Migrations table did not create"
def test_migrations_list(conn, queries):
"""Test that we can list out available migrations."""
ms = list(anosql_migrations.available_migrations(queries, conn))
assert any(
m.name == "migration_0000_create_kv" for m in ms
), f"Didn't find in {ms!r}"
def test_committed_migrations(conn, queries):
"""Assert that only the bootstrap migration is committed to the empty connection."""
ms = list(anosql_migrations.committed_migrations(queries, conn))
assert len(ms) == 1
assert ms[0].name == "anosql_migrations_create_table"
def test_apply_migrations(conn, queries):
"""Assert that if we apply migrations, the requisite table is created."""
anosql_migrations.run_migrations(queries, conn)
assert table_exists(conn, "kv")
@pytest.fixture
def migrated_conn(conn, queries):
"""Generate a connection whithin which the `kv` migration has already been run."""
anosql_migrations.run_migrations(queries, conn)
return conn
def test_post_committed_migrations(migrated_conn, queries):
"""Assert that the create_kv migration has been committed."""
ms = list(anosql_migrations.committed_migrations(queries, migrated_conn))
assert any(m.name == "migration_0000_create_kv" for m in ms), "\n".join(
migrated_conn.iterdump()
)

View file

@ -0,0 +1,10 @@
py_project(
name="anosql",
test_deps = [
py_requirement("pytest-postgresql"),
py_requirement("psycopg2"),
],
test_tags = [
"known-to-fail",
],
)

26
projects/anosql/LICENSE Normal file
View file

@ -0,0 +1,26 @@
Copyright (c) 2014-2017, Honza Pokorny
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The views and conclusions contained in the software and documentation are those
of the authors and should not be interpreted as representing official policies,
either expressed or implied, of the FreeBSD Project.

View file

@ -0,0 +1,16 @@
anosql
======
A Python library for using SQL
Inspired by the excellent Yesql library by Kris Jenkins. In my mother tongue, ano means yes.
This is a vendored copy of `anosql`_, which exists to capture a few pre-deprecation patches that never made it into the last release, and bazel-ify everything for my use.
License
-------
BSD, short and sweet
.. _anosql: https://github.com/honza/anosql
.. _Yesql: https://github.com/krisajenkins/yesql/

View file

@ -0,0 +1,225 @@
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " applehelp to make an Apple Help Book"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " epub3 to make an epub3"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
@echo " coverage to run coverage check of the documentation (if enabled)"
@echo " dummy to check syntax errors of document sources"
.PHONY: clean
clean:
rm -rf $(BUILDDIR)/*
.PHONY: html
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
.PHONY: dirhtml
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
.PHONY: singlehtml
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
.PHONY: pickle
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
.PHONY: json
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
.PHONY: htmlhelp
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
.PHONY: qthelp
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/anosql.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/anosql.qhc"
.PHONY: applehelp
applehelp:
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@echo
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
@echo "N.B. You won't be able to view it unless you put it in" \
"~/Library/Documentation/Help or install it in your application" \
"bundle."
.PHONY: devhelp
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/anosql"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/anosql"
@echo "# devhelp"
.PHONY: epub
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
.PHONY: epub3
epub3:
$(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3
@echo
@echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3."
.PHONY: latex
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
.PHONY: latexpdf
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
.PHONY: latexpdfja
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
.PHONY: text
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
.PHONY: man
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
.PHONY: texinfo
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
.PHONY: info
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
.PHONY: gettext
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
.PHONY: changes
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
.PHONY: linkcheck
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
.PHONY: doctest
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
.PHONY: coverage
coverage:
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
@echo "Testing of coverage in the sources finished, look at the " \
"results in $(BUILDDIR)/coverage/python.txt."
.PHONY: xml
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
.PHONY: pseudoxml
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
.PHONY: dummy
dummy:
$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
@echo
@echo "Build finished. Dummy builder generates no files."

339
projects/anosql/doc/conf.py Normal file
View file

@ -0,0 +1,339 @@
# -*- coding: utf-8 -*-
#
# anosql documentation build configuration file, created by
# sphinx-quickstart on Mon Jul 25 09:16:20 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import pkg_resources
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = ".rst"
# The encoding of source files.
#
# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = "index"
# General information about the project.
project = u"anosql"
copyright = u"2014-2017, Honza Pokorny"
author = u"Honza Pokorny"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = pkg_resources.get_distribution("anosql").version
# The full version, including alpha/beta/rc tags.
release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#
# today = ''
#
# Else, today_fmt is used as the format for a strftime call.
#
# today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
# The reST default role (used for this markup: `text`) to use for all
# documents.
#
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"
# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []
# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = "alabaster"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []
# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
#
# html_title = u'anosql v0.1.2'
# A shorter title for the navigation bar. Default is the same as html_title.
#
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#
# html_logo = None
# The name of an image file (relative to this directory) to use as a favicon of
# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#
# html_extra_path = []
# If not None, a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# The empty string is equivalent to '%b %d, %Y'.
#
# html_last_updated_fmt = None
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#
# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#
# html_additional_pages = {}
# If false, no module index is generated.
#
# html_domain_indices = True
# If false, no index is generated.
#
# html_use_index = True
# If true, the index is split into individual pages for each letter.
#
# html_split_index = False
# If true, links to the reST sources are added to the pages.
#
# html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#
# html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#
# html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#
# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None
# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'
#
# html_search_language = 'en'
# A dictionary with options for the search language support, empty by default.
# 'ja' uses this config value.
# 'zh' user can custom change `jieba` dictionary path.
#
# html_search_options = {'type': 'default'}
# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#
# html_search_scorer = 'scorer.js'
# Output file base name for HTML help builder.
htmlhelp_basename = "anosqldoc"
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, "anosql.tex", u"anosql Documentation", u"Honza Pokorny", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#
# latex_use_parts = False
# If true, show page references after internal links.
#
# latex_show_pagerefs = False
# If true, show URL addresses after external links.
#
# latex_show_urls = False
# Documents to append as an appendix to all manuals.
#
# latex_appendices = []
# It false, will not define \strong, \code, itleref, \crossref ... but only
# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
# packages.
#
# latex_keep_old_macro_names = True
# If false, no module index is generated.
#
# latex_domain_indices = True
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [(master_doc, "anosql", u"anosql Documentation", [author], 1)]
# If true, show URL addresses after external links.
#
# man_show_urls = False
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(
master_doc,
"anosql",
u"anosql Documentation",
author,
"anosql",
"One line description of project.",
"Miscellaneous",
),
]
# Documents to append as an appendix to all manuals.
#
# texinfo_appendices = []
# If false, no module index is generated.
#
# texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#
# texinfo_show_urls = 'footnote'
# If true, do not generate a @detailmenu in the "Top" node's menu.
#
# texinfo_no_detailmenu = False

View file

@ -0,0 +1,185 @@
####################
Defining SQL Queries
####################
Query Names & Comments
======================
Name definitions are how ``anosql`` determines how to name the SQL code blocks which are loaded.
A query name definition is a normal SQL comment starting with "\-\- name:" and is followed by the
name of the query. You can use ``-`` or ``_`` in your query names, but the methods in Python
will always be valid Python names using underscores.
.. code-block:: sql
-- name: get-all-blogs
select * from blogs;
The above example when loaded by ``anosql.from_path`` will return an object with a
``.get_all_blogs(conn)`` method.
Your SQL comments will be added to your methods as Python docstrings, and accessible by calling
``help()`` on them.
.. code-block:: sql
-- name: get-all-blogs
-- Fetch all fields for every blog in the database.
select * from blogs;
.. code-block:: python
queries = anosql.from_path("blogs.sql", "sqlite3")
help(anosql.get_all_blogs)
returns
.. code-block:: text
Help on function get_user_blogs in module anosql.anosql:
get_all_blogs(conn, *args, **kwargs)
Fetch all fields for every blog in the database.
.. _query-operations:
Query Operations
================
Adding query operator symbols to the end of query names will inform ``anosql`` of how to
execute and return results. In the above section the ``get-all-blogs`` name has no special operator
characters trailing it. This lack of operator is actually the most basic operator which performs
SQL ``select`` statements and returns a list of rows. When writing an application you will often
need to perform other operations besides selects, like inserts, deletes, and bulk opearations. The
operators detailed in this section let you declare in your SQL how your code should be executed
by the database driver.
Insert/Update/Delete with ``!``
-------------------------------
The ``!`` operator will execute SQL without returning any results. It is meant for use with ``insert``,
``update``, and ``delete`` statements for which returned data is not required.
.. code-block:: sql
-- name: publish-blog!
insert into blogs(userid, title, content) values (:userid, :title, :content);
-- name: remove-blog!
-- Remove a blog from the database
delete from blogs where blogid = :blogid;
The methods generated are:
- ``publish_blog(conn, *args, **kwargs)``
- ``remove_blog(conn, *args, **kwargs)``
Each of them can be run to alter the database, but both will return ``None``.
Insert Returning with ``<!``
----------------------------
Sometimes when performing an insert it is necessary to receive some information back about the
newly created database row. The ``<!`` operator tells anosql to perform execute the insert query, but to also expect and
return some data.
In SQLite this means the ``cur.lastrowid`` will be returned.
.. code-block:: sql
-- name: publish-blog<!
insert into blogs(userid, title, content) values (:userid, :title, :content);
Will return the ``blogid`` of the inserted row.
PostgreSQL however allows returning multiple values via the ``returning`` clause of insert
queries.
.. code-block:: sql
-- name: publish-blog<!
insert into blogs (
userid,
title,
content
)
values (
:userid,
:title,
:content
)
returning blogid, title;
This will insert the new blog row and return both it's ``blogid`` and ``title`` value as follows::
queries = anosql.from_path("blogs.sql", "psycopg2")
blogid, title = queries.publish_blog(conn, userid=1, title="Hi", content="word.")
Insert/Update/Delete Many with ``*!``
-------------------------------------
The DB-API 2.0 drivers like ``sqlite3`` and ``psycopg2`` have an ``executemany`` method which
execute a SQL command against all parameter sequences or mappings found in a sequence. This
is useful for bulk updates to the database. The below example is a PostgreSQL statement to insert
many blog rows.
.. code-block:: sql
-- name: bulk-publish*!
-- Insert many blogs at once
insert into blogs (
userid,
title,
content,
published
)
values (
:userid,
:title,
:content,
:published
)
Applying this to a list of blogs in Python::
queries = anosql.from_path("blogs.sql", "psycopg2")
blogs = [
{"userid": 1, "title": "First Blog", "content": "...", published: datetime(2018, 1, 1)},
{"userid": 1, "title": "Next Blog", "content": "...", published: datetime(2018, 1, 2)},
{"userid": 2, "title": "Hey, Hey!", "content": "...", published: datetime(2018, 7, 28)},
]
queries.bulk_publish(conn, blogs)
Execute SQL script statements with ``#``
---------------------------------------------
Executes some SQL statements as a script. These methods don't do variable substitution, or return
any rows. An example use case is using data definition statements like `create` table in order to
setup your database.
.. code-block:: sql
-- name: create-schema#
create table users (
userid integer not null primary key,
username text not null,
firstname integer not null,
lastname text not null
);
create table blogs (
blogid integer not null primary key,
userid integer not null,
title text not null,
content text not null,
published date not null default CURRENT_DATE,
foreign key(userid) references users(userid)
);
From code::
queries = anosql.from_path("create_schema.sql", "sqlite3")
queries.create_schema(conn)

View file

@ -0,0 +1,50 @@
.. _extending-anosql:
################
Extending anosql
################
.. _driver-adapters:
Driver Adapters
---------------
Database driver adapters in ``anosql`` are duck-typed classes which follow the below interface.::
class MyDbAdapter():
def process_sql(self, name, op_type, sql):
pass
def select(self, conn, sql, parameters):
pass
@contextmanager
def select_cursor(self, conn, sql, parameters):
pass
def insert_update_delete(self, conn, sql, parameters):
pass
def insert_update_delete_many(self, conn, sql, parameters):
pass
def insert_returning(self, conn, sql, parameters):
pass
def execute_script(self, conn, sql):
pass
anosql.core.register_driver_adapter("mydb", MyDbAdapter)
If your adapter constructor takes arguments you can register a function which can build
your adapter instance::
def adapter_factory():
return MyDbAdapter("foo", 42)
anosql.core.register_driver_adapter("mydb", adapter_factory)
Looking at the source of the builtin
`adapters/ <https://github.com/honza/anosql/tree/master/anosql/adapters>`_ is a great place
to start seeing how you may write your own database driver adapter.

View file

@ -0,0 +1,56 @@
###############
Getting Started
###############
Below is an example of a program which can print ``"{greeting}, {world_name}!"`` from data held in a minimal SQLite
database containing greetings and worlds.
The SQL is in a ``greetings.sql`` file with ``-- name:`` definitions on each query to tell ``anosql`` under which name
we would like to be able to execute them. For example, the query under the name ``get-all-greetings`` in the example
below will be available to us after loading via ``anosql.from_path`` as a method ``get_all_greetings(conn)``.
Each method on an ``anosql.Queries`` object accepts a database connection to use in communicating with the database.
.. code-block:: sql
-- name: get-all-greetings
-- Get all the greetings in the database
select greeting_id, greeting from greetings;
-- name: get-worlds-by-name
-- Get all the world record from the database.
select world_id,
world_name,
location
from worlds
where world_name = :world_name;
By specifying ``db_driver="sqlite3"`` we can use the Python stdlib ``sqlite3`` driver to execute these SQL queries and
get the results. We're also using the ``sqlite3.Row`` type for our records to make it easy to access our data via
their column names rather than as tuple indices.
.. code-block:: python
import sqlite3
import anosql
queries = anosql.from_path("greetings.sql", db_driver="sqlite3")
conn = sqlite3.connect("greetings.db")
conn.row_factory = sqlite3.Row
greetings = queries.get_greetings(conn)
worlds = queries.get_worlds_by_name(conn, world_name="Earth")
# greetings = [
# <Row greeting_id=1, greeting="Hi">,
# <Row greeting_id=2, greeting="Aloha">,
# <Row greeting_id=3, greeting="Hola">
# ]
# worlds = [<Row world_id=1, world_name="Earth">]
for world_row in worlds:
for greeting_row in greetings:
print(f"{greeting_row['greeting']}, {world_row['world_name']}!")
# Hi, Earth!
# Aloha, Earth!
# Hola, Earth!
conn.close()

View file

@ -0,0 +1,142 @@
.. anosql documentation master file, created by
sphinx-quickstart on Mon Jul 25 09:16:20 2016.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to anosql's documentation!
==================================
.. image:: https://badge.fury.io/py/anosql.svg
:target: https://badge.fury.io/py/anosql
:alt: pypi package version
.. image:: http://readthedocs.org/projects/anosql/badge/?version=latest
:target: http://anosql.readthedocs.io/en/latest/?badge=latest
:alt: Documentation status
.. image:: https://travis-ci.org/honza/anosql.svg?branch=master
:target: https://travis-ci.org/honza/anosql
:alt: Travis build status
A Python library for using SQL
Inspired by the excellent `Yesql`_ library by Kris Jenkins. In my mother
tongue, *ano* means *yes*.
If you are on python3.6+ or need ``anosql`` to work with ``asyncio`` based database drivers, see the related project `aiosql <https://github.com/nackjicholson/aiosql>`_.
Installation
------------
::
$ pip install anosql
Usage
-----
Basics
******
Given a ``queries.sql`` file:
.. code-block:: sql
-- name: get-all-greetings
-- Get all the greetings in the database
SELECT * FROM greetings;
We can issue SQL queries, like so:
.. code-block:: python
import anosql
import psycopg2
import sqlite3
# PostgreSQL
conn = psycopg2.connect('...')
queries = anosql.from_path('queries.sql', 'psycopg2')
# Or, Sqlite3...
conn = sqlite3.connect('cool.db')
queries = anosql.from_path('queries.sql', 'sqlite3')
queries.get_all_greetings(conn)
# => [(1, 'Hi')]
queries.get_all_greetings.__doc__
# => Get all the greetings in the database
queries.get_all_greetings.sql
# => SELECT * FROM greetings;
queries.available_queries
# => ['get_all_greetings']
Parameters
**********
Often, you want to change parts of the query dynamically, particularly values in the ``WHERE`` clause.
You can use parameters to do this:
.. code-block:: sql
-- name: get-greetings-for-language
-- Get all the greetings in the database for a given language
SELECT *
FROM greetings
WHERE lang = %s;
And they become positional parameters:
.. code-block:: python
visitor_language = "en"
queries.get_all_greetings_for_language(conn, visitor_language)
Named Parameters
****************
To make queries with many parameters more understandable and maintainable, you can give the parameters names:
.. code-block:: sql
-- name: get-greetings-for-language
-- Get all the greetings in the database for given language and length
SELECT *
FROM greetings
WHERE lang = :lang
AND len(greeting) <= :length_limit;
If you were writing a Postgresql query, you could also format the parameters as ``%s(lang)`` and ``%s(length_limit)``.
Then, call your queries like you would any Python function with named parameters:
.. code-block:: python
visitor_language = "en"
greetings_for_texting = queries.get_all_greetings(conn, lang=visitor_language, length_limit=140)
Contents
--------
.. toctree::
:maxdepth: 2
Getting Started <getting_started>
Defining Queries <defining_queries>
Extending anosql <extending>
Upgrading <upgrading>
API <source/modules>
License
-------
BSD, short and sweet
.. _Yesql: https://github.com/krisajenkins/yesql/

View file

@ -0,0 +1,281 @@
@ECHO OFF
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set BUILDDIR=_build
set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
set I18NSPHINXOPTS=%SPHINXOPTS% .
if NOT "%PAPER%" == "" (
set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
)
if "%1" == "" goto help
if "%1" == "help" (
:help
echo.Please use `make ^<target^>` where ^<target^> is one of
echo. html to make standalone HTML files
echo. dirhtml to make HTML files named index.html in directories
echo. singlehtml to make a single large HTML file
echo. pickle to make pickle files
echo. json to make JSON files
echo. htmlhelp to make HTML files and a HTML help project
echo. qthelp to make HTML files and a qthelp project
echo. devhelp to make HTML files and a Devhelp project
echo. epub to make an epub
echo. epub3 to make an epub3
echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
echo. text to make text files
echo. man to make manual pages
echo. texinfo to make Texinfo files
echo. gettext to make PO message catalogs
echo. changes to make an overview over all changed/added/deprecated items
echo. xml to make Docutils-native XML files
echo. pseudoxml to make pseudoxml-XML files for display purposes
echo. linkcheck to check all external links for integrity
echo. doctest to run all doctests embedded in the documentation if enabled
echo. coverage to run coverage check of the documentation if enabled
echo. dummy to check syntax errors of document sources
goto end
)
if "%1" == "clean" (
for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
del /q /s %BUILDDIR%\*
goto end
)
REM Check if sphinx-build is available and fallback to Python version if any
%SPHINXBUILD% 1>NUL 2>NUL
if errorlevel 9009 goto sphinx_python
goto sphinx_ok
:sphinx_python
set SPHINXBUILD=python -m sphinx.__init__
%SPHINXBUILD% 2> nul
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
:sphinx_ok
if "%1" == "html" (
%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/html.
goto end
)
if "%1" == "dirhtml" (
%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
goto end
)
if "%1" == "singlehtml" (
%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
goto end
)
if "%1" == "pickle" (
%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the pickle files.
goto end
)
if "%1" == "json" (
%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the JSON files.
goto end
)
if "%1" == "htmlhelp" (
%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run HTML Help Workshop with the ^
.hhp project file in %BUILDDIR%/htmlhelp.
goto end
)
if "%1" == "qthelp" (
%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run "qcollectiongenerator" with the ^
.qhcp project file in %BUILDDIR%/qthelp, like this:
echo.^> qcollectiongenerator %BUILDDIR%\qthelp\anosql.qhcp
echo.To view the help file:
echo.^> assistant -collectionFile %BUILDDIR%\qthelp\anosql.ghc
goto end
)
if "%1" == "devhelp" (
%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished.
goto end
)
if "%1" == "epub" (
%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The epub file is in %BUILDDIR%/epub.
goto end
)
if "%1" == "epub3" (
%SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The epub3 file is in %BUILDDIR%/epub3.
goto end
)
if "%1" == "latex" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
if errorlevel 1 exit /b 1
echo.
echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "latexpdf" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
cd %BUILDDIR%/latex
make all-pdf
cd %~dp0
echo.
echo.Build finished; the PDF files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "latexpdfja" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
cd %BUILDDIR%/latex
make all-pdf-ja
cd %~dp0
echo.
echo.Build finished; the PDF files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "text" (
%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The text files are in %BUILDDIR%/text.
goto end
)
if "%1" == "man" (
%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The manual pages are in %BUILDDIR%/man.
goto end
)
if "%1" == "texinfo" (
%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
goto end
)
if "%1" == "gettext" (
%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
goto end
)
if "%1" == "changes" (
%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
if errorlevel 1 exit /b 1
echo.
echo.The overview file is in %BUILDDIR%/changes.
goto end
)
if "%1" == "linkcheck" (
%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
if errorlevel 1 exit /b 1
echo.
echo.Link check complete; look for any errors in the above output ^
or in %BUILDDIR%/linkcheck/output.txt.
goto end
)
if "%1" == "doctest" (
%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
if errorlevel 1 exit /b 1
echo.
echo.Testing of doctests in the sources finished, look at the ^
results in %BUILDDIR%/doctest/output.txt.
goto end
)
if "%1" == "coverage" (
%SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage
if errorlevel 1 exit /b 1
echo.
echo.Testing of coverage in the sources finished, look at the ^
results in %BUILDDIR%/coverage/python.txt.
goto end
)
if "%1" == "xml" (
%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The XML files are in %BUILDDIR%/xml.
goto end
)
if "%1" == "pseudoxml" (
%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
goto end
)
if "%1" == "dummy" (
%SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy
if errorlevel 1 exit /b 1
echo.
echo.Build finished. Dummy builder generates no files.
goto end
)
:end

View file

@ -0,0 +1,7 @@
anosql.adapters.psycopg2 module
===============================
.. automodule:: anosql.adapters.psycopg2
:members:
:undoc-members:
:show-inheritance:

View file

@ -0,0 +1,18 @@
anosql.adapters package
=======================
Submodules
----------
.. toctree::
anosql.adapters.psycopg2
anosql.adapters.sqlite3
Module contents
---------------
.. automodule:: anosql.adapters
:members:
:undoc-members:
:show-inheritance:

View file

@ -0,0 +1,7 @@
anosql.adapters.sqlite3 module
==============================
.. automodule:: anosql.adapters.sqlite3
:members:
:undoc-members:
:show-inheritance:

View file

@ -0,0 +1,7 @@
anosql.core module
==================
.. automodule:: anosql.core
:members:
:undoc-members:
:show-inheritance:

View file

@ -0,0 +1,7 @@
anosql.exceptions module
========================
.. automodule:: anosql.exceptions
:members:
:undoc-members:
:show-inheritance:

View file

@ -0,0 +1,7 @@
anosql.patterns module
======================
.. automodule:: anosql.patterns
:members:
:undoc-members:
:show-inheritance:

View file

@ -0,0 +1,26 @@
anosql package
==============
Subpackages
-----------
.. toctree::
anosql.adapters
Submodules
----------
.. toctree::
anosql.core
anosql.exceptions
anosql.patterns
Module contents
---------------
.. automodule:: anosql
:members:
:undoc-members:
:show-inheritance:

View file

@ -0,0 +1,7 @@
anosql
======
.. toctree::
:maxdepth: 4
anosql

View file

@ -0,0 +1,83 @@
#########
Upgrading
#########
Upgrading from 0.x to 1.x
=========================
Changed ``load_queries`` and ``load_queries_from_string``
---------------------------------------------------------
These methods were changed, mostly for brevity. To load ``anosql`` queries, you should now use
the ``anosql.from_str`` to load queries from a SQL string, and ``anosql.from_path`` to load queries
from a SQL file, or directory of SQL files.
Removed the ``$`` "record" operator
-----------------------------------
Because most database drivers have more efficient, robust, and featureful ways of controlling the
rows and records output, this feature was removed.
See:
* `sqlite.Row <https://docs.python.org/2/library/sqlite3.html#sqlite3.Row>`_
* `psycopg2 - Connection and Cursor subclasses <http://initd.org/psycopg/docs/extras.html#connection-and-cursor-subclasses>`_
SQLite example::
conn = sqlite3.connect("...")
conn.row_factory = sqlite3.Row
actual = queries.get_all_users(conn)
assert actual[0]["userid"] == 1
assert actual[0]["username"] == "bobsmith"
assert actual[0][2] == "Bob"
assert actual[0]["lastname" == "Smith"
PostgreSQL example::
with psycopg2.connect("...", cursor_factory=psycopg2.extras.RealDictCursor) as conn:
actual = queries.get_all_users(conn)
assert actual[0] == {
"userid": 1,
"username": "bobsmith",
"firstname": "Bob",
"lastname": "Smith",
}
Driver Adapter classes instead of QueryLoader
---------------------------------------------
I'm not aware of anyone who actually has made or distributed an extension for ``anosql``, as it was
only available in its current form for a few weeks. So this notice is really just for completeness.
For ``0.3.x`` versions of ``anosql`` in order to add a new database extensions you had to build a
subclass of ``anosql.QueryLoader``. This base class is no longer available, and driver adapters no
longer have to extend from any class at all. They are duck-typed classes which are expected to
adhere to a standard interface. For more information about this see :ref:`Extending anosql <extending-anosql>`.
New Things
==========
Use the database driver ``cursor`` directly
-------------------------------------------
All the queries with a `SELECT` type have a duplicate method suffixed by `_cursor` which is a context manager to the database cursor. So `get_all_blogs(conn)` can also be used as:
::
rows = queries.get_all_blogs(conn)
# [(1, "My Blog", "yadayada"), ...]
with queries.get_all_blogs_cursor(conn) as cur:
# All the power of the underlying cursor object! Not limited to just a list of rows.
for row in cur:
print(row)
New operator types for runnings scripts ``#`` and bulk-inserts ``*!``
---------------------------------------------------------------------
See :ref:`Query Operations <query-operations>`

View file

@ -0,0 +1,18 @@
from .core import (
from_path,
from_str,
SQLOperationType,
)
from .exceptions import (
SQLLoadException,
SQLParseException,
)
__all__ = [
"from_path",
"from_str",
"SQLOperationType",
"SQLLoadException",
"SQLParseException",
]

View file

@ -0,0 +1,61 @@
from contextlib import contextmanager
from ..patterns import var_pattern
def replacer(match):
gd = match.groupdict()
if gd["dblquote"] is not None:
return gd["dblquote"]
elif gd["quote"] is not None:
return gd["quote"]
else:
return "{lead}%({var_name})s{trail}".format(
lead=gd["lead"],
var_name=gd["var_name"],
trail=gd["trail"],
)
class PsycoPG2Adapter(object):
@staticmethod
def process_sql(_query_name, _op_type, sql):
return var_pattern.sub(replacer, sql)
@staticmethod
def select(conn, _query_name, sql, parameters):
with conn.cursor() as cur:
cur.execute(sql, parameters)
return cur.fetchall()
@staticmethod
@contextmanager
def select_cursor(conn, _query_name, sql, parameters):
with conn.cursor() as cur:
cur.execute(sql, parameters)
yield cur
@staticmethod
def insert_update_delete(conn, _query_name, sql, parameters):
with conn.cursor() as cur:
cur.execute(sql, parameters)
@staticmethod
def insert_update_delete_many(conn, _query_name, sql, parameters):
with conn.cursor() as cur:
cur.executemany(sql, parameters)
@staticmethod
def insert_returning(conn, _query_name, sql, parameters):
with conn.cursor() as cur:
cur.execute(sql, parameters)
res = cur.fetchone()
if res:
return res[0] if len(res) == 1 else res
else:
return None
@staticmethod
def execute_script(conn, sql):
with conn.cursor() as cur:
cur.execute(sql)

View file

@ -0,0 +1,96 @@
"""
A driver object implementing support for SQLite3
"""
from contextlib import contextmanager
import logging
import re
import sqlite3
log = logging.getLogger(__name__)
class SQLite3DriverAdapter(object):
@staticmethod
def process_sql(_query_name, _op_type, sql):
"""Munge queries.
Args:
_query_name (str): The name of the sql query.
_op_type (anosql.SQLOperationType): The type of SQL operation performed by the sql.
sql (str): The sql as written before processing.
Returns:
str: A normalized form of the query suitable to logging or copy/paste.
"""
# Normalize out comments
sql = re.sub(r"-{2,}.*?\n", "", sql)
# Normalize out a variety of syntactically irrelevant whitespace
#
# FIXME: This is technically invalid, because what if you had `foo ` as
# a table name. Shit idea, but this won't handle it correctly.
sql = re.sub(r"\s+", " ", sql)
sql = re.sub(r"\(\s+", "(", sql)
sql = re.sub(r"\s+\)", ")", sql)
sql = re.sub(r"\s+,", ",", sql)
sql = re.sub(r"\s+;", ";", sql)
return sql
@staticmethod
def select(conn, _query_name, sql, parameters):
cur = conn.cursor()
log.debug({"sql": sql, "parameters": parameters})
cur.execute(sql, parameters)
results = cur.fetchall()
cur.close()
return results
@staticmethod
@contextmanager
def select_cursor(conn: sqlite3.Connection, _query_name, sql, parameters):
cur = conn.cursor()
log.debug({"sql": sql, "parameters": parameters})
cur.execute(sql, parameters)
try:
yield cur
finally:
cur.close()
@staticmethod
def insert_update_delete(conn: sqlite3.Connection, _query_name, sql, parameters):
log.debug({"sql": sql, "parameters": parameters})
conn.execute(sql, parameters)
@staticmethod
def insert_update_delete_many(
conn: sqlite3.Connection, _query_name, sql, parameters
):
log.debug({"sql": sql, "parameters": parameters})
conn.executemany(sql, parameters)
@staticmethod
def insert_returning(conn: sqlite3.Connection, _query_name, sql, parameters):
cur = conn.cursor()
log.debug({"sql": sql, "parameters": parameters})
cur.execute(sql, parameters)
if "returning" not in sql.lower():
# Original behavior - return the last row ID
results = cur.lastrowid
else:
# New behavior - honor a `RETURNING` clause
results = cur.fetchall()
log.debug({"results": results})
cur.close()
return results
@staticmethod
def execute_script(conn: sqlite3.Connection, sql):
log.debug({"sql": sql, "parameters": None})
conn.executescript(sql)

View file

@ -0,0 +1,362 @@
import os
from .adapters.psycopg2 import PsycoPG2Adapter
from .adapters.sqlite3 import SQLite3DriverAdapter
from .exceptions import (
SQLLoadException,
SQLParseException,
)
from .patterns import (
doc_comment_pattern,
empty_pattern,
query_name_definition_pattern,
valid_query_name_pattern,
)
_ADAPTERS = {
"psycopg2": PsycoPG2Adapter,
"sqlite3": SQLite3DriverAdapter,
}
def register_driver_adapter(driver_name, driver_adapter):
"""Registers custom driver adapter classes to extend ``anosql`` to to handle additional drivers.
For details on how to create a new driver adapter see :ref:`driver-adapters` documentation.
Args:
driver_name (str): The driver type name.
driver_adapter (callable): Either n class or function which creates an instance of a
driver adapter.
Returns:
None
Examples:
To register a new loader::
class MyDbAdapter():
def process_sql(self, name, op_type, sql):
pass
def select(self, conn, sql, parameters):
pass
@contextmanager
def select_cursor(self, conn, sql, parameters):
pass
def insert_update_delete(self, conn, sql, parameters):
pass
def insert_update_delete_many(self, conn, sql, parameters):
pass
def insert_returning(self, conn, sql, parameters):
pass
def execute_script(self, conn, sql):
pass
anosql.register_driver_adapter("mydb", MyDbAdapter)
If your adapter constructor takes arguments you can register a function which can build
your adapter instance::
def adapter_factory():
return MyDbAdapter("foo", 42)
anosql.register_driver_adapter("mydb", adapter_factory)
"""
_ADAPTERS[driver_name] = driver_adapter
def get_driver_adapter(driver_name):
"""Get the driver adapter instance registered by the ``driver_name``.
Args:
driver_name (str): The database driver name.
Returns:
object: A driver adapter class.
"""
try:
driver_adapter = _ADAPTERS[driver_name]
except KeyError:
raise ValueError("Encountered unregistered driver_name: {}".format(driver_name))
return driver_adapter()
class SQLOperationType(object):
"""Enumeration (kind of) of anosql operation types"""
INSERT_RETURNING = 0
INSERT_UPDATE_DELETE = 1
INSERT_UPDATE_DELETE_MANY = 2
SCRIPT = 3
SELECT = 4
SELECT_ONE_ROW = 5
class Queries:
"""Container object with dynamic methods built from SQL queries.
The ``-- name`` definition comments in the SQL content determine what the dynamic
methods of this class will be named.
@DynamicAttrs
"""
def __init__(self, queries=None):
"""Queries constructor.
Args:
queries (list(tuple)):
"""
if queries is None:
queries = []
self._available_queries = set()
for query_name, fn in queries:
self.add_query(query_name, fn)
@property
def available_queries(self):
"""Returns listing of all the available query methods loaded in this class.
Returns:
list(str): List of dot-separated method accessor names.
"""
return sorted(self._available_queries)
def __repr__(self):
return "Queries(" + self.available_queries.__repr__() + ")"
def add_query(self, query_name, fn):
"""Adds a new dynamic method to this class.
Args:
query_name (str): The method name as found in the SQL content.
fn (function): The loaded query function.
Returns:
"""
setattr(self, query_name, fn)
self._available_queries.add(query_name)
def add_child_queries(self, child_name, child_queries):
"""Adds a Queries object as a property.
Args:
child_name (str): The property name to group the child queries under.
child_queries (Queries): Queries instance to add as sub-queries.
Returns:
None
"""
setattr(self, child_name, child_queries)
for child_query_name in child_queries.available_queries:
self._available_queries.add("{}.{}".format(child_name, child_query_name))
def _create_fns(query_name, docs, op_type, sql, driver_adapter):
def fn(conn, *args, **kwargs):
parameters = kwargs if len(kwargs) > 0 else args
if op_type == SQLOperationType.INSERT_RETURNING:
return driver_adapter.insert_returning(conn, query_name, sql, parameters)
elif op_type == SQLOperationType.INSERT_UPDATE_DELETE:
return driver_adapter.insert_update_delete(
conn, query_name, sql, parameters
)
elif op_type == SQLOperationType.INSERT_UPDATE_DELETE_MANY:
return driver_adapter.insert_update_delete_many(
conn, query_name, sql, *parameters
)
elif op_type == SQLOperationType.SCRIPT:
return driver_adapter.execute_script(conn, sql)
elif op_type == SQLOperationType.SELECT_ONE_ROW:
res = driver_adapter.select(conn, query_name, sql, parameters)
return res[0] if len(res) == 1 else None
elif op_type == SQLOperationType.SELECT:
return driver_adapter.select(conn, query_name, sql, parameters)
else:
raise ValueError("Unknown op_type: {}".format(op_type))
fn.__name__ = query_name
fn.__doc__ = docs
fn.sql = sql
ctx_mgr_method_name = "{}_cursor".format(query_name)
def ctx_mgr(conn, *args, **kwargs):
parameters = kwargs if len(kwargs) > 0 else args
return driver_adapter.select_cursor(conn, query_name, sql, parameters)
ctx_mgr.__name__ = ctx_mgr_method_name
ctx_mgr.__doc__ = docs
ctx_mgr.sql = sql
if op_type == SQLOperationType.SELECT:
return [(query_name, fn), (ctx_mgr_method_name, ctx_mgr)]
return [(query_name, fn)]
def load_methods(sql_text, driver_adapter):
lines = sql_text.strip().splitlines()
query_name = lines[0].replace("-", "_")
if query_name.endswith("<!"):
op_type = SQLOperationType.INSERT_RETURNING
query_name = query_name[:-2]
elif query_name.endswith("*!"):
op_type = SQLOperationType.INSERT_UPDATE_DELETE_MANY
query_name = query_name[:-2]
elif query_name.endswith("!"):
op_type = SQLOperationType.INSERT_UPDATE_DELETE
query_name = query_name[:-1]
elif query_name.endswith("#"):
op_type = SQLOperationType.SCRIPT
query_name = query_name[:-1]
elif query_name.endswith("?"):
op_type = SQLOperationType.SELECT_ONE_ROW
query_name = query_name[:-1]
else:
op_type = SQLOperationType.SELECT
if not valid_query_name_pattern.match(query_name):
raise SQLParseException(
'name must convert to valid python variable, got "{}".'.format(query_name)
)
docs = ""
sql = ""
for line in lines[1:]:
match = doc_comment_pattern.match(line)
if match:
docs += match.group(1) + "\n"
else:
sql += line + "\n"
docs = docs.strip()
sql = driver_adapter.process_sql(query_name, op_type, sql.strip())
return _create_fns(query_name, docs, op_type, sql, driver_adapter)
def load_queries_from_sql(sql, driver_adapter):
queries = []
for query_text in query_name_definition_pattern.split(sql):
if not empty_pattern.match(query_text):
for method_pair in load_methods(query_text, driver_adapter):
queries.append(method_pair)
return queries
def load_queries_from_file(file_path, driver_adapter):
with open(file_path) as fp:
return load_queries_from_sql(fp.read(), driver_adapter)
def load_queries_from_dir_path(dir_path, query_loader):
if not os.path.isdir(dir_path):
raise ValueError("The path {} must be a directory".format(dir_path))
def _recurse_load_queries(path):
queries = Queries()
for item in os.listdir(path):
item_path = os.path.join(path, item)
if os.path.isfile(item_path) and not item.endswith(".sql"):
continue
elif os.path.isfile(item_path) and item.endswith(".sql"):
for name, fn in load_queries_from_file(item_path, query_loader):
queries.add_query(name, fn)
elif os.path.isdir(item_path):
child_queries = _recurse_load_queries(item_path)
queries.add_child_queries(item, child_queries)
else:
# This should be practically unreachable.
raise SQLLoadException(
"The path must be a directory or file, got {}".format(item_path)
)
return queries
return _recurse_load_queries(dir_path)
def from_str(sql, driver_name):
"""Load queries from a SQL string.
Args:
sql (str) A string containing SQL statements and anosql name:
driver_name (str): The database driver to use to load and execute queries.
Returns:
Queries
Example:
Loading queries from a SQL string::
import sqlite3
import anosql
sql_text = \"""
-- name: get-all-greetings
-- Get all the greetings in the database
select * from greetings;
-- name: get-users-by-username
-- Get all the users from the database,
-- and return it as a dict
select * from users where username =:username;
\"""
queries = anosql.from_str(sql_text, db_driver="sqlite3")
queries.get_all_greetings(conn)
queries.get_users_by_username(conn, username="willvaughn")
"""
driver_adapter = get_driver_adapter(driver_name)
return Queries(load_queries_from_sql(sql, driver_adapter))
def from_path(sql_path, driver_name):
"""Load queries from a sql file, or a directory of sql files.
Args:
sql_path (str): Path to a ``.sql`` file or directory containing ``.sql`` files.
driver_name (str): The database driver to use to load and execute queries.
Returns:
Queries
Example:
Loading queries paths::
import sqlite3
import anosql
queries = anosql.from_path("./greetings.sql", driver_name="sqlite3")
queries2 = anosql.from_path("./sql_dir", driver_name="sqlite3")
"""
if not os.path.exists(sql_path):
raise SQLLoadException("File does not exist: {}.".format(sql_path), sql_path)
driver_adapter = get_driver_adapter(driver_name)
if os.path.isdir(sql_path):
return load_queries_from_dir_path(sql_path, driver_adapter)
elif os.path.isfile(sql_path):
return Queries(load_queries_from_file(sql_path, driver_adapter))
else:
raise SQLLoadException(
"The sql_path must be a directory or file, got {}".format(sql_path),
sql_path,
)

View file

@ -0,0 +1,6 @@
class SQLLoadException(Exception):
pass
class SQLParseException(Exception):
pass

View file

@ -0,0 +1,31 @@
import re
query_name_definition_pattern = re.compile(r"--\s*name\s*:\s*")
"""
Pattern: Identifies name definition comments.
"""
empty_pattern = re.compile(r"^\s*$")
"""
Pattern: Identifies empty lines.
"""
valid_query_name_pattern = re.compile(r"\w+")
"""
Pattern: Enforces names are valid python variable names.
"""
doc_comment_pattern = re.compile(r"\s*--\s*(.*)$")
"""
Pattern: Identifies SQL comments.
"""
var_pattern = re.compile(
r'(?P<dblquote>"[^"]+")|'
r"(?P<quote>\'[^\']+\')|"
r"(?P<lead>[^:]):(?P<var_name>[\w-]+)(?P<trail>[^:]?)"
)
"""
Pattern: Identifies variable definitions in SQL code.
"""

View file

@ -0,0 +1,3 @@
1,What I did Today,"I mowed the lawn - washed some clothes - ate a burger.",2017-07-28
3,Testing,Is this thing on?,2018-01-01
1,How to make a pie.,"1. Make crust\n2. Fill\n3. Bake\n4.Eat",2018-11-23
1 1 What I did Today I mowed the lawn - washed some clothes - ate a burger. 2017-07-28
2 3 Testing Is this thing on? 2018-01-01
3 1 How to make a pie. 1. Make crust\n2. Fill\n3. Bake\n4.Eat 2018-11-23

View file

@ -0,0 +1,3 @@
bobsmith,Bob,Smith
johndoe,John,Doe
janedoe,Jane,Doe
1 bobsmith Bob Smith
2 johndoe John Doe
3 janedoe Jane Doe

View file

@ -0,0 +1,26 @@
-- name: publish-blog<!
insert into blogs (
userid,
title,
content,
published
)
values (
:userid,
:title,
:content,
:published
)
-- name: remove-blog!
-- Remove a blog from the database
delete from blogs where blogid = :blogid;
-- name: get-user-blogs
-- Get blogs authored by a user.
select title,
published
from blogs
where userid = :userid
order by published desc;

View file

@ -0,0 +1,41 @@
-- name: pg-get-blogs-published-after
-- Get all blogs by all authors published after the given date.
select title,
username,
to_char(published, 'YYYY-MM-DD HH24:MI') as published
from blogs
join users using(userid)
where published >= :published
order by published desc;
-- name: pg-publish-blog<!
insert into blogs (
userid,
title,
content,
published
)
values (
:userid,
:title,
:content,
:published
)
returning blogid, title;
-- name: pg-bulk-publish*!
-- Insert many blogs at once
insert into blogs (
userid,
title,
content,
published
)
values (
:userid,
:title,
:content,
:published
)

View file

@ -0,0 +1,20 @@
-- name: sqlite-get-blogs-published-after
-- Get all blogs by all authors published after the given date.
select b.title,
u.username,
strftime('%Y-%m-%d %H:%M', b.published) as published
from blogs b
inner join users u on b.userid = u.userid
where b.published >= :published
order by b.published desc;
-- name: sqlite-bulk-publish*!
-- Insert many blogs at once
insert into blogs (
userid,
title,
content,
published
)
values (?, ?, ? , ?);

View file

@ -0,0 +1,11 @@
-- name: get-all
-- Get all user records
select * from users;
-- name: get-all-sorted
-- Get all user records sorted by username
select * from users order by username asc;
-- name: get-one?
-- Get one user based on its id
select username, firstname, lastname from users where userid = %s;

View file

@ -0,0 +1,122 @@
import csv
import os
import sqlite3
import pytest
BLOGDB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "blogdb")
USERS_DATA_PATH = os.path.join(BLOGDB_PATH, "data", "users_data.csv")
BLOGS_DATA_PATH = os.path.join(BLOGDB_PATH, "data", "blogs_data.csv")
def populate_sqlite3_db(db_path):
conn = sqlite3.connect(db_path)
cur = conn.cursor()
cur.executescript(
"""
create table users (
userid integer not null primary key,
username text not null,
firstname integer not null,
lastname text not null
);
create table blogs (
blogid integer not null primary key,
userid integer not null,
title text not null,
content text not null,
published date not null default CURRENT_DATE,
foreign key(userid) references users(userid)
);
"""
)
with open(USERS_DATA_PATH) as fp:
users = list(csv.reader(fp))
cur.executemany(
"""
insert into users (
username,
firstname,
lastname
) values (?, ?, ?);""",
users,
)
with open(BLOGS_DATA_PATH) as fp:
blogs = list(csv.reader(fp))
cur.executemany(
"""
insert into blogs (
userid,
title,
content,
published
) values (?, ?, ?, ?);""",
blogs,
)
conn.commit()
conn.close()
@pytest.fixture()
def sqlite3_db_path(tmpdir):
db_path = os.path.join(tmpdir.strpath, "blogdb.db")
populate_sqlite3_db(db_path)
return db_path
@pytest.fixture()
def sqlite3_conn(sqlite3_db_path):
conn = sqlite3.connect(sqlite3_db_path)
yield conn
conn.close()
@pytest.fixture
def pg_conn(postgresql):
with postgresql:
# Loads data from blogdb fixture data
with postgresql.cursor() as cur:
cur.execute(
"""
create table users (
userid serial not null primary key,
username varchar(32) not null,
firstname varchar(255) not null,
lastname varchar(255) not null
);"""
)
cur.execute(
"""
create table blogs (
blogid serial not null primary key,
userid integer not null references users(userid),
title varchar(255) not null,
content text not null,
published date not null default CURRENT_DATE
);"""
)
with postgresql.cursor() as cur:
with open(USERS_DATA_PATH) as fp:
cur.copy_from(
fp, "users", sep=",", columns=["username", "firstname", "lastname"]
)
with open(BLOGS_DATA_PATH) as fp:
cur.copy_from(
fp,
"blogs",
sep=",",
columns=["userid", "title", "content", "published"],
)
return postgresql
@pytest.fixture()
def pg_dsn(pg_conn):
p = pg_conn.get_dsn_parameters()
return "postgres://{user}@{host}:{port}/{dbname}".format(**p)

View file

@ -0,0 +1,132 @@
from datetime import date
import os
import anosql
import psycopg2
import psycopg2.extras
import pytest
@pytest.fixture()
def queries():
dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "blogdb", "sql")
return anosql.from_path(dir_path, "psycopg2")
def test_record_query(pg_conn, queries):
dsn = pg_conn.get_dsn_parameters()
with psycopg2.connect(cursor_factory=psycopg2.extras.RealDictCursor, **dsn) as conn:
actual = queries.users.get_all(conn)
assert len(actual) == 3
assert actual[0] == {
"userid": 1,
"username": "bobsmith",
"firstname": "Bob",
"lastname": "Smith",
}
def test_parameterized_query(pg_conn, queries):
actual = queries.blogs.get_user_blogs(pg_conn, userid=1)
expected = [
("How to make a pie.", date(2018, 11, 23)),
("What I did Today", date(2017, 7, 28)),
]
assert actual == expected
def test_parameterized_record_query(pg_conn, queries):
dsn = pg_conn.get_dsn_parameters()
with psycopg2.connect(cursor_factory=psycopg2.extras.RealDictCursor, **dsn) as conn:
actual = queries.blogs.pg_get_blogs_published_after(
conn, published=date(2018, 1, 1)
)
expected = [
{
"title": "How to make a pie.",
"username": "bobsmith",
"published": "2018-11-23 00:00",
},
{"title": "Testing", "username": "janedoe", "published": "2018-01-01 00:00"},
]
assert actual == expected
def test_select_cursor_context_manager(pg_conn, queries):
with queries.blogs.get_user_blogs_cursor(pg_conn, userid=1) as cursor:
actual = cursor.fetchall()
expected = [
("How to make a pie.", date(2018, 11, 23)),
("What I did Today", date(2017, 7, 28)),
]
assert actual == expected
def test_insert_returning(pg_conn, queries):
with pg_conn:
blogid, title = queries.blogs.pg_publish_blog(
pg_conn,
userid=2,
title="My first blog",
content="Hello, World!",
published=date(2018, 12, 4),
)
with pg_conn.cursor() as cur:
cur.execute(
"""\
select blogid,
title
from blogs
where blogid = %s;
""",
(blogid,),
)
expected = cur.fetchone()
assert (blogid, title) == expected
def test_delete(pg_conn, queries):
# Removing the "janedoe" blog titled "Testing"
actual = queries.blogs.remove_blog(pg_conn, blogid=2)
assert actual is None
janes_blogs = queries.blogs.get_user_blogs(pg_conn, userid=3)
assert len(janes_blogs) == 0
def test_insert_many(pg_conn, queries):
blogs = [
{
"userid": 2,
"title": "Blog Part 1",
"content": "content - 1",
"published": date(2018, 12, 4),
},
{
"userid": 2,
"title": "Blog Part 2",
"content": "content - 2",
"published": date(2018, 12, 5),
},
{
"userid": 2,
"title": "Blog Part 3",
"content": "content - 3",
"published": date(2018, 12, 6),
},
]
with pg_conn:
actual = queries.blogs.pg_bulk_publish(pg_conn, blogs)
assert actual is None
johns_blogs = queries.blogs.get_user_blogs(pg_conn, userid=2)
assert johns_blogs == [
("Blog Part 3", date(2018, 12, 6)),
("Blog Part 2", date(2018, 12, 5)),
("Blog Part 1", date(2018, 12, 4)),
]

View file

@ -0,0 +1,238 @@
import anosql
import pytest
@pytest.fixture
def sqlite(request):
import sqlite3
sqlconnection = sqlite3.connect(":memory:")
def fin():
"teardown"
print("teardown")
sqlconnection.close()
request.addfinalizer(fin)
return sqlconnection
def test_simple_query(sqlite):
_test_create_insert = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (a, b, c);\n\n"
"-- name: insert-some-value!\n"
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n"
)
q = anosql.from_str(_test_create_insert, "sqlite3")
q.create_some_table(sqlite)
q.insert_some_value(sqlite)
def test_auto_insert_query(sqlite):
_test_create_insert = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (a, b, c);\n\n"
"-- name: insert-some-value<!\n"
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n"
)
q = anosql.from_str(_test_create_insert, "sqlite3")
q.create_some_table(sqlite)
assert q.insert_some_value(sqlite) == 1
assert q.insert_some_value(sqlite) == 2
assert q.insert_some_value(sqlite) == 3
def test_parametrized_insert(sqlite):
_test_create_insert = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (a, b, c);\n\n"
"-- name: insert-some-value!\n"
"INSERT INTO foo (a, b, c) VALUES (?, ?, ?);\n\n"
"-- name: get-all-values\n"
"SELECT * FROM foo;\n"
)
q = anosql.from_str(_test_create_insert, "sqlite3")
q.create_some_table(sqlite)
q.insert_some_value(sqlite, 10, 11, 12)
assert q.get_all_values(sqlite) == [(10, 11, 12)]
def test_parametrized_insert_named(sqlite):
_test_create_insert = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (a, b, c);\n\n"
"-- name: insert-some-value!\n"
"INSERT INTO foo (a, b, c) VALUES (:a, :b, :c);\n\n"
"-- name: get-all-values\n"
"SELECT * FROM foo;\n"
)
q = anosql.from_str(_test_create_insert, "sqlite3")
q.create_some_table(sqlite)
q.insert_some_value(sqlite, c=12, b=11, a=10)
assert q.get_all_values(sqlite) == [(10, 11, 12)]
def test_one_row(sqlite):
_test_one_row = (
"-- name: one-row?\n"
"SELECT 1, 'hello';\n\n"
"-- name: two-rows?\n"
"SELECT 1 UNION SELECT 2;\n"
)
q = anosql.from_str(_test_one_row, "sqlite3")
assert q.one_row(sqlite) == (1, "hello")
assert q.two_rows(sqlite) is None
def test_simple_query_pg(postgresql):
_queries = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
"-- name: insert-some-value!\n"
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3);\n\n"
"-- name: get-all-values\n"
"SELECT a, b, c FROM foo;\n"
)
q = anosql.from_str(_queries, "psycopg2")
q.create_some_table(postgresql)
q.insert_some_value(postgresql)
assert q.get_all_values(postgresql) == [(1, 2, 3)]
def test_auto_insert_query_pg(postgresql):
_queries = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
"-- name: insert-some-value<!\n"
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3) returning id;\n\n"
"-- name: get-all-values\n"
"SELECT a, b, c FROM foo;\n"
)
q = anosql.from_str(_queries, "psycopg2")
q.create_some_table(postgresql)
assert q.insert_some_value(postgresql) == 1
assert q.insert_some_value(postgresql) == 2
def test_parameterized_insert_pg(postgresql):
_queries = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
"-- name: insert-some-value!\n"
"INSERT INTO foo (a, b, c) VALUES (%s, %s, %s);\n\n"
"-- name: get-all-values\n"
"SELECT a, b, c FROM foo;\n"
)
q = anosql.from_str(_queries, "psycopg2")
q.create_some_table(postgresql)
q.insert_some_value(postgresql, 1, 2, 3)
assert q.get_all_values(postgresql) == [(1, 2, 3)]
def test_auto_parameterized_insert_query_pg(postgresql):
_queries = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
"-- name: insert-some-value<!\n"
"INSERT INTO foo (a, b, c) VALUES (%s, %s, %s) returning id;\n\n"
"-- name: get-all-values\n"
"SELECT a, b, c FROM foo;\n"
)
q = anosql.from_str(_queries, "psycopg2")
q.create_some_table(postgresql)
assert q.insert_some_value(postgresql, 1, 2, 3) == 1
assert q.get_all_values(postgresql) == [(1, 2, 3)]
assert q.insert_some_value(postgresql, 1, 2, 3) == 2
def test_parameterized_select_pg(postgresql):
_queries = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
"-- name: insert-some-value!\n"
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3)\n\n"
"-- name: get-all-values\n"
"SELECT a, b, c FROM foo WHERE a = %s;\n"
)
q = anosql.from_str(_queries, "psycopg2")
q.create_some_table(postgresql)
q.insert_some_value(postgresql)
assert q.get_all_values(postgresql, 1) == [(1, 2, 3)]
def test_parameterized_insert_named_pg(postgresql):
_queries = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
"-- name: insert-some-value!\n"
"INSERT INTO foo (a, b, c) VALUES (%(a)s, %(b)s, %(c)s)\n\n"
"-- name: get-all-values\n"
"SELECT a, b, c FROM foo;\n"
)
q = anosql.from_str(_queries, "psycopg2")
q.create_some_table(postgresql)
q.insert_some_value(postgresql, a=1, b=2, c=3)
assert q.get_all_values(postgresql) == [(1, 2, 3)]
def test_parameterized_select_named_pg(postgresql):
_queries = (
"-- name: create-some-table#\n"
"-- testing insertion\n"
"CREATE TABLE foo (id serial primary key, a int, b int, c int);\n\n"
"-- name: insert-some-value!\n"
"INSERT INTO foo (a, b, c) VALUES (1, 2, 3)\n\n"
"-- name: get-all-values\n"
"SELECT a, b, c FROM foo WHERE a = %(a)s;\n"
)
q = anosql.from_str(_queries, "psycopg2")
q.create_some_table(postgresql)
q.insert_some_value(postgresql)
assert q.get_all_values(postgresql, a=1) == [(1, 2, 3)]
def test_without_trailing_semi_colon_pg():
"""Make sure keywords ending queries are recognized even without
semi-colons.
"""
_queries = "-- name: get-by-a\n" "SELECT a, b, c FROM foo WHERE a = :a\n"
q = anosql.from_str(_queries, "psycopg2")
assert q.get_by_a.sql == "SELECT a, b, c FROM foo WHERE a = %(a)s"

View file

@ -0,0 +1,114 @@
import os
import anosql
import pytest
def dict_factory(cursor, row):
d = {}
for idx, col in enumerate(cursor.description):
d[col[0]] = row[idx]
return d
@pytest.fixture()
def queries():
dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "blogdb", "sql")
return anosql.from_path(dir_path, "sqlite3")
def test_record_query(sqlite3_conn, queries):
sqlite3_conn.row_factory = dict_factory
actual = queries.users.get_all(sqlite3_conn)
assert len(actual) == 3
assert actual[0] == {
"userid": 1,
"username": "bobsmith",
"firstname": "Bob",
"lastname": "Smith",
}
def test_parameterized_query(sqlite3_conn, queries):
actual = queries.blogs.get_user_blogs(sqlite3_conn, userid=1)
expected = [
("How to make a pie.", "2018-11-23"),
("What I did Today", "2017-07-28"),
]
assert actual == expected
def test_parameterized_record_query(sqlite3_conn, queries):
sqlite3_conn.row_factory = dict_factory
actual = queries.blogs.sqlite_get_blogs_published_after(
sqlite3_conn, published="2018-01-01"
)
expected = [
{
"title": "How to make a pie.",
"username": "bobsmith",
"published": "2018-11-23 00:00",
},
{"title": "Testing", "username": "janedoe", "published": "2018-01-01 00:00"},
]
assert actual == expected
def test_select_cursor_context_manager(sqlite3_conn, queries):
with queries.blogs.get_user_blogs_cursor(sqlite3_conn, userid=1) as cursor:
actual = cursor.fetchall()
expected = [
("How to make a pie.", "2018-11-23"),
("What I did Today", "2017-07-28"),
]
assert actual == expected
def test_insert_returning(sqlite3_conn, queries):
with sqlite3_conn:
blogid = queries.blogs.publish_blog(
sqlite3_conn,
userid=2,
title="My first blog",
content="Hello, World!",
published="2018-12-04",
)
print(blogid, type(blogid))
cur = sqlite3_conn.cursor()
cur.execute("SELECT `title` FROM `blogs` WHERE `blogid` = ?;", (blogid,))
actual = cur.fetchone()
cur.close()
expected = ("My first blog",)
assert actual == expected
def test_delete(sqlite3_conn, queries):
# Removing the "janedoe" blog titled "Testing"
actual = queries.blogs.remove_blog(sqlite3_conn, blogid=2)
assert actual is None
janes_blogs = queries.blogs.get_user_blogs(sqlite3_conn, userid=3)
assert len(janes_blogs) == 0
def test_insert_many(sqlite3_conn, queries):
blogs = [
(2, "Blog Part 1", "content - 1", "2018-12-04"),
(2, "Blog Part 2", "content - 2", "2018-12-05"),
(2, "Blog Part 3", "content - 3", "2018-12-06"),
]
with sqlite3_conn:
actual = queries.blogs.sqlite_bulk_publish(sqlite3_conn, blogs)
assert actual is None
johns_blogs = queries.blogs.get_user_blogs(sqlite3_conn, userid=2)
assert johns_blogs == [
("Blog Part 3", "2018-12-06"),
("Blog Part 2", "2018-12-05"),
("Blog Part 1", "2018-12-04"),
]

View file

@ -0,0 +1,20 @@
zapp_binary(
name = "hash_copy",
main = "hash_copy.py",
srcs = [
"util.py"
],
)
zapp_binary(
name = "org_photos",
main = "org_photos.py",
srcs = [
"util.py"
],
deps = [
py_requirement("ExifRead"),
py_requirement("yaspin"),
],
shebang = "/usr/bin/env python3"
)

View file

@ -0,0 +1,56 @@
"""
A tree deduplicator and archiver tool.
"""
import argparse
from pathlib import Path
from shutil import copy2 as copyfile
from .util import *
parser = argparse.ArgumentParser()
parser.add_argument("from_dir", type=Path)
parser.add_argument("to_dir", type=Path)
def main():
opts, args = parser.parse_known_args()
for abs_src_path in opts.from_dir.glob("**/*"):
rel_src_path = abs_src_path.relative_to(opts.from_dir)
abs_dest_path = opts.to_dir / rel_src_path
if abs_src_path.is_dir():
print("dir", abs_src_path, "->", abs_dest_path)
abs_dest_path.mkdir(exist_ok=True)
elif abs_src_path.is_file():
print("file", abs_src_path, "->", abs_dest_path)
if not abs_dest_path.exists():
copyfile(abs_src_path, abs_dest_path)
else:
# If you trust mtime, this can go a lot faster
trust_mtime = False
if (
trust_mtime
and abs_dest_path.stat().st_mtime < abs_src_path.stat().st_mtime
):
pass
elif (src_checksum := checksum_path(abs_src_path)) != (
dest_checksum := checksum_path(abs_dest_path)
):
print(
f"file conflict (src {src_checksum}, dest {dest_checksum}), correcting..."
)
copyfile(abs_src_path, abs_dest_path)
abs_src_path.unlink()
if __name__ == "__main__":
main()

288
projects/archiver/notes.md Normal file
View file

@ -0,0 +1,288 @@
# EXIF tags dataset (exifread edition)
- 'EXIF ApertureValue'
- 'EXIF BodySerialNumber'
- 'EXIF BrightnessValue'
- 'EXIF CVAPattern'
- 'EXIF CameraOwnerName'
- 'EXIF ColorSpace'
- 'EXIF ComponentsConfiguration'
- 'EXIF CompressedBitsPerPixel'
- 'EXIF Contrast'
- 'EXIF CustomRendered'
- 'EXIF DateTimeDigitized'
- 'EXIF DateTimeOriginal'
- 'EXIF DeviceSettingDescription'
- 'EXIF DigitalZoomRatio'
- 'EXIF ExifImageLength'
- 'EXIF ExifImageWidth'
- 'EXIF ExifVersion'
- 'EXIF ExposureBiasValue'
- 'EXIF ExposureIndex'
- 'EXIF ExposureMode'
- 'EXIF ExposureProgram'
- 'EXIF ExposureTime'
- 'EXIF FNumber'
- 'EXIF FileSource'
- 'EXIF Flash'
- 'EXIF FlashEnergy'
- 'EXIF FlashPixVersion'
- 'EXIF FocalLength'
- 'EXIF FocalLengthIn35mmFilm'
- 'EXIF FocalPlaneResolutionUnit'
- 'EXIF FocalPlaneXResolution'
- 'EXIF FocalPlaneYResolution'
- 'EXIF GainControl'
- 'EXIF ISOSpeedRatings'
- 'EXIF ImageUniqueID'
- 'EXIF InteroperabilityOffset'
- 'EXIF JPEGInterchangeFormat'
- 'EXIF JPEGInterchangeFormatLength'
- 'EXIF LensMake'
- 'EXIF LensModel'
- 'EXIF LensSerialNumber'
- 'EXIF LensSpecification'
- 'EXIF LightSource'
- 'EXIF MakerNote'
- 'EXIF MaxApertureValue'
- 'EXIF MeteringMode'
- 'EXIF OffsetSchema'
- 'EXIF OffsetTime'
- 'EXIF OffsetTimeDigitized'
- 'EXIF OffsetTimeOriginal'
- 'EXIF Padding'
- 'EXIF RecommendedExposureIndex'
- 'EXIF Saturation'
- 'EXIF SceneCaptureType'
- 'EXIF SceneType'
- 'EXIF SensingMethod'
- 'EXIF SensitivityType'
- 'EXIF Sharpness'
- 'EXIF ShutterSpeedValue'
- 'EXIF SubSecTime'
- 'EXIF SubSecTimeDigitized'
- 'EXIF SubSecTimeOriginal'
- 'EXIF SubjectArea'
- 'EXIF SubjectDistance'
- 'EXIF SubjectDistanceRange'
- 'EXIF UserComment'
- 'EXIF WhiteBalance'
- 'GPS GPSAltitude'
- 'GPS GPSAltitudeRef'
- 'GPS GPSDOP'
- 'GPS GPSDate'
- 'GPS GPSImgDirection'
- 'GPS GPSImgDirectionRef'
- 'GPS GPSLatitude'
- 'GPS GPSLatitudeRef'
- 'GPS GPSLongitude'
- 'GPS GPSLongitudeRef'
- 'GPS GPSMapDatum'
- 'GPS GPSMeasureMode'
- 'GPS GPSProcessingMethod'
- 'GPS GPSTimeStamp'
- 'GPS GPSVersionID'
- 'GPS Tag 0xEA1C'
- 'Image Artist'
- 'Image BitsPerSample'
- 'Image Copyright'
- 'Image DateTime'
- 'Image DateTimeDigitized'
- 'Image ExifOffset'
- 'Image ExposureMode'
- 'Image ExposureProgram'
- 'Image ExposureTime'
- 'Image FNumber'
- 'Image Flash'
- 'Image FocalLength'
- 'Image GPSInfo'
- 'Image ISOSpeedRatings'
- 'Image ImageDescription'
- 'Image ImageLength'
- 'Image ImageWidth'
- 'Image JPEGInterchangeFormat'
- 'Image JPEGInterchangeFormatLength'
- 'Image LightSource'
- 'Image Make'
- 'Image MeteringMode'
- 'Image Model'
- 'Image Orientation'
- 'Image Padding'
- 'Image PhotometricInterpretation'
- 'Image PrintIM'
- 'Image ResolutionUnit'
- 'Image SamplesPerPixel'
- 'Image Software'
- 'Image UserComment'
- 'Image WhiteBalance'
- 'Image XPComment'
- 'Image XPKeywords'
- 'Image XPTitle'
- 'Image XResolution'
- 'Image YCbCrPositioning'
- 'Image YResolution'
- 'Interoperability InteroperabilityIndex'
- 'Interoperability InteroperabilityVersion'
- 'JPEGThumbnail'
- 'MakerNote AEBracketCompensationApplied'
- 'MakerNote AESetting'
- 'MakerNote AFAreaMode'
- 'MakerNote AFInfo2'
- 'MakerNote AFPointSelected'
- 'MakerNote AFPointUsed'
- 'MakerNote ActiveDLighting'
- 'MakerNote AspectInfo'
- 'MakerNote AutoBracketRelease'
- 'MakerNote AutoFlashMode'
- 'MakerNote BracketMode'
- 'MakerNote BracketShotNumber'
- 'MakerNote BracketValue'
- 'MakerNote BracketingMode'
- 'MakerNote CanonImageWidth'
- 'MakerNote ColorBalance'
- 'MakerNote ColorSpace'
- 'MakerNote ContinuousDriveMode'
- 'MakerNote Contrast'
- 'MakerNote CropHiSpeed'
- 'MakerNote CropInfo'
- 'MakerNote DigitalVariProgram'
- 'MakerNote DigitalZoom'
- 'MakerNote DustRemovalData'
- 'MakerNote EasyShootingMode'
- 'MakerNote ExposureDifference'
- 'MakerNote ExposureMode'
- 'MakerNote ExposureTuning'
- 'MakerNote ExternalFlashExposureComp'
- 'MakerNote FileInfo'
- 'MakerNote FileNumber'
- 'MakerNote FilterEffect'
- 'MakerNote FirmwareVersion'
- 'MakerNote FlashActivity'
- 'MakerNote FlashBias'
- 'MakerNote FlashBracketCompensationApplied'
- 'MakerNote FlashCompensation'
- 'MakerNote FlashDetails'
- 'MakerNote FlashExposureLock'
- 'MakerNote FlashInfo'
- 'MakerNote FlashMode'
- 'MakerNote FlashSetting'
- 'MakerNote FocalLength'
- 'MakerNote FocalType'
- 'MakerNote FocalUnitsPerMM'
- 'MakerNote FocusMode'
- 'MakerNote FocusType'
- 'MakerNote HDRImageType'
- 'MakerNote HighISONoiseReduction'
- 'MakerNote ISO'
- 'MakerNote ISOInfo'
- 'MakerNote ISOSetting'
- 'MakerNote ISOSpeedRequested'
- 'MakerNote ImageDataSize'
- 'MakerNote ImageSize'
- 'MakerNote ImageStabilization'
- 'MakerNote ImageType'
- 'MakerNote InternalSerialNumber'
- 'MakerNote LensData'
- 'MakerNote LensFStops'
- 'MakerNote LensMinMaxFocalMaxAperture'
- 'MakerNote LensModel'
- 'MakerNote LensType'
- 'MakerNote LiveViewShooting'
- 'MakerNote LongExposureNoiseReduction2'
- 'MakerNote LongFocalLengthOfLensInFocalUnits'
- 'MakerNote MacroMagnification'
- 'MakerNote Macromode'
- 'MakerNote MakernoteVersion'
- 'MakerNote ManualFlashOutput'
- 'MakerNote MeteringMode'
- 'MakerNote ModelID'
- 'MakerNote MultiExposure'
- 'MakerNote NikonPreview'
- 'MakerNote NoiseReduction'
- 'MakerNote NumAFPoints'
- 'MakerNote OwnerName'
- 'MakerNote PhotoCornerCoordinates'
- 'MakerNote PictureControl'
- 'MakerNote PowerUpTime'
- 'MakerNote ProgramShift'
- 'MakerNote Quality'
- 'MakerNote RawJpgQuality'
- 'MakerNote RawJpgSize'
- 'MakerNote RecordMode'
- 'MakerNote RetouchHistory'
- 'MakerNote Saturation'
- 'MakerNote SelfTimer'
- 'MakerNote SequenceNumber'
- 'MakerNote SerialNumber'
- 'MakerNote Sharpness'
- 'MakerNote ShortFocalLengthOfLensInFocalUnits'
- 'MakerNote ShotInfo'
- 'MakerNote SlowShutter'
- 'MakerNote SpotMeteringMode'
- 'MakerNote SubjectDistance'
- 'MakerNote Tag 0x0001'
- 'MakerNote Tag 0x0002'
- 'MakerNote Tag 0x0003'
- 'MakerNote Tag 0x0004'
- 'MakerNote Tag 0x0005'
- 'MakerNote Tag 0x0006'
- 'MakerNote Tag 0x0007'
- 'MakerNote Tag 0x0008'
- 'MakerNote Tag 0x0009'
- 'MakerNote Tag 0x000E'
- 'MakerNote Tag 0x0014'
- 'MakerNote Tag 0x0015'
- 'MakerNote Tag 0x0019'
- 'MakerNote Tag 0x002B'
- 'MakerNote Tag 0x002C'
- 'MakerNote Tag 0x002D'
- 'MakerNote Tag 0x0083'
- 'MakerNote Tag 0x0099'
- 'MakerNote Tag 0x009D'
- 'MakerNote Tag 0x00A0'
- 'MakerNote Tag 0x00A3'
- 'MakerNote Tag 0x00AA'
- 'MakerNote Tag 0x00BB'
- 'MakerNote Tag 0x00D0'
- 'MakerNote Tag 0x00E0'
- 'MakerNote Tag 0x4001'
- 'MakerNote Tag 0x4008'
- 'MakerNote Tag 0x4009'
- 'MakerNote Tag 0x4010'
- 'MakerNote Tag 0x4011'
- 'MakerNote Tag 0x4012'
- 'MakerNote Tag 0x4015'
- 'MakerNote Tag 0x4016'
- 'MakerNote Tag 0x4017'
- 'MakerNote Tag 0x4018'
- 'MakerNote Tag 0x4019'
- 'MakerNote Tag 0x4020'
- 'MakerNote ThumbnailImageValidArea'
- 'MakerNote ToningEffect'
- 'MakerNote TotalShutterReleases'
- 'MakerNote Unknown'
- 'MakerNote VRInfo'
- 'MakerNote ValidAFPoints'
- 'MakerNote WBBracketMode'
- 'MakerNote WBBracketValueAB'
- 'MakerNote WBBracketValueGM'
- 'MakerNote WhiteBalance'
- 'MakerNote WhiteBalanceBias'
- 'MakerNote WhiteBalanceRBCoeff'
- 'MakerNote Whitebalance'
- 'MakerNote WorldTime'
- 'Thumbnail Compression'
- 'Thumbnail DateTime'
- 'Thumbnail ImageDescription'
- 'Thumbnail ImageLength'
- 'Thumbnail ImageWidth'
- 'Thumbnail JPEGInterchangeFormat'
- 'Thumbnail JPEGInterchangeFormatLength'
- 'Thumbnail Make'
- 'Thumbnail Model'
- 'Thumbnail Orientation'
- 'Thumbnail ResolutionUnit'
- 'Thumbnail Software'
- 'Thumbnail XResolution'
- 'Thumbnail YCbCrPositioning'
- 'Thumbnail YResolution'

View file

@ -0,0 +1,454 @@
"""Organize (and annotate) photos predictably.
This script is designed to eat a Dropbox style "camera uploads" directory containing many years of photos with various
metadata (or data only in the file name) and stomp that all down to something consistent by doing EXIF parsing and
falling back to filename parsing if need be.
Note that this script does NOT do EXIF rewriting, although it probably should.
Generates an output tree of the format -
{year}/v1_{iso_date}_{camera_make}_{camera_model}_{device_fingerprint}.{ext}
The idea being that chronological-ish order is the default way users will want to view (or process) photos.
Note that this means if you do interleaved shooting between devices, it should Just Work (TM).
Inspired by https://github.com/herval/org_photos/blob/main/org_photos.rb
"""
import argparse
from datetime import datetime, timedelta
from hashlib import sha256, sha512
import os
from pathlib import Path
import re
from shutil import copy2 as copyfile
import stat
import sys
import typing as t
from .util import *
# FIXME: use piexif, which supports writeback not exifread.
import exifread
from yaspin import Spinner, yaspin
_print = print
def print(*strs, **kwargs):
_print("\r", *strs, **kwargs)
parser = argparse.ArgumentParser()
parser.add_argument("src_dir", type=Path)
parser.add_argument("dest_dir", type=Path)
parser.add_argument("destructive", action="store_true", default=False)
MODIFIED_ISO_DATE = "%Y:%m:%dT%H:%M:%SF%f"
SPINNER = Spinner(["|", "/", "-", "\\"], 200)
KNOWN_IMG_TYPES = {
".jpg": ".jpeg",
".jpeg": ".jpeg",
".png": ".png",
".mov": ".mov",
".gif": ".gif",
".mp4": ".mp4",
".m4a": ".m4a",
".oga": ".oga", # How the hell do I have ogg files kicking around
}
def exif_tags(p: Path) -> object:
"""Return the EXIF tags on an image."""
with open(p, "rb") as fp:
return exifread.process_file(fp)
def sanitize(s: str) -> str:
"""Something like b64encode; sanitize a string to a path-friendly version."""
to_del = [" ", ";", ":", "_", "-", "/", "\\", "."]
s = s.lower()
for c in to_del:
s = s.replace(c, "")
return s
def safe_strptime(date, format):
try:
return datetime.strptime(date, format)
except ValueError:
pass
def safe_ymdhmms(date):
fmt = (
r"(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})"
r" "
r"(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2})(?P<millisecond>\d{3})"
)
m = re.match(fmt, date)
if m:
return datetime(
year=int(m.group("year")),
month=int(m.group("month")),
day=int(m.group("day")),
hour=int(m.group("hour")),
minute=int(m.group("minute")),
second=int(m.group("second")),
microsecond=int(m.group("millisecond")) * 1000,
)
def date_from_name(fname: str):
# Discard common prefixes
fname = fname.replace("IMG_", "")
fname = fname.replace("PXL_", "")
fname = fname.replace("VID_", "")
if fname.startswith("v1_"):
# The date segment of the v1 file name
fname = fname.split("_")[1]
# Guess if we have a v0 file name, which neglected the prefix
elif fname.count("_") == 3:
# A bug
fname = fname.split("_")[-1].replace("rc", "")[:25]
# We have some other garbage, make our date parsing life easier.
else:
# A couple of date formats use _ as field separators, consistently choice " " instead so that we can write fewer
# date patterns and be more correct.
fname = fname.replace("_", " ")
fname = re.sub(
r"(-\d+)(-\d+)*$", r"\1", fname
) # deal with -1-2 etc. crap from Dropbox
# Try to guess the date
# File date formats:
for unfmt in [
# Our date format
lambda d: safe_strptime(d, MODIFIED_ISO_DATE),
# A bug
# 2014:08:21T19:4640F1408672000
# 2015:12:14T23:0933F1450159773
lambda d: safe_strptime(d, "%Y:%m:%dT%H:%M%SF%f"),
# 2020-12-21 17.15.09.0
lambda d: safe_strptime(d, "%Y-%m-%d %H.%M.%S.%f"),
# 2020-12-21 17.15.09
lambda d: safe_strptime(d, "%Y-%m-%d %H.%M.%S"),
# 2019-02-09 12.45.32-6
# 2019-01-13 13.43.45-16
lambda d: safe_strptime(d, "%Y-%m-%d %H.%M.%S-%f"),
# Note the _1 or such may not be millis, but we assume it is.
# 20171113_130826_1
# 20171113 130826 1
lambda d: safe_strptime(d, "%Y%m%d %H%M%S %f"),
# 20180404_114639
# 20180404 114639
lambda d: safe_strptime(d, "%Y%m%d %H%M%S"),
# 2017-11-05_15:15:55
# 2017-11-05 15:15:55
lambda d: safe_strptime(d, "%Y-%m-%d %H:%M:%S"),
lambda d: safe_strptime(d, "%Y%m%d %h%m%s%f"),
# From the dashcam(s)
# 20220427_211616_00023F
# 20220510_213347_04187R
lambda d: safe_strptime(d, "%Y%m%d %h%m%s %fF"),
lambda d: safe_strptime(d, "%Y%m%d %h%m%s %fR"),
# HACK:
# Python doesn't support %s as milliseconds; these don't quite work.
# So use a custom matcher.
# 20210526 002327780
# 20210417_220753284
# 20210417 220753284
# 20210304 204755545
# 20211111 224304117
safe_ymdhmms,
]:
val = unfmt(fname)
if val is not None:
return val
def date_from_path(p: Path):
"""Try to munge a datestamp out of a path."""
fname = ".".join(p.name.split(".")[:-1])
date = date_from_name(fname)
if not date:
print(f"Warning: Unable to infer datetime from {fname!r}", file=sys.stderr)
return date
def normalize_ext(p: Path):
renaming = KNOWN_IMG_TYPES
exts = [e.lower() for e in p.suffixes]
# Guess an ext out of potentially many, allowing only for folding of effective dupes
exts = set(renaming[e] for e in exts if e in renaming)
assert len(exts) == 1
return list(exts)[0]
class ImgInfo(t.NamedTuple):
file_path: Path
tags: dict
camera_make: str
camera_model: str
camera_sn: str
lens_make: str
lens_model: str
lens_sn: str
software: str
date: datetime
# A dirty bit, indicating whether the info needs to be written back
dirty: bool = False
shasum_prefix: int = 9
def device_fingerprint(self):
"""Compute a stable 'fingerprint' for the device that took the shot."""
return checksum_list(
[
self.camera_make,
self.camera_model,
self.camera_sn,
self.lens_make,
self.lens_model,
self.lens_sn,
self.software,
]
)[: self.shasum_prefix]
def file_fingerprint(self):
"""Compute a 'fingerprint' for the file itself.
Note that this hash DOES include EXIF data, and is not stable.
"""
return self.file_sha256sum()[: self.shasum_prefix]
def file_sha256sum(self):
return checksum_path(self.file_path, sha256)
def file_sha512sum(self):
return checksum_path(self.file_path, sha512)
def incr(self, offset: int) -> "ImgInfo":
return ImgInfo(
self.file_path,
self.tags,
self.camera_make,
self.camera_model,
self.camera_sn,
self.lens_make,
self.lens_model,
self.lens_sn,
self.software,
self.date + timedelta(microseconds=offset),
True,
)
def img_info(p: Path) -> ImgInfo:
"""Figure out everything we know from the image info."""
tags = exif_tags(p)
def get_tag(tag, default=None):
if v := tags.get(tag):
if isinstance(v.values, list):
return v.values[0]
elif isinstance(v.values, str):
return v.values
else:
raise ValueError(f"Don't know how to simplify {v!r}")
else:
return default
## Camera data
camera_make = get_tag("Image Make", "Unknown")
camera_model = get_tag("Image Model", "Unknown")
camera_sn = get_tag("MakerNote SerialNumber", "Unknown")
lens_make = get_tag("EXIF LensMake", "Unknown")
lens_model = get_tag("EXIF LensModel", "Unknown")
lens_sn = get_tag("EXIF LensSerialNumber", "Unknown")
software = get_tag("Image Software", "Unknown")
dirty = False
# Fixup magic for "correcting" metadata where possible
# FIXME: This could be a postprocessing pass
if camera_make == "Unknown" and p.name.lower().startswith("dji"):
# Magic constants from found examples
camera_make = "DJI"
camera_model = "FC220"
software = "v02.09.514"
dirty |= True
elif camera_make == "Unknown" and p.name.startswith("PXL_"):
camera_make = "Google"
camera_model = "Pixel"
dirty |= True
## Timestamp data
stat = p.stat()
# 2019:03:31 15:59:26
date = (
get_tag("Image DateTime")
or get_tag("EXIF DateTimeOriginal")
or get_tag("EXIF DateTimeDigitized")
)
if date and (date := safe_strptime(date, "%Y:%m:%d %H:%M:%S")):
pass
elif date := date_from_path(p):
dirty |= True
else:
# The oldest of the mtime and the ctime
date = datetime.fromtimestamp(min([stat.st_mtime, stat.st_ctime]))
dirty |= True
# 944404
subsec = int(
get_tag("EXIF SubSecTime")
or get_tag("EXIF SubSecTimeOriginal")
or get_tag("EXIF SubSecTimeDigitized")
or "0"
)
# GoPro burst format is G%f.JPG or something close to it
if subsec == 0 and (m := re.match(r"g.*(\d{6}).jpe?g", p.name.lower())):
subsec = int(m.group(1))
date = date.replace(microsecond=subsec)
if not (2015 <= date.year <= datetime.now().year):
raise ValueError(f"{p}'s inferred date ({date!r}) is beyond the sanity-check range!")
return ImgInfo(
p,
tags,
camera_make,
camera_model,
camera_sn,
lens_make,
lens_model,
lens_sn,
software,
date,
dirty,
)
def is_readonly(src: Path) -> bool:
statinfo = os.stat(src, dir_fd=None, follow_symlinks=True)
if getattr(statinfo, "st_flags", 0) & stat.UF_IMMUTABLE:
return True
return False
def main():
opts, args = parser.parse_known_args()
def _delete(src):
if opts.destructive:
if is_readonly(src):
print(f" warning: {src} is read-only, unable to remove")
else:
try:
src.unlink()
print(" unlink: ok")
except PermissionError:
print(" unlink: fail")
def _copy(src, target):
print(f" rename: {target}")
try:
if not opts.destructive:
raise OSError()
src.rename(target) # Execute the rename
except OSError: # cross-device move
with yaspin(SPINNER):
copyfile(src, target)
_delete(src)
print("---")
sequence_name = None
sequence = 0
for src in opts.src_dir.glob("**/*"):
print(f"{src}:")
ext = "." + src.name.lower().split(".")[-1]
print(f" msg: ext inferred as {ext}")
if src.is_dir():
continue
elif src.name.lower().startswith("."):
continue
elif ext in ["thm", "lrv", "ico", "sav"] or src.name.startswith("._"):
_delete(src)
continue
elif ext in KNOWN_IMG_TYPES:
info = img_info(src)
year_dir = Path(opts.dest_dir / str(info.date.year))
year_dir.mkdir(exist_ok=True) # Ignore existing and continue
# Figure out a stable file name
stable_name = f"v1_{info.date.strftime(MODIFIED_ISO_DATE)}_{sanitize(info.camera_make)}_{sanitize(info.camera_model)}_{info.device_fingerprint()}"
# De-conflict using a sequence number added to the sub-seconds field
if sequence_name == stable_name:
sequence += 1
info = info.incr(sequence)
print(f" warning: de-conflicting filenames with sequence {sequence}")
stable_name = f"v1_{info.date.strftime(MODIFIED_ISO_DATE)}_{sanitize(info.camera_make)}_{sanitize(info.camera_model)}_{info.device_fingerprint()}"
else:
sequence = 0
sequence_name = stable_name
try:
ext = normalize_ext(src)
except AssertionError:
continue # Just skip fucked up files
target = Path(year_dir / f"{stable_name}{ext}")
if not target.exists():
# src & !target => copy
_copy(src, target)
elif src == target:
# src == target; skip DO NOT DELETE SRC
pass
elif checksum_path_blocks(src) == checksum_path_blocks(target):
print(f" ok: {target}")
# src != target && id(src) == id(target); delete src
_delete(src)
else:
# src != target && id(src) != id(target); replace target with src?
print(f" warning: {target} is a content-id collision with a different checksum; skipping")
else:
print(f" msg: unknown filetype {ext}")
if __name__ == "__main__":
main()

68
projects/archiver/util.py Normal file
View file

@ -0,0 +1,68 @@
from hashlib import sha256
from pathlib import Path
import typing as t
def iter_chunks(fp):
yield from iter(lambda: fp.read(4096), b"")
def take(n, iter):
"""Take the first N items lazily off of an iterable."""
for _ in range(0, n):
try:
yield next(iter)
except StopIteration:
break
def checksum_list(iter, sum=sha256, salt=b";"):
"""Compute the checksum of a bunch of stuff from an iterable."""
sum = sum()
for i in iter:
if salt:
sum.update(salt) # Merkle tree salting.
if isinstance(i, str):
i = str.encode(i, "utf-8")
try:
sum.update(i)
except Exception as e:
print(i, type(i))
raise e
return sum.hexdigest()
def checksum_path_blocks(p: Path, sum=sha256) -> t.Iterable[str]:
"""Compute block-wise checksums of a file.
Inspired by the Dropbox content-hashing interface -
https://www.dropbox.com/developers/reference/content-hash
"""
def _helper():
with open(p, "rb") as fp:
for chunk in iter_chunks(fp):
digest = sum()
digest.update(chunk)
yield digest.hexdigest()
return list(_helper())
def checksum_path(p: Path, sum=sha256) -> str:
"""Compute 'the' checksum of an entire file.
Note that this does semi-streaming I/O.
"""
sum = sum()
with open(p, "rb") as fp:
for chunk in iter_chunks(fp):
sum.update(chunk)
return sum.hexdigest()

View file

@ -0,0 +1,94 @@
import os
import re
import sys
def main(opts, args):
"""Usage: python rollback.py date
Parse /var/log/pacman.log, enumerating package transactions since the
specified date and building a plan for restoring the state of your system to
what it was at the specified date.
Assumes:
- /var/log/pacman.log has not been truncated
- /var/cache/pacman/pkg has not been flushed and still contains all required
packages
- The above paths are Arch default and have not been customized
- That it is not necessary to remove any "installed" packages
Note: no attempt is made to inspect the dependency graph of packages to be
downgraded to detect when a package is already transitively listed for
downgrading. This can create some annoying errors where eg. systemd will be
downgraded, meaning libsystemd will also be downgraded, but pacman considers
explicitly listing the downgrade of libsystemd when it will already be
transitively downgraded an error.
"""
date, = args
print("Attempting to roll back package state to that of {0}...\n"
.format(date),
file=sys.stderr)
# These patterns can't be collapsed because we want to select different
# version identifying strings depending on which case we're in. Not ideal,
# but it works.
# Ex. [2017-04-01 09:51] [ALPM] upgraded filesystem (2016.12-2 -> 2017.03-2)
upgraded_pattern = re.compile(
".*? upgraded (?P<name>\w+) \((?P<from>[^ ]+) -> (?P<to>[^\)]+)\)")
# Ex: [2018-02-23 21:18] [ALPM] downgraded emacs (25.3-3 -> 25.3-2)
downgraded_pattern = re.compile(
".*? downgraded (?P<name>\w+) \((?P<to>[^ ]+) -> (?P<from>[^\)]+)\)")
# Ex: [2017-03-31 07:05] [ALPM] removed gdm (3.22.3-1)
removed_pattern = re.compile(
".*? removed (?P<name>\w+) \((?P<from>[^ ]+)\)")
checkpoint = {}
flag = False
with open("/var/log/pacman.log") as logfile:
for line in logfile:
if date in line:
flag = True
elif not flag:
continue
match = re.match(upgraded_pattern, line)\
or re.match(downgraded_pattern, line)\
or re.match(removed_pattern, line)
if match:
package = match.group("name")
from_rev = match.group("from")
if package not in checkpoint:
checkpoint[package] = from_rev
continue
print("Checkpoint state:")
for k in checkpoint.keys():
print("{0} -> {1}".format(k, checkpoint[k]), file=sys.stderr)
pkgcache = "/var/cache/pacman/pkg"
pkgs = os.listdir(pkgcache)
pkgnames = ["{0}-{1}".format(k, v) for k, v in checkpoint.items()]
selected_pkgs = set([os.path.join(pkgcache, p)
for n in pkgnames
for p in pkgs
if n in p])
print("Suggested incantation:\n", file=sys.stderr)
print("sudo pacman --noconfirm -U {}"
.format("\\\n ".join(selected_pkgs)))
if __name__ == "__main__":
main(None, sys.argv[1:])

View file

@ -0,0 +1,3 @@
py_project(
name = "async_cache"
)

View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Rajat Singh
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View file

@ -0,0 +1,13 @@
# Async cache
An LRU and TTL cache for async functions in Python.
- `alru_cache` provides an LRU cache decorator with configurable size.
- `attl_cache` provides a TTL+LRU cache decorator with configurable size.
Neither cache proactively expires keys.
Maintenance occurs only when requesting keys out of the cache.
## License
Derived from https://github.com/iamsinghrajat/async-cache, published under the MIT license.

View file

@ -0,0 +1,4 @@
"""The interface to the package. Just re-exports implemented caches."""
from .lru import ALRU, alru_cache # noqa
from .ttl import ATTL, attl_cache # noqa

View file

@ -0,0 +1,23 @@
from typing import Any
class KEY:
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
def __eq__(self, obj):
return hash(self) == hash(obj)
def __hash__(self):
def _hash(param: Any):
if isinstance(param, tuple):
return tuple(map(_hash, param))
if isinstance(param, dict):
return tuple(map(_hash, param.items()))
elif hasattr(param, "__dict__"):
return str(vars(param))
else:
return str(param)
return hash(_hash(self.args) + _hash(self.kwargs))

View file

@ -0,0 +1,44 @@
from collections import OrderedDict
from .key import KEY
class LRU(OrderedDict):
def __init__(self, maxsize, *args, **kwargs):
self.maxsize = maxsize
super().__init__(*args, **kwargs)
def __getitem__(self, key):
value = super().__getitem__(key)
self.move_to_end(key)
return value
def __setitem__(self, key, value):
super().__setitem__(key, value)
if self.maxsize and len(self) > self.maxsize:
oldest = next(iter(self))
del self[oldest]
class ALRU(object):
def __init__(self, maxsize=128):
"""
:param maxsize: Use maxsize as None for unlimited size cache
"""
self.lru = LRU(maxsize=maxsize)
def __call__(self, func):
async def wrapper(*args, **kwargs):
key = KEY(args, kwargs)
if key in self.lru:
return self.lru[key]
else:
self.lru[key] = await func(*args, **kwargs)
return self.lru[key]
wrapper.__name__ += func.__name__
return wrapper
alru_cache = ALRU

View file

@ -0,0 +1,77 @@
import datetime
from typing import Optional, Union
from .key import KEY
from .lru import LRU
class ATTL:
class _TTL(LRU):
def __init__(self, ttl: Optional[Union[datetime.timedelta, int, float]], maxsize: int):
super().__init__(maxsize=maxsize)
if isinstance(ttl, datetime.timedelta):
self.ttl = ttl
elif isinstance(ttl, (int, float)):
self.ttl = datetime.timedelta(seconds=ttl)
elif ttl is None:
self.ttl = None
else:
raise ValueError("TTL must be int or timedelta")
self.maxsize = maxsize
def __contains__(self, key):
if key not in self.keys():
return False
else:
key_expiration = super().__getitem__(key)[1]
if key_expiration and key_expiration < datetime.datetime.now():
del self[key]
return False
else:
return True
def __getitem__(self, key):
if key in self:
value = super().__getitem__(key)[0]
return value
raise KeyError
def __setitem__(self, key, value):
ttl_value = (
datetime.datetime.now() + self.ttl
) if self.ttl else None
super().__setitem__(key, (value, ttl_value))
def __init__(
self,
ttl: Optional[Union[datetime.timedelta, int]] = datetime.timedelta(seconds=60),
maxsize: int = 1024,
skip_args: int = 0
):
"""
:param ttl: Use ttl as None for non expiring cache
:param maxsize: Use maxsize as None for unlimited size cache
:param skip_args: Use `1` to skip first arg of func in determining cache key
"""
self.ttl = self._TTL(ttl=ttl, maxsize=maxsize)
self.skip_args = skip_args
def __call__(self, func):
async def wrapper(*args, **kwargs):
key = KEY(args[self.skip_args:], kwargs)
if key in self.ttl:
val = self.ttl[key]
else:
self.ttl[key] = await func(*args, **kwargs)
val = self.ttl[key]
return val
wrapper.__name__ += func.__name__
return wrapper
attl_cache = ATTL

View file

@ -0,0 +1,82 @@
import asyncio
import time
from async_cache import ALRU, ATTL
@ALRU(maxsize=128)
async def func(wait: int):
await asyncio.sleep(wait)
class TestClassFunc:
@ALRU(maxsize=128)
async def obj_func(self, wait: int):
await asyncio.sleep(wait)
@staticmethod
@ATTL(maxsize=128, ttl=None, skip_args=1)
async def skip_arg_func(arg: int, wait: int):
await asyncio.sleep(wait)
@classmethod
@ALRU(maxsize=128)
async def class_func(cls, wait: int):
await asyncio.sleep(wait)
def test():
t1 = time.time()
asyncio.get_event_loop().run_until_complete(func(4))
t2 = time.time()
asyncio.get_event_loop().run_until_complete(func(4))
t3 = time.time()
t_first_exec = (t2 - t1) * 1000
t_second_exec = (t3 - t2) * 1000
print(t_first_exec)
print(t_second_exec)
assert t_first_exec > 4000
assert t_second_exec < 4000
def test_obj_fn():
t1 = time.time()
obj = TestClassFunc()
asyncio.get_event_loop().run_until_complete(obj.obj_func(4))
t2 = time.time()
asyncio.get_event_loop().run_until_complete(obj.obj_func(4))
t3 = time.time()
t_first_exec = (t2 - t1) * 1000
t_second_exec = (t3 - t2) * 1000
print(t_first_exec)
print(t_second_exec)
assert t_first_exec > 4000
assert t_second_exec < 4000
def test_class_fn():
t1 = time.time()
asyncio.get_event_loop().run_until_complete(TestClassFunc.class_func(4))
t2 = time.time()
asyncio.get_event_loop().run_until_complete(TestClassFunc.class_func(4))
t3 = time.time()
t_first_exec = (t2 - t1) * 1000
t_second_exec = (t3 - t2) * 1000
print(t_first_exec)
print(t_second_exec)
assert t_first_exec > 4000
assert t_second_exec < 4000
def test_skip_args():
t1 = time.time()
asyncio.get_event_loop().run_until_complete(TestClassFunc.skip_arg_func(5, 4))
t2 = time.time()
asyncio.get_event_loop().run_until_complete(TestClassFunc.skip_arg_func(6, 4))
t3 = time.time()
t_first_exec = (t2 - t1) * 1000
t_second_exec = (t3 - t2) * 1000
print(t_first_exec)
print(t_second_exec)
assert t_first_exec > 4000
assert t_second_exec < 4000

View file

@ -0,0 +1,57 @@
import asyncio
import time
from async_cache import ATTL
@ATTL(ttl=60)
async def long_expiration_fn(wait: int):
await asyncio.sleep(wait)
return wait
@ATTL(ttl=5)
async def short_expiration_fn(wait: int):
await asyncio.sleep(wait)
return wait
@ATTL(ttl=3)
async def short_cleanup_fn(wait: int):
await asyncio.sleep(wait)
return wait
def test_cache_hit():
t1 = time.time()
asyncio.get_event_loop().run_until_complete(long_expiration_fn(4))
t2 = time.time()
asyncio.get_event_loop().run_until_complete(long_expiration_fn(4))
t3 = time.time()
t_first_exec = (t2 - t1) * 1000
t_second_exec = (t3 - t2) * 1000
print(t_first_exec)
print(t_second_exec)
assert t_first_exec > 4000
assert t_second_exec < 4000
def test_cache_expiration():
t1 = time.time()
asyncio.get_event_loop().run_until_complete(short_expiration_fn(1))
t2 = time.time()
asyncio.get_event_loop().run_until_complete(short_expiration_fn(1))
t3 = time.time()
time.sleep(5)
t4 = time.time()
asyncio.get_event_loop().run_until_complete(short_expiration_fn(1))
t5 = time.time()
t_first_exec = (t2 - t1) * 1000
t_second_exec = (t3 - t2) * 1000
t_third_exec = (t5 - t4) * 1000
print(t_first_exec)
print(t_second_exec)
print(t_third_exec)
assert t_first_exec > 1000
assert t_second_exec < 1000
assert t_third_exec > 1000

View file

@ -0,0 +1,4 @@
py_project(
name = "bazelshim",
main = "src/bazelshim/__main__.py",
)

View file

@ -0,0 +1,217 @@
#!/usr/bin/env python3
# Since this can't run under Bazel, we have to set up the sys.path ourselves
import os
import sys
from pathlib import Path
if (p := str(Path(sys.argv[0]).absolute().parent.parent)) not in sys.path:
sys.path.pop(0) # Remove '.' / ''
sys.path.insert(0, p) # Insert the bazelshim root
# Now that's out of the way...
from dataclasses import dataclass
from shlex import quote, split as shlex
import sys
import tomllib
from pathlib import Path
from typing import List, Optional
from itertools import chain
# FIXME: Allow user-defined extensions here
VERBS = [
"aquery",
"build",
"clean",
"coverage",
"cquery",
"dump",
"fetch",
"help",
"info",
"mod",
"query",
"run",
"sync",
"test",
"watch",
]
def path():
for it in os.getenv("PATH").split(":"):
yield Path(it)
def which(cmd):
for it in path():
f: Path = (it / cmd).absolute()
if f.exists() and f.stat().st_mode & 0x700:
yield f
def normalize_opts(args: List[str]) -> List[str]:
acc = []
if args[0].endswith("bazel") or args[0].endswith("bazelis"):
acc.append(args.pop(0))
while len(args) >= 2:
if args[0] == "--":
# Break
acc.extend(args)
break
elif "=" in args[0]:
# If it's a k/v form pass it through
acc.append(args.pop(0))
elif args[0].startswith("--no"):
# Convert --no<foo> args to --<foo>=no
acc.append("--" + args.pop(0).lstrip("--no") + "=false")
elif args[0] == "--isatty=0":
acc.append("--isatty=false")
args.pop(0)
elif (
args[0].startswith("--")
and not args[1].startswith("--")
and args[1] not in VERBS
):
# If the next thing isn't an opt, assume it's a '--a b' form
acc.append(args[0] + "=" + args[1])
args.pop(0)
args.pop(0)
elif args[0].startswith("--"):
# Assume it's a boolean true flag
acc.append(args.pop(0) + "=true")
else:
acc.append(args.pop(0))
else:
if args:
acc.extend(args)
return acc
@dataclass
class BazelCli:
binary: str
startup_opts: List[str]
command: Optional[str]
command_opts: List[str]
subprocess_opts: List[str]
@classmethod
def parse_cli(cls, args: List[str]) -> "BazelCli":
args = normalize_opts(args)
binary = args.pop(0)
startup_opts = []
while args and args[0].startswith("--"):
startup_opts.append(args.pop(0))
command = None
if args and args[0] in VERBS:
command = args.pop(0)
command_opts = []
while args and args[0] != "--":
command_opts.append(args.pop(0))
subprocess_opts = []
if args:
if args[0] == "--":
args.pop(0)
subprocess_opts.extend(args)
return cls(
binary=binary,
startup_opts=startup_opts,
command=command,
command_opts=command_opts,
subprocess_opts=subprocess_opts,
)
def render_cli(self):
acc = [
self.binary,
*self.startup_opts,
]
if self.command:
if self.command == "watch":
acc.extend(self.command_opts)
else:
acc.append(self.command)
acc.extend(self.command_opts)
if self.command == "test":
acc.extend(["--test_arg=" + it for it in self.subprocess_opts])
elif self.command == "run":
acc.append("--")
acc.extend(self.subprocess_opts)
elif self.command and not self.subprocess_opts:
pass
else:
print(
f"Warning: {self.command} does not support -- args! {self.subprocess_opts!r}",
file=sys.stderr,
)
return acc
def executable(self, exclude: List[Path]):
"""Try to resolve as via which() an executable to delegate to."""
if self.command == "watch":
for p in chain(which("ibazel")):
if p.parent not in exclude:
return str(p)
else:
for p in chain(which("bazelisk"), which("bazel")):
if p.parent not in exclude:
return str(p)
def render_text(self, next):
lines = []
lines.append(" " + next)
base_prefix = " "
for arg in self.render_cli()[1:]:
prefix = base_prefix
if arg in VERBS or arg == self.command:
prefix = " "
elif arg == "--":
base_prefix += " "
lines.append(prefix + arg)
return "\\\n".join(lines)
# FIXME: Use some sort of plugin model here to implement interceptors
def middleware(cli):
return cli
if __name__ == "__main__":
# This script has a magical flag to help with resolving bazel
exclude = []
while len(sys.argv) > 1 and sys.argv[1].startswith("--bazelshim_exclude"):
exclude.append(Path(sys.argv.pop(1).split("=")[1]).absolute())
us = Path(sys.argv[0]).absolute()
exclude.append(us.parent)
cli = BazelCli.parse_cli(["bazel"] + sys.argv[1:])
cli = middleware(cli)
next = cli.executable(exclude=exclude)
if sys.stderr.isatty() and not "--isatty=false" in cli.command_opts:
print(
"\u001b[33mInfo\u001b[0m: Executing\n" + cli.render_text(next),
file=sys.stderr,
)
os.execv(next, cli.render_cli())

View file

@ -0,0 +1,74 @@
#!/usr/bin/env python3
from shlex import split as shlex
from bazelshim.__main__ import normalize_opts
import pytest
@pytest.mark.parametrize(
"a, b",
[
(
"bazel clean",
[
"bazel",
"clean",
],
),
(
"bazel --client_debug clean",
[
"bazel",
"--client_debug=true",
"clean",
],
),
(
"bazel build //foo:bar //baz:*",
[
"bazel",
"build",
"//foo:bar",
"//baz:*",
],
),
(
"bazel test //foo:bar //baz:* -- -vvv",
[
"bazel",
"test",
"//foo:bar",
"//baz:*",
"--",
"-vvv",
],
),
(
"bazel test --shell_executable /bin/bish //foo:bar //baz:* -- -vvv",
[
"bazel",
"test",
"--shell_executable=/bin/bish",
"//foo:bar",
"//baz:*",
"--",
"-vvv",
],
),
(
"bazel run //foo:bar -- --foo=bar --baz=qux",
[
"bazel",
"run",
"//foo:bar",
"--",
"--foo=bar",
"--baz=qux",
],
),
],
)
def test_normalize_opts(a, b):
assert normalize_opts(shlex(a)) == b

View file

@ -0,0 +1,52 @@
#!/usr/bin/env python3
from shlex import split as shlex
from bazelshim.__main__ import BazelCli
import pytest
@pytest.mark.parametrize(
"a, b",
[
(
"bazel clean",
BazelCli("bazel", [], "clean", [], []),
),
(
"bazel --client_debug clean",
BazelCli("bazel", ["--client_debug=true"], "clean", [], []),
),
(
"bazel build //foo:bar //baz:*",
BazelCli("bazel", [], "build", ["//foo:bar", "//baz:*"], []),
),
(
"bazel test //foo:bar //baz:* -- -vvv",
BazelCli("bazel", [], "test", ["//foo:bar", "//baz:*"], ["-vvv"]),
),
(
"bazel test --shell_executable /bin/bish //foo:bar //baz:* -- -vvv",
BazelCli(
"bazel",
[],
"test",
["--shell_executable=/bin/bish", "//foo:bar", "//baz:*"],
["-vvv"],
),
),
(
"bazel run //foo:bar -- --foo=bar --baz=qux",
BazelCli(
"bazel",
[],
"run",
["//foo:bar"],
["--foo=bar", "--baz=qux"],
),
),
],
)
def test_normalize_opts(a, b):
assert BazelCli.parse_cli(shlex(a)) == b

View file

@ -9,7 +9,10 @@ zonefiles through the parser.
from types import LambdaType from types import LambdaType
from bussard.gen.parser import parse as _parse, Parser # noqa from bussard.gen.parser import ( # noqa
parse as _parse,
Parser,
)
from bussard.gen.types import * # noqa from bussard.gen.types import * # noqa

View file

@ -3,7 +3,12 @@ Tests of the Bussard reader.
""" """
import bussard.reader as t import bussard.reader as t
from bussard.reader import Actions, Parser, read, read1 from bussard.reader import (
Actions,
Parser,
read,
read1,
)
def parse_word(input): def parse_word(input):

Some files were not shown because too many files have changed in this diff Show more