From 82d79660467703464c0ce5ad9b8e06f347f1518a Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Tue, 3 Aug 2021 08:43:19 -0600 Subject: [PATCH] Add kook as-is --- projects/kook/.editorconfig | 8 + projects/kook/.eggs/README.txt | 6 + .../setuptools_git_version-1.0.3-py3.7.egg | Bin 0 -> 2715 bytes .../setuptools_git_version-1.0.3-py3.8.egg | Bin 0 -> 2720 bytes projects/kook/LICENSE.md | 7 + projects/kook/README.md | 74 ++ projects/kook/bin/kook | 742 ++++++++++++++++++ projects/kook/bin/kook-inventory | 136 ++++ projects/kook/kook-archive.json | 554 +++++++++++++ projects/kook/setup.py | 38 + projects/kook/src/kook/__init__.py | 0 projects/kook/src/kook/client.py | 730 +++++++++++++++++ projects/kook/src/kook/config.py | 46 ++ projects/kook/test.sh | 109 +++ projects/kook/test/test_client.py | 3 + 15 files changed, 2453 insertions(+) create mode 100644 projects/kook/.editorconfig create mode 100644 projects/kook/.eggs/README.txt create mode 100644 projects/kook/.eggs/setuptools_git_version-1.0.3-py3.7.egg create mode 100644 projects/kook/.eggs/setuptools_git_version-1.0.3-py3.8.egg create mode 100644 projects/kook/LICENSE.md create mode 100644 projects/kook/README.md create mode 100755 projects/kook/bin/kook create mode 100755 projects/kook/bin/kook-inventory create mode 100644 projects/kook/kook-archive.json create mode 100644 projects/kook/setup.py create mode 100644 projects/kook/src/kook/__init__.py create mode 100644 projects/kook/src/kook/client.py create mode 100644 projects/kook/src/kook/config.py create mode 100644 projects/kook/test.sh create mode 100644 projects/kook/test/test_client.py diff --git a/projects/kook/.editorconfig b/projects/kook/.editorconfig new file mode 100644 index 0000000..ccba29b --- /dev/null +++ b/projects/kook/.editorconfig @@ -0,0 +1,8 @@ +root=true + +[*] +indent_style=space +indent_size=2 +trim_trailing_whitespace=true +insert_final_newline=true +max_line_length=100 diff --git a/projects/kook/.eggs/README.txt b/projects/kook/.eggs/README.txt new file mode 100644 index 0000000..5d01668 --- /dev/null +++ b/projects/kook/.eggs/README.txt @@ -0,0 +1,6 @@ +This directory contains eggs that were downloaded by setuptools to build, test, and run plug-ins. + +This directory caches those eggs to prevent repeated downloads. + +However, it is safe to delete this directory. + diff --git a/projects/kook/.eggs/setuptools_git_version-1.0.3-py3.7.egg b/projects/kook/.eggs/setuptools_git_version-1.0.3-py3.7.egg new file mode 100644 index 0000000000000000000000000000000000000000..2face623623b4b312783f59affa18c16c011a95c GIT binary patch literal 2715 zcmb7`d00~E9>)AFtt(Em(%Q<(7tq zOyg2&nQ19$Vz^^gW_Lu#i&kXrHd^;MJtZ->Kkm!(J_mTt_j`Wt@Av*b@9}aI|3Mi9 z0!e{74r_U4jgmEvi-ADHU=TaeUjv%!{!>K3eODEQjjM{4+gO;DeKbRsE9okjEo_GgRky z%O2ii7VDJk;=1kRT!F7el2KGmAwBibHo*dXK3T&Dp0sv~-pN4J21d-U!1YJw4QgP~ z26gF%%nvgR%6IR zHGjm+Qu057cSu@KDRwoavn5&LbE<4c+R8qy)}fZ5xu~#n@?y~tO2y69$Sd|Ty$%gC ze`Fi-pUBMic-^lLpjl+_p2&7S6+g6|dt-ONfW@$}*EePH^1eH50F-4Jn2JDI9G#q? zF7TtCz-^zu#9nEGvrOd9z9Y#zIE>hZwJ|j{-PKs_qgG`9I&Oe!?K)W2dH`lf(Yot! zYkht7ll6XgroRgsx1)R)B;<;*LzR*1xqQjebMBne#kc0u6_UG}*M8xqHteA&<=XH3 zm5QaAPUuP2T+NA|8` zEA7u^AHSm&mr`2dv@%Of z0O`a4Y2E07uW9x5^!KrM^o0=vx6BzgX14H5AuAwd`ZJCm_HX>2XFHFxX0Z&%| zriySBG3e-6Obq(32n^xS-B1H|U*GP=+YSBZzl$!VRBf$%V+pFj+%9ZF#Nm;V=s0xb zMg}%rE_wDDoi}!a?&;<9{VgxTS^cKfPGj&;LP$9JZwe_Znf6qGm(@TqOo7KEk$4g+ z1a$(9L>hmGpC~+ucmjuonp*)VMR`#7t{@;5Xh)13T_k;OWx+ zM(7~;!;`qTk(PIipT9CVzoZUEe`E!-<;$a`hu72nD9cK;U{dGAW7~s$&6%0nZSVTb z);|;~&E?sy^zXvq8L9@><5}6@PEZ`25L9c*veDT8a$F#97T5GRiK{Jt$>nZi{Xshu zL{-0HA1(JeeIU@E7j(!*g>Au954QR+L%KR04D(BYJQCm;6MkjgznVFAm{%EZi_4>> zvRLn5?Xt9sw&{0Z6~Vgpa?~;;IK>hLb(iUypt=Wf2(#rk9}5nlN?n>(&-1ULp?@r# z?cLGa^*m^jkb=2AkTc599uINOOoq|h)Bdr4H}P4v<(3JEzxN}3t7_)FyT;{dbikI~JCJV? z*t-l_;{H;S)6+4_o7LHu&}2MFL`siw?=fRkblVd@B`#KwQmW15bOox{{kM`kp5D4t_tHb0^Ku1)PhmeI>QL-9S#eC4 z94mK64PveU&aC93YeJd~dZrK+Y4y;hBi*j*IRD%4=LAu959|$t6We}6 zRpmrC3#gvW6*&=SvHMpH-*;VzRo+g0@uq+Axa^18Ocoi7Q~nTsy`=JrGV^5EPA*gN zG_!zv%MKAc_7n8k*7*3hNAf?_xz92g3E3A%j*Gw8^4h-C#)!Wz?&T&1Ru=#MdjPPl z{~C1i#^pP^8c!0n5k0p)n}9%cavngfugLjwiiyfa`MfC?0R8q=E<|fl2jOnsVC0uU zC*K6R^{d1GBBrR7C@?pzW`wOo@hNH}O1w=Q{0~AA3G+|XLX>x#77fA{-_TFgLpU)T snENv5WUz!#7+;_#Di>wmrrZLM|K9}ka+8qUu#g5m$v|l(r8a*37o4gb#sB~S literal 0 HcmV?d00001 diff --git a/projects/kook/.eggs/setuptools_git_version-1.0.3-py3.8.egg b/projects/kook/.eggs/setuptools_git_version-1.0.3-py3.8.egg new file mode 100644 index 0000000000000000000000000000000000000000..c5e5862c1ed095968d4463344db4473c066b2679 GIT binary patch literal 2720 zcmb7`dpy(oAIE2!CgAl?!?`mx$@-0uN|2x7DrLpRtVOyM@b@pKa_|VwYiuSepfe(hfCYX-X zOGiAOo2)W2bM7e9*;Ae-u?F~(tDJ*Zn1(>93I!d)c5$@vYYnMoNfu6 zta~TPu0KP@b!;r}qSk)9MfWQy-cwIc_iFikD0gvKYlqg$<+PdhogR}SoR~-qvr^^C z!nW>G#dKr{bunYsj^~Z9mb8Km9UF8(%rTd z6R$I&{7&hahse5eAC^hVXr~NsQ2g*J`<9+hugQQR@|&_CJ*viafwC+CQywUbor43+ z$<5v!xb5?oh>J~b%|@G(r)ko^Tlw!Hm>U}#@2MVT_w#w$R9IzQrO&UYNX()c zMlvEPy1BbRCkR6f7>~VcbBxKiKJB}*1?GZ zq!S0Eb-f3^rq$Ek8)akX38zL=(>s6uDc(pfY2;v3-nAxOwO~bY@^i@Mhb^HuBPFbf zC0u`-sJPjNT%*%YcDKaI7_KE{h1Hpqdgrg6^lCXpjEFqYN3BwUiQP!973Jm&ItvVc%=?x{Gy{AX7@@~Z=#7ohoD494&~{0t`1s3(X77{UxdX)v03fT(CJcXsuu1N^dQYW&&TyJ}$YJSz|BL@P1AD&ybB$}GW zNH+8wK66H3C-)(J?PFu|u|}l36{&@s9B$e4q}$0p+#=8D))t;>R6~ARGkS_1Ssj-c zU!gv^yD=0YZ!uQTw>M|d0wzE&;7DN@81k}=oI`pV%LKd3J; zow2C?RL-r~$t5t^ZOHn zPpkgbqODSzxC@GFQhmWj+g4|oqJh+5w~xH6!1fjt5smCbNo=i#~dBV6jaXvBL`K>fuS|SE^5P+f@WnM~<_#=!*d_ z+IG*=D~lrh?vbkyyJt)DCQ1VxxJ|dp1JnDfD7@#8H-GMfOsp*{UjT;(OTBpV;dLyu z5_V9~@>1)1vh^v~LD;I-11y@KCS!LyQF?0r{r-{E^&n{!}(1ZAAMW=Hc zCuf`=93vGvp2TJscaAl8m(RO4;g>)z^_<>T*oHZmQgg# I AM NOT A LOONY! +> +> ~ John Cleese + +[Apache Zookeeper](https://zookeeper.apache.org/) is a mature distributed coordination system, providing locking, monitoring, leader election and other such capabilities comparable to [Google Chubby](https://ai.google/research/pubs/pub27897) in many ways. +Google has deployed Chubby to solve a number of interesting problems around service discovery (DNS), leadership management & coarse coordination, as well as cluster membership and configuration management. +While these problems prima face appear to be unrelated, they have deep similarities in the requirements they introduce for reliability, and the distributed consensus capabilities needed to offer them. + +Kook is a Python library - backed by the Kazoo Zookeeper client - which sketches at a leveraging a Zookeeper cluster to build out host management and coordination capabilities. + +Kook's data model consists of hosts and groups. + +A host is a model of a physical device. +It has key/value storage in the form of `attributes` and may be a member of one or more `groups`. +Groups likewise have `attributes` and may themselves be members (children) of other groups. + +``` +>>> from kook.client import KookClient +>>> client = KookClient(hosts="zookeeper:2181") +>>> client.servers() +>>> client.hosts() +[, + , + , + ...] +>>> client.server("ethos").groups() +[, + , + , + ] +``` + +## With Ansible + +The [`import_inventory.py`](https://git.arrdem.com/arrdem/kook/tree/import_inventory.py) script can be used to convert an existing Ansible inventory into host and group records in Kook. +The script contains shims for importing both host vars and group vars. +However kook is only intended for tracking slow-moving host level vars, and group membership. +I believe configuration data (group vars) should be on groups, and live in source control. +Only inventory and membership should be fast-enough moving to live in Zookeeper. + +The [`kook_inventory.py`](https://git.arrdem.com/arrdem/kook/tree/kook_inventory.py) script uses the `KookClient` to provide an [Ansible dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_dynamic_inventory.html). +To use `kook_inventory.py` with Ansible, you'll need the following lines in your `ansible.cfg`: + +``` +[default] +inventory = kook_inventory.py +... + +[inventory] +enable_plugins = script +... +``` + +This tells Ansible to use the `kook_inventory.py` script as its sole source of inventory. + +## Status + +Kook is currently prototype-grade. +The client works, the inventory script works. +The `kook` CLI script is woefully incomplete. + +Both the client and client and inventory feature significant limitations. +Zookeeper really wasn't designed for a "full scan" workload like this. +It's a coordination system not a database - for all it may seem appropriate to overload its usage. +Really making this viable would require a meaningful effort to leverage client-side data caching. + +The real concern is improving the locking story, using watches to update fetched nodes rather than re-fetching every time. +For coarse-grained actions on slow moving inventory the current naive strategies should be close enough to correct. + +## License + +Published under the MIT license. diff --git a/projects/kook/bin/kook b/projects/kook/bin/kook new file mode 100755 index 0000000..159cee2 --- /dev/null +++ b/projects/kook/bin/kook @@ -0,0 +1,742 @@ +#!/usr/bin/env python3 + +# See kook --help for usage. + +import argparse +from datetime import datetime +import getpass +import json +import re +from subprocess import call +import sys +from time import sleep +from typing import Iterable + +import jinja2 +from kook.client import KookClient, lock + + +# Note colored is an optional dependency +try: + from colored import attr, fg + +except ImportError: + # Provide stub functions + def fg(_): + return "" + + def attr(_): + return "" + + +def _var_filter_ctor(s: str): + if s.startswith("!"): + s = s[1:] + if ":" in s: + s = s.split(":", 1) + return ("not_match", s[0], s[1]) + else: + return ("unset", s) + elif ":" in s: + return ("match", *s.split(":", 1)) + else: + raise ValueError("Got invalid filter") + + +def _compile_filters(filters): + if not filters: + return lambda x: True + + else: + unset = object() + + def _compile_filter(f): + if f[0] == "unset": + return lambda x: x.canonicalized_vars().get(f[1], unset) is unset + + elif f[0] in ["match", "not_match"]: + # There's no reasonable thing such as regex antimatch, so we do a + # positive match and invert the result, which makes the two regex + # operations differ only by a complement() which python doesn't have so + # we do it with a lambda. + + pattern = re.compile(f[2]) + + def _helper(x): + v = x.canonicalized_vars().get(f[1], unset) + if v is unset: + return False + else: + return re.match(pattern, v) is not None + + if f[0] == "match": + return _helper + else: + return lambda x: not _helper(x) + + _filters = [_compile_filter(f) for f in filters] + + return lambda x: all(f(x) for f in _filters) + + +def _hostlist(client, opts, key=lambda h: h.name): + """Given the shared host/group machinery, build and return a host list.""" + + hosts = set() + + if not opts.hosts and not opts.groups: + for host in client.hosts(): + hosts.add(host) + + elif opts.hosts: + for host in opts.hosts: + h = client.host(host) + if h is not None: + hosts.add(h) + + elif opts.groups: + for group in opts.groups: + g = client.group(group) + if g is not None: + for h in g.hosts(): + hosts.add(h) + + hosts = list(filter(_compile_filters(opts.var_filters), hosts)) + + return sorted(hosts, key=key) + + +class ExtendAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + items = getattr(namespace, self.dest) or [] + items.extend(values) + setattr(namespace, self.dest, items) + + +class CommandRegistry(object): + def __init__(self): + self._registry = {} + + def __call__(self, *args, **kwargs): + return self.register(*args, **kwargs) + + def register(self, command_tuple): + """Intended for use as a decorator. + + Registers a function into the registry as a command.""" + + def _wrapper(func): + self._registry[command_tuple] = func + return func + + return _wrapper + + def dispatch(self, command_tuple): + """Look up the longest prefix match, returning its function.""" + + match, val = None, None + for _tuple, func in self._registry.items(): + if command_tuple >= _tuple and all( + a == b for a, b in zip(command_tuple, _tuple) + ): + if val and len(_tuple) > len(match): + match, val = _tuple, func + elif match is None: + match, val = _tuple, func + return match, val + + def __iter__(self): + return iter(self._registry.items()) + + +REGISTRY = CommandRegistry() +register_command = REGISTRY.__call__ + +CMD_COLOR = 2 + +TEMPLATE_ENVIRONMENT = jinja2.Environment() +TEMPLATE_ENVIRONMENT.filters['datetime'] = lambda o: datetime.fromtimestamp(int(o)) +TEMPLATE_ENVIRONMENT.filters['rpad'] = lambda s, width: str(s) + " " * max(0, width - len(str(s))) +TEMPLATE_ENVIRONMENT.filters['lpad'] = lambda s, width: " " * max(0, width - len(str(s))) + str(s) + + +LIST_TEMPLATE = """{{host_name | rpad(32)}} {{ geo | rpad(9)}} {{rack | rpad(9)}} {{host_reported_address | rpad(15)}} {{host_checkin | default(-1) | datetime}} ({{host_checkin | default(-1)}})""" + + +@register_command(("format",)) +def cli_format(client: KookClient, opts, args: Iterable[str]): + """Usage: kook format + + Wipe all Kook data out of Zookeeper. + """ + + print(f"Deleted {client.config.group_prefix}") + if opts.execute: + client.delete(client.config.group_prefix, recursive=True) + + print(f"Deleted {client.config.host_prefix}") + if opts.execute: + client.delete(client.config.host_prefix, recursive=True) + + +def set_in(dict, path, blob): + if len(path) == 1: + dict[path[0]] = blob + return blob + + elif len(path) > 1: + k = path[0] + if k not in dict: + dict[k] = {} + return set_in(dict[k], path[1:], blob) + + +@register_command(("dump",)) +def cli_dump(client: KookClient, opts, args: Iterable[str]): + """Usage: kook dump + + Create a JSON archive of all durable host and group datat which could be + loaded with the `kook load` command. + + """ + + groups = {} + for g in client.groups(): + group_json, _stat = g.fetch() + groups[g.name] = group_json + + hosts = {} + for h in client.hosts(): + host_json, _stat = h.fetch() + hosts[h.name] = host_json + + meta = {} + blob = {"__meta__": meta} + for type, k, data in [('groups', client.config.group_prefix, groups), + ('hosts', client.config.host_prefix, hosts)]: + meta[type] = k + blob[k] = data + + json.dump(blob, sys.stdout, indent=2, sort_keys=True) + + +@register_command(("load",)) +def cli_load(client: KookClient, opts, args: Iterable[str]): + """Usage: kook load [--execute] + + Format the store, read a JSON archive from stdin and backfill the store from + the JSON archive. + + """ + + blob = json.load(sys.stdin) + assert "__meta__" in blob, "Sanity check failed, no __meta__ header present!" + assert isinstance(blob["__meta__"], dict), "Sanity check failed, header was not a dict!" + + assert "groups" in blob["__meta__"], "Sanity check failed, no 'groups' source specified in the header!" + assert blob["__meta__"]["groups"] in blob, "Sanity check failed, specified 'groups' source is absent!" + assert isinstance(blob[blob["__meta__"]["groups"]], dict), "Sanity check failed, specified 'groups' source was not a dict!" + + assert "hosts" in blob["__meta__"], "Sanity check failed, no 'hosts' source specified in the header!" + assert blob["__meta__"]["hosts"] in blob, "Sanity check failed, specified 'hosts' source is absent!" + assert isinstance(blob[blob["__meta__"]["hosts"]], dict), "Sanity check failed, specified 'hosts' source was not a dict!" + + cli_format(client, opts, args) # Destroy any state currently in Kook + + # Insert all groups first + for group_name, group_blob in blob[blob["__meta__"]["groups"]].items(): + k, v = f"{client.config.group_prefix}/{group_name}", json.dumps(group_blob).encode("utf-8") + print(f"Set {k} => {v}") + if opts.execute: + client.create(k, v, makepath=True) + + # Insert all groups first + for host_name, host_blob in blob[blob["__meta__"]["hosts"]].items(): + k, v = f"{client.config.host_prefix}/{host_name}", json.dumps(host_blob).encode("utf-8") + print(f"Set {k} => {v}") + if opts.execute: + client.create(k, v, makepath=True) + + +@register_command(("run",)) +def run(client: KookClient, opts, args: Iterable[str]): + """Usage: kook [-H | -g ]+ run [-u username] [cmd ...] + + Run a command on all selected hosts, printing output. + """ + + reset = attr("reset") + + for h in _hostlist(client, opts): + with lock(h.lock): + cmd = [ + "ssh", + "-o", + "ConnectTimeout=5", + f"{opts.user}@{h.vars().get('host_reported_address')}", + *args, + ] + print(fg(CMD_COLOR) + f"exec] {h.name}] " + " ".join(cmd), reset, file=sys.stderr) + res = call(cmd) + print() + + +@register_command(("reboot",)) +def reboot(client: KookClient, opts, args: Iterable[str]): + """Usage: kook [-H | -g ]+ reboot + + Power cycle all the host(s), maintaining the host lock(s) to prevent other automation intervening. + + """ + + reset = attr("reset") + + for h in _hostlist(client, opts): + with lock(h.lock): + prefix = [ + "ssh", + "-o", + "ConnectTimeout=3", + f"{opts.user}@{h.vars().get('host_reported_address')}", + ] + rbt = prefix + ["reboot"] + upt = prefix + ["uptime"] + print(fg(CMD_COLOR) + "exec] " + " ".join(rbt), reset, file=sys.stderr) + call(rbt) + while True: + sleep(5) + if call(upt) == 0: + print() + break + + +######################################## +# Host CRUD +@register_command(("host", "create")) +def cli_host_create(client: KookClient, opts, args: Iterable[str]): + """Usage: kook host create + + Enter a host into the Kook service with no key/value pairs. + + Attributes such as `ansible_host` and groups must be added separately. + """ + + try: + hostname, *_ = args + except ValueError: + return + + h = client.create_host(hostname) + print("Created host", h, file=sys.stderr) + + json.dump({"_created": [hostname]}, sys.stdout, indent=2, sort_keys=True) + + +@register_command(("host", "delete")) +def cli_host_delete(client: KookClient, opts, args: Iterable[str]): + """Usage: kook host delete []+ + + Delete host(s) out of the Kook service. + """ + + hosts = [] + + for hostname in args: + h = client.host(hostname) + if h is not None: + print("Deleting host", h, file=sys.stderr) + h.delete() + hosts.append(hostname) + + json.dump({"_deleted": hosts}, sys.stdout, indent=2, sort_keys=True) + + +@register_command(("host", "list")) +def list_hosts(client: KookClient, opts, args: Iterable[str]): + """Usage: kook [-g ] host list [--format FMT] + + If -g is provided, list hosts in the specified groups. + Otherwise, list all hosts. + """ + + def _key(h): + vars = h.canonicalized_vars() + return (vars.get("rack", ""), vars.get("host_address", "")) + + hosts = _hostlist(client, opts, key=_key) + + if opts.format: + t = TEMPLATE_ENVIRONMENT.from_string(opts.format) + for h in hosts: + vars = h.canonicalized_vars() + #print(repr(vars)) + print(t.render(**vars)) + else: + json.dump( + {"all": [h.name for h in hosts]}, sys.stdout, indent=2, sort_keys=True + ) + + +@register_command(("host", "details")) +def cli_host_details(client: KookClient, opts, args: Iterable[str]): + """Usage: kook -H host details + + Print host details. + """ + + json.dump( + { + host.name: host.canonicalized_vars(meta=opts.meta) + for host in _hostlist(client, opts) + }, + sys.stdout, + indent=2, + sort_keys=True, + ) + + +######################################## +# Group CRUD +@register_command(("group", "create")) +def cli_group_create(client: KookClient, opts, args: Iterable[str]): + """Usage: kook group create []+ + + Create the specified group (if a group of the same name does not exist). + """ + + groups = [] + for group in args: + g = client.create_group(group) + print("Created group", g, file=sys.stderr) + groups.append(group) + + json.dump({"_created": groups}, sys.stdout, indent=2, sort_keys=True) + + +@register_command(("group", "add")) +def cli_group_add(client: KookClient, opts, args: Iterable[str]): + """Usage: kook [-H | -g ]+ group add + + Adds all specified hosts and groups to the named group - here . + """ + + try: + group, *_ = args + except ValueError: + print("Usage: group add ", file=sys.stderr) + return + + group_to_add = client.create_group(group) + + for group in opts.groups: + g = client.group(group) + if g: + g.add_group(group_to_add) + print("Added group to group", g, file=sys.stderr) + else: + print("No such group", group, file=sys.stderr) + + for host in opts.hosts: + h = client.host(host) + if f: + h.add_group(group_to_add) + print("Added group to host", h, file=sys.stderr) + else: + print("No such host", host, file=sys.stderr) + + +@register_command(("group", "remove")) +def cli_group_group(client: KookClient, opts, args: Iterable[str]): + """Usage: kook [-H | -g ]+ group remove + + Removes all specified hosts and groups from the named group - here . + """ + + try: + group, *_ = args + except ValueError: + print("Usage: group add ", file=sys.stderr) + return + + group_to_remove = client.group(group) + if not group_to_remove: + return + + for group in opts.groups: + g = client.group(group) + if g: + g.remove_group(group_to_remove) + print(f"Removed {group_to_remove} from {g}", file=sys.stderr) + else: + print("No such group", group, file=sys.stderr) + + for host in opts.hosts: + h = client.host(host) + if f: + h.remove_group(group_to_remove) + print("Removed host from group", h, file=sys.stderr) + else: + print("No such host", host, file=sys.stderr) + + +@register_command(("group", "list")) +def cli_group_list(client: KookClient, opts, args: Iterable[str]): + """Usage: kook group list + + Produce a list of all groups. + """ + + data = {"all": [g.name for g in client.groups()]} + + json.dump(data, sys.stdout, indent=2, sort_keys=True) + + +@register_command(("group", "details")) +def cli_group_detail(client: KookClient, opts, args: Iterable[str]): + """Usage: kook [-g ]+ group details + + Print details (vars) for each named group. + """ + + data = {} + + for groupname in opts.groups: + group = client.group(groupname) + if group: + data[groupname] = v = group.canonicalized_vars(meta=opts.meta) + v["_hostvars"] = group.host_vars() + else: + print("Unable to locate group", group, file=sys.stderr) + data[groupname] = None + + json.dump(data, sys.stdout, indent=2, sort_keys=True) + + +@register_command(("group", "delete")) +def cli_group_delete(client: KookClient, opts, args: Iterable[str]): + """Usage: kook group delete []+ + + Delete the named group(s). + """ + + groups = [] + + for group in args: + g = client.group(group) + if g is not None: + g.delete() + print("Deleted group", g, file=sys.stderr) + groups.append(group) + + json.dump({"_deleted": groups}, sys.stdout, indent=2, sort_keys=True) + + +######################################## +# Attribute CRUD +@register_command(("var", "create")) +def cli_var_add(client: KookClient, opts, args: Iterable[str]): + """Usage: kook [-H | -g ]+ var add + + Add a var to groups and hosts. + """ + + try: + key, value = args + except ValueError: + print("Usage: var create ", file=sys.stderr) + return + + for group in opts.groups: + g = client.group(group) + if g: + g.set_var(key, value) + print("Set var on group", g, file=sys.stderr) + else: + print("No such group", group, file=sys.stderr) + + for host in opts.hosts: + h = client.host(host) + if h: + h.set_var(key, value) + print("Set var on host", h, file=sys.stderr) + else: + print("No such host", host, file=sys.stderr) + + +@register_command(("var", "delete")) +def cli_var_remove(client: KookClient, opts, args: Iterable[str]): + """Usage: kook [-H | -g ]+ var remove + + Delete a var from provided group(s) and host(s). + """ + + try: + var, *_ = args + except ValueError: + print("Usage: var remove ") + return + + for group in opts.groups: + g = client.group(group) + if g: + g.del_var(var) + print("Removed var on group", g, file=sys.stderr) + else: + print("No such group", group, file=sys.stderr) + + for host in opts.hosts: + h = client.host(host) + if h: + h.del_var(var) + print("Removed var on host", h, file=sys.stderr) + else: + print("No such host", host, file=sys.stderr) + + +######################################## +# And now for argparsing... +PARSER = argparse.ArgumentParser("kook") +PARSER.register("action", "extend", ExtendAction) +PARSER.add_argument( + "-b", + "--bootstrap-server", + help="host:port at which to connect to zookeeper, overriding the config file(s)", +) + +PARSER.add_argument( + "-u", "--user", dest="user", help="User to run as", default=getpass.getuser() +) + +PARSER.add_argument( + "--format", + metavar="FORMAT", + dest="format", + default=LIST_TEMPLATE, +) + +PARSER.add_argument( + "-1", + action="store_const", + const="{{host_name}}", + dest="format", +) + +PARSER.add_argument( + "--execute", + dest="execute", + action="store_true", + default=False, + help="Execute changes instead of dryrunning them" +) + +# --hosts and --groups are mutually exclusive +HG = PARSER.add_mutually_exclusive_group() +HG.add_argument( + "-H", + "--hosts", + help="Comma joined list of hostnames on which to operate. Many may be provided.", + # Lower, split, trim whitespace, and reduce to the first name segment + type=lambda s: [e.strip() for e in s.lower().split(",")], + action="extend", + dest="hosts", + default=[], +) + +HG.add_argument( + "-g", + "--groups", + help="Comma joined list of group(s) on which to operate", + type=lambda s: [e.strip() for e in s.split(",")], + action="extend", + dest="groups", + default=[], +) + +PARSER.add_argument( + "-v", + "--var", + help="Filters to applied to hosts. Either : for positive match or ! for unset or !: for antimatch.", + dest="var_filters", + type=_var_filter_ctor, + action="append", + default=[], +) + +# --synchronization and --no-synchronization are mutually exclusive +SG = PARSER.add_mutually_exclusive_group() +SG.add_argument( + "--synchronization", + dest="use_synchronization", + default=True, + action="store_true", + help="Enable synchronization reading Kook data", +) + +SG.add_argument( + "--no-synchronization", + dest="use_synchronization", + action="store_false", + help="DISABLE synchronization reading Kook data", +) + +META = PARSER.add_mutually_exclusive_group() +META.add_argument( + "--meta", dest="meta", action="store_true", help="Enable metadata in details" +) + +META.add_argument( + "--no-meta", + dest="meta", + default=False, + action="store_false", + help="DISABLE metadata in details", +) + +PARSER.add_argument("subcommand", nargs="+", help="The subcommand to execute.") + + +def indent(str, width): + lines = str.strip().split("\n") + lines = [(" " * width) + line.strip() for line in lines] + return "\n".join(lines) + + +PARSER.usage = "\n".join( + [ + "\n kook [-b BOOSTRAP_SERVER = zookeeper:2181] \\\n" + " [-H ]* \\\n" + " [-g ]* \\\n" + " [--[no-]synchronization] \\\n" + " [--[no-]meta] \\\n" + " & args\n" + ] + + ["subcommands:\n" "--------------------------------------------------"] + + [ + "\n " + + " ".join(cmd) + + "\n" + + " --------------------\n" + + indent(f.__doc__ or f.__name__, 4) + for cmd, f in iter(REGISTRY) + ] +) + +######################################## +# And now for main... +if __name__ == "__main__": + opts = PARSER.parse_args(sys.argv[1:]) + command = tuple(opts.subcommand) + + client = KookClient( + hosts=opts.bootstrap_server, use_synchronization=opts.use_synchronization + ) + + match, f = REGISTRY.dispatch(command) + + if not opts.execute: + print("Running in dryrun mode, no changes will be committed", file=sys.stderr) + + if f is not None: + f(client, opts, command[len(match) :]) + else: + print(PARSER.format_usage()) diff --git a/projects/kook/bin/kook-inventory b/projects/kook/bin/kook-inventory new file mode 100755 index 0000000..0247d6c --- /dev/null +++ b/projects/kook/bin/kook-inventory @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +""" +Usage: + kook-inventory [--list | --host | --import ] [--[no-]synchronization] + +Create a Kook client, and use it to generate Ansible formatted JSON inventory. +In keeping with the Ansible inventory protocol, requires --list or --host. +Otherwise this message will be printed. +""" + +import argparse +import json +import sys + +from kook.client import KookClient +import yaml + + +def merge(m1, m2): + return {**m1, **m2} + + +def get_inventory(client): + inventory = {} + hostvars = {} + inventory["_meta"] = {"hostvars": hostvars} + + # Import hosts + for host in client.hosts(): + hostvars[host.name] = host.canonicalized_vars(meta=False) + + for group in client.groups(): + inventory[group.name] = {"children": [], "hosts": [], "vars": {}} + inventory[group.name]["children"].extend([g.name for g in group.children()]) + inventory[group.name]["hosts"].extend([h.name for h in group.hosts()]) + + return inventory + + +def get_host(client, hostname): + host = client.host(hostname) + if host: + return host.canonicalized_vars() + + +def import_inventory(client, filename): + with open(filename, "r") as _f: + data = yaml.safe_load(_f) + + # mapcat out all the group(s) + groups = list(data.items()) + for groupname, groupdata in groups: + groups.extend((groupdata or {}).get("children", {}).items()) + + # Import all the group(s) + for groupname, groupdata in groups: + g = client.create_group(groupname) + print("Created,", g) + + if groupdata is None: + continue + + # Import the vars + for var, val in groupdata.get("vars", {}).items(): + # Fixup my key(s) + if var == "ansible_host": + var = "host_address" + + g.set_var(var, val) + + # Import the direct host bindings + for hostname, hostvars in groupdata.get("hosts", {}).items(): + h = client.create_host(hostname) + h.add_group(g) + print(g, "added host", h) + + for var, val in (hostvars or {}).items(): + g.set_host_var(hostname, var, val) + + # Import the child group(s) + for childname, childvars in groupdata.get("children", {}).items(): + child = client.create_group(childname) + child.add_group(g) + # FIXME (arrdem 2019-08-10): + # Support group on child group vars? + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--synchronization", + dest="use_synchronization", + default=True, + action="store_true", + ) + parser.add_argument( + "--no-synchronization", dest="use_synchronization", action="store_false" + ) + parser.add_argument( + "--list", + action="store_true", + dest="list", + default=False, + help="Output inventory groups and hosts", + ) + parser.add_argument( + "--host", + dest="host", + default=None, + metavar="HOST", + help="Output variables only for the given hostname", + ) + parser.add_argument( + "--import", + dest="_import", + default=None, + metavar="IMPORT", + help="Import a YAML inventory file", + ) + parser.add_argument("-b", "--bootstrap", help="Bootstrap servers list to use") + + args = parser.parse_args(sys.argv[1:]) + + client = KookClient( + hosts=args.bootstrap, use_synchronization=args.use_synchronization + ) + + if args.host: + print(json.dumps(get_host(client, args.host), indent=2, sort_keys=True)) + elif args.list: + print(json.dumps(get_inventory(client), indent=2, sort_keys=True)) + elif args._import: + import_inventory(client, args._import) + else: + print(__doc__) diff --git a/projects/kook/kook-archive.json b/projects/kook/kook-archive.json new file mode 100644 index 0000000..00d6ce6 --- /dev/null +++ b/projects/kook/kook-archive.json @@ -0,0 +1,554 @@ +{ + "/kook/group": { + "battery_apartment_larval": { + "vars": { + "battery": "larval", + "battery_password": "secret", + "battery_powervalue": 1, + "battery_system": "ups@10.0.0.22", + "battery_username": "monuser" + } + }, + "chasis_ds416play": { + "members": [ + "/kook/host/hieroglyph.apartment.arrdem.com" + ], + "vars": {} + }, + "chasis_pi_b": { + "members": [ + "/kook/host/rijom-mapul.apartment.arrdem.com", + "/kook/host/zipas-goloh.apartment.arrdem.com", + "/kook/host/fumiv-jifid.apartment.arrdem.com" + ], + "vars": { + "chasis": "pi_b", + "default_eth_iface": "eth0" + } + }, + "chasis_pi_bp": { + "host_vars": {}, + "members": [ + "/kook/host/dazav-hutiz.apartment.arrdem.com", + "/kook/host/fumiv-jifid.apartment.arrdem.com", + "/kook/host/vavor-nahub.apartment.arrdem.com", + "/kook/host/girof-fomuf.apartment.arrdem.com", + "/kook/host/fipol-gufop.apartment.arrdem.com", + "/kook/host/bufih-jakib.apartment.arrdem.com", + "/kook/host/sibar-pupuf.apartment.arrdem.com", + "/kook/host/takil-bolus.apartment.arrdem.com", + "/kook/host/zipas-goloh.apartment.arrdem.com", + "/kook/host/rijom-mapul.apartment.arrdem.com", + "/kook/host/nalos-suvav.apartment.arrdem.com", + "/kook/host/kupik-totos.apartment.arrdem.com", + "/kook/host/fozim-tasin.apartment.arrdem.com" + ], + "vars": { + "chasis": "pi_bp", + "default_eth_iface": "eth0" + } + }, + "chasis_ryzen0": { + "members": [ + "/kook/host/pathos.apartment.arrdem.com", + "/kook/host/logos.apartment.arrdem.com", + "/kook/host/ethos.apartment.arrdem.com" + ], + "vars": { + "chasis": "ryzen0", + "default_eth_iface": "enp9s0", + "resources": { + "cpu": 4, + "disk": { + "/": "32GiB", + "/data": "1TiB" + }, + "ram": "16GiB" + } + } + }, + "geo_apartment": { + "host_vars": { + "stormclad.apartment.arrdem.com": { + "host_address": "10.0.0.153" + } + }, + "members": [ + "/kook/group/rack_apartment_test", + "/kook/group/rack_apartment_pis", + "/kook/group/rack_apartment_modes", + "/kook/group/rack_apartment_infra", + "/kook/host/stormclad.apartment.arrdem.com" + ], + "vars": { + "dns_resolvers": [ + "10.0.0.32", + "10.0.0.33", + "10.0.0.34" + ], + "geo": "apartment" + } + }, + "pdu_apartment_sucker": { + "host_vars": { + "ethos.apartment.arrdem.com": { + "host_address": "10.0.0.34", + "pdu_outlet": 1 + }, + "logos.apartment.arrdem.com": { + "host_address": "10.0.0.33", + "pdu_outlet": 3 + }, + "pathos.apartment.arrdem.com": { + "host_address": "10.0.0.32", + "pdu_outlet": 2 + } + }, + "members": [ + "/kook/host/ethos.apartment.arrdem.com", + "/kook/host/pathos.apartment.arrdem.com", + "/kook/host/logos.apartment.arrdem.com" + ], + "vars": { + "pdu": "sucker", + "pdu_uri": "sucker.apartment.arrdem.com:23" + } + }, + "rack_apartment_infra": { + "groups": [ + "/kook/group/geo_apartment" + ], + "host_vars": { + "hieroglyph.apartment.arrdem.com": { + "host_address": "10.0.0.22" + }, + "sucker.apartment.arrdem.com": { + "host_address": "10.0.0.16" + } + }, + "members": [ + "/kook/host/hieroglyph.apartment.arrdem.com", + "/kook/host/sucker.apartment.arrdem.com", + "/kook/host/chisel.apartment.arrdem.com" + ], + "vars": { + "rack": "infra", + "rack_cidr": "10.0.0.0/27" + } + }, + "rack_apartment_modes": { + "groups": [ + "/kook/group/service_apartment_resolvers", + "/kook/group/geo_apartment" + ], + "members": [ + "/kook/host/logos.apartment.arrdem.com", + "/kook/host/ethos.apartment.arrdem.com", + "/kook/host/pathos.apartment.arrdem.com" + ], + "vars": { + "rack": "modes", + "rack_cidr": "10.0.0.32/29" + } + }, + "rack_apartment_pis": { + "groups": [ + "/kook/group/geo_apartment" + ], + "host_vars": {}, + "members": [ + "/kook/host/fumiv-jifid.apartment.arrdem.com", + "/kook/host/zipas-goloh.apartment.arrdem.com", + "/kook/host/rijom-mapul.apartment.arrdem.com", + "/kook/host/nalos-suvav.apartment.arrdem.com", + "/kook/host/kupik-totos.apartment.arrdem.com", + "/kook/host/fozim-tasin.apartment.arrdem.com" + ], + "vars": { + "rack": "pis", + "rack_cidr": "10.0.0.40/29" + } + }, + "rack_apartment_test": { + "groups": [ + "/kook/group/geo_apartment" + ], + "host_vars": {}, + "members": [ + "/kook/host/vavor-nahub.apartment.arrdem.com", + "/kook/host/girof-fomuf.apartment.arrdem.com", + "/kook/host/fipol-gufop.apartment.arrdem.com", + "/kook/host/bufih-jakib.apartment.arrdem.com", + "/kook/host/sibar-pupuf.apartment.arrdem.com", + "/kook/host/takil-bolus.apartment.arrdem.com" + ], + "vars": { + "rack": "test", + "rack_cidr": "10.0.0.48/28" + } + }, + "service_apartment_git": { + "members": [ + "/kook/host/ethos.apartment.arrdem.com" + ], + "vars": {} + }, + "service_apartment_mirror": { + "members": [ + "/kook/host/logos.apartment.arrdem.com" + ], + "vars": {} + }, + "service_apartment_postgres": { + "members": [ + "/kook/host/logos.apartment.arrdem.com" + ], + "vars": {} + }, + "service_apartment_resolvers": { + "members": [ + "/kook/group/rack_apartment_modes" + ], + "vars": {} + }, + "service_apartment_www": { + "members": [ + "/kook/host/ethos.apartment.arrdem.com" + ], + "vars": {} + }, + "service_apartment_zookeeper": { + "members": [ + "/kook/host/fumiv-jifid.apartment.arrdem.com", + "/kook/host/zipas-goloh.apartment.arrdem.com", + "/kook/host/rijom-mapul.apartment.arrdem.com", + "/kook/host/nalos-suvav.apartment.arrdem.com", + "/kook/host/kupik-totos.apartment.arrdem.com", + "/kook/host/fozim-tasin.apartment.arrdem.com" + ], + "vars": {}, + "host_vars": { + "fumiv-jifid.apartment.arrdem.com": { + "zookeeper_id": 1 + }, + "zipas-goloh.apartment.arrdem.com": { + "zookeeper_id": 2 + }, + "rijom-mapul.apartment.arrdem.com": { + "zookeeper_id": 3 + }, + "nalos-suvav.apartment.arrdem.com": { + "zookeeper_id": 4 + }, + "kupik-totos.apartment.arrdem.com": { + "zookeeper_id": 5 + }, + "fozim-tasin.apartment.arrdem.com": { + "zookeeper_id": 6 + } + } + } + }, + "/kook/host": { + "bufih-jakib.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_test" + ], + "vars": { + "host_address": "10.0.0.51", + "host_checkin": "1583735436", + "host_id": "2a049fb210ad40a2ae1619c4f1e2effa", + "host_link": "b8:27:eb:1c:00:eb", + "host_name": "bufih-jakib.apartment.arrdem.com", + "host_reported_address": "10.0.0.51", + "host_serial_number": "00000000b91c00eb", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILPdhyvSF20nXtoiNYwaaO09bMBRkl0jhQ5bXwUr6GvF root@bufih-jakib.apartment.arrdem.com" + } + }, + "chisel.apartment.arrdem.com": { + "groups": [ + "/kook/group/rack_apartment_infra" + ], + "vars": { + "host_address": "10.0.0.2", + "host_name": "chisel.apartment.arrdem.com" + } + }, + "dazav-hutiz.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp" + ], + "vars": { + "host_name": "dazav-hutiz.apartment.arrdem.com" + } + }, + "ethos.apartment.arrdem.com": { + "groups": [ + "/kook/group/service_apartment_www", + "/kook/group/service_apartment_git", + "/kook/group/rack_apartment_modes", + "/kook/group/chasis_ryzen0", + "/kook/group/pdu_apartment_sucker" + ], + "vars": { + "host_address": "10.0.0.34", + "host_checkin": "1588960015", + "host_id": "a3dbc867034b4055a8b9ba8d23af85eb", + "host_link": "e0:d5:5e:a0:84:65", + "host_name": "ethos.apartment.arrdem.com", + "host_reported_address": "10.0.0.34", + "host_serial_number": "03d502e0-045e-05a0-8406-650700080009", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDubv05J2uavdCLGaPk0GKHeWaz+BT6dEvR3IdiI/ooq root@ethos.apartment.arrdem.com.apartment.arrdem.com" + } + }, + "fipol-gufop.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_test" + ], + "vars": { + "host_address": "10.0.0.52", + "host_checkin": "1583735436", + "host_id": "f3de497d08c14625a0a7c8f5d30a0b8c", + "host_link": "b8:27:eb:7b:44:8a", + "host_name": "fipol-gufop.apartment.arrdem.com", + "host_reported_address": "10.0.0.52", + "host_serial_number": "00000000957b448a", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0i4+OkkFz9oLXF3L+U1UoxDZ8ON8n6LuKj3fdDvEPJ root@fipol-gufop.apartment.arrdem.com" + } + }, + "fozim-tasin.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_pis", + "/kook/group/service_apartment_zookeeper" + ], + "vars": { + "host_address": "10.0.0.40", + "host_checkin": "1588415231", + "host_id": "fa85dc9a95e24174ba104af3e60615e1", + "host_link": "b8:27:eb:0b:d6:a4", + "host_name": "fozim-tasin.apartment.arrdem.com", + "host_reported_address": "10.0.0.40", + "host_serial_number": "00000000fb0bd6a4", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMYjfeoBUSfymceFUKFJPcp7fmDxDp6cwzsJyp+kEGNz root@fozim-tasin.apartment.arrdem.com" + } + }, + "fumiv-jifid.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_pis", + "/kook/group/service_apartment_zookeeper" + ], + "vars": { + "host_address": "10.0.0.44", + "host_checkin": "1588415231", + "host_id": "fc63d668e4d94989901e6f2686a5682d", + "host_link": "b8:27:eb:54:d2:0a", + "host_name": "fumiv-jifid.apartment.arrdem.com", + "host_reported_address": "10.0.0.44", + "host_serial_number": "00000000cf54d20a", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMPoAOPTU4aZNQyrUGzJvy6z+w96QYZO7P8GwZl42eAK root@fumiv-jivid.apartment.arrdem.com" + } + }, + "girof-fomuf.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_test" + ], + "vars": { + "host_address": "10.0.0.53", + "host_checkin": "1583735435", + "host_id": "c4da3a07aee84bfd963dff6d0aacc532", + "host_link": "b8:27:eb:7f:db:6f", + "host_name": "girof-fomuf.apartment.arrdem.com", + "host_reported_address": "10.0.0.53", + "host_serial_number": "000000008e7fdb6f", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILdwdP0lYnVBOLOLFX/GAB9STY5JeDu4dCoIw0MIcgxv root@girof-fomuf.apartment.arrdem.com" + } + }, + "hieroglyph.apartment.arrdem.com": { + "groups": [ + "/kook/group/rack_apartment_infra", + "/kook/group/chasis_ds416play" + ], + "vars": { + "host_name": "hieroglyph.apartment.arrdem.com" + } + }, + "kupik-totos.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_pis", + "/kook/group/service_apartment_zookeeper" + ], + "vars": { + "host_address": "10.0.0.41", + "host_checkin": "1588415231", + "host_id": "75f74d85493f42dc840b65b6a8f3aa7b", + "host_link": "b8:27:eb:f5:d7:44", + "host_name": "kupik-totos.apartment.arrdem.com", + "host_reported_address": "10.0.0.41", + "host_serial_number": "00000000d2f5d744", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM1o1UhMVeJLCXkmLdz2FUn85ZTZuS/fB0ERFTeT58U8 root@kupik-totos.apartment.arrdem.com" + } + }, + "logos.apartment.arrdem.com": { + "groups": [ + "/kook/group/service_apartment_mirror", + "/kook/group/service_apartment_postgres", + "/kook/group/rack_apartment_modes", + "/kook/group/chasis_ryzen0", + "/kook/group/pdu_apartment_sucker" + ], + "vars": { + "host_address": "10.0.0.33", + "host_checkin": "1588960025", + "host_id": "58fe0fa321254f5f992963f530ccb59d", + "host_link": "e0:d5:5e:a2:9d:6d", + "host_name": "logos.apartment.arrdem.com", + "host_reported_address": "10.0.0.33", + "host_serial_number": "58fe0fa321254f5f992963f530ccb59d", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGa4N4uvRKsRgHUHzUenlLfu5mej4XN8SRyn6YG+beq3 root@logos.apartment.arrdem.com" + } + }, + "nalos-suvav.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_pis", + "/kook/group/service_apartment_zookeeper" + ], + "vars": { + "host_address": "10.0.0.42", + "host_checkin": "1588415231", + "host_id": "2c7801c24cd74908bee951843aaf9e61", + "host_link": "b8:27:eb:2c:4d:ba", + "host_name": "nalos-suvav.apartment.arrdem.com", + "host_reported_address": "10.0.0.42", + "host_serial_number": "00000000582c4dba", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBoQNyaVG5a4so63LgeagQIolB5HlDUr+q+v5e9CnnPy root@nalos-suvav.apartment.arrdem.com" + } + }, + "pathos.apartment.arrdem.com": { + "groups": [ + "/kook/group/rack_apartment_modes", + "/kook/group/chasis_ryzen0", + "/kook/group/pdu_apartment_sucker" + ], + "vars": { + "host_address": "10.0.0.32", + "host_checkin": "1588960019", + "host_id": "e061b32df58c44a08e08dd767e2bd37b", + "host_link": "e0:d5:5e:a0:84:5f", + "host_name": "pathos.apartment.arrdem.com", + "host_reported_address": "10.0.0.32", + "host_serial_number": "e061b32df58c44a08e08dd767e2bd37b", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBfKQi7Y2mTykU4GYP/xg8jbWYWbsFdpVRtfDKiVbkpV root@pathos.apartment.arrdem.com" + } + }, + "rijom-mapul.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_b", + "/kook/group/rack_apartment_pis", + "/kook/group/service_apartment_zookeeper" + ], + "vars": { + "host_address": "10.0.0.43", + "host_checkin": "1588415231", + "host_id": "1638c72712b2448f8133d4ae3c9cf82c", + "host_link": "b8:27:eb:1b:20:62", + "host_name": "rijom-mapul.apartment.arrdem.com", + "host_reported_address": "10.0.0.43", + "host_serial_number": "00000000421b2062", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFc0m7TdPz6lFsNaeyko8TPNbaqmUdJO12NH2ha25XmU root@rijom-mapul.apartment.arrdem.com" + } + }, + "sibar-pupuf.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_test" + ], + "vars": { + "host_address": "10.0.0.50", + "host_checkin": "1583774080", + "host_id": "663ed032848a44b3a79041c05e837d04", + "host_link": "b8:27:eb:73:6b:2a", + "host_name": "sibar-pupuf.apartment.arrdem.com", + "host_reported_address": "10.0.0.50", + "host_serial_number": "00000000ce736b2a", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAWgtzwngCD9Vol5jubkkG6LTJ7SkHYPGD89SpSjx+2z root@sibar-pupuf.apartment.arrdem.com" + } + }, + "stormclad.apartment.arrdem.com": { + "groups": [ + "/kook/group/service_apartment_zookeeper", + "/kook/group/geo_apartment" + ], + "vars": { + "host_name": "stormclad.apartment.arrdem.com" + } + }, + "sucker.apartment.arrdem.com": { + "groups": [ + "/kook/group/rack_apartment_infra" + ], + "vars": { + "host_name": "sucker.apartment.arrdem.com" + } + }, + "takil-bolus.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_test" + ], + "vars": { + "host_address": "10.0.0.49", + "host_checkin": "1583735438", + "host_id": "5926783985a941b0bb7f1bf542e0edcf", + "host_link": "b8:27:eb:df:d7:d1", + "host_name": "takil-bolus.apartment.arrdem.com", + "host_reported_address": "10.0.0.49", + "host_serial_number": "00000000a6dfd7d1", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINygpJO5LTr+mUUEFJoZTyN85Q095fAnsarR4GNYqvEV root@takil-bolus.apartment.arrdem.com" + } + }, + "vavor-nahub.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_bp", + "/kook/group/rack_apartment_test" + ], + "vars": { + "host_address": "10.0.0.54", + "host_checkin": "1584852209", + "host_id": "4a7ed5da84d54c4ba25a05e7204ba1af", + "host_link": "70:88:6b:81:58:06", + "host_name": "vavor-nahub.apartment.arrdem.com", + "host_reported_address": "10.0.0.54", + "host_serial_number": "000000005722c20d", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILtsrXsEksmBipO7pDy1xSghJqOi5YO+ROI2F0IQeJRU root@vavor-nahub.apartment.arrdem.com" + } + }, + "zipas-goloh.apartment.arrdem.com": { + "groups": [ + "/kook/group/chasis_pi_b", + "/kook/group/rack_apartment_pis", + "/kook/group/service_apartment_zookeeper" + ], + "vars": { + "host_address": "10.0.0.45", + "host_checkin": "1588415231", + "host_id": "f349f4886ca9472a8213541a98569523", + "host_link": "b8:27:eb:e7:50:58", + "host_name": "zipas-goloh.apartment.arrdem.com", + "host_reported_address": "10.0.0.45", + "host_serial_number": "000000005ee75058", + "host_sshd_fingerprint": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFF3Iy7XsOlnvLHns0zQXsog2lRSGy9gxJnP90qeP6qN root@zipas-goloh.apartment.arrdem.com" + } + } + }, + "__meta__": { + "groups": "/kook/group", + "hosts": "/kook/host" + } +} diff --git a/projects/kook/setup.py b/projects/kook/setup.py new file mode 100644 index 0000000..1cd7b77 --- /dev/null +++ b/projects/kook/setup.py @@ -0,0 +1,38 @@ +from setuptools import setup + + +setup( + name="arrdem.kook", + # Package metadata + version="0.1.19", + license="MIT", + description="A Kazoo based inventory management system", + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + author="Reid 'arrdem' McKenzie", + author_email="me@arrdem.com", + url="https://git.arrdem.com/arrdem/kook", + classifiers=[ + "License :: OSI Approved :: MIT License", + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + ], + # Package setup + package_dir={"": "src"}, + packages=[ + "kook", + ], + scripts=[ + "bin/kook", + "bin/kook-inventory", + ], + install_requires=[ + "kazoo>=2.6.1", + "toolz>=0.10.0", + "PyYAML>=5.1.0", + "Jinja2>=2.11.0", + ], + extras_require={"color": ["colored>=1.4.2"]}, +) diff --git a/projects/kook/src/kook/__init__.py b/projects/kook/src/kook/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/projects/kook/src/kook/client.py b/projects/kook/src/kook/client.py new file mode 100644 index 0000000..83a76f3 --- /dev/null +++ b/projects/kook/src/kook/client.py @@ -0,0 +1,730 @@ +"""The core Kook client library.""" + +from contextlib import contextmanager +from itertools import chain +import json +import sys +import time +from typing import Any, Iterable, Optional, Tuple, Union + +from kazoo.client import KazooClient +from kazoo.exceptions import NodeExistsError +from kazoo.protocol.states import ZnodeStat +from kazoo.recipe.lock import Lock, ReadLock, WriteLock +from kazoo.recipe.watchers import ChildrenWatch, DataWatch +from kook.config import current_config, KookConfig +from toolz.dicttoolz import assoc as _assoc, dissoc as _dissoc, merge as _merge, update_in + + +def assoc(m, k, v): + return _assoc(m or {}, k, v) + + +def dissoc(m, k): + if m: + return _dissoc(m, k) + else: + return {} + + +def merge(a, b): + return _merge(a or {}, b or {}) + + +class Timer(object): + """A context manager for counting elapsed time.""" + + def __enter__(self): + self.start = time.time() + return self + + def __exit__(self, *args): + self.end = time.time() + self.interval = self.end - self.start + + def elapsed(self): + return self.interval + + +@contextmanager +def lock(l: Lock, other: Optional[Lock] = None, **kwargs): + """A context manager for acquiring a non-reentrant lock if it isn't held.""" + + if l and other: + flag = not l.is_acquired and not other.is_acquired + if l and not other: + flag = not l.is_acquired + if l is None: + raise ValueError("A Lock instance is required!") + + if flag: + l.acquire(**kwargs) + yield + if flag: + l.release() + + +def cl(f, *args): + """Curry-Last. + + Lets me stop writing lambda x: f(x, ....) + """ + + def _helper(*inner_args): + return f(*[*inner_args, *args]) + + return _helper + + +def conj(l: Optional[list], v: Any) -> list: + """conj(oin) a value onto a list, returning a new list with the value appended.""" + + l = l or [] + if v not in l: + return [v, *l] + return l + + +def disj(l: Optional[list], v: Any) -> list: + """disj(oin) a value from a list, returning a list with the value removed.""" + + l = l or [] + return [e for e in l if e != v] + + +def as_str(v: Union[bytes, str]) -> str: + """Force to a str.""" + + if isinstance(v, bytes): + return v.decode("utf-8") + elif isinstance(v, str): + return v + else: + raise TypeError("Unable to str-coerce " + str(type(v))) + + +class KookBase(object): + """Base class for Kook types.""" + + def __init__( + self, config: KookConfig, client: "KookClient", key: str, exists=False + ): + self.config = config + self.key = key + self.meta_lkey = key + "/" + config.meta_lock_suffix + self.client = client + self._cache = None + self._valid = True + + if not exists: + if not self.client.exists(self.meta_lkey): + self.client.create(self.meta_lkey, makepath=True) + + # self.rl = self.client.create_rlock(self.meta_lkey) + self.wl = self.client.create_lock(self.meta_lkey) + + def __enter__(self): + self.wl.acquire() + return self + + def __exit__(self, type, value, traceback): + self.wl.release() + + def fetch(self, watch=None) -> Tuple[Any, ZnodeStat]: + """Read the current value, using only the read lock.""" + + assert self._valid + + def hook(*args, **kwargs): + self._cache = None + if watch is not None: + watch(*args, **kwargs) + + if not self._cache: + # with lock(self.rl, other=self.wl): + data, stat = self.client.get(self.key, watch=hook) + data = json.loads(as_str(data) or "{}") + self._cache = data, stat + + return self._cache + + def update(self, xform) -> ZnodeStat: + """With the appropriate lock, update the underlying key with the transformer.""" + + # We take the write lock so we go after all live reads. + with lock(self.wl): + data, stat = self.fetch() + newval = xform(data) + newdata = json.dumps(newval).encode("utf-8") + self.client.set(self.key, newdata) + return self.fetch() + + def vars(self, watch=None) -> dict: + """Read the entity's vars, optionally setting a watch. + + Note that the watch will be called the first time ANYTHING + changes, not just if the vars change. Clients wanting to watch for + something in specific must re-enter the watch after it is invoked, + as it will not be called twice. + + """ + + attrs, stat = self.fetch(watch=watch) + return attrs.get("vars", {}) + + def set_var(self, k, v) -> ZnodeStat: + """Set a k/v pair on the entity.""" + + return self.update(cl(update_in, ["vars"], cl(assoc, k, v))) + + def del_var(self, k) -> ZnodeStat: + """Clear a k/v pair on the entity.""" + + return self.update(cl(update_in, ["vars"], cl(dissoc, k))) + + def groups(self, watch=None): + """Read the entity's groups, optionally setting a watch. + + Note that the watch will be called the first time ANYTHING changes, not + just if the groups change. Clients wanting to watch for something in + specific must re-enter the watch after it is invoked, as it will not be + called twice. + + """ + + data, stat = self.fetch(watch=watch) + p = self.config.group_prefix + "/" + return [self.client.group(g.replace(p, "")) for g in data.get("groups", [])] + + def group_closure(self): + """Compute the transitive closure of group membership.""" + + groups = list(self.groups()) + for g in groups: + if not g: + print(f"Error - got nil group! {self}", file=sys.stderr) + continue + + for _g in g.groups(): + if _g not in groups: + groups.append(_g) + groups.reverse() + return groups + + def inherited_vars(self): + """Merge all vars from all parent groups. + + Record keys' sources in _meta, but do not throw on conflicts. + + """ + + groups = self.group_closure() + keys = {} + _groups = [] + attrs = {"_meta": keys, "_groups": _groups} + for group in groups: + _groups.append(group.name) + for k, v in group.vars().items(): + attrs[k] = v + s = keys[k] = keys.get(k, []) + if group.name not in s: + s.append(group.name) + + return attrs + + def canonicalized_vars(self, meta=True): + """Merge host vars with inherited vars. + + Inherited vars are merged in no order whatsoever. Conflicts will produce + thrown exceptions. However host vars win over inherited vars. + + """ + + attrs = self.inherited_vars() + vars = self.vars() + _meta = {} + for k, v in vars.items(): + attrs[k] = v + l = _meta[k] = _meta.get(k, []) + if "host" not in l: + l.append("host") + + if meta: + vars["_meta"] = _meta + + return attrs + + def add_group(self, group: Union["KookGroup", str]): + """Add the entity to a group.""" + + assert self._valid + group = self.client.group(group) + + with lock(self.wl): + with lock(group.wl): + self.update(cl(update_in, ["groups"], cl(conj, group.key))) + group.update(cl(update_in, ["members"], cl(conj, self.key))) + + def remove_group(self, group: Union["KookGroup", str]): + """Remove the entity from a group.""" + + assert self._valid + group = self.client.group(group) + + with lock(self.wl): + with lock(group.wl): + self.update(cl(update_in, ["groups"], cl(disj, group.key))) + group.update(cl(update_in, ["members"], cl(disj, self.key))) + + +class KookGroup(KookBase): + """Representation of a group.""" + + def __init__( + self, config: KookConfig, client: "KookClient", group_name: str, exists=False + ): + assert "/" not in group_name or group_name.startswith(config.group_prefix) + + if not group_name.startswith(config.group_prefix): + key = "/".join([config.group_prefix, group_name]) + else: + key = group_name + group_name = group_name.replace(config.group_prefix + "/", "") + + super().__init__(config, client, key, exists=exists) + self.name = group_name + + def _hosts(self, watch=None): + """Return a list of hosts who are directly members of this group.""" + + data, stat = self.fetch(watch=watch) + p = self.config.host_prefix + "/" + res = [] + for m in data.get("members", []): + if m.startswith(p): + if host := self.client.host(m.replace(p, "")): + res.append(host) + else: + raise RuntimeError(f"Unable to locate host {m} on group {self}") + + return res + + def hosts(self, watch=None): + """Return a list of hosts who are directly members of this group.""" + + # Compute the group closure + groups = list(self.children()) + for g in groups: + for _g in g.children(): + if _g not in groups: + groups.append(_g) + + # Compute the host list + hosts = list(self._hosts()) + for g in groups: + for h in g._hosts(): + if h not in hosts: + hosts.append(h) + + return hosts + + def host_vars(self, watch=None): + """Get the per-host vars assigned by this group.""" + + attrs, stat = self.fetch(watch=watch) + return attrs.get("host_vars") or {} + + def set_host_var(self, host: Union["KookHost", str], key, value): + host = self.client.host(host) + self.update(cl(update_in, ["host_vars", host.name], cl(assoc, key, value))) + + def del_host_var(self, host: Union["KookHost", str], key): + host = self.client.host(host) + self.update(cl(update_in, ["host_vars", host.name], cl(dissoc, key))) + + def del_host_vars(self, host: Union["KookHost", str]): + host = self.client.host(host) + self.update(cl(update_in, ["host_vars"], cl(dissoc, host.name))) + + def children(self, watch=None): + """Return a list of groups which are children of this group.""" + + data, stat = self.fetch(watch=watch) + p = self.config.group_prefix + "/" + return [ + self.client.group(m.replace(p, "")) + for m in data.get("members", []) + if m.startswith(p) + ] + + def delete(self): + """ "Remove parent relations + + Note - prevents group deletion while processing group updates + Interleaved writes on non-locked groups still OK. + + """ + + # with lock(self.client.grl, other=self.client.gwl): + for group in self.groups(): + self.remove_group(group) + + # Remove member relations + for member in chain(self.hosts(), self.children()): + member.remove_group(self) + + # With the write lock on the server list + # FIXME (arrdem 2019-06-25): + # Do we actually need this? + with lock(self.client.swl): + self.client.delete(self.key, recursive=True) + self._valid = False + + def __repr__(self): + return "<{} {}>".format(__class__.__name__, self.key) + + +class KookHost(KookBase): + """ + Representation of a physical device. + """ + + def __init__(self, config, client: "KookClient", host_name: str, exists=False): + assert "/" not in host_name or host_name.startswith(config.host_prefix) + + if not host_name.startswith(config.host_prefix): + key = "{}/{}".format(config.host_prefix, host_name) + else: + key = host_name + host_name = host_name.replace(config.host_prefix + "/", "") + + super().__init__(config, client, key, exists=exists) + self.name = host_name + host_lock = self.key + "/" + config.lock_suffix + if not exists and not self.client.exists(host_lock): + self.client.create(host_lock, makepath=True) + self.lock = client.create_lock(host_lock) + + def canonicalized_vars(self, meta=True): + """Factors in group host_vars atop inheriting group vars.""" + + _meta = {} + groups = [] + vars = {"_groups": groups} + + for group in self.group_closure(): + groups.append(group.name) + for k, v in group.vars().items(): + vars[k] = v + l = _meta[k] = _meta.get(k, []) + if group.name not in l: + l.append(group.name) + + for k, v in group.host_vars().get(self.name, {}).items(): + vars[k] = v + l = _meta[k] = _meta.get(k, []) + if group.name not in l: + l.append("host[{0}]".format(group.name)) + + for k, v in self.vars().items(): + vars[k] = v + l = _meta[k] = _meta.get(k, []) + if "host" not in l: + l.append("host") + + if meta: + vars["_meta"] = _meta + + return vars + + @property + def lock_path(self): + return self.lock.create_path.rsplit("/", 1)[0] + + def is_locked(self): + """Return True if the host is locked by any client.""" + + # A Kazoo lock is a ZK "directory" of files, the lowest sequence + # number of which currently holds the lock and the rest of which + # are waiting/contending. Consequently there being children at the + # lock's create path is enough to say the lock is held. + return bool(self.client.get_children(self.lock_path)) + + def delete(self): + for group in self.groups(): + group.update(cl(update_in, ["host_vars"], cl(dissoc, self.name))) + self.remove_group(group) + + self.client.delete(self.key, recursive=True) + self._valid = False + + def remove_group(self, group: Union[KookGroup, str]): + group = self.client.group(group) + group.del_host_vars(self) + super().remove_group(group) + + def __repr__(self): + return "<{} {}>".format(__class__.__name__, self.key) + + +class LockProxy(object): + """Shim which behaves enough like a Kazoo lock to let me toggle locking.""" + + def __init__(self, lock, use_synchronization=True): + self.__lock = lock + self.__elapsed = 0 + self.__use_synchronization = use_synchronization + + def acquire(self, **kwargs): + if self.__use_synchronization: + try: + with Timer() as t: + return self.__lock.acquire(**kwargs) + finally: + self.__elapsed += t.elapsed() + + def release(self, **kwargs): + if self.__use_synchronization: + try: + with Timer() as t: + return self.__lock.release(**kwargs) + finally: + self.__elapsed += t.elapsed() + self.__elapsed = 0 + + @property + def create_path(self): + return self.__lock.create_path + + @property + def is_acquired(self): + if hasattr(self.__lock, "is_acquired"): + return self.__lock.is_acquired + elif isinstance(self.__lock, ReadLock): + return super(ReadLock, self.__lock).is_acquired + elif isinstance(self.__lock, WriteLock): + return super(WriteLock, self.__lock).is_acquired + + +class KookClient(object): + """Client to the Kook metadata store. + + Connects to a Zookeeper deployment - by default using the uri + `zookeeper:2181` - and uses a `KookConfig` instance (or the default instance + if none is provided) to begin fetching data out of the Kook data store. + + As a convenience to the implementation of Kook, the `KookClient` proxies a + number of methods on the `KazooClient`. Consumers of the `KookClient` are + encouraged not to rely on these methods. + + """ + + def __init__(self, hosts=None, client=None, config=None, use_synchronization=True): + self.config = config or current_config() + self.client = client or KazooClient(hosts=hosts or self.config.hosts) + self.client.start() + self.use_synchronization = use_synchronization + + # The various read locks + _slk = self.config.host_prefix + "/" + self.config.meta_lock_suffix + if not self.client.exists(_slk): + self.client.create(_slk, makepath=True) + # self.srl = self.create_rlock(self.slk) + self.swl = self.create_lock(_slk) + + _glk = self.config.group_prefix + "/" + self.config.meta_lock_suffix + if not self.client.exists(_glk): + self.client.create(_glk, makepath=True) + # self.grl = self.create_rlock(self.glk) + self.gwl = self.create_lock(_glk) + + self._groups = {} + self._hosts = {} + + # Proxies to the Kazoo client + #################################################################### + def start(self): + """Proxy to `KazooClient.start`.""" + + return self.client.start() + + def stop(self): + """Proxy to `KazooClient.stop`.""" + + return self.client.stop() + + def restart(self): + """Proxy to `KazooClient.restart`.""" + + return self.client.restart() + + def get(self, k, *args, **kwargs): + """Proxy to `KazooClient.get`.""" + + return self.client.get(k, *args, **kwargs) + + def get_children(self, *args, **kwargs): + """Proxy to `KazooClient.get_children`.""" + + return self.client.get_children(*args, **kwargs) + + def set(self, *args, **kwargs): + """Proxy to `KazooClient.set`.""" + + return self.client.set(*args, **kwargs) + + def exists(self, *args, **kwargs): + """Proxy to `KazooClient.exists`.""" + + return self.client.exists(*args, **kwargs) + + def create(self, *args, **kwargs): + """Proxy to `KazooClient.create`.""" + + try: + return self.client.create(*args, **kwargs) + except NodeExistsError: + pass + + def delete(self, *args, **kwargs): + """Proxy to `KazooClient.delete`.""" + + return self.client.delete(*args, **kwargs) + + def create_lock(self, *args, **kwargs): + """Wrapper around the Lock recipe.""" + + return LockProxy( + Lock(self.client, *args, **kwargs), + use_synchronization=self.use_synchronization, + ) + + def create_rlock(self, *args, **kwargs): + """Wrapper around the ReadLock recipe.""" + + return LockProxy( + ReadLock(self.client, *args, **kwargs), + use_synchronization=self.use_synchronization, + ) + + def create_wlock(self, *args, **kwargs): + """Wrapper around the WriteLock recipe.""" + + return LockProxy( + WriteLock(self.client, *args, **kwargs), + use_synchronization=self.use_synchronization, + ) + + # The intentional API + #################################################################### + def group(self, group, value=None) -> Optional[KookGroup]: + """Attempt to fetch a single group record.""" + + if isinstance(group, KookGroup): + return group + + if value is not None: + group = "{group}_{value}".format(**locals()) + + g = self._groups.get(group) + if g is None: + _key = "{}/{}".format(self.config.group_prefix, group) + if self.exists(_key): + g = KookGroup(self.config, self, group, exists=True) + + if g and not g._valid: + g = None + + if g: + self._groups[group] = g + + if not g: + print(f"Warning: unable to resolve group {g}", file=sys.stderr) + + return g + + def create_group(self, group, value=None, vars=None) -> KookGroup: + """Fetch a group if it exists, otherwise cause one to be created.""" + + vars = vars or {} + + if value is not None: + vars = merge(vars, {group: value}) + group = "{group}_{value}".format(**locals()) + + g = self._groups.get(group) + if g is None: + with lock(self.gwl): + g = self._groups[group] = KookGroup(self.config, self, group) + + # Apply the definitional k/v, and any extras. + g.update(cl(update_in, ["vars"], cl(merge, vars))) + return g + + def groups(self, watch=None) -> Iterable[KookGroup]: + """In no order, produce a list (eagerly!) of all the group records.""" + + mpath = self.config.meta_suffix.split("/", 1)[0] + return [ + self.group(groupname) + for groupname in self.client.get_children( + self.config.group_prefix, watch=watch + ) + if groupname != mpath + ] + + def delete_group(self, group: Union[KookGroup, str]): + """Given a group instance or string naming one, delete the group.""" + + self.group(group).delete() + + def host(self, host) -> Optional[KookHost]: + """Attempt to fetch a single host by name.""" + + if isinstance(host, KookHost): + return host + + h = self._hosts.get(host) + if h is None: + _key = "{}/{}".format(self.config.host_prefix, host) + if self.exists(_key): + h = KookHost(self.config, self, host, exists=True) + else: + return None + + self._hosts[host] = h + + if h._valid is False: + return None + + return h + + def create_host(self, host_name, vars=None) -> KookHost: + """Fetch a host if it exists, otherwise cause one to be created.""" + + vars = merge(vars or {}, {"host_name": host_name}) + h = self._hosts.get(host_name) + if h is None: + with lock(self.swl): + h = self._hosts[host_name] = KookHost(self.config, self, host_name) + + # Apply any host vars + h.update(cl(update_in, ["vars"], cl(merge, vars))) + return h + + def delete_host(self, host: Union[KookHost, str]): + """Given a host instance or a string naming one, delete the host.""" + + self.host(host).delete() + + def hosts(self, watch=None) -> Iterable[KookHost]: + """In no order, produce a list (eagerly!) of all the host records.""" + + mpath = self.config.meta_suffix.split("/", 1)[0] + return [ + self.host(host_name) + for host_name in self.client.get_children( + self.config.host_prefix, watch=watch + ) + if host_name != mpath + ] diff --git a/projects/kook/src/kook/config.py b/projects/kook/src/kook/config.py new file mode 100644 index 0000000..82ef3c3 --- /dev/null +++ b/projects/kook/src/kook/config.py @@ -0,0 +1,46 @@ +"""Configuration for the Kook client.""" + +import os + +import yaml + + +class KookConfig(object): + """A config type used to control the keys Kook uses.""" + + def __init__( + self, + hosts="zookeeper:2181", + host_prefix="/kook/host", + host_schema=None, + host_ttl=None, + group_prefix="/kook/group", + lock_suffix="lock", + meta_suffix="meta", + ): + + if isinstance(hosts, list): + hosts = ",".join(hosts) + + self.hosts = hosts + self.host_prefix = host_prefix + self.host_schema = host_schema + self.host_ttl = host_ttl + + self.group_prefix = group_prefix + + assert not lock_suffix.startswith("/") + self.lock_suffix = lock_suffix + + assert not meta_suffix.startswith("/") + self.meta_suffix = meta_suffix + + self.meta_lock_suffix = meta_suffix + "/" + lock_suffix + + +def current_config(path="/etc/kook.yml") -> KookConfig: + """Either the default config, or a config drawn from /etc/kook.yml""" + + path = os.getenv("KOOK_CONFIG", path) + + return KookConfig(**(yaml.safe_load(open(path)) if os.path.exists(path) else {})) diff --git a/projects/kook/test.sh b/projects/kook/test.sh new file mode 100644 index 0000000..70313bf --- /dev/null +++ b/projects/kook/test.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +TEST_HOST=sabod-rital.test.arrdem.com + +function kook_setup { + kook -g geo_test group delete + kook -g rack_test_test group delete + kook -H $TEST_HOST host delete +} + +function kook_test { + ################################################## + # Group CRU + if kook group create geo_test; then + echo "[OK] Geo group created" + else + echo "[FAIL] Geo group not created" + fi + + if kook -g geo_test var create geo test; then + echo "[OK] Var created" + else + echo "[FAIL] Var create failed" + fi + + if [ $(kook -g geo_test group details | jq '.geo_test.geo == "test"') == "true" ]; then + echo "[OK] Var read after write" + else + echo "[FAIL] Var not read back" + fi + + # These two paths have already been tested + kook group create rack_test_test + kook -g rack_test_test var create rack test + + # Testing vars on parent groups + if kook -g rack_test_test group add geo_test; then + echo "[OK] Group-group added" + else + echo "[FAIL] Group-group not added" + fi + + if [ $(kook -g rack_test_test group details | jq '.rack_test_test.geo == "test"') == "true" ]; then + echo "[OK] 'geo' var inherited from supergroup" + else + echo "[FAIL] Var not inherited!" + fi + + if [ $(kook -g rack_test_test group details | jq '.rack_test_test.rack == "test"') == "true" ]; then + echo "[OK] 'rack' var set" + else + echo "[FAIL] Var not set!" + fi + + ################################################## + # Host CRU + if kook host create $TEST_HOST; then + echo "[OK] Host created" + else + echo "[FAIL] Host not created" + fi + + if kook host list | grep -q $TEST_HOST > /dev/null; then + echo "[OK] Host read after write" + else + echo "[FAIL] Host not read" + fi + + if kook -H $TEST_HOST var create foo bar; then + echo "[OK] Var created" + else + echo "[FAIL] Unable to create host var" + fi + + if [ $(kook -H $TEST_HOST host details | jq ".[\"${TEST_HOST}\"].foo == \"bar\"") == "true" ]; then + echo "[OK] Able to read var back" + else + echo "[FAIL] Unable to read host var back!" + fi + + # And for a host... + if kook -H $TEST_HOST group add rack_test_test; then + echo "[OK] Added host to rack group" + else + echo "[FAIL] Unable to add host to rack group" + fi + + # Checking that the host is in both group's list. + if kook -g rack_test_test host list | grep -q $TEST_HOST; then + echo "[OK] Host is in the rack group" + else + echo "[FAIL] Host is not in the rack group!" + fi + + # And in the geo's list. + if kook -g geo_test host list | grep -q $TEST_HOST; then + echo "[OK] Host is in the geo group" + else + echo "[FAIL] Host is not in the geo group!" + fi +} + +function kook_teardown { + kook_setup +} + +kook_setup 1>/dev/null 2>/dev/null +kook_test +kook_teardown 1>/dev/null 2>/dev/null diff --git a/projects/kook/test/test_client.py b/projects/kook/test/test_client.py new file mode 100644 index 0000000..834c6bf --- /dev/null +++ b/projects/kook/test/test_client.py @@ -0,0 +1,3 @@ +""" +Client tests. +"""