A second cut at the cram tool, now with TOML and tests

This commit is contained in:
Reid D. 'arrdem' McKenzie 2022-02-15 02:17:54 -07:00
parent 011bd6019b
commit 735a3c65d8
17 changed files with 482 additions and 150 deletions

View file

@ -1,12 +1,28 @@
zapp_binary( py_library(
name = "cram", name = "lib",
main = "src/python/cram/__main__.py", srcs = glob(["src/python/**/*.py"]),
imports = [
"src/python"
],
deps = [ deps = [
"//projects/vfs", "//projects/vfs",
py_requirement("click"), py_requirement("click"),
py_requirement("toposort"), py_requirement("toposort"),
py_requirement("toml"),
] ]
) )
zapp_binary(
name = "cram",
main = "src/python/cram/__main__.py",
shebang = "/usr/bin/env python3",
imports = [
"src/python"
],
deps = [
":lib",
],
)
sh_test(
name = "test_cram",
srcs = glob(["test.sh"]),
data = glob(["test/**/*"]) + [":cram"],
)

View file

@ -1,13 +1,15 @@
#!/usr/bin/env python3 """Cram's entry point."""
from itertools import chain from itertools import chain
import logging import logging
import os import os
from pathlib import Path from pathlib import Path
import pickle import pickle
import re
import sys import sys
from typing import NamedTuple from typing import List
from .v0 import PackageV0, ProfileV0
from .v1 import PackageV1, ProfileV1
from vfs import Vfs from vfs import Vfs
@ -18,138 +20,51 @@ from toposort import toposort_flatten
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def stow(fs: Vfs, src_dir: Path, dest_dir: Path, skip=[]): def load(root: Path, name: str, clss):
"""Recursively 'stow' (link) the contents of the source into the destination.""" for c in clss:
i = c(root, name)
dest_root = Path(dest_dir) if i.test():
src_root = Path(src_dir) return i
skip = [src_root / n for n in skip]
for src in src_root.glob("**/*"):
if src in skip:
continue
dest = dest_root / src.relative_to(src_root)
if src.is_dir():
fs.mkdir(dest)
fs.chmod(dest, src.stat().st_mode)
elif src.is_file():
fs.link(src, dest)
class PackageV0(NamedTuple): def load_package(root, name):
"""The original package format from install.sh.""" return load(root, name, [PackageV1, PackageV0])
root: Path
name: str
subpackages: bool = False
SPECIAL_FILES = ["BUILD", "PRE_INSTALL", "INSTALL", "POST_INSTALL", "REQUIRES"]
def requires(self):
"""Get the dependencies of this package."""
requiresf = self.root / "REQUIRES"
requires = []
# Listed dependencies
if requiresf.exists():
with open(requiresf) as fp:
for l in fp:
l = l.strip()
l = re.sub(r"\s*#.*\n", "", l)
if l:
requires.append(l)
# Implicitly depended subpackages
if self.subpackages:
for p in self.root.glob("*"):
if p.is_dir():
requires.append(self.name + "/" + p.name)
return requires
def install(self, fs: Vfs, dest: Path):
"""Install this package."""
buildf = self.root / "BUILD"
if buildf.exists():
fs.exec(self.root, ["bash", str(buildf)])
pref = self.root / "PRE_INSTALL"
if pref.exists():
fs.exec(self.root, ["bash", str(pref)])
installf = self.root / "INSTALL"
if installf.exists():
fs.exec(self.root, ["bash", str(installf)])
else:
stow(fs, self.root, dest, self.SPECIAL_FILES)
postf = self.root / "POST_INSTALL"
if postf.exists():
fs.exec(self.root, ["bash", str(postf)])
class ProfileV0(PackageV0): def load_profile(root, name):
def install(self, fs: Vfs, dest: Path): return load(root, name, [ProfileV1, ProfileV0])
"""Profiles differ from Packages in that they don't support literal files."""
buildf = self.root / "BUILD"
if buildf.exists():
fs.exec(self.root, ["bash", str(buildf)])
pref = self.root / "PRE_INSTALL"
if pref.exists():
fs.exec(self.root, ["bash", str(pref)])
installf = self.root / "INSTALL"
if installf.exists():
fs.exec(self.root, ["bash", str(installf)])
postf = self.root / "POST_INSTALL"
if postf.exists():
fs.exec(self.root, ["bash", str(postf)])
def load_config(root: Path) -> dict: def load_packages(root: Path) -> dict:
"""Load the configured packages.""" """Load the configured packages."""
packages = { packages = {}
str(p.relative_to(root)): PackageV0(p, str(p.relative_to(root))) for p in (root / "packages.d").glob("*"):
for p in (root / "packages.d").glob("*") name = str(p.relative_to(root))
} packages[name] = load_package(p, name)
# Add profiles, hosts which contain subpackages. # Add profiles, hosts which contain subpackages.
for mp_root in chain((root / "profiles.d").glob("*"), (root / "hosts.d").glob("*")): for mp_root in chain((root / "profiles.d").glob("*"), (root / "hosts.d").glob("*")):
# First find all subpackages # First find all subpackages
for p in mp_root.glob( for p in mp_root.glob("*"):
"*",
):
if p.is_dir(): if p.is_dir():
packages[str(p.relative_to(root))] = PackageV0( name = str(p.relative_to(root))
p, str(p.relative_to(root)) packages[name] = load_package(p, name)
)
# Register the metapackages themselves using the profile type # Register the metapackages themselves using the profile type
packages[str(mp_root.relative_to(root))] = ProfileV0( mp_name = str(mp_root.relative_to(root))
mp_root, str(mp_root.relative_to(root)), True packages[mp_name] = load_profile(mp_root, mp_name)
)
return packages return packages
def build_fs(root: Path, dest: Path) -> Vfs: def build_fs(root: Path, dest: Path, prelude: List[str]) -> Vfs:
"""Build a VFS by configuring dest from the given config root.""" """Build a VFS by configuring dest from the given config root."""
packages = load_config(root) packages = load_packages(root)
requirements = []
hostname = os.uname()[1] requirements.extend(prelude)
# Compute the closure of packages to install
requirements = [
f"hosts.d/{hostname}",
"profiles.d/default",
]
for r in requirements: for r in requirements:
try: try:
@ -172,7 +87,7 @@ def build_fs(root: Path, dest: Path) -> Vfs:
return fs return fs
def load_fs(statefile: Path) -> Vfs: def load_state(statefile: Path) -> Vfs:
"""Load a persisted VFS state from disk. Sort of.""" """Load a persisted VFS state from disk. Sort of."""
oldfs = Vfs() oldfs = Vfs()
@ -187,7 +102,7 @@ def load_fs(statefile: Path) -> Vfs:
return oldfs return oldfs
def simplify(old_fs: Vfs, new_fs: Vfs) -> Vfs: def simplify(old_fs: Vfs, new_fs: Vfs, /, exec_idempotent=True) -> Vfs:
"""Try to reduce a new VFS using diff from the original VFS.""" """Try to reduce a new VFS using diff from the original VFS."""
old_fs = old_fs.copy() old_fs = old_fs.copy()
@ -196,11 +111,21 @@ def simplify(old_fs: Vfs, new_fs: Vfs) -> Vfs:
# Scrub anything in the new log that's in the old log # Scrub anything in the new log that's in the old log
for txn in list(old_fs._log): for txn in list(old_fs._log):
# Except for execs which are stateful # Except for execs which are stateful
if txn[0] == "exec": if txn[0] == "exec" and not exec_idempotent:
continue continue
try:
new_fs._log.remove(txn) new_fs._log.remove(txn)
old_fs._log.remove(txn) except ValueError:
pass
# Dedupe the new log while preserving order
distinct = set()
for txn, idx in zip(new_fs._log, range(len(new_fs._log))):
if txn in distinct:
new_fs._log.pop(idx)
else:
distinct.add(txn)
return new_fs return new_fs
@ -227,13 +152,15 @@ def cli():
pass pass
@cli.command() @cli.command("apply")
@click.option("--execute/--dry-run", default=False) @click.option("--execute/--dry-run", default=False)
@click.option("--state-file", default=".cram.log", type=Path) @click.option("--state-file", default=".cram.log", type=Path)
@click.option("--optimize/--no-optimize", default=False) @click.option("--optimize/--no-optimize", default=True)
@click.option("--require", type=str, multiple=True, default=[f"hosts.d/{os.uname()[1]}", "profiles.d/default"])
@click.option("--exec-idempotent/--exec-always", "exec_idempotent", default=True)
@click.argument("confdir", type=Path) @click.argument("confdir", type=Path)
@click.argument("destdir", type=Path) @click.argument("destdir", type=Path)
def apply(confdir, destdir, state_file, execute, optimize): def do_apply(confdir, destdir, state_file, execute, optimize, require, exec_idempotent):
"""The entry point of cram.""" """The entry point of cram."""
# Resolve the two input paths to absolutes # Resolve the two input paths to absolutes
@ -242,46 +169,37 @@ def apply(confdir, destdir, state_file, execute, optimize):
if not state_file.is_absolute(): if not state_file.is_absolute():
state_file = root / state_file state_file = root / state_file
new_fs = build_fs(root, dest) new_fs = build_fs(root, dest, require)
old_fs = load_fs(state_file) old_fs = load_state(state_file)
# Middleware processing of the resulting filesystem(s) # Middleware processing of the resulting filesystem(s)
executable_fs = scrub(old_fs, new_fs) executable_fs = scrub(old_fs, new_fs)
if optimize: if optimize:
executable_fs = simplify(old_fs, new_fs) executable_fs = simplify(old_fs, new_fs,
exec_idempotent=exec_idempotent)
# Dump the new state. # Dump the new state.
# Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states. # Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states.
def cb(e):
print("-", *e)
if execute: if execute:
executable_fs.execute() executable_fs.execute(callback=cb)
with open(state_file, "wb") as fp: with open(state_file, "wb") as fp:
pickle.dump(new_fs._log, fp) pickle.dump(new_fs._log, fp)
else: else:
for e in executable_fs._log: for e in executable_fs._log:
print("-", e) cb(e)
@cli.command() @cli.command("list")
@click.option("--state-file", default=".cram.log", type=Path)
@click.argument("confdir", type=Path)
def show(confdir, state_file):
"""List out the last `apply` state in the <confdir>/.cram.log or --state-file."""
root = confdir.resolve()
if not state_file.is_absolute():
state_file = root / state_file
fs = load_fs(state_file)
for e in fs._log:
print(*e)
@cli.command()
@click.argument("confdir", type=Path) @click.argument("confdir", type=Path)
@click.argument("list_packages", nargs=-1) @click.argument("list_packages", nargs=-1)
def list(confdir, list_packages): def do_list(confdir, list_packages):
"""List out packages, profiles, hosts and subpackages in the <confdir>.""" """List out packages, profiles, hosts and subpackages in the <confdir>."""
packages = load_config(confdir) packages = load_packages(confdir)
if list_packages: if list_packages:
dest = Path("~/") dest = Path("~/")
@ -305,9 +223,22 @@ def list(confdir, list_packages):
print(f"- {d}") print(f"- {d}")
@cli.command("state")
@click.option("--state-file", default=".cram.log", type=Path)
@click.argument("confdir", type=Path)
def do_state(confdir, state_file):
"""List out the last `apply` state in the <confdir>/.cram.log or --state-file."""
root = confdir.resolve()
if not state_file.is_absolute():
state_file = root / state_file
fs = load_state(state_file)
for e in fs._log:
print(*e)
if __name__ == "__main__" or 1: if __name__ == "__main__" or 1:
logging.basicConfig( logging.basicConfig(
level=logging.DEBUG, level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
) )

View file

@ -0,0 +1,73 @@
#!/usr/bin/env python3
from abc import abstractmethod
from pathlib import Path
from shlex import quote as sh_quote
from typing import List, Optional
from vfs import Vfs
SHELL = "/bin/sh"
def sh(cmd: List[str], /,
env: Optional[dict] = None):
prefix = []
if env:
prefix.append("/usr/bin/env")
for k, v in env.items():
v = sh_quote(str(v))
prefix.append(f"{k}={v}")
return tuple(prefix + [SHELL, *cmd])
def stow(fs: Vfs, src_dir: Path, dest_dir: Path, skip=[]):
"""Recursively 'stow' (link) the contents of the source into the destination."""
dest_root = Path(dest_dir)
src_root = Path(src_dir)
skip = [src_root / n for n in skip]
for src in src_root.glob("**/*"):
if src in skip:
continue
dest = dest_root / src.relative_to(src_root)
if src.is_dir():
fs.mkdir(dest)
fs.chmod(dest, src.stat().st_mode)
elif src.is_file():
fs.link(src, dest)
class Package(object):
def __init__(self, root: Path, name: str):
self.root = root
self.name = name
def test(self):
return True
def requires(self):
return []
def install(self, fs: Vfs, dest: Path):
self.do_build(fs, dest)
self.pre_install(fs, dest)
self.do_install(fs, dest)
self.post_install(fs, dest)
def do_build(self, fs: Vfs, dest: Path):
pass
def pre_install(self, fs: Vfs, dest: Path):
pass
def do_install(self, fs: Vfs, dest: Path):
pass
def post_install(self, fs: Vfs, dest: Path):
pass

View file

@ -0,0 +1,82 @@
"""Cram's original (v0) configs.
An ill-considered pseudo-format.
"""
from pathlib import Path
import re
from typing import NamedTuple
from .common import Package, sh, stow
from vfs import Vfs
class PackageV0(Package):
"""The original package format from install.sh."""
SPECIAL_FILES = ["BUILD", "PRE_INSTALL", "INSTALL", "POST_INSTALL", "REQUIRES"]
def requires(self):
"""Get the dependencies of this package."""
requiresf = self.root / "REQUIRES"
requires = []
# Listed dependencies
if requiresf.exists():
with open(requiresf) as fp:
for l in fp:
l = l.strip()
l = re.sub(r"\s*#.*\n", "", l)
if l:
requires.append(l)
return requires
def install(self, fs: Vfs, dest: Path):
"""Install this package."""
buildf = self.root / "BUILD"
if buildf.exists():
fs.exec(self.root, sh([str(buildf)]))
pref = self.root / "PRE_INSTALL"
if pref.exists():
fs.exec(self.root, sh([str(pref)]))
installf = self.root / "INSTALL"
if installf.exists():
fs.exec(self.root, sh([str(installf)]))
else:
stow(fs, self.root, dest, self.SPECIAL_FILES)
postf = self.root / "POST_INSTALL"
if postf.exists():
fs.exec(self.root, sh([str(postf)]))
class ProfileV0(PackageV0):
def requires(self):
requires = super().requires()
for p in self.root.glob("*"):
if p.is_dir():
requires.append(self.name + "/" + p.name)
return requires
def install(self, fs: Vfs, dest: Path):
"""Profiles differ from Packages in that they don't support literal files."""
buildf = self.root / "BUILD"
if buildf.exists():
fs.exec(self.root, sh([str(buildf)]))
pref = self.root / "PRE_INSTALL"
if pref.exists():
fs.exec(self.root, sh([str(pref)]))
installf = self.root / "INSTALL"
if installf.exists():
fs.exec(self.root, sh([str(installf)]))
postf = self.root / "POST_INSTALL"
if postf.exists():
fs.exec(self.root, sh([str(postf)]))

View file

@ -0,0 +1,103 @@
"""Cram's v1 configs.
Based on well* defined TOML manifests, rather than many files.
*Okay. Better.
"""
from hashlib import sha256
from pathlib import Path
from typing import List, Optional, Union
from typing import NamedTuple
from .common import Package, sh, stow
from vfs import Vfs
import toml
def tempf(name):
root = Path("/tmp/stow")
root.mkdir(exist_ok=True, parents=True)
return root / name
class PackageV1(Package):
"""The v1 package format."""
SPECIAL_FILES = ["config.toml"]
_config = None
def config(self):
if not self._config:
with open(self.root / self.SPECIAL_FILES[0], "r") as fp:
self._config = toml.load(fp)
return self._config
def test(self):
return (self.root / self.SPECIAL_FILES[0]).exists() and self.config().get("cram", {}).get("version") == 1
def requires(self):
"""Get the dependencies of this package."""
return self.config().get("package", {}).get("requires") or [
it["name"] for it in self.config().get("package", {}).get("require", [])
]
def do_sh_or_script(self, content: Optional[Union[List[str], str]], fs: Vfs, dest: Path, cwd: Path = "/tmp"):
if content is None:
pass
elif isinstance(content, list):
return any(self.do_sh_or_script(c, fs, dest) for c in content)
elif isinstance(content, dict):
return self.do_sh_or_script(content["run"], fs, dest, {"cwd": self.root}.get(content.get("root"), "/tmp"))
elif isinstance(content, str):
sum = sha256()
sum.update(content.encode("utf-8"))
sum = sum.hexdigest()
installf = self.root / content
if installf.exists():
with open(installf, "r") as fp:
return self.do_sh_or_script(fp.read(), fs, dest)
elif content:
f = tempf(f"{sum}.sh")
with open(f, "w") as fp:
fp.write(content)
fs.exec(cwd, sh([f]))
return True
def do_build(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("build"), fs, dest)
def pre_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("pre_install"), fs, dest)
def do_install(self, fs: Vfs, dest: Path):
if not self.do_sh_or_script(self.config().get("package", {}).get("install"), fs, dest):
stow(fs, self.root, dest, self.SPECIAL_FILES)
def post_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("post_install"), fs, dest)
class ProfileV1(PackageV1):
"""Unline packages, profiles don't support recursive stow of contents."""
def do_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("install"), fs, dest)
def requires(self):
requires = super().requires()
# Implicitly depended subpackages
for p in self.root.glob("*"):
if p.is_dir():
requires.append(self.name + "/" + p.name)
return requires

85
projects/cram/test.sh Executable file
View file

@ -0,0 +1,85 @@
#!/usr/bin/env bash
set -ex
cd projects/cram
dest=$(mktemp -d)
./cram --help
# Should be able to list all packages
./cram list test/ | grep "packages.d/p1"
# P3 depends on P1, should show up in the listing
./cram list test/ packages.d/p3 | grep "packages.d/p1"
# P4 depends on P3, should show up in the listing
./cram list test/ packages.d/p4 | grep "packages.d/p3"
# The default profile should depend on its subpackage
./cram list test/ profiles.d/default | grep "profiles.d/default/subpackage"
# And the subpackage has a dep
./cram list test/ profiles.d/default/subpackage | grep "packages.d/p3"
# Install one package
./cram apply --no-optimize --require packages.d/p1 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
./cram state test/ | grep "${dest}/foo"
rm -r "${dest}"/*
# Install two transitively (legacy)
./cram apply --no-optimize --require packages.d/p3 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
./cram state test/ | grep "${dest}/foo"
./cram state test/ | grep "${dest}/bar"
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require hosts.d/test --require profiles.d/default --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# INSTALL scripts get run as-is
./cram list test/ packages.d/p5 | grep "packages.d/p5/INSTALL"
# Inline scripts get pulled out repeatably
./cram list test/ packages.d/p6 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"
# Inline scripts get pulled out repeatably, even from the list format
./cram list test/ packages.d/p7 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"
# Test log-based optimization
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
# These paths were already linked, they shouldn't be re-linked when optimizing.
! ./cram apply --require packages.d/p4 --optimize --execute test/ "${dest}" | grep "${dest}/foo"
! ./cram apply --require packages.d/p4 --optimize --execute test/ "${dest}" | grep "${dest}/bar"
rm -r "${dest}"/*
# Likewise, if we've exec'd this once we shouldn't do it again
./cram apply --no-optimize --require packages.d/p5 --execute test/ "${dest}"
! ./cram apply --require packages.d/p5 --execute test/ "${dest}" | grep "exec"
# ... unless the user tells us to
./cram apply --no-optimize --require packages.d/p5 --execute test/ "${dest}"
./cram apply --exec-always --require packages.d/p5 --execute test/ "${dest}" | grep "exec"
# If multiple packages provide the same _effective_ script, do it once
./cram apply --require packages.d/p6 --require packages.d/p7 --execute test/ "${dest}" | sort | uniq -c | grep "/tmp/stow/b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b.sh" | grep "1 - exec"

View file

@ -0,0 +1,6 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p1"

View file

@ -0,0 +1 @@
bar

View file

@ -0,0 +1 @@
qux

View file

@ -0,0 +1,2 @@
packages.d/p1
packages.d/p2

View file

@ -0,0 +1,6 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p3"

View file

@ -0,0 +1,3 @@
#!/bin/bash
# A legacy custom install script
true

View file

@ -0,0 +1,5 @@
[cram]
version = 1
[package]
install = "true"

View file

@ -0,0 +1,6 @@
[cram]
version = 1
[package]
[[package.install]]
run = "true"

View file

@ -0,0 +1,4 @@
[cram]
version = 1
[package]

View file

@ -0,0 +1,6 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p3"

View file

@ -16,9 +16,11 @@ class Vfs(object):
def __init__(self, log=None): def __init__(self, log=None):
self._log = log or [] self._log = log or []
def execute(self): def execute(self, /, callback=None):
for e in self._log: for e in self._log:
_log.debug(e) _log.debug(e)
if callback:
callback(e)
if e[0] == "exec": if e[0] == "exec":
_, dir, cmd = e _, dir, cmd = e