From 75a56695c04ff1607ecd521224c02ffb1dc0aee6 Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Sun, 31 Oct 2021 10:59:27 -0600 Subject: [PATCH] Teach cram about profile and host subpackages --- projects/cram/BUILD | 1 + projects/cram/README.md | 1 + projects/cram/src/python/cram/__main__.py | 156 ++++++++++++++++------ 3 files changed, 116 insertions(+), 42 deletions(-) diff --git a/projects/cram/BUILD b/projects/cram/BUILD index e2c8edd..b700b33 100644 --- a/projects/cram/BUILD +++ b/projects/cram/BUILD @@ -6,6 +6,7 @@ zapp_binary( ], deps = [ "//projects/vfs", + py_requirement("click"), py_requirement("toposort"), ] ) diff --git a/projects/cram/README.md b/projects/cram/README.md index d94c70e..458d482 100644 --- a/projects/cram/README.md +++ b/projects/cram/README.md @@ -29,6 +29,7 @@ Cram reads a config dir with three groups of packages Configuration should be left to profiles. - `profiles.d/` contains a profile; a group of related profiles and packages that should be installed together. - `hosts.d/` contains one package for each host, and should pull in a list of profiles. +- Both profiles and hosts entries may specify their own "inline" packages as a convenience. The intent of this tool is to keep GNU Stow's intuitive model of deploying configs via symlinks, and augment it with a useful pattern for talking about "layers" / "packages" of related configs. diff --git a/projects/cram/src/python/cram/__main__.py b/projects/cram/src/python/cram/__main__.py index 9dd9684..e09e104 100644 --- a/projects/cram/src/python/cram/__main__.py +++ b/projects/cram/src/python/cram/__main__.py @@ -1,29 +1,21 @@ #!/usr/bin/env python3 -import argparse from itertools import chain import logging import os from pathlib import Path import pickle +import re import sys from typing import NamedTuple +import click from toposort import toposort_flatten from vfs import Vfs log = logging.getLogger(__name__) -parser = argparse.ArgumentParser() -parser.add_argument("-x", "--execute", dest="execute", action="store_true", default=False) -parser.add_argument("-d", "--dry-run", dest="execute", action="store_false") -parser.add_argument("-s", "--state-file", dest="statefile", default=".cram.log") -parser.add_argument("--optimize", dest="optimize", default=False, action="store_true") -parser.add_argument("--no-optimize", dest="optimize", action="store_false") -parser.add_argument("confdir", type=Path) -parser.add_argument("destdir", type=Path) - def stow(fs: Vfs, src_dir: Path, dest_dir: Path, skip=[]): """Recursively 'stow' (link) the contents of the source into the destination.""" @@ -50,16 +42,31 @@ class PackageV0(NamedTuple): root: Path name: str + subpackages: bool = False SPECIAL_FILES = ["BUILD", "PRE_INSTALL", "INSTALL", "POST_INSTALL", "REQUIRES"] def requires(self): """Get the dependencies of this package.""" requiresf = self.root / "REQUIRES" + requires = [] + + # Listed dependencies if requiresf.exists(): with open(requiresf) as fp: - return [l.strip() for l in fp] - return [] + for l in fp: + l = l.strip() + l = re.sub(r"\s*#.*\n", "", l) + if l: + requires.append(l) + + # Implicitly depended subpackages + if self.subpackages: + for p in self.root.glob("*"): + if p.is_dir(): + requires.append(self.name + "/" + p.name) + + return requires def install(self, fs: Vfs, dest): """Install this package.""" @@ -82,13 +89,31 @@ class PackageV0(NamedTuple): fs.exec(self.root, ["bash", str(postf)]) +def load_config(root: Path) -> dict: + """Load the configured packages.""" + + packages = {str(p.relative_to(root)): PackageV0(p, str(p.relative_to(root))) + for p in (root / "packages.d").glob("*")} + + # Add profiles, hosts which contain subpackages. + for mp_root in chain((root / "profiles.d").glob("*"), + (root / "hosts.d").glob("*")): + + # First find all subpackages + for p in mp_root.glob("*", ): + if p.is_dir(): + packages[str(p.relative_to(root))] = PackageV0(p, str(p.relative_to(root))) + + # Register the metapackages themselves + packages[str(mp_root.relative_to(root))] = PackageV0(mp_root, str(mp_root.relative_to(root)), True) + + return packages + + def build_fs(root: Path, dest: Path) -> Vfs: """Build a VFS by configuring dest from the given config root.""" - - packages = {str(p.relative_to(root)): PackageV0(p, str(p)) - for p in chain((root / "packages.d").glob("*"), - (root / "profiles.d").glob("*"), - (root / "hosts.d").glob("*"))} + + packages = load_config(root) hostname = os.uname()[1] @@ -125,8 +150,11 @@ def load_fs(statefile: Path) -> Vfs: oldfs = Vfs() if statefile.exists(): + log.debug("Loading statefile %s", statefile) with open(statefile, "rb") as fp: oldfs._log = pickle.load(fp) + else: + log.warning("No previous statefile %s", statefile) return oldfs @@ -146,6 +174,15 @@ def simplify(old_fs: Vfs, new_fs: Vfs) -> Vfs: new_fs._log.remove(txn) old_fs._log.remove(txn) + return new_fs + + +def scrub(old_fs: Vfs, new_fs: Vfs) -> Vfs: + """Try to eliminate files which were previously installed but are no longer used.""" + + old_fs = old_fs.copy() + new_fs = new_fs.copy() + # Look for files in the old log which are no longer present in the new log for txn in old_fs._log: if txn[0] == "link" and txn not in new_fs._log: @@ -157,36 +194,71 @@ def simplify(old_fs: Vfs, new_fs: Vfs) -> Vfs: return new_fs -def main(): +@click.group() +def cli(): + pass + + +@cli.command() +@click.option("--execute/--dry-run", default=False) +@click.option("--state-file", default=".cram.log") +@click.option("--optimize/--no-optimize", default=False) +@click.argument("confdir", type=Path) +@click.argument("destdir", type=Path) +def apply(confdir, destdir, state_file, execute, optimize): """The entry point of cram.""" - opts, args = parser.parse_known_args() + # Resolve the two input paths to absolutes + root = confdir.resolve() + dest = destdir.resolve() + statef = root / state_file + new_fs = build_fs(root, dest) + old_fs = load_fs(statef) + + # Middleware processing of the resulting filesystem(s) + executable_fs = scrub(old_fs, new_fs) + if optimize: + executable_fs = simplify(old_fs, new_fs) + + # Dump the new state. + # Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states. + if execute: + executable_fs.execute() + + with open(statef, "wb") as fp: + pickle.dump(new_fs._log, fp) + + else: + for e in executable_fs._log: + print("-", e) + + +@cli.command() +@click.option("--state-file", default=".cram.log") +@click.argument("confdir", type=Path) +def show(confdir, state_file): + root = confdir.resolve() + statef = root / state_file + fs = load_fs(statef) + for e in fs._log: + print(*e) + + +@cli.command() +@click.argument("confdir", type=Path) +def list(confdir): + packages = load_config(confdir) + for pname in sorted(packages.keys()): + p = packages[pname] + print(f"{pname}:") + for d in p.requires(): + print(f" - {d}") + +if __name__ == "__main__" or 1: logging.basicConfig( level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) - # Resolve the two input paths to absolutes - root = opts.confdir.resolve() - dest = opts.destdir.resolve() - statef = root / opts.statefile - - new_fs = build_fs(root, dest) - old_fs = load_fs(statef) - - if opts.optimize: - fast_fs = simplify(old_fs, new_fs) - fast_fs.execute(opts.execute) - else: - new_fs.execute(opts.execute) - - # Dump the new state. - # Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states. - if opts.execute: - with open(statef, "wb") as fp: - pickle.dump(new_fs._log, fp) - - -if __name__ == "__main__" or 1: - main() + cli()