Teach cram about profile and host subpackages

projects/cram/BUILD

@@ -6,6 +6,7 @@ zapp_binary(
     ],
     deps = [
         "//projects/vfs",
+        py_requirement("click"),
         py_requirement("toposort"),
     ]
 )

projects/cram/README.md

@@ -29,6 +29,7 @@ Cram reads a config dir with three groups of packages
   Configuration should be left to profiles.
 - `profiles.d/<profilename>` contains a profile; a group of related profiles and packages that should be installed together.
 - `hosts.d/<hostname>` contains one package for each host, and should pull in a list of profiles.
+- Both profiles and hosts entries may specify their own "inline" packages as a convenience.
 
 The intent of this tool is to keep GNU Stow's intuitive model of deploying configs via symlinks, and augment it with a useful pattern for talking about "layers" / "packages" of related configs.
 

projects/cram/src/python/cram/__main__.py

@@ -1,29 +1,21 @@
 #!/usr/bin/env python3
 
-import argparse
 from itertools import chain
 import logging
 import os
 from pathlib import Path
 import pickle
+import re
 import sys
 from typing import NamedTuple
 
+import click
 from toposort import toposort_flatten
 from vfs import Vfs
 
 
 log = logging.getLogger(__name__)
 
-parser = argparse.ArgumentParser()
-parser.add_argument("-x", "--execute", dest="execute", action="store_true", default=False)
-parser.add_argument("-d", "--dry-run", dest="execute", action="store_false")
-parser.add_argument("-s", "--state-file", dest="statefile", default=".cram.log")
-parser.add_argument("--optimize", dest="optimize", default=False, action="store_true")
-parser.add_argument("--no-optimize", dest="optimize", action="store_false")
-parser.add_argument("confdir", type=Path)
-parser.add_argument("destdir", type=Path)
-
 
 def stow(fs: Vfs, src_dir: Path, dest_dir: Path, skip=[]):
     """Recursively 'stow' (link) the contents of the source into the destination."""
@@ -50,16 +42,31 @@ class PackageV0(NamedTuple):
 
     root: Path
     name: str
+    subpackages: bool = False
 
     SPECIAL_FILES = ["BUILD", "PRE_INSTALL", "INSTALL", "POST_INSTALL", "REQUIRES"]
 
     def requires(self):
         """Get the dependencies of this package."""
         requiresf = self.root / "REQUIRES"
+        requires = []
+
+        # Listed dependencies
         if requiresf.exists():
             with open(requiresf) as fp:
-                return [l.strip() for l in fp]
+                for l in fp:
-        return []
+                    l = l.strip()
+                    l = re.sub(r"\s*#.*\n", "", l)
+                    if l:
+                        requires.append(l)
+
+        # Implicitly depended subpackages
+        if self.subpackages:
+            for p in self.root.glob("*"):
+                if p.is_dir():
+                    requires.append(self.name + "/" + p.name)
+
+        return requires
 
     def install(self, fs: Vfs, dest):
         """Install this package."""
@@ -82,13 +89,31 @@ class PackageV0(NamedTuple):
             fs.exec(self.root, ["bash", str(postf)])
 
 
+def load_config(root: Path) -> dict:
+    """Load the configured packages."""
+
+    packages = {str(p.relative_to(root)): PackageV0(p, str(p.relative_to(root)))
+                for p in (root / "packages.d").glob("*")}
+
+    # Add profiles, hosts which contain subpackages.
+    for mp_root in chain((root / "profiles.d").glob("*"),
+                         (root / "hosts.d").glob("*")):
+
+        # First find all subpackages
+        for p in mp_root.glob("*", ):
+            if p.is_dir():
+                packages[str(p.relative_to(root))] = PackageV0(p, str(p.relative_to(root)))
+
+        # Register the metapackages themselves
+        packages[str(mp_root.relative_to(root))] = PackageV0(mp_root, str(mp_root.relative_to(root)), True)
+
+    return packages
+
+
 def build_fs(root: Path, dest: Path) -> Vfs:
     """Build a VFS by configuring dest from the given config root."""
-    
+
-    packages = {str(p.relative_to(root)): PackageV0(p, str(p))
+    packages = load_config(root)
-                for p in chain((root / "packages.d").glob("*"),
-                               (root / "profiles.d").glob("*"),
-                               (root / "hosts.d").glob("*"))}
 
     hostname = os.uname()[1]
 
@@ -125,8 +150,11 @@ def load_fs(statefile: Path) -> Vfs:
     oldfs = Vfs()
 
     if statefile.exists():
+        log.debug("Loading statefile %s", statefile)
         with open(statefile, "rb") as fp:
             oldfs._log = pickle.load(fp)
+    else:
+        log.warning("No previous statefile %s", statefile)
 
     return oldfs
     
@@ -146,6 +174,15 @@ def simplify(old_fs: Vfs, new_fs: Vfs) -> Vfs:
         new_fs._log.remove(txn)
         old_fs._log.remove(txn)
 
+    return new_fs
+
+
+def scrub(old_fs: Vfs, new_fs: Vfs) -> Vfs:
+    """Try to eliminate files which were previously installed but are no longer used."""
+
+    old_fs = old_fs.copy()
+    new_fs = new_fs.copy()
+
     # Look for files in the old log which are no longer present in the new log
     for txn in old_fs._log:
         if txn[0] == "link" and txn not in new_fs._log:
@@ -157,36 +194,71 @@ def simplify(old_fs: Vfs, new_fs: Vfs) -> Vfs:
     return new_fs
 
 
-def main():
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.option("--execute/--dry-run", default=False)
+@click.option("--state-file", default=".cram.log")
+@click.option("--optimize/--no-optimize", default=False)
+@click.argument("confdir", type=Path)
+@click.argument("destdir", type=Path)
+def apply(confdir, destdir, state_file, execute, optimize):
     """The entry point of cram."""
 
-    opts, args = parser.parse_known_args()
+    # Resolve the two input paths to absolutes
+    root = confdir.resolve()
+    dest = destdir.resolve()
+    statef = root / state_file
 
+    new_fs = build_fs(root, dest)
+    old_fs = load_fs(statef)
+
+    # Middleware processing of the resulting filesystem(s)
+    executable_fs = scrub(old_fs, new_fs)
+    if optimize:
+        executable_fs = simplify(old_fs, new_fs)
+
+    # Dump the new state.
+    # Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states.
+    if execute:
+        executable_fs.execute()
+
+        with open(statef, "wb") as fp:
+            pickle.dump(new_fs._log, fp)
+
+    else:
+        for e in executable_fs._log:
+            print("-", e)
+
+
+@cli.command()
+@click.option("--state-file", default=".cram.log")
+@click.argument("confdir", type=Path)
+def show(confdir, state_file):
+    root = confdir.resolve()
+    statef = root / state_file
+    fs = load_fs(statef)
+    for e in fs._log:
+        print(*e)
+
+
+@cli.command()
+@click.argument("confdir", type=Path)
+def list(confdir):
+    packages = load_config(confdir)
+    for pname in sorted(packages.keys()):
+        p = packages[pname]
+        print(f"{pname}:")
+        for d in p.requires():
+            print(f"  - {d}")
+
+if __name__ == "__main__" or 1:
     logging.basicConfig(
         level=logging.DEBUG,
         format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
     )
 
-    # Resolve the two input paths to absolutes
+    cli()
-    root = opts.confdir.resolve()
-    dest = opts.destdir.resolve()
-    statef = root / opts.statefile
-
-    new_fs = build_fs(root, dest)
-    old_fs = load_fs(statef)
-
-    if opts.optimize:
-        fast_fs = simplify(old_fs, new_fs)
-        fast_fs.execute(opts.execute)
-    else:
-        new_fs.execute(opts.execute)
-
-    # Dump the new state.
-    # Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states.
-    if opts.execute:
-        with open(statef, "wb") as fp:
-            pickle.dump(new_fs._log, fp)
-
-
-if __name__ == "__main__" or 1:
-    main()