WIP: support for TOML root config

src/python/cram/__main__.py

@@ -5,7 +5,11 @@ import logging
 import os
 from pathlib import Path
 import pickle
-from typing import List
+from pprint import pprint
+from typing import List, Optional, Union
+import re
+import platform
+from collections import defaultdict
 
 from . import (
     __author__,
@@ -30,6 +34,78 @@ def _exit(val):
     exit(val)
 
 
+def first(iter):
+    return iter[0]
+
+
+def second(iter):
+    return iter[1]
+
+
+def groupby(iter, keyfn=lambda x: x, mapfn=lambda x: x):
+    bag = defaultdict(list)
+    for it in iter:
+        bag[keyfn(it)].append(mapfn(it))
+    return dict(bag)
+
+
+def upsearch(name: Union[str, Path], limit=10) -> Optional[Path]:
+    d = Path(os.getcwd())
+    while True:
+        if limit == 0:
+            return
+
+        p = d.parent
+        if d == p:
+            return
+
+        dest = d / name
+        if dest.exists():
+            return dest
+
+        d = p
+        limit -= 1
+
+
+def expandvars(s: str) -> str:
+    def _repl(m: re.Match) -> str:
+        var = m.group(0).lower()
+        match var:
+            case "${hostname}":
+                return os.uname()[1].split(".")[0]
+            case "${fqdn}":
+                return os.uname()[1]
+            case "${home}":
+                return os.path.expanduser("~")
+            case "${uname}" | "${sysname}":
+                return platform.system().lower()
+            case "${arch}":
+                return platform.machine()
+            case "${release}":
+                return platform.release()
+            case _:
+                return os.path.expandvars(m.group(0))
+
+    s = re.sub(r"\${?([^\s}]+)}?", "${\\1}", s)
+    return re.sub(r"\$\{.*?\}", _repl, s)
+
+
+def handle_vars(_ctx, _param, s):
+    return expandvars(s)
+
+
+def handle_path_vars(_ctx, _param, p):
+    return Path(expandvars(str(p)))
+
+
+def expand_requires(requires):
+    return [expandvars(r) for r in requires]
+
+
+def handle_requires(_ctx, _params, requires):
+    return expand_requires(requires)
+
+
 def load(root: Path, name: str, clss):
     for c in clss:
         i = c(root, name)
@@ -47,19 +123,24 @@ def load_profile(root, name):
     return load(root, name, [ProfileV1, ProfileV0])
 
 
-def load_packages(root: Path) -> dict:
+def load_packages(root: Path, roots) -> dict:
     """Load the configured packages."""
 
     packages = {}
-    log.debug(f"Trying to load packages from {root}...")
+    for r in roots.get("package", []):
-    for p in (root / "packages.d").glob("*"):
+        r = Path(r)
+        log.debug(f"Trying to load packages from {r}...")
+        for p in r.glob("*"):
             name = str(p.relative_to(root))
             packages[name] = load_package(p, name)
 
+    for r in roots.get("profile", []):
         # Add profiles, hosts which contain subpackages.
-    for mp_root in chain((root / "profiles.d").glob("*"), (root / "hosts.d").glob("*")):
+        r = Path(r)
-
+        log.debug(f"Trying to load profiles from {r}...")
+        for mp_root in r.glob("*"):
             # First find all subpackages
+            log.debug(f"Trying to load packages from {mp_root}...")
             for p in mp_root.glob("*"):
                 if p.is_dir():
                     name = str(p.relative_to(root))
@@ -69,13 +150,27 @@ def load_packages(root: Path) -> dict:
             mp_name = str(mp_root.relative_to(root))
             packages[mp_name] = load_profile(mp_root, mp_name)
 
+    log.debug(f"Loaded {len(packages)} packages...")
     return packages
 
 
-def build_fs(root: Path, dest: Path, prelude: List[str]) -> Vfs:
+def missing_require_fatal(r):
+    log.fatal(f"Unable to load package {r}")
+    _exit(1)
+
+
+def missing_require_warn(r):
+    log.warn(f"Unable to load package {r}")
+
+
+def missing_require_ignore(r):
+    pass
+
+
+def build_fs(root: Path, roots, dest: Path, prelude: List[str], missing_require_handler) -> Vfs:
     """Build a VFS by configuring dest from the given config root."""
 
-    packages = load_packages(root)
+    packages = load_packages(root, roots)
     requirements = []
     requirements.extend(prelude)
 
@@ -85,14 +180,14 @@ def build_fs(root: Path, dest: Path, prelude: List[str]) -> Vfs:
     else:
         log.warning("Loaded no packages!")
 
-    for r in requirements:
+    for r in list(requirements):
         try:
             for d in packages[r].requires():
                 if d not in requirements:
                     requirements.append(d)
         except KeyError:
-            log.fatal(f"Error: Unable to load package {r}")
+            missing_require_handler(r)
-            _exit(1)
+            requirements.remove(r)
 
     # Compute the topsort graph
     requirements = {r: packages[r].requires() for r in requirements}
@@ -173,22 +268,74 @@ def scrub(old_fs: Vfs, new_fs: Vfs) -> Vfs:
     return cleanup_fs
 
 
-@click.group()
+def configure(ctx, param, filename: Optional[Path]):
-@click.version_option(version=__version__, message=f"""Cram {__version__}
+    if filename and filename.exists():
+        ctx.default_map = {}
+        os.chdir(filename.parent)
+        log.debug(f"Loading config from {filename}")
+        cfg = toml.load(filename)
+        assert cfg["cram"]["version"] >= 1
+
+        task_cfg = cfg.get("cram", {}).get("task", {})
+        defaults = task_cfg.get("default", {})
+        subcommands = ctx.command.commands.keys()
+        error = False
+        for subcommand in task_cfg.keys():
+            if subcommand not in subcommands and subcommand != "default":
+                error |= True
+                log.fatal(f"Erroneous config [cram.task.{subcommand}]")
+
+            elif subcommand == "default":
+                pass
+
+        for subcommand in subcommands:
+            # Take all defaults
+            m = defaults.copy()
+            # Splice in task defaults
+            m.update(task_cfg.get(subcommand, {}))
+            # And prune out anything that doesn't map to a param
+            cmd = ctx.command.commands[subcommand]
+            param_names = {it.name for it in cmd.params}
+            for k in list(m.keys()):
+                if k not in param_names:
+                    del m[k]
+            ctx.default_map[subcommand] = m
+
+        if error:
+            exit(1)
+
+
+@click.group(invoke_without_command=True)
+@click.option(
+    "--config",
+    type=click.Path(dir_okay=False),
+    default=upsearch("cram.toml"),
+    callback=configure,
+    is_eager=True,
+    expose_value=False,
+    help="Read option defaults from the specified TOML file",
+    show_default=True,
+)
+@click.version_option(
+    version=__version__,
+    message=f"""Cram {__version__}
 
 Documentation
-  https://github.com/arrdem/source/tree/trunk/projects/cram/
+  https://git.arrdem.com/arrdem/cram
 
 Features
  - 0.0.0 legacy config format
  - 0.1.0 TOML config format
  - 0.1.0 log based optimizer
  - 0.1.0 idempotent default for scripts
+ - 0.2.0 initial release
+ - 0.2.1 TOML root config
 
 About
   {__copyright__}, {__author__}.
   Published under the terms of the {__license__} license.
-""")
+""",
+)
 def cli():
     pass
 
@@ -198,11 +345,33 @@ def cli():
 @click.option("--force/--no-force", default=False)
 @click.option("--state-file", default=".cram.log", type=Path)
 @click.option("--optimize/--no-optimize", default=True)
-@click.option("--require", type=str, multiple=True, default=[f"hosts.d/{os.uname()[1].split('.')[0]}", "profiles.d/default"])
+@click.option(
+    "--require",
+    type=str,
+    multiple=True,
+    default=[
+        "hosts.d/${HOSTNAME}",
+        "profiles.d/default"
+    ],
+    callback=handle_requires,
+)
+@click.option(
+    "--require-root",
+    type=(str, click.Choice(["profile", "package"])),
+    multiple=True,
+    callback=lambda ctx, param, pairs: groupby([(y, expandvars(x)) for x, y in pairs], first, second)
+)
+@click.option(
+    "--missing-require",
+    type=click.Choice(["error", "warn", "ignore"]),
+    callback=lambda ctx, param, it: {"ignore": missing_require_ignore, "warn": missing_require_warn, "error": missing_require_fatal}[it]
+)
 @click.option("--exec-idempotent/--exec-always", "exec_idempotent", default=True)
-@click.argument("confdir", type=Path)
+@click.argument("confdir", type=Path, callback=handle_path_vars)
-@click.argument("destdir", type=Path)
+@click.argument("destdir", type=Path, callback=handle_path_vars)
-def do_apply(confdir, destdir, state_file, execute, optimize, force, require, exec_idempotent):
+def do_apply(
+        confdir, destdir, state_file, execute, optimize, force, require, require_root, missing_require, exec_idempotent,
+):
     """The entry point of cram."""
 
     # Resolve the two input paths to absolutes
@@ -223,14 +392,13 @@ def do_apply(confdir, destdir, state_file, execute, optimize, force, require, ex
         # Force an empty state
         old_fs = Vfs([])
 
-    new_fs = build_fs(root, dest, require)
+    new_fs = build_fs(root, require_root, dest, require, missing_require)
     log.debug(f"Built new state consisting of {len(new_fs._log)} steps")
 
     # Middleware processing of the resulting filesystem(s)
     executable_fs = scrub(old_fs, new_fs)
     if optimize:
-        executable_fs = simplify(old_fs, executable_fs,
+        executable_fs = simplify(old_fs, executable_fs, exec_idempotent=exec_idempotent)
-                                 exec_idempotent=exec_idempotent)
 
     # Dump the new state.
     # Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states.
@@ -249,10 +417,21 @@ def do_apply(confdir, destdir, state_file, execute, optimize, force, require, ex
 
 
 @cli.command("list")
-@click.option("-1", "--oneline", is_flag=True, default=False, help="Only list names of resources")
+@click.option(
-@click.argument("confdir", type=Path)
+    "-1", "--oneline",
-@click.argument("requirements", nargs=-1)
+    is_flag=True,
-def do_list(confdir, requirements, oneline):
+    default=False,
+    help="Only list names of resources"
+)
+@click.option(
+    "--require-root",
+    type=(str, click.Choice(["profile", "package"])),
+    multiple=True,
+    callback=lambda ctx, param, pairs: groupby([(y, expandvars(x)) for x, y in pairs], first, second)
+)
+@click.argument("confdir", type=Path, callback=handle_path_vars)
+@click.argument("requirements", nargs=-1, callback=handle_requires)
+def do_list(confdir, requirements, require_root, oneline):
     """List out packages, profiles, hosts and subpackages in the <confdir>."""
     root = confdir.resolve()
 
@@ -260,7 +439,7 @@ def do_list(confdir, requirements, oneline):
         log.fatal(f"{confdir} does not exist!")
         _exit(1)
 
-    packages = load_packages(root)
+    packages = load_packages(root, require_root)
 
     if requirements:
         dest = Path("~/")
@@ -289,7 +468,7 @@ def do_list(confdir, requirements, oneline):
 
 @cli.command("state")
 @click.option("--state-file", default=".cram.log", type=Path)
-@click.argument("confdir", type=Path)
+@click.argument("confdir", type=Path, callback=handle_path_vars)
 def do_state(confdir, state_file):
     """List out the last `apply` state in the <confdir>/.cram.log or --state-file."""
     root = confdir.resolve()
@@ -307,9 +486,15 @@ def do_state(confdir, state_file):
 
 
 @cli.command("fmt")
-@click.argument("confdir", type=Path)
+@click.argument("confdir", type=Path, callback=handle_path_vars)
-@click.argument("requirement", type=str)
+@click.option(
-def do_fmt(confdir, requirement):
+    "--require-root",
+    type=(str, click.Choice(["profile", "package"])),
+    multiple=True,
+    callback=lambda ctx, param, pairs: groupby([(y, expandvars(x)) for x, y in pairs], first, second)
+)
+@click.argument("requirement", type=str, callback=handle_path_vars)
+def do_fmt(confdir, requirement, require_root):
     """Format the specified requirement to a canonical-ish representation."""
 
     root = confdir.resolve()
@@ -318,21 +503,65 @@ def do_fmt(confdir, requirement):
         log.fatal(f"{confdir} does not exist!")
         _exit(1)
 
-    packages = load_packages(root)
+    packages = load_packages(root, require_root)
     pkg = packages[requirement]
     json = pkg.json()
 
     for suffix in pkg.SPECIAL_FILES:
-        f = (root / requirement / suffix)
+        f = root / requirement / suffix
         if f.exists():
             f.unlink()
 
     with open(root / requirement / "pkg.toml", "w") as fp:
         toml.dump(json, fp)
 
+
+@cli.command("init")
+@click.argument("confdir", type=Path, callback=handle_path_vars)
+@click.option("-f", "--force", is_flag=True)
+def do_init(confdir: Path, force: bool):
+    """Initialize an empty Cram repo."""
+
+    confdir.mkdir(parents=True, exist_ok=True)
+    rootfile = confdir / 'cram.toml'
+    if rootfile.exists() and not force:
+        log.fatal("cram.toml exists! Provide -f/--force to overwrite")
+        exit(1)
+
+    log.info("Writing initial cram.toml...")
+    with open(rootfile, 'w') as fp:
+        fp.write("""\
+[cram]
+version = 1
+
+[cram.task.default]
+confdir = "${PWD}"
+destdir = "${HOME}"
+state_file = "${PWD}/.cram.log"
+optimize = true
+exec_idempotent = true
+require_roots = {
+  "packages.d" = "package",
+  "profiles.d" = "profile"
+  "hosts.d" = "profile",
+}
+missing_require = "error"
+
+[cram.task.apply]
+require = [
+  "hosts.d/${FQDN}",
+  "hosts.d/${HOSTNAME}",
+  "profiles.d/default",
+]
+""")
+
+    for d in ['profiles.d', 'packages.d', 'hosts.d']:
+        (confdir / d).mkdir(parents=True, exist_ok=True)
+
+
 if __name__ == "__main__" or 1:
     logging.basicConfig(
-        level=logging.INFO,
+        level=logging.DEBUG,
         format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
     )
 

test/integration/cram.toml

@@ -0,0 +1,26 @@
+[cram]
+version = 1
+
+[cram.task.default]
+confdir = "${PWD}"
+destdir = "${HOME}"
+state_file = "${PWD}/.cram.log"
+optimize = true
+exec_idempotent = true
+# Where to load requirements from
+# Types: package, profile
+require_root = [
+  ["${PWD}/packages.d", "package"],
+  ["${PWD}/profiles.d", "profile"],
+  ["${PWD}/hosts.d", "profile"],
+]
+# Choice([error, warn, ignore])
+missing_require = "error"
+
+[cram.task.apply]
+missing_require = "ignore"
+require = [
+  "hosts.d/${FQDN}",
+  "hosts.d/${HOSTNAME}",
+  "profiles.d/default",
+]