From 027f8c24fc0e1b4b0af2fc4b1c570782b4140d19 Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Tue, 13 Sep 2022 23:21:18 -0600 Subject: [PATCH] And get TOML configs working --- integration_test.sh | 95 +++++++++++++++++++++++-------------- src/python/cram/__main__.py | 50 ++++++++++--------- test/integration/cram.toml | 13 +++-- 3 files changed, 92 insertions(+), 66 deletions(-) diff --git a/integration_test.sh b/integration_test.sh index 878a23b..7cd4e07 100755 --- a/integration_test.sh +++ b/integration_test.sh @@ -4,93 +4,116 @@ set -ex dest=$(mktemp -d) -./cram --help +PATH="$PATH:$(realpath .)" +PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' -root="test/integration/" +cram --help + +root="test/integration" + +function ,scrub() { + rm -rf "${dest}"/* .cram.log +} # Should be able to list all packages -./cram list "$root" | grep "packages.d/p1" +cram list "${root}" | grep "packages.d/p1" +,scrub # P3 depends on P1, should show up in the listing -./cram list "$root" packages.d/p3 | grep "packages.d/p1" +cram list "${root}" packages.d/p3 | grep "packages.d/p1" +,scrub # P4 depends on P3, should show up in the listing -./cram list "$root" packages.d/p4 | grep "packages.d/p3" +cram list "${root}" packages.d/p4 | grep "packages.d/p3" +,scrub # The default profile should depend on its subpackage -./cram list "$root" profiles.d/default | grep "profiles.d/default/subpackage" +cram list "${root}" profiles.d/default | grep "profiles.d/default/subpackage" +,scrub # And the subpackage has a dep -./cram list "$root" profiles.d/default/subpackage | grep "packages.d/p3" +cram list "${root}" profiles.d/default/subpackage | grep "packages.d/p3" +,scrub # Install one package -./cram apply --no-optimize --require packages.d/p1 --execute "$root" "${dest}" +cram apply --no-optimize --require packages.d/p1 --execute "${root}" "${dest}" [ -L "${dest}"/foo ] -./cram state "$root" | grep "${dest}/foo" -rm -r "${dest}"/* +cram state "$root" | grep "${dest}/foo" +,scrub # Install two transitively (legacy) -./cram apply --no-optimize --require packages.d/p3 --execute "$root" "${dest}" +cram apply --no-optimize --require packages.d/p3 --execute "${root}" "${dest}" [ -L "${dest}"/foo ] [ -L "${dest}"/bar ] -./cram state "$root" | grep "${dest}/foo" -./cram state "$root" | grep "${dest}/bar" -rm -r "${dest}"/* +cram state "$root" | grep "${dest}/foo" +cram state "$root" | grep "${dest}/bar" +,scrub # Install two transitively (current) -./cram apply --no-optimize --require packages.d/p4 --execute "$root" "${dest}" +cram apply --no-optimize --require packages.d/p4 --execute "${root}" "${dest}" [ -L "${dest}"/foo ] [ -L "${dest}"/bar ] -rm -r "${dest}"/* +,scrub # Install two transitively (current) -./cram apply --no-optimize --require packages.d/p4 --execute "$root" "${dest}" +cram apply --no-optimize --require packages.d/p4 --execute "${root}" "${dest}" [ -L "${dest}"/foo ] [ -L "${dest}"/bar ] -rm -r "${dest}"/* +,scrub # Install two transitively (current) -./cram apply --no-optimize --require hosts.d/test --require profiles.d/default --execute "$root" "${dest}" +cram apply --no-optimize --require hosts.d/test --require profiles.d/default --execute "${root}" "${dest}" +tree "${dest}" [ -L "${dest}"/foo ] [ -L "${dest}"/bar ] -rm -r "${dest}"/* +,scrub # INSTALL scripts get run as-is -./cram list "$root" packages.d/p5 | grep "packages.d/p5/INSTALL" +cram list "${root}" packages.d/p5 | grep "packages.d/p5/INSTALL" +,scrub # Inline scripts get pulled out repeatably -./cram list "$root" packages.d/p6 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b" +cram list "${root}" packages.d/p6 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b" +,scrub # Inline scripts get pulled out repeatably, even from the list format -./cram list "$root" packages.d/p7 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b" +cram list "${root}" packages.d/p7 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b" +,scrub # Test log-based optimization -./cram apply --no-optimize --require packages.d/p4 --execute "$root" "${dest}" +cram apply --no-optimize --require packages.d/p4 --execute "${root}" "${dest}" [ -L "${dest}"/foo ] [ -L "${dest}"/bar ] # These paths were already linked, they shouldn't be re-linked when optimizing. -! ./cram apply --require packages.d/p4 --optimize --execute "$root" "${dest}" | grep "${dest}/foo" -! ./cram apply --require packages.d/p4 --optimize --execute "$root" "${dest}" | grep "${dest}/bar" -rm -r "${dest}"/* +! cram apply --require packages.d/p4 --optimize --execute "${root}" "${dest}" | grep "${dest}/foo" +! cram apply --require packages.d/p4 --optimize --execute "${root}" "${dest}" | grep "${dest}/bar" +,scrub # Likewise, if we've exec'd this once we shouldn't do it again -./cram apply --no-optimize --require packages.d/p5 --execute "$root" "${dest}" -! ./cram apply --require packages.d/p5 --execute "$root" "${dest}" | grep "exec" +cram apply --no-optimize --require packages.d/p5 --execute "${root}" "${dest}" +! cram apply --require packages.d/p5 --execute "${root}" "${dest}" | grep "exec" +,scrub # ... unless the user tells us to -./cram apply --no-optimize --require packages.d/p5 --execute "$root" "${dest}" -./cram apply --exec-always --require packages.d/p5 --execute "$root" "${dest}" | grep "exec" +cram apply --no-optimize --require packages.d/p5 --execute "${root}" "${dest}" +cram apply --exec-always --require packages.d/p5 --execute "${root}" "${dest}" | grep "exec" +,scrub # If multiple packages provide the same _effective_ script, do it once -./cram apply --require packages.d/p6 --require packages.d/p7 --execute "$root" "${dest}" | sort | uniq -c | grep "/tmp/stow/b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b.sh" | grep "1 - exec" +cram apply --require packages.d/p6 --require packages.d/p7 --execute "${root}" "${dest}" | sort | uniq -c | grep "/tmp/stow/b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b.sh" | grep "1 - exec" +,scrub # Test log-based cleanup -./cram apply --require packages.d/p1 --require packages.d/p2 --execute "$root" "${dest}" +cram apply --require packages.d/p1 --require packages.d/p2 --execute "${root}" "${dest}" +cram apply --require packages.d/p1 --require packages.d/p2 --execute "${root}" "${dest}" [ -L "${dest}"/foo ] [ -L "${dest}"/bar ] +,scrub + # And how bar shouldn't be installed... -./cram state test/ -./cram apply --require packages.d/p1 --execute "$root" "${dest}" -./cram state test/ +cram state "${root}" +cram apply --require packages.d/p1 --execute "${root}" "${dest}" +cram state "${root}" [ -L "${dest}"/foo ] [ ! -L "${dest}"/bar ] +,scrub diff --git a/src/python/cram/__main__.py b/src/python/cram/__main__.py index b2e22e3..6ed5097 100644 --- a/src/python/cram/__main__.py +++ b/src/python/cram/__main__.py @@ -1,11 +1,9 @@ """Cram's entry point.""" -from itertools import chain import logging import os from pathlib import Path import pickle -from pprint import pprint from typing import List, Optional, Union import re import platform @@ -128,7 +126,7 @@ def load_packages(root: Path, roots) -> dict: packages = {} for r in roots.get("package", []): - r = Path(r) + r = root / r log.debug(f"Trying to load packages from {r}...") for p in r.glob("*"): name = str(p.relative_to(root)) @@ -136,7 +134,7 @@ def load_packages(root: Path, roots) -> dict: for r in roots.get("profile", []): # Add profiles, hosts which contain subpackages. - r = Path(r) + r = root / r log.debug(f"Trying to load profiles from {r}...") for mp_root in r.glob("*"): # First find all subpackages @@ -181,7 +179,7 @@ def build_fs( log.warning("Loaded no packages!") requirements = list(prelude) - for r in list(requirements): + for r in requirements: try: for d in packages[r].requires(): if d not in requirements: @@ -190,7 +188,7 @@ def build_fs( missing_require_handler(r, e) requirements.remove(r) - log.debug(f"Mapped requirements {prelude} to {requirements}") + log.info(f"Mapped requirements {prelude} to {requirements}") # Compute the topsort graph requirements_graph = {r: packages[r].requires() for r in requirements} fs = Vfs() @@ -273,7 +271,6 @@ def scrub(old_fs: Vfs, new_fs: Vfs) -> Vfs: def configure(ctx, param, filename: Optional[Path]): if filename and filename.exists(): ctx.default_map = {} - os.chdir(filename.parent) log.debug(f"Loading config from {filename}") cfg = toml.load(filename) assert cfg["cram"]["version"] >= 1 @@ -306,8 +303,6 @@ def configure(ctx, param, filename: Optional[Path]): if error: exit(1) - pprint(ctx.default_map) - @click.group(invoke_without_command=True) @click.option( @@ -363,12 +358,14 @@ def arg_require(*args, **kwargs): def arg_require_root(*args, **kwargs): return click.option( "--require-root", - type=(str, click.Choice(["profile", "package"])), + type=(click.Choice(["profile", "package"]), str), multiple=True, - default=[("profiles.d", "profile"), ("hosts.d", "profile"), ("packages.d", "package")], - callback=lambda ctx, param, pairs: groupby( - [(y, expandvars(x)) for x, y in pairs], first, second - ), + default=[ + ("profile", "profiles.d"), + ("profile", "hosts.d"), + ("package", "packages.d"), + ], + callback=lambda ctx, param, pairs: groupby(pairs, first, second), )(*args, **kwargs) @@ -376,6 +373,7 @@ def arg_missing_require(*args, **kwargs): return click.option( "--missing-require", type=click.Choice(["error", "warn", "ignore"]), + default="error", callback=lambda ctx, param, it: { "ignore": missing_require_ignore, "warn": missing_require_warn, @@ -414,7 +412,8 @@ def do_apply( """The entry point of cram.""" # Resolve the two input paths to absolutes - root = confdir.resolve() + root = confdir.resolve().absolute() + os.chdir(root) dest = destdir.resolve() if not root.is_dir(): @@ -464,7 +463,9 @@ def do_apply( @click.argument("requirements", nargs=-1, callback=handle_requires) def do_list(confdir, requirements, require_root, oneline): """List out packages, profiles, hosts and subpackages in the .""" - root = confdir.resolve() + + root = confdir.resolve().absolute() + os.chdir(root) log.debug(f"confdir: {root}, roots: {require_root}, requirements: {requirements}") if not root.is_dir(): @@ -503,7 +504,9 @@ def do_list(confdir, requirements, require_root, oneline): @click.argument("confdir", type=Path, callback=handle_path_vars) def do_state(confdir, state_file): """List out the last `apply` state in the /.cram.log or --state-file.""" - root = confdir.resolve() + + root = confdir.resolve().absolute() + os.chdir(root) if not root.is_dir(): log.fatal(f"{confdir} does not exist!") @@ -521,10 +524,11 @@ def do_state(confdir, state_file): @arg_require_root @click.argument("confdir", type=Path, callback=handle_path_vars) @click.argument("requirement", type=str, callback=handle_path_vars) -def do_fmt(confdir, requirement, require_root): +def do_fmt(confdir: Path, requirement, require_root): """Format the specified requirement to a canonical-ish representation.""" - root = confdir.resolve() + root = confdir.resolve().absolute() + os.chdir(root) if not root.is_dir(): log.fatal(f"{confdir} does not exist!") @@ -571,9 +575,9 @@ exec_idempotent = true # Where to load requirements from, and the requirement type # Types: package, profile require_root = [ - ["${PWD}/packages.d", "package"], - ["${PWD}/profiles.d", "profile"], - ["${PWD}/hosts.d", "profile"], + ["package", "${PWD}/packages.d"], + ["profile", "${PWD}/profiles.d"], + ["profile", "${PWD}/hosts.d"], ] # Choice([error, warn, ignore]) missing_require = "error" @@ -594,7 +598,7 @@ require = [ if __name__ == "__main__" or 1: logging.basicConfig( - level=logging.DEBUG, + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) diff --git a/test/integration/cram.toml b/test/integration/cram.toml index c38d62c..7d54f9f 100644 --- a/test/integration/cram.toml +++ b/test/integration/cram.toml @@ -7,12 +7,12 @@ destdir = "${HOME}" state_file = "${PWD}/.cram.log" optimize = true exec_idempotent = true -# Where to load requirements from +# Where to load requirements from, and the requirement type # Types: package, profile require_root = [ - ["${PWD}/packages.d", "package"], - ["${PWD}/profiles.d", "profile"], - ["${PWD}/hosts.d", "profile"], + ["package", "${PWD}/packages.d"], + ["profile", "${PWD}/profiles.d"], + ["profile", "${PWD}/hosts.d"], ] # Choice([error, warn, ignore]) missing_require = "error" @@ -20,8 +20,7 @@ missing_require = "error" [cram.task.apply] missing_require = "ignore" require = [ -# "hosts.d/${FQDN}", -# "hosts.d/${HOSTNAME}", - "hosts.d/test", + "hosts.d/${FQDN}", + "hosts.d/${HOSTNAME}", "profiles.d/default", ]