arrdem/cram
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

And get TOML configs working

Browse source
This commit is contained in:
Reid 'arrdem' McKenzie 2022-09-13 23:21:18 -06:00
parent 2bd82a7a32
commit 027f8c24fc
3 changed files with 92 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
integration_test.sh
View file

@ -4,93 +4,116 @@ set -ex
dest=$(mktemp -d) dest=$(mktemp -d)
./cram --help PATH="$PATH:$(realpath .)"
PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
root="test/integration/" cram --help
root="test/integration"
function ,scrub() {
rm -rf "${dest}"/* .cram.log
}
# Should be able to list all packages # Should be able to list all packages
./cram list "$root" | grep "packages.d/p1" cram list "${root}" | grep "packages.d/p1"
,scrub
# P3 depends on P1, should show up in the listing # P3 depends on P1, should show up in the listing
./cram list "$root" packages.d/p3 | grep "packages.d/p1" cram list "${root}" packages.d/p3 | grep "packages.d/p1"
,scrub
# P4 depends on P3, should show up in the listing # P4 depends on P3, should show up in the listing
./cram list "$root" packages.d/p4 | grep "packages.d/p3" cram list "${root}" packages.d/p4 | grep "packages.d/p3"
,scrub
# The default profile should depend on its subpackage # The default profile should depend on its subpackage
./cram list "$root" profiles.d/default | grep "profiles.d/default/subpackage" cram list "${root}" profiles.d/default | grep "profiles.d/default/subpackage"
,scrub
# And the subpackage has a dep # And the subpackage has a dep
./cram list "$root" profiles.d/default/subpackage | grep "packages.d/p3" cram list "${root}" profiles.d/default/subpackage | grep "packages.d/p3"
,scrub
# Install one package # Install one package
./cram apply --no-optimize --require packages.d/p1 --execute "$root" "${dest}" cram apply --no-optimize --require packages.d/p1 --execute "${root}" "${dest}"
[ -L "${dest}"/foo ] [ -L "${dest}"/foo ]
./cram state "$root" | grep "${dest}/foo" cram state "$root" | grep "${dest}/foo"
rm -r "${dest}"/* ,scrub
# Install two transitively (legacy) # Install two transitively (legacy)
./cram apply --no-optimize --require packages.d/p3 --execute "$root" "${dest}" cram apply --no-optimize --require packages.d/p3 --execute "${root}" "${dest}"
[ -L "${dest}"/foo ] [ -L "${dest}"/foo ]
[ -L "${dest}"/bar ] [ -L "${dest}"/bar ]
./cram state "$root" | grep "${dest}/foo" cram state "$root" | grep "${dest}/foo"
./cram state "$root" | grep "${dest}/bar" cram state "$root" | grep "${dest}/bar"
rm -r "${dest}"/* ,scrub
# Install two transitively (current) # Install two transitively (current)
./cram apply --no-optimize --require packages.d/p4 --execute "$root" "${dest}" cram apply --no-optimize --require packages.d/p4 --execute "${root}" "${dest}"
[ -L "${dest}"/foo ] [ -L "${dest}"/foo ]
[ -L "${dest}"/bar ] [ -L "${dest}"/bar ]
rm -r "${dest}"/* ,scrub
# Install two transitively (current) # Install two transitively (current)
./cram apply --no-optimize --require packages.d/p4 --execute "$root" "${dest}" cram apply --no-optimize --require packages.d/p4 --execute "${root}" "${dest}"
[ -L "${dest}"/foo ] [ -L "${dest}"/foo ]
[ -L "${dest}"/bar ] [ -L "${dest}"/bar ]
rm -r "${dest}"/* ,scrub
# Install two transitively (current) # Install two transitively (current)
./cram apply --no-optimize --require hosts.d/test --require profiles.d/default --execute "$root" "${dest}" cram apply --no-optimize --require hosts.d/test --require profiles.d/default --execute "${root}" "${dest}"
tree "${dest}"
[ -L "${dest}"/foo ] [ -L "${dest}"/foo ]
[ -L "${dest}"/bar ] [ -L "${dest}"/bar ]
rm -r "${dest}"/* ,scrub
# INSTALL scripts get run as-is # INSTALL scripts get run as-is
./cram list "$root" packages.d/p5 | grep "packages.d/p5/INSTALL" cram list "${root}" packages.d/p5 | grep "packages.d/p5/INSTALL"
,scrub
# Inline scripts get pulled out repeatably # Inline scripts get pulled out repeatably
./cram list "$root" packages.d/p6 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b" cram list "${root}" packages.d/p6 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"
,scrub
# Inline scripts get pulled out repeatably, even from the list format # Inline scripts get pulled out repeatably, even from the list format
./cram list "$root" packages.d/p7 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b" cram list "${root}" packages.d/p7 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"
,scrub
# Test log-based optimization # Test log-based optimization
./cram apply --no-optimize --require packages.d/p4 --execute "$root" "${dest}" cram apply --no-optimize --require packages.d/p4 --execute "${root}" "${dest}"
[ -L "${dest}"/foo ] [ -L "${dest}"/foo ]
[ -L "${dest}"/bar ] [ -L "${dest}"/bar ]
# These paths were already linked, they shouldn't be re-linked when optimizing. # These paths were already linked, they shouldn't be re-linked when optimizing.
! ./cram apply --require packages.d/p4 --optimize --execute "$root" "${dest}" | grep "${dest}/foo" ! cram apply --require packages.d/p4 --optimize --execute "${root}" "${dest}" | grep "${dest}/foo"
! ./cram apply --require packages.d/p4 --optimize --execute "$root" "${dest}" | grep "${dest}/bar" ! cram apply --require packages.d/p4 --optimize --execute "${root}" "${dest}" | grep "${dest}/bar"
rm -r "${dest}"/* ,scrub
# Likewise, if we've exec'd this once we shouldn't do it again # Likewise, if we've exec'd this once we shouldn't do it again
./cram apply --no-optimize --require packages.d/p5 --execute "$root" "${dest}" cram apply --no-optimize --require packages.d/p5 --execute "${root}" "${dest}"
! ./cram apply --require packages.d/p5 --execute "$root" "${dest}" | grep "exec" ! cram apply --require packages.d/p5 --execute "${root}" "${dest}" | grep "exec"
,scrub
# ... unless the user tells us to # ... unless the user tells us to
./cram apply --no-optimize --require packages.d/p5 --execute "$root" "${dest}" cram apply --no-optimize --require packages.d/p5 --execute "${root}" "${dest}"
./cram apply --exec-always --require packages.d/p5 --execute "$root" "${dest}" | grep "exec" cram apply --exec-always --require packages.d/p5 --execute "${root}" "${dest}" | grep "exec"
,scrub
# If multiple packages provide the same _effective_ script, do it once # If multiple packages provide the same _effective_ script, do it once
./cram apply --require packages.d/p6 --require packages.d/p7 --execute "$root" "${dest}" | sort | uniq -c | grep "/tmp/stow/b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b.sh" | grep "1 - exec" cram apply --require packages.d/p6 --require packages.d/p7 --execute "${root}" "${dest}" | sort | uniq -c | grep "/tmp/stow/b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b.sh" | grep "1 - exec"
,scrub
# Test log-based cleanup # Test log-based cleanup
./cram apply --require packages.d/p1 --require packages.d/p2 --execute "$root" "${dest}" cram apply --require packages.d/p1 --require packages.d/p2 --execute "${root}" "${dest}"
cram apply --require packages.d/p1 --require packages.d/p2 --execute "${root}" "${dest}"
[ -L "${dest}"/foo ] [ -L "${dest}"/foo ]
[ -L "${dest}"/bar ] [ -L "${dest}"/bar ]
,scrub
# And how bar shouldn't be installed... # And how bar shouldn't be installed...
./cram state test/ cram state "${root}"
./cram apply --require packages.d/p1 --execute "$root" "${dest}" cram apply --require packages.d/p1 --execute "${root}" "${dest}"
./cram state test/ cram state "${root}"
[ -L "${dest}"/foo ] [ -L "${dest}"/foo ]
[ ! -L "${dest}"/bar ] [ ! -L "${dest}"/bar ]
,scrub

50
src/python/cram/__main__.py
View file

@ -1,11 +1,9 @@
"""Cram's entry point.""" """Cram's entry point."""
from itertools import chain
import logging import logging
import os import os
from pathlib import Path from pathlib import Path
import pickle import pickle
from pprint import pprint
from typing import List, Optional, Union from typing import List, Optional, Union
import re import re
import platform import platform
@ -128,7 +126,7 @@ def load_packages(root: Path, roots) -> dict:
packages = {} packages = {}
for r in roots.get("package", []): for r in roots.get("package", []):
r = Path(r) r = root / r
log.debug(f"Trying to load packages from {r}...") log.debug(f"Trying to load packages from {r}...")
for p in r.glob("*"): for p in r.glob("*"):
name = str(p.relative_to(root)) name = str(p.relative_to(root))
@ -136,7 +134,7 @@ def load_packages(root: Path, roots) -> dict:
for r in roots.get("profile", []): for r in roots.get("profile", []):
# Add profiles, hosts which contain subpackages. # Add profiles, hosts which contain subpackages.
r = Path(r) r = root / r
log.debug(f"Trying to load profiles from {r}...") log.debug(f"Trying to load profiles from {r}...")
for mp_root in r.glob("*"): for mp_root in r.glob("*"):
# First find all subpackages # First find all subpackages
@ -181,7 +179,7 @@ def build_fs(
log.warning("Loaded no packages!") log.warning("Loaded no packages!")
requirements = list(prelude) requirements = list(prelude)
for r in list(requirements): for r in requirements:
try: try:
for d in packages[r].requires(): for d in packages[r].requires():
if d not in requirements: if d not in requirements:
@ -190,7 +188,7 @@ def build_fs(
missing_require_handler(r, e) missing_require_handler(r, e)
requirements.remove(r) requirements.remove(r)
log.debug(f"Mapped requirements {prelude} to {requirements}") log.info(f"Mapped requirements {prelude} to {requirements}")
# Compute the topsort graph # Compute the topsort graph
requirements_graph = {r: packages[r].requires() for r in requirements} requirements_graph = {r: packages[r].requires() for r in requirements}
fs = Vfs() fs = Vfs()
@ -273,7 +271,6 @@ def scrub(old_fs: Vfs, new_fs: Vfs) -> Vfs:
def configure(ctx, param, filename: Optional[Path]): def configure(ctx, param, filename: Optional[Path]):
if filename and filename.exists(): if filename and filename.exists():
ctx.default_map = {} ctx.default_map = {}
os.chdir(filename.parent)
log.debug(f"Loading config from {filename}") log.debug(f"Loading config from {filename}")
cfg = toml.load(filename) cfg = toml.load(filename)
assert cfg["cram"]["version"] >= 1 assert cfg["cram"]["version"] >= 1
@ -306,8 +303,6 @@ def configure(ctx, param, filename: Optional[Path]):
if error: if error:
exit(1) exit(1)
pprint(ctx.default_map)
@click.group(invoke_without_command=True) @click.group(invoke_without_command=True)
@click.option( @click.option(
@ -363,12 +358,14 @@ def arg_require(*args, **kwargs):
def arg_require_root(*args, **kwargs): def arg_require_root(*args, **kwargs):
return click.option( return click.option(
"--require-root", "--require-root",
type=(str, click.Choice(["profile", "package"])), type=(click.Choice(["profile", "package"]), str),
multiple=True, multiple=True,
default=[("profiles.d", "profile"), ("hosts.d", "profile"), ("packages.d", "package")], default=[
callback=lambda ctx, param, pairs: groupby( ("profile", "profiles.d"),
[(y, expandvars(x)) for x, y in pairs], first, second ("profile", "hosts.d"),
), ("package", "packages.d"),
],
callback=lambda ctx, param, pairs: groupby(pairs, first, second),
)(*args, **kwargs) )(*args, **kwargs)
@ -376,6 +373,7 @@ def arg_missing_require(*args, **kwargs):
return click.option( return click.option(
"--missing-require", "--missing-require",
type=click.Choice(["error", "warn", "ignore"]), type=click.Choice(["error", "warn", "ignore"]),
default="error",
callback=lambda ctx, param, it: { callback=lambda ctx, param, it: {
"ignore": missing_require_ignore, "ignore": missing_require_ignore,
"warn": missing_require_warn, "warn": missing_require_warn,
@ -414,7 +412,8 @@ def do_apply(
"""The entry point of cram.""" """The entry point of cram."""
# Resolve the two input paths to absolutes # Resolve the two input paths to absolutes
root = confdir.resolve() root = confdir.resolve().absolute()
os.chdir(root)
dest = destdir.resolve() dest = destdir.resolve()
if not root.is_dir(): if not root.is_dir():
@ -464,7 +463,9 @@ def do_apply(
@click.argument("requirements", nargs=-1, callback=handle_requires) @click.argument("requirements", nargs=-1, callback=handle_requires)
def do_list(confdir, requirements, require_root, oneline): def do_list(confdir, requirements, require_root, oneline):
"""List out packages, profiles, hosts and subpackages in the <confdir>.""" """List out packages, profiles, hosts and subpackages in the <confdir>."""
root = confdir.resolve()
root = confdir.resolve().absolute()
os.chdir(root)
log.debug(f"confdir: {root}, roots: {require_root}, requirements: {requirements}") log.debug(f"confdir: {root}, roots: {require_root}, requirements: {requirements}")
if not root.is_dir(): if not root.is_dir():
@ -503,7 +504,9 @@ def do_list(confdir, requirements, require_root, oneline):
@click.argument("confdir", type=Path, callback=handle_path_vars) @click.argument("confdir", type=Path, callback=handle_path_vars)
def do_state(confdir, state_file): def do_state(confdir, state_file):
"""List out the last `apply` state in the <confdir>/.cram.log or --state-file.""" """List out the last `apply` state in the <confdir>/.cram.log or --state-file."""
root = confdir.resolve()
root = confdir.resolve().absolute()
os.chdir(root)
if not root.is_dir(): if not root.is_dir():
log.fatal(f"{confdir} does not exist!") log.fatal(f"{confdir} does not exist!")
@ -521,10 +524,11 @@ def do_state(confdir, state_file):
@arg_require_root @arg_require_root
@click.argument("confdir", type=Path, callback=handle_path_vars) @click.argument("confdir", type=Path, callback=handle_path_vars)
@click.argument("requirement", type=str, callback=handle_path_vars) @click.argument("requirement", type=str, callback=handle_path_vars)
def do_fmt(confdir, requirement, require_root): def do_fmt(confdir: Path, requirement, require_root):
"""Format the specified requirement to a canonical-ish representation.""" """Format the specified requirement to a canonical-ish representation."""
root = confdir.resolve() root = confdir.resolve().absolute()
os.chdir(root)
if not root.is_dir(): if not root.is_dir():
log.fatal(f"{confdir} does not exist!") log.fatal(f"{confdir} does not exist!")
@ -571,9 +575,9 @@ exec_idempotent = true
# Where to load requirements from, and the requirement type # Where to load requirements from, and the requirement type
# Types: package, profile # Types: package, profile
require_root = [ require_root = [
["${PWD}/packages.d", "package"], ["package", "${PWD}/packages.d"],
["${PWD}/profiles.d", "profile"], ["profile", "${PWD}/profiles.d"],
["${PWD}/hosts.d", "profile"], ["profile", "${PWD}/hosts.d"],
] ]
# Choice([error, warn, ignore]) # Choice([error, warn, ignore])
missing_require = "error" missing_require = "error"
@ -594,7 +598,7 @@ require = [
if __name__ == "__main__" or 1: if __name__ == "__main__" or 1:
logging.basicConfig( logging.basicConfig(
level=logging.DEBUG, level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
) )

13
test/integration/cram.toml
View file

@ -7,12 +7,12 @@ destdir = "${HOME}"
state_file = "${PWD}/.cram.log" state_file = "${PWD}/.cram.log"
optimize = true optimize = true
exec_idempotent = true exec_idempotent = true
# Where to load requirements from # Where to load requirements from, and the requirement type
# Types: package, profile # Types: package, profile
require_root = [ require_root = [
["${PWD}/packages.d", "package"], ["package", "${PWD}/packages.d"],
["${PWD}/profiles.d", "profile"], ["profile", "${PWD}/profiles.d"],
["${PWD}/hosts.d", "profile"], ["profile", "${PWD}/hosts.d"],
] ]
# Choice([error, warn, ignore]) # Choice([error, warn, ignore])
missing_require = "error" missing_require = "error"
@ -20,8 +20,7 @@ missing_require = "error"
[cram.task.apply] [cram.task.apply]
missing_require = "ignore" missing_require = "ignore"
require = [ require = [
# "hosts.d/${FQDN}", "hosts.d/${FQDN}",
# "hosts.d/${HOSTNAME}", "hosts.d/${HOSTNAME}",
"hosts.d/test",
"profiles.d/default", "profiles.d/default",
] ]