Promote cram out

This commit is contained in:
Reid 'arrdem' McKenzie 2022-07-28 23:45:00 -06:00
parent 5a34f095d7
commit a7b236fa5f
19 changed files with 7 additions and 814 deletions

View file

@ -8,6 +8,7 @@ And so I'm going the other way; Bazel in a monorepo with subprojects so I'm able
- [Datalog](projects/datalog) and the matching [shell](projects/datalog-shell) - [Datalog](projects/datalog) and the matching [shell](projects/datalog-shell)
- [YAML Schema](projects/yamlschema) (JSON schema with knowledge of PyYAML's syntax structure & nice errors) - [YAML Schema](projects/yamlschema) (JSON schema with knowledge of PyYAML's syntax structure & nice errors)
- [Zapp! (now with a new home and releases)](https://github.com/arrdem/rules_zapp) - [Zapp! (now with a new home and releases)](https://github.com/arrdem/rules_zapp)
- [Cram (now with a new home)](https://github.com/arrdem/cram)
- [Flowmetal](projects/flowmetal) - [Flowmetal](projects/flowmetal)
- [Lilith](projects/lilith) - [Lilith](projects/lilith)

View file

@ -1,28 +0,0 @@
py_library(
name = "lib",
srcs = glob(["src/python/**/*.py"]),
deps = [
"//projects/vfs",
py_requirement("click"),
py_requirement("toposort"),
py_requirement("toml"),
]
)
zapp_binary(
name = "cram",
main = "src/python/cram/__main__.py",
shebang = "/usr/bin/env python3",
imports = [
"src/python"
],
deps = [
":lib",
],
)
sh_test(
name = "test_cram",
srcs = glob(["test.sh"]),
data = glob(["test/**/*"]) + [":cram"],
)

View file

@ -1,5 +1,11 @@
# Cram # Cram
Cram has graduated [to its own repo](https://github.com/arrdem/cram)!
------
> To force (people or things) into a place or container that is or appears to be too small to contain them. > To force (people or things) into a place or container that is or appears to be too small to contain them.
An alternative to GNU Stow, more some notion of packages with dependencies and install scripts. An alternative to GNU Stow, more some notion of packages with dependencies and install scripts.

View file

@ -1,4 +0,0 @@
__version__ = "0.1.0"
__author__ = "Reid D. 'arrdem' McKenzie <me@arrdem.com>"
__copyright__ = "Copyright 2020"
__license__ = "https://anticapitalist.software/"

View file

@ -1,335 +0,0 @@
"""Cram's entry point."""
from itertools import chain
import logging
import os
from pathlib import Path
import pickle
from typing import List
from . import (
__author__,
__copyright__,
__license__,
__version__,
)
from .v0 import PackageV0, ProfileV0
from .v1 import PackageV1, ProfileV1
import click
import toml
from toposort import toposort_flatten
from vfs import Vfs
log = logging.getLogger(__name__)
def _exit(val):
logging.shutdown()
exit(val)
def load(root: Path, name: str, clss):
for c in clss:
i = c(root, name)
if i.test():
return i
def load_package(root, name):
log.debug(f"Attempting to load package {name} from {root}")
return load(root, name, [PackageV1, PackageV0])
def load_profile(root, name):
log.debug(f"Attempting to load profile {name} from {root}")
return load(root, name, [ProfileV1, ProfileV0])
def load_packages(root: Path) -> dict:
"""Load the configured packages."""
packages = {}
log.debug(f"Trying to load packages from {root}...")
for p in (root / "packages.d").glob("*"):
name = str(p.relative_to(root))
packages[name] = load_package(p, name)
# Add profiles, hosts which contain subpackages.
for mp_root in chain((root / "profiles.d").glob("*"), (root / "hosts.d").glob("*")):
# First find all subpackages
for p in mp_root.glob("*"):
if p.is_dir():
name = str(p.relative_to(root))
packages[name] = load_package(p, name)
# Register the metapackages themselves using the profile type
mp_name = str(mp_root.relative_to(root))
packages[mp_name] = load_profile(mp_root, mp_name)
return packages
def build_fs(root: Path, dest: Path, prelude: List[str]) -> Vfs:
"""Build a VFS by configuring dest from the given config root."""
packages = load_packages(root)
requirements = []
requirements.extend(prelude)
if packages:
for p in packages:
log.debug(f"Loaded package {p}")
else:
log.warning("Loaded no packages!")
for r in requirements:
try:
for d in packages[r].requires():
if d not in requirements:
requirements.append(d)
except KeyError:
log.fatal(f"Error: Unable to load package {r}")
_exit(1)
# Compute the topsort graph
requirements = {r: packages[r].requires() for r in requirements}
fs = Vfs()
# Abstractly execute the current packages
for r in toposort_flatten(requirements):
r = packages[r]
r.install(fs, dest)
return fs
def load_state(statefile: Path) -> Vfs:
"""Load a persisted VFS state from disk. Sort of."""
oldfs = Vfs([])
if statefile.exists():
log.debug("Loading statefile %s", statefile)
with open(statefile, "rb") as fp:
oldfs._log = pickle.load(fp)
else:
log.warning("No previous statefile %s", statefile)
return oldfs
def simplify(old_fs: Vfs, new_fs: Vfs, /, exec_idempotent=True) -> Vfs:
"""Try to reduce a new VFS using diff from the original VFS."""
old_fs = old_fs.copy()
new_fs = new_fs.copy()
# Scrub anything in the new log that's in the old log
for txn in list(old_fs._log):
# Except for execs which are stateful
if txn[0] == "exec" and not exec_idempotent:
continue
try:
new_fs._log.remove(txn)
except ValueError:
pass
# Dedupe the new log while preserving order.
keys = set()
deduped = []
for op in new_fs._log:
key = str(op)
if key not in keys:
keys.add(key)
deduped.append(op)
new_fs._log = deduped
return new_fs
def scrub(old_fs: Vfs, new_fs: Vfs) -> Vfs:
"""Try to eliminate files which were previously installed but are no longer used."""
old_fs = old_fs.copy()
new_fs = new_fs.copy()
cleanup_fs = Vfs([])
# Look for files in the old log which are no longer present in the new log
for txn in old_fs._log:
if txn[0] == "link" and txn not in new_fs._log:
cleanup_fs.unlink(txn[2])
elif txn[0] == "mkdir" and txn not in new_fs._log:
cleanup_fs.unlink(txn[1])
# Do unlink operations before we do install operations.
# This works around being unable to finely straify uninstall operations over their source packages.
cleanup_fs.merge(new_fs)
return cleanup_fs
@click.group()
@click.version_option(version=1, message=f"""Cram {__version__}
Documentation
https://github.com/arrdem/source/tree/trunk/projects/cram/
Features
- 0.0.0 legacy config format
- 0.1.0 TOML config format
- 0.1.0 log based optimizer
- 0.1.0 idempotent default for scripts
About
{__copyright__}, {__author__}.
Published under the terms of the {__license__} license.
""")
def cli():
pass
@cli.command("apply")
@click.option("--execute/--dry-run", default=False)
@click.option("--force/--no-force", default=False)
@click.option("--state-file", default=".cram.log", type=Path)
@click.option("--optimize/--no-optimize", default=True)
@click.option("--require", type=str, multiple=True, default=[f"hosts.d/{os.uname()[1].split('.')[0]}", "profiles.d/default"])
@click.option("--exec-idempotent/--exec-always", "exec_idempotent", default=True)
@click.argument("confdir", type=Path)
@click.argument("destdir", type=Path)
def do_apply(confdir, destdir, state_file, execute, optimize, force, require, exec_idempotent):
"""The entry point of cram."""
# Resolve the two input paths to absolutes
root = confdir.resolve()
dest = destdir.resolve()
if not root.is_dir():
log.fatal(f"{confdir} does not exist!")
_exit(1)
if not state_file.is_absolute():
state_file = root / state_file
if not force:
old_fs = load_state(state_file)
log.debug(f"Loaded old state consisting of {len(old_fs._log)} steps")
else:
# Force an empty state
old_fs = Vfs([])
new_fs = build_fs(root, dest, require)
log.debug(f"Built new state consisting of {len(new_fs._log)} steps")
# Middleware processing of the resulting filesystem(s)
executable_fs = scrub(old_fs, new_fs)
if optimize:
executable_fs = simplify(old_fs, executable_fs,
exec_idempotent=exec_idempotent)
# Dump the new state.
# Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states.
def cb(e):
print("-", *e)
if execute:
executable_fs.execute(callback=cb)
with open(state_file, "wb") as fp:
pickle.dump(new_fs._log, fp)
else:
for e in executable_fs._log:
cb(e)
@cli.command("list")
@click.argument("confdir", type=Path)
@click.argument("list_packages", nargs=-1)
def do_list(confdir, list_packages):
"""List out packages, profiles, hosts and subpackages in the <confdir>."""
root = confdir.resolve()
if not root.is_dir():
log.fatal(f"{confdir} does not exist!")
_exit(1)
packages = load_packages(root)
if list_packages:
dest = Path("~/")
for pname in list_packages:
fs = Vfs()
p = packages[pname]
p.install(fs, dest)
print(f"{pname}: ({type(p).__name__})")
print("requires:")
for e in p.requires():
print(" -", e)
print("log:")
for e in fs._log:
print(" -", *e)
else:
for pname in sorted(packages.keys()):
p = packages[pname]
print(f"{pname}: ({type(p).__name__})")
for d in p.requires():
print(f"- {d}")
@cli.command("state")
@click.option("--state-file", default=".cram.log", type=Path)
@click.argument("confdir", type=Path)
def do_state(confdir, state_file):
"""List out the last `apply` state in the <confdir>/.cram.log or --state-file."""
root = confdir.resolve()
if not root.is_dir():
log.fatal(f"{confdir} does not exist!")
_exit(1)
if not state_file.is_absolute():
state_file = root / state_file
fs = load_state(state_file)
for e in fs._log:
print(*e)
@cli.command("fmt")
@click.argument("confdir", type=Path)
@click.argument("requirement", type=str)
def do_migrate(confdir, requirement):
"""Format the specified requirement to a canonical-ish representation."""
root = confdir.resolve()
if not root.is_dir():
log.fatal(f"{confdir} does not exist!")
_exit(1)
packages = load_packages(root)
pkg = packages[requirement]
json = pkg.json()
for suffix in pkg.SPECIAL_FILES:
f = (root / requirement / suffix)
if f.exists():
f.unlink()
with open(root / requirement / "pkg.toml", "w") as fp:
toml.dump(json, fp)
if __name__ == "__main__" or 1:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
cli()

View file

@ -1,86 +0,0 @@
#!/usr/bin/env python3
import os
from pathlib import Path
from shlex import quote as sh_quote
import sys
from typing import List, Optional
from vfs import Vfs
# FIXME: This should be a config somewhere
SHELL = "/bin/sh"
# Light monkeypatching because macos ships a "stable" a py
if sys.version_info <= (3, 9, 0):
Path.readlink = lambda p: Path(os.readlink(str(p)))
def sh(cmd: List[str], /,
env: Optional[dict] = None):
prefix = []
if env:
prefix.append("/usr/bin/env")
for k, v in env.items():
v = sh_quote(str(v))
prefix.append(f"{k}={v}")
return tuple(prefix + [SHELL, *cmd])
def stow(fs: Vfs, src_dir: Path, dest_dir: Path, skip=[]):
"""Recursively 'stow' (link) the contents of the source into the destination."""
dest_root = Path(dest_dir)
src_root = Path(src_dir)
skip = [src_root / n for n in skip]
for src in src_root.glob("**/*"):
if src in skip:
continue
elif src.name.endswith(".gitkeep"):
continue
dest = dest_root / src.relative_to(src_root)
if src.is_symlink():
fs.link(src.readlink().resolve(), dest)
elif src.is_dir():
fs.mkdir(dest)
fs.chmod(dest, src.stat().st_mode)
elif src.is_file():
fs.link(src, dest)
class Package(object):
def __init__(self, root: Path, name: str):
self.root = root
self.name = name
def test(self):
return True
def requires(self):
return []
def install(self, fs: Vfs, dest: Path):
self.do_build(fs, dest)
self.pre_install(fs, dest)
self.do_install(fs, dest)
self.post_install(fs, dest)
def do_build(self, fs: Vfs, dest: Path):
pass
def pre_install(self, fs: Vfs, dest: Path):
pass
def do_install(self, fs: Vfs, dest: Path):
pass
def post_install(self, fs: Vfs, dest: Path):
pass

View file

@ -1,109 +0,0 @@
"""Cram's original (v0) configs.
An ill-considered pseudo-format.
"""
from pathlib import Path
import re
from .common import Package, sh, stow
from vfs import Vfs
class PackageV0(Package):
"""The original package format from install.sh."""
SPECIAL_FILES = ["BUILD", "PRE_INSTALL", "INSTALL", "POST_INSTALL", "REQUIRES"]
def requires(self):
"""Get the dependencies of this package."""
requiresf = self.root / "REQUIRES"
requires = []
# Listed dependencies
if requiresf.exists():
with open(requiresf) as fp:
for l in fp:
l = l.strip()
l = re.sub(r"\s*#.*\n", "", l)
if l:
requires.append(l)
return requires
def install(self, fs: Vfs, dest: Path):
"""Install this package."""
buildf = self.root / "BUILD"
if buildf.exists():
fs.exec(self.root, sh([str(buildf)]))
pref = self.root / "PRE_INSTALL"
if pref.exists():
fs.exec(self.root, sh([str(pref)]))
installf = self.root / "INSTALL"
if installf.exists():
fs.exec(self.root, sh([str(installf)]))
else:
stow(fs, self.root, dest, self.SPECIAL_FILES)
postf = self.root / "POST_INSTALL"
if postf.exists():
fs.exec(self.root, sh([str(postf)]))
def _read(self, p: Path):
if p.exists():
with open(p) as fp:
return fp.read()
else:
return None
def json(self):
buildt = self._read(self.root / "BUILD")
pret = self._read(self.root / "PRE_INSTALL")
installt = self._read(self.root / "INSTALL")
postt = self._read(self.root / "POST_INSTALL")
o = {"cram": {"version": 1}, "package": {"require": []}}
if buildt:
o["package"]["build"] = [{"run": buildt}]
if pret:
o["package"]["pre_install"] = [{"run": pret}]
if installt:
o["package"]["install"] = [{"run": installt}]
if postt:
o["package"]["install"] = [{"run": postt}]
o["package"]["require"] = [{"name": it} for it in sorted(self.requires())]
return o
class ProfileV0(PackageV0):
def requires(self):
requires = super().requires()
for p in self.root.glob("*"):
if p.is_dir():
requires.append(self.name + "/" + p.name)
return requires
def install(self, fs: Vfs, dest: Path):
"""Profiles differ from Packages in that they don't support literal files."""
buildf = self.root / "BUILD"
if buildf.exists():
fs.exec(self.root, sh([str(buildf)]))
pref = self.root / "PRE_INSTALL"
if pref.exists():
fs.exec(self.root, sh([str(pref)]))
installf = self.root / "INSTALL"
if installf.exists():
fs.exec(self.root, sh([str(installf)]))
postf = self.root / "POST_INSTALL"
if postf.exists():
fs.exec(self.root, sh([str(postf)]))

View file

@ -1,116 +0,0 @@
"""Cram's v1 configs.
Based on well* defined TOML manifests, rather than many files.
*Okay. Better.
"""
from hashlib import sha256
from pathlib import Path
from typing import List, Optional, Union
from .common import Package, sh, stow
import toml
from vfs import Vfs
def tempf(name):
root = Path("/tmp/stow")
root.mkdir(exist_ok=True, parents=True)
return root / name
class PackageV1(Package):
"""The v1 package format."""
SPECIAL_FILES = ["pkg.toml"]
_config = None
def config(self):
if not self._config:
with open(self.root / self.SPECIAL_FILES[0], "r") as fp:
self._config = toml.load(fp)
return self._config
def test(self):
return (self.root / self.SPECIAL_FILES[0]).exists() and self.config().get("cram", {}).get("version") == 1
def requires(self):
"""Get the dependencies of this package."""
def _name(it):
if isinstance(it, str):
return it
elif isinstance(it, dict):
return it["name"]
return [
_name(it) for it in self.config().get("package", {}).get("require", [])
]
def do_sh_or_script(self, content: Optional[Union[List[str], str]], fs: Vfs, dest: Path, cwd: Path = "/tmp"):
if content is None:
pass
elif isinstance(content, list):
for c in content:
self.do_sh_or_script(c, fs, dest)
elif isinstance(content, dict):
self.do_sh_or_script(
content["run"],
fs,
dest,
{"cwd": self.root}.get(content.get("root"), "/tmp")
)
elif isinstance(content, str):
sum = sha256()
sum.update(content.encode("utf-8"))
sum = sum.hexdigest()
installf = self.root / content
if installf.exists():
with open(installf, "r") as fp:
self.do_sh_or_script(fp.read(), fs, dest)
elif content:
f = tempf(f"{sum}.sh")
with open(f, "w") as fp:
fp.write(content)
fs.exec(cwd, sh([f]))
def do_build(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("build"), fs, dest)
def pre_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("pre_install"), fs, dest)
def do_install(self, fs: Vfs, dest: Path):
if not self.do_sh_or_script(self.config().get("package", {}).get("install"), fs, dest):
stow(fs, self.root, dest, self.SPECIAL_FILES)
def post_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("post_install"), fs, dest)
def json(self):
return self.config()
class ProfileV1(PackageV1):
"""Unline packages, profiles don't support recursive stow of contents."""
def do_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("install"), fs, dest)
def requires(self):
requires = super().requires()
# Implicitly depended subpackages
for p in self.root.glob("*"):
if p.is_dir():
requires.append(self.name + "/" + p.name)
return requires

View file

@ -1,96 +0,0 @@
#!/usr/bin/env bash
set -ex
cd projects/cram
dest=$(mktemp -d)
./cram --help
# Should be able to list all packages
./cram list test/ | grep "packages.d/p1"
# P3 depends on P1, should show up in the listing
./cram list test/ packages.d/p3 | grep "packages.d/p1"
# P4 depends on P3, should show up in the listing
./cram list test/ packages.d/p4 | grep "packages.d/p3"
# The default profile should depend on its subpackage
./cram list test/ profiles.d/default | grep "profiles.d/default/subpackage"
# And the subpackage has a dep
./cram list test/ profiles.d/default/subpackage | grep "packages.d/p3"
# Install one package
./cram apply --no-optimize --require packages.d/p1 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
./cram state test/ | grep "${dest}/foo"
rm -r "${dest}"/*
# Install two transitively (legacy)
./cram apply --no-optimize --require packages.d/p3 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
./cram state test/ | grep "${dest}/foo"
./cram state test/ | grep "${dest}/bar"
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require hosts.d/test --require profiles.d/default --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# INSTALL scripts get run as-is
./cram list test/ packages.d/p5 | grep "packages.d/p5/INSTALL"
# Inline scripts get pulled out repeatably
./cram list test/ packages.d/p6 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"
# Inline scripts get pulled out repeatably, even from the list format
./cram list test/ packages.d/p7 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"
# Test log-based optimization
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
# These paths were already linked, they shouldn't be re-linked when optimizing.
! ./cram apply --require packages.d/p4 --optimize --execute test/ "${dest}" | grep "${dest}/foo"
! ./cram apply --require packages.d/p4 --optimize --execute test/ "${dest}" | grep "${dest}/bar"
rm -r "${dest}"/*
# Likewise, if we've exec'd this once we shouldn't do it again
./cram apply --no-optimize --require packages.d/p5 --execute test/ "${dest}"
! ./cram apply --require packages.d/p5 --execute test/ "${dest}" | grep "exec"
# ... unless the user tells us to
./cram apply --no-optimize --require packages.d/p5 --execute test/ "${dest}"
./cram apply --exec-always --require packages.d/p5 --execute test/ "${dest}" | grep "exec"
# If multiple packages provide the same _effective_ script, do it once
./cram apply --require packages.d/p6 --require packages.d/p7 --execute test/ "${dest}" | sort | uniq -c | grep "/tmp/stow/b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b.sh" | grep "1 - exec"
# Test log-based cleanup
./cram apply --require packages.d/p1 --require packages.d/p2 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
# And how bar shouldn't be installed...
./cram state test/
./cram apply --require packages.d/p1 --execute test/ "${dest}"
./cram state test/
[ -L "${dest}"/foo ]
[ ! -L "${dest}"/bar ]

View file

@ -1,6 +0,0 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p1"

View file

@ -1 +0,0 @@
bar

View file

@ -1 +0,0 @@
qux

View file

@ -1,2 +0,0 @@
packages.d/p1
packages.d/p2

View file

@ -1,6 +0,0 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p3"

View file

@ -1,3 +0,0 @@
#!/bin/bash
# A legacy custom install script
true

View file

@ -1,5 +0,0 @@
[cram]
version = 1
[package]
install = "true"

View file

@ -1,6 +0,0 @@
[cram]
version = 1
[package]
[[package.install]]
run = "true"

View file

@ -1,4 +0,0 @@
[cram]
version = 1
[package]

View file

@ -1,6 +0,0 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p3"