More improvements and an execution optimizer

This commit is contained in:
Reid 'arrdem' McKenzie 2021-10-11 00:09:51 -06:00
parent b9e40b5bb1
commit aab2ec1c33
2 changed files with 91 additions and 16 deletions

View file

@ -2,6 +2,7 @@
import argparse import argparse
from itertools import chain from itertools import chain
import pickle
import logging import logging
import os import os
from pathlib import Path from pathlib import Path
@ -16,6 +17,9 @@ log = logging.getLogger(__name__)
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("-x", "--execute", dest="execute", action="store_true", default=False) parser.add_argument("-x", "--execute", dest="execute", action="store_true", default=False)
parser.add_argument("-d", "--dry-run", dest="execute", action="store_false") parser.add_argument("-d", "--dry-run", dest="execute", action="store_false")
parser.add_argument("-s", "--state-file", dest="statefile", default=".cram.log")
parser.add_argument("--optimize", dest="optimize", default=False, action="store_true")
parser.add_argument("--no-optimize", dest="optimize", action="store_false")
parser.add_argument("confdir", type=Path) parser.add_argument("confdir", type=Path)
parser.add_argument("destdir", type=Path) parser.add_argument("destdir", type=Path)
@ -35,6 +39,7 @@ def stow(fs: Vfs, src_dir: Path, dest_dir: Path, skip=[]):
if src.is_dir(): if src.is_dir():
fs.mkdir(dest) fs.mkdir(dest)
fs.chmod(dest, src.stat().st_mode) fs.chmod(dest, src.stat().st_mode)
elif src.is_file(): elif src.is_file():
fs.link(src, dest) fs.link(src, dest)
@ -76,18 +81,9 @@ class PackageV0(NamedTuple):
fs.exec(self.root, ["bash", str(postf)]) fs.exec(self.root, ["bash", str(postf)])
def main(): def build_fs(root: Path, dest: Path) -> Vfs:
"""The entry point of cram.""" """Build a VFS by configuring dest from the given config root."""
opts, args = parser.parse_known_args()
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
root = opts.confdir
packages = {str(p.relative_to(root)): PackageV0(p, str(p)) packages = {str(p.relative_to(root)): PackageV0(p, str(p))
for p in chain((root / "packages.d").glob("*"), for p in chain((root / "packages.d").glob("*"),
(root / "profiles.d").glob("*"), (root / "profiles.d").glob("*"),
@ -114,11 +110,80 @@ def main():
requirements = {r: packages[r].requires() for r in requirements} requirements = {r: packages[r].requires() for r in requirements}
fs = Vfs() fs = Vfs()
# Abstractly execute the current packages
for r in toposort_flatten(requirements): for r in toposort_flatten(requirements):
r = packages[r] r = packages[r]
r.install(fs, opts.destdir) r.install(fs, dest)
fs.execute(opts.execute) return fs
def load_fs(statefile: Path) -> Vfs:
"""Load a persisted VFS state from disk. Sort of."""
oldfs = Vfs()
if statefile.exists():
with open(statefile, "rb") as fp:
oldfs._log = pickle.load(fp)
return oldfs
def simplify(old_fs: Vfs, new_fs: Vfs) -> Vfs:
"""Try to reduce a new VFS using diff from the original VFS."""
old_fs = old_fs.copy()
new_fs = new_fs.copy()
# Scrub anything in the new log that's in the old log
for txn in list(old_fs._log):
# Except for execs which are stateful
if txn[0] == "exec":
continue
new_fs._log.remove(txn)
old_fs._log.remove(txn)
# Look for files in the old log which are no longer present in the new log
for txn in old_fs._log:
if txn[0] == "link" and txn not in new_fs._log:
new_fs.unlink(txn[2])
elif txn[0] == "mkdir" and txn not in new_fs.log:
new_fs.unlink(txn[1])
return new_fs
def main():
"""The entry point of cram."""
opts, args = parser.parse_known_args()
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
# Resolve the two input paths to absolutes
root = opts.confdir.resolve()
dest = opts.destdir.resolve()
statef = root / opts.statefile
new_fs = build_fs(root, dest)
old_fs = load_fs(statef)
if opts.optimize:
fast_fs = simplify(old_fs, new_fs)
fast_fs.execute(opts.execute)
else:
new_fs.execute(opts.execute)
# Dump the new state.
# Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states.
if opts.execute:
with open(statef, "wb") as fp:
pickle.dump(new_fs._log, fp)
if __name__ == "__main__" or 1: if __name__ == "__main__" or 1:

View file

@ -12,8 +12,8 @@ _log = logging.getLogger(__name__)
class Vfs(object): class Vfs(object):
"""An abstract filesystem device which can accumulate changes, and apply them in a batch.""" """An abstract filesystem device which can accumulate changes, and apply them in a batch."""
def __init__(self): def __init__(self, log=None):
self._log = [] self._log = log or []
def execute(self, execute=False): def execute(self, execute=False):
for e in self._log: for e in self._log:
@ -46,6 +46,10 @@ class Vfs(object):
_, dest = e _, dest = e
dest.mkdir(exist_ok=True) dest.mkdir(exist_ok=True)
elif e[0] == "unlink":
_, dest = e
dest.unlink()
def _append(self, msg): def _append(self, msg):
self._log.append(msg) self._log.append(msg)
@ -63,3 +67,9 @@ class Vfs(object):
def exec(self, dest, cmd): def exec(self, dest, cmd):
self._append(("exec", dest, cmd)) self._append(("exec", dest, cmd))
def unlink(self, dest):
self._append(("unlink", dest))
def copy(self):
return Vfs(list(self._log))