Enable zapp to chew through whls with source file conflicts

This commit is contained in:
Reid 'arrdem' McKenzie 2021-09-24 22:43:52 -06:00
parent da9203f81a
commit 9761b5c6d3
2 changed files with 28 additions and 11 deletions

View file

@ -6,10 +6,12 @@ import argparse
import io import io
import json import json
import os import os
import ast
import pathlib import pathlib
from itertools import chain from itertools import chain
import stat import stat
import sys import sys
from collections import defaultdict
import zipfile import zipfile
from email.parser import Parser from email.parser import Parser
from shutil import move from shutil import move
@ -57,7 +59,7 @@ for script in {scripts!r}:
def dsub(d1, d2): def dsub(d1, d2):
"""Dictionary subtraction. Remove k/vs from d1 if they occur in d2.""" """Dictionary subtraction. Remove k/vs from d1 if they occur in d2."""
return {k: v for k, v in d1.items() if k not in d2 or d2[k] != v} return [(k, v) for k, v in d1 if not k in (_k for _k, _ in d2)]
def make_dunder_main(manifest): def make_dunder_main(manifest):
@ -104,7 +106,7 @@ def load_wheel(opts, manifest, path):
prefix = os.path.dirname(path) prefix = os.path.dirname(path)
sources = {k: v for k, v in manifest["sources"].items() if v["source"].startswith(prefix)} sources = [(dest, spec,) for dest, spec in manifest["sources"] if spec["source"].startswith(prefix)]
return { return {
# "record": record, # "record": record,
@ -143,7 +145,7 @@ def zip_wheel(tmpdir, wheel):
wheel_file = os.path.join(tmpdir, wheel_name(wheel)) wheel_file = os.path.join(tmpdir, wheel_name(wheel))
with zipfile.ZipFile(wheel_file, "w") as whl: with zipfile.ZipFile(wheel_file, "w") as whl:
for dest, src in wheel["sources"].items(): for dest, src in wheel["sources"]:
whl.write(src["source"], dest) whl.write(src["source"], dest)
return wheel_file return wheel_file
@ -158,7 +160,7 @@ def rezip_wheels(opts, manifest):
wheels = [ wheels = [
load_wheel(opts, manifest, os.path.dirname(s["source"])) load_wheel(opts, manifest, os.path.dirname(s["source"]))
for s in manifest["sources"].values() for _, s in manifest["sources"]
if s["source"].endswith("/WHEEL") if s["source"].endswith("/WHEEL")
] ]
@ -169,6 +171,11 @@ def rezip_wheels(opts, manifest):
# Expunge sources available in the wheel # Expunge sources available in the wheel
manifest["sources"] = dsub(manifest["sources"], w["sources"]) manifest["sources"] = dsub(manifest["sources"], w["sources"])
if opts.debug:
from pprint import pprint
print("---")
pprint({"$type": "whl", **w})
# We may have a double-path dependency. # We may have a double-path dependency.
# If we DON'T, we have to zip # If we DON'T, we have to zip
if wn not in manifest["wheels"]: if wn not in manifest["wheels"]:
@ -182,12 +189,19 @@ def rezip_wheels(opts, manifest):
else: else:
wf = zip_wheel(opts.tmpdir, w) wf = zip_wheel(opts.tmpdir, w)
# Insert a new wheel source # Insert a new wheel source
manifest["wheels"][wn] = {"hashes": [], "source": wf} manifest["wheels"][wn] = {"hashes": [], "source": wf}
return manifest return manifest
def ensure_srcs_map(opts, manifest):
manifest["sources"] = dict(manifest["sources"])
return manifest
def generate_dunder_inits(opts, manifest): def generate_dunder_inits(opts, manifest):
"""Hack the manifest to insert __init__ files as needed.""" """Hack the manifest to insert __init__ files as needed."""
@ -259,6 +273,7 @@ def main():
setattr(opts, "tmpdir", d) setattr(opts, "tmpdir", d)
manifest = rezip_wheels(opts, manifest) manifest = rezip_wheels(opts, manifest)
manifest = ensure_srcs_map(opts, manifest)
manifest = enable_unzipping(opts, manifest) manifest = enable_unzipping(opts, manifest)
# Patch the manifest to insert needed __init__ files # Patch the manifest to insert needed __init__ files
manifest = generate_dunder_inits(opts, manifest) manifest = generate_dunder_inits(opts, manifest)
@ -270,8 +285,10 @@ def main():
if opts.debug: if opts.debug:
from pprint import pprint from pprint import pprint
print("---")
pprint( pprint(
{ {
"$type": "zapp",
"opts": { "opts": {
k: getattr(opts, k) for k in dir(opts) if not k.startswith("_") k: getattr(opts, k) for k in dir(opts) if not k.startswith("_")
}, },

View file

@ -51,7 +51,7 @@ def _check_script(point, sources_map):
"""Check that a given 'script' (eg. module:fn ref.) maps to a file in sources.""" """Check that a given 'script' (eg. module:fn ref.) maps to a file in sources."""
fname = point.split(":")[0].replace(".", "/") + ".py" fname = point.split(":")[0].replace(".", "/") + ".py"
if fname not in sources_map: if fname not in [e[0] for e in sources_map]:
fail("Point %s (%s) is not a known source!" % (fname, sources_map)) fail("Point %s (%s) is not a known source!" % (fname, sources_map))
@ -98,17 +98,17 @@ def _zapp_impl(ctx):
# Make a manifest of files to store in the .zapp file. The # Make a manifest of files to store in the .zapp file. The
# runfiles manifest is not quite right, so we make our own. # runfiles manifest is not quite right, so we make our own.
sources_map = {} sources_map = []
# Now add the regular (source and generated) files # Now add the regular (source and generated) files
for input_file in srcs: for input_file in srcs:
stored_path = _store_path(input_file.short_path, ctx, import_roots) stored_path = _store_path(input_file.short_path, ctx, import_roots)
if stored_path: if stored_path:
local_path = input_file.path local_path = input_file.path
if stored_path in sources_map and sources_map[stored_path] != '': conflicts = [e for e in sources_map if e[0] == stored_path]
fail("File path conflict between %s and %s" % sources_map[stored_path], local_path) if conflicts:
print("File %s conflicts with others, %s" % (input_file, conflicts))
sources_map[stored_path] = local_path sources_map.append([stored_path, local_path])
_check_script(main_py_ref, sources_map) _check_script(main_py_ref, sources_map)
for p in ctx.attr.prelude_points: for p in ctx.attr.prelude_points:
@ -135,7 +135,7 @@ def _zapp_impl(ctx):
output = manifest_file, output = manifest_file,
content = json.encode({ content = json.encode({
"shebang": ctx.attr.shebang.replace("%py3%", py3), "shebang": ctx.attr.shebang.replace("%py3%", py3),
"sources": {d: {"hashes": [], "source": s} for d, s in sources_map.items()}, "sources": [[d, {"hashes": [], "source": s}] for d, s in sources_map],
"zip_safe": ctx.attr.zip_safe, "zip_safe": ctx.attr.zip_safe,
"prelude_points": ctx.attr.prelude_points, "prelude_points": ctx.attr.prelude_points,
"entry_point": main_py_ref, "entry_point": main_py_ref,