Enable zapp to chew through whls with source file conflicts
This commit is contained in:
parent
da9203f81a
commit
9761b5c6d3
2 changed files with 28 additions and 11 deletions
|
@ -6,10 +6,12 @@ import argparse
|
|||
import io
|
||||
import json
|
||||
import os
|
||||
import ast
|
||||
import pathlib
|
||||
from itertools import chain
|
||||
import stat
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
import zipfile
|
||||
from email.parser import Parser
|
||||
from shutil import move
|
||||
|
@ -57,7 +59,7 @@ for script in {scripts!r}:
|
|||
def dsub(d1, d2):
|
||||
"""Dictionary subtraction. Remove k/vs from d1 if they occur in d2."""
|
||||
|
||||
return {k: v for k, v in d1.items() if k not in d2 or d2[k] != v}
|
||||
return [(k, v) for k, v in d1 if not k in (_k for _k, _ in d2)]
|
||||
|
||||
|
||||
def make_dunder_main(manifest):
|
||||
|
@ -104,7 +106,7 @@ def load_wheel(opts, manifest, path):
|
|||
|
||||
prefix = os.path.dirname(path)
|
||||
|
||||
sources = {k: v for k, v in manifest["sources"].items() if v["source"].startswith(prefix)}
|
||||
sources = [(dest, spec,) for dest, spec in manifest["sources"] if spec["source"].startswith(prefix)]
|
||||
|
||||
return {
|
||||
# "record": record,
|
||||
|
@ -143,7 +145,7 @@ def zip_wheel(tmpdir, wheel):
|
|||
wheel_file = os.path.join(tmpdir, wheel_name(wheel))
|
||||
|
||||
with zipfile.ZipFile(wheel_file, "w") as whl:
|
||||
for dest, src in wheel["sources"].items():
|
||||
for dest, src in wheel["sources"]:
|
||||
whl.write(src["source"], dest)
|
||||
|
||||
return wheel_file
|
||||
|
@ -158,7 +160,7 @@ def rezip_wheels(opts, manifest):
|
|||
|
||||
wheels = [
|
||||
load_wheel(opts, manifest, os.path.dirname(s["source"]))
|
||||
for s in manifest["sources"].values()
|
||||
for _, s in manifest["sources"]
|
||||
if s["source"].endswith("/WHEEL")
|
||||
]
|
||||
|
||||
|
@ -169,6 +171,11 @@ def rezip_wheels(opts, manifest):
|
|||
# Expunge sources available in the wheel
|
||||
manifest["sources"] = dsub(manifest["sources"], w["sources"])
|
||||
|
||||
if opts.debug:
|
||||
from pprint import pprint
|
||||
print("---")
|
||||
pprint({"$type": "whl", **w})
|
||||
|
||||
# We may have a double-path dependency.
|
||||
# If we DON'T, we have to zip
|
||||
if wn not in manifest["wheels"]:
|
||||
|
@ -182,12 +189,19 @@ def rezip_wheels(opts, manifest):
|
|||
else:
|
||||
wf = zip_wheel(opts.tmpdir, w)
|
||||
|
||||
|
||||
# Insert a new wheel source
|
||||
manifest["wheels"][wn] = {"hashes": [], "source": wf}
|
||||
|
||||
return manifest
|
||||
|
||||
|
||||
def ensure_srcs_map(opts, manifest):
|
||||
manifest["sources"] = dict(manifest["sources"])
|
||||
|
||||
return manifest
|
||||
|
||||
|
||||
def generate_dunder_inits(opts, manifest):
|
||||
"""Hack the manifest to insert __init__ files as needed."""
|
||||
|
||||
|
@ -259,6 +273,7 @@ def main():
|
|||
setattr(opts, "tmpdir", d)
|
||||
|
||||
manifest = rezip_wheels(opts, manifest)
|
||||
manifest = ensure_srcs_map(opts, manifest)
|
||||
manifest = enable_unzipping(opts, manifest)
|
||||
# Patch the manifest to insert needed __init__ files
|
||||
manifest = generate_dunder_inits(opts, manifest)
|
||||
|
@ -270,8 +285,10 @@ def main():
|
|||
if opts.debug:
|
||||
from pprint import pprint
|
||||
|
||||
print("---")
|
||||
pprint(
|
||||
{
|
||||
"$type": "zapp",
|
||||
"opts": {
|
||||
k: getattr(opts, k) for k in dir(opts) if not k.startswith("_")
|
||||
},
|
||||
|
|
|
@ -51,7 +51,7 @@ def _check_script(point, sources_map):
|
|||
"""Check that a given 'script' (eg. module:fn ref.) maps to a file in sources."""
|
||||
|
||||
fname = point.split(":")[0].replace(".", "/") + ".py"
|
||||
if fname not in sources_map:
|
||||
if fname not in [e[0] for e in sources_map]:
|
||||
fail("Point %s (%s) is not a known source!" % (fname, sources_map))
|
||||
|
||||
|
||||
|
@ -98,17 +98,17 @@ def _zapp_impl(ctx):
|
|||
|
||||
# Make a manifest of files to store in the .zapp file. The
|
||||
# runfiles manifest is not quite right, so we make our own.
|
||||
sources_map = {}
|
||||
sources_map = []
|
||||
|
||||
# Now add the regular (source and generated) files
|
||||
for input_file in srcs:
|
||||
stored_path = _store_path(input_file.short_path, ctx, import_roots)
|
||||
if stored_path:
|
||||
local_path = input_file.path
|
||||
if stored_path in sources_map and sources_map[stored_path] != '':
|
||||
fail("File path conflict between %s and %s" % sources_map[stored_path], local_path)
|
||||
|
||||
sources_map[stored_path] = local_path
|
||||
conflicts = [e for e in sources_map if e[0] == stored_path]
|
||||
if conflicts:
|
||||
print("File %s conflicts with others, %s" % (input_file, conflicts))
|
||||
sources_map.append([stored_path, local_path])
|
||||
|
||||
_check_script(main_py_ref, sources_map)
|
||||
for p in ctx.attr.prelude_points:
|
||||
|
@ -135,7 +135,7 @@ def _zapp_impl(ctx):
|
|||
output = manifest_file,
|
||||
content = json.encode({
|
||||
"shebang": ctx.attr.shebang.replace("%py3%", py3),
|
||||
"sources": {d: {"hashes": [], "source": s} for d, s in sources_map.items()},
|
||||
"sources": [[d, {"hashes": [], "source": s}] for d, s in sources_map],
|
||||
"zip_safe": ctx.attr.zip_safe,
|
||||
"prelude_points": ctx.attr.prelude_points,
|
||||
"entry_point": main_py_ref,
|
||||
|
|
Loading…
Reference in a new issue