Enable zapp to chew through whls with source file conflicts
This commit is contained in:
parent
da9203f81a
commit
9761b5c6d3
2 changed files with 28 additions and 11 deletions
|
@ -6,10 +6,12 @@ import argparse
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import ast
|
||||||
import pathlib
|
import pathlib
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
import stat
|
import stat
|
||||||
import sys
|
import sys
|
||||||
|
from collections import defaultdict
|
||||||
import zipfile
|
import zipfile
|
||||||
from email.parser import Parser
|
from email.parser import Parser
|
||||||
from shutil import move
|
from shutil import move
|
||||||
|
@ -57,7 +59,7 @@ for script in {scripts!r}:
|
||||||
def dsub(d1, d2):
|
def dsub(d1, d2):
|
||||||
"""Dictionary subtraction. Remove k/vs from d1 if they occur in d2."""
|
"""Dictionary subtraction. Remove k/vs from d1 if they occur in d2."""
|
||||||
|
|
||||||
return {k: v for k, v in d1.items() if k not in d2 or d2[k] != v}
|
return [(k, v) for k, v in d1 if not k in (_k for _k, _ in d2)]
|
||||||
|
|
||||||
|
|
||||||
def make_dunder_main(manifest):
|
def make_dunder_main(manifest):
|
||||||
|
@ -104,7 +106,7 @@ def load_wheel(opts, manifest, path):
|
||||||
|
|
||||||
prefix = os.path.dirname(path)
|
prefix = os.path.dirname(path)
|
||||||
|
|
||||||
sources = {k: v for k, v in manifest["sources"].items() if v["source"].startswith(prefix)}
|
sources = [(dest, spec,) for dest, spec in manifest["sources"] if spec["source"].startswith(prefix)]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
# "record": record,
|
# "record": record,
|
||||||
|
@ -143,7 +145,7 @@ def zip_wheel(tmpdir, wheel):
|
||||||
wheel_file = os.path.join(tmpdir, wheel_name(wheel))
|
wheel_file = os.path.join(tmpdir, wheel_name(wheel))
|
||||||
|
|
||||||
with zipfile.ZipFile(wheel_file, "w") as whl:
|
with zipfile.ZipFile(wheel_file, "w") as whl:
|
||||||
for dest, src in wheel["sources"].items():
|
for dest, src in wheel["sources"]:
|
||||||
whl.write(src["source"], dest)
|
whl.write(src["source"], dest)
|
||||||
|
|
||||||
return wheel_file
|
return wheel_file
|
||||||
|
@ -158,7 +160,7 @@ def rezip_wheels(opts, manifest):
|
||||||
|
|
||||||
wheels = [
|
wheels = [
|
||||||
load_wheel(opts, manifest, os.path.dirname(s["source"]))
|
load_wheel(opts, manifest, os.path.dirname(s["source"]))
|
||||||
for s in manifest["sources"].values()
|
for _, s in manifest["sources"]
|
||||||
if s["source"].endswith("/WHEEL")
|
if s["source"].endswith("/WHEEL")
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -169,6 +171,11 @@ def rezip_wheels(opts, manifest):
|
||||||
# Expunge sources available in the wheel
|
# Expunge sources available in the wheel
|
||||||
manifest["sources"] = dsub(manifest["sources"], w["sources"])
|
manifest["sources"] = dsub(manifest["sources"], w["sources"])
|
||||||
|
|
||||||
|
if opts.debug:
|
||||||
|
from pprint import pprint
|
||||||
|
print("---")
|
||||||
|
pprint({"$type": "whl", **w})
|
||||||
|
|
||||||
# We may have a double-path dependency.
|
# We may have a double-path dependency.
|
||||||
# If we DON'T, we have to zip
|
# If we DON'T, we have to zip
|
||||||
if wn not in manifest["wheels"]:
|
if wn not in manifest["wheels"]:
|
||||||
|
@ -182,12 +189,19 @@ def rezip_wheels(opts, manifest):
|
||||||
else:
|
else:
|
||||||
wf = zip_wheel(opts.tmpdir, w)
|
wf = zip_wheel(opts.tmpdir, w)
|
||||||
|
|
||||||
|
|
||||||
# Insert a new wheel source
|
# Insert a new wheel source
|
||||||
manifest["wheels"][wn] = {"hashes": [], "source": wf}
|
manifest["wheels"][wn] = {"hashes": [], "source": wf}
|
||||||
|
|
||||||
return manifest
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_srcs_map(opts, manifest):
|
||||||
|
manifest["sources"] = dict(manifest["sources"])
|
||||||
|
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
def generate_dunder_inits(opts, manifest):
|
def generate_dunder_inits(opts, manifest):
|
||||||
"""Hack the manifest to insert __init__ files as needed."""
|
"""Hack the manifest to insert __init__ files as needed."""
|
||||||
|
|
||||||
|
@ -259,6 +273,7 @@ def main():
|
||||||
setattr(opts, "tmpdir", d)
|
setattr(opts, "tmpdir", d)
|
||||||
|
|
||||||
manifest = rezip_wheels(opts, manifest)
|
manifest = rezip_wheels(opts, manifest)
|
||||||
|
manifest = ensure_srcs_map(opts, manifest)
|
||||||
manifest = enable_unzipping(opts, manifest)
|
manifest = enable_unzipping(opts, manifest)
|
||||||
# Patch the manifest to insert needed __init__ files
|
# Patch the manifest to insert needed __init__ files
|
||||||
manifest = generate_dunder_inits(opts, manifest)
|
manifest = generate_dunder_inits(opts, manifest)
|
||||||
|
@ -270,8 +285,10 @@ def main():
|
||||||
if opts.debug:
|
if opts.debug:
|
||||||
from pprint import pprint
|
from pprint import pprint
|
||||||
|
|
||||||
|
print("---")
|
||||||
pprint(
|
pprint(
|
||||||
{
|
{
|
||||||
|
"$type": "zapp",
|
||||||
"opts": {
|
"opts": {
|
||||||
k: getattr(opts, k) for k in dir(opts) if not k.startswith("_")
|
k: getattr(opts, k) for k in dir(opts) if not k.startswith("_")
|
||||||
},
|
},
|
||||||
|
|
|
@ -51,7 +51,7 @@ def _check_script(point, sources_map):
|
||||||
"""Check that a given 'script' (eg. module:fn ref.) maps to a file in sources."""
|
"""Check that a given 'script' (eg. module:fn ref.) maps to a file in sources."""
|
||||||
|
|
||||||
fname = point.split(":")[0].replace(".", "/") + ".py"
|
fname = point.split(":")[0].replace(".", "/") + ".py"
|
||||||
if fname not in sources_map:
|
if fname not in [e[0] for e in sources_map]:
|
||||||
fail("Point %s (%s) is not a known source!" % (fname, sources_map))
|
fail("Point %s (%s) is not a known source!" % (fname, sources_map))
|
||||||
|
|
||||||
|
|
||||||
|
@ -98,17 +98,17 @@ def _zapp_impl(ctx):
|
||||||
|
|
||||||
# Make a manifest of files to store in the .zapp file. The
|
# Make a manifest of files to store in the .zapp file. The
|
||||||
# runfiles manifest is not quite right, so we make our own.
|
# runfiles manifest is not quite right, so we make our own.
|
||||||
sources_map = {}
|
sources_map = []
|
||||||
|
|
||||||
# Now add the regular (source and generated) files
|
# Now add the regular (source and generated) files
|
||||||
for input_file in srcs:
|
for input_file in srcs:
|
||||||
stored_path = _store_path(input_file.short_path, ctx, import_roots)
|
stored_path = _store_path(input_file.short_path, ctx, import_roots)
|
||||||
if stored_path:
|
if stored_path:
|
||||||
local_path = input_file.path
|
local_path = input_file.path
|
||||||
if stored_path in sources_map and sources_map[stored_path] != '':
|
conflicts = [e for e in sources_map if e[0] == stored_path]
|
||||||
fail("File path conflict between %s and %s" % sources_map[stored_path], local_path)
|
if conflicts:
|
||||||
|
print("File %s conflicts with others, %s" % (input_file, conflicts))
|
||||||
sources_map[stored_path] = local_path
|
sources_map.append([stored_path, local_path])
|
||||||
|
|
||||||
_check_script(main_py_ref, sources_map)
|
_check_script(main_py_ref, sources_map)
|
||||||
for p in ctx.attr.prelude_points:
|
for p in ctx.attr.prelude_points:
|
||||||
|
@ -135,7 +135,7 @@ def _zapp_impl(ctx):
|
||||||
output = manifest_file,
|
output = manifest_file,
|
||||||
content = json.encode({
|
content = json.encode({
|
||||||
"shebang": ctx.attr.shebang.replace("%py3%", py3),
|
"shebang": ctx.attr.shebang.replace("%py3%", py3),
|
||||||
"sources": {d: {"hashes": [], "source": s} for d, s in sources_map.items()},
|
"sources": [[d, {"hashes": [], "source": s}] for d, s in sources_map],
|
||||||
"zip_safe": ctx.attr.zip_safe,
|
"zip_safe": ctx.attr.zip_safe,
|
||||||
"prelude_points": ctx.attr.prelude_points,
|
"prelude_points": ctx.attr.prelude_points,
|
||||||
"entry_point": main_py_ref,
|
"entry_point": main_py_ref,
|
||||||
|
|
Loading…
Reference in a new issue