Enable zapp to chew through whls with source file conflicts

zapp/compiler/__main__.py

@@ -6,10 +6,12 @@ import argparse
 import io
 import json
 import os
+import ast
 import pathlib
 from itertools import chain
 import stat
 import sys
+from collections import defaultdict
 import zipfile
 from email.parser import Parser
 from shutil import move
@@ -57,7 +59,7 @@ for script in {scripts!r}:
 def dsub(d1, d2):
     """Dictionary subtraction. Remove k/vs from d1 if they occur in d2."""
 
-    return {k: v for k, v in d1.items() if k not in d2 or d2[k] != v}
+    return [(k, v) for k, v in d1 if not k in (_k for _k, _ in d2)]
 
 
 def make_dunder_main(manifest):
@@ -104,7 +106,7 @@ def load_wheel(opts, manifest, path):
 
     prefix = os.path.dirname(path)
 
-    sources = {k: v for k, v in manifest["sources"].items() if v["source"].startswith(prefix)}
+    sources = [(dest, spec,) for dest, spec in manifest["sources"] if spec["source"].startswith(prefix)]
 
     return {
         # "record": record,
@@ -143,7 +145,7 @@ def zip_wheel(tmpdir, wheel):
     wheel_file = os.path.join(tmpdir, wheel_name(wheel))
 
     with zipfile.ZipFile(wheel_file, "w") as whl:
-        for dest, src in wheel["sources"].items():
+        for dest, src in wheel["sources"]:
             whl.write(src["source"], dest)
 
     return wheel_file
@@ -158,7 +160,7 @@ def rezip_wheels(opts, manifest):
 
     wheels = [
         load_wheel(opts, manifest, os.path.dirname(s["source"]))
-        for s in manifest["sources"].values()
+        for _, s in manifest["sources"]
         if s["source"].endswith("/WHEEL")
     ]
 
@@ -169,6 +171,11 @@ def rezip_wheels(opts, manifest):
         # Expunge sources available in the wheel
         manifest["sources"] = dsub(manifest["sources"], w["sources"])
 
+        if opts.debug:
+            from pprint import pprint
+            print("---")
+            pprint({"$type": "whl", **w})
+
         # We may have a double-path dependency.
         # If we DON'T, we have to zip
         if wn not in manifest["wheels"]:
@@ -182,12 +189,19 @@ def rezip_wheels(opts, manifest):
             else:
                 wf = zip_wheel(opts.tmpdir, w)
 
+
             # Insert a new wheel source
             manifest["wheels"][wn] = {"hashes": [], "source": wf}
 
     return manifest
 
 
+def ensure_srcs_map(opts, manifest):
+    manifest["sources"] = dict(manifest["sources"])
+
+    return manifest
+
+
 def generate_dunder_inits(opts, manifest):
     """Hack the manifest to insert __init__ files as needed."""
 
@@ -259,6 +273,7 @@ def main():
         setattr(opts, "tmpdir", d)
 
         manifest = rezip_wheels(opts, manifest)
+        manifest = ensure_srcs_map(opts, manifest)
         manifest = enable_unzipping(opts, manifest)
         # Patch the manifest to insert needed __init__ files
         manifest = generate_dunder_inits(opts, manifest)
@@ -270,8 +285,10 @@ def main():
         if opts.debug:
             from pprint import pprint
 
+            print("---")
             pprint(
                 {
+                    "$type": "zapp",
                     "opts": {
                         k: getattr(opts, k) for k in dir(opts) if not k.startswith("_")
                     },

zapp/zapp.bzl

@@ -51,7 +51,7 @@ def _check_script(point, sources_map):
     """Check that a given 'script' (eg. module:fn ref.) maps to a file in sources."""
 
     fname = point.split(":")[0].replace(".", "/") + ".py"
-    if fname not in sources_map:
+    if fname not in [e[0] for e in sources_map]:
         fail("Point %s (%s) is not a known source!" % (fname, sources_map))
 
 
@@ -98,17 +98,17 @@ def _zapp_impl(ctx):
 
     # Make a manifest of files to store in the .zapp file.  The
     # runfiles manifest is not quite right, so we make our own.
-    sources_map = {}
+    sources_map = []
 
     # Now add the regular (source and generated) files
     for input_file in srcs:
         stored_path = _store_path(input_file.short_path, ctx, import_roots)
         if stored_path:
             local_path = input_file.path
-            if stored_path in sources_map and sources_map[stored_path] != '':
-                fail("File path conflict between %s and %s" % sources_map[stored_path], local_path)
-
-            sources_map[stored_path] = local_path
+            conflicts = [e for e in sources_map if e[0] == stored_path]
+            if conflicts:
+                print("File %s conflicts with others, %s" % (input_file, conflicts))
+            sources_map.append([stored_path, local_path])
 
     _check_script(main_py_ref, sources_map)
     for p in ctx.attr.prelude_points:
@@ -135,7 +135,7 @@ def _zapp_impl(ctx):
         output = manifest_file,
         content = json.encode({
             "shebang": ctx.attr.shebang.replace("%py3%", py3),
-            "sources": {d: {"hashes": [], "source": s} for d, s in sources_map.items()},
+            "sources": [[d, {"hashes": [], "source": s}] for d, s in sources_map],
             "zip_safe": ctx.attr.zip_safe,
             "prelude_points": ctx.attr.prelude_points,
             "entry_point": main_py_ref,