Ready to try interpreting

projects/lilith/src/lilith/designdoc.lil

@@ -1,9 +1,4 @@
-!defscope[designdoc]
-!scope[designdoc]
-
-!def[pitch]
-!frag[lang: md]
-
+!def[pitch, frag[lang: md]]
 # The Lilith Pitch
 
 Code is more than .. just code for the compiler.
@@ -40,11 +35,9 @@ FIXME: we need bare words, we need strings
 
 FIXME: We need the object language
 
-!def[openapi]
-!frag[lang: yaml]
+!def[openapi, frag[lang: yaml]]
 
-!def[main]
-!frag[lang: lil]
+!def[main, frag[lang: lil]]
 ; is importing a bang-operation?
 
 import[tagle]

projects/lilith/src/python/lilith/grammar.lark

@@ -1,21 +1,29 @@
 %import common.WORD
-%import common.NUMBER
+%import common.INT
+%import common.FLOAT
 %import common.WS
 %ignore WS
 
 STRING: /""".*?"""/ | /".*?"/
 
-atom: NUMBER | WORD | STRING
+int: INT
+float: FLOAT
+number: int | float
+word: WORD ("." WORD)*
+string: STRING
+atom: number | word | string
 
-expr: atom
+application: word "[" arguments? "]"
+
+expr: application | atom
 
 args: expr ("," args)?
 
 kwargs: expr ":" expr ("," kwargs)?
 
-_args_kwargs: args ";" kwargs
-_args: args
-_kwargs: kwargs
-arguments: _args_kwargs | _args | _kwargs
+a_args_kwargs: args ";" kwargs
+a_args: args
+a_kwargs: kwargs
+arguments: a_args_kwargs | a_args | a_kwargs
 
-header: "!" WORD "[" arguments? "]"
+header: "!" application

projects/lilith/src/python/lilith/parser.py

@@ -21,25 +21,56 @@ class Args(t.NamedTuple):
     kwargs: object = {}
 
 
-class Block(t.NamedTuple):
+class Apply(t.NamedTuple):
     tag: str
     args: Args
+
+
+class Block(t.NamedTuple):
+    app: Apply
     body_lines: list
 
+    @property
+    def tag(self):
+        return self.app.tag
+
+    @property
+    def args(self):
+        return self.app.args
+
     @property
     def body(self):
         return "\n".join(self.body_lines)
 
 
 class TreeToTuples(lark.Transformer):
+    def int(self, args):
+        return int(args[0])
+
+    def float(self, args):
+        return float(args[0])
+
+    def number(self, args):
+        return args[0]
+
+    def word(self, args):
+        """args: ['a'] ['a' ['b', 'c', 'd']]"""
+        return ".".join(a.value for a in args)
+
     def atom(self, args):
         return args[0]
 
     def expr(self, args):
         return args[0]
 
+    def application(self, args):
+        tag = args[0]
+        args = args[1] if len(args) > 1 else Args()
+        print(args)
+        return Apply(tag, args)
+
     def args(self, args):
-        _args = [args[0].value]
+        _args = [args[0]]
         if len(args) == 2:
             _args = _args + args[1]
         return _args
@@ -49,28 +80,23 @@ class TreeToTuples(lark.Transformer):
         key, val = args[0:2]
         if len(args) == 3:
             d.update(args[2])
-        d[key.value] = val.value
+        d[key] = val
         return d
 
-    def _args_kwargs(self, args):
-        return lark.Tree('args', (args[0], args[1]))
+    def a_args_kwargs(self, args):
+        return Args(args[0], args[1])
 
-    def _args(self, args):
-        return lark.Tree('args', (args[0], {}))
+    def a_args(self, args):
+        return Args(args[0], {})
 
-    def _kwargs(self, args):
-        return lark.Tree('args', ([], args[0]))
+    def a_kwargs(self, args):
+        return Args([], args[0])
 
     def arguments(self, args):
-        return args
+        return args[0]
 
     def header(self, args):
-        print("Header", args)
-        tag = args[0]
-        arguments = args[1] if len(args) > 1 else ([], {})
-        body = []
-
-        return Block(tag, Args(*arguments), body)
+        return Block(args[0], [])
 
 
 def parser_with_transformer(grammar, start="header"):
@@ -80,16 +106,21 @@ def parser_with_transformer(grammar, start="header"):
                      transformer=TreeToTuples())
 
 
-def shotgun_parse(buff: str) -> t.List[object]:
+
+def parse_buffer(buff: str, name: str = "&buff") -> t.List[object]:
+    header_parser = parser_with_transformer(GRAMMAR, "header")
+
     def _parse():
         block = None
         for line in buff.splitlines():
             if line.startswith("!"):
                 if block:
                     yield block
-                block = [line]
+                block = header_parser.parse(line)
+            elif block:
+                block.body_lines.append(line)
             else:
-                block.append(line)
+                raise SyntaxError("Buffers must start with a ![] block")
         if block:
             yield block
 

projects/lilith/src/python/lilith/reader.py

@@ -0,0 +1,44 @@
+"""
+Lilith's reader takes parsed blocks and applies languages, building a module structure.
+"""
+
+import logging
+import typing as t
+from .parser import Block, Args, parse_buffer
+from warnings import warn
+
+log = logging.getLogger(__name__)
+
+class Module(t.NamedTuple):
+    name: str
+    defs: t.Dict[str, Block]
+
+
+def read_buffer(buffer: str, name: str = "&buff") -> Module:
+    """Read a module out of a string [or file]"""
+
+    m = Module(name, {})
+    for block in parse_buffer(buffer, name):
+        log.debug(f"{name}, Got a block", block)
+
+        if block.tag == "def":
+            if len(block.args.positionals) == 2:
+                def_name, expr = block.args.positionals
+                m.defs[def_name] = Block(expr, block.body_lines)
+            else:
+                raise SyntaxError("!def[<name>, <expr>; <kwargs>] <body>")
+
+            if block.args.kwargs:
+                warn("!def[<kwargs>] are ignored")
+
+        else:
+            raise SyntaxError(f"Unsupported block !{block.tag}[..]")
+
+    return m
+
+
+def read_file(path: str):
+    """Read a module out of a file."""
+
+    with open(path) as fp:
+        return read_buffer(fp.read(), path)

projects/lilith/test/python/test_parser.py

@@ -1,45 +1,60 @@
 """tests covering the Lilith parser."""
 
-from lilith.parser import Args, Block, parser_with_transformer, GRAMMAR
-
+from lilith.parser import Apply, Args, Block, GRAMMAR, parse_buffer, parser_with_transformer
 import pytest
 
 
-@pytest.mark.parametrize('example, result', [
-    ("1", ["1"]),
-    ("1, 2", ["1", "2"]),
-    ("1, 2, 3", ["1", "2", "3"]),
+@pytest.mark.parametrize('example, expected', [
+    ("1", [1]),
+    ("1, 2", [1, 2]),
+    ("1, 2, 3", [1, 2, 3]),
 ])
-def test_parse_args(args_grammar, example, result):
-    assert args_grammar.parse(example) == result
+def test_parse_args(args_grammar, example, expected):
+    assert args_grammar.parse(example) == expected
 
 
-@pytest.mark.parametrize('example, result', [
+@pytest.mark.parametrize('example, expected', [
     ("foo: bar", {"foo": "bar"}),
     ("foo: bar, baz: qux", {"foo": "bar", "baz": "qux"}),
 ])
-def test_parse_kwargs(kwargs_grammar, example, result):
-    assert kwargs_grammar.parse(example) == result
+def test_parse_kwargs(kwargs_grammar, example, expected):
+    assert kwargs_grammar.parse(example) == expected
 
 
-@pytest.mark.parametrize('example, result', [
-    ("1", (["1"], {})),
-    ("1, 2", (["1", "2"], {})),
-    ("1, 2, 3", (["1", "2", "3"], {})),
+@pytest.mark.parametrize('example, expected', [
+    ("1", ([1], {})),
+    ("1, 2", ([1, 2], {})),
+    ("1, 2, 3", ([1, 2, 3], {})),
     ("foo: bar", ([], {"foo": "bar"})),
     ("foo: bar, baz: qux", ([], {"foo": "bar", "baz": "qux"})),
-    ("1; foo: bar, baz: qux", (["1"], {"foo": "bar", "baz": "qux"})),
+    ("1; foo: bar, baz: qux", ([1], {"foo": "bar", "baz": "qux"})),
 ])
-def test_parse_arguments(arguments_grammar, example, result):
-    assert arguments_grammar.parse(example) == result
+def test_parse_arguments(arguments_grammar, example, expected):
+    assert arguments_grammar.parse(example) == expected
 
-@pytest.mark.parametrize('example, result', [
+
+@pytest.mark.parametrize('example, expected', [
     ('!def[syntax]',
-     Block('def', Args(['syntax'], {}), [])),
+     Block(Apply('def', Args(['syntax'], {})), [])),
     ('!frag[lang: md]',
-     Block('frag', Args([], {'lang': 'md'}), [])),
+     Block(Apply('frag', Args([], {'lang': 'md'})), [])),
     ('!frag[foo; lang: md]',
-     Block('frag', Args(['foo'], {'lang': 'md'}), [])),
+     Block(Apply('frag', Args(['foo'], {'lang': 'md'})), [])),
+    ("!int.add[1, 2]",
+     Block(Apply('int.add', Args([1, 2], {})), [])),
 ])
-def test_parse_header(header_grammar, example, result):
-    assert header_grammar.parse(example) == result
+def test_parse_header(header_grammar, example, expected):
+    assert header_grammar.parse(example) == expected
+
+
+@pytest.mark.parametrize('example, expected', [
+    ("!frag[lang: md]",
+     [Block(Apply('frag', Args([], {"lang": "md"})), [])]),
+    ("""!frag[lang: md]\nHello, world!\n\n""",
+     [Block(Apply('frag', Args([], {"lang": "md"})), ["Hello, world!", ""])]),
+    ("""!frag[lang: md]\nHello, world!\n\n!def[bar]""",
+     [Block(Apply('frag', Args([], {"lang": "md"})), ["Hello, world!", ""]),
+      Block(Apply('def', Args(["bar"], {})), [])]),
+])
+def test_block_parser(example, expected):
+    assert parse_buffer(example) == expected

projects/lilith/test/python/test_reader.py

@@ -0,0 +1,16 @@
+"""Tests covering the reader."""
+
+from lilith.parser import Apply, Args, Block
+from lilith.reader import Module, read_buffer
+
+import pytest
+
+
+@pytest.mark.parametrize('example, expected', [
+    ("""!def[main, lang[lil]]\nprint["hello, world"]\n""",
+     Module("&buff", {"main": Block(Apply('lang', Args(["lil"], {})), ["print[\"hello, world\"]"])}))
+])
+def test_read(example, expected):
+    got = read_buffer(example)
+    print(got)
+    assert got == expected