Working Lilith block header parser

projects/lilith/BUILD

@@ -1,7 +1,7 @@
 py_project(
     name = "lilith",
     lib_deps = [
-
+        py_requirement("lark"),
     ],
     test_deps = [
         py_requirement("hypothesis"),

projects/lilith/src/lilith/designdoc.lil

@@ -40,8 +40,12 @@ FIXME: we need bare words, we need strings
 
 FIXME: We need the object language
 
+!def[openapi]
+!frag[lang: yaml]
+
 !def[main]
 !frag[lang: lil]
 ; is importing a bang-operation?
+
 import[tagle]
-print[tangle[pitch, syntax]]
+print[str.join["", list[pitch, syntax]]]

projects/lilith/src/python/lilith/grammar.lark

@@ -0,0 +1,21 @@
+%import common.WORD
+%import common.NUMBER
+%import common.WS
+%ignore WS
+
+STRING: /""".*?"""/ | /".*?"/
+
+atom: NUMBER | WORD | STRING
+
+expr: atom
+
+args: expr ("," args)?
+
+kwargs: expr ":" expr ("," kwargs)?
+
+_args_kwargs: args ";" kwargs
+_args: args
+_kwargs: kwargs
+arguments: _args_kwargs | _args | _kwargs
+
+header: "!" WORD "[" arguments? "]"

projects/lilith/src/python/lilith/parser.py

@@ -4,13 +4,26 @@ Variously poor parsing for Lilith.
 
 import typing as t
 import re
+from importlib.resources import read_text
 
 import lark
 
+GRAMMAR = read_text('lilith', 'grammar.lark')
+
+
+# !foo[bar]
+# !def[name]
+# !frag[lang: yaml]
+# !end
+# all this following tex
+class Args(t.NamedTuple):
+    positionals: object = []
+    kwargs: object = {}
+
+
 class Block(t.NamedTuple):
     tag: str
-    args: list
-    kwargs: list
+    args: Args
     body_lines: list
 
     @property
@@ -19,6 +32,18 @@ class Block(t.NamedTuple):
 
 
 class TreeToTuples(lark.Transformer):
+    def atom(self, args):
+        return args[0]
+
+    def expr(self, args):
+        return args[0]
+
+    def args(self, args):
+        _args = [args[0].value]
+        if len(args) == 2:
+            _args = _args + args[1]
+        return _args
+
     def kwargs(self, args):
         d = {}
         key, val = args[0:2]
@@ -27,47 +52,34 @@ class TreeToTuples(lark.Transformer):
         d[key.value] = val.value
         return d
 
-    def args(self, args):
-        _args = [args[0].value]
-        if len(args) == 2:
-            _args = _args + args[1]
-        return _args
+    def _args_kwargs(self, args):
+        return lark.Tree('args', (args[0], args[1]))
 
-    def header(self, parse_args):
-        print("Header", parse_args)
-        tag = None
-        args = None
-        kwargs = None
+    def _args(self, args):
+        return lark.Tree('args', (args[0], {}))
+
+    def _kwargs(self, args):
+        return lark.Tree('args', ([], args[0]))
+
+    def arguments(self, args):
+        return args
+
+    def header(self, args):
+        print("Header", args)
+        tag = args[0]
+        arguments = args[1] if len(args) > 1 else ([], {})
         body = []
 
-        iargs = iter(parse_args[1])
-        tag = parse_args[0]
-        v = next(iargs, None)
-        if isinstance(v, list):
-            args = v
-            v = next(iargs, None)
-        if isinstance(v, dict):
-            kwargs = v
-
-        return Block(tag, args, kwargs, body)
+        return Block(tag, Args(*arguments), body)
 
 
-block_grammar = lark.Lark("""
-%import common.WORD
-%import common.WS
-%ignore WS
-?start: header
-
-args: WORD ("," args)?
-kwargs: WORD ":" WORD ("," kwargs)?
-arguments: args "," kwargs | args | kwargs
-header: "!" WORD "[" arguments? "]"
-""",
+def parser_with_transformer(grammar, start="header"):
+    return lark.Lark(grammar,
+                     start=start,
                      parser='lalr',
                      transformer=TreeToTuples())
 
 
-
 def shotgun_parse(buff: str) -> t.List[object]:
     def _parse():
         block = None

projects/lilith/test/python/conftest.py

@@ -1,3 +1,26 @@
 """
 Pytest fixtures.
 """
+
+from lilith.parser import Block, parser_with_transformer, GRAMMAR
+
+import pytest
+
+
+@pytest.fixture
+def args_grammar():
+    return parser_with_transformer(GRAMMAR, "args")
+
+
+@pytest.fixture
+def kwargs_grammar():
+    return parser_with_transformer(GRAMMAR, "kwargs")
+
+
+@pytest.fixture
+def arguments_grammar():
+    return parser_with_transformer(GRAMMAR, "arguments")
+
+@pytest.fixture
+def header_grammar():
+    return parser_with_transformer(GRAMMAR, "header")

projects/lilith/test/python/test_parser.py

@@ -1,26 +1,45 @@
 """tests covering the Lilith parser."""
 
-from lilith.parser import block_grammar, Block
+from lilith.parser import Args, Block, parser_with_transformer, GRAMMAR
 
 import pytest
 
+
+@pytest.mark.parametrize('example, result', [
+    ("1", ["1"]),
+    ("1, 2", ["1", "2"]),
+    ("1, 2, 3", ["1", "2", "3"]),
+])
+def test_parse_args(args_grammar, example, result):
+    assert args_grammar.parse(example) == result
+
+
+@pytest.mark.parametrize('example, result', [
+    ("foo: bar", {"foo": "bar"}),
+    ("foo: bar, baz: qux", {"foo": "bar", "baz": "qux"}),
+])
+def test_parse_kwargs(kwargs_grammar, example, result):
+    assert kwargs_grammar.parse(example) == result
+
+
+@pytest.mark.parametrize('example, result', [
+    ("1", (["1"], {})),
+    ("1, 2", (["1", "2"], {})),
+    ("1, 2, 3", (["1", "2", "3"], {})),
+    ("foo: bar", ([], {"foo": "bar"})),
+    ("foo: bar, baz: qux", ([], {"foo": "bar", "baz": "qux"})),
+    ("1; foo: bar, baz: qux", (["1"], {"foo": "bar", "baz": "qux"})),
+])
+def test_parse_arguments(arguments_grammar, example, result):
+    assert arguments_grammar.parse(example) == result
+
 @pytest.mark.parametrize('example, result', [
     ('!def[syntax]',
-     Block('def', ['syntax'], None, [])),
+     Block('def', Args(['syntax'], {}), [])),
     ('!frag[lang: md]',
-     Block('frag', None, {'lang': 'md'}, [])),
-    ('!frag[foo, lang: md]',
-     Block('frag', ['foo'], {'lang': 'md'}, [])),
+     Block('frag', Args([], {'lang': 'md'}), [])),
+    ('!frag[foo; lang: md]',
+     Block('frag', Args(['foo'], {'lang': 'md'}), [])),
 ])
-def test_parse_header(example, result):
-    assert block_grammar.parse(example) == result
-
-
-(
-    """!def[designdoc]
-!frag[lang: md]
-# Designdoc
-
-A design document""",
-     None
-)
+def test_parse_header(header_grammar, example, result):
+    assert header_grammar.parse(example) == result