From 014ce0b21d1734f80d87e11aa8471b5c59eb9c0b Mon Sep 17 00:00:00 2001
From: Reid 'arrdem' McKenzie <me@arrdem.com>
Date: Sat, 21 Aug 2021 11:49:46 -0600
Subject: [PATCH] Working Lilith block header parser

---
 projects/lilith/BUILD                         |  2 +-
 projects/lilith/src/lilith/designdoc.lil      |  6 +-
 .../lilith/src/python/lilith/grammar.lark     | 21 +++++
 projects/lilith/src/python/lilith/parser.py   | 84 +++++++++++--------
 projects/lilith/test/python/conftest.py       | 23 +++++
 projects/lilith/test/python/test_parser.py    | 53 ++++++++----
 6 files changed, 134 insertions(+), 55 deletions(-)
 create mode 100644 projects/lilith/src/python/lilith/grammar.lark

diff --git a/projects/lilith/BUILD b/projects/lilith/BUILD
index 58e3348..c050566 100644
--- a/projects/lilith/BUILD
+++ b/projects/lilith/BUILD
@@ -1,7 +1,7 @@
 py_project(
     name = "lilith",
     lib_deps = [
-
+        py_requirement("lark"),
     ],
     test_deps = [
         py_requirement("hypothesis"),
diff --git a/projects/lilith/src/lilith/designdoc.lil b/projects/lilith/src/lilith/designdoc.lil
index e5d18fc..de3b123 100644
--- a/projects/lilith/src/lilith/designdoc.lil
+++ b/projects/lilith/src/lilith/designdoc.lil
@@ -40,8 +40,12 @@ FIXME: we need bare words, we need strings
 
 FIXME: We need the object language
 
+!def[openapi]
+!frag[lang: yaml]
+
 !def[main]
 !frag[lang: lil]
 ; is importing a bang-operation?
+
 import[tagle]
-print[tangle[pitch, syntax]]
+print[str.join["", list[pitch, syntax]]]
diff --git a/projects/lilith/src/python/lilith/grammar.lark b/projects/lilith/src/python/lilith/grammar.lark
new file mode 100644
index 0000000..a7c03f3
--- /dev/null
+++ b/projects/lilith/src/python/lilith/grammar.lark
@@ -0,0 +1,21 @@
+%import common.WORD
+%import common.NUMBER
+%import common.WS
+%ignore WS
+
+STRING: /""".*?"""/ | /".*?"/
+
+atom: NUMBER | WORD | STRING
+
+expr: atom
+
+args: expr ("," args)?
+
+kwargs: expr ":" expr ("," kwargs)?
+
+_args_kwargs: args ";" kwargs
+_args: args
+_kwargs: kwargs
+arguments: _args_kwargs | _args | _kwargs
+
+header: "!" WORD "[" arguments? "]"
diff --git a/projects/lilith/src/python/lilith/parser.py b/projects/lilith/src/python/lilith/parser.py
index f183c3d..6e58da3 100644
--- a/projects/lilith/src/python/lilith/parser.py
+++ b/projects/lilith/src/python/lilith/parser.py
@@ -4,13 +4,26 @@ Variously poor parsing for Lilith.
 
 import typing as t
 import re
+from importlib.resources import read_text
 
 import lark
 
+GRAMMAR = read_text('lilith', 'grammar.lark')
+
+
+# !foo[bar]
+# !def[name]
+# !frag[lang: yaml]
+# !end
+# all this following tex
+class Args(t.NamedTuple):
+    positionals: object = []
+    kwargs: object = {}
+
+
 class Block(t.NamedTuple):
     tag: str
-    args: list
-    kwargs: list
+    args: Args
     body_lines: list
 
     @property
@@ -19,6 +32,18 @@ class Block(t.NamedTuple):
 
 
 class TreeToTuples(lark.Transformer):
+    def atom(self, args):
+        return args[0]
+
+    def expr(self, args):
+        return args[0]
+
+    def args(self, args):
+        _args = [args[0].value]
+        if len(args) == 2:
+            _args = _args + args[1]
+        return _args
+
     def kwargs(self, args):
         d = {}
         key, val = args[0:2]
@@ -27,45 +52,32 @@ class TreeToTuples(lark.Transformer):
         d[key.value] = val.value
         return d
 
-    def args(self, args):
-        _args = [args[0].value]
-        if len(args) == 2:
-            _args = _args + args[1]
-        return _args
+    def _args_kwargs(self, args):
+        return lark.Tree('args', (args[0], args[1]))
 
-    def header(self, parse_args):
-        print("Header", parse_args)
-        tag = None
-        args = None
-        kwargs = None
+    def _args(self, args):
+        return lark.Tree('args', (args[0], {}))
+
+    def _kwargs(self, args):
+        return lark.Tree('args', ([], args[0]))
+
+    def arguments(self, args):
+        return args
+
+    def header(self, args):
+        print("Header", args)
+        tag = args[0]
+        arguments = args[1] if len(args) > 1 else ([], {})
         body = []
 
-        iargs = iter(parse_args[1])
-        tag = parse_args[0]
-        v = next(iargs, None)
-        if isinstance(v, list):
-            args = v
-            v = next(iargs, None)
-        if isinstance(v, dict):
-            kwargs = v
-
-        return Block(tag, args, kwargs, body)
+        return Block(tag, Args(*arguments), body)
 
 
-block_grammar = lark.Lark("""
-%import common.WORD
-%import common.WS
-%ignore WS
-?start: header
-
-args: WORD ("," args)?
-kwargs: WORD ":" WORD ("," kwargs)?
-arguments: args "," kwargs | args | kwargs
-header: "!" WORD "[" arguments? "]"
-""",
-                          parser='lalr',
-                          transformer=TreeToTuples())
-
+def parser_with_transformer(grammar, start="header"):
+    return lark.Lark(grammar,
+                     start=start,
+                     parser='lalr',
+                     transformer=TreeToTuples())
 
 
 def shotgun_parse(buff: str) -> t.List[object]:
diff --git a/projects/lilith/test/python/conftest.py b/projects/lilith/test/python/conftest.py
index 58a7168..cbf694b 100644
--- a/projects/lilith/test/python/conftest.py
+++ b/projects/lilith/test/python/conftest.py
@@ -1,3 +1,26 @@
 """
 Pytest fixtures.
 """
+
+from lilith.parser import Block, parser_with_transformer, GRAMMAR
+
+import pytest
+
+
+@pytest.fixture
+def args_grammar():
+    return parser_with_transformer(GRAMMAR, "args")
+
+
+@pytest.fixture
+def kwargs_grammar():
+    return parser_with_transformer(GRAMMAR, "kwargs")
+
+
+@pytest.fixture
+def arguments_grammar():
+    return parser_with_transformer(GRAMMAR, "arguments")
+
+@pytest.fixture
+def header_grammar():
+    return parser_with_transformer(GRAMMAR, "header")
diff --git a/projects/lilith/test/python/test_parser.py b/projects/lilith/test/python/test_parser.py
index f6bec6f..0d2acb0 100644
--- a/projects/lilith/test/python/test_parser.py
+++ b/projects/lilith/test/python/test_parser.py
@@ -1,26 +1,45 @@
 """tests covering the Lilith parser."""
 
-from lilith.parser import block_grammar, Block
+from lilith.parser import Args, Block, parser_with_transformer, GRAMMAR
 
 import pytest
 
+
+@pytest.mark.parametrize('example, result', [
+    ("1", ["1"]),
+    ("1, 2", ["1", "2"]),
+    ("1, 2, 3", ["1", "2", "3"]),
+])
+def test_parse_args(args_grammar, example, result):
+    assert args_grammar.parse(example) == result
+
+
+@pytest.mark.parametrize('example, result', [
+    ("foo: bar", {"foo": "bar"}),
+    ("foo: bar, baz: qux", {"foo": "bar", "baz": "qux"}),
+])
+def test_parse_kwargs(kwargs_grammar, example, result):
+    assert kwargs_grammar.parse(example) == result
+
+
+@pytest.mark.parametrize('example, result', [
+    ("1", (["1"], {})),
+    ("1, 2", (["1", "2"], {})),
+    ("1, 2, 3", (["1", "2", "3"], {})),
+    ("foo: bar", ([], {"foo": "bar"})),
+    ("foo: bar, baz: qux", ([], {"foo": "bar", "baz": "qux"})),
+    ("1; foo: bar, baz: qux", (["1"], {"foo": "bar", "baz": "qux"})),
+])
+def test_parse_arguments(arguments_grammar, example, result):
+    assert arguments_grammar.parse(example) == result
+
 @pytest.mark.parametrize('example, result', [
     ('!def[syntax]',
-     Block('def', ['syntax'], None, [])),
+     Block('def', Args(['syntax'], {}), [])),
     ('!frag[lang: md]',
-     Block('frag', None, {'lang': 'md'}, [])),
-    ('!frag[foo, lang: md]',
-     Block('frag', ['foo'], {'lang': 'md'}, [])),
+     Block('frag', Args([], {'lang': 'md'}), [])),
+    ('!frag[foo; lang: md]',
+     Block('frag', Args(['foo'], {'lang': 'md'}), [])),
 ])
-def test_parse_header(example, result):
-    assert block_grammar.parse(example) == result
-
-
-(
-    """!def[designdoc]
-!frag[lang: md]
-# Designdoc
-
-A design document""",
-     None
-)
+def test_parse_header(header_grammar, example, result):
+    assert header_grammar.parse(example) == result