From 1b97cfb41d0e7fb1f0fa8a85dbc817b4c61ec8d0 Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Sat, 21 Aug 2021 13:13:56 -0600 Subject: [PATCH] Ready to try interpreting --- projects/lilith/src/lilith/designdoc.lil | 13 +--- .../lilith/src/python/lilith/grammar.lark | 24 ++++--- projects/lilith/src/python/lilith/parser.py | 69 ++++++++++++++----- projects/lilith/src/python/lilith/reader.py | 44 ++++++++++++ projects/lilith/test/python/test_parser.py | 63 ++++++++++------- projects/lilith/test/python/test_reader.py | 16 +++++ 6 files changed, 168 insertions(+), 61 deletions(-) create mode 100644 projects/lilith/src/python/lilith/reader.py create mode 100644 projects/lilith/test/python/test_reader.py diff --git a/projects/lilith/src/lilith/designdoc.lil b/projects/lilith/src/lilith/designdoc.lil index de3b123..917d0b5 100644 --- a/projects/lilith/src/lilith/designdoc.lil +++ b/projects/lilith/src/lilith/designdoc.lil @@ -1,9 +1,4 @@ -!defscope[designdoc] -!scope[designdoc] - -!def[pitch] -!frag[lang: md] - +!def[pitch, frag[lang: md]] # The Lilith Pitch Code is more than .. just code for the compiler. @@ -40,11 +35,9 @@ FIXME: we need bare words, we need strings FIXME: We need the object language -!def[openapi] -!frag[lang: yaml] +!def[openapi, frag[lang: yaml]] -!def[main] -!frag[lang: lil] +!def[main, frag[lang: lil]] ; is importing a bang-operation? import[tagle] diff --git a/projects/lilith/src/python/lilith/grammar.lark b/projects/lilith/src/python/lilith/grammar.lark index a7c03f3..4ea9440 100644 --- a/projects/lilith/src/python/lilith/grammar.lark +++ b/projects/lilith/src/python/lilith/grammar.lark @@ -1,21 +1,29 @@ %import common.WORD -%import common.NUMBER +%import common.INT +%import common.FLOAT %import common.WS %ignore WS STRING: /""".*?"""/ | /".*?"/ -atom: NUMBER | WORD | STRING +int: INT +float: FLOAT +number: int | float +word: WORD ("." WORD)* +string: STRING +atom: number | word | string -expr: atom +application: word "[" arguments? "]" + +expr: application | atom args: expr ("," args)? kwargs: expr ":" expr ("," kwargs)? -_args_kwargs: args ";" kwargs -_args: args -_kwargs: kwargs -arguments: _args_kwargs | _args | _kwargs +a_args_kwargs: args ";" kwargs +a_args: args +a_kwargs: kwargs +arguments: a_args_kwargs | a_args | a_kwargs -header: "!" WORD "[" arguments? "]" +header: "!" application diff --git a/projects/lilith/src/python/lilith/parser.py b/projects/lilith/src/python/lilith/parser.py index 6e58da3..6a1d24a 100644 --- a/projects/lilith/src/python/lilith/parser.py +++ b/projects/lilith/src/python/lilith/parser.py @@ -21,25 +21,56 @@ class Args(t.NamedTuple): kwargs: object = {} -class Block(t.NamedTuple): +class Apply(t.NamedTuple): tag: str args: Args + + +class Block(t.NamedTuple): + app: Apply body_lines: list + @property + def tag(self): + return self.app.tag + + @property + def args(self): + return self.app.args + @property def body(self): return "\n".join(self.body_lines) class TreeToTuples(lark.Transformer): + def int(self, args): + return int(args[0]) + + def float(self, args): + return float(args[0]) + + def number(self, args): + return args[0] + + def word(self, args): + """args: ['a'] ['a' ['b', 'c', 'd']]""" + return ".".join(a.value for a in args) + def atom(self, args): return args[0] def expr(self, args): return args[0] + def application(self, args): + tag = args[0] + args = args[1] if len(args) > 1 else Args() + print(args) + return Apply(tag, args) + def args(self, args): - _args = [args[0].value] + _args = [args[0]] if len(args) == 2: _args = _args + args[1] return _args @@ -49,28 +80,23 @@ class TreeToTuples(lark.Transformer): key, val = args[0:2] if len(args) == 3: d.update(args[2]) - d[key.value] = val.value + d[key] = val return d - def _args_kwargs(self, args): - return lark.Tree('args', (args[0], args[1])) + def a_args_kwargs(self, args): + return Args(args[0], args[1]) - def _args(self, args): - return lark.Tree('args', (args[0], {})) + def a_args(self, args): + return Args(args[0], {}) - def _kwargs(self, args): - return lark.Tree('args', ([], args[0])) + def a_kwargs(self, args): + return Args([], args[0]) def arguments(self, args): - return args + return args[0] def header(self, args): - print("Header", args) - tag = args[0] - arguments = args[1] if len(args) > 1 else ([], {}) - body = [] - - return Block(tag, Args(*arguments), body) + return Block(args[0], []) def parser_with_transformer(grammar, start="header"): @@ -80,16 +106,21 @@ def parser_with_transformer(grammar, start="header"): transformer=TreeToTuples()) -def shotgun_parse(buff: str) -> t.List[object]: + +def parse_buffer(buff: str, name: str = "&buff") -> t.List[object]: + header_parser = parser_with_transformer(GRAMMAR, "header") + def _parse(): block = None for line in buff.splitlines(): if line.startswith("!"): if block: yield block - block = [line] + block = header_parser.parse(line) + elif block: + block.body_lines.append(line) else: - block.append(line) + raise SyntaxError("Buffers must start with a ![] block") if block: yield block diff --git a/projects/lilith/src/python/lilith/reader.py b/projects/lilith/src/python/lilith/reader.py new file mode 100644 index 0000000..fc6a1d4 --- /dev/null +++ b/projects/lilith/src/python/lilith/reader.py @@ -0,0 +1,44 @@ +""" +Lilith's reader takes parsed blocks and applies languages, building a module structure. +""" + +import logging +import typing as t +from .parser import Block, Args, parse_buffer +from warnings import warn + +log = logging.getLogger(__name__) + +class Module(t.NamedTuple): + name: str + defs: t.Dict[str, Block] + + +def read_buffer(buffer: str, name: str = "&buff") -> Module: + """Read a module out of a string [or file]""" + + m = Module(name, {}) + for block in parse_buffer(buffer, name): + log.debug(f"{name}, Got a block", block) + + if block.tag == "def": + if len(block.args.positionals) == 2: + def_name, expr = block.args.positionals + m.defs[def_name] = Block(expr, block.body_lines) + else: + raise SyntaxError("!def[, ; ] ") + + if block.args.kwargs: + warn("!def[] are ignored") + + else: + raise SyntaxError(f"Unsupported block !{block.tag}[..]") + + return m + + +def read_file(path: str): + """Read a module out of a file.""" + + with open(path) as fp: + return read_buffer(fp.read(), path) diff --git a/projects/lilith/test/python/test_parser.py b/projects/lilith/test/python/test_parser.py index 0d2acb0..f6d28ee 100644 --- a/projects/lilith/test/python/test_parser.py +++ b/projects/lilith/test/python/test_parser.py @@ -1,45 +1,60 @@ """tests covering the Lilith parser.""" -from lilith.parser import Args, Block, parser_with_transformer, GRAMMAR - +from lilith.parser import Apply, Args, Block, GRAMMAR, parse_buffer, parser_with_transformer import pytest -@pytest.mark.parametrize('example, result', [ - ("1", ["1"]), - ("1, 2", ["1", "2"]), - ("1, 2, 3", ["1", "2", "3"]), +@pytest.mark.parametrize('example, expected', [ + ("1", [1]), + ("1, 2", [1, 2]), + ("1, 2, 3", [1, 2, 3]), ]) -def test_parse_args(args_grammar, example, result): - assert args_grammar.parse(example) == result +def test_parse_args(args_grammar, example, expected): + assert args_grammar.parse(example) == expected -@pytest.mark.parametrize('example, result', [ +@pytest.mark.parametrize('example, expected', [ ("foo: bar", {"foo": "bar"}), ("foo: bar, baz: qux", {"foo": "bar", "baz": "qux"}), ]) -def test_parse_kwargs(kwargs_grammar, example, result): - assert kwargs_grammar.parse(example) == result +def test_parse_kwargs(kwargs_grammar, example, expected): + assert kwargs_grammar.parse(example) == expected -@pytest.mark.parametrize('example, result', [ - ("1", (["1"], {})), - ("1, 2", (["1", "2"], {})), - ("1, 2, 3", (["1", "2", "3"], {})), +@pytest.mark.parametrize('example, expected', [ + ("1", ([1], {})), + ("1, 2", ([1, 2], {})), + ("1, 2, 3", ([1, 2, 3], {})), ("foo: bar", ([], {"foo": "bar"})), ("foo: bar, baz: qux", ([], {"foo": "bar", "baz": "qux"})), - ("1; foo: bar, baz: qux", (["1"], {"foo": "bar", "baz": "qux"})), + ("1; foo: bar, baz: qux", ([1], {"foo": "bar", "baz": "qux"})), ]) -def test_parse_arguments(arguments_grammar, example, result): - assert arguments_grammar.parse(example) == result +def test_parse_arguments(arguments_grammar, example, expected): + assert arguments_grammar.parse(example) == expected -@pytest.mark.parametrize('example, result', [ + +@pytest.mark.parametrize('example, expected', [ ('!def[syntax]', - Block('def', Args(['syntax'], {}), [])), + Block(Apply('def', Args(['syntax'], {})), [])), ('!frag[lang: md]', - Block('frag', Args([], {'lang': 'md'}), [])), + Block(Apply('frag', Args([], {'lang': 'md'})), [])), ('!frag[foo; lang: md]', - Block('frag', Args(['foo'], {'lang': 'md'}), [])), + Block(Apply('frag', Args(['foo'], {'lang': 'md'})), [])), + ("!int.add[1, 2]", + Block(Apply('int.add', Args([1, 2], {})), [])), ]) -def test_parse_header(header_grammar, example, result): - assert header_grammar.parse(example) == result +def test_parse_header(header_grammar, example, expected): + assert header_grammar.parse(example) == expected + + +@pytest.mark.parametrize('example, expected', [ + ("!frag[lang: md]", + [Block(Apply('frag', Args([], {"lang": "md"})), [])]), + ("""!frag[lang: md]\nHello, world!\n\n""", + [Block(Apply('frag', Args([], {"lang": "md"})), ["Hello, world!", ""])]), + ("""!frag[lang: md]\nHello, world!\n\n!def[bar]""", + [Block(Apply('frag', Args([], {"lang": "md"})), ["Hello, world!", ""]), + Block(Apply('def', Args(["bar"], {})), [])]), +]) +def test_block_parser(example, expected): + assert parse_buffer(example) == expected diff --git a/projects/lilith/test/python/test_reader.py b/projects/lilith/test/python/test_reader.py new file mode 100644 index 0000000..bf62f29 --- /dev/null +++ b/projects/lilith/test/python/test_reader.py @@ -0,0 +1,16 @@ +"""Tests covering the reader.""" + +from lilith.parser import Apply, Args, Block +from lilith.reader import Module, read_buffer + +import pytest + + +@pytest.mark.parametrize('example, expected', [ + ("""!def[main, lang[lil]]\nprint["hello, world"]\n""", + Module("&buff", {"main": Block(Apply('lang', Args(["lil"], {})), ["print[\"hello, world\"]"])})) +]) +def test_read(example, expected): + got = read_buffer(example) + print(got) + assert got == expected