Refactor some of the types, get keywords working

This commit is contained in:
Reid 'arrdem' McKenzie 2020-07-18 16:49:32 -06:00
parent 43d1040d76
commit beea56fad5
2 changed files with 58 additions and 18 deletions

View file

@ -5,7 +5,7 @@ A parser for s-expressions.
from abc import ABC, abstractmethod
from enum import Enum
from io import StringIO, BufferedReader
from typing import IO, NamedTuple
from typing import IO, NamedTuple, Any
from fractions import Fraction
import re
@ -40,54 +40,73 @@ class TokenBase(object):
def raw(self):
"""The raw token as scanned."""
class ConstTokenBase(TokenBase):
"""The shared interface for constant tokens"""
@property
@abstractmethod
def read(self):
"""Return a runtime value for the token, discarding any whitespace and soforth."""
def data(self) -> Any:
"""The value of the token."""
# Hash according to data
def __hash__(self):
return hash(self.data)
# And make sure it's orderable
def __eq__(self, other):
return self.data == other
def __lt__(self, other):
return self.data < other
def __gt__(self, other):
return self.data > other
class IntegerToken(NamedTuple, TokenBase):
class BooleanToken(ConstTokenBase, Enum):
"""A read boolean."""
data: bool
raw: str
pos: Position
class IntegerToken(ConstTokenBase, NamedTuple):
"""A read integer, including position."""
data: int
raw: str
pos: Position
def read(self):
return
class FractionToken(NamedTuple, TokenBase):
class FractionToken(ConstTokenBase, NamedTuple):
"""A read fraction, including position."""
data: Fraction
raw: str
pos: Position
def read(self):
return
class FloatToken(NamedTuple, TokenBase):
class FloatToken(ConstTokenBase, NamedTuple):
"""A read floating point number, including position."""
data: float
raw: str
pos: Position
def read(self):
return
class SymbolToken(NamedTuple, TokenBase):
class SymbolToken(ConstTokenBase, NamedTuple):
"""A read symbol, including position."""
data: str
raw: str
pos: Position
class KeywordToken(NamedTuple, TokenBase):
class KeywordToken(ConstTokenBase, NamedTuple):
"""A read keyword."""
data: str
raw: str
pos: Position
class StringToken(NamedTuple, TokenBase):
class StringToken(ConstTokenBase, NamedTuple):
"""A read string, including position."""
data: str
raw: str
@ -323,6 +342,17 @@ class Parser(SexpParser):
if re.fullmatch(r"([+-]?)\d[\d_]*(\.\d[\d_]*)?(e[+-]?\d[\d_]*)?", buff):
return FloatToken(float(buff), buff, pos)
# Booleans
if buff == "true":
return BooleanToken(True, buff, pos)
if buff == "false":
return BooleanToken(False, buff, pos)
# Keywords
if buff.startswith(":"):
return KeywordToken(buff, buff, pos)
# Default behavior
return SymbolToken(buff, buff, pos)

View file

@ -149,3 +149,13 @@ def test_ambiguous_floats(txt, tokenization):
def test_string(txt):
"""Some examples of strings, and of escape sequences."""
assert isinstance(p.parses(txt), p.StringToken)
@pytest.mark.parametrize('txt,', [
':foo',
':foo/bar',
':foo.bar/baz?',
])
def test_keyword(txt):
"""Some examples of keywords."""
assert isinstance(p.parses(txt), p.KeywordToken)