Refactor some of the types, get keywords working

src/python/flowmetal/parser.py

@@ -5,7 +5,7 @@ A parser for s-expressions.
 from abc import ABC, abstractmethod
 from enum import Enum
 from io import StringIO, BufferedReader
-from typing import IO, NamedTuple
+from typing import IO, NamedTuple, Any
 from fractions import Fraction
 import re
 
@@ -40,54 +40,73 @@ class TokenBase(object):
     def raw(self):
         """The raw token as scanned."""
 
+
+class ConstTokenBase(TokenBase):
+    """The shared interface for constant tokens"""
+
+    @property
     @abstractmethod
-    def read(self):
+    def data(self) -> Any:
-        """Return a runtime value for the token, discarding any whitespace and soforth."""
+        """The value of the token."""
+
+    # Hash according to data
+    def __hash__(self):
+        return hash(self.data)
+
+    # And make sure it's orderable
+    def __eq__(self, other):
+        return self.data == other
+
+    def __lt__(self, other):
+        return self.data < other
+
+    def __gt__(self, other):
+        return self.data > other
 
 
-class IntegerToken(NamedTuple, TokenBase):
+class BooleanToken(ConstTokenBase, Enum):
+    """A read boolean."""
+    data: bool
+    raw: str
+    pos: Position
+
+
+class IntegerToken(ConstTokenBase, NamedTuple):
     """A read integer, including position."""
     data: int
     raw: str
     pos: Position
 
-    def read(self):
-        return
 
-
+class FractionToken(ConstTokenBase, NamedTuple):
-class FractionToken(NamedTuple, TokenBase):
     """A read fraction, including position."""
     data: Fraction
     raw: str
     pos: Position
 
-    def read(self):
-        return
 
-
+class FloatToken(ConstTokenBase, NamedTuple):
-class FloatToken(NamedTuple, TokenBase):
     """A read floating point number, including position."""
     data: float
     raw: str
     pos: Position
 
-    def read(self):
-        return
 
-
+class SymbolToken(ConstTokenBase, NamedTuple):
-class SymbolToken(NamedTuple, TokenBase):
     """A read symbol, including position."""
     data: str
     raw: str
     pos: Position
 
-class KeywordToken(NamedTuple, TokenBase):
+
+class KeywordToken(ConstTokenBase, NamedTuple):
     """A read keyword."""
     data: str
+    raw: str
     pos: Position
 
 
-class StringToken(NamedTuple, TokenBase):
+class StringToken(ConstTokenBase, NamedTuple):
     """A read string, including position."""
     data: str
     raw: str
@@ -323,6 +342,17 @@ class Parser(SexpParser):
         if re.fullmatch(r"([+-]?)\d[\d_]*(\.\d[\d_]*)?(e[+-]?\d[\d_]*)?", buff):
             return FloatToken(float(buff), buff, pos)
 
+        # Booleans
+        if buff == "true":
+            return BooleanToken(True, buff, pos)
+
+        if buff == "false":
+            return BooleanToken(False, buff, pos)
+
+        # Keywords
+        if buff.startswith(":"):
+            return KeywordToken(buff, buff, pos)
+
         # Default behavior
         return SymbolToken(buff, buff, pos)
 

test/python/flowmetal/test_parser.py

@@ -149,3 +149,13 @@ def test_ambiguous_floats(txt, tokenization):
 def test_string(txt):
     """Some examples of strings, and of escape sequences."""
     assert isinstance(p.parses(txt), p.StringToken)
+
+
+@pytest.mark.parametrize('txt,', [
+    ':foo',
+    ':foo/bar',
+    ':foo.bar/baz?',
+])
+def test_keyword(txt):
+    """Some examples of keywords."""
+    assert isinstance(p.parses(txt), p.KeywordToken)