diff --git a/src/python/flowmetal/parser.py b/src/python/flowmetal/parser.py index 3843a88..6ab72b5 100644 --- a/src/python/flowmetal/parser.py +++ b/src/python/flowmetal/parser.py @@ -272,32 +272,55 @@ class Parser(SexpParser): def parse_sqlist(cls, f: PosTrackingBufferedReader): return cls.parse_delimeted(f, "[", "]", lambda *args: ListToken(*args, ListType.SQUARE)) + # FIXME (arrdem 2020-07-18): + # Break this apart into middleware or composable features somehow? @classmethod def handle_symbol(cls, buff, pos): - # Parsing integers - if m := re.fullmatch(r"[+-]?\d[\d_]*", buff): - return IntegerToken(int(buff.replace("_", "")), buff, pos) + def _sign(m, idx): + if m.group(idx) == '-': + return -1 + else: + return 1 # Parsing integers with bases - elif m := re.fullmatch(r"(\d+)r([a-z0-9_]+)", buff): - return IntegerToken(int(m.group(2).replace("_", "")), int(m.group(1)), buff, pos) + if m := re.fullmatch(r"([+-]?)(\d+)r([a-z0-9_]+)", buff): + return IntegerToken( + _sign(m, 1) * int(m.group(3).replace("_", ""), + int(m.group(2))), + buff, + pos, + ) + + # Parsing hex numbers + if m := re.fullmatch(r"([+-]?)0[xX]([A-Fa-f0-9_]*)", buff): + val = m.group(2).replace("_", "") + return IntegerToken(_sign(m, 1) * int(val, 16), buff, pos) + + # Parsing octal numbers + if m := re.fullmatch(r"([+-]?)0([\d_]*)", buff): + val = m.group(2).replace("_", "") + return IntegerToken(_sign(m, 1) * int(val, 8), buff, pos) + + # Parsing integers + if m := re.fullmatch(r"([+-]?)\d[\d_]*", buff): + return IntegerToken(int(buff.replace("_", "")), buff, pos) # Parsing fractions - elif m := re.fullmatch(r"([+-]?\d[\d_]*)/(\d[\d_]*)", buff): + if m := re.fullmatch(r"([+-]?)(\d[\d_]*)/(\d[\d_]*)", buff): return FractionToken( Fraction( - int(m.group(1).replace("_", "")), - int(m.group(2).replace("_", ""))), + int(m.group(2).replace("_", "")), + int(m.group(3).replace("_", ""))), buff, pos, ) # Parsing floats - elif re.fullmatch(r"[+-]?\d[\d_]*(\.\d[\d_]*)?(e[+-]?\d[\d_]*)?", buff): + if re.fullmatch(r"([+-]?)\d[\d_]*(\.\d[\d_]*)?(e[+-]?\d[\d_]*)?", buff): return FloatToken(float(buff), buff, pos) - else: - return SymbolToken(buff, buff, pos) + # Default behavior + return SymbolToken(buff, buff, pos) @classmethod def parse_symbol(cls, f: PosTrackingBufferedReader): @@ -331,6 +354,7 @@ class Parser(SexpParser): buff = str(rtb) return CommentToken(buff, buff, pos) + ## Parsing interface def parses(buff: str, parser: SexpParser = Parser, diff --git a/test/python/flowmetal/test_parser.py b/test/python/flowmetal/test_parser.py index bac2344..11fda87 100644 --- a/test/python/flowmetal/test_parser.py +++ b/test/python/flowmetal/test_parser.py @@ -20,10 +20,32 @@ def test_parse_list(): ('2', 2), ('103', 103), ('504', 504), + # Sign prefixes ('-1', -1), ('+1', +1), + # Underscores as whitespace + ('1_000_000', 1e6), ('+1_000', 1000), ('-1_000', -1000), + # Variable base + ('2r1', 1), + ('2r10', 2), + ('2r100', 4), + ('2r101', 5), + ('+2r10', 2), + ('-2r10', -2), + # Octal + ('00', 0), + ('01', 1), + ('010', 8), + ('+010', 8), + ('-010', -8), + # Hex + ('0x0', 0), + ('0xF', 15), + ('0x10', 16), + ('+0x10', 16), + ('-0x10', -16), ]) def test_parse_num(txt, val): """Some trivial cases of parsing numbers."""