source/projects/calf/tests/python/test_lexer.py
2021-08-29 22:35:10 -06:00

126 lines
2.7 KiB
Python

"""
Tests of calf.lexer
Tests both basic functionality, some examples and makes sure that arbitrary token sequences round
trip through the lexer.
"""
import calf.lexer as cl
from conftest import parametrize
def lex_single_token(buffer):
"""Lexes a single token from the buffer."""
return next(iter(cl.lex_buffer(buffer)))
@parametrize(
"text,token_type",
[
(
"(",
"PAREN_LEFT",
),
(
")",
"PAREN_RIGHT",
),
(
"[",
"BRACKET_LEFT",
),
(
"]",
"BRACKET_RIGHT",
),
(
"{",
"BRACE_LEFT",
),
(
"}",
"BRACE_RIGHT",
),
(
"^",
"META",
),
(
"#",
"MACRO_DISPATCH",
),
("'", "SINGLE_QUOTE"),
(
"foo",
"SYMBOL",
),
("foo/bar", "SYMBOL"),
(
":foo",
"KEYWORD",
),
(
":foo/bar",
"KEYWORD",
),
(
" ,,\t ,, \t",
"WHITESPACE",
),
("\n\r", "WHITESPACE"),
("\n", "WHITESPACE"),
(
" , ",
"WHITESPACE",
),
("; this is a sample comment\n", "COMMENT"),
('"foo"', "STRING"),
('"foo bar baz"', "STRING"),
],
)
def test_lex_examples(text, token_type):
t = lex_single_token(text)
assert t.value == text
assert t.type == token_type
@parametrize(
"text,token_types",
[
("foo^bar", ["SYMBOL", "META", "SYMBOL"]),
("foo bar", ["SYMBOL", "WHITESPACE", "SYMBOL"]),
("foo-bar", ["SYMBOL"]),
("foo\nbar", ["SYMBOL", "WHITESPACE", "SYMBOL"]),
(
"{[^#()]}",
[
"BRACE_LEFT",
"BRACKET_LEFT",
"META",
"MACRO_DISPATCH",
"PAREN_LEFT",
"PAREN_RIGHT",
"BRACKET_RIGHT",
"BRACE_RIGHT",
],
),
("+", ["SYMBOL"]),
("-", ["SYMBOL"]),
("1", ["INTEGER"]),
("-1", ["INTEGER"]),
("-1.0", ["FLOAT"]),
("-1e3", ["FLOAT"]),
("+1.3e", ["FLOAT"]),
("f", ["SYMBOL"]),
("f1", ["SYMBOL"]),
("f1g2", ["SYMBOL"]),
("foo13-bar", ["SYMBOL"]),
("foo+13-12bar", ["SYMBOL"]),
("+-+-+-+-+", ["SYMBOL"]),
],
)
def test_lex_compound_examples(text, token_types):
t = cl.lex_buffer(text)
result_types = [token.type for token in t]
assert result_types == token_types