From d7f175a987f33c644fd5baf36c491b28af5c07cd Mon Sep 17 00:00:00 2001 From: Reid 'arrdem' McKenzie Date: Sat, 18 Jul 2020 13:46:10 -0600 Subject: [PATCH] Get floating point numbers working too --- src/flowmetal/com.twitter.wilson.flow | 83 +++++++++++++++++++++++ src/python/flowmetal/parser.py | 89 ++++++++++++------------- src/python/flowmetal/syntax_analyzer.py | 16 ++++- test/python/flowmetal/test_parser.py | 35 +++++++++- 4 files changed, 174 insertions(+), 49 deletions(-) create mode 100644 src/flowmetal/com.twitter.wilson.flow diff --git a/src/flowmetal/com.twitter.wilson.flow b/src/flowmetal/com.twitter.wilson.flow new file mode 100644 index 0000000..9d61a52 --- /dev/null +++ b/src/flowmetal/com.twitter.wilson.flow @@ -0,0 +1,83 @@ +(defpackage com.twitter.wilson + (require + ;; The log lets you record status information into a program's trace + [flowmetal.log + :refer [log!]] + ;; The time system lets you put bounds on the implicit awaiting Flowmetal does + [flowmetal.time + :refer [with-timeout!, timeout?, make-duration, duration?, +seconds+, +hours+, sleep!]] + ;; JSON. Simple enough + [flowmetal.json + :refer [loads, dumps, json?]] + ;; Extensions! Provided by other systems. + ;; + ;; This one allows for an external service to receive HTTP callbacks on Flowmetal's behalf. + [http.callback + :refer [make-callback!, get-callback!, callback?]] + ;; This one allows for an external service to make HTTP requests on Flowmetal's behalf. + [http.request + :refer [post!, error?, dns-error?, connection-error?, response-error?]]) + + (defenum stage + +reboot+ + +bios-update+ + +reinstall+) + + ;; FIXME:- how to do table optimization? + (defn fib [x] + (match x + [0 1] + [1 1] + [_ (+ (fib (- x 1) (- x 2)))])) + + (defn retry-http [f + :- (fn? [] a?) + backoff-fn + :- (fn? [int?] duration?) + :default (fn [x :- int?] + :- duration? + (make-duration (fib x) +seconds+)) + backoff-count + :- int? + :default 0] + :- a + "The implementation of HTTP with retrying." + (let [response (f)] + (if (not (error? response)) + response + ;; FIXME:- how does auth denied get represented? + (if (or (dns-error? response) + (connection-error? response) + (response-error? response)) + (do (sleep (backoff-fn backoff-count)) + (retry-http* f backoff-fn (+ backoff-count 1))))))) + + (defn job [hostname + :- str? + stages + :- (list? stage?) + job-timeout + :- duration? + :default (duration 3 :hours)] + :- (union? [timeout? json?]) + "Run a wilson job, wait for the callback and process the result. + + By default the job is only waited for three hours. + + " + (let [callback :- callback? (make-callback!) + job (retry-http + (fn [] + (post "http://wilson.local.twitter.com" + :data + (dumps + {:host hostname + :stages [stages] + :callbacks [{:type :http, :url callback}]}))))] + + (let [result (with-timeout! (duration 3 :hours) + (fn [] + (get-callback callback)))] + (if-not (timeout? result) + (loads result) + result))))) diff --git a/src/python/flowmetal/parser.py b/src/python/flowmetal/parser.py index eda3494..3843a88 100644 --- a/src/python/flowmetal/parser.py +++ b/src/python/flowmetal/parser.py @@ -7,6 +7,7 @@ from enum import Enum from io import StringIO, BufferedReader from typing import IO, NamedTuple from fractions import Fraction +import re ## Types @@ -54,9 +55,9 @@ class IntegerToken(NamedTuple, TokenBase): return -class RationalToken(NamedTuple, TokenBase): - """A read integer, including position.""" - data: int +class FractionToken(NamedTuple, TokenBase): + """A read fraction, including position.""" + data: Fraction raw: str pos: Position @@ -66,7 +67,7 @@ class RationalToken(NamedTuple, TokenBase): class FloatToken(NamedTuple, TokenBase): """A read floating point number, including position.""" - data: int + data: float raw: str pos: Position @@ -225,8 +226,6 @@ class Parser(SexpParser): return cls.parse_sqlist(f) elif cls.isspace(f.peek()): return cls.parse_whitespace(f) - elif f.peek().isdigit(): - return cls.parse_num(f) elif f.peek() == ";": return cls.parse_comment(f) else: @@ -235,7 +234,21 @@ class Parser(SexpParser): @classmethod def isspace(cls, ch: str): """An extension point allowing for a more expansive concept of whitespace.""" - return ch.isspace() + return ch.isspace() or ch == ',' + + @classmethod + def ispunct(cls, ch: str): + return cls.isspace(ch) or ch in ( + ';' # Semicolon + '()' # Parens + '⟮⟯' # 'flat' parens + '[]' # Square brackets + '⟦⟧' # 'white' square brackets + '{}' # Curly brackets + '⟨⟩' # Angle brackets + '《》' # Double angle brackets + '⟪⟫' # Another kind of double angle brackets + ) @classmethod def parse_delimeted(cls, f: PosTrackingBufferedReader, openc, closec, ctor): @@ -260,57 +273,41 @@ class Parser(SexpParser): return cls.parse_delimeted(f, "[", "]", lambda *args: ListToken(*args, ListType.SQUARE)) @classmethod - def parse_unum(cls, f: PosTrackingBufferedReader): - with ReadThroughBuffer(f) as rtb: - assert rtb.peek().isdigit() - pos = f.pos() - while rtb.peek().isdigit(): - rtb.read() - buff = str(rtb) - return IntegerToken(int(buff), buff, pos) + def handle_symbol(cls, buff, pos): + # Parsing integers + if m := re.fullmatch(r"[+-]?\d[\d_]*", buff): + return IntegerToken(int(buff.replace("_", "")), buff, pos) - @classmethod - def parse_num(cls, f: PosTrackingBufferedReader): - with ReadThroughBuffer(f) as rtb: - num: IntegerToken = cls.parse_unum(rtb) + # Parsing integers with bases + elif m := re.fullmatch(r"(\d+)r([a-z0-9_]+)", buff): + return IntegerToken(int(m.group(2).replace("_", "")), int(m.group(1)), buff, pos) - # Various cases of more interesting numbers - if rtb.peek() == "/": - ## Case of a rational - # Discard the delimeter - rtb.read() - denom = cls.parse_num(rtb) + # Parsing fractions + elif m := re.fullmatch(r"([+-]?\d[\d_]*)/(\d[\d_]*)", buff): + return FractionToken( + Fraction( + int(m.group(1).replace("_", "")), + int(m.group(2).replace("_", ""))), + buff, + pos, + ) - return RationalToken(Fraction(num.data, denom.data), str(rtb), num.pos) + # Parsing floats + elif re.fullmatch(r"[+-]?\d[\d_]*(\.\d[\d_]*)?(e[+-]?\d[\d_]*)?", buff): + return FloatToken(float(buff), buff, pos) - elif rtb.peek() == "r": - ## Case of a number with a base - # Discard thd delimeter - rtb.read() - body = cls.parse_symbol(rtb) - return IntegerToken(int(body.raw, num.data), str(rtb), num.pos) - - elif rtb.peek() == ".": - ## Case of a number with a decimal component - ## Note there may be a trailing exponent - raise NotImplementedError() - - elif rtb.peek() == "e": - ## Case of a number with a floating point exponent - raise NotImplementedError() - - else: - return num + else: + return SymbolToken(buff, buff, pos) @classmethod def parse_symbol(cls, f: PosTrackingBufferedReader): with ReadThroughBuffer(f) as rtb: pos = None - while rtb.peek() and not cls.isspace(rtb.peek()): + while rtb.peek() and not cls.ispunct(rtb.peek()): pos = pos or rtb.pos() rtb.read() buff = str(rtb) - return SymbolToken(buff, buff, pos) + return cls.handle_symbol(buff, pos) @classmethod def parse_whitespace(cls, f: PosTrackingBufferedReader): diff --git a/src/python/flowmetal/syntax_analyzer.py b/src/python/flowmetal/syntax_analyzer.py index 98c32eb..81d0717 100644 --- a/src/python/flowmetal/syntax_analyzer.py +++ b/src/python/flowmetal/syntax_analyzer.py @@ -12,7 +12,7 @@ import flowmetal.parser as p ### Types ## We are not, in fact, sponsored by Typelevel LLC. -class TypeLevelExpr(ABC): +class TypeLevelExpr(object): """A base class for type-level expressions.""" pass @@ -27,24 +27,29 @@ class TypeExpr(TypeLevelExpr, NamedTuple): ## Now down to reality -class ValueLevelExpr(ABC): +class ValueLevelExpr(object): """A base class for value-level expressions.""" + pass class AscribeExpr(ValueLevelExpr, NamedTuple): """Ascribe a type (via type-level expression) to a value-level expression.""" + pass class InvokeExpr(ValueLevelExpr, NamedTuple): """(a ⊢ (fn A ⊢ B) [...] ⊢ A) ⊢ B""" + pass class IfExpr(ValueLevelExpr, NamedTuple): """(if test a ⊢ A b ⊢ B) ⊢ (Variant A B).""" + pass class LetExpr(ValueLevelExpr, NamedTuple): """Let a single binding and wrap a body. Yes one. N-ary let is an abstraction.""" + pass class DoExpr(ValueError, NamedTuple): @@ -52,24 +57,31 @@ class DoExpr(ValueError, NamedTuple): (do a b c ... ω ⊢ Ω) ⊢ Ω """ + pass + ProcExpr = DoExpr # ain't broke don't fix it + class MappingExpr(ValueLevelExpr, NamedTuple): """Mappings require their own constructor expression due to local/symbol references.""" + pass class SetExpr(ValueLevelExpr, NamedTuple): """Sets require their own constructor expression due to local/symbol references.""" + pass class ListExpr(ValueLevelExpr, NamedTuple): """While round () lists are generally InvokeExprs, [] lists are constructors like sets and maps.""" + pass ## Reader implementation class SexpAnalyzer(ABC): """A base class for Analyzers.""" + pass class Analyzer(SexpAnalyzer): diff --git a/test/python/flowmetal/test_parser.py b/test/python/flowmetal/test_parser.py index dbac466..7665e1f 100644 --- a/test/python/flowmetal/test_parser.py +++ b/test/python/flowmetal/test_parser.py @@ -2,6 +2,8 @@ Tests covering the Flowmetal parser. """ +from math import nan + import flowmetal.parser as p import pytest @@ -27,7 +29,7 @@ def test_parse_num(num): ]) def test_parse_ratio(frac): """Test covering the ratio notation.""" - assert isinstance(p.parses(frac), p.RationalToken) + assert isinstance(p.parses(frac), p.FractionToken) assert p.parses(frac).data == p.Fraction(frac) @@ -64,3 +66,34 @@ def test_list_contents(txt, tokenization): for (type, text), token in zip(tokenization, lelems): assert isinstance(token, type) assert token.raw == text + + +@pytest.mark.parametrize('txt, value', [ + ('1.0', 1.0), + ('-1.0', -1.0), + ('1.01', 1.01), + ('1e0', 1e0), + ('1e3', 1e3), + ('1e-3', 1e-3), + ('1.01e3', 1.01e3), +]) +def test_float_values(txt, value): + """Some examples of floats.""" + assert isinstance(p.parses(txt), p.FloatToken) + assert p.parses(txt).data == value + + +@pytest.mark.parametrize('txt, tokenization', [ + ('+1', p.IntegerToken), + ('+1+', p.SymbolToken), + ('+1e', p.SymbolToken), + ('+1e3', p.FloatToken), + ('+1.0', p.FloatToken), + ('+1.0e3', p.FloatToken), + ('a.b', p.SymbolToken), + ('1.b', p.SymbolToken), +]) +def test_ambiguous_floats(txt, tokenization): + """Parse examples of 'difficult' floats and symbols.""" + assert isinstance(p.parses(txt), tokenization), "Token type didn't match!" + assert p.parses(txt).raw == txt, "Parse wasn't total!"