Get floating point numbers working too

src/flowmetal/com.twitter.wilson.flow

@@ -0,0 +1,83 @@
+(defpackage com.twitter.wilson
+  (require
+    ;; The log lets you record status information into a program's trace
+    [flowmetal.log
+     :refer [log!]]
+    ;; The time system lets you put bounds on the implicit awaiting Flowmetal does
+    [flowmetal.time
+     :refer [with-timeout!, timeout?, make-duration, duration?, +seconds+, +hours+, sleep!]]
+    ;; JSON. Simple enough
+    [flowmetal.json
+     :refer [loads, dumps, json?]]
+    ;; Extensions! Provided by other systems.
+    ;;
+    ;; This one allows for an external service to receive HTTP callbacks on Flowmetal's behalf.
+    [http.callback
+     :refer [make-callback!, get-callback!, callback?]]
+     ;; This one allows for an external service to make HTTP requests on Flowmetal's behalf.
+    [http.request
+     :refer [post!, error?, dns-error?, connection-error?, response-error?]])
+
+  (defenum stage
+    +reboot+
+    +bios-update+
+    +reinstall+)
+
+  ;; FIXME:- how to do table optimization?
+  (defn fib [x]
+    (match x
+       [0 1]
+       [1 1]
+       [_ (+ (fib (- x 1) (- x 2)))]))
+
+  (defn retry-http [f
+                      :- (fn? [] a?)
+                    backoff-fn
+                      :- (fn? [int?] duration?)
+                      :default (fn [x :- int?]
+                                   :- duration?
+                                  (make-duration (fib x) +seconds+))
+                    backoff-count
+                      :- int?
+                      :default 0]
+                    :- a
+    "The implementation of HTTP with retrying."
+    (let [response (f)]
+      (if (not (error? response))
+         response
+         ;; FIXME:- how does auth denied get represented?
+         (if (or (dns-error? response)
+                 (connection-error? response)
+                 (response-error? response))
+             (do (sleep (backoff-fn backoff-count))
+                 (retry-http* f backoff-fn (+ backoff-count 1)))))))
+
+  (defn job [hostname
+               :- str?
+             stages
+               :- (list? stage?)
+             job-timeout
+               :- duration?
+               :default (duration 3 :hours)]
+             :- (union? [timeout? json?])
+    "Run a wilson job, wait for the callback and process the result.
+
+    By default the job is only waited for three hours.
+
+    "
+    (let [callback :- callback? (make-callback!)
+          job (retry-http
+                (fn []
+                  (post "http://wilson.local.twitter.com"
+                   :data
+                   (dumps
+                     {:host hostname
+                      :stages [stages]
+                      :callbacks [{:type :http, :url callback}]}))))]
+
+      (let [result (with-timeout! (duration 3 :hours)
+                     (fn []
+                        (get-callback callback)))]
+         (if-not (timeout? result)
+           (loads result)
+           result)))))

src/python/flowmetal/parser.py

@@ -7,6 +7,7 @@ from enum import Enum
 from io import StringIO, BufferedReader
 from typing import IO, NamedTuple
 from fractions import Fraction
+import re
 
 
 ## Types
@@ -54,9 +55,9 @@ class IntegerToken(NamedTuple, TokenBase):
         return
 
 
-class RationalToken(NamedTuple, TokenBase):
+class FractionToken(NamedTuple, TokenBase):
-    """A read integer, including position."""
+    """A read fraction, including position."""
-    data: int
+    data: Fraction
     raw: str
     pos: Position
 
@@ -66,7 +67,7 @@ class RationalToken(NamedTuple, TokenBase):
 
 class FloatToken(NamedTuple, TokenBase):
     """A read floating point number, including position."""
-    data: int
+    data: float
     raw: str
     pos: Position
 
@@ -225,8 +226,6 @@ class Parser(SexpParser):
             return cls.parse_sqlist(f)
         elif cls.isspace(f.peek()):
             return cls.parse_whitespace(f)
-        elif f.peek().isdigit():
-            return cls.parse_num(f)
         elif f.peek() == ";":
             return cls.parse_comment(f)
         else:
@@ -235,7 +234,21 @@ class Parser(SexpParser):
     @classmethod
     def isspace(cls, ch: str):
         """An extension point allowing for a more expansive concept of whitespace."""
-        return ch.isspace()
+        return ch.isspace() or ch == ','
+
+    @classmethod
+    def ispunct(cls, ch: str):
+        return cls.isspace(ch) or ch in (
+            ';'   # Semicolon
+            '()'  # Parens
+            '⟮⟯'   # 'flat' parens
+            '[]'  # Square brackets
+            '⟦⟧'  # 'white' square brackets
+            '{}'  # Curly brackets
+            '⟨⟩'  # Angle brackets
+            '《》'  # Double angle brackets
+            '⟪⟫'  # Another kind of double angle brackets
+        )
 
     @classmethod
     def parse_delimeted(cls, f: PosTrackingBufferedReader, openc, closec, ctor):
@@ -260,57 +273,41 @@ class Parser(SexpParser):
         return cls.parse_delimeted(f, "[", "]", lambda *args: ListToken(*args, ListType.SQUARE))
 
     @classmethod
-    def parse_unum(cls, f: PosTrackingBufferedReader):
+    def handle_symbol(cls, buff, pos):
-        with ReadThroughBuffer(f) as rtb:
+        # Parsing integers
-            assert rtb.peek().isdigit()
+        if m := re.fullmatch(r"[+-]?\d[\d_]*", buff):
-            pos = f.pos()
+            return IntegerToken(int(buff.replace("_", "")), buff, pos)
-            while rtb.peek().isdigit():
-                rtb.read()
-            buff = str(rtb)
-            return IntegerToken(int(buff), buff, pos)
 
-    @classmethod
+        # Parsing integers with bases
-    def parse_num(cls, f: PosTrackingBufferedReader):
+        elif m := re.fullmatch(r"(\d+)r([a-z0-9_]+)", buff):
-        with ReadThroughBuffer(f) as rtb:
+            return IntegerToken(int(m.group(2).replace("_", "")), int(m.group(1)), buff, pos)
-            num: IntegerToken = cls.parse_unum(rtb)
 
-            # Various cases of more interesting numbers
+        # Parsing fractions
-            if rtb.peek() == "/":
+        elif m := re.fullmatch(r"([+-]?\d[\d_]*)/(\d[\d_]*)", buff):
-                ## Case of a rational
+            return FractionToken(
-                # Discard the delimeter
+                Fraction(
-                rtb.read()
+                    int(m.group(1).replace("_", "")),
-                denom = cls.parse_num(rtb)
+                    int(m.group(2).replace("_", ""))),
+                buff,
+                pos,
+            )
 
-                return RationalToken(Fraction(num.data, denom.data), str(rtb), num.pos)
+        # Parsing floats
+        elif re.fullmatch(r"[+-]?\d[\d_]*(\.\d[\d_]*)?(e[+-]?\d[\d_]*)?", buff):
+            return FloatToken(float(buff), buff, pos)
 
-            elif rtb.peek() == "r":
+        else:
-                ## Case of a number with a base
+            return SymbolToken(buff, buff, pos)
-                # Discard thd delimeter
-                rtb.read()
-                body = cls.parse_symbol(rtb)
-                return IntegerToken(int(body.raw, num.data), str(rtb), num.pos)
-
-            elif rtb.peek() == ".":
-                ## Case of a number with a decimal component
-                ## Note there may be a trailing exponent
-                raise NotImplementedError()
-
-            elif rtb.peek() == "e":
-                ## Case of a number with a floating point exponent
-                raise NotImplementedError()
-
-            else:
-                return num
 
     @classmethod
     def parse_symbol(cls, f: PosTrackingBufferedReader):
         with ReadThroughBuffer(f) as rtb:
             pos = None
-            while rtb.peek() and not cls.isspace(rtb.peek()):
+            while rtb.peek() and not cls.ispunct(rtb.peek()):
                 pos = pos or rtb.pos()
                 rtb.read()
             buff = str(rtb)
-            return SymbolToken(buff, buff, pos)
+            return cls.handle_symbol(buff, pos)
 
     @classmethod
     def parse_whitespace(cls, f: PosTrackingBufferedReader):

src/python/flowmetal/syntax_analyzer.py

@@ -12,7 +12,7 @@ import flowmetal.parser as p
 
 ### Types
 ## We are not, in fact, sponsored by Typelevel LLC.
-class TypeLevelExpr(ABC):
+class TypeLevelExpr(object):
     """A base class for type-level expressions."""
     pass
 
@@ -27,24 +27,29 @@ class TypeExpr(TypeLevelExpr, NamedTuple):
 
 
 ## Now down to reality
-class ValueLevelExpr(ABC):
+class ValueLevelExpr(object):
     """A base class for value-level expressions."""
+    pass
 
 
 class AscribeExpr(ValueLevelExpr, NamedTuple):
     """Ascribe a type (via type-level expression) to a value-level expression."""
+    pass
 
 
 class InvokeExpr(ValueLevelExpr, NamedTuple):
     """(a ⊢ (fn A ⊢ B) [...] ⊢ A) ⊢ B"""
+    pass
 
 
 class IfExpr(ValueLevelExpr, NamedTuple):
     """(if test a ⊢ A b ⊢ B) ⊢ (Variant A B)."""
+    pass
 
 
 class LetExpr(ValueLevelExpr, NamedTuple):
     """Let a single binding and wrap a body. Yes one. N-ary let is an abstraction."""
+    pass
 
 
 class DoExpr(ValueError, NamedTuple):
@@ -52,24 +57,31 @@ class DoExpr(ValueError, NamedTuple):
 
     (do a b c ... ω ⊢ Ω) ⊢ Ω
     """
+    pass
+
 
 ProcExpr = DoExpr  # ain't broke don't fix it
 
+
 class MappingExpr(ValueLevelExpr, NamedTuple):
     """Mappings require their own constructor expression due to local/symbol references."""
+    pass
 
 
 class SetExpr(ValueLevelExpr, NamedTuple):
     """Sets require their own constructor expression due to local/symbol references."""
+    pass
 
 
 class ListExpr(ValueLevelExpr, NamedTuple):
     """While round () lists are generally InvokeExprs, [] lists are constructors like sets and maps."""
+    pass
 
 
 ## Reader implementation
 class SexpAnalyzer(ABC):
     """A base class for Analyzers."""
+    pass
 
 
 class Analyzer(SexpAnalyzer):

test/python/flowmetal/test_parser.py

@@ -2,6 +2,8 @@
 Tests covering the Flowmetal parser.
 """
 
+from math import nan
+
 import flowmetal.parser as p
 
 import pytest
@@ -27,7 +29,7 @@ def test_parse_num(num):
 ])
 def test_parse_ratio(frac):
     """Test covering the ratio notation."""
-    assert isinstance(p.parses(frac), p.RationalToken)
+    assert isinstance(p.parses(frac), p.FractionToken)
     assert p.parses(frac).data == p.Fraction(frac)
 
 
@@ -64,3 +66,34 @@ def test_list_contents(txt, tokenization):
     for (type, text), token in zip(tokenization, lelems):
         assert isinstance(token, type)
         assert token.raw == text
+
+
+@pytest.mark.parametrize('txt, value', [
+    ('1.0', 1.0),
+    ('-1.0', -1.0),
+    ('1.01', 1.01),
+    ('1e0', 1e0),
+    ('1e3', 1e3),
+    ('1e-3', 1e-3),
+    ('1.01e3', 1.01e3),
+])
+def test_float_values(txt, value):
+    """Some examples of floats."""
+    assert isinstance(p.parses(txt), p.FloatToken)
+    assert p.parses(txt).data == value
+
+
+@pytest.mark.parametrize('txt, tokenization', [
+    ('+1', p.IntegerToken),
+    ('+1+', p.SymbolToken),
+    ('+1e', p.SymbolToken),
+    ('+1e3', p.FloatToken),
+    ('+1.0', p.FloatToken),
+    ('+1.0e3', p.FloatToken),
+    ('a.b', p.SymbolToken),
+    ('1.b', p.SymbolToken),
+])
+def test_ambiguous_floats(txt, tokenization):
+    """Parse examples of 'difficult' floats and symbols."""
+    assert isinstance(p.parses(txt), tokenization), "Token type didn't match!"
+    assert p.parses(txt).raw == txt, "Parse wasn't total!"