Get floating point numbers working too
This commit is contained in:
parent
d60e690445
commit
d7f175a987
4 changed files with 174 additions and 49 deletions
83
src/flowmetal/com.twitter.wilson.flow
Normal file
83
src/flowmetal/com.twitter.wilson.flow
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
(defpackage com.twitter.wilson
|
||||
(require
|
||||
;; The log lets you record status information into a program's trace
|
||||
[flowmetal.log
|
||||
:refer [log!]]
|
||||
;; The time system lets you put bounds on the implicit awaiting Flowmetal does
|
||||
[flowmetal.time
|
||||
:refer [with-timeout!, timeout?, make-duration, duration?, +seconds+, +hours+, sleep!]]
|
||||
;; JSON. Simple enough
|
||||
[flowmetal.json
|
||||
:refer [loads, dumps, json?]]
|
||||
;; Extensions! Provided by other systems.
|
||||
;;
|
||||
;; This one allows for an external service to receive HTTP callbacks on Flowmetal's behalf.
|
||||
[http.callback
|
||||
:refer [make-callback!, get-callback!, callback?]]
|
||||
;; This one allows for an external service to make HTTP requests on Flowmetal's behalf.
|
||||
[http.request
|
||||
:refer [post!, error?, dns-error?, connection-error?, response-error?]])
|
||||
|
||||
(defenum stage
|
||||
+reboot+
|
||||
+bios-update+
|
||||
+reinstall+)
|
||||
|
||||
;; FIXME:- how to do table optimization?
|
||||
(defn fib [x]
|
||||
(match x
|
||||
[0 1]
|
||||
[1 1]
|
||||
[_ (+ (fib (- x 1) (- x 2)))]))
|
||||
|
||||
(defn retry-http [f
|
||||
:- (fn? [] a?)
|
||||
backoff-fn
|
||||
:- (fn? [int?] duration?)
|
||||
:default (fn [x :- int?]
|
||||
:- duration?
|
||||
(make-duration (fib x) +seconds+))
|
||||
backoff-count
|
||||
:- int?
|
||||
:default 0]
|
||||
:- a
|
||||
"The implementation of HTTP with retrying."
|
||||
(let [response (f)]
|
||||
(if (not (error? response))
|
||||
response
|
||||
;; FIXME:- how does auth denied get represented?
|
||||
(if (or (dns-error? response)
|
||||
(connection-error? response)
|
||||
(response-error? response))
|
||||
(do (sleep (backoff-fn backoff-count))
|
||||
(retry-http* f backoff-fn (+ backoff-count 1)))))))
|
||||
|
||||
(defn job [hostname
|
||||
:- str?
|
||||
stages
|
||||
:- (list? stage?)
|
||||
job-timeout
|
||||
:- duration?
|
||||
:default (duration 3 :hours)]
|
||||
:- (union? [timeout? json?])
|
||||
"Run a wilson job, wait for the callback and process the result.
|
||||
|
||||
By default the job is only waited for three hours.
|
||||
|
||||
"
|
||||
(let [callback :- callback? (make-callback!)
|
||||
job (retry-http
|
||||
(fn []
|
||||
(post "http://wilson.local.twitter.com"
|
||||
:data
|
||||
(dumps
|
||||
{:host hostname
|
||||
:stages [stages]
|
||||
:callbacks [{:type :http, :url callback}]}))))]
|
||||
|
||||
(let [result (with-timeout! (duration 3 :hours)
|
||||
(fn []
|
||||
(get-callback callback)))]
|
||||
(if-not (timeout? result)
|
||||
(loads result)
|
||||
result)))))
|
||||
|
|
@ -7,6 +7,7 @@ from enum import Enum
|
|||
from io import StringIO, BufferedReader
|
||||
from typing import IO, NamedTuple
|
||||
from fractions import Fraction
|
||||
import re
|
||||
|
||||
|
||||
## Types
|
||||
|
|
@ -54,9 +55,9 @@ class IntegerToken(NamedTuple, TokenBase):
|
|||
return
|
||||
|
||||
|
||||
class RationalToken(NamedTuple, TokenBase):
|
||||
"""A read integer, including position."""
|
||||
data: int
|
||||
class FractionToken(NamedTuple, TokenBase):
|
||||
"""A read fraction, including position."""
|
||||
data: Fraction
|
||||
raw: str
|
||||
pos: Position
|
||||
|
||||
|
|
@ -66,7 +67,7 @@ class RationalToken(NamedTuple, TokenBase):
|
|||
|
||||
class FloatToken(NamedTuple, TokenBase):
|
||||
"""A read floating point number, including position."""
|
||||
data: int
|
||||
data: float
|
||||
raw: str
|
||||
pos: Position
|
||||
|
||||
|
|
@ -225,8 +226,6 @@ class Parser(SexpParser):
|
|||
return cls.parse_sqlist(f)
|
||||
elif cls.isspace(f.peek()):
|
||||
return cls.parse_whitespace(f)
|
||||
elif f.peek().isdigit():
|
||||
return cls.parse_num(f)
|
||||
elif f.peek() == ";":
|
||||
return cls.parse_comment(f)
|
||||
else:
|
||||
|
|
@ -235,7 +234,21 @@ class Parser(SexpParser):
|
|||
@classmethod
|
||||
def isspace(cls, ch: str):
|
||||
"""An extension point allowing for a more expansive concept of whitespace."""
|
||||
return ch.isspace()
|
||||
return ch.isspace() or ch == ','
|
||||
|
||||
@classmethod
|
||||
def ispunct(cls, ch: str):
|
||||
return cls.isspace(ch) or ch in (
|
||||
';' # Semicolon
|
||||
'()' # Parens
|
||||
'⟮⟯' # 'flat' parens
|
||||
'[]' # Square brackets
|
||||
'⟦⟧' # 'white' square brackets
|
||||
'{}' # Curly brackets
|
||||
'⟨⟩' # Angle brackets
|
||||
'《》' # Double angle brackets
|
||||
'⟪⟫' # Another kind of double angle brackets
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def parse_delimeted(cls, f: PosTrackingBufferedReader, openc, closec, ctor):
|
||||
|
|
@ -260,57 +273,41 @@ class Parser(SexpParser):
|
|||
return cls.parse_delimeted(f, "[", "]", lambda *args: ListToken(*args, ListType.SQUARE))
|
||||
|
||||
@classmethod
|
||||
def parse_unum(cls, f: PosTrackingBufferedReader):
|
||||
with ReadThroughBuffer(f) as rtb:
|
||||
assert rtb.peek().isdigit()
|
||||
pos = f.pos()
|
||||
while rtb.peek().isdigit():
|
||||
rtb.read()
|
||||
buff = str(rtb)
|
||||
return IntegerToken(int(buff), buff, pos)
|
||||
def handle_symbol(cls, buff, pos):
|
||||
# Parsing integers
|
||||
if m := re.fullmatch(r"[+-]?\d[\d_]*", buff):
|
||||
return IntegerToken(int(buff.replace("_", "")), buff, pos)
|
||||
|
||||
@classmethod
|
||||
def parse_num(cls, f: PosTrackingBufferedReader):
|
||||
with ReadThroughBuffer(f) as rtb:
|
||||
num: IntegerToken = cls.parse_unum(rtb)
|
||||
# Parsing integers with bases
|
||||
elif m := re.fullmatch(r"(\d+)r([a-z0-9_]+)", buff):
|
||||
return IntegerToken(int(m.group(2).replace("_", "")), int(m.group(1)), buff, pos)
|
||||
|
||||
# Various cases of more interesting numbers
|
||||
if rtb.peek() == "/":
|
||||
## Case of a rational
|
||||
# Discard the delimeter
|
||||
rtb.read()
|
||||
denom = cls.parse_num(rtb)
|
||||
# Parsing fractions
|
||||
elif m := re.fullmatch(r"([+-]?\d[\d_]*)/(\d[\d_]*)", buff):
|
||||
return FractionToken(
|
||||
Fraction(
|
||||
int(m.group(1).replace("_", "")),
|
||||
int(m.group(2).replace("_", ""))),
|
||||
buff,
|
||||
pos,
|
||||
)
|
||||
|
||||
return RationalToken(Fraction(num.data, denom.data), str(rtb), num.pos)
|
||||
|
||||
elif rtb.peek() == "r":
|
||||
## Case of a number with a base
|
||||
# Discard thd delimeter
|
||||
rtb.read()
|
||||
body = cls.parse_symbol(rtb)
|
||||
return IntegerToken(int(body.raw, num.data), str(rtb), num.pos)
|
||||
|
||||
elif rtb.peek() == ".":
|
||||
## Case of a number with a decimal component
|
||||
## Note there may be a trailing exponent
|
||||
raise NotImplementedError()
|
||||
|
||||
elif rtb.peek() == "e":
|
||||
## Case of a number with a floating point exponent
|
||||
raise NotImplementedError()
|
||||
# Parsing floats
|
||||
elif re.fullmatch(r"[+-]?\d[\d_]*(\.\d[\d_]*)?(e[+-]?\d[\d_]*)?", buff):
|
||||
return FloatToken(float(buff), buff, pos)
|
||||
|
||||
else:
|
||||
return num
|
||||
return SymbolToken(buff, buff, pos)
|
||||
|
||||
@classmethod
|
||||
def parse_symbol(cls, f: PosTrackingBufferedReader):
|
||||
with ReadThroughBuffer(f) as rtb:
|
||||
pos = None
|
||||
while rtb.peek() and not cls.isspace(rtb.peek()):
|
||||
while rtb.peek() and not cls.ispunct(rtb.peek()):
|
||||
pos = pos or rtb.pos()
|
||||
rtb.read()
|
||||
buff = str(rtb)
|
||||
return SymbolToken(buff, buff, pos)
|
||||
return cls.handle_symbol(buff, pos)
|
||||
|
||||
@classmethod
|
||||
def parse_whitespace(cls, f: PosTrackingBufferedReader):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import flowmetal.parser as p
|
|||
|
||||
### Types
|
||||
## We are not, in fact, sponsored by Typelevel LLC.
|
||||
class TypeLevelExpr(ABC):
|
||||
class TypeLevelExpr(object):
|
||||
"""A base class for type-level expressions."""
|
||||
pass
|
||||
|
||||
|
|
@ -27,24 +27,29 @@ class TypeExpr(TypeLevelExpr, NamedTuple):
|
|||
|
||||
|
||||
## Now down to reality
|
||||
class ValueLevelExpr(ABC):
|
||||
class ValueLevelExpr(object):
|
||||
"""A base class for value-level expressions."""
|
||||
pass
|
||||
|
||||
|
||||
class AscribeExpr(ValueLevelExpr, NamedTuple):
|
||||
"""Ascribe a type (via type-level expression) to a value-level expression."""
|
||||
pass
|
||||
|
||||
|
||||
class InvokeExpr(ValueLevelExpr, NamedTuple):
|
||||
"""(a ⊢ (fn A ⊢ B) [...] ⊢ A) ⊢ B"""
|
||||
pass
|
||||
|
||||
|
||||
class IfExpr(ValueLevelExpr, NamedTuple):
|
||||
"""(if test a ⊢ A b ⊢ B) ⊢ (Variant A B)."""
|
||||
pass
|
||||
|
||||
|
||||
class LetExpr(ValueLevelExpr, NamedTuple):
|
||||
"""Let a single binding and wrap a body. Yes one. N-ary let is an abstraction."""
|
||||
pass
|
||||
|
||||
|
||||
class DoExpr(ValueError, NamedTuple):
|
||||
|
|
@ -52,24 +57,31 @@ class DoExpr(ValueError, NamedTuple):
|
|||
|
||||
(do a b c ... ω ⊢ Ω) ⊢ Ω
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
ProcExpr = DoExpr # ain't broke don't fix it
|
||||
|
||||
|
||||
class MappingExpr(ValueLevelExpr, NamedTuple):
|
||||
"""Mappings require their own constructor expression due to local/symbol references."""
|
||||
pass
|
||||
|
||||
|
||||
class SetExpr(ValueLevelExpr, NamedTuple):
|
||||
"""Sets require their own constructor expression due to local/symbol references."""
|
||||
pass
|
||||
|
||||
|
||||
class ListExpr(ValueLevelExpr, NamedTuple):
|
||||
"""While round () lists are generally InvokeExprs, [] lists are constructors like sets and maps."""
|
||||
pass
|
||||
|
||||
|
||||
## Reader implementation
|
||||
class SexpAnalyzer(ABC):
|
||||
"""A base class for Analyzers."""
|
||||
pass
|
||||
|
||||
|
||||
class Analyzer(SexpAnalyzer):
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
Tests covering the Flowmetal parser.
|
||||
"""
|
||||
|
||||
from math import nan
|
||||
|
||||
import flowmetal.parser as p
|
||||
|
||||
import pytest
|
||||
|
|
@ -27,7 +29,7 @@ def test_parse_num(num):
|
|||
])
|
||||
def test_parse_ratio(frac):
|
||||
"""Test covering the ratio notation."""
|
||||
assert isinstance(p.parses(frac), p.RationalToken)
|
||||
assert isinstance(p.parses(frac), p.FractionToken)
|
||||
assert p.parses(frac).data == p.Fraction(frac)
|
||||
|
||||
|
||||
|
|
@ -64,3 +66,34 @@ def test_list_contents(txt, tokenization):
|
|||
for (type, text), token in zip(tokenization, lelems):
|
||||
assert isinstance(token, type)
|
||||
assert token.raw == text
|
||||
|
||||
|
||||
@pytest.mark.parametrize('txt, value', [
|
||||
('1.0', 1.0),
|
||||
('-1.0', -1.0),
|
||||
('1.01', 1.01),
|
||||
('1e0', 1e0),
|
||||
('1e3', 1e3),
|
||||
('1e-3', 1e-3),
|
||||
('1.01e3', 1.01e3),
|
||||
])
|
||||
def test_float_values(txt, value):
|
||||
"""Some examples of floats."""
|
||||
assert isinstance(p.parses(txt), p.FloatToken)
|
||||
assert p.parses(txt).data == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize('txt, tokenization', [
|
||||
('+1', p.IntegerToken),
|
||||
('+1+', p.SymbolToken),
|
||||
('+1e', p.SymbolToken),
|
||||
('+1e3', p.FloatToken),
|
||||
('+1.0', p.FloatToken),
|
||||
('+1.0e3', p.FloatToken),
|
||||
('a.b', p.SymbolToken),
|
||||
('1.b', p.SymbolToken),
|
||||
])
|
||||
def test_ambiguous_floats(txt, tokenization):
|
||||
"""Parse examples of 'difficult' floats and symbols."""
|
||||
assert isinstance(p.parses(txt), tokenization), "Token type didn't match!"
|
||||
assert p.parses(txt).raw == txt, "Parse wasn't total!"
|
||||
|
|
|
|||
Loading…
Reference in a new issue