Playing with the Python ASTs
This commit is contained in:
parent
03b37675b5
commit
df3dac01c9
4 changed files with 1102 additions and 0 deletions
14
projects/flowmetal/scratch/BUILD
Normal file
14
projects/flowmetal/scratch/BUILD
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
py_binary(
|
||||
name = "astdump",
|
||||
main = "astdump.py",
|
||||
deps = [
|
||||
py_requirement("pyyaml"),
|
||||
]
|
||||
)
|
||||
|
||||
py_binary(
|
||||
name = "astinterp",
|
||||
main = "astinterp.py",
|
||||
deps = [
|
||||
]
|
||||
)
|
||||
96
projects/flowmetal/scratch/astdump.py
Normal file
96
projects/flowmetal/scratch/astdump.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
A (toy) tool for emitting Python ASTs as YAML formatted data.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import optparse
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def propnames(node):
|
||||
"""return names of attributes specific for the current node"""
|
||||
|
||||
props = {x for x in dir(node) if not x.startswith("_")}
|
||||
|
||||
if isinstance(node, ast.Module):
|
||||
props -= {"body"}
|
||||
|
||||
if isinstance(node, (ast.Expr, ast.Attribute)):
|
||||
props -= {"value"}
|
||||
|
||||
if isinstance(node, ast.Constant):
|
||||
props -= {"n", "s"}
|
||||
|
||||
if isinstance(node, ast.ClassDef):
|
||||
props -= {"body"}
|
||||
|
||||
return props
|
||||
|
||||
|
||||
# Note that ast.NodeTransformer exists for mutations.
|
||||
# This is just for reads.
|
||||
class TreeDumper(ast.NodeVisitor):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._stack = []
|
||||
|
||||
def dump(self, node):
|
||||
self.visit(node)
|
||||
|
||||
def visit(self, node):
|
||||
nodetype = type(node)
|
||||
nodename = node.__class__.__name__
|
||||
indent = " " * len(self._stack) * 2
|
||||
print(indent + nodename)
|
||||
for n in propnames(node):
|
||||
print(indent + "%s: %s" % (n, node.__dict__[n]))
|
||||
|
||||
self._stack.append(node)
|
||||
self.generic_visit(node)
|
||||
self._stack.pop()
|
||||
|
||||
|
||||
class YAMLTreeDumper(ast.NodeVisitor):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._stack = []
|
||||
|
||||
def node2yml(self, node):
|
||||
try:
|
||||
nodetype = type(node)
|
||||
nodename = node.__class__.__name__
|
||||
return {
|
||||
"op": nodename,
|
||||
"props": {n: node.__dict__[n] for n in propnames(node)},
|
||||
"children": [],
|
||||
}
|
||||
except:
|
||||
print(repr(node), propnames(node), dir(node))
|
||||
|
||||
def visit(self, node):
|
||||
yml_node = self.node2yml(node)
|
||||
self._stack.append(yml_node)
|
||||
old_stack = self._stack
|
||||
self._stack = yml_node["children"]
|
||||
self.generic_visit(node)
|
||||
self._stack = old_stack
|
||||
return yml_node
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = optparse.OptionParser(usage="%prog [options] <filename.py>")
|
||||
opts, args = parser.parse_args()
|
||||
|
||||
if len(args) == 0:
|
||||
parser.print_help()
|
||||
sys.exit(-1)
|
||||
filename = args[0]
|
||||
|
||||
with open(filename) as f:
|
||||
root = ast.parse(f.read(), filename)
|
||||
|
||||
print(yaml.dump(YAMLTreeDumper().visit(root), default_flow_style=False, sort_keys=False))
|
||||
967
projects/flowmetal/scratch/astinterp.py
Normal file
967
projects/flowmetal/scratch/astinterp.py
Normal file
|
|
@ -0,0 +1,967 @@
|
|||
# Python AST interpreter written in Python
|
||||
#
|
||||
# This module is part of the Pycopy https://github.com/pfalcon/pycopy
|
||||
# project.
|
||||
#
|
||||
# Copyright (c) 2019 Paul Sokolovsky
|
||||
#
|
||||
# The MIT License
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
#
|
||||
# Modified by Reid D. 'ardem' Mckenzie in 2021 to be a bit more fully-featured
|
||||
# and usable for running 'real' code as part of an experiment in implementing a
|
||||
# durable Python interpreter atop the original pycopy substrate.
|
||||
|
||||
import ast
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
|
||||
if sys.version_info < (3, 0, 0):
|
||||
builtins = __builtins__
|
||||
else:
|
||||
import builtins
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StrictNodeVisitor(ast.NodeVisitor):
|
||||
def generic_visit(self, node):
|
||||
n = node.__class__.__name__
|
||||
raise NotImplementedError("Visitor for node {} not implemented".format(n))
|
||||
|
||||
|
||||
class ANamespace:
|
||||
def __init__(self, node):
|
||||
self.d = {}
|
||||
self.parent = None
|
||||
# Cross-link namespace to AST node. Note that we can't do the
|
||||
# opposite, because for one node, there can be different namespaces.
|
||||
self.node = node
|
||||
|
||||
def __getitem__(self, k):
|
||||
return self.d[k]
|
||||
|
||||
def get(self, k, default=None):
|
||||
return self.d.get(k, default)
|
||||
|
||||
def __setitem__(self, k, v):
|
||||
self.d[k] = v
|
||||
|
||||
def __delitem__(self, k):
|
||||
del self.d[k]
|
||||
|
||||
def __contains__(self, k):
|
||||
return k in self.d
|
||||
|
||||
def __str__(self):
|
||||
return "<{} {}>".format(self.__class__.__name__, self.d)
|
||||
|
||||
|
||||
class ModuleNS(ANamespace):
|
||||
pass
|
||||
|
||||
|
||||
class FunctionNS(ANamespace):
|
||||
pass
|
||||
|
||||
|
||||
class ClassNS(ANamespace):
|
||||
pass
|
||||
|
||||
|
||||
# Pycopy by default doesn't support direct slice construction, use helper
|
||||
# object to construct it.
|
||||
class SliceGetter:
|
||||
def __getitem__(self, idx):
|
||||
return idx
|
||||
|
||||
|
||||
slice_getter = SliceGetter()
|
||||
|
||||
|
||||
def arg_name(arg):
|
||||
if sys.version_info < (3, 0, 0):
|
||||
return arg.id
|
||||
else:
|
||||
return arg.arg
|
||||
|
||||
def kwarg_defaults(args):
|
||||
if sys.version_info < (3, 0, 0):
|
||||
return args.defaults
|
||||
else:
|
||||
return args.kw_defaults
|
||||
|
||||
|
||||
class TargetNonlocalFlow(Exception):
|
||||
"""Base exception class to simulate non-local control flow transfers in
|
||||
a target application."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TargetBreak(TargetNonlocalFlow):
|
||||
pass
|
||||
|
||||
|
||||
class TargetContinue(TargetNonlocalFlow):
|
||||
pass
|
||||
|
||||
|
||||
class TargetReturn(TargetNonlocalFlow):
|
||||
pass
|
||||
|
||||
|
||||
class VarScopeSentinel:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
|
||||
NO_VAR = VarScopeSentinel("no_var")
|
||||
GLOBAL = VarScopeSentinel("global")
|
||||
NONLOCAL = VarScopeSentinel("nonlocal")
|
||||
|
||||
|
||||
class InterpFuncWrap:
|
||||
"Callable wrapper for AST functions (FunctionDef nodes)."
|
||||
|
||||
def __init__(self, node, interp):
|
||||
self.node = node
|
||||
self.interp = interp
|
||||
self.lexical_scope = interp.ns
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self.interp.call_func(self.node, self, *args, **kwargs)
|
||||
|
||||
|
||||
# Python don't fully treat objects, even those defining __call__() special
|
||||
# method, as a true callable. For example, such objects aren't automatically
|
||||
# converted to bound methods if looked up as another object's attributes.
|
||||
# As we want our "interpreted functions" to behave as close as possible to
|
||||
# real functions, we just wrap function object with a real function. An
|
||||
# alternative might have been to perform needed checks and explicitly
|
||||
# bind a method using types.MethodType() in visit_Attribute (but then maybe
|
||||
# there would be still other cases of "callable object" vs "function"
|
||||
# discrepancies).
|
||||
def InterpFunc(fun):
|
||||
def func(*args, **kwargs):
|
||||
return fun.__call__(*args, **kwargs)
|
||||
|
||||
return func
|
||||
|
||||
|
||||
class InterpWith:
|
||||
def __init__(self, ctx):
|
||||
self.ctx = ctx
|
||||
|
||||
def __enter__(self):
|
||||
return self.ctx.__enter__()
|
||||
|
||||
def __exit__(self, tp, exc, tb):
|
||||
# Don't leak meta-level exceptions into target
|
||||
if isinstance(exc, TargetNonlocalFlow):
|
||||
tp = exc = tb = None
|
||||
return self.ctx.__exit__(tp, exc, tb)
|
||||
|
||||
|
||||
class InterpModule:
|
||||
def __init__(self, ns):
|
||||
self.ns = ns
|
||||
|
||||
def __getattr__(self, name):
|
||||
try:
|
||||
return self.ns[name]
|
||||
except KeyError:
|
||||
raise AttributeError
|
||||
|
||||
def __dir__(self):
|
||||
return list(self.ns.d.keys())
|
||||
|
||||
|
||||
class ModuleInterpreter(StrictNodeVisitor):
|
||||
"""An interpreter specific to a single module."""
|
||||
|
||||
def __init__(self, system, fname, node):
|
||||
self.system = system
|
||||
self.fname = fname
|
||||
self.ns = self.module_ns = ModuleNS(node)
|
||||
# Call stack (in terms of function AST nodes).
|
||||
self.call_stack = []
|
||||
# To implement "store" operation, we need to arguments: location and
|
||||
# value to store. The operation itself is handled by a node visitor
|
||||
# (e.g. visit_Name), and location is represented by AST node, but
|
||||
# there's no support to pass additional arguments to a visitor
|
||||
# (likely, because it would be a burden to explicit pass such
|
||||
# additional arguments thru the chain of visitors). So instead, we
|
||||
# store this value as field. As interpretation happens sequentially,
|
||||
# there's no risk that it will be overwritten "concurrently".
|
||||
self.store_val = None
|
||||
# Current active exception, for bare "raise", which doesn't work
|
||||
# across function boundaries (and that's how we have it - exception
|
||||
# would be caught in visit_Try, while re-rasing would happen in
|
||||
# visit_Raise).
|
||||
self.cur_exc = []
|
||||
|
||||
def push_ns(self, new_ns):
|
||||
new_ns.parent = self.ns
|
||||
self.ns = new_ns
|
||||
|
||||
def pop_ns(self):
|
||||
self.ns = self.ns.parent
|
||||
|
||||
def stmt_list_visit(self, lst):
|
||||
res = None
|
||||
for s in lst:
|
||||
res = self.visit(s)
|
||||
return res
|
||||
|
||||
def wrap_decorators(self, obj, node):
|
||||
for deco_n in reversed(list(node.decorator_list)):
|
||||
deco = self.visit(deco_n)
|
||||
obj = deco(obj)
|
||||
return obj
|
||||
|
||||
def visit(self, node):
|
||||
val = super(StrictNodeVisitor, self).visit(node)
|
||||
return val
|
||||
|
||||
def visit_Module(self, node):
|
||||
self.stmt_list_visit(node.body)
|
||||
|
||||
def visit_Expression(self, node):
|
||||
return self.visit(node.body)
|
||||
|
||||
def visit_ClassDef(self, node):
|
||||
self.push_ns(ClassNS(node))
|
||||
try:
|
||||
self.stmt_list_visit(node.body)
|
||||
except:
|
||||
self.pop_ns()
|
||||
raise
|
||||
ns = self.ns
|
||||
self.pop_ns()
|
||||
cls = type(node.name, tuple([self.visit(b) for b in node.bases]), ns.d)
|
||||
cls = self.wrap_decorators(cls, node)
|
||||
self.ns[node.name] = cls
|
||||
# Store reference to class object in the namespace object
|
||||
ns.cls = cls
|
||||
|
||||
def visit_Lambda(self, node):
|
||||
node.name = "<lambda>"
|
||||
return self.prepare_func(node)
|
||||
|
||||
def visit_FunctionDef(self, node):
|
||||
# Defaults are evaluated at function definition time, so we
|
||||
# need to do that now.
|
||||
func = self.prepare_func(node)
|
||||
func = self.wrap_decorators(func, node)
|
||||
self.ns[node.name] = func
|
||||
|
||||
def prepare_func(self, node):
|
||||
"""Prepare function AST node for future interpretation: pre-calculate
|
||||
and cache useful information, etc."""
|
||||
|
||||
func = InterpFuncWrap(node, self)
|
||||
args = node.args or node.posonlyargs
|
||||
num_required = len(args.args) - len(args.defaults)
|
||||
all_args = set()
|
||||
d = {}
|
||||
for i, a in enumerate(args.args):
|
||||
all_args.add(arg_name(a))
|
||||
if i >= num_required:
|
||||
d[arg_name(a)] = self.visit(args.defaults[i - num_required])
|
||||
|
||||
for a, v in zip(getattr(args, "kwonlyargs", ()), kwarg_defaults(args)):
|
||||
all_args.add(arg_name(a))
|
||||
if v is not None:
|
||||
d[arg_name(a)] = self.visit(v)
|
||||
# We can store cached argument names of a function in its node -
|
||||
# it's static.
|
||||
node.args.all_args = all_args
|
||||
# We can't store the values of default arguments - they're dynamic,
|
||||
# may depend on the lexical scope.
|
||||
func.defaults_dict = d
|
||||
|
||||
return InterpFunc(func)
|
||||
|
||||
def prepare_func_args(self, node, interp_func, *args, **kwargs):
|
||||
def arg_num_mismatch():
|
||||
raise TypeError(
|
||||
"{}() takes {} positional arguments but {} were given".format(
|
||||
node.name, len(argspec.args), len(args)
|
||||
)
|
||||
)
|
||||
|
||||
argspec = node.args
|
||||
# If there's vararg, either offload surplus of args to it, or init
|
||||
# it to empty tuple (all in one statement). If no vararg, error on
|
||||
# too many args.
|
||||
#
|
||||
# Note that we have to do the .posonlyargs dance
|
||||
if argspec.vararg:
|
||||
self.ns[argspec.vararg.arg] = args[len(argspec.args):]
|
||||
else:
|
||||
if len(args) > len(argspec.args or getattr(argspec, "posonlyargs", ())):
|
||||
arg_num_mismatch()
|
||||
|
||||
if argspec.args:
|
||||
for i in range(min(len(args), len(argspec.args))):
|
||||
self.ns[arg_name(argspec.args[i])] = args[i]
|
||||
elif getattr(argspec, "posonlyargs", ()):
|
||||
if len(args) != len(argspec.posonlyargs):
|
||||
arg_num_mismatch()
|
||||
|
||||
for a, value in zip(argspec.posonlyargs, args):
|
||||
self.ns[arg_name(a)] = value
|
||||
|
||||
# Process incoming keyword arguments, putting them in namespace if
|
||||
# actual arg exists by that name, or offload to function's kwarg
|
||||
# if any. All make needed checks and error out.
|
||||
func_kwarg = {}
|
||||
for k, v in kwargs.items():
|
||||
if k in argspec.all_args:
|
||||
if k in self.ns:
|
||||
raise TypeError(
|
||||
"{}() got multiple values for argument '{}'".format(
|
||||
node.name, k
|
||||
)
|
||||
)
|
||||
self.ns[k] = v
|
||||
elif argspec.kwarg:
|
||||
func_kwarg[k] = v
|
||||
else:
|
||||
raise TypeError(
|
||||
"{}() got an unexpected keyword argument '{}'".format(node.name, k)
|
||||
)
|
||||
if argspec.kwarg:
|
||||
self.ns[arg_name(argspec.kwarg)] = func_kwarg
|
||||
|
||||
# Finally, overlay default values for arguments not yet initialized.
|
||||
# We need to do this last for "multiple values for the same arg"
|
||||
# check to work.
|
||||
for k, v in interp_func.defaults_dict.items():
|
||||
if k not in self.ns:
|
||||
self.ns[k] = v
|
||||
|
||||
# And now go thru and check for any missing arguments.
|
||||
for a in argspec.args:
|
||||
if arg_name(a) not in self.ns:
|
||||
raise TypeError(
|
||||
"{}() missing required positional argument: '{}'".format(
|
||||
node.name, arg_name(a)
|
||||
)
|
||||
)
|
||||
for a in getattr(argspec, "kwonlyargs", ()):
|
||||
if a.arg not in self.ns:
|
||||
raise TypeError(
|
||||
"{}() missing required keyword-only argument: '{}'".format(
|
||||
node.name, arg_name(a)
|
||||
)
|
||||
)
|
||||
|
||||
def call_func(self, node, interp_func, *args, **kwargs):
|
||||
self.call_stack.append(node)
|
||||
# We need to switch from dynamic execution scope to lexical scope
|
||||
# in which function was defined (then switch back on return).
|
||||
dyna_scope = self.ns
|
||||
self.ns = interp_func.lexical_scope
|
||||
self.push_ns(FunctionNS(node))
|
||||
try:
|
||||
self.prepare_func_args(node, interp_func, *args, **kwargs)
|
||||
if isinstance(node.body, list):
|
||||
res = self.stmt_list_visit(node.body)
|
||||
else:
|
||||
res = self.visit(node.body)
|
||||
except TargetReturn as e:
|
||||
res = e.args[0]
|
||||
finally:
|
||||
self.pop_ns()
|
||||
self.ns = dyna_scope
|
||||
self.call_stack.pop()
|
||||
return res
|
||||
|
||||
def visit_Return(self, node):
|
||||
if not isinstance(self.ns, FunctionNS):
|
||||
raise SyntaxError("'return' outside function")
|
||||
raise TargetReturn(node.value and self.visit(node.value))
|
||||
|
||||
def visit_With(self, node):
|
||||
assert len(node.items) == 1
|
||||
ctx = self.visit(node.items[0].context_expr)
|
||||
with InterpWith(ctx) as val:
|
||||
if node.items[0].optional_vars is not None:
|
||||
self.handle_assign(node.items[0].optional_vars, val)
|
||||
self.stmt_list_visit(node.body)
|
||||
|
||||
def visit_Try(self, node):
|
||||
try:
|
||||
self.stmt_list_visit(node.body)
|
||||
except TargetNonlocalFlow:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.cur_exc.append(e)
|
||||
try:
|
||||
for h in getattr(node, "handlers", ()):
|
||||
if h.type is None or isinstance(e, self.visit(h.type)):
|
||||
if h.name:
|
||||
self.ns[h.name] = e
|
||||
self.stmt_list_visit(h.body)
|
||||
if h.name:
|
||||
del self.ns[h.name]
|
||||
break
|
||||
else:
|
||||
raise
|
||||
finally:
|
||||
self.cur_exc.pop()
|
||||
else:
|
||||
self.stmt_list_visit(node.orelse)
|
||||
finally:
|
||||
if getattr(node, "finalbody", None):
|
||||
self.stmt_list_visit(node.finalbody)
|
||||
|
||||
def visit_TryExcept(self, node):
|
||||
# Py2k only; py3k merged all this into one node type.
|
||||
return self.visit_Try(node)
|
||||
|
||||
def visit_TryFinally(self, node):
|
||||
# Py2k only; py3k merged all this into one node type.
|
||||
return self.visit_Try(node)
|
||||
|
||||
def visit_For(self, node):
|
||||
iter = self.visit(node.iter)
|
||||
for item in iter:
|
||||
self.handle_assign(node.target, item)
|
||||
try:
|
||||
self.stmt_list_visit(node.body)
|
||||
except TargetBreak:
|
||||
break
|
||||
except TargetContinue:
|
||||
continue
|
||||
else:
|
||||
self.stmt_list_visit(node.orelse)
|
||||
|
||||
def visit_While(self, node):
|
||||
while self.visit(node.test):
|
||||
try:
|
||||
self.stmt_list_visit(node.body)
|
||||
except TargetBreak:
|
||||
break
|
||||
except TargetContinue:
|
||||
continue
|
||||
else:
|
||||
self.stmt_list_visit(node.orelse)
|
||||
|
||||
def visit_Break(self, node):
|
||||
raise TargetBreak
|
||||
|
||||
def visit_Continue(self, node):
|
||||
raise TargetContinue
|
||||
|
||||
def visit_If(self, node):
|
||||
test = self.visit(node.test)
|
||||
if test:
|
||||
self.stmt_list_visit(node.body)
|
||||
else:
|
||||
self.stmt_list_visit(node.orelse)
|
||||
|
||||
def visit_Import(self, node):
|
||||
for n in node.names:
|
||||
self.ns[n.asname or n.name] = self.system.handle_import(n.name)
|
||||
|
||||
def visit_ImportFrom(self, node):
|
||||
mod = self.system.handle_import(
|
||||
node.module, None, None, [n.name for n in node.names], node.level
|
||||
)
|
||||
for n in node.names:
|
||||
if n.name == "*":
|
||||
# This is the special case of the wildcard import. Copy
|
||||
# everything over.
|
||||
for n in getattr(mod, "__all__", dir(mod)):
|
||||
self.ns[n] = getattr(mod, n)
|
||||
else:
|
||||
self.ns[n.asname or n.name] = getattr(mod, n.name)
|
||||
|
||||
def visit_Raise(self, node):
|
||||
if node.exc is None:
|
||||
if not self.cur_exc:
|
||||
raise RuntimeError("No active exception to reraise")
|
||||
raise self.cur_exc[-1]
|
||||
elif node.cause is None:
|
||||
raise self.visit(node.exc)
|
||||
# else:
|
||||
# raise self.visit(node.exc) from self.visit(node.cause)
|
||||
|
||||
def visit_AugAssign(self, node):
|
||||
assert isinstance(node.target.ctx, ast.Store)
|
||||
# Not functional style, oops. Node in AST has store context, but we
|
||||
# need to read its value first. To not construct a copy of the entire
|
||||
# node with load context, we temporarily patch it in-place.
|
||||
save_ctx = node.target.ctx
|
||||
node.target.ctx = ast.Load()
|
||||
var_val = self.visit(node.target)
|
||||
node.target.ctx = save_ctx
|
||||
|
||||
rval = self.visit(node.value)
|
||||
|
||||
# As augmented assignment is statement, not operator, we can't put them
|
||||
# all into map. We could instead directly lookup special inplace methods
|
||||
# (__iadd__ and friends) and use them, with a fallback to normal binary
|
||||
# operations, but from the point of view of this interpreter, presence
|
||||
# of such methods is an implementation detail of the object system, it's
|
||||
# not concerned with it.
|
||||
op = type(node.op)
|
||||
if op is ast.Add:
|
||||
var_val += rval
|
||||
elif op is ast.Sub:
|
||||
var_val -= rval
|
||||
elif op is ast.Mult:
|
||||
var_val *= rval
|
||||
elif op is ast.Div:
|
||||
var_val /= rval
|
||||
elif op is ast.FloorDiv:
|
||||
var_val //= rval
|
||||
elif op is ast.Mod:
|
||||
var_val %= rval
|
||||
elif op is ast.Pow:
|
||||
var_val **= rval
|
||||
elif op is ast.LShift:
|
||||
var_val <<= rval
|
||||
elif op is ast.RShift:
|
||||
var_val >>= rval
|
||||
elif op is ast.BitAnd:
|
||||
var_val &= rval
|
||||
elif op is ast.BitOr:
|
||||
var_val |= rval
|
||||
elif op is ast.BitXor:
|
||||
var_val ^= rval
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
self.store_val = var_val
|
||||
self.visit(node.target)
|
||||
|
||||
def visit_Assign(self, node):
|
||||
val = self.visit(node.value)
|
||||
for n in node.targets:
|
||||
self.handle_assign(n, val)
|
||||
|
||||
def handle_assign(self, target, val):
|
||||
if isinstance(target, ast.Tuple):
|
||||
it = iter(val)
|
||||
try:
|
||||
for elt_idx, t in enumerate(target.elts):
|
||||
if getattr(ast, "Starred", None ) and isinstance(t, ast.Starred):
|
||||
t = t.value
|
||||
all_elts = list(it)
|
||||
break_i = len(all_elts) - (len(target.elts) - elt_idx - 1)
|
||||
self.store_val = all_elts[:break_i]
|
||||
it = iter(all_elts[break_i:])
|
||||
else:
|
||||
self.store_val = next(it)
|
||||
self.visit(t)
|
||||
except StopIteration:
|
||||
raise ValueError(
|
||||
"not enough values to unpack (expected {})".format(len(target.elts))
|
||||
)
|
||||
|
||||
try:
|
||||
next(it)
|
||||
raise ValueError(
|
||||
"too many values to unpack (expected {})".format(len(target.elts))
|
||||
)
|
||||
except StopIteration:
|
||||
# Expected
|
||||
pass
|
||||
else:
|
||||
self.store_val = val
|
||||
self.visit(target)
|
||||
|
||||
def visit_Delete(self, node):
|
||||
for n in node.targets:
|
||||
self.visit(n)
|
||||
|
||||
def visit_Pass(self, node):
|
||||
pass
|
||||
|
||||
def visit_Assert(self, node):
|
||||
if node.msg is None:
|
||||
assert self.visit(node.test)
|
||||
else:
|
||||
assert self.visit(node.test), self.visit(node.msg)
|
||||
|
||||
def visit_Expr(self, node):
|
||||
# Produced value is ignored
|
||||
self.visit(node.value)
|
||||
|
||||
def enumerate_comps(self, iters):
|
||||
"""Enumerate thru all possible values of comprehension clauses,
|
||||
including multiple "for" clauses, each optionally associated
|
||||
with multiple "if" clauses. Current result of the enumeration
|
||||
is stored in the namespace."""
|
||||
|
||||
def eval_ifs(iter):
|
||||
"""Evaluate all "if" clauses."""
|
||||
for cond in iter.ifs:
|
||||
if not self.visit(cond):
|
||||
return False
|
||||
return True
|
||||
|
||||
if not iters:
|
||||
yield
|
||||
return
|
||||
for el in self.visit(iters[0].iter):
|
||||
self.store_val = el
|
||||
self.visit(iters[0].target)
|
||||
for t in self.enumerate_comps(iters[1:]):
|
||||
if eval_ifs(iters[0]):
|
||||
yield
|
||||
|
||||
def visit_ListComp(self, node):
|
||||
self.push_ns(FunctionNS(node))
|
||||
try:
|
||||
return [self.visit(node.elt) for _ in self.enumerate_comps(node.generators)]
|
||||
finally:
|
||||
self.pop_ns()
|
||||
|
||||
def visit_SetComp(self, node):
|
||||
self.push_ns(FunctionNS(node))
|
||||
try:
|
||||
return {self.visit(node.elt) for _ in self.enumerate_comps(node.generators)}
|
||||
finally:
|
||||
self.pop_ns()
|
||||
|
||||
def visit_DictComp(self, node):
|
||||
self.push_ns(FunctionNS(node))
|
||||
try:
|
||||
return {
|
||||
self.visit(node.key): self.visit(node.value)
|
||||
for _ in self.enumerate_comps(node.generators)
|
||||
}
|
||||
finally:
|
||||
self.pop_ns()
|
||||
|
||||
def visit_IfExp(self, node):
|
||||
if self.visit(node.test):
|
||||
return self.visit(node.body)
|
||||
else:
|
||||
return self.visit(node.orelse)
|
||||
|
||||
def visit_Call(self, node):
|
||||
func = self.visit(node.func)
|
||||
|
||||
args = []
|
||||
for a in node.args:
|
||||
if getattr(ast, "Starred", None) and isinstance(a, ast.Starred):
|
||||
args.extend(self.visit(a.value))
|
||||
else:
|
||||
args.append(self.visit(a))
|
||||
|
||||
kwargs = {}
|
||||
for kw in node.keywords:
|
||||
val = self.visit(kw.value)
|
||||
if kw.arg is None:
|
||||
kwargs.update(val)
|
||||
else:
|
||||
kwargs[kw.arg] = val
|
||||
|
||||
if func is builtins.super and not args:
|
||||
if not self.ns.parent or not isinstance(self.ns.parent, ClassNS):
|
||||
raise RuntimeError("super(): no arguments")
|
||||
# As we're creating methods dynamically outside of class, super()
|
||||
# without argument won't work, as that requires __class__ cell.
|
||||
# Creating that would be cumbersome (Pycopy definitely lacks
|
||||
# enough introspection for that), so we substitute 2 implied
|
||||
# args (which argumentless super() would take from cell and
|
||||
# 1st arg to func). In our case, we take them from prepared
|
||||
# bookkeeping info.
|
||||
args = (self.ns.parent.cls, self.ns["self"])
|
||||
|
||||
return func(*args, **kwargs)
|
||||
|
||||
def visit_Compare(self, node):
|
||||
cmpop_map = {
|
||||
ast.Eq: lambda x, y: x == y,
|
||||
ast.NotEq: lambda x, y: x != y,
|
||||
ast.Lt: lambda x, y: x < y,
|
||||
ast.LtE: lambda x, y: x <= y,
|
||||
ast.Gt: lambda x, y: x > y,
|
||||
ast.GtE: lambda x, y: x >= y,
|
||||
ast.Is: lambda x, y: x is y,
|
||||
ast.IsNot: lambda x, y: x is not y,
|
||||
ast.In: lambda x, y: x in y,
|
||||
ast.NotIn: lambda x, y: x not in y,
|
||||
}
|
||||
lv = self.visit(node.left)
|
||||
for op, r in zip(node.ops, node.comparators):
|
||||
rv = self.visit(r)
|
||||
if not cmpop_map[type(op)](lv, rv):
|
||||
return False
|
||||
lv = rv
|
||||
return True
|
||||
|
||||
def visit_BoolOp(self, node):
|
||||
if isinstance(node.op, ast.And):
|
||||
res = True
|
||||
for v in node.values:
|
||||
res = res and self.visit(v)
|
||||
elif isinstance(node.op, ast.Or):
|
||||
res = False
|
||||
for v in node.values:
|
||||
res = res or self.visit(v)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
return res
|
||||
|
||||
def visit_BinOp(self, node):
|
||||
binop_map = {
|
||||
ast.Add: lambda x, y: x + y,
|
||||
ast.Sub: lambda x, y: x - y,
|
||||
ast.Mult: lambda x, y: x * y,
|
||||
ast.Div: lambda x, y: x / y,
|
||||
ast.FloorDiv: lambda x, y: x // y,
|
||||
ast.Mod: lambda x, y: x % y,
|
||||
ast.Pow: lambda x, y: x ** y,
|
||||
ast.LShift: lambda x, y: x << y,
|
||||
ast.RShift: lambda x, y: x >> y,
|
||||
ast.BitAnd: lambda x, y: x & y,
|
||||
ast.BitOr: lambda x, y: x | y,
|
||||
ast.BitXor: lambda x, y: x ^ y,
|
||||
}
|
||||
l = self.visit(node.left)
|
||||
r = self.visit(node.right)
|
||||
return binop_map[type(node.op)](l, r)
|
||||
|
||||
def visit_UnaryOp(self, node):
|
||||
unop_map = {
|
||||
ast.UAdd: lambda x: +x,
|
||||
ast.USub: lambda x: -x,
|
||||
ast.Invert: lambda x: ~x,
|
||||
ast.Not: lambda x: not x,
|
||||
}
|
||||
val = self.visit(node.operand)
|
||||
return unop_map[type(node.op)](val)
|
||||
|
||||
def visit_Subscript(self, node):
|
||||
obj = self.visit(node.value)
|
||||
idx = self.visit(node.slice)
|
||||
if isinstance(node.ctx, ast.Load):
|
||||
return obj[idx]
|
||||
elif isinstance(node.ctx, ast.Store):
|
||||
obj[idx] = self.store_val
|
||||
elif isinstance(node.ctx, ast.Del):
|
||||
del obj[idx]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
def visit_Index(self, node):
|
||||
return self.visit(node.value)
|
||||
|
||||
def visit_Slice(self, node):
|
||||
# Any of these can be None
|
||||
lower = node.lower and self.visit(node.lower)
|
||||
upper = node.upper and self.visit(node.upper)
|
||||
step = node.step and self.visit(node.step)
|
||||
slice = slice_getter[lower:upper:step]
|
||||
return slice
|
||||
|
||||
def visit_Attribute(self, node):
|
||||
obj = self.visit(node.value)
|
||||
if isinstance(node.ctx, ast.Load):
|
||||
return getattr(obj, node.attr)
|
||||
elif isinstance(node.ctx, ast.Store):
|
||||
setattr(obj, node.attr, self.store_val)
|
||||
elif isinstance(node.ctx, ast.Del):
|
||||
delattr(obj, node.attr)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
def visit_Global(self, node):
|
||||
for n in node.names:
|
||||
if n in self.ns and self.ns[n] is not GLOBAL:
|
||||
raise SyntaxError(
|
||||
"SyntaxError: name '{}' is assigned to before global declaration"
|
||||
.format(n)
|
||||
)
|
||||
# Don't store GLOBAL in the top-level namespace
|
||||
if self.ns.parent:
|
||||
self.ns[n] = GLOBAL
|
||||
|
||||
def visit_Nonlocal(self, node):
|
||||
if isinstance(self.ns, ModuleNS):
|
||||
raise SyntaxError("nonlocal declaration not allowed at module level")
|
||||
for n in node.names:
|
||||
self.ns[n] = NONLOCAL
|
||||
|
||||
def resolve_nonlocal(self, id, ns):
|
||||
while ns:
|
||||
res = ns.get(id, NO_VAR)
|
||||
if res is GLOBAL:
|
||||
return self.module_ns
|
||||
if res is not NO_VAR and res is not NONLOCAL:
|
||||
if isinstance(ns, ModuleNS):
|
||||
break
|
||||
return ns
|
||||
ns = ns.parent
|
||||
raise SyntaxError("no binding for nonlocal '{}' found".format(id))
|
||||
|
||||
def visit_Name(self, node):
|
||||
if isinstance(node.ctx, ast.Load):
|
||||
res = NO_VAR
|
||||
ns = self.ns
|
||||
# We always lookup in the current namespace (on the first
|
||||
# iteration), but afterwards we always skip class namespaces.
|
||||
# Or put it another way, class code can look up in its own
|
||||
# namespace, but that's the only case when the class namespace
|
||||
# is consulted.
|
||||
skip_classes = False
|
||||
while ns:
|
||||
if not (skip_classes and isinstance(ns, ClassNS)):
|
||||
res = ns.get(node.id, NO_VAR)
|
||||
if res is not NO_VAR:
|
||||
break
|
||||
ns = ns.parent
|
||||
skip_classes = True
|
||||
|
||||
if res is NONLOCAL:
|
||||
ns = self.resolve_nonlocal(node.id, ns.parent)
|
||||
return ns[node.id]
|
||||
if res is GLOBAL:
|
||||
res = self.module_ns.get(node.id, NO_VAR)
|
||||
if res is not NO_VAR:
|
||||
return res
|
||||
|
||||
try:
|
||||
return getattr(builtins, node.id)
|
||||
except AttributeError:
|
||||
raise NameError("name '{}' is not defined".format(node.id))
|
||||
elif isinstance(node.ctx, ast.Store):
|
||||
res = self.ns.get(node.id, NO_VAR)
|
||||
if res is GLOBAL:
|
||||
self.module_ns[node.id] = self.store_val
|
||||
elif res is NONLOCAL:
|
||||
ns = self.resolve_nonlocal(node.id, self.ns.parent)
|
||||
ns[node.id] = self.store_val
|
||||
else:
|
||||
self.ns[node.id] = self.store_val
|
||||
elif isinstance(node.ctx, ast.Del):
|
||||
res = self.ns.get(node.id, NO_VAR)
|
||||
if res is NO_VAR:
|
||||
raise NameError("name '{}' is not defined".format(node.id))
|
||||
elif res is GLOBAL:
|
||||
del self.module_ns[node.id]
|
||||
elif res is NONLOCAL:
|
||||
ns = self.resolve_nonlocal(node.id, self.ns.parent)
|
||||
del ns[node.id]
|
||||
else:
|
||||
del self.ns[node.id]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
def visit_Dict(self, node):
|
||||
return {self.visit(p[0]): self.visit(p[1]) for p in zip(node.keys, node.values)}
|
||||
|
||||
def visit_Set(self, node):
|
||||
return {self.visit(e) for e in node.elts}
|
||||
|
||||
def visit_List(self, node):
|
||||
return [self.visit(e) for e in node.elts]
|
||||
|
||||
def visit_Tuple(self, node):
|
||||
return tuple([self.visit(e) for e in node.elts])
|
||||
|
||||
def visit_NameConstant(self, node):
|
||||
return node.value
|
||||
|
||||
def visit_Ellipsis(self, node):
|
||||
# In Py3k only
|
||||
from ast import Ellipsis
|
||||
return Ellipsis
|
||||
|
||||
def visit_Print(self, node):
|
||||
# In Py2k only
|
||||
raise NotImplementedError("Absolutely not. Use __future__.")
|
||||
|
||||
def visit_Str(self, node):
|
||||
return node.s
|
||||
|
||||
def visit_Bytes(self, node):
|
||||
return node.s
|
||||
|
||||
def visit_Num(self, node):
|
||||
return node.n
|
||||
|
||||
|
||||
class InterpreterSystem(object):
|
||||
"""A bag of shared state."""
|
||||
|
||||
def __init__(self, path=None):
|
||||
self.modules = {}
|
||||
self.path = path or sys.path
|
||||
|
||||
def handle_import(self, name, globals=None, locals=None, fromlist=(), level=0):
|
||||
log.debug(" Attempting to import '{}'".format(name))
|
||||
if name not in self.modules:
|
||||
if name in sys.modules:
|
||||
log.debug(" Short-circuited from bootstrap sys.modules")
|
||||
self.modules[name] = sys.modules[name]
|
||||
|
||||
else:
|
||||
name = name.replace(".", os.path.sep)
|
||||
for e in self.path:
|
||||
for ext in [
|
||||
# ".flow",
|
||||
".py",
|
||||
]:
|
||||
if os.path.isdir(e):
|
||||
f = os.path.join(e, name + ext)
|
||||
log.debug(" Checking {}".format(f))
|
||||
if os.path.exists(f):
|
||||
mod = self.load(f)
|
||||
self.modules[name] = mod.ns
|
||||
break
|
||||
elif os.path.isfile(e):
|
||||
# FIXME (arrdem 2021-05-)
|
||||
raise RuntimeError("Import from .zip/.whl/.egg archives aren't supported yet")
|
||||
else:
|
||||
self.modules[name] = __import__(name, globals, locals, fromlist, level)
|
||||
|
||||
return self.modules[name]
|
||||
|
||||
def load(self, fname):
|
||||
with open(fname) as f:
|
||||
tree = ast.parse(f.read())
|
||||
interp = ModuleInterpreter(self, fname, tree)
|
||||
interp.visit(tree)
|
||||
return interp
|
||||
|
||||
def execute(self, fname):
|
||||
with open(fname) as f:
|
||||
tree = ast.parse(f.read())
|
||||
interp = ModuleInterpreter(self, fname, tree)
|
||||
interp.ns["__name__"] = "__main__"
|
||||
self.modules["__main__"] = InterpModule(interp.ns)
|
||||
interp.visit(tree)
|
||||
return interp
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
InterpreterSystem().execute(sys.argv[1])
|
||||
25
projects/flowmetal/scratch/test.py
Normal file
25
projects/flowmetal/scratch/test.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
#!astinterp
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
class Foo(object):
|
||||
def bar(self):
|
||||
return 1
|
||||
|
||||
@property
|
||||
def baz(self):
|
||||
print("'Computing' baz...")
|
||||
return 2
|
||||
|
||||
|
||||
a = Foo()
|
||||
print(a.bar())
|
||||
print(a.baz)
|
||||
|
||||
import random
|
||||
for _ in range(10):
|
||||
print(random.randint(0, 1024))
|
||||
|
||||
|
||||
def bar(a, b, **bs):
|
||||
pass
|
||||
Loading…
Reference in a new issue