diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/.bazelignore @@ -0,0 +1 @@ +.git diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 0000000..260a611 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,2 @@ +test --test_output=errors +build --keep_going diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000..09b254e --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +6.0.0 diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..84c7d18 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +export VIRTUAL_ENV=/home/arrdem/.virtualenvs/flowmetal diff --git a/.gitignore b/.gitignore index 8792abd..477386b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /**/__pycache__ /**/*.egg-info +scratch +bazel-* diff --git a/MODULE.bazel b/MODULE.bazel deleted file mode 100644 index 57257d5..0000000 --- a/MODULE.bazel +++ /dev/null @@ -1,24 +0,0 @@ -bazel_dep(name = "rules_python", version = "0.19.0") - -pip = use_extension("@rules_python//python:extensions.bzl", "pip") - -pip.parse( - name = "pypa", - requirements_lock = "//tools/python:requirements_lock.txt", -) - -use_repo(pip, "pypa") - -# (Optional) Register a specific python toolchain instead of using the host version -python = use_extension("@rules_python//python:extensions.bzl", "python") - -python.toolchain( - name = "python3_10", - python_version = "3.10", -) - -use_repo(python, "python3_10_toolchains") - -register_toolchains( - "@python3_10_toolchains//:all", -) diff --git a/WORKSPACE b/WORKSPACE index 85830e1..3770ab1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,16 +1,52 @@ workspace( - name = "arrdem_flowmetal", + name = "flowmetal" ) load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + http_archive( - name = "rules_python", - sha256 = "ffc7b877c95413c82bfd5482c017edcf759a6250d8b24e82f41f3c8b8d9e287e", - strip_prefix = "rules_python-0.19.0", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.19.0/rules_python-0.19.0.tar.gz", + name = "bazel_skylib", + sha256 = "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz", + ], ) -load("@rules_python//python:repositories.bzl", "py_repositories") +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") -py_repositories() +bazel_skylib_workspace() + +rules_python_version = "c504355672223144cefb2cbf3f69e2d38e7e2726" + +http_archive( + name = "rules_python", + sha256 = "3f12b492dbf7d56b0e3deed81f21d56c3241babaa52d7eb525cb7c657bba9125", + strip_prefix = "rules_python-{}".format(rules_python_version), + url = "https://github.com/bazelbuild/rules_python/archive/{}.zip".format(rules_python_version), +) + +load("@rules_python//python:repositories.bzl", "python_register_toolchains") + +python_register_toolchains( + name = "python3_10", + python_version = "3.10", +) + +load("@python3_10//:defs.bzl", python3_10="interpreter") + +load("@rules_python//python:pip.bzl", "pip_parse") + +# Create a central repo that knows about the dependencies needed from +# requirements_lock.txt. +pip_parse( + name = "pypi", + python_interpreter_target = python3_10, + requirements_lock = "//tools/python:requirements_lock.txt", +) + +load("@pypi//:requirements.bzl", "install_deps") + +install_deps() diff --git a/components/hydra/src/python/hydra/__init__.py b/components/hydra/src/python/hydra/__init__.py index 9b33cf5..93721c5 100644 --- a/components/hydra/src/python/hydra/__init__.py +++ b/components/hydra/src/python/hydra/__init__.py @@ -1,29 +1,32 @@ #!/usr/bin/env python3.10 +"""Hydra; the multi-headed Python interpreter. + +> Chop off one head and two more grow back in its place. + + + +""" + import ast +import builtins import logging import os import sys -import builtins -from typing import Optional, Type +from typing import Optional, Type, Union, List, Callable +from pathlib import Path +from importlib import __import__ +from attrs import Factory, define, field log = logging.getLogger(__name__) -class StrictNodeVisitor(ast.NodeVisitor): - def generic_visit(self, node): - n = node.__class__.__name__ - raise NotImplementedError("Visitor for node {} not implemented".format(n)) - - +@define class ANamespace: - def __init__(self, node): - self.d = {} - self.parent: Optional[Type["ANamespace"]] = None - # Cross-link namespace to AST node. Note that we can't do the - # opposite, because for one node, there can be different namespaces. - self.node = node + node: ast.AST + d: dict = {} + parent: Optional[Type["ANamespace"]] = None def __getitem__(self, k): return self.d[k] @@ -44,118 +47,22 @@ class ANamespace: return "<{} {}>".format(self.__class__.__name__, self.d) +@define class ModuleNS(ANamespace): # parent: Optional["ModuleNS"] = None pass +@define class FunctionNS(ANamespace): pass +@define class ClassNS(ANamespace): cls: Optional[type] = None -# Pycopy by default doesn't support direct slice construction, use helper -# object to construct it. -class SliceGetter: - def __getitem__(self, idx): - return idx - - -slice_getter = SliceGetter() - - -def arg_name(arg): - return arg.arg - - -def kwarg_defaults(args): - return args.kw_defaults - - -class TargetNonlocalFlow(Exception): - """Base exception class to simulate non-local control flow transfers in - a target application.""" - - -class TargetBreak(TargetNonlocalFlow): - pass - - -class TargetContinue(TargetNonlocalFlow): - pass - - -class TargetReturn(TargetNonlocalFlow): - pass - - -class VarScopeSentinel: - def __init__(self, name): - self.name = name - - -NO_VAR = VarScopeSentinel("no_var") -GLOBAL = VarScopeSentinel("global") -NONLOCAL = VarScopeSentinel("nonlocal") - - -class InterpFuncWrap: - "Callable wrapper for AST functions (FunctionDef nodes)." - - def __init__(self, node, interp): - self.node = node - self.interp = interp - self.lexical_scope = interp.ns - - def __call__(self, *args, **kwargs): - return self.interp.call_func(self.node, self, *args, **kwargs) - - -# Python don't fully treat objects, even those defining __call__() special method, as a true -# callable. For example, such objects aren't automatically converted to bound methods if looked up -# as another object's attributes. As we want our "interpreted functions" to behave as closely as -# possible to real functions, we just wrap function object with a real function. An alternative -# might have been to perform needed checks and explicitly bind a method using types.MethodType() in -# visit_Attribute (but then maybe there would be still other cases of "callable object" vs -# "function" discrepancies). -def InterpFunc(fun): - def func(*args, **kwargs): - return fun.__call__(*args, **kwargs) - - return func - - -class InterpWith: - def __init__(self, ctx): - self.ctx = ctx - - def __enter__(self): - return self.ctx.__enter__() - - def __exit__(self, tp, exc, tb): - # Don't leak meta-level exceptions into target - if isinstance(exc, TargetNonlocalFlow): - tp = exc = tb = None - return self.ctx.__exit__(tp, exc, tb) - - -class InterpModule: - def __init__(self, ns): - self.ns = ns - - def __getattr__(self, name): - try: - return self.ns[name] - except KeyError: - raise AttributeError - - def __dir__(self): - return list(self.ns.d.keys()) - - # TODO (arrdem 2023-03-08): # This interpreter works well enough to import `requests` and many other libraries and do some # work, but is unsuited to Flowmetal's needs for checkpointing. Because this interpreter uses @@ -177,749 +84,84 @@ class InterpModule: # representing where to resume. The logical equivalent of a program counter, but a tree path. -class ModuleInterpreter(StrictNodeVisitor): - """An interpreter specific to a single module.""" +@define +class Module: + fname: Union[Path, str] + tree: ast.AST + ns: "ANamespace" = field() - def __init__(self, system, fname, node): - self.system = system - self.fname = fname - self.module_ns: ModuleNS = ModuleNS(node) - self.ns: ANamespace = self.module_ns + @ns.default + def _ns_default(self): + return ModuleNS(self.tree) - # Call stack (in terms of function AST nodes). - self.call_stack = [] +@define +class Pc: + """A 'program counter' as a list of AST indices.""" - # To implement "store" operation, we need to arguments: location and value to store. The - # operation itself is handled by a node visitor (e.g. visit_Name), and location is - # represented by AST node, but there's no support to pass additional arguments to a visitor - # (likely, because it would be a burden to explicit pass such additional arguments thru the - # chain of visitors). So instead, we store this value as field. As interpretation happens - # sequentially, there's no risk that it will be overwritten "concurrently". - self.store_val = None + idxs: List[int] = Factory(list) - # Current active exception, for bare "raise", which doesn't work across function boundaries - # (and that's how we have it - exception would be caught in visit_Try, while re-rasing would - # happen in visit_Raise). - self.cur_exc = [] - def push_ns(self, new_ns): - new_ns.parent = self.ns - self.ns = new_ns +@define +class Frame: + """An 'execution frame' as a PC, AST and namespace.""" + pc: Pc + ast: ast.AST + ns: ANamespace - def pop_ns(self): - assert self.ns is not None - self.ns = self.ns.parent - def stmt_list_visit(self, lst): - res = None - for s in lst: - res = self.visit(s) - return res +@define +class Cont: + """A 'Continuation' (thread/coroutine) of execution.""" + id: int + entry: Frame + stack: List[Frame] = field() - def wrap_decorators(self, obj, node): - for deco_n in reversed(list(node.decorator_list)): - deco = self.visit(deco_n) - obj = deco(obj) - return obj + @stack.default + def _stack_default(self): + return [self.entry] - def visit(self, node): - val = super(StrictNodeVisitor, self).visit(node) - return val - def visit_Module(self, node): - self.stmt_list_visit(node.body) +@define +class Vm: + """A bag of shared state. - def visit_Expression(self, node): - return self.visit(node.body) + :attribute path: The equivalent of sys.path + :attribute modules: The equivalent of sys.modules + :attribute conts: All interpreter continuations + :attribute log: A log of all statements executed by any continuation - def visit_ClassDef(self, node): - ns: ClassNS = ClassNS(node) - self.push_ns(ns) - try: - self.stmt_list_visit(node.body) - except Exception: - self.pop_ns() - raise - self.pop_ns() - cls = type(node.name, tuple([self.visit(b) for b in node.bases]), ns.d) - cls = self.wrap_decorators(cls, node) - self.ns[node.name] = cls - # Store reference to class object in the namespace object - ns.cls = cls + It should be possible to reconstruct the VM's state simply by replaying the + log in statement order, as a fallback for dealing with C-extension state, + connections and soforth. - def visit_Lambda(self, node): - node.name = "" - return self.prepare_func(node) + """ - def visit_FunctionDef(self, node): - # Defaults are evaluated at function definition time, so we need to do that now. - func = self.prepare_func(node) - func = self.wrap_decorators(func, node) - self.ns[node.name] = func + path: list = Factory(lambda: list(sys.path)) + modules: dict = Factory(dict) + conts: dict = Factory(dict) + log: list = Factory(list) - def prepare_func(self, node): - """Prepare function AST node for future interpretation: pre-calculate - and cache useful information, etc.""" - - func = InterpFuncWrap(node, self) - args = node.args or node.posonlyargs - num_required = len(args.args) - len(args.defaults) - all_args = set() - d = {} - for i, a in enumerate(args.args): - all_args.add(arg_name(a)) - if i >= num_required: - d[arg_name(a)] = self.visit(args.defaults[i - num_required]) - - for a, v in zip(getattr(args, "kwonlyargs", ()), kwarg_defaults(args)): - all_args.add(arg_name(a)) - if v is not None: - d[arg_name(a)] = self.visit(v) - # We can store cached argument names of a function in its node - it's static. - node.args.all_args = all_args - # We can't store the values of default arguments - they're dynamic, may depend on the - # lexical scope. - func.defaults_dict = d - - return InterpFunc(func) - - def prepare_func_args(self, node, interp_func, *args, **kwargs): - def arg_num_mismatch(): - raise TypeError( - "{}() takes {} positional arguments but {} were given".format( - node.name, len(argspec.args), len(args) - ) - ) - - argspec = node.args - # If there's vararg, either offload surplus of args to it, or init it to empty tuple (all in - # one statement). If no vararg, error on too many args. - # - # Note that we have to do the .posonlyargs dance - if argspec.vararg: - self.ns[argspec.vararg.arg] = args[len(argspec.args) :] - else: - if len(args) > len(argspec.args or getattr(argspec, "posonlyargs", ())): - arg_num_mismatch() - - if argspec.args: - for i in range(min(len(args), len(argspec.args))): - self.ns[arg_name(argspec.args[i])] = args[i] - elif getattr(argspec, "posonlyargs", ()): - if len(args) != len(argspec.posonlyargs): - arg_num_mismatch() - - for a, value in zip(argspec.posonlyargs, args): - self.ns[arg_name(a)] = value - - # Process incoming keyword arguments, putting them in namespace if actual arg exists by that - # name, or offload to function's kwarg if any. All make needed checks and error out. - func_kwarg = {} - for k, v in kwargs.items(): - if k in argspec.all_args: - if k in self.ns: - raise TypeError( - "{}() got multiple values for argument '{}'".format( - node.name, k - ) - ) - self.ns[k] = v - elif argspec.kwarg: - func_kwarg[k] = v - else: - raise TypeError( - "{}() got an unexpected keyword argument '{}'".format(node.name, k) - ) - if argspec.kwarg: - self.ns[arg_name(argspec.kwarg)] = func_kwarg - - # Finally, overlay default values for arguments not yet initialized. We need to do this last - # for "multiple values for the same arg" check to work. - for k, v in interp_func.defaults_dict.items(): - if k not in self.ns: - self.ns[k] = v - - # And now go thru and check for any missing arguments. - for a in argspec.args: - if arg_name(a) not in self.ns: - raise TypeError( - "{}() missing required positional argument: '{}'".format( - node.name, arg_name(a) - ) - ) - for a in getattr(argspec, "kwonlyargs", ()): - if a.arg not in self.ns: - raise TypeError( - "{}() missing required keyword-only argument: '{}'".format( - node.name, arg_name(a) - ) - ) - - def call_func(self, node, interp_func, *args, **kwargs): - self.call_stack.append(node) - # We need to switch from dynamic execution scope to lexical scope in which function was - # defined (then switch back on return). - dyna_scope = self.ns - self.ns = interp_func.lexical_scope - self.push_ns(FunctionNS(node)) - try: - self.prepare_func_args(node, interp_func, *args, **kwargs) - if isinstance(node.body, list): - res = self.stmt_list_visit(node.body) - else: - res = self.visit(node.body) - except TargetReturn as e: - res = e.args[0] - finally: - self.pop_ns() - self.ns = dyna_scope - self.call_stack.pop() - return res - - def visit_Return(self, node): - if not isinstance(self.ns, FunctionNS): - raise SyntaxError("'return' outside function") - raise TargetReturn(node.value and self.visit(node.value)) - - def visit_With(self, node): - assert len(node.items) == 1 - ctx = self.visit(node.items[0].context_expr) - with InterpWith(ctx) as val: - if node.items[0].optional_vars is not None: - self.handle_assign(node.items[0].optional_vars, val) - self.stmt_list_visit(node.body) - - def visit_Try(self, node): - try: - self.stmt_list_visit(node.body) - except TargetNonlocalFlow: - raise - except Exception as e: - self.cur_exc.append(e) - try: - for h in getattr(node, "handlers", ()): - if h.type is None or isinstance(e, self.visit(h.type)): - if h.name: - self.ns[h.name] = e - self.stmt_list_visit(h.body) - if h.name: - del self.ns[h.name] - break - else: - raise - finally: - self.cur_exc.pop() - else: - self.stmt_list_visit(node.orelse) - finally: - if getattr(node, "finalbody", None): - self.stmt_list_visit(node.finalbody) - - def visit_TryExcept(self, node): - # Py2k only; py3k merged all this into one node type. - return self.visit_Try(node) - - def visit_TryFinally(self, node): - # Py2k only; py3k merged all this into one node type. - return self.visit_Try(node) - - def visit_For(self, node): - iter = self.visit(node.iter) - for item in iter: - self.handle_assign(node.target, item) - try: - self.stmt_list_visit(node.body) - except TargetBreak: - break - except TargetContinue: - continue - else: - self.stmt_list_visit(node.orelse) - - def visit_While(self, node): - while self.visit(node.test): - try: - self.stmt_list_visit(node.body) - except TargetBreak: - break - except TargetContinue: - continue - else: - self.stmt_list_visit(node.orelse) - - def visit_Break(self, node): - raise TargetBreak - - def visit_Continue(self, node): - raise TargetContinue - - def visit_If(self, node): - test = self.visit(node.test) - if test: - self.stmt_list_visit(node.body) - else: - self.stmt_list_visit(node.orelse) - - def visit_Import(self, node): - for n in node.names: - self.ns[n.asname or n.name] = self.system.handle_import(n.name) - - def visit_ImportFrom(self, node): - mod = self.system.handle_import( - node.module, None, None, [n.name for n in node.names], node.level - ) - for n in node.names: - if n.name == "*": - # This is the special case of the wildcard import. Copy - # everything over. - for n in getattr(mod, "__all__", dir(mod)): - self.ns[n] = getattr(mod, n) - else: - self.ns[n.asname or n.name] = getattr(mod, n.name) - - def visit_Raise(self, node): - if node.exc is None: - if not self.cur_exc: - raise RuntimeError("No active exception to reraise") - raise self.cur_exc[-1] - elif node.cause is None: - raise self.visit(node.exc) - # else: - # raise self.visit(node.exc) from self.visit(node.cause) - - def visit_AugAssign(self, node): - assert isinstance(node.target.ctx, ast.Store) - # Not functional style, oops. Node in AST has store context, but we need to read its value - # first. To not construct a copy of the entire node with load context, we temporarily patch - # it in-place. - save_ctx = node.target.ctx - node.target.ctx = ast.Load() - var_val = self.visit(node.target) - node.target.ctx = save_ctx - - rval = self.visit(node.value) - - # As augmented assignment is statement, not operator, we can't put them all into map. We - # could instead directly lookup special inplace methods (__iadd__ and friends) and use them, - # with a fallback to normal binary operations, but from the point of view of this - # interpreter, presence of such methods is an implementation detail of the object system, - # it's not concerned with it. - op = type(node.op) - if op is ast.Add: - var_val += rval - elif op is ast.Sub: - var_val -= rval - elif op is ast.Mult: - var_val *= rval - elif op is ast.Div: - var_val /= rval - elif op is ast.FloorDiv: - var_val //= rval - elif op is ast.Mod: - var_val %= rval - elif op is ast.Pow: - var_val **= rval - elif op is ast.LShift: - var_val <<= rval - elif op is ast.RShift: - var_val >>= rval - elif op is ast.BitAnd: - var_val &= rval - elif op is ast.BitOr: - var_val |= rval - elif op is ast.BitXor: - var_val ^= rval - else: - raise NotImplementedError - - self.store_val = var_val - self.visit(node.target) - - def visit_Assign(self, node): - val = self.visit(node.value) - for n in node.targets: - self.handle_assign(n, val) - - def handle_assign(self, target, val): - if isinstance(target, ast.Tuple): - it = iter(val) - try: - for elt_idx, t in enumerate(target.elts): - if getattr(ast, "Starred", None) and isinstance(t, ast.Starred): - t = t.value - all_elts = list(it) - break_i = len(all_elts) - (len(target.elts) - elt_idx - 1) - self.store_val = all_elts[:break_i] - it = iter(all_elts[break_i:]) - else: - self.store_val = next(it) - self.visit(t) - except StopIteration: - raise ValueError( - "not enough values to unpack (expected {})".format(len(target.elts)) - ) - - try: - next(it) - raise ValueError( - "too many values to unpack (expected {})".format(len(target.elts)) - ) - except StopIteration: - # Expected - pass - else: - self.store_val = val - self.visit(target) - - def visit_Delete(self, node): - for n in node.targets: - self.visit(n) - - def visit_Pass(self, node): - pass - - def visit_Assert(self, node): - if node.msg is None: - assert self.visit(node.test) - else: - assert self.visit(node.test), self.visit(node.msg) - - def visit_Expr(self, node): - # Produced value is ignored - self.visit(node.value) - - def enumerate_comps(self, iters): - """Enumerate thru all possible values of comprehension clauses, - including multiple "for" clauses, each optionally associated - with multiple "if" clauses. Current result of the enumeration - is stored in the namespace.""" - - def eval_ifs(iter): - """Evaluate all "if" clauses.""" - for cond in iter.ifs: - if not self.visit(cond): - return False - return True - - if not iters: - yield - return - for el in self.visit(iters[0].iter): - self.store_val = el - self.visit(iters[0].target) - for t in self.enumerate_comps(iters[1:]): - if eval_ifs(iters[0]): - yield - - def visit_ListComp(self, node): - self.push_ns(FunctionNS(node)) - try: - return [self.visit(node.elt) for _ in self.enumerate_comps(node.generators)] - finally: - self.pop_ns() - - def visit_SetComp(self, node): - self.push_ns(FunctionNS(node)) - try: - return {self.visit(node.elt) for _ in self.enumerate_comps(node.generators)} - finally: - self.pop_ns() - - def visit_DictComp(self, node): - self.push_ns(FunctionNS(node)) - try: - return { - self.visit(node.key): self.visit(node.value) - for _ in self.enumerate_comps(node.generators) - } - finally: - self.pop_ns() - - def visit_IfExp(self, node): - if self.visit(node.test): - return self.visit(node.body) - else: - return self.visit(node.orelse) - - def visit_Call(self, node): - func = self.visit(node.func) - - args = [] - for a in node.args: - if getattr(ast, "Starred", None) and isinstance(a, ast.Starred): - args.extend(self.visit(a.value)) - else: - args.append(self.visit(a)) - - kwargs = {} - for kw in node.keywords: - val = self.visit(kw.value) - if kw.arg is None: - kwargs.update(val) - else: - kwargs[kw.arg] = val - - if func is builtins.super and not args: - if not self.ns.parent or not isinstance(self.ns.parent, ClassNS): - raise RuntimeError("super(): no arguments") - # As we're creating methods dynamically outside of class, super() without argument won't - # work, as that requires __class__ cell. Creating that would be cumbersome (Pycopy - # definitely lacks enough introspection for that), so we substitute 2 implied args - # (which argumentless super() would take from cell and 1st arg to func). In our case, we - # take them from prepared bookkeeping info. - args = (self.ns.parent.cls, self.ns["self"]) - - return func(*args, **kwargs) - - def visit_Compare(self, node): - cmpop_map = { - ast.Eq: lambda x, y: x == y, - ast.NotEq: lambda x, y: x != y, - ast.Lt: lambda x, y: x < y, - ast.LtE: lambda x, y: x <= y, - ast.Gt: lambda x, y: x > y, - ast.GtE: lambda x, y: x >= y, - ast.Is: lambda x, y: x is y, - ast.IsNot: lambda x, y: x is not y, - ast.In: lambda x, y: x in y, - ast.NotIn: lambda x, y: x not in y, - } - lv = self.visit(node.left) - for op, r in zip(node.ops, node.comparators): - rv = self.visit(r) - if not cmpop_map[type(op)](lv, rv): - return False - lv = rv - return True - - def visit_BoolOp(self, node): - if isinstance(node.op, ast.And): - res = True - for v in node.values: - res = res and self.visit(v) - elif isinstance(node.op, ast.Or): - res = False - for v in node.values: - res = res or self.visit(v) - else: - raise NotImplementedError - return res - - def visit_BinOp(self, node): - binop_map = { - ast.Add: lambda x, y: x + y, - ast.Sub: lambda x, y: x - y, - ast.Mult: lambda x, y: x * y, - ast.Div: lambda x, y: x / y, - ast.FloorDiv: lambda x, y: x // y, - ast.Mod: lambda x, y: x % y, - ast.Pow: lambda x, y: x**y, - ast.LShift: lambda x, y: x << y, - ast.RShift: lambda x, y: x >> y, - ast.BitAnd: lambda x, y: x & y, - ast.BitOr: lambda x, y: x | y, - ast.BitXor: lambda x, y: x ^ y, - } - l = self.visit(node.left) - r = self.visit(node.right) - return binop_map[type(node.op)](l, r) - - def visit_UnaryOp(self, node): - unop_map = { - ast.UAdd: lambda x: +x, - ast.USub: lambda x: -x, - ast.Invert: lambda x: ~x, - ast.Not: lambda x: not x, - } - val = self.visit(node.operand) - return unop_map[type(node.op)](val) - - def visit_Subscript(self, node): - obj = self.visit(node.value) - idx = self.visit(node.slice) - if isinstance(node.ctx, ast.Load): - return obj[idx] - elif isinstance(node.ctx, ast.Store): - obj[idx] = self.store_val - elif isinstance(node.ctx, ast.Del): - del obj[idx] - else: - raise NotImplementedError - - def visit_Index(self, node): - return self.visit(node.value) - - def visit_Slice(self, node): - # Any of these can be None - lower = node.lower and self.visit(node.lower) - upper = node.upper and self.visit(node.upper) - step = node.step and self.visit(node.step) - slice = slice_getter[lower:upper:step] - return slice - - def visit_Attribute(self, node): - obj = self.visit(node.value) - if isinstance(node.ctx, ast.Load): - return getattr(obj, node.attr) - elif isinstance(node.ctx, ast.Store): - setattr(obj, node.attr, self.store_val) - elif isinstance(node.ctx, ast.Del): - delattr(obj, node.attr) - else: - raise NotImplementedError - - def visit_Global(self, node): - for n in node.names: - if n in self.ns and self.ns[n] is not GLOBAL: - raise SyntaxError( - "SyntaxError: name '{}' is assigned to before global declaration".format( - n - ) - ) - # Don't store GLOBAL in the top-level namespace - if self.ns.parent: - self.ns[n] = GLOBAL - - def visit_Nonlocal(self, node): - if isinstance(self.ns, ModuleNS): - raise SyntaxError("nonlocal declaration not allowed at module level") - for n in node.names: - self.ns[n] = NONLOCAL - - def resolve_nonlocal(self, id, ns): - while ns: - res = ns.get(id, NO_VAR) - if res is GLOBAL: - return self.module_ns - if res is not NO_VAR and res is not NONLOCAL: - if isinstance(ns, ModuleNS): - break - return ns - ns = ns.parent - raise SyntaxError("no binding for nonlocal '{}' found".format(id)) - - def visit_Name(self, node): - if isinstance(node.ctx, ast.Load): - res = NO_VAR - ns = self.ns - # We always lookup in the current namespace (on the first iteration), but afterwards we always skip class - # namespaces. Or put it another way, class code can look up in its own namespace, but that's the only case - # when the class namespace is consulted. - skip_classes = False - while ns: - if not (skip_classes and isinstance(ns, ClassNS)): - res = ns.get(node.id, NO_VAR) - if res is not NO_VAR: - break - ns = ns.parent - skip_classes = True - - if res is NONLOCAL: - ns = self.resolve_nonlocal(node.id, ns.parent) - return ns[node.id] - - if res is GLOBAL: - res = self.module_ns.get(node.id, NO_VAR) - - if res is not NO_VAR: - return res - - try: - return getattr(builtins, node.id) - except AttributeError: - raise NameError("name '{}' is not defined".format(node.id)) - - elif isinstance(node.ctx, ast.Store): - res = self.ns.get(node.id, NO_VAR) - if res is GLOBAL: - self.module_ns[node.id] = self.store_val - - elif res is NONLOCAL: - ns = self.resolve_nonlocal(node.id, self.ns.parent) - ns[node.id] = self.store_val - - else: - self.ns[node.id] = self.store_val - - elif isinstance(node.ctx, ast.Del): - res = self.ns.get(node.id, NO_VAR) - if res is NO_VAR: - raise NameError("name '{}' is not defined".format(node.id)) - - elif res is GLOBAL: - del self.module_ns[node.id] - - elif res is NONLOCAL: - ns = self.resolve_nonlocal(node.id, self.ns.parent) - del ns[node.id] - - else: - del self.ns[node.id] - - else: - raise NotImplementedError - - def visit_Dict(self, node): - return {self.visit(p[0]): self.visit(p[1]) for p in zip(node.keys, node.values)} - - def visit_Set(self, node): - return {self.visit(e) for e in node.elts} - - def visit_List(self, node): - return [self.visit(e) for e in node.elts] - - def visit_Tuple(self, node): - return tuple([self.visit(e) for e in node.elts]) - - def visit_NameConstant(self, node): - return node.value - - def visit_Ellipsis(self, node): - # In Py3k only - from ast import Ellipsis - - return Ellipsis - - def visit_Print(self, node): - # In Py2k only - raise SyntaxError("Absolutely not. Use __future__.") - - def visit_Str(self, node): - return node.s - - def visit_Bytes(self, node): - return node.s - - def visit_Num(self, node): - return node.n - - -class InterpreterSystem(object): - """A bag of shared state.""" - - def __init__(self, path=None): - self.modules = {} - self.path = path or sys.path - - def handle_import(self, name, globals=None, locals=None, fromlist=(), level=0): + def handle_import( + self, thread, name, globals=None, locals=None, fromlist=(), level=0 + ): log.debug(" Attempting to import '{}'".format(name)) if name not in self.modules: if name in sys.modules: - log.debug(" Short-circuited from bootstrap sys.modules") + # FIXME: Need to hack sys, os and several other built-in packages here + log.debug("Short-circuited loading %r from bootstrap sys.modules", name) self.modules[name] = sys.modules[name] else: name = name.replace(".", os.path.sep) for e in self.path: for ext in [ - # ".flow", ".py", ]: if os.path.isdir(e): f = os.path.join(e, name + ext) log.debug(" Checking {}".format(f)) if os.path.exists(f): - mod = self.load(f) + mod = self.execute_load(thread, f, name) self.modules[name] = mod.ns break @@ -930,24 +172,30 @@ class InterpreterSystem(object): ) else: + log.debug("Falling back to native import for %r", name) self.modules[name] = __import__( name, globals, locals, fromlist, level ) return self.modules[name] - def load(self, fname): - with open(fname) as f: - tree = ast.parse(f.read()) - interp = ModuleInterpreter(self, fname, tree) - interp.visit(tree) - return interp + def execute_module(self, module: Module): + """Execute all the Expressions and Statements in a given Module sequentially, as in a single thread.""" - def execute(self, fname): + def execute_load(self, fname, name): + """Execute the given file as if it were an imported module.""" + + # FIXME: Choose encoding here with open(fname) as f: tree = ast.parse(f.read()) - interp = ModuleInterpreter(self, fname, tree) - interp.ns["__name__"] = "__main__" - self.modules["__main__"] = InterpModule(interp.ns) - interp.visit(tree) - return interp + + mod = Module(fname, tree) + mod.ns["__name__"] = name + self.modules[name] = mod.ns + self.execute_module(mod) + return mod + + def execute_dunder_main(self, fname): + """Execute the given file as if it were a script entrypoint.""" + + return self.execute_load(fname, "__main__") diff --git a/components/uast/BUILD b/components/uast/BUILD deleted file mode 100644 index 5c15d3f..0000000 --- a/components/uast/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -py_project( - name = "uast", - lib_deps = [ - "//components/utokenize", - ] -) diff --git a/components/uast/README.md b/components/uast/README.md deleted file mode 100644 index 404d5fd..0000000 --- a/components/uast/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# pycopy-ast - -A small parallel implementation of Python's `ast` module, vendored from `pycopy-ast==2.9.1`. - -[pycopy-ast](https://pypi.org/project/pycopy-ast/) is released under the MIT license, copyright © Paul Sokolovsky 2021. diff --git a/components/uast/src/python/uast/__init__.py b/components/uast/src/python/uast/__init__.py deleted file mode 100644 index 419290e..0000000 --- a/components/uast/src/python/uast/__init__.py +++ /dev/null @@ -1,126 +0,0 @@ -# (c) 2019 Paul Sokolovsky. MIT license. - -from .types import * - - -def dump_to_stream(t, file): - if isinstance(t, AST): - file.write(type(t).__name__) - file.write("(") - comma = False - for k in t._fields: - if k.startswith("_"): - continue - res = "" - if comma: - res += ", " - res += k + "=" - file.write(res) - dump_to_stream(getattr(t, k, None), file) - comma = True - file.write(")") - elif isinstance(t, list): - file.write("[") - comma = False - for v in t: - if comma: - file.write(", ") - dump_to_stream(v, file) - comma = True - file.write("]") - else: - file.write(repr(t)) - - -def dump(t): - import io - - buf = io.StringIO() - dump_to_stream(t, buf) - return buf.getvalue() - - -def iter_fields(t): - for k in t._fields: - if k.startswith("_"): - continue - yield (k, getattr(t, k, None)) - - -def copy_location(new_node, old_node): - return new_node - - -def parse_tokens(token_stream, filename="", mode="exec"): - import utokenize as tokenize - from . import parser - - p = parser.Parser(token_stream) - p.match(tokenize.ENCODING) - if mode == "exec": - t = p.match_mod() - elif mode == "eval": - t = Expression(body=p.require_expr()) - elif mode == "single": - t = Interactive(body=p.match_stmt()) - else: - raise ValueError - return t - - -def parse_stream(stream, filename="", mode="exec"): - import utokenize as tokenize - - tstream = tokenize.tokenize(stream.readline) - return parse_tokens(tstream) - - -def parse(source, filename="", mode="exec"): - import io - - return parse_stream(io.StringIO(source), filename, mode) - - -class NodeVisitor: - def visit(self, node): - n = node.__class__.__name__ - m = getattr(self, "visit_" + n, None) - if m: - return m(node) - else: - return self.generic_visit(node) - - def generic_visit(self, node): - for f in node._fields: - val = getattr(node, f) - if isinstance(val, list): - for v in val: - if isinstance(v, AST): - self.visit(v) - elif isinstance(val, AST): - self.visit(val) - - -class NodeTransformer(NodeVisitor): - def generic_visit(self, node): - for f in node._fields: - val = getattr(node, f) - if isinstance(val, list): - newl = [] - for v in val: - if not isinstance(v, AST): - newl.append(v) - continue - newv = self.visit(v) - if newv is None: - pass - elif isinstance(newv, list): - newl.extend(newv) - else: - newl.append(newv) - setattr(node, f, newl) - elif isinstance(val, AST): - newv = self.visit(val) - setattr(node, f, newv) - - return node diff --git a/components/uast/src/python/uast/parser.py b/components/uast/src/python/uast/parser.py deleted file mode 100644 index 82bf694..0000000 --- a/components/uast/src/python/uast/parser.py +++ /dev/null @@ -1,1278 +0,0 @@ -# (c) 2019 Paul Sokolovsky. MIT license. - -import sys -import logging -from utokenize import * -import utokenize -from . import types as ast - - -log = logging.Logger(__name__) - - -TOK_TYPE = 0 -TOK_STRING = 1 - -BP_UNTIL_COMMA = 6 -BP_LVALUE = 160 - 1 - - -class GenComp(ast.expr): - _fields = ("elt", "generators") - - -def literal_eval(s): - if s.endswith('"') or s.endswith("'"): - if s.endswith('"""') or s.endswith("'''"): - sep = s[-3:] - else: - sep = s[-1] - prefix, s = s.split(sep, 1) - s = s[: -len(sep)] - is_bytes = "b" in prefix - if "r" in prefix: - res = s - if is_bytes: - res = bytes(res, "utf-8") - else: - if is_bytes: - res = b"" - else: - res = "" - while s: - if s[0] == "\\": - c = s[1] - if c >= "0" and c <= "7": - s = s[1:] - ns = "" - while s and s[0] >= "0" and s[0] <= "7" and len(ns) < 3: - ns += s[0] - s = s[1:] - if is_bytes: - nc = bytes([int(ns, 8)]) - else: - nc = chr(int(ns, 8)) - elif c == "x": - if is_bytes: - nc = bytes([int(s[2:4], 16)]) - else: - nc = chr(int(s[2:4], 16)) - s = s[4:] - elif c == "u" and not is_bytes: - nc = chr(int(s[2:6], 16)) - s = s[6:] - elif c == "U" and not is_bytes: - nc = chr(int(s[2:10], 16)) - s = s[10:] - else: - nc = { - "a": "\a", - "b": "\b", - "f": "\f", - "n": "\n", - "r": "\r", - "t": "\t", - "v": "\v", - "\\": "\\", - "'": "'", - '"': '"', - "\n": "", - }.get(c) - if nc is None: - nc = s[0:2] - s = s[2:] - if is_bytes and not isinstance(nc, bytes): - nc = nc.encode() - res += nc - else: - if is_bytes: - res += s[0].encode() - else: - res += s[0] - s = s[1:] - return res - - raise NotImplementedError - - -# Pratt parser token root base class -class TokBase: - # Left denotation and null denotation binding powers, effectively, - # operator precedence. As such, we use operator's row number - # (starting from 1) in the precedence table at - # https://docs.python.org/3.5/reference/expressions.html#operator-precedence - # multiplied by 10. Note that not all operators are in that table, - # and some may get precedence inbetween (including before 10). - # For Pratt algorithm, only LBP needs to be set on each class - # (which can be at "left arg available" position, i.e. infix or - # postfix operators. NBP is mostly given as commented value for - # documentation purposes (actual NBP is encoded as numeric value - # in nud() method "for efficiency"). - lbp = 0 - # nbp = 0 - - -# Token base classes - - -class TokDelim(TokBase): - pass - - -class TokPrefix(TokBase): - @classmethod - def nud(cls, p, t): - arg = p.expr(cls.nbp) - node = ast.UnaryOp(op=cls.ast_un_op(), operand=arg) - return node - - -class TokInfix(TokBase): - @classmethod - def led(cls, p, left, t): - right = p.expr(cls.lbp) - if ( - cls.ast_bin_op in (ast.And, ast.Or) - and isinstance(left, ast.BoolOp) - and isinstance(left.op, cls.ast_bin_op) - and not getattr(left, "parenform", False) - ): - left.values.append(right) - return left - node = cls.bin_op(cls.ast_bin_op, left, right) - return node - - @staticmethod - def bin_op(op, left, right): - if op in (ast.And, ast.Or): - return ast.BoolOp(op=op(), values=[left, right]) - elif issubclass(op, ast.cmpop): - if isinstance(left, ast.Compare) and not getattr(left, "parenform", False): - left.ops.append(op()) - left.comparators.append(right) - return left - return ast.Compare(ops=[op()], left=left, comparators=[right]) - else: - return ast.BinOp(op=op(), left=left, right=right) - - -class TokInfixRAssoc(TokBase): - @classmethod - def led(cls, p, left, t): - right = p.expr(cls.lbp - 1) - node = ast.BinOp(op=cls.ast_bin_op(), left=left, right=right) - return node - - -# Concrete tokens - - -class TokYield(TokBase): - # nbp = 3 - @classmethod - def nud(cls, p, t): - yield_from = False - if p.match("from"): - yield_from = True - value = p.match_expr(rbp=4) - if yield_from: - return ast.YieldFrom(value=value) - else: - return ast.Yield(value=value) - - -class TokComma(TokBase): - lbp = 5 - - # Tuple creation operator - @classmethod - def led(cls, p, left, t): - elts = [left] - while not p.is_delim(): - e = p.expr(5) - elts.append(e) - if not p.match(","): - break - node = ast.Tuple(elts=elts, ctx=ast.Load()) - return node - - -class TokFor(TokBase): - lbp = 7 - - @classmethod - def led(cls, p, left, t): - is_async = 0 - if t.string == "async": - is_async = 1 - t = p.tok - p.next() - assert t.string == "for" - target, expr = p.match_for_in(20) - ifs = [] - while p.match("if"): - ifs.append(p.expr(20)) - comp = ast.comprehension(target=target, iter=expr, ifs=ifs, is_async=is_async) - if isinstance(left, GenComp): - left.generators.append(comp) - return left - return GenComp(elt=left, generators=[comp]) - - -class TokLambda(TokBase): - # nbp = 10 - @classmethod - def nud(cls, p, t): - arg_spec = p.require_typedargslist(True) - p.expect(":") - body = p.expr(10) - node = ast.Lambda(args=arg_spec, body=body) - return node - - -class TokIf(TokBase): - lbp = 20 - - @classmethod - def led(cls, p, left, t): - cond = p.expr(20) - p.expect("else") - orelse = p.expr(19) - node = ast.IfExp(test=cond, body=left, orelse=orelse) - return node - - -class TokOr(TokInfix): - lbp = 30 - ast_bin_op = ast.Or - - -class TokAnd(TokInfix): - lbp = 40 - ast_bin_op = ast.And - - -class TokNot(TokPrefix, TokInfix): - nbp = 50 - ast_un_op = ast.Not - - lbp = 60 # not in - ast_bin_op = ast.NotIn - - @classmethod - def led(cls, p, left, t): - p.expect("in") - return super().led(p, left, t) - - -class TokEq(TokInfix): - lbp = 60 - ast_bin_op = ast.Eq - - -class TokNotEq(TokInfix): - lbp = 60 - ast_bin_op = ast.NotEq - - -class TokLt(TokInfix): - lbp = 60 - ast_bin_op = ast.Lt - - -class TokGt(TokInfix): - lbp = 60 - ast_bin_op = ast.Gt - - -class TokLtE(TokInfix): - lbp = 60 - ast_bin_op = ast.LtE - - -class TokGtE(TokInfix): - lbp = 60 - ast_bin_op = ast.GtE - - -class TokIs(TokInfix): - lbp = 60 - - # Overriden to handle both "is" and "is not" - @classmethod - def led(cls, p, left, t): - op = ast.Is - if p.match("not"): - op = ast.IsNot - right = p.expr(cls.lbp) - node = cls.bin_op(op, left, right) - return node - - -class TokIn(TokInfix): - lbp = 60 - ast_bin_op = ast.In - - -class TokBinOr(TokInfix): - lbp = 70 - ast_bin_op = ast.BitOr - - -class TokBinXor(TokInfix): - lbp = 80 - ast_bin_op = ast.BitXor - - -class TokBinAnd(TokInfix): - lbp = 90 - ast_bin_op = ast.BitAnd - - -class TokLShift(TokInfix): - lbp = 100 - ast_bin_op = ast.LShift - - -class TokRShift(TokInfix): - lbp = 100 - ast_bin_op = ast.RShift - - -class TokPlus(TokInfix, TokPrefix): - lbp = 110 - ast_bin_op = ast.Add - nbp = 130 - ast_un_op = ast.UAdd - - -class TokMinus(TokInfix, TokPrefix): - lbp = 110 - ast_bin_op = ast.Sub - nbp = 130 - ast_un_op = ast.USub - - -class TokMul(TokInfix): - lbp = 120 - ast_bin_op = ast.Mult - - @classmethod - def nud(cls, p, t): - value = p.expr(160 - 1) - return ast.Starred(value=value, ctx=ast.Load()) - - -class TokMatMul(TokInfix): - lbp = 120 - ast_bin_op = ast.MatMult - - -class TokDiv(TokInfix): - lbp = 120 - ast_bin_op = ast.Div - - -class TokFloorDiv(TokInfix): - lbp = 120 - ast_bin_op = ast.FloorDiv - - -class TokMod(TokInfix): - lbp = 120 - ast_bin_op = ast.Mod - - -class TokInvert(TokPrefix): - nbp = 130 - ast_un_op = ast.Invert - - -class TokPow(TokInfixRAssoc): - lbp = 140 - ast_bin_op = ast.Pow - - -class TokAwait(TokPrefix): - # nbp = 150 - @classmethod - def nud(cls, p, t): - value = p.expr(150) - return ast.Await(value=value) - - -class TokDot(TokBase): - lbp = 160 - - @classmethod - def led(cls, p, left, t): - attr = p.expect(NAME) - node = ast.Attribute(value=left, attr=attr, ctx=ast.Load()) - return node - - -class TokOpenSquare(TokBase): - lbp = 160 - - @classmethod - def led(cls, p, left, t): - dims = [] - rbp = 0 - while True: - idx = p.match_expr(rbp=rbp) - if p.match(":"): - upper = p.match_expr(rbp=BP_UNTIL_COMMA) - step = None - if p.match(":"): - step = p.match_expr(rbp=BP_UNTIL_COMMA) - slc = ast.Slice(lower=idx, upper=upper, step=step) - else: - slc = ast.Index(value=idx) - dims.append(slc) - if not p.match(","): - break - rbp = BP_UNTIL_COMMA - p.expect("]") - if len(dims) == 1: - node = ast.Subscript(value=left, slice=slc, ctx=ast.Load()) - else: - node = ast.Subscript( - value=left, slice=ast.ExtSlice(dims=dims), ctx=ast.Load() - ) - return node - - @classmethod - def nud(cls, p, t): - elts = [] - while not p.match("]"): - val = p.expr(BP_UNTIL_COMMA) - if isinstance(val, GenComp): - p.expect("]") - return ast.ListComp(elt=val.elt, generators=val.generators) - - elts.append(val) - p.match(",") - node = ast.List(elts=elts, ctx=ast.Load()) - return node - - -class TokOpenBrace(TokBase): - @classmethod - def nud(cls, p, t): - keys = [] - vals = [] - # dict or set - is_dict = None - while not p.match("}"): - if p.match("**"): - is_dict = True - v = p.expr(BP_UNTIL_COMMA) - keys.append(None) - vals.append(v) - p.match(",") - continue - - k = p.expr(BP_UNTIL_COMMA) - if isinstance(k, GenComp): - p.expect("}") - return ast.SetComp(elt=k.elt, generators=k.generators) - - keys.append(k) - if is_dict is None: - is_dict = bool(p.check(":")) - if is_dict: - p.expect(":") - v = p.expr(BP_UNTIL_COMMA) - if isinstance(v, GenComp): - p.expect("}") - return ast.DictComp(key=k, value=v.elt, generators=v.generators) - - vals.append(v) - p.match(",") - if is_dict or is_dict is None: - node = ast.Dict(keys=keys, values=vals) - else: - node = ast.Set(elts=keys) - return node - - -class TokOpenParens(TokBase): - lbp = 160 - - @classmethod - def led(cls, p, left, t): - args, keywords = p.match_call_args() - node = ast.Call(func=left, args=args, keywords=keywords) - return node - - # nbp = 170 - @classmethod - def nud(cls, p, t): - if p.match(")"): - # Empty tuple - return ast.Tuple(elts=[], ctx=ast.Load()) - e = p.expr() - p.expect(")") - e.parenform = True - if isinstance(e, GenComp): - return ast.GeneratorExp(elt=e.elt, generators=e.generators) - return e - - -class TokNumber(TokBase): - @classmethod - def nud(cls, p, t): - try: - v = int(t.string, 0) - except ValueError: - if t.string.endswith("j"): - v = complex(t.string) - else: - v = float(t.string) - node = ast.Num(n=v) - return node - - -class TokString(TokBase): - lbp = 200 - - # Adjacent strings - @classmethod - def led(cls, p, left, t): - assert isinstance(left, (ast.Str, ast.Bytes)) - left.s += literal_eval(t.string) - return left - - @classmethod - def nud(cls, p, t): - v = literal_eval(t.string) - if isinstance(v, bytes): - return ast.Bytes(s=v) - else: - return ast.Str(s=v) - - -class TokName(TokBase): - @classmethod - def nud(cls, p, t): - return ast.Name(id=t[TOK_STRING], ctx=ast.Load()) - - -class TokConst(TokBase): - @classmethod - def nud(cls, p, t): - return ast.NameConstant(value=cls.value) - - -class TokNone(TokConst): - value = None - - -class TokTrue(TokConst): - value = True - - -class TokFalse(TokConst): - value = False - - -class TokEllipsis(TokBase): - @classmethod - def nud(cls, p, t): - return ast.Ellipsis() - - -pratt_token_map = { - NEWLINE: TokDelim, - "=": TokDelim, - ",": TokComma, - "yield": TokYield, - "for": TokFor, - "async": TokFor, - "lambda": TokLambda, - "if": TokIf, - "else": TokDelim, - "or": TokOr, - "and": TokAnd, - "not": TokNot, - "==": TokEq, - "!=": TokNotEq, - "<": TokLt, - "<=": TokLtE, - ">": TokGt, - ">=": TokGtE, - "is": TokIs, - "in": TokIn, - "|": TokBinOr, - "^": TokBinXor, - "&": TokBinAnd, - "<<": TokLShift, - ">>": TokRShift, - "+": TokPlus, - "-": TokMinus, - "*": TokMul, - "@": TokMatMul, - "/": TokDiv, - "//": TokFloorDiv, - "%": TokMod, - "~": TokInvert, - "**": TokPow, - "await": TokAwait, - ".": TokDot, - "[": TokOpenSquare, - "]": TokDelim, - "{": TokOpenBrace, - "}": TokDelim, - "(": TokOpenParens, - ")": TokDelim, - "None": TokNone, - "True": TokTrue, - "False": TokFalse, - "...": TokEllipsis, - NUMBER: TokNumber, - STRING: TokString, -} - - -class Parser: - def __init__(self, token_stream): - self.tstream = token_stream - self.tok = None - self.next() - self.decorators = [] - - def next(self): - while True: - self.tok = next(self.tstream) - log.debug("next: %r", self.tok) - if self.tok[TOK_TYPE] not in (utokenize.COMMENT, utokenize.NL): - break - - def error(self, msg="syntax error"): - sys.stderr.write(":%d: error: %s\n" % (self.tok.start, msg)) - raise Exception - - # Recursively set "lvalue" node access context (to other value than - # default ast.Load). - @staticmethod - def set_ctx(t, ctx): - if isinstance(t, list): - for e in t: - Parser.set_ctx(e, ctx) - elif isinstance(t, ast.AST): - t.ctx = ctx - if isinstance(t, ast.Subscript): - return - for k in t._fields: - v = getattr(t, k, None) - if not (isinstance(t, ast.Attribute) and k == "value"): - Parser.set_ctx(v, ctx) - - def check(self, what): - if isinstance(what, str): - if self.tok[TOK_STRING] == what and self.tok[TOK_TYPE] in ( - utokenize.NAME, - utokenize.OP, - ): - return True - return None - - if isinstance(what, int): - if self.tok[TOK_TYPE] == what: - if what == utokenize.ENDMARKER: - return True - res = self.tok[TOK_STRING] - if res == "": - return True - return res - return None - - assert False, "Unknown value type to match" - - def match(self, what): - res = self.check(what) - if res and what != utokenize.ENDMARKER: - self.next() - return res - - def expect(self, what): - res = self.match(what) - if not res: - if isinstance(what, int): - self.error("expected %s" % utokenize.tok_name[what]) - else: - self.error("expected '%s'" % what) - return res - - def is_end_of_stmt(self): - return self.check(NEWLINE) or self.check(";") or self.check(ENDMARKER) - - def is_delim(self): - if self.is_end_of_stmt(): - return True - if self.tok[TOK_TYPE] == OP and self.tok[TOK_STRING] in ( - "]", - "}", - ")", - ":", - "=", - ): - return True - - @staticmethod - def make_name(id, ctx=None): - node = ast.Name(id=id) - if ctx: - node.ctx = ctx() - return node - - def match_funcdef(self, is_async=False): - lineno = self.tok.start - if not self.match("def"): - return - name = self.expect(NAME) - self.expect("(") - arg_spec = self.require_typedargslist() - self.expect(")") - - returns = None - if self.match("->"): - returns = self.require_expr() - - self.expect(":") - decorator_list = self.decorators - self.decorators = [] - body = self.match_suite() - if is_async: - asttype = ast.AsyncFunctionDef - else: - asttype = ast.FunctionDef - node = asttype( - name=name, - args=arg_spec, - body=body, - decorator_list=decorator_list, - returns=returns, - lineno=lineno, - ) - return node - - def match_classdef(self): - lineno = self.tok.start - if not self.match("class"): - return - name = self.expect(NAME) - bases = [] - keywords = [] - if self.match("("): - bases, keywords = self.match_call_args() - self.expect(":") - decorator_list = self.decorators - self.decorators = [] - body = self.match_suite() - return ast.ClassDef( - name=name, - body=body, - bases=bases, - keywords=keywords, - decorator_list=decorator_list, - lineno=lineno, - ) - - def match_stmt(self): - while True: - res = self.match_compound_stmt() - # True means a decorator matched - if res is not True: - break - if res: - log.debug("match_stmt: %r", res) - return [res] - res = self.match_simple_stmt() - if res: - return res - self.error("expected statement") - - def match_simple_stmt(self): - res = self.match_small_stmt() - if res is None: - return None - - body = [res] - while True: - if not self.match(";"): - break - if self.check(NEWLINE): - break - res = self.match_small_stmt() - if res is None: - break - body.append(res) - self.expect(NEWLINE) - return body - - def match_small_stmt(self): - res = self.match_import_stmt() - if res: - return res - - if self.match("break"): - return ast.Break() - if self.match("continue"): - return ast.Continue() - if self.match("pass"): - return ast.Pass() - - if self.match("return"): - expr = self.match_expr() - return ast.Return(value=expr) - - if self.match("raise"): - expr = self.match_expr() - cause = None - if self.match("from"): - cause = self.match_expr() - return ast.Raise(exc=expr, cause=cause) - - if self.match("assert"): - expr = self.match_expr(rbp=BP_UNTIL_COMMA) - msg = None - if self.match(","): - msg = self.match_expr(rbp=BP_UNTIL_COMMA) - return ast.Assert(test=expr, msg=msg) - - if self.match("del"): - exprs = self.match_exprlist(ctx=ast.Del) - return ast.Delete(targets=exprs) - - if self.match("global"): - names = self.match_namelist() - return ast.Global(names=names) - if self.match("nonlocal"): - names = self.match_namelist() - return ast.Nonlocal(names=names) - - res = self.match_expr() - if not res: - return None - - ann = None - if self.match(":"): - ann = self.match_expr() - - if self.check("="): - targets = [] - while self.match("="): - self.set_ctx(res, ast.Store()) - targets.append(res) - res = self.match_expr() - if ann is None: - return ast.Assign(targets=targets, value=res) - else: - assert len(targets) == 1 - return ast.AnnAssign( - target=targets[0], annotation=ann, value=res, simple=1 - ) - - elif self.check(OP) and self.tok.string.endswith("="): - self.set_ctx(res, ast.Store()) - op_type = { - "+=": ast.Add, - "-=": ast.Sub, - "*=": ast.Mult, - "/=": ast.Div, - "//=": ast.FloorDiv, - "%=": ast.Mod, - "**=": ast.Pow, - "@=": ast.MatMult, - "|=": ast.BitOr, - "^=": ast.BitXor, - "&=": ast.BitAnd, - "<<=": ast.LShift, - ">>=": ast.RShift, - }[self.tok.string] - self.next() - val = self.match_expr() - return ast.AugAssign(target=res, op=op_type(), value=val) - - if ann is None: - return ast.Expr(value=res) - else: - self.set_ctx(res, ast.Store()) - return ast.AnnAssign(target=res, annotation=ann, value=None, simple=1) - - def match_compound_stmt(self): - if self.match("@"): - decor = self.match_expr(rbp=BP_LVALUE) - self.expect(NEWLINE) - self.decorators.append(decor) - return True - - is_async = False - if self.match("async"): - is_async = True - - res = self.match_funcdef(is_async) - if res: - return res - res = self.match_for_stmt(is_async) - if res: - return res - res = self.match_with_stmt(is_async) - if res: - return res - - if is_async: - self.error("Unexpected async keyword") - - res = self.match_classdef() - if res: - return res - - if self.decorators: - self.error("Unexpected decorator") - - res = self.match_if_stmt() - if res: - return res - res = self.match_while_stmt() - if res: - return res - res = self.match_try_stmt() - if res: - return res - return None - - def match_suite(self): - if self.match(NEWLINE): - self.expect(INDENT) - body = [] - while self.match(DEDENT) is None: - body.extend(self.match_stmt()) - return body - else: - return self.match_simple_stmt() - - def match_import_stmt(self): - if self.match("import"): - names = [] - while True: - name = self.match_dotted_name() - asname = None - if self.match("as"): - asname = self.expect(NAME) - names.append(ast.alias(name=name, asname=asname)) - if not self.match(","): - break - return ast.Import(names=names) - elif self.match("from"): - level = 0 - while True: - if self.match("."): - level += 1 - # "..." is a single token (ellipsis) - elif self.match("..."): - level += 3 - else: - break - module = None - if not self.check("import"): - module = self.match_dotted_name() - self.expect("import") - names = [] - if self.match("*"): - name = "*" - names.append(ast.alias(name="*", asname=None)) - else: - is_paren = self.match("(") - while True: - name = self.expect(NAME) - asname = None - if self.match("as"): - asname = self.expect(NAME) - names.append(ast.alias(name=name, asname=asname)) - if not self.match(","): - break - if self.check(")"): - break - if is_paren: - self.match(")") - return ast.ImportFrom(module=module, names=names, level=level) - - def match_if_stmt(self): - def handle_if(): - expr = self.require_expr() - self.expect(":") - body = self.match_suite() - orelse = [] - - if self.match("elif"): - orelse = [handle_if()] - - elif self.match("else"): - self.expect(":") - orelse = self.match_suite() - - return ast.If(test=expr, body=body, orelse=orelse) - - if self.match("if"): - return handle_if() - - def match_for_in(self, rbp=0): - target = self.match_expr(ctx=ast.Store, rbp=BP_LVALUE) - if self.check(","): - target = ast.Tuple(elts=[target], ctx=ast.Store()) - while self.match(","): - if self.check("in"): - break - target.elts.append(self.match_expr(ctx=ast.Store, rbp=BP_LVALUE)) - self.expect("in") - expr = self.require_expr(rbp=rbp) - return target, expr - - def match_for_stmt(self, is_async=False): - if not self.match("for"): - return None - target, expr = self.match_for_in() - self.expect(":") - body = self.match_suite() - orelse = [] - if self.match("else"): - self.expect(":") - orelse = self.match_suite() - if is_async: - asttype = ast.AsyncFor - else: - asttype = ast.For - return asttype(target=target, iter=expr, body=body, orelse=orelse) - - def match_while_stmt(self): - if not self.match("while"): - return None - expr = self.require_expr() - self.expect(":") - body = self.match_suite() - orelse = [] - if self.match("else"): - self.expect(":") - orelse = self.match_suite() - return ast.While(test=expr, body=body, orelse=orelse) - - def match_with_stmt(self, is_async=False): - if not self.match("with"): - return None - items = [] - while True: - expr = self.require_expr(rbp=BP_UNTIL_COMMA) - asname = None - if self.match("as"): - asname = self.match_expr(rbp=BP_UNTIL_COMMA) - self.set_ctx(asname, ast.Store()) - items.append(ast.withitem(context_expr=expr, optional_vars=asname)) - if not self.match(","): - break - self.expect(":") - body = self.match_suite() - if is_async: - asttype = ast.AsyncWith - else: - asttype = ast.With - return asttype(items=items, body=body) - - def match_try_stmt(self): - if not self.match("try"): - return None - self.expect(":") - body = self.match_suite() - - handlers = [] - while self.match("except"): - exc_sel = None - capture_var = None - if not self.match(":"): - exc_sel = self.require_expr() - if self.match("as"): - capture_var = self.expect(NAME) - self.expect(":") - exc_body = self.match_suite() - handlers.append( - ast.ExceptHandler(type=exc_sel, name=capture_var, body=exc_body) - ) - - orelse = [] - if self.match("else"): - self.expect(":") - orelse = self.match_suite() - - finalbody = [] - if self.match("finally"): - self.expect(":") - finalbody = self.match_suite() - - return ast.Try(body=body, handlers=handlers, orelse=orelse, finalbody=finalbody) - - @staticmethod - def get_token_class(t): - cls = pratt_token_map.get(t[TOK_TYPE]) - if cls: - return cls - cls = pratt_token_map.get(t[TOK_STRING]) - if cls: - return cls - return TokName - - def expr(self, rbp=0): - t = self.tok - self.next() - cls_nud = self.get_token_class(t) - left = cls_nud.nud(self, t) - t = self.tok - cls_led = self.get_token_class(t) - while rbp < cls_led.lbp: - self.next() - left = cls_led.led(self, left, t) - t = self.tok - cls_led = self.get_token_class(t) - return left - - def match_expr(self, ctx=None, rbp=0): - # Adhoc, consider making suitable TokDelim.nud() return None - if self.is_delim(): - return None - if rbp >= BP_UNTIL_COMMA and self.check(","): - return None - - n = self.expr(rbp) - if ctx: - self.set_ctx(n, ctx()) - return n - - def require_expr(self, ctx=None, rbp=0): - res = self.match_expr(ctx, rbp) - if res is not None: - return res - self.error("expected expression") - - def match_exprlist(self, ctx=None): - res = [] - while True: - expr = self.match_expr(ctx, rbp=BP_UNTIL_COMMA) - if not expr: - break - res.append(expr) - if not self.match(","): - break - assert res - return res - - def match_namelist(self): - res = [] - while True: - name = self.expect(NAME) - res.append(name) - if not self.match(","): - break - return res - - def require_typedargslist(self, is_lambda=False): - args = [] - defaults = [] - kwonlyargs = [] - kw_defaults = [] - vararg = None - kwarg = None - kwonly = False - # TODO: This is somewhat adhoc, relies on terminating token for funcdef vs lambda - while not self.check(")") and not self.check(":"): - if self.match("*"): - kwonly = True - if self.match(","): - continue - if vararg: - self.error(">1 vararg") - vararg = True - elif self.match("**"): - if kwarg: - self.error(">1 kwarg") - kwarg = True - arg = self.expect(NAME) - arg = ast.arg(arg=arg, annotation=None) - if not is_lambda and self.match(":"): - arg.annotation = self.require_expr(rbp=BP_UNTIL_COMMA) - - if vararg is True: - vararg = arg - self.match(",") - continue - elif kwarg is True: - kwarg = arg - self.match(",") - continue - if kwonly: - kwonlyargs.append(arg) - else: - args.append(arg) - if self.match("="): - dflt = self.require_expr(rbp=BP_UNTIL_COMMA) - if kwonly: - kw_defaults.append(dflt) - else: - defaults.append(dflt) - elif kwonly: - kw_defaults.append(None) - self.match(",") - - arg_spec = ast.arguments( - args=args, - vararg=vararg, - kwonlyargs=kwonlyargs, - kw_defaults=kw_defaults, - kwarg=kwarg, - defaults=defaults, - ) - return arg_spec - - def match_call_args(self): - args = [] - keywords = [] - if not self.check(")"): - while True: - starred = None - if self.match("*"): - starred = "*" - elif self.match("**"): - starred = "**" - arg = self.expr(BP_UNTIL_COMMA) - if isinstance(arg, GenComp): - arg = ast.GeneratorExp(elt=arg.elt, generators=arg.generators) - if self.match("="): - assert isinstance(arg, ast.Name) - val = self.expr(BP_UNTIL_COMMA) - keywords.append(ast.keyword(arg=arg.id, value=val)) - else: - if starred == "**": - keywords.append(ast.keyword(arg=None, value=arg)) - else: - if starred == "*": - arg = ast.Starred(value=arg, ctx=ast.Load()) - args.append(arg) - self.match(",") - if self.check(")"): - break - self.expect(")") - return args, keywords - - def match_dotted_name(self): - name = self.match(NAME) - if name is None: - return None - while self.match("."): - name += "." + self.expect(NAME) - return name - - def match_mod(self): - body = [] - while not self.match(ENDMARKER): - stmt = self.match_stmt() - body.extend(stmt) - return ast.Module(body=body) diff --git a/components/uast/src/python/uast/types.py b/components/uast/src/python/uast/types.py deleted file mode 100644 index c866015..0000000 --- a/components/uast/src/python/uast/types.py +++ /dev/null @@ -1,467 +0,0 @@ -# (c) 2019 Paul Sokolovsky. MIT license. - - -class AST: - def __init__(self, **fields): - for k, v in fields.items(): - setattr(self, k, v) - - -class mod(AST): - pass - - -class Module(mod): - _fields = ("body",) - - -class Interactive(mod): - _fields = ("body",) - - -class Expression(mod): - _fields = ("body",) - - -class Suite(mod): - _fields = ("body",) - - -class stmt(AST): - pass - - -class FunctionDef(stmt): - _fields = ("name", "args", "body", "decorator_list", "returns") - - -class AsyncFunctionDef(stmt): - _fields = ("name", "args", "body", "decorator_list", "returns") - - -class ClassDef(stmt): - _fields = ("name", "bases", "keywords", "body", "decorator_list") - - -class Return(stmt): - _fields = ("value",) - - -class Delete(stmt): - _fields = ("targets",) - - -class Assign(stmt): - _fields = ("targets", "value") - - -class AugAssign(stmt): - _fields = ("target", "op", "value") - - -class AnnAssign(stmt): - _fields = ("target", "annotation", "value", "simple") - - -class For(stmt): - _fields = ("target", "iter", "body", "orelse") - - -class AsyncFor(stmt): - _fields = ("target", "iter", "body", "orelse") - - -class While(stmt): - _fields = ("test", "body", "orelse") - - -class If(stmt): - _fields = ("test", "body", "orelse") - - -class With(stmt): - _fields = ("items", "body") - - -class AsyncWith(stmt): - _fields = ("items", "body") - - -class Raise(stmt): - _fields = ("exc", "cause") - - -class Try(stmt): - _fields = ("body", "handlers", "orelse", "finalbody") - - -class Assert(stmt): - _fields = ("test", "msg") - - -class Import(stmt): - _fields = ("names",) - - -class ImportFrom(stmt): - _fields = ("module", "names", "level") - - -class Global(stmt): - _fields = ("names",) - - -class Nonlocal(stmt): - _fields = ("names",) - - -class Expr(stmt): - _fields = ("value",) - - -class Pass(stmt): - _fields = () - - -class Break(stmt): - _fields = () - - -class Continue(stmt): - _fields = () - - -class expr(AST): - pass - - -class BoolOp(expr): - _fields = ("op", "values") - - -class BinOp(expr): - _fields = ("left", "op", "right") - - -class UnaryOp(expr): - _fields = ("op", "operand") - - -class Lambda(expr): - _fields = ("args", "body") - - -class IfExp(expr): - _fields = ("test", "body", "orelse") - - -class Dict(expr): - _fields = ("keys", "values") - - -class Set(expr): - _fields = ("elts",) - - -class ListComp(expr): - _fields = ("elt", "generators") - - -class SetComp(expr): - _fields = ("elt", "generators") - - -class DictComp(expr): - _fields = ("key", "value", "generators") - - -class GeneratorExp(expr): - _fields = ("elt", "generators") - - -class Await(expr): - _fields = ("value",) - - -class Yield(expr): - _fields = ("value",) - - -class YieldFrom(expr): - _fields = ("value",) - - -class Compare(expr): - _fields = ("left", "ops", "comparators") - - -class Call(expr): - _fields = ("func", "args", "keywords") - - -class Num(expr): - _fields = ("n",) - - -class Str(expr): - _fields = ("s",) - - -class FormattedValue(expr): - _fields = ("value", "conversion", "format_spec") - - -class JoinedStr(expr): - _fields = ("values",) - - -class Bytes(expr): - _fields = ("s",) - - -class NameConstant(expr): - _fields = ("value",) - - -class Ellipsis(expr): - _fields = () - - -class Constant(expr): - _fields = ("value",) - - -class Attribute(expr): - _fields = ("value", "attr", "ctx") - - -class Subscript(expr): - _fields = ("value", "slice", "ctx") - - -class Starred(expr): - _fields = ("value", "ctx") - - -class Name(expr): - _fields = ("id", "ctx") - - -class List(expr): - _fields = ("elts", "ctx") - - -class Tuple(expr): - _fields = ("elts", "ctx") - - -class expr_context(AST): - pass - - -class Load(expr_context): - _fields = () - - -class Store(expr_context): - _fields = () - - -class StoreConst(expr_context): - _fields = () - - -class Del(expr_context): - _fields = () - - -class AugLoad(expr_context): - _fields = () - - -class AugStore(expr_context): - _fields = () - - -class Param(expr_context): - _fields = () - - -class slice(AST): - pass - - -class Slice(slice): - _fields = ("lower", "upper", "step") - - -class ExtSlice(slice): - _fields = ("dims",) - - -class Index(slice): - _fields = ("value",) - - -class boolop(AST): - pass - - -class And(boolop): - _fields = () - - -class Or(boolop): - _fields = () - - -class operator(AST): - pass - - -class Add(operator): - _fields = () - - -class Sub(operator): - _fields = () - - -class Mult(operator): - _fields = () - - -class MatMult(operator): - _fields = () - - -class Div(operator): - _fields = () - - -class Mod(operator): - _fields = () - - -class Pow(operator): - _fields = () - - -class LShift(operator): - _fields = () - - -class RShift(operator): - _fields = () - - -class BitOr(operator): - _fields = () - - -class BitXor(operator): - _fields = () - - -class BitAnd(operator): - _fields = () - - -class FloorDiv(operator): - _fields = () - - -class unaryop(AST): - pass - - -class Invert(unaryop): - _fields = () - - -class Not(unaryop): - _fields = () - - -class UAdd(unaryop): - _fields = () - - -class USub(unaryop): - _fields = () - - -class cmpop(AST): - pass - - -class Eq(cmpop): - _fields = () - - -class NotEq(cmpop): - _fields = () - - -class Lt(cmpop): - _fields = () - - -class LtE(cmpop): - _fields = () - - -class Gt(cmpop): - _fields = () - - -class GtE(cmpop): - _fields = () - - -class Is(cmpop): - _fields = () - - -class IsNot(cmpop): - _fields = () - - -class In(cmpop): - _fields = () - - -class NotIn(cmpop): - _fields = () - - -class comprehension(AST): - _fields = ("target", "iter", "ifs", "is_async") - - -class excepthandler(AST): - pass - - -class ExceptHandler(excepthandler): - _fields = ("type", "name", "body") - - -class arguments(AST): - _fields = ("args", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults") - - -class arg(AST): - _fields = ("arg", "annotation") - - -class keyword(AST): - _fields = ("arg", "value") - - -class alias(AST): - _fields = ("name", "asname") - - -class withitem(AST): - _fields = ("context_expr", "optional_vars") diff --git a/components/uast/test/python/test_ast.py b/components/uast/test/python/test_ast.py deleted file mode 100644 index 7c9a57c..0000000 --- a/components/uast/test/python/test_ast.py +++ /dev/null @@ -1,2702 +0,0 @@ -import ast -import builtins -import dis -import enum -import os -import sys -import types -import unittest -import warnings -import weakref -from textwrap import dedent - -from test import support -from test.support.ast_helper import ASTTestMixin - -def to_tuple(t): - if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis: - return t - elif isinstance(t, list): - return [to_tuple(e) for e in t] - result = [t.__class__.__name__] - if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): - result.append((t.lineno, t.col_offset)) - if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): - result[-1] += (t.end_lineno, t.end_col_offset) - if t._fields is None: - return tuple(result) - for f in t._fields: - result.append(to_tuple(getattr(t, f))) - return tuple(result) - - -# These tests are compiled through "exec" -# There should be at least one test per statement -exec_tests = [ - # None - "None", - # Module docstring - "'module docstring'", - # FunctionDef - "def f(): pass", - # FunctionDef with docstring - "def f(): 'function docstring'", - # FunctionDef with arg - "def f(a): pass", - # FunctionDef with arg and default value - "def f(a=0): pass", - # FunctionDef with varargs - "def f(*args): pass", - # FunctionDef with varargs as TypeVarTuple - "def f(*args: *Ts): pass", - # FunctionDef with varargs as unpacked Tuple - "def f(*args: *tuple[int, ...]): pass", - # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple - "def f(*args: *tuple[int, *Ts]): pass", - # FunctionDef with kwargs - "def f(**kwargs): pass", - # FunctionDef with all kind of args and docstring - "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", - # FunctionDef with type annotation on return involving unpacking - "def f() -> tuple[*Ts]: pass", - "def f() -> tuple[int, *Ts]: pass", - "def f() -> tuple[int, *tuple[int, ...]]: pass", - # ClassDef - "class C:pass", - # ClassDef with docstring - "class C: 'docstring for class C'", - # ClassDef, new style class - "class C(object): pass", - # Return - "def f():return 1", - # Delete - "del v", - # Assign - "v = 1", - "a,b = c", - "(a,b) = c", - "[a,b] = c", - # AnnAssign with unpacked types - "x: tuple[*Ts]", - "x: tuple[int, *Ts]", - "x: tuple[int, *tuple[str, ...]]", - # AugAssign - "v += 1", - # For - "for v in v:pass", - # While - "while v:pass", - # If - "if v:pass", - # If-Elif - "if a:\n pass\nelif b:\n pass", - # If-Elif-Else - "if a:\n pass\nelif b:\n pass\nelse:\n pass", - # With - "with x as y: pass", - "with x as y, z as q: pass", - # Raise - "raise Exception('string')", - # TryExcept - "try:\n pass\nexcept Exception:\n pass", - # TryFinally - "try:\n pass\nfinally:\n pass", - # TryStarExcept - "try:\n pass\nexcept* Exception:\n pass", - # Assert - "assert v", - # Import - "import sys", - # ImportFrom - "from sys import v", - # Global - "global v", - # Expr - "1", - # Pass, - "pass", - # Break - "for v in v:break", - # Continue - "for v in v:continue", - # for statements with naked tuples (see http://bugs.python.org/issue6704) - "for a,b in c: pass", - "for (a,b) in c: pass", - "for [a,b] in c: pass", - # Multiline generator expression (test for .lineno & .col_offset) - """( - ( - Aa - , - Bb - ) - for - Aa - , - Bb in Cc - )""", - # dictcomp - "{a : b for w in x for m in p if g}", - # dictcomp with naked tuple - "{a : b for v,w in x}", - # setcomp - "{r for l in x if g}", - # setcomp with naked tuple - "{r for l,m in x}", - # AsyncFunctionDef - "async def f():\n 'async function'\n await something()", - # AsyncFor - "async def f():\n async for e in i: 1\n else: 2", - # AsyncWith - "async def f():\n async with a as b: 1", - # PEP 448: Additional Unpacking Generalizations - "{**{1:2}, 2:3}", - "{*{1, 2}, 3}", - # Asynchronous comprehensions - "async def f():\n [i async for b in c]", - # Decorated FunctionDef - "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", - # Decorated AsyncFunctionDef - "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", - # Decorated ClassDef - "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", - # Decorator with generator argument - "@deco(a for a in b)\ndef f(): pass", - # Decorator with attribute - "@a.b.c\ndef f(): pass", - # Simple assignment expression - "(a := 1)", - # Positional-only arguments - "def f(a, /,): pass", - "def f(a, /, c, d, e): pass", - "def f(a, /, c, *, d, e): pass", - "def f(a, /, c, *, d, e, **kwargs): pass", - # Positional-only arguments with defaults - "def f(a=1, /,): pass", - "def f(a=1, /, b=2, c=4): pass", - "def f(a=1, /, b=2, *, c=4): pass", - "def f(a=1, /, b=2, *, c): pass", - "def f(a=1, /, b=2, *, c=4, **kwargs): pass", - "def f(a=1, /, b=2, *, c, **kwargs): pass", - -] - -# These are compiled through "single" -# because of overlap with "eval", it just tests what -# can't be tested with "eval" -single_tests = [ - "1+2" -] - -# These are compiled through "eval" -# It should test all expressions -eval_tests = [ - # None - "None", - # BoolOp - "a and b", - # BinOp - "a + b", - # UnaryOp - "not v", - # Lambda - "lambda:None", - # Dict - "{ 1:2 }", - # Empty dict - "{}", - # Set - "{None,}", - # Multiline dict (test for .lineno & .col_offset) - """{ - 1 - : - 2 - }""", - # ListComp - "[a for b in c if d]", - # GeneratorExp - "(a for b in c if d)", - # Comprehensions with multiple for targets - "[(a,b) for a,b in c]", - "[(a,b) for (a,b) in c]", - "[(a,b) for [a,b] in c]", - "{(a,b) for a,b in c}", - "{(a,b) for (a,b) in c}", - "{(a,b) for [a,b] in c}", - "((a,b) for a,b in c)", - "((a,b) for (a,b) in c)", - "((a,b) for [a,b] in c)", - # Yield - yield expressions can't work outside a function - # - # Compare - "1 < 2 < 3", - # Call - "f(1,2,c=3,*d,**e)", - # Call with multi-character starred - "f(*[0, 1])", - # Call with a generator argument - "f(a for a in b)", - # Num - "10", - # Str - "'string'", - # Attribute - "a.b", - # Subscript - "a[b:c]", - # Name - "v", - # List - "[1,2,3]", - # Empty list - "[]", - # Tuple - "1,2,3", - # Tuple - "(1,2,3)", - # Empty tuple - "()", - # Combination - "a.b.c.d(a.b[1:2])", - -] - -# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension -# excepthandler, arguments, keywords, alias - -class AST_Tests(unittest.TestCase): - - def _is_ast_node(self, name, node): - if not isinstance(node, type): - return False - if "ast" not in node.__module__: - return False - return name != 'AST' and name[0].isupper() - - def _assertTrueorder(self, ast_node, parent_pos): - if not isinstance(ast_node, ast.AST) or ast_node._fields is None: - return - if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)): - node_pos = (ast_node.lineno, ast_node.col_offset) - self.assertGreaterEqual(node_pos, parent_pos) - parent_pos = (ast_node.lineno, ast_node.col_offset) - for name in ast_node._fields: - value = getattr(ast_node, name) - if isinstance(value, list): - first_pos = parent_pos - if value and name == 'decorator_list': - first_pos = (value[0].lineno, value[0].col_offset) - for child in value: - self._assertTrueorder(child, first_pos) - elif value is not None: - self._assertTrueorder(value, parent_pos) - self.assertEqual(ast_node._fields, ast_node.__match_args__) - - def test_AST_objects(self): - x = ast.AST() - self.assertEqual(x._fields, ()) - x.foobar = 42 - self.assertEqual(x.foobar, 42) - self.assertEqual(x.__dict__["foobar"], 42) - - with self.assertRaises(AttributeError): - x.vararg - - with self.assertRaises(TypeError): - # "ast.AST constructor takes 0 positional arguments" - ast.AST(2) - - def test_AST_garbage_collection(self): - class X: - pass - a = ast.AST() - a.x = X() - a.x.a = a - ref = weakref.ref(a.x) - del a - support.gc_collect() - self.assertIsNone(ref()) - - def test_snippets(self): - for input, output, kind in ((exec_tests, exec_results, "exec"), - (single_tests, single_results, "single"), - (eval_tests, eval_results, "eval")): - for i, o in zip(input, output): - with self.subTest(action="parsing", input=i): - ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) - self.assertEqual(to_tuple(ast_tree), o) - self._assertTrueorder(ast_tree, (0, 0)) - with self.subTest(action="compiling", input=i, kind=kind): - compile(ast_tree, "?", kind) - - def test_ast_validation(self): - # compile() is the only function that calls PyAST_Validate - snippets_to_validate = exec_tests + single_tests + eval_tests - for snippet in snippets_to_validate: - tree = ast.parse(snippet) - compile(tree, '', 'exec') - - def test_invalid_position_information(self): - invalid_linenos = [ - (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) - ] - - for lineno, end_lineno in invalid_linenos: - with self.subTest(f"Check invalid linenos {lineno}:{end_lineno}"): - snippet = "a = 1" - tree = ast.parse(snippet) - tree.body[0].lineno = lineno - tree.body[0].end_lineno = end_lineno - with self.assertRaises(ValueError): - compile(tree, '', 'exec') - - invalid_col_offsets = [ - (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) - ] - for col_offset, end_col_offset in invalid_col_offsets: - with self.subTest(f"Check invalid col_offset {col_offset}:{end_col_offset}"): - snippet = "a = 1" - tree = ast.parse(snippet) - tree.body[0].col_offset = col_offset - tree.body[0].end_col_offset = end_col_offset - with self.assertRaises(ValueError): - compile(tree, '', 'exec') - - def test_compilation_of_ast_nodes_with_default_end_position_values(self): - tree = ast.Module(body=[ - ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), - ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) - ], type_ignores=[]) - - # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. - compile(tree, "", "exec") - - def test_slice(self): - slc = ast.parse("x[::]").body[0].value.slice - self.assertIsNone(slc.upper) - self.assertIsNone(slc.lower) - self.assertIsNone(slc.step) - - def test_from_import(self): - im = ast.parse("from . import y").body[0] - self.assertIsNone(im.module) - - def test_non_interned_future_from_ast(self): - mod = ast.parse("from __future__ import division") - self.assertIsInstance(mod.body[0], ast.ImportFrom) - mod.body[0].module = " __future__ ".strip() - compile(mod, "", "exec") - - def test_alias(self): - im = ast.parse("from bar import y").body[0] - self.assertEqual(len(im.names), 1) - alias = im.names[0] - self.assertEqual(alias.name, 'y') - self.assertIsNone(alias.asname) - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 17) - - im = ast.parse("from bar import *").body[0] - alias = im.names[0] - self.assertEqual(alias.name, '*') - self.assertIsNone(alias.asname) - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 17) - - im = ast.parse("from bar import y as z").body[0] - alias = im.names[0] - self.assertEqual(alias.name, "y") - self.assertEqual(alias.asname, "z") - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 22) - - im = ast.parse("import bar as foo").body[0] - alias = im.names[0] - self.assertEqual(alias.name, "bar") - self.assertEqual(alias.asname, "foo") - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 7) - self.assertEqual(alias.end_col_offset, 17) - - def test_base_classes(self): - self.assertTrue(issubclass(ast.For, ast.stmt)) - self.assertTrue(issubclass(ast.Name, ast.expr)) - self.assertTrue(issubclass(ast.stmt, ast.AST)) - self.assertTrue(issubclass(ast.expr, ast.AST)) - self.assertTrue(issubclass(ast.comprehension, ast.AST)) - self.assertTrue(issubclass(ast.Gt, ast.AST)) - - def test_field_attr_existence(self): - for name, item in ast.__dict__.items(): - if self._is_ast_node(name, item): - if name == 'Index': - # Index(value) just returns value now. - # The argument is required. - continue - x = item() - if isinstance(x, ast.AST): - self.assertEqual(type(x._fields), tuple) - - def test_arguments(self): - x = ast.arguments() - self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', - 'kw_defaults', 'kwarg', 'defaults')) - - with self.assertRaises(AttributeError): - x.args - self.assertIsNone(x.vararg) - - x = ast.arguments(*range(1, 8)) - self.assertEqual(x.args, 2) - self.assertEqual(x.vararg, 3) - - def test_field_attr_writable(self): - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - - def test_classattrs(self): - x = ast.Num() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) - with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): - ast.Constant(1, value=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual(ast.Constant(42).value, 42) - self.assertEqual(ast.Constant(4.25).value, 4.25) - self.assertEqual(ast.Constant(4.25j).value, 4.25j) - self.assertEqual(ast.Constant('42').value, '42') - self.assertEqual(ast.Constant(b'42').value, b'42') - self.assertIs(ast.Constant(True).value, True) - self.assertIs(ast.Constant(False).value, False) - self.assertIs(ast.Constant(None).value, None) - self.assertIs(ast.Constant(...).value, ...) - - def test_realtype(self): - self.assertEqual(type(ast.Num(42)), ast.Constant) - self.assertEqual(type(ast.Num(4.25)), ast.Constant) - self.assertEqual(type(ast.Num(4.25j)), ast.Constant) - self.assertEqual(type(ast.Str('42')), ast.Constant) - self.assertEqual(type(ast.Bytes(b'42')), ast.Constant) - self.assertEqual(type(ast.NameConstant(True)), ast.Constant) - self.assertEqual(type(ast.NameConstant(False)), ast.Constant) - self.assertEqual(type(ast.NameConstant(None)), ast.Constant) - self.assertEqual(type(ast.Ellipsis()), ast.Constant) - - def test_isinstance(self): - self.assertTrue(isinstance(ast.Num(42), ast.Num)) - self.assertTrue(isinstance(ast.Num(4.2), ast.Num)) - self.assertTrue(isinstance(ast.Num(4.2j), ast.Num)) - self.assertTrue(isinstance(ast.Str('42'), ast.Str)) - self.assertTrue(isinstance(ast.Bytes(b'42'), ast.Bytes)) - self.assertTrue(isinstance(ast.NameConstant(True), ast.NameConstant)) - self.assertTrue(isinstance(ast.NameConstant(False), ast.NameConstant)) - self.assertTrue(isinstance(ast.NameConstant(None), ast.NameConstant)) - self.assertTrue(isinstance(ast.Ellipsis(), ast.Ellipsis)) - - self.assertTrue(isinstance(ast.Constant(42), ast.Num)) - self.assertTrue(isinstance(ast.Constant(4.2), ast.Num)) - self.assertTrue(isinstance(ast.Constant(4.2j), ast.Num)) - self.assertTrue(isinstance(ast.Constant('42'), ast.Str)) - self.assertTrue(isinstance(ast.Constant(b'42'), ast.Bytes)) - self.assertTrue(isinstance(ast.Constant(True), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(False), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(None), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(...), ast.Ellipsis)) - - self.assertFalse(isinstance(ast.Str('42'), ast.Num)) - self.assertFalse(isinstance(ast.Num(42), ast.Str)) - self.assertFalse(isinstance(ast.Str('42'), ast.Bytes)) - self.assertFalse(isinstance(ast.Num(42), ast.NameConstant)) - self.assertFalse(isinstance(ast.Num(42), ast.Ellipsis)) - self.assertFalse(isinstance(ast.NameConstant(True), ast.Num)) - self.assertFalse(isinstance(ast.NameConstant(False), ast.Num)) - - self.assertFalse(isinstance(ast.Constant('42'), ast.Num)) - self.assertFalse(isinstance(ast.Constant(42), ast.Str)) - self.assertFalse(isinstance(ast.Constant('42'), ast.Bytes)) - self.assertFalse(isinstance(ast.Constant(42), ast.NameConstant)) - self.assertFalse(isinstance(ast.Constant(42), ast.Ellipsis)) - self.assertFalse(isinstance(ast.Constant(True), ast.Num)) - self.assertFalse(isinstance(ast.Constant(False), ast.Num)) - - self.assertFalse(isinstance(ast.Constant(), ast.Num)) - self.assertFalse(isinstance(ast.Constant(), ast.Str)) - self.assertFalse(isinstance(ast.Constant(), ast.Bytes)) - self.assertFalse(isinstance(ast.Constant(), ast.NameConstant)) - self.assertFalse(isinstance(ast.Constant(), ast.Ellipsis)) - - class S(str): pass - self.assertTrue(isinstance(ast.Constant(S('42')), ast.Str)) - self.assertFalse(isinstance(ast.Constant(S('42')), ast.Num)) - - def test_subclasses(self): - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, 'spam') - self.assertEqual(type(n), N) - self.assertTrue(isinstance(n, N)) - self.assertTrue(isinstance(n, ast.Num)) - self.assertFalse(isinstance(n, N2)) - self.assertFalse(isinstance(ast.Num(42), N)) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertEqual(type(n), N) - - def test_module(self): - body = [ast.Num(42)] - x = ast.Module(body, []) - self.assertEqual(x.body, body) - - def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed - x = ast.BinOp() - self.assertEqual(x._fields, ('left', 'op', 'right')) - - # Random attribute allowed too - x.foobarbaz = 5 - self.assertEqual(x.foobarbaz, 5) - - n1 = ast.Num(1) - n3 = ast.Num(3) - addop = ast.Add() - x = ast.BinOp(n1, addop, n3) - self.assertEqual(x.left, n1) - self.assertEqual(x.op, addop) - self.assertEqual(x.right, n3) - - x = ast.BinOp(1, 2, 3) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - - x = ast.BinOp(1, 2, 3, lineno=0) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - self.assertEqual(x.lineno, 0) - - # node raises exception when given too many arguments - self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4) - # node raises exception when given too many arguments - self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4, lineno=0) - - # can set attributes through kwargs too - x = ast.BinOp(left=1, op=2, right=3, lineno=0) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - self.assertEqual(x.lineno, 0) - - # Random kwargs also allowed - x = ast.BinOp(1, 2, 3, foobarbaz=42) - self.assertEqual(x.foobarbaz, 42) - - def test_no_fields(self): - # this used to fail because Sub._fields was None - x = ast.Sub() - self.assertEqual(x._fields, ()) - - def test_pickling(self): - import pickle - - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) - - def test_invalid_sum(self): - pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) - with self.assertRaises(TypeError) as cm: - compile(m, "", "exec") - self.assertIn("but got ", "exec") - self.assertIn("identifier must be of type str", str(cm.exception)) - - def test_invalid_constant(self): - for invalid_constant in int, (1, 2, int), frozenset((1, 2, int)): - e = ast.Expression(body=ast.Constant(invalid_constant)) - ast.fix_missing_locations(e) - with self.assertRaisesRegex( - TypeError, "invalid type in Constant: type" - ): - compile(e, "", "eval") - - def test_empty_yield_from(self): - # Issue 16546: yield from value is not optional. - empty_yield_from = ast.parse("def f():\n yield from g()") - empty_yield_from.body[0].body[0].value.value = None - with self.assertRaises(ValueError) as cm: - compile(empty_yield_from, "", "exec") - self.assertIn("field 'value' is required", str(cm.exception)) - - @support.cpython_only - def test_issue31592(self): - # There shouldn't be an assertion failure in case of a bad - # unicodedata.normalize(). - import unicodedata - def bad_normalize(*args): - return None - with support.swap_attr(unicodedata, 'normalize', bad_normalize): - self.assertRaises(TypeError, ast.parse, '\u03D5') - - def test_issue18374_binop_col_offset(self): - tree = ast.parse('4+5+6+7') - parent_binop = tree.body[0].value - child_binop = parent_binop.left - grandchild_binop = child_binop.left - self.assertEqual(parent_binop.col_offset, 0) - self.assertEqual(parent_binop.end_col_offset, 7) - self.assertEqual(child_binop.col_offset, 0) - self.assertEqual(child_binop.end_col_offset, 5) - self.assertEqual(grandchild_binop.col_offset, 0) - self.assertEqual(grandchild_binop.end_col_offset, 3) - - tree = ast.parse('4+5-\\\n 6-7') - parent_binop = tree.body[0].value - child_binop = parent_binop.left - grandchild_binop = child_binop.left - self.assertEqual(parent_binop.col_offset, 0) - self.assertEqual(parent_binop.lineno, 1) - self.assertEqual(parent_binop.end_col_offset, 4) - self.assertEqual(parent_binop.end_lineno, 2) - - self.assertEqual(child_binop.col_offset, 0) - self.assertEqual(child_binop.lineno, 1) - self.assertEqual(child_binop.end_col_offset, 2) - self.assertEqual(child_binop.end_lineno, 2) - - self.assertEqual(grandchild_binop.col_offset, 0) - self.assertEqual(grandchild_binop.lineno, 1) - self.assertEqual(grandchild_binop.end_col_offset, 3) - self.assertEqual(grandchild_binop.end_lineno, 1) - - def test_issue39579_dotted_name_end_col_offset(self): - tree = ast.parse('@a.b.c\ndef f(): pass') - attr_b = tree.body[0].decorator_list[0].value - self.assertEqual(attr_b.end_col_offset, 4) - - def test_ast_asdl_signature(self): - self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)") - self.assertEqual(ast.GtE.__doc__, "GtE") - self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)") - self.assertEqual(ast.cmpop.__doc__, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn") - expressions = [f" | {node.__doc__}" for node in ast.expr.__subclasses__()] - expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" - self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) - - def test_positional_only_feature_version(self): - ast.parse('def foo(x, /): ...', feature_version=(3, 8)) - ast.parse('def bar(x=1, /): ...', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('def foo(x, /): ...', feature_version=(3, 7)) - with self.assertRaises(SyntaxError): - ast.parse('def bar(x=1, /): ...', feature_version=(3, 7)) - - ast.parse('lambda x, /: ...', feature_version=(3, 8)) - ast.parse('lambda x=1, /: ...', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('lambda x, /: ...', feature_version=(3, 7)) - with self.assertRaises(SyntaxError): - ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - - def test_parenthesized_with_feature_version(self): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 10)) - # While advertised as a feature in Python 3.10, this was allowed starting 3.9 - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 9)) - with self.assertRaises(SyntaxError): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 8)) - ast.parse('with CtxManager() as example: ...', feature_version=(3, 8)) - - def test_debug_f_string_feature_version(self): - ast.parse('f"{x=}"', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('f"{x=}"', feature_version=(3, 7)) - - def test_assignment_expression_feature_version(self): - ast.parse('(x := 0)', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('(x := 0)', feature_version=(3, 7)) - - def test_exception_groups_feature_version(self): - code = dedent(''' - try: ... - except* Exception: ... - ''') - ast.parse(code) - with self.assertRaises(SyntaxError): - ast.parse(code, feature_version=(3, 10)) - - def test_invalid_major_feature_version(self): - with self.assertRaises(ValueError): - ast.parse('pass', feature_version=(2, 7)) - with self.assertRaises(ValueError): - ast.parse('pass', feature_version=(4, 0)) - - def test_constant_as_name(self): - for constant in "True", "False", "None": - expr = ast.Expression(ast.Name(constant, ast.Load())) - ast.fix_missing_locations(expr) - with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): - compile(expr, "", "eval") - - def test_precedence_enum(self): - class _Precedence(enum.IntEnum): - """Precedence table that originated from python grammar.""" - NAMED_EXPR = enum.auto() # := - TUPLE = enum.auto() # , - YIELD = enum.auto() # 'yield', 'yield from' - TEST = enum.auto() # 'if'-'else', 'lambda' - OR = enum.auto() # 'or' - AND = enum.auto() # 'and' - NOT = enum.auto() # 'not' - CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', - # 'in', 'not in', 'is', 'is not' - EXPR = enum.auto() - BOR = EXPR # '|' - BXOR = enum.auto() # '^' - BAND = enum.auto() # '&' - SHIFT = enum.auto() # '<<', '>>' - ARITH = enum.auto() # '+', '-' - TERM = enum.auto() # '*', '@', '/', '%', '//' - FACTOR = enum.auto() # unary '+', '-', '~' - POWER = enum.auto() # '**' - AWAIT = enum.auto() # 'await' - ATOM = enum.auto() - def next(self): - try: - return self.__class__(self + 1) - except ValueError: - return self - enum._test_simple_enum(_Precedence, ast._Precedence) - - @support.cpython_only - def test_ast_recursion_limit(self): - fail_depth = support.EXCEEDS_RECURSION_LIMIT - crash_depth = 100_000 - success_depth = 1200 - - def check_limit(prefix, repeated): - expect_ok = prefix + repeated * success_depth - ast.parse(expect_ok) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format( - prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - with support.infinite_recursion(): - ast.parse(broken) - - check_limit("a", "()") - check_limit("a", ".b") - check_limit("a", "[0]") - check_limit("a", "*a") - - def test_null_bytes(self): - with self.assertRaises(SyntaxError, - msg="source code string cannot contain null bytes"): - ast.parse("a\0b") - -class ASTHelpers_Test(unittest.TestCase): - maxDiff = None - - def test_parse(self): - a = ast.parse('foo(1 + 1)') - b = compile('foo(1 + 1)', '', 'exec', ast.PyCF_ONLY_AST) - self.assertEqual(ast.dump(a), ast.dump(b)) - - def test_parse_in_error(self): - try: - 1/0 - except Exception: - with self.assertRaises(SyntaxError) as e: - ast.literal_eval(r"'\U'") - self.assertIsNotNone(e.exception.__context__) - - def test_dump(self): - node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual(ast.dump(node), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " - "keywords=[]))], type_ignores=[])" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))], [])" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " - "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " - "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " - "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" - ) - - def test_dump_indent(self): - node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual(ast.dump(node, indent=3), """\ -Module( - body=[ - Expr( - value=Call( - func=Name(id='spam', ctx=Load()), - args=[ - Name(id='eggs', ctx=Load()), - Constant(value='and cheese')], - keywords=[]))], - type_ignores=[])""") - self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ -Module( -\t[ -\t\tExpr( -\t\t\tCall( -\t\t\t\tName('spam', Load()), -\t\t\t\t[ -\t\t\t\t\tName('eggs', Load()), -\t\t\t\t\tConstant('and cheese')], -\t\t\t\t[]))], -\t[])""") - self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ -Module( - body=[ - Expr( - value=Call( - func=Name( - id='spam', - ctx=Load(), - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=4), - args=[ - Name( - id='eggs', - ctx=Load(), - lineno=1, - col_offset=5, - end_lineno=1, - end_col_offset=9), - Constant( - value='and cheese', - lineno=1, - col_offset=11, - end_lineno=1, - end_col_offset=23)], - keywords=[], - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=24), - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=24)], - type_ignores=[])""") - - def test_dump_incomplete(self): - node = ast.Raise(lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), - "Raise()" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(lineno=3, col_offset=4)" - ) - node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), - "Raise(exc=Name(id='e', ctx=Load()))" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(Name('e', Load()))" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" - ) - self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), - "Raise(Name('e', Load()), lineno=3, col_offset=4)" - ) - node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) - self.assertEqual(ast.dump(node), - "Raise(cause=Name(id='e', ctx=Load()))" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(cause=Name('e', Load()))" - ) - - def test_copy_location(self): - src = ast.parse('1 + 1', mode='eval') - src.body.right = ast.copy_location(ast.Num(2), src.body.right) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' - 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' - 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' - 'col_offset=0, end_lineno=1, end_col_offset=5))' - ) - src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1) - new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None)) - self.assertIsNone(new.end_lineno) - self.assertIsNone(new.end_col_offset) - self.assertEqual(new.lineno, 1) - self.assertEqual(new.col_offset, 1) - - def test_fix_missing_locations(self): - src = ast.parse('write("spam")') - src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), - [ast.Str('eggs')], []))) - self.assertEqual(src, ast.fix_missing_locations(src)) - self.maxDiff = None - self.assertEqual(ast.dump(src, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " - "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " - "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " - "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], " - "type_ignores=[])" - ) - - def test_increment_lineno(self): - src = ast.parse('1 + 1', mode='eval') - self.assertEqual(ast.increment_lineno(src, n=3), src) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' - 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' - 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' - 'col_offset=0, end_lineno=4, end_col_offset=5))' - ) - # issue10869: do not increment lineno of root twice - src = ast.parse('1 + 1', mode='eval') - self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' - 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' - 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' - 'col_offset=0, end_lineno=4, end_col_offset=5))' - ) - src = ast.Call( - func=ast.Name("test", ast.Load()), args=[], keywords=[], lineno=1 - ) - self.assertEqual(ast.increment_lineno(src).lineno, 2) - self.assertIsNone(ast.increment_lineno(src).end_lineno) - - def test_increment_lineno_on_module(self): - src = ast.parse(dedent("""\ - a = 1 - b = 2 # type: ignore - c = 3 - d = 4 # type: ignore@tag - """), type_comments=True) - ast.increment_lineno(src, n=5) - self.assertEqual(src.type_ignores[0].lineno, 7) - self.assertEqual(src.type_ignores[1].lineno, 9) - self.assertEqual(src.type_ignores[1].tag, '@tag') - - def test_iter_fields(self): - node = ast.parse('foo()', mode='eval') - d = dict(ast.iter_fields(node.body)) - self.assertEqual(d.pop('func').id, 'foo') - self.assertEqual(d, {'keywords': [], 'args': []}) - - def test_iter_child_nodes(self): - node = ast.parse("spam(23, 42, eggs='leek')", mode='eval') - self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) - iterator = ast.iter_child_nodes(node.body) - self.assertEqual(next(iterator).id, 'spam') - self.assertEqual(next(iterator).value, 23) - self.assertEqual(next(iterator).value, 42) - self.assertEqual(ast.dump(next(iterator)), - "keyword(arg='eggs', value=Constant(value='leek'))" - ) - - def test_get_docstring(self): - node = ast.parse('"""line one\n line two"""') - self.assertEqual(ast.get_docstring(node), - 'line one\nline two') - - node = ast.parse('class foo:\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), - 'line one\nline two') - - node = ast.parse('def foo():\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), - 'line one\nline two') - - node = ast.parse('async def foo():\n """spam\n ham"""') - self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') - - def test_get_docstring_none(self): - self.assertIsNone(ast.get_docstring(ast.parse(''))) - node = ast.parse('x = "not docstring"') - self.assertIsNone(ast.get_docstring(node)) - node = ast.parse('def foo():\n pass') - self.assertIsNone(ast.get_docstring(node)) - - node = ast.parse('class foo:\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('class foo:\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('class foo:\n def bar(self): pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - - node = ast.parse('def foo():\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('def foo():\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - - node = ast.parse('async def foo():\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('async def foo():\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - - def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): - node = ast.parse( - '"""line one\nline two"""\n\n' - 'def foo():\n """line one\n line two"""\n\n' - ' def bar():\n """line one\n line two"""\n' - ' """line one\n line two"""\n' - '"""line one\nline two"""\n\n' - ) - self.assertEqual(node.body[0].col_offset, 0) - self.assertEqual(node.body[0].lineno, 1) - self.assertEqual(node.body[1].body[0].col_offset, 2) - self.assertEqual(node.body[1].body[0].lineno, 5) - self.assertEqual(node.body[1].body[1].body[0].col_offset, 4) - self.assertEqual(node.body[1].body[1].body[0].lineno, 9) - self.assertEqual(node.body[1].body[2].col_offset, 2) - self.assertEqual(node.body[1].body[2].lineno, 11) - self.assertEqual(node.body[2].col_offset, 0) - self.assertEqual(node.body[2].lineno, 13) - - def test_elif_stmt_start_position(self): - node = ast.parse('if a:\n pass\nelif b:\n pass\n') - elif_stmt = node.body[0].orelse[0] - self.assertEqual(elif_stmt.lineno, 3) - self.assertEqual(elif_stmt.col_offset, 0) - - def test_elif_stmt_start_position_with_else(self): - node = ast.parse('if a:\n pass\nelif b:\n pass\nelse:\n pass\n') - elif_stmt = node.body[0].orelse[0] - self.assertEqual(elif_stmt.lineno, 3) - self.assertEqual(elif_stmt.col_offset, 0) - - def test_starred_expr_end_position_within_call(self): - node = ast.parse('f(*[0, 1])') - starred_expr = node.body[0].value.args[0] - self.assertEqual(starred_expr.end_lineno, 1) - self.assertEqual(starred_expr.end_col_offset, 9) - - def test_literal_eval(self): - self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3]) - self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) - self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None)) - self.assertEqual(ast.literal_eval('{1, 2, 3}'), {1, 2, 3}) - self.assertEqual(ast.literal_eval('b"hi"'), b"hi") - self.assertEqual(ast.literal_eval('set()'), set()) - self.assertRaises(ValueError, ast.literal_eval, 'foo()') - self.assertEqual(ast.literal_eval('6'), 6) - self.assertEqual(ast.literal_eval('+6'), 6) - self.assertEqual(ast.literal_eval('-6'), -6) - self.assertEqual(ast.literal_eval('3.25'), 3.25) - self.assertEqual(ast.literal_eval('+3.25'), 3.25) - self.assertEqual(ast.literal_eval('-3.25'), -3.25) - self.assertEqual(repr(ast.literal_eval('-0.0')), '-0.0') - self.assertRaises(ValueError, ast.literal_eval, '++6') - self.assertRaises(ValueError, ast.literal_eval, '+True') - self.assertRaises(ValueError, ast.literal_eval, '2+3') - - def test_literal_eval_str_int_limit(self): - with support.adjust_int_max_str_digits(4000): - ast.literal_eval('3'*4000) # no error - with self.assertRaises(SyntaxError) as err_ctx: - ast.literal_eval('3'*4001) - self.assertIn('Exceeds the limit ', str(err_ctx.exception)) - self.assertIn(' Consider hexadecimal ', str(err_ctx.exception)) - - def test_literal_eval_complex(self): - # Issue #4907 - self.assertEqual(ast.literal_eval('6j'), 6j) - self.assertEqual(ast.literal_eval('-6j'), -6j) - self.assertEqual(ast.literal_eval('6.75j'), 6.75j) - self.assertEqual(ast.literal_eval('-6.75j'), -6.75j) - self.assertEqual(ast.literal_eval('3+6j'), 3+6j) - self.assertEqual(ast.literal_eval('-3+6j'), -3+6j) - self.assertEqual(ast.literal_eval('3-6j'), 3-6j) - self.assertEqual(ast.literal_eval('-3-6j'), -3-6j) - self.assertEqual(ast.literal_eval('3.25+6.75j'), 3.25+6.75j) - self.assertEqual(ast.literal_eval('-3.25+6.75j'), -3.25+6.75j) - self.assertEqual(ast.literal_eval('3.25-6.75j'), 3.25-6.75j) - self.assertEqual(ast.literal_eval('-3.25-6.75j'), -3.25-6.75j) - self.assertEqual(ast.literal_eval('(3+6j)'), 3+6j) - self.assertRaises(ValueError, ast.literal_eval, '-6j+3') - self.assertRaises(ValueError, ast.literal_eval, '-6j+3j') - self.assertRaises(ValueError, ast.literal_eval, '3+-6j') - self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') - self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') - - def test_literal_eval_malformed_dict_nodes(self): - malformed = ast.Dict(keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)]) - self.assertRaises(ValueError, ast.literal_eval, malformed) - malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) - self.assertRaises(ValueError, ast.literal_eval, malformed) - - def test_literal_eval_trailing_ws(self): - self.assertEqual(ast.literal_eval(" -1"), -1) - self.assertEqual(ast.literal_eval("\t\t-1"), -1) - self.assertEqual(ast.literal_eval(" \t -1"), -1) - self.assertRaises(IndentationError, ast.literal_eval, "\n -1") - - def test_literal_eval_malformed_lineno(self): - msg = r'malformed node or string on line 3:' - with self.assertRaisesRegex(ValueError, msg): - ast.literal_eval("{'a': 1,\n'b':2,\n'c':++3,\n'd':4}") - - node = ast.UnaryOp( - ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) - self.assertIsNone(getattr(node, 'lineno', None)) - msg = r'malformed node or string:' - with self.assertRaisesRegex(ValueError, msg): - ast.literal_eval(node) - - def test_literal_eval_syntax_errors(self): - with self.assertRaisesRegex(SyntaxError, "unexpected indent"): - ast.literal_eval(r''' - \ - (\ - \ ''') - - def test_bad_integer(self): - # issue13436: Bad error message with invalid numeric values - body = [ast.ImportFrom(module='time', - names=[ast.alias(name='sleep')], - level=None, - lineno=None, col_offset=None)] - mod = ast.Module(body, []) - with self.assertRaises(ValueError) as cm: - compile(mod, 'test', 'exec') - self.assertIn("invalid integer value: None", str(cm.exception)) - - def test_level_as_none(self): - body = [ast.ImportFrom(module='time', - names=[ast.alias(name='sleep', - lineno=0, col_offset=0)], - level=None, - lineno=0, col_offset=0)] - mod = ast.Module(body, []) - code = compile(mod, 'test', 'exec') - ns = {} - exec(code, ns) - self.assertIn('sleep', ns) - - def test_recursion_direct(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - e.operand = e - with self.assertRaises(RecursionError): - with support.infinite_recursion(): - compile(ast.Expression(e), "", "eval") - - def test_recursion_indirect(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - e.operand = f - f.operand = e - with self.assertRaises(RecursionError): - with support.infinite_recursion(): - compile(ast.Expression(e), "", "eval") - - -class ASTValidatorTests(unittest.TestCase): - - def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): - mod.lineno = mod.col_offset = 0 - ast.fix_missing_locations(mod) - if msg is None: - compile(mod, "", mode) - else: - with self.assertRaises(exc) as cm: - compile(mod, "", mode) - self.assertIn(msg, str(cm.exception)) - - def expr(self, node, msg=None, *, exc=ValueError): - mod = ast.Module([ast.Expr(node)], []) - self.mod(mod, msg, exc=exc) - - def stmt(self, stmt, msg=None): - mod = ast.Module([stmt], []) - self.mod(mod, msg) - - def test_module(self): - m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) - self.mod(m, "must have Load context", "single") - m = ast.Expression(ast.Name("x", ast.Store())) - self.mod(m, "must have Load context", "eval") - - def _check_arguments(self, fac, check): - def arguments(args=None, posonlyargs=None, vararg=None, - kwonlyargs=None, kwarg=None, - defaults=None, kw_defaults=None): - if args is None: - args = [] - if posonlyargs is None: - posonlyargs = [] - if kwonlyargs is None: - kwonlyargs = [] - if defaults is None: - defaults = [] - if kw_defaults is None: - kw_defaults = [] - args = ast.arguments(args, posonlyargs, vararg, kwonlyargs, - kw_defaults, kwarg, defaults) - return fac(args) - args = [ast.arg("x", ast.Name("x", ast.Store()))] - check(arguments(args=args), "must have Load context") - check(arguments(posonlyargs=args), "must have Load context") - check(arguments(kwonlyargs=args), "must have Load context") - check(arguments(defaults=[ast.Num(3)]), - "more positional defaults than args") - check(arguments(kw_defaults=[ast.Num(4)]), - "length of kwonlyargs is not the same as kw_defaults") - args = [ast.arg("x", ast.Name("x", ast.Load()))] - check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), - "must have Load context") - args = [ast.arg("a", ast.Name("x", ast.Load())), - ast.arg("b", ast.Name("y", ast.Load()))] - check(arguments(kwonlyargs=args, - kw_defaults=[None, ast.Name("x", ast.Store())]), - "must have Load context") - - def test_funcdef(self): - a = ast.arguments([], [], None, [], [], None, []) - f = ast.FunctionDef("x", a, [], [], None) - self.stmt(f, "empty body on FunctionDef") - f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], - None) - self.stmt(f, "must have Load context") - f = ast.FunctionDef("x", a, [ast.Pass()], [], - ast.Name("x", ast.Store())) - self.stmt(f, "must have Load context") - def fac(args): - return ast.FunctionDef("x", args, [ast.Pass()], [], None) - self._check_arguments(fac, self.stmt) - - def test_classdef(self): - def cls(bases=None, keywords=None, body=None, decorator_list=None): - if bases is None: - bases = [] - if keywords is None: - keywords = [] - if body is None: - body = [ast.Pass()] - if decorator_list is None: - decorator_list = [] - return ast.ClassDef("myclass", bases, keywords, - body, decorator_list) - self.stmt(cls(bases=[ast.Name("x", ast.Store())]), - "must have Load context") - self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), - "must have Load context") - self.stmt(cls(body=[]), "empty body on ClassDef") - self.stmt(cls(body=[None]), "None disallowed") - self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]), - "must have Load context") - - def test_delete(self): - self.stmt(ast.Delete([]), "empty targets on Delete") - self.stmt(ast.Delete([None]), "None disallowed") - self.stmt(ast.Delete([ast.Name("x", ast.Load())]), - "must have Del context") - - def test_assign(self): - self.stmt(ast.Assign([], ast.Num(3)), "empty targets on Assign") - self.stmt(ast.Assign([None], ast.Num(3)), "None disallowed") - self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Num(3)), - "must have Store context") - self.stmt(ast.Assign([ast.Name("x", ast.Store())], - ast.Name("y", ast.Store())), - "must have Load context") - - def test_augassign(self): - aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(), - ast.Name("y", ast.Load())) - self.stmt(aug, "must have Store context") - aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(), - ast.Name("y", ast.Store())) - self.stmt(aug, "must have Load context") - - def test_for(self): - x = ast.Name("x", ast.Store()) - y = ast.Name("y", ast.Load()) - p = ast.Pass() - self.stmt(ast.For(x, y, [], []), "empty body on For") - self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []), - "must have Store context") - self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []), - "must have Load context") - e = ast.Expr(ast.Name("x", ast.Store())) - self.stmt(ast.For(x, y, [e], []), "must have Load context") - self.stmt(ast.For(x, y, [p], [e]), "must have Load context") - - def test_while(self): - self.stmt(ast.While(ast.Num(3), [], []), "empty body on While") - self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), - "must have Load context") - self.stmt(ast.While(ast.Num(3), [ast.Pass()], - [ast.Expr(ast.Name("x", ast.Store()))]), - "must have Load context") - - def test_if(self): - self.stmt(ast.If(ast.Num(3), [], []), "empty body on If") - i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) - self.stmt(i, "must have Load context") - i = ast.If(ast.Num(3), [ast.Expr(ast.Name("x", ast.Store()))], []) - self.stmt(i, "must have Load context") - i = ast.If(ast.Num(3), [ast.Pass()], - [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(i, "must have Load context") - - def test_with(self): - p = ast.Pass() - self.stmt(ast.With([], [p]), "empty items on With") - i = ast.withitem(ast.Num(3), None) - self.stmt(ast.With([i], []), "empty body on With") - i = ast.withitem(ast.Name("x", ast.Store()), None) - self.stmt(ast.With([i], [p]), "must have Load context") - i = ast.withitem(ast.Num(3), ast.Name("x", ast.Load())) - self.stmt(ast.With([i], [p]), "must have Store context") - - def test_raise(self): - r = ast.Raise(None, ast.Num(3)) - self.stmt(r, "Raise with cause but no exception") - r = ast.Raise(ast.Name("x", ast.Store()), None) - self.stmt(r, "must have Load context") - r = ast.Raise(ast.Num(4), ast.Name("x", ast.Store())) - self.stmt(r, "must have Load context") - - def test_try(self): - p = ast.Pass() - t = ast.Try([], [], [], [p]) - self.stmt(t, "empty body on Try") - t = ast.Try([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) - self.stmt(t, "must have Load context") - t = ast.Try([p], [], [], []) - self.stmt(t, "Try has neither except handlers nor finalbody") - t = ast.Try([p], [], [p], [p]) - self.stmt(t, "Try has orelse but no except handlers") - t = ast.Try([p], [ast.ExceptHandler(None, "x", [])], [], []) - self.stmt(t, "empty body on ExceptHandler") - e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] - self.stmt(ast.Try([p], e, [], []), "must have Load context") - e = [ast.ExceptHandler(None, "x", [p])] - t = ast.Try([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) - self.stmt(t, "must have Load context") - t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(t, "must have Load context") - - def test_try_star(self): - p = ast.Pass() - t = ast.TryStar([], [], [], [p]) - self.stmt(t, "empty body on TryStar") - t = ast.TryStar([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) - self.stmt(t, "must have Load context") - t = ast.TryStar([p], [], [], []) - self.stmt(t, "TryStar has neither except handlers nor finalbody") - t = ast.TryStar([p], [], [p], [p]) - self.stmt(t, "TryStar has orelse but no except handlers") - t = ast.TryStar([p], [ast.ExceptHandler(None, "x", [])], [], []) - self.stmt(t, "empty body on ExceptHandler") - e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] - self.stmt(ast.TryStar([p], e, [], []), "must have Load context") - e = [ast.ExceptHandler(None, "x", [p])] - t = ast.TryStar([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) - self.stmt(t, "must have Load context") - t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(t, "must have Load context") - - def test_assert(self): - self.stmt(ast.Assert(ast.Name("x", ast.Store()), None), - "must have Load context") - assrt = ast.Assert(ast.Name("x", ast.Load()), - ast.Name("y", ast.Store())) - self.stmt(assrt, "must have Load context") - - def test_import(self): - self.stmt(ast.Import([]), "empty names on Import") - - def test_importfrom(self): - imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) - self.stmt(imp, "Negative ImportFrom level") - self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") - - def test_global(self): - self.stmt(ast.Global([]), "empty names on Global") - - def test_nonlocal(self): - self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") - - def test_expr(self): - e = ast.Expr(ast.Name("x", ast.Store())) - self.stmt(e, "must have Load context") - - def test_boolop(self): - b = ast.BoolOp(ast.And(), []) - self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Num(3)]) - self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Num(4), None]) - self.expr(b, "None disallowed") - b = ast.BoolOp(ast.And(), [ast.Num(4), ast.Name("x", ast.Store())]) - self.expr(b, "must have Load context") - - def test_unaryop(self): - u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) - self.expr(u, "must have Load context") - - def test_lambda(self): - a = ast.arguments([], [], None, [], [], None, []) - self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), - "must have Load context") - def fac(args): - return ast.Lambda(args, ast.Name("x", ast.Load())) - self._check_arguments(fac, self.expr) - - def test_ifexp(self): - l = ast.Name("x", ast.Load()) - s = ast.Name("y", ast.Store()) - for args in (s, l, l), (l, s, l), (l, l, s): - self.expr(ast.IfExp(*args), "must have Load context") - - def test_dict(self): - d = ast.Dict([], [ast.Name("x", ast.Load())]) - self.expr(d, "same number of keys as values") - d = ast.Dict([ast.Name("x", ast.Load())], [None]) - self.expr(d, "None disallowed") - - def test_set(self): - self.expr(ast.Set([None]), "None disallowed") - s = ast.Set([ast.Name("x", ast.Store())]) - self.expr(s, "must have Load context") - - def _check_comprehension(self, fac): - self.expr(fac([]), "comprehension with no generators") - g = ast.comprehension(ast.Name("x", ast.Load()), - ast.Name("x", ast.Load()), [], 0) - self.expr(fac([g]), "must have Store context") - g = ast.comprehension(ast.Name("x", ast.Store()), - ast.Name("x", ast.Store()), [], 0) - self.expr(fac([g]), "must have Load context") - x = ast.Name("x", ast.Store()) - y = ast.Name("y", ast.Load()) - g = ast.comprehension(x, y, [None], 0) - self.expr(fac([g]), "None disallowed") - g = ast.comprehension(x, y, [ast.Name("x", ast.Store())], 0) - self.expr(fac([g]), "must have Load context") - - def _simple_comp(self, fac): - g = ast.comprehension(ast.Name("x", ast.Store()), - ast.Name("x", ast.Load()), [], 0) - self.expr(fac(ast.Name("x", ast.Store()), [g]), - "must have Load context") - def wrap(gens): - return fac(ast.Name("x", ast.Store()), gens) - self._check_comprehension(wrap) - - def test_listcomp(self): - self._simple_comp(ast.ListComp) - - def test_setcomp(self): - self._simple_comp(ast.SetComp) - - def test_generatorexp(self): - self._simple_comp(ast.GeneratorExp) - - def test_dictcomp(self): - g = ast.comprehension(ast.Name("y", ast.Store()), - ast.Name("p", ast.Load()), [], 0) - c = ast.DictComp(ast.Name("x", ast.Store()), - ast.Name("y", ast.Load()), [g]) - self.expr(c, "must have Load context") - c = ast.DictComp(ast.Name("x", ast.Load()), - ast.Name("y", ast.Store()), [g]) - self.expr(c, "must have Load context") - def factory(comps): - k = ast.Name("x", ast.Load()) - v = ast.Name("y", ast.Load()) - return ast.DictComp(k, v, comps) - self._check_comprehension(factory) - - def test_yield(self): - self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") - self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") - - def test_compare(self): - left = ast.Name("x", ast.Load()) - comp = ast.Compare(left, [ast.In()], []) - self.expr(comp, "no comparators") - comp = ast.Compare(left, [ast.In()], [ast.Num(4), ast.Num(5)]) - self.expr(comp, "different number of comparators and operands") - comp = ast.Compare(ast.Num("blah"), [ast.In()], [left]) - self.expr(comp) - comp = ast.Compare(left, [ast.In()], [ast.Num("blah")]) - self.expr(comp) - - def test_call(self): - func = ast.Name("x", ast.Load()) - args = [ast.Name("y", ast.Load())] - keywords = [ast.keyword("w", ast.Name("z", ast.Load()))] - call = ast.Call(ast.Name("x", ast.Store()), args, keywords) - self.expr(call, "must have Load context") - call = ast.Call(func, [None], keywords) - self.expr(call, "None disallowed") - bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))] - call = ast.Call(func, args, bad_keywords) - self.expr(call, "must have Load context") - - def test_num(self): - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - def test_attribute(self): - attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) - self.expr(attr, "must have Load context") - - def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Num(3), - ast.Load()) - self.expr(sub, "must have Load context") - x = ast.Name("x", ast.Load()) - sub = ast.Subscript(x, ast.Name("y", ast.Store()), - ast.Load()) - self.expr(sub, "must have Load context") - s = ast.Name("x", ast.Store()) - for args in (s, None, None), (None, s, None), (None, None, s): - sl = ast.Slice(*args) - self.expr(ast.Subscript(x, sl, ast.Load()), - "must have Load context") - sl = ast.Tuple([], ast.Load()) - self.expr(ast.Subscript(x, sl, ast.Load())) - sl = ast.Tuple([s], ast.Load()) - self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") - - def test_starred(self): - left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], - ast.Store()) - assign = ast.Assign([left], ast.Num(4)) - self.stmt(assign, "must have Store context") - - def _sequence(self, fac): - self.expr(fac([None], ast.Load()), "None disallowed") - self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), - "must have Load context") - - def test_list(self): - self._sequence(ast.List) - - def test_tuple(self): - self._sequence(ast.Tuple) - - def test_nameconstant(self): - self.expr(ast.NameConstant(4)) - - def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: - with self.subTest(module): - fn = os.path.join(stdlib, module) - with open(fn, "r", encoding="utf-8") as fp: - source = fp.read() - mod = ast.parse(source, fn) - compile(mod, fn, "exec") - - constant_1 = ast.Constant(1) - pattern_1 = ast.MatchValue(constant_1) - - constant_x = ast.Constant('x') - pattern_x = ast.MatchValue(constant_x) - - constant_true = ast.Constant(True) - pattern_true = ast.MatchSingleton(True) - - name_carter = ast.Name('carter', ast.Load()) - - _MATCH_PATTERNS = [ - ast.MatchValue( - ast.Attribute( - ast.Attribute( - ast.Name('x', ast.Store()), - 'y', ast.Load() - ), - 'z', ast.Load() - ) - ), - ast.MatchValue( - ast.Attribute( - ast.Attribute( - ast.Name('x', ast.Load()), - 'y', ast.Store() - ), - 'z', ast.Load() - ) - ), - ast.MatchValue( - ast.Constant(...) - ), - ast.MatchValue( - ast.Constant(True) - ), - ast.MatchValue( - ast.Constant((1,2,3)) - ), - ast.MatchSingleton('string'), - ast.MatchSequence([ - ast.MatchSingleton('string') - ]), - ast.MatchSequence( - [ - ast.MatchSequence( - [ - ast.MatchSingleton('string') - ] - ) - ] - ), - ast.MatchMapping( - [constant_1, constant_true], - [pattern_x] - ), - ast.MatchMapping( - [constant_true, constant_1], - [pattern_x, pattern_1], - rest='True' - ), - ast.MatchMapping( - [constant_true, ast.Starred(ast.Name('lol', ast.Load()), ast.Load())], - [pattern_x, pattern_1], - rest='legit' - ), - ast.MatchClass( - ast.Attribute( - ast.Attribute( - constant_x, - 'y', ast.Load()), - 'z', ast.Load()), - patterns=[], kwd_attrs=[], kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=['True'], - kwd_patterns=[pattern_1] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=[], - kwd_patterns=[pattern_1] - ), - ast.MatchClass( - name_carter, - patterns=[ast.MatchSingleton('string')], - kwd_attrs=[], - kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[ast.MatchStar()], - kwd_attrs=[], - kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=[], - kwd_patterns=[ast.MatchStar()] - ), - ast.MatchSequence( - [ - ast.MatchStar("True") - ] - ), - ast.MatchAs( - name='False' - ), - ast.MatchOr( - [] - ), - ast.MatchOr( - [pattern_1] - ), - ast.MatchOr( - [pattern_1, pattern_x, ast.MatchSingleton('xxx')] - ), - ast.MatchAs(name="_"), - ast.MatchStar(name="x"), - ast.MatchSequence([ast.MatchStar("_")]), - ast.MatchMapping([], [], rest="_"), - ] - - def test_match_validation_pattern(self): - name_x = ast.Name('x', ast.Load()) - for pattern in self._MATCH_PATTERNS: - with self.subTest(ast.dump(pattern, indent=4)): - node = ast.Match( - subject=name_x, - cases = [ - ast.match_case( - pattern=pattern, - body = [ast.Pass()] - ) - ] - ) - node = ast.fix_missing_locations(node) - module = ast.Module([node], []) - with self.assertRaises(ValueError): - compile(module, "", "exec") - - -class ConstantTests(unittest.TestCase): - """Tests on the ast.Constant node type.""" - - def compile_constant(self, value): - tree = ast.parse("x = 123") - - node = tree.body[0].value - new_node = ast.Constant(value=value) - ast.copy_location(new_node, node) - tree.body[0].value = new_node - - code = compile(tree, "", "exec") - - ns = {} - exec(code, ns) - return ns['x'] - - def test_validation(self): - with self.assertRaises(TypeError) as cm: - self.compile_constant([1, 2, 3]) - self.assertEqual(str(cm.exception), - "got an invalid type in Constant: list") - - def test_singletons(self): - for const in (None, False, True, Ellipsis, b'', frozenset()): - with self.subTest(const=const): - value = self.compile_constant(const) - self.assertIs(value, const) - - def test_values(self): - nested_tuple = (1,) - nested_frozenset = frozenset({1}) - for level in range(3): - nested_tuple = (nested_tuple, 2) - nested_frozenset = frozenset({nested_frozenset, 2}) - values = (123, 123.0, 123j, - "unicode", b'bytes', - tuple("tuple"), frozenset("frozenset"), - nested_tuple, nested_frozenset) - for value in values: - with self.subTest(value=value): - result = self.compile_constant(value) - self.assertEqual(result, value) - - def test_assign_to_constant(self): - tree = ast.parse("x = 1") - - target = tree.body[0].targets[0] - new_target = ast.Constant(value=1) - ast.copy_location(new_target, target) - tree.body[0].targets[0] = new_target - - with self.assertRaises(ValueError) as cm: - compile(tree, "string", "exec") - self.assertEqual(str(cm.exception), - "expression which can't be assigned " - "to in Store context") - - def test_get_docstring(self): - tree = ast.parse("'docstring'\nx = 1") - self.assertEqual(ast.get_docstring(tree), 'docstring') - - def get_load_const(self, tree): - # Compile to bytecode, disassemble and get parameter of LOAD_CONST - # instructions - co = compile(tree, '', 'exec') - consts = [] - for instr in dis.get_instructions(co): - if instr.opname == 'LOAD_CONST' or instr.opname == 'RETURN_CONST': - consts.append(instr.argval) - return consts - - @support.cpython_only - def test_load_const(self): - consts = [None, - True, False, - 124, - 2.0, - 3j, - "unicode", - b'bytes', - (1, 2, 3)] - - code = '\n'.join(['x={!r}'.format(const) for const in consts]) - code += '\nx = ...' - consts.extend((Ellipsis, None)) - - tree = ast.parse(code) - self.assertEqual(self.get_load_const(tree), - consts) - - # Replace expression nodes with constants - for assign, const in zip(tree.body, consts): - assert isinstance(assign, ast.Assign), ast.dump(assign) - new_node = ast.Constant(value=const) - ast.copy_location(new_node, assign.value) - assign.value = new_node - - self.assertEqual(self.get_load_const(tree), - consts) - - def test_literal_eval(self): - tree = ast.parse("1 + 2") - binop = tree.body[0].value - - new_left = ast.Constant(value=10) - ast.copy_location(new_left, binop.left) - binop.left = new_left - - new_right = ast.Constant(value=20j) - ast.copy_location(new_right, binop.right) - binop.right = new_right - - self.assertEqual(ast.literal_eval(binop), 10+20j) - - def test_string_kind(self): - c = ast.parse('"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, None) - - c = ast.parse('u"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, "u") - - c = ast.parse('r"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, None) - - c = ast.parse('b"x"', mode='eval').body - self.assertEqual(c.value, b"x") - self.assertEqual(c.kind, None) - - -class EndPositionTests(unittest.TestCase): - """Tests for end position of AST nodes. - - Testing end positions of nodes requires a bit of extra care - because of how LL parsers work. - """ - def _check_end_pos(self, ast_node, end_lineno, end_col_offset): - self.assertEqual(ast_node.end_lineno, end_lineno) - self.assertEqual(ast_node.end_col_offset, end_col_offset) - - def _check_content(self, source, ast_node, content): - self.assertEqual(ast.get_source_segment(source, ast_node), content) - - def _parse_value(self, s): - # Use duck-typing to support both single expression - # and a right hand side of an assignment statement. - return ast.parse(s).body[0].value - - def test_lambda(self): - s = 'lambda x, *y: None' - lam = self._parse_value(s) - self._check_content(s, lam.body, 'None') - self._check_content(s, lam.args.args[0], 'x') - self._check_content(s, lam.args.vararg, 'y') - - def test_func_def(self): - s = dedent(''' - def func(x: int, - *args: str, - z: float = 0, - **kwargs: Any) -> bool: - return True - ''').strip() - fdef = ast.parse(s).body[0] - self._check_end_pos(fdef, 5, 15) - self._check_content(s, fdef.body[0], 'return True') - self._check_content(s, fdef.args.args[0], 'x: int') - self._check_content(s, fdef.args.args[0].annotation, 'int') - self._check_content(s, fdef.args.kwarg, 'kwargs: Any') - self._check_content(s, fdef.args.kwarg.annotation, 'Any') - - def test_call(self): - s = 'func(x, y=2, **kw)' - call = self._parse_value(s) - self._check_content(s, call.func, 'func') - self._check_content(s, call.keywords[0].value, '2') - self._check_content(s, call.keywords[1].value, 'kw') - - def test_call_noargs(self): - s = 'x[0]()' - call = self._parse_value(s) - self._check_content(s, call.func, 'x[0]') - self._check_end_pos(call, 1, 6) - - def test_class_def(self): - s = dedent(''' - class C(A, B): - x: int = 0 - ''').strip() - cdef = ast.parse(s).body[0] - self._check_end_pos(cdef, 2, 14) - self._check_content(s, cdef.bases[1], 'B') - self._check_content(s, cdef.body[0], 'x: int = 0') - - def test_class_kw(self): - s = 'class S(metaclass=abc.ABCMeta): pass' - cdef = ast.parse(s).body[0] - self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta') - - def test_multi_line_str(self): - s = dedent(''' - x = """Some multi-line text. - - It goes on starting from same indent.""" - ''').strip() - assign = ast.parse(s).body[0] - self._check_end_pos(assign, 3, 40) - self._check_end_pos(assign.value, 3, 40) - - def test_continued_str(self): - s = dedent(''' - x = "first part" \\ - "second part" - ''').strip() - assign = ast.parse(s).body[0] - self._check_end_pos(assign, 2, 13) - self._check_end_pos(assign.value, 2, 13) - - def test_suites(self): - # We intentionally put these into the same string to check - # that empty lines are not part of the suite. - s = dedent(''' - while True: - pass - - if one(): - x = None - elif other(): - y = None - else: - z = None - - for x, y in stuff: - assert True - - try: - raise RuntimeError - except TypeError as e: - pass - - pass - ''').strip() - mod = ast.parse(s) - while_loop = mod.body[0] - if_stmt = mod.body[1] - for_loop = mod.body[2] - try_stmt = mod.body[3] - pass_stmt = mod.body[4] - - self._check_end_pos(while_loop, 2, 8) - self._check_end_pos(if_stmt, 9, 12) - self._check_end_pos(for_loop, 12, 15) - self._check_end_pos(try_stmt, 17, 8) - self._check_end_pos(pass_stmt, 19, 4) - - self._check_content(s, while_loop.test, 'True') - self._check_content(s, if_stmt.body[0], 'x = None') - self._check_content(s, if_stmt.orelse[0].test, 'other()') - self._check_content(s, for_loop.target, 'x, y') - self._check_content(s, try_stmt.body[0], 'raise RuntimeError') - self._check_content(s, try_stmt.handlers[0].type, 'TypeError') - - def test_fstring(self): - s = 'x = f"abc {x + y} abc"' - fstr = self._parse_value(s) - binop = fstr.values[1].value - self._check_content(s, binop, 'x + y') - - def test_fstring_multi_line(self): - s = dedent(''' - f"""Some multi-line text. - { - arg_one - + - arg_two - } - It goes on...""" - ''').strip() - fstr = self._parse_value(s) - binop = fstr.values[1].value - self._check_end_pos(binop, 5, 7) - self._check_content(s, binop.left, 'arg_one') - self._check_content(s, binop.right, 'arg_two') - - def test_import_from_multi_line(self): - s = dedent(''' - from x.y.z import ( - a, b, c as c - ) - ''').strip() - imp = ast.parse(s).body[0] - self._check_end_pos(imp, 3, 1) - self._check_end_pos(imp.names[2], 2, 16) - - def test_slices(self): - s1 = 'f()[1, 2] [0]' - s2 = 'x[ a.b: c.d]' - sm = dedent(''' - x[ a.b: f () , - g () : c.d - ] - ''').strip() - i1, i2, im = map(self._parse_value, (s1, s2, sm)) - self._check_content(s1, i1.value, 'f()[1, 2]') - self._check_content(s1, i1.value.slice, '1, 2') - self._check_content(s2, i2.slice.lower, 'a.b') - self._check_content(s2, i2.slice.upper, 'c.d') - self._check_content(sm, im.slice.elts[0].upper, 'f ()') - self._check_content(sm, im.slice.elts[1].lower, 'g ()') - self._check_end_pos(im, 3, 3) - - def test_binop(self): - s = dedent(''' - (1 * 2 + (3 ) + - 4 - ) - ''').strip() - binop = self._parse_value(s) - self._check_end_pos(binop, 2, 6) - self._check_content(s, binop.right, '4') - self._check_content(s, binop.left, '1 * 2 + (3 )') - self._check_content(s, binop.left.right, '3') - - def test_boolop(self): - s = dedent(''' - if (one_condition and - (other_condition or yet_another_one)): - pass - ''').strip() - bop = ast.parse(s).body[0].test - self._check_end_pos(bop, 2, 44) - self._check_content(s, bop.values[1], - 'other_condition or yet_another_one') - - def test_tuples(self): - s1 = 'x = () ;' - s2 = 'x = 1 , ;' - s3 = 'x = (1 , 2 ) ;' - sm = dedent(''' - x = ( - a, b, - ) - ''').strip() - t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm)) - self._check_content(s1, t1, '()') - self._check_content(s2, t2, '1 ,') - self._check_content(s3, t3, '(1 , 2 )') - self._check_end_pos(tm, 3, 1) - - def test_attribute_spaces(self): - s = 'func(x. y .z)' - call = self._parse_value(s) - self._check_content(s, call, s) - self._check_content(s, call.args[0], 'x. y .z') - - def test_redundant_parenthesis(self): - s = '( ( ( a + b ) ) )' - v = ast.parse(s).body[0].value - self.assertEqual(type(v).__name__, 'BinOp') - self._check_content(s, v, 'a + b') - s2 = 'await ' + s - v = ast.parse(s2).body[0].value.value - self.assertEqual(type(v).__name__, 'BinOp') - self._check_content(s2, v, 'a + b') - - def test_trailers_with_redundant_parenthesis(self): - tests = ( - ('( ( ( a ) ) ) ( )', 'Call'), - ('( ( ( a ) ) ) ( b )', 'Call'), - ('( ( ( a ) ) ) [ b ]', 'Subscript'), - ('( ( ( a ) ) ) . b', 'Attribute'), - ) - for s, t in tests: - with self.subTest(s): - v = ast.parse(s).body[0].value - self.assertEqual(type(v).__name__, t) - self._check_content(s, v, s) - s2 = 'await ' + s - v = ast.parse(s2).body[0].value.value - self.assertEqual(type(v).__name__, t) - self._check_content(s2, v, s) - - def test_displays(self): - s1 = '[{}, {1, }, {1, 2,} ]' - s2 = '{a: b, f (): g () ,}' - c1 = self._parse_value(s1) - c2 = self._parse_value(s2) - self._check_content(s1, c1.elts[0], '{}') - self._check_content(s1, c1.elts[1], '{1, }') - self._check_content(s1, c1.elts[2], '{1, 2,}') - self._check_content(s2, c2.keys[1], 'f ()') - self._check_content(s2, c2.values[1], 'g ()') - - def test_comprehensions(self): - s = dedent(''' - x = [{x for x, y in stuff - if cond.x} for stuff in things] - ''').strip() - cmp = self._parse_value(s) - self._check_end_pos(cmp, 2, 37) - self._check_content(s, cmp.generators[0].iter, 'things') - self._check_content(s, cmp.elt.generators[0].iter, 'stuff') - self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x') - self._check_content(s, cmp.elt.generators[0].target, 'x, y') - - def test_yield_await(self): - s = dedent(''' - async def f(): - yield x - await y - ''').strip() - fdef = ast.parse(s).body[0] - self._check_content(s, fdef.body[0].value, 'yield x') - self._check_content(s, fdef.body[1].value, 'await y') - - def test_source_segment_multi(self): - s_orig = dedent(''' - x = ( - a, b, - ) + () - ''').strip() - s_tuple = dedent(''' - ( - a, b, - ) - ''').strip() - binop = self._parse_value(s_orig) - self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) - - def test_source_segment_padded(self): - s_orig = dedent(''' - class C: - def fun(self) -> None: - "ЖЖЖЖЖ" - ''').strip() - s_method = ' def fun(self) -> None:\n' \ - ' "ЖЖЖЖЖ"' - cdef = ast.parse(s_orig).body[0] - self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True), - s_method) - - def test_source_segment_endings(self): - s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n' - v, w, x, y, z = ast.parse(s).body - self._check_content(s, v, 'v = 1') - self._check_content(s, w, 'w = 1') - self._check_content(s, x, 'x = 1') - self._check_content(s, y, 'y = 1') - self._check_content(s, z, 'z = 1') - - def test_source_segment_tabs(self): - s = dedent(''' - class C: - \t\f def fun(self) -> None: - \t\f pass - ''').strip() - s_method = ' \t\f def fun(self) -> None:\n' \ - ' \t\f pass' - - cdef = ast.parse(s).body[0] - self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method) - - def test_source_segment_missing_info(self): - s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n' - v, w, x, y = ast.parse(s).body - del v.lineno - del w.end_lineno - del x.col_offset - del y.end_col_offset - self.assertIsNone(ast.get_source_segment(s, v)) - self.assertIsNone(ast.get_source_segment(s, w)) - self.assertIsNone(ast.get_source_segment(s, x)) - self.assertIsNone(ast.get_source_segment(s, y)) - -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, 'Num', node.n)) - def visit_Str(self, node): - log.append((node.lineno, 'Str', node.s)) - def visit_Bytes(self, node): - log.append((node.lineno, 'Bytes', node.s)) - def visit_NameConstant(self, node): - log.append((node.lineno, 'NameConstant', node.value)) - def visit_Ellipsis(self, node): - log.append((node.lineno, 'Ellipsis', ...)) - mod = ast.parse(dedent('''\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - ''')) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - visitor.visit(mod) - self.assertEqual(log, [ - (1, 'Num', 42), - (2, 'Num', 4.25), - (3, 'Num', 4.25j), - (4, 'Str', 'string'), - (5, 'Bytes', b'bytes'), - (6, 'NameConstant', True), - (7, 'NameConstant', None), - (8, 'Ellipsis', ...), - ]) - self.assertEqual([str(w.message) for w in wlog], [ - 'visit_Num is deprecated; add visit_Constant', - 'visit_Num is deprecated; add visit_Constant', - 'visit_Num is deprecated; add visit_Constant', - 'visit_Str is deprecated; add visit_Constant', - 'visit_Bytes is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_Ellipsis is deprecated; add visit_Constant', - ]) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer - - def assertASTTransformation(self, tranformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) - expected_ast = ast.parse(dedent(expected_code)) - - tranformer = tranformer_class() - result_ast = ast.fix_missing_locations(tranformer.visit(initial_ast)) - - self.assertASTEqual(result_ast, expected_ast) - - def test_node_remove_single(self): - code = 'def func(arg) -> SomeType: ...' - expected = 'def func(arg): ...' - - # Since `FunctionDef.returns` is defined as a single value, we test - # the `if isinstance(old_value, AST):` branch here. - class SomeTypeRemover(ast.NodeTransformer): - def visit_Name(self, node: ast.Name): - self.generic_visit(node) - if node.id == 'SomeType': - return None - return node - - self.assertASTTransformation(SomeTypeRemover, code, expected) - - def test_node_remove_from_list(self): - code = """ - def func(arg): - print(arg) - yield arg - """ - expected = """ - def func(arg): - print(arg) - """ - - # Since `FunctionDef.body` is defined as a list, we test - # the `if isinstance(old_value, list):` branch here. - class YieldRemover(ast.NodeTransformer): - def visit_Expr(self, node: ast.Expr): - self.generic_visit(node) - if isinstance(node.value, ast.Yield): - return None # Remove `yield` from a function - return node - - self.assertASTTransformation(YieldRemover, code, expected) - - def test_node_return_list(self): - code = """ - class DSL(Base, kw1=True): ... - """ - expected = """ - class DSL(Base, kw1=True, kw2=True, kw3=False): ... - """ - - class ExtendKeywords(ast.NodeTransformer): - def visit_keyword(self, node: ast.keyword): - self.generic_visit(node) - if node.arg == 'kw1': - return [ - node, - ast.keyword('kw2', ast.Constant(True)), - ast.keyword('kw3', ast.Constant(False)), - ] - return node - - self.assertASTTransformation(ExtendKeywords, code, expected) - - def test_node_mutate(self): - code = """ - def func(arg): - print(arg) - """ - expected = """ - def func(arg): - log(arg) - """ - - class PrintToLog(ast.NodeTransformer): - def visit_Call(self, node: ast.Call): - self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == 'print': - node.func.id = 'log' - return node - - self.assertASTTransformation(PrintToLog, code, expected) - - def test_node_replace(self): - code = """ - def func(arg): - print(arg) - """ - expected = """ - def func(arg): - logger.log(arg, debug=True) - """ - - class PrintToLog(ast.NodeTransformer): - def visit_Call(self, node: ast.Call): - self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == 'print': - return ast.Call( - func=ast.Attribute( - ast.Name('logger', ctx=ast.Load()), - attr='log', - ctx=ast.Load(), - ), - args=node.args, - keywords=[ast.keyword('debug', ast.Constant(True))], - ) - return node - - self.assertASTTransformation(PrintToLog, code, expected) - - -@support.cpython_only -class ModuleStateTests(unittest.TestCase): - # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. - - def check_ast_module(self): - # Check that the _ast module still works as expected - code = 'x + 1' - filename = '' - mode = 'eval' - - # Create _ast.AST subclasses instances - ast_tree = compile(code, filename, mode, flags=ast.PyCF_ONLY_AST) - - # Call PyAST_Check() - code = compile(ast_tree, filename, mode) - self.assertIsInstance(code, types.CodeType) - - def test_reload_module(self): - # bpo-41194: Importing the _ast module twice must not crash. - with support.swap_item(sys.modules, '_ast', None): - del sys.modules['_ast'] - import _ast as ast1 - - del sys.modules['_ast'] - import _ast as ast2 - - self.check_ast_module() - - # Unloading the two _ast module instances must not crash. - del ast1 - del ast2 - support.gc_collect() - - self.check_ast_module() - - def test_sys_modules(self): - # bpo-41631: Test reproducing a Mercurial crash when PyAST_Check() - # imported the _ast module internally. - lazy_mod = object() - - def my_import(name, *args, **kw): - sys.modules[name] = lazy_mod - return lazy_mod - - with support.swap_item(sys.modules, '_ast', None): - del sys.modules['_ast'] - - with support.swap_attr(builtins, '__import__', my_import): - # Test that compile() does not import the _ast module - self.check_ast_module() - self.assertNotIn('_ast', sys.modules) - - # Sanity check of the test itself - import _ast - self.assertIs(_ast, lazy_mod) - - def test_subinterpreter(self): - # bpo-41631: Importing and using the _ast module in a subinterpreter - # must not crash. - code = dedent(''' - import _ast - import ast - import gc - import sys - import types - - # Create _ast.AST subclasses instances and call PyAST_Check() - ast_tree = compile('x+1', '', 'eval', - flags=ast.PyCF_ONLY_AST) - code = compile(ast_tree, 'string', 'eval') - if not isinstance(code, types.CodeType): - raise AssertionError - - # Unloading the _ast module must not crash. - del ast, _ast - del sys.modules['ast'], sys.modules['_ast'] - gc.collect() - ''') - res = support.run_in_subinterp(code) - self.assertEqual(res, 0) - - -def main(): - if __name__ != '__main__': - return - if sys.argv[1:] == ['-g']: - for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), - (eval_tests, "eval")): - print(kind+"_results = [") - for statement in statements: - tree = ast.parse(statement, "?", kind) - print("%r," % (to_tuple(tree),)) - print("]") - print("main()") - raise SystemExit - unittest.main() - -#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### -exec_results = [ -('Module', [('Expr', (1, 0, 1, 4), ('Constant', (1, 0, 1, 4), None, None))], []), -('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), -('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []), -('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []), -('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [])], []), -('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [])], []), -('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None)], []), -('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), -('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), -('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), -('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), -('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), -('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), -('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), -('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), -('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), -('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), -('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), -('Module', [('Global', (1, 0, 1, 8), ['v'])], []), -('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), -('Module', [('Pass', (1, 0, 1, 4))], []), -('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), -('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), -('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None)], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None)], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None)], []), -('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), -('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None)], []), -('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None)], []), -('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None)], []), -('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None)], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None)], []), -('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None)], []), -('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None)], []), -] -single_results = [ -('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), -] -eval_results = [ -('Expression', ('Constant', (1, 0, 1, 4), None, None)), -('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), -('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), -('Expression', ('Dict', (1, 0, 1, 2), [], [])), -('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), -('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), -('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), -('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), -('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), -('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), -('Expression', ('Constant', (1, 0, 1, 2), 10, None)), -('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), -('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), -('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), -('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), -] -main() diff --git a/components/utokenize/BUILD b/components/utokenize/BUILD deleted file mode 100644 index 5203a42..0000000 --- a/components/utokenize/BUILD +++ /dev/null @@ -1,3 +0,0 @@ -py_project( - name = "utokenize" -) diff --git a/components/utokenize/README.md b/components/utokenize/README.md deleted file mode 100644 index 0ce0449..0000000 --- a/components/utokenize/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# pycopy-utokenize - -A small Python parser, vendored from `pycopy-utokenize==2.0`. - -[pycopy-utokenize](https://pypi.org/project/pycopy-utokenize/) is released under the MIT license, copyright © Paul Sokolovsky 2021. diff --git a/components/utokenize/src/python/utokenize.py b/components/utokenize/src/python/utokenize.py deleted file mode 100644 index 2c70716..0000000 --- a/components/utokenize/src/python/utokenize.py +++ /dev/null @@ -1,240 +0,0 @@ -# (c) 2019 Paul Sokolovsky, MIT license - -import token -from collections import namedtuple -import io - -# Hacking in comments, newline and encoding as tokens -COMMENT = token.N_TOKENS + 0 -NL = token.N_TOKENS + 1 -ENCODING = token.N_TOKENS + 2 -token.tok_name[COMMENT] = "COMMENT" -token.tok_name[NL] = "NL" -token.tok_name[ENCODING] = "ENCODING" - - -class TokenInfo(namedtuple("TokenInfo", ("type", "string", "start", "end", "line"))): - def __str__(self): - return "TokenInfo(type=%d (%s), string=%r, startl=%d, line=%r)" % ( - self.type, - token.tok_name[self.type], - self.string, - self.start, - self.line, - ) - - -def get_indent(l): - for i in range(len(l)): - if l[i] != " " and l[i] != "\t": - return i, l[i:] - - -def get_str(l, readline): - lineno = 0 - s = io.StringIO() - - if l.startswith('"""') or l.startswith("'''"): - sep = l[0:3] - s += sep - l = l[3:] - pos = 0 - while True: - i = l.find(sep, pos) - if i >= 0: - if i > 0 and l[i - 1] == "\\": - pos = i + 1 - continue - break - s += l - l = readline() - pos = 0 - assert l - lineno += 1 - s += l[: i + 3] - return s.getvalue(), l[i + 3 :], lineno - - lbuf = io.StringIO(l) - sep = lbuf.read(1) - s += sep - while True: - c = lbuf.read(1) - if not c: - break - s += c - if c == "\\": - c = lbuf.read(1) - s += c - if c == "\n": - lbuf = io.StringIO(readline()) - lineno += 1 - continue - elif c == sep: - break - return s.getvalue(), lbuf.read(), lineno - - -def generate_tokens(readline): - indent_stack = [0] - lineno = 0 - paren_level = 0 - no_newline = False - - # generate_tokens() doesn't yield this, only tokenine() does. - # yield TokenInfo(ENCODING, "utf-8", 0, 0, "") - - while True: - l = readline() - lineno += 1 - org_l = l - if not l: - break - if not l.endswith("\n"): - l += "\n" - no_newline = True - i, l = get_indent(l) - - if l == "\n": - yield TokenInfo(NL, l, lineno, 0, org_l) - continue - elif l == "\x0c\n": - yield TokenInfo(NL, "\n", lineno, 0, org_l) - continue - - if l.startswith("#"): - yield TokenInfo(COMMENT, l.rstrip("\n"), lineno, 0, org_l) - yield TokenInfo(NL, "\n", lineno, 0, org_l) - continue - - if paren_level == 0: - if i > indent_stack[-1]: - yield TokenInfo(token.INDENT, org_l[:i], lineno, 0, org_l) - indent_stack.append(i) - elif i < indent_stack[-1]: - while i != indent_stack[-1]: - yield TokenInfo(token.DEDENT, "", lineno, 0, org_l) - indent_stack.pop() - - while l: - if l[0].isdigit() or (l.startswith(".") and len(l) > 1 and l[1].isdigit()): - seen_dot = False - t = "" - if l.startswith("0x") or l.startswith("0X"): - t = "0x" - l = l[2:] - elif l.startswith("0o") or l.startswith("0O"): - t = "0o" - l = l[2:] - elif l.startswith("0b") or l.startswith("0B"): - t = "0b" - l = l[2:] - while l and ( - l[0].isdigit() - or l[0] == "." - or l[0] == "_" - or (t.startswith("0x") and l[0] in "ABCDEFabcdef") - ): - if l[0] == ".": - if seen_dot: - break - seen_dot = True - t += l[0] - l = l[1:] - if l.startswith("e") or l.startswith("E"): - t += l[0] - l = l[1:] - if l[0] in ("+", "-"): - t += l[0] - l = l[1:] - while l and (l[0].isdigit() or l[0] == "_"): - t += l[0] - l = l[1:] - if l.startswith("j"): - t += l[0] - l = l[1:] - yield TokenInfo(token.NUMBER, t, lineno, 0, org_l) - elif l[0].isalpha() or l.startswith("_") or ord(l[0]) >= 0xAA: - name = "" - while l and ( - l[0].isalpha() - or l[0].isdigit() - or l.startswith("_") - or ord(l[0]) >= 0xAA - ): - name += l[0] - l = l[1:] - if (l.startswith('"') or l.startswith("'")) and name in ( - "b", - "r", - "rb", - "br", - "u", - "f", - ): - s, l, lineno_delta = get_str(l, readline) - yield TokenInfo(token.STRING, name + s, lineno, 0, org_l) - lineno += lineno_delta - else: - yield TokenInfo(token.NAME, name, lineno, 0, org_l) - elif l == "\\\n": - l = readline() - lineno += 1 - elif l[0] == "\n": - nl = "" if no_newline else "\n" - if paren_level > 0: - yield TokenInfo(NL, nl, lineno, 0, org_l) - else: - yield TokenInfo(token.NEWLINE, nl, lineno, 0, org_l) - break - elif l[0].isspace(): - l = l[1:] - elif l.startswith('"') or l.startswith("'"): - s, l, lineno_delta = get_str(l, readline) - yield TokenInfo(token.STRING, s, lineno, 0, org_l) - lineno += lineno_delta - elif l.startswith("#"): - yield TokenInfo(COMMENT, l.rstrip("\n"), lineno, 0, org_l) - l = "\n" - else: - for op in ( - "**=", - "//=", - ">>=", - "<<=", - "+=", - "-=", - "*=", - "/=", - "%=", - "@=", - "&=", - "|=", - "^=", - "**", - "//", - "<<", - ">>", - "==", - "!=", - ">=", - "<=", - "...", - "->", - ): - if l.startswith(op): - yield TokenInfo(token.OP, op, lineno, 0, org_l) - l = l[len(op) :] - break - else: - yield TokenInfo(token.OP, l[0], lineno, 0, org_l) - if l[0] in ("(", "[", "{"): - paren_level += 1 - elif l[0] in (")", "]", "}"): - paren_level -= 1 - l = l[1:] - - while indent_stack[-1] > 0: - yield TokenInfo(token.DEDENT, "", lineno, 0, "") - indent_stack.pop() - - yield TokenInfo(token.ENDMARKER, "", lineno, 0, "") diff --git a/components/utokenize/test/python/test_tokenize.py b/components/utokenize/test/python/test_tokenize.py deleted file mode 100644 index 63c2501..0000000 --- a/components/utokenize/test/python/test_tokenize.py +++ /dev/null @@ -1,2651 +0,0 @@ -from test import support -from test.support import os_helper -from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP, - STRING, ENDMARKER, ENCODING, tok_name, detect_encoding, - open as tokenize_open, Untokenizer, generate_tokens, - NEWLINE, _generate_tokens_from_c_tokenizer, DEDENT) -from io import BytesIO, StringIO -import unittest -from textwrap import dedent -from unittest import TestCase, mock -from test.test_grammar import (VALID_UNDERSCORE_LITERALS, - INVALID_UNDERSCORE_LITERALS) -from test.support import os_helper -from test.support.script_helper import run_test_script, make_script -import os -import token - -# Converts a source string into a list of textual representation -# of the tokens such as: -# ` NAME 'if' (1, 0) (1, 2)` -# to make writing tests easier. -def stringify_tokens_from_source(token_generator, source_string): - result = [] - num_lines = len(source_string.splitlines()) - missing_trailing_nl = source_string[-1] not in '\r\n' - - for type, token, start, end, line in token_generator: - if type == ENDMARKER: - break - # Ignore the new line on the last line if the input lacks one - if missing_trailing_nl and type == NEWLINE and end[0] == num_lines: - continue - type = tok_name[type] - result.append(f" {type:10} {token!r:13} {start} {end}") - - return result - -class TokenizeTest(TestCase): - # Tests for the tokenize module. - - # The tests can be really simple. Given a small fragment of source - # code, print out a table with tokens. The ENDMARKER, ENCODING and - # final NEWLINE are omitted for brevity. - - def check_tokenize(self, s, expected): - # Format the tokens in s in a table format. - # The ENDMARKER and final NEWLINE are omitted. - f = BytesIO(s.encode('utf-8')) - result = stringify_tokens_from_source(tokenize(f.readline), s) - self.assertEqual(result, - [" ENCODING 'utf-8' (0, 0) (0, 0)"] + - expected.rstrip().splitlines()) - - def test_implicit_newline(self): - # Make sure that the tokenizer puts in an implicit NEWLINE - # when the input lacks a trailing new line. - f = BytesIO("x".encode('utf-8')) - tokens = list(tokenize(f.readline)) - self.assertEqual(tokens[-2].type, NEWLINE) - self.assertEqual(tokens[-1].type, ENDMARKER) - - def test_basic(self): - self.check_tokenize("1 + 1", """\ - NUMBER '1' (1, 0) (1, 1) - OP '+' (1, 2) (1, 3) - NUMBER '1' (1, 4) (1, 5) - """) - self.check_tokenize("if False:\n" - " # NL\n" - " \n" - " True = False # NEWLINE\n", """\ - NAME 'if' (1, 0) (1, 2) - NAME 'False' (1, 3) (1, 8) - OP ':' (1, 8) (1, 9) - NEWLINE '\\n' (1, 9) (1, 10) - COMMENT '# NL' (2, 4) (2, 8) - NL '\\n' (2, 8) (2, 9) - NL '\\n' (3, 4) (3, 5) - INDENT ' ' (4, 0) (4, 4) - NAME 'True' (4, 4) (4, 8) - OP '=' (4, 9) (4, 10) - NAME 'False' (4, 11) (4, 16) - COMMENT '# NEWLINE' (4, 17) (4, 26) - NEWLINE '\\n' (4, 26) (4, 27) - DEDENT '' (5, 0) (5, 0) - """) - indent_error_file = b"""\ -def k(x): - x += 2 - x += 5 -""" - readline = BytesIO(indent_error_file).readline - with self.assertRaisesRegex(IndentationError, - "unindent does not match any " - "outer indentation level"): - for tok in tokenize(readline): - pass - - def test_int(self): - # Ordinary integers and binary operators - self.check_tokenize("0xff <= 255", """\ - NUMBER '0xff' (1, 0) (1, 4) - OP '<=' (1, 5) (1, 7) - NUMBER '255' (1, 8) (1, 11) - """) - self.check_tokenize("0b10 <= 255", """\ - NUMBER '0b10' (1, 0) (1, 4) - OP '<=' (1, 5) (1, 7) - NUMBER '255' (1, 8) (1, 11) - """) - self.check_tokenize("0o123 <= 0O123", """\ - NUMBER '0o123' (1, 0) (1, 5) - OP '<=' (1, 6) (1, 8) - NUMBER '0O123' (1, 9) (1, 14) - """) - self.check_tokenize("1234567 > ~0x15", """\ - NUMBER '1234567' (1, 0) (1, 7) - OP '>' (1, 8) (1, 9) - OP '~' (1, 10) (1, 11) - NUMBER '0x15' (1, 11) (1, 15) - """) - self.check_tokenize("2134568 != 1231515", """\ - NUMBER '2134568' (1, 0) (1, 7) - OP '!=' (1, 8) (1, 10) - NUMBER '1231515' (1, 11) (1, 18) - """) - self.check_tokenize("(-124561-1) & 200000000", """\ - OP '(' (1, 0) (1, 1) - OP '-' (1, 1) (1, 2) - NUMBER '124561' (1, 2) (1, 8) - OP '-' (1, 8) (1, 9) - NUMBER '1' (1, 9) (1, 10) - OP ')' (1, 10) (1, 11) - OP '&' (1, 12) (1, 13) - NUMBER '200000000' (1, 14) (1, 23) - """) - self.check_tokenize("0xdeadbeef != -1", """\ - NUMBER '0xdeadbeef' (1, 0) (1, 10) - OP '!=' (1, 11) (1, 13) - OP '-' (1, 14) (1, 15) - NUMBER '1' (1, 15) (1, 16) - """) - self.check_tokenize("0xdeadc0de & 12345", """\ - NUMBER '0xdeadc0de' (1, 0) (1, 10) - OP '&' (1, 11) (1, 12) - NUMBER '12345' (1, 13) (1, 18) - """) - self.check_tokenize("0xFF & 0x15 | 1234", """\ - NUMBER '0xFF' (1, 0) (1, 4) - OP '&' (1, 5) (1, 6) - NUMBER '0x15' (1, 7) (1, 11) - OP '|' (1, 12) (1, 13) - NUMBER '1234' (1, 14) (1, 18) - """) - - def test_long(self): - # Long integers - self.check_tokenize("x = 0", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '0' (1, 4) (1, 5) - """) - self.check_tokenize("x = 0xfffffffffff", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '0xfffffffffff' (1, 4) (1, 17) - """) - self.check_tokenize("x = 123141242151251616110", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '123141242151251616110' (1, 4) (1, 25) - """) - self.check_tokenize("x = -15921590215012591", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - OP '-' (1, 4) (1, 5) - NUMBER '15921590215012591' (1, 5) (1, 22) - """) - - def test_float(self): - # Floating point numbers - self.check_tokenize("x = 3.14159", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '3.14159' (1, 4) (1, 11) - """) - self.check_tokenize("x = 314159.", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '314159.' (1, 4) (1, 11) - """) - self.check_tokenize("x = .314159", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '.314159' (1, 4) (1, 11) - """) - self.check_tokenize("x = 3e14159", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '3e14159' (1, 4) (1, 11) - """) - self.check_tokenize("x = 3E123", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '3E123' (1, 4) (1, 9) - """) - self.check_tokenize("x+y = 3e-1230", """\ - NAME 'x' (1, 0) (1, 1) - OP '+' (1, 1) (1, 2) - NAME 'y' (1, 2) (1, 3) - OP '=' (1, 4) (1, 5) - NUMBER '3e-1230' (1, 6) (1, 13) - """) - self.check_tokenize("x = 3.14e159", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '3.14e159' (1, 4) (1, 12) - """) - - def test_underscore_literals(self): - def number_token(s): - f = BytesIO(s.encode('utf-8')) - for toktype, token, start, end, line in tokenize(f.readline): - if toktype == NUMBER: - return token - return 'invalid token' - for lit in VALID_UNDERSCORE_LITERALS: - if '(' in lit: - # this won't work with compound complex inputs - continue - self.assertEqual(number_token(lit), lit) - for lit in INVALID_UNDERSCORE_LITERALS: - self.assertNotEqual(number_token(lit), lit) - - def test_string(self): - # String literals - self.check_tokenize("x = ''; y = \"\"", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - STRING "''" (1, 4) (1, 6) - OP ';' (1, 6) (1, 7) - NAME 'y' (1, 8) (1, 9) - OP '=' (1, 10) (1, 11) - STRING '""' (1, 12) (1, 14) - """) - self.check_tokenize("x = '\"'; y = \"'\"", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - STRING '\\'"\\'' (1, 4) (1, 7) - OP ';' (1, 7) (1, 8) - NAME 'y' (1, 9) (1, 10) - OP '=' (1, 11) (1, 12) - STRING '"\\'"' (1, 13) (1, 16) - """) - self.check_tokenize("x = \"doesn't \"shrink\", does it\"", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - STRING '"doesn\\'t "' (1, 4) (1, 14) - NAME 'shrink' (1, 14) (1, 20) - STRING '", does it"' (1, 20) (1, 31) - """) - self.check_tokenize("x = 'abc' + 'ABC'", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - STRING "'abc'" (1, 4) (1, 9) - OP '+' (1, 10) (1, 11) - STRING "'ABC'" (1, 12) (1, 17) - """) - self.check_tokenize('y = "ABC" + "ABC"', """\ - NAME 'y' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - STRING '"ABC"' (1, 4) (1, 9) - OP '+' (1, 10) (1, 11) - STRING '"ABC"' (1, 12) (1, 17) - """) - self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - STRING "r'abc'" (1, 4) (1, 10) - OP '+' (1, 11) (1, 12) - STRING "r'ABC'" (1, 13) (1, 19) - OP '+' (1, 20) (1, 21) - STRING "R'ABC'" (1, 22) (1, 28) - OP '+' (1, 29) (1, 30) - STRING "R'ABC'" (1, 31) (1, 37) - """) - self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\ - NAME 'y' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - STRING 'r"abc"' (1, 4) (1, 10) - OP '+' (1, 11) (1, 12) - STRING 'r"ABC"' (1, 13) (1, 19) - OP '+' (1, 20) (1, 21) - STRING 'R"ABC"' (1, 22) (1, 28) - OP '+' (1, 29) (1, 30) - STRING 'R"ABC"' (1, 31) (1, 37) - """) - - self.check_tokenize("u'abc' + U'abc'", """\ - STRING "u'abc'" (1, 0) (1, 6) - OP '+' (1, 7) (1, 8) - STRING "U'abc'" (1, 9) (1, 15) - """) - self.check_tokenize('u"abc" + U"abc"', """\ - STRING 'u"abc"' (1, 0) (1, 6) - OP '+' (1, 7) (1, 8) - STRING 'U"abc"' (1, 9) (1, 15) - """) - - self.check_tokenize("b'abc' + B'abc'", """\ - STRING "b'abc'" (1, 0) (1, 6) - OP '+' (1, 7) (1, 8) - STRING "B'abc'" (1, 9) (1, 15) - """) - self.check_tokenize('b"abc" + B"abc"', """\ - STRING 'b"abc"' (1, 0) (1, 6) - OP '+' (1, 7) (1, 8) - STRING 'B"abc"' (1, 9) (1, 15) - """) - self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\ - STRING "br'abc'" (1, 0) (1, 7) - OP '+' (1, 8) (1, 9) - STRING "bR'abc'" (1, 10) (1, 17) - OP '+' (1, 18) (1, 19) - STRING "Br'abc'" (1, 20) (1, 27) - OP '+' (1, 28) (1, 29) - STRING "BR'abc'" (1, 30) (1, 37) - """) - self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\ - STRING 'br"abc"' (1, 0) (1, 7) - OP '+' (1, 8) (1, 9) - STRING 'bR"abc"' (1, 10) (1, 17) - OP '+' (1, 18) (1, 19) - STRING 'Br"abc"' (1, 20) (1, 27) - OP '+' (1, 28) (1, 29) - STRING 'BR"abc"' (1, 30) (1, 37) - """) - self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\ - STRING "rb'abc'" (1, 0) (1, 7) - OP '+' (1, 8) (1, 9) - STRING "rB'abc'" (1, 10) (1, 17) - OP '+' (1, 18) (1, 19) - STRING "Rb'abc'" (1, 20) (1, 27) - OP '+' (1, 28) (1, 29) - STRING "RB'abc'" (1, 30) (1, 37) - """) - self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\ - STRING 'rb"abc"' (1, 0) (1, 7) - OP '+' (1, 8) (1, 9) - STRING 'rB"abc"' (1, 10) (1, 17) - OP '+' (1, 18) (1, 19) - STRING 'Rb"abc"' (1, 20) (1, 27) - OP '+' (1, 28) (1, 29) - STRING 'RB"abc"' (1, 30) (1, 37) - """) - # Check 0, 1, and 2 character string prefixes. - self.check_tokenize(r'"a\ -de\ -fg"', """\ - STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3) - """) - self.check_tokenize(r'u"a\ -de"', """\ - STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3) - """) - self.check_tokenize(r'rb"a\ -d"', """\ - STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2) - """) - self.check_tokenize(r'"""a\ -b"""', """\ - STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) - """) - self.check_tokenize(r'u"""a\ -b"""', """\ - STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) - """) - self.check_tokenize(r'rb"""a\ -b\ -c"""', """\ - STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) - """) - self.check_tokenize('f"abc"', """\ - STRING 'f"abc"' (1, 0) (1, 6) - """) - self.check_tokenize('fR"a{b}c"', """\ - STRING 'fR"a{b}c"' (1, 0) (1, 9) - """) - self.check_tokenize('f"""abc"""', """\ - STRING 'f\"\"\"abc\"\"\"' (1, 0) (1, 10) - """) - self.check_tokenize(r'f"abc\ -def"', """\ - STRING 'f"abc\\\\\\ndef"' (1, 0) (2, 4) - """) - self.check_tokenize(r'Rf"abc\ -def"', """\ - STRING 'Rf"abc\\\\\\ndef"' (1, 0) (2, 4) - """) - - def test_function(self): - self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\ - NAME 'def' (1, 0) (1, 3) - NAME 'd22' (1, 4) (1, 7) - OP '(' (1, 7) (1, 8) - NAME 'a' (1, 8) (1, 9) - OP ',' (1, 9) (1, 10) - NAME 'b' (1, 11) (1, 12) - OP ',' (1, 12) (1, 13) - NAME 'c' (1, 14) (1, 15) - OP '=' (1, 15) (1, 16) - NUMBER '2' (1, 16) (1, 17) - OP ',' (1, 17) (1, 18) - NAME 'd' (1, 19) (1, 20) - OP '=' (1, 20) (1, 21) - NUMBER '2' (1, 21) (1, 22) - OP ',' (1, 22) (1, 23) - OP '*' (1, 24) (1, 25) - NAME 'k' (1, 25) (1, 26) - OP ')' (1, 26) (1, 27) - OP ':' (1, 27) (1, 28) - NAME 'pass' (1, 29) (1, 33) - """) - self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\ - NAME 'def' (1, 0) (1, 3) - NAME 'd01v_' (1, 4) (1, 9) - OP '(' (1, 9) (1, 10) - NAME 'a' (1, 10) (1, 11) - OP '=' (1, 11) (1, 12) - NUMBER '1' (1, 12) (1, 13) - OP ',' (1, 13) (1, 14) - OP '*' (1, 15) (1, 16) - NAME 'k' (1, 16) (1, 17) - OP ',' (1, 17) (1, 18) - OP '**' (1, 19) (1, 21) - NAME 'w' (1, 21) (1, 22) - OP ')' (1, 22) (1, 23) - OP ':' (1, 23) (1, 24) - NAME 'pass' (1, 25) (1, 29) - """) - self.check_tokenize("def d23(a: str, b: int=3) -> int: pass", """\ - NAME 'def' (1, 0) (1, 3) - NAME 'd23' (1, 4) (1, 7) - OP '(' (1, 7) (1, 8) - NAME 'a' (1, 8) (1, 9) - OP ':' (1, 9) (1, 10) - NAME 'str' (1, 11) (1, 14) - OP ',' (1, 14) (1, 15) - NAME 'b' (1, 16) (1, 17) - OP ':' (1, 17) (1, 18) - NAME 'int' (1, 19) (1, 22) - OP '=' (1, 22) (1, 23) - NUMBER '3' (1, 23) (1, 24) - OP ')' (1, 24) (1, 25) - OP '->' (1, 26) (1, 28) - NAME 'int' (1, 29) (1, 32) - OP ':' (1, 32) (1, 33) - NAME 'pass' (1, 34) (1, 38) - """) - - def test_comparison(self): - # Comparison - self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " - "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\ - NAME 'if' (1, 0) (1, 2) - NUMBER '1' (1, 3) (1, 4) - OP '<' (1, 5) (1, 6) - NUMBER '1' (1, 7) (1, 8) - OP '>' (1, 9) (1, 10) - NUMBER '1' (1, 11) (1, 12) - OP '==' (1, 13) (1, 15) - NUMBER '1' (1, 16) (1, 17) - OP '>=' (1, 18) (1, 20) - NUMBER '5' (1, 21) (1, 22) - OP '<=' (1, 23) (1, 25) - NUMBER '0x15' (1, 26) (1, 30) - OP '<=' (1, 31) (1, 33) - NUMBER '0x12' (1, 34) (1, 38) - OP '!=' (1, 39) (1, 41) - NUMBER '1' (1, 42) (1, 43) - NAME 'and' (1, 44) (1, 47) - NUMBER '5' (1, 48) (1, 49) - NAME 'in' (1, 50) (1, 52) - NUMBER '1' (1, 53) (1, 54) - NAME 'not' (1, 55) (1, 58) - NAME 'in' (1, 59) (1, 61) - NUMBER '1' (1, 62) (1, 63) - NAME 'is' (1, 64) (1, 66) - NUMBER '1' (1, 67) (1, 68) - NAME 'or' (1, 69) (1, 71) - NUMBER '5' (1, 72) (1, 73) - NAME 'is' (1, 74) (1, 76) - NAME 'not' (1, 77) (1, 80) - NUMBER '1' (1, 81) (1, 82) - OP ':' (1, 82) (1, 83) - NAME 'pass' (1, 84) (1, 88) - """) - - def test_shift(self): - # Shift - self.check_tokenize("x = 1 << 1 >> 5", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '1' (1, 4) (1, 5) - OP '<<' (1, 6) (1, 8) - NUMBER '1' (1, 9) (1, 10) - OP '>>' (1, 11) (1, 13) - NUMBER '5' (1, 14) (1, 15) - """) - - def test_additive(self): - # Additive - self.check_tokenize("x = 1 - y + 15 - 1 + 0x124 + z + a[5]", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '1' (1, 4) (1, 5) - OP '-' (1, 6) (1, 7) - NAME 'y' (1, 8) (1, 9) - OP '+' (1, 10) (1, 11) - NUMBER '15' (1, 12) (1, 14) - OP '-' (1, 15) (1, 16) - NUMBER '1' (1, 17) (1, 18) - OP '+' (1, 19) (1, 20) - NUMBER '0x124' (1, 21) (1, 26) - OP '+' (1, 27) (1, 28) - NAME 'z' (1, 29) (1, 30) - OP '+' (1, 31) (1, 32) - NAME 'a' (1, 33) (1, 34) - OP '[' (1, 34) (1, 35) - NUMBER '5' (1, 35) (1, 36) - OP ']' (1, 36) (1, 37) - """) - - def test_multiplicative(self): - # Multiplicative - self.check_tokenize("x = 1//1*1/5*12%0x12@42", """\ - NAME 'x' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - NUMBER '1' (1, 4) (1, 5) - OP '//' (1, 5) (1, 7) - NUMBER '1' (1, 7) (1, 8) - OP '*' (1, 8) (1, 9) - NUMBER '1' (1, 9) (1, 10) - OP '/' (1, 10) (1, 11) - NUMBER '5' (1, 11) (1, 12) - OP '*' (1, 12) (1, 13) - NUMBER '12' (1, 13) (1, 15) - OP '%' (1, 15) (1, 16) - NUMBER '0x12' (1, 16) (1, 20) - OP '@' (1, 20) (1, 21) - NUMBER '42' (1, 21) (1, 23) - """) - - def test_unary(self): - # Unary - self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\ - OP '~' (1, 0) (1, 1) - NUMBER '1' (1, 1) (1, 2) - OP '^' (1, 3) (1, 4) - NUMBER '1' (1, 5) (1, 6) - OP '&' (1, 7) (1, 8) - NUMBER '1' (1, 9) (1, 10) - OP '|' (1, 11) (1, 12) - NUMBER '1' (1, 12) (1, 13) - OP '^' (1, 14) (1, 15) - OP '-' (1, 16) (1, 17) - NUMBER '1' (1, 17) (1, 18) - """) - self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\ - OP '-' (1, 0) (1, 1) - NUMBER '1' (1, 1) (1, 2) - OP '*' (1, 2) (1, 3) - NUMBER '1' (1, 3) (1, 4) - OP '/' (1, 4) (1, 5) - NUMBER '1' (1, 5) (1, 6) - OP '+' (1, 6) (1, 7) - NUMBER '1' (1, 7) (1, 8) - OP '*' (1, 8) (1, 9) - NUMBER '1' (1, 9) (1, 10) - OP '//' (1, 10) (1, 12) - NUMBER '1' (1, 12) (1, 13) - OP '-' (1, 14) (1, 15) - OP '-' (1, 16) (1, 17) - OP '-' (1, 17) (1, 18) - OP '-' (1, 18) (1, 19) - NUMBER '1' (1, 19) (1, 20) - OP '**' (1, 20) (1, 22) - NUMBER '1' (1, 22) (1, 23) - """) - - def test_selector(self): - # Selector - self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\ - NAME 'import' (1, 0) (1, 6) - NAME 'sys' (1, 7) (1, 10) - OP ',' (1, 10) (1, 11) - NAME 'time' (1, 12) (1, 16) - NEWLINE '\\n' (1, 16) (1, 17) - NAME 'x' (2, 0) (2, 1) - OP '=' (2, 2) (2, 3) - NAME 'sys' (2, 4) (2, 7) - OP '.' (2, 7) (2, 8) - NAME 'modules' (2, 8) (2, 15) - OP '[' (2, 15) (2, 16) - STRING "'time'" (2, 16) (2, 22) - OP ']' (2, 22) (2, 23) - OP '.' (2, 23) (2, 24) - NAME 'time' (2, 24) (2, 28) - OP '(' (2, 28) (2, 29) - OP ')' (2, 29) (2, 30) - """) - - def test_method(self): - # Methods - self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\ - OP '@' (1, 0) (1, 1) - NAME 'staticmethod' (1, 1) (1, 13) - NEWLINE '\\n' (1, 13) (1, 14) - NAME 'def' (2, 0) (2, 3) - NAME 'foo' (2, 4) (2, 7) - OP '(' (2, 7) (2, 8) - NAME 'x' (2, 8) (2, 9) - OP ',' (2, 9) (2, 10) - NAME 'y' (2, 10) (2, 11) - OP ')' (2, 11) (2, 12) - OP ':' (2, 12) (2, 13) - NAME 'pass' (2, 14) (2, 18) - """) - - def test_tabs(self): - # Evil tabs - self.check_tokenize("def f():\n" - "\tif x\n" - " \tpass", """\ - NAME 'def' (1, 0) (1, 3) - NAME 'f' (1, 4) (1, 5) - OP '(' (1, 5) (1, 6) - OP ')' (1, 6) (1, 7) - OP ':' (1, 7) (1, 8) - NEWLINE '\\n' (1, 8) (1, 9) - INDENT '\\t' (2, 0) (2, 1) - NAME 'if' (2, 1) (2, 3) - NAME 'x' (2, 4) (2, 5) - NEWLINE '\\n' (2, 5) (2, 6) - INDENT ' \\t' (3, 0) (3, 9) - NAME 'pass' (3, 9) (3, 13) - DEDENT '' (4, 0) (4, 0) - DEDENT '' (4, 0) (4, 0) - """) - - def test_non_ascii_identifiers(self): - # Non-ascii identifiers - self.check_tokenize("Örter = 'places'\ngrün = 'green'", """\ - NAME 'Örter' (1, 0) (1, 5) - OP '=' (1, 6) (1, 7) - STRING "'places'" (1, 8) (1, 16) - NEWLINE '\\n' (1, 16) (1, 17) - NAME 'grün' (2, 0) (2, 4) - OP '=' (2, 5) (2, 6) - STRING "'green'" (2, 7) (2, 14) - """) - - def test_unicode(self): - # Legacy unicode literals: - self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\ - NAME 'Örter' (1, 0) (1, 5) - OP '=' (1, 6) (1, 7) - STRING "u'places'" (1, 8) (1, 17) - NEWLINE '\\n' (1, 17) (1, 18) - NAME 'grün' (2, 0) (2, 4) - OP '=' (2, 5) (2, 6) - STRING "U'green'" (2, 7) (2, 15) - """) - - def test_async(self): - # Async/await extension: - self.check_tokenize("async = 1", """\ - NAME 'async' (1, 0) (1, 5) - OP '=' (1, 6) (1, 7) - NUMBER '1' (1, 8) (1, 9) - """) - - self.check_tokenize("a = (async = 1)", """\ - NAME 'a' (1, 0) (1, 1) - OP '=' (1, 2) (1, 3) - OP '(' (1, 4) (1, 5) - NAME 'async' (1, 5) (1, 10) - OP '=' (1, 11) (1, 12) - NUMBER '1' (1, 13) (1, 14) - OP ')' (1, 14) (1, 15) - """) - - self.check_tokenize("async()", """\ - NAME 'async' (1, 0) (1, 5) - OP '(' (1, 5) (1, 6) - OP ')' (1, 6) (1, 7) - """) - - self.check_tokenize("class async(Bar):pass", """\ - NAME 'class' (1, 0) (1, 5) - NAME 'async' (1, 6) (1, 11) - OP '(' (1, 11) (1, 12) - NAME 'Bar' (1, 12) (1, 15) - OP ')' (1, 15) (1, 16) - OP ':' (1, 16) (1, 17) - NAME 'pass' (1, 17) (1, 21) - """) - - self.check_tokenize("class async:pass", """\ - NAME 'class' (1, 0) (1, 5) - NAME 'async' (1, 6) (1, 11) - OP ':' (1, 11) (1, 12) - NAME 'pass' (1, 12) (1, 16) - """) - - self.check_tokenize("await = 1", """\ - NAME 'await' (1, 0) (1, 5) - OP '=' (1, 6) (1, 7) - NUMBER '1' (1, 8) (1, 9) - """) - - self.check_tokenize("foo.async", """\ - NAME 'foo' (1, 0) (1, 3) - OP '.' (1, 3) (1, 4) - NAME 'async' (1, 4) (1, 9) - """) - - self.check_tokenize("async for a in b: pass", """\ - NAME 'async' (1, 0) (1, 5) - NAME 'for' (1, 6) (1, 9) - NAME 'a' (1, 10) (1, 11) - NAME 'in' (1, 12) (1, 14) - NAME 'b' (1, 15) (1, 16) - OP ':' (1, 16) (1, 17) - NAME 'pass' (1, 18) (1, 22) - """) - - self.check_tokenize("async with a as b: pass", """\ - NAME 'async' (1, 0) (1, 5) - NAME 'with' (1, 6) (1, 10) - NAME 'a' (1, 11) (1, 12) - NAME 'as' (1, 13) (1, 15) - NAME 'b' (1, 16) (1, 17) - OP ':' (1, 17) (1, 18) - NAME 'pass' (1, 19) (1, 23) - """) - - self.check_tokenize("async.foo", """\ - NAME 'async' (1, 0) (1, 5) - OP '.' (1, 5) (1, 6) - NAME 'foo' (1, 6) (1, 9) - """) - - self.check_tokenize("async", """\ - NAME 'async' (1, 0) (1, 5) - """) - - self.check_tokenize("async\n#comment\nawait", """\ - NAME 'async' (1, 0) (1, 5) - NEWLINE '\\n' (1, 5) (1, 6) - COMMENT '#comment' (2, 0) (2, 8) - NL '\\n' (2, 8) (2, 9) - NAME 'await' (3, 0) (3, 5) - """) - - self.check_tokenize("async\n...\nawait", """\ - NAME 'async' (1, 0) (1, 5) - NEWLINE '\\n' (1, 5) (1, 6) - OP '...' (2, 0) (2, 3) - NEWLINE '\\n' (2, 3) (2, 4) - NAME 'await' (3, 0) (3, 5) - """) - - self.check_tokenize("async\nawait", """\ - NAME 'async' (1, 0) (1, 5) - NEWLINE '\\n' (1, 5) (1, 6) - NAME 'await' (2, 0) (2, 5) - """) - - self.check_tokenize("foo.async + 1", """\ - NAME 'foo' (1, 0) (1, 3) - OP '.' (1, 3) (1, 4) - NAME 'async' (1, 4) (1, 9) - OP '+' (1, 10) (1, 11) - NUMBER '1' (1, 12) (1, 13) - """) - - self.check_tokenize("async def foo(): pass", """\ - NAME 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'foo' (1, 10) (1, 13) - OP '(' (1, 13) (1, 14) - OP ')' (1, 14) (1, 15) - OP ':' (1, 15) (1, 16) - NAME 'pass' (1, 17) (1, 21) - """) - - self.check_tokenize('''\ -async def foo(): - def foo(await): - await = 1 - if 1: - await -async += 1 -''', """\ - NAME 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'foo' (1, 10) (1, 13) - OP '(' (1, 13) (1, 14) - OP ')' (1, 14) (1, 15) - OP ':' (1, 15) (1, 16) - NEWLINE '\\n' (1, 16) (1, 17) - INDENT ' ' (2, 0) (2, 2) - NAME 'def' (2, 2) (2, 5) - NAME 'foo' (2, 6) (2, 9) - OP '(' (2, 9) (2, 10) - NAME 'await' (2, 10) (2, 15) - OP ')' (2, 15) (2, 16) - OP ':' (2, 16) (2, 17) - NEWLINE '\\n' (2, 17) (2, 18) - INDENT ' ' (3, 0) (3, 4) - NAME 'await' (3, 4) (3, 9) - OP '=' (3, 10) (3, 11) - NUMBER '1' (3, 12) (3, 13) - NEWLINE '\\n' (3, 13) (3, 14) - DEDENT '' (4, 2) (4, 2) - NAME 'if' (4, 2) (4, 4) - NUMBER '1' (4, 5) (4, 6) - OP ':' (4, 6) (4, 7) - NEWLINE '\\n' (4, 7) (4, 8) - INDENT ' ' (5, 0) (5, 4) - NAME 'await' (5, 4) (5, 9) - NEWLINE '\\n' (5, 9) (5, 10) - DEDENT '' (6, 0) (6, 0) - DEDENT '' (6, 0) (6, 0) - NAME 'async' (6, 0) (6, 5) - OP '+=' (6, 6) (6, 8) - NUMBER '1' (6, 9) (6, 10) - NEWLINE '\\n' (6, 10) (6, 11) - """) - - self.check_tokenize('''\ -async def foo(): - async for i in 1: pass''', """\ - NAME 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'foo' (1, 10) (1, 13) - OP '(' (1, 13) (1, 14) - OP ')' (1, 14) (1, 15) - OP ':' (1, 15) (1, 16) - NEWLINE '\\n' (1, 16) (1, 17) - INDENT ' ' (2, 0) (2, 2) - NAME 'async' (2, 2) (2, 7) - NAME 'for' (2, 8) (2, 11) - NAME 'i' (2, 12) (2, 13) - NAME 'in' (2, 14) (2, 16) - NUMBER '1' (2, 17) (2, 18) - OP ':' (2, 18) (2, 19) - NAME 'pass' (2, 20) (2, 24) - DEDENT '' (3, 0) (3, 0) - """) - - self.check_tokenize('''async def foo(async): await''', """\ - NAME 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'foo' (1, 10) (1, 13) - OP '(' (1, 13) (1, 14) - NAME 'async' (1, 14) (1, 19) - OP ')' (1, 19) (1, 20) - OP ':' (1, 20) (1, 21) - NAME 'await' (1, 22) (1, 27) - """) - - self.check_tokenize('''\ -def f(): - - def baz(): pass - async def bar(): pass - - await = 2''', """\ - NAME 'def' (1, 0) (1, 3) - NAME 'f' (1, 4) (1, 5) - OP '(' (1, 5) (1, 6) - OP ')' (1, 6) (1, 7) - OP ':' (1, 7) (1, 8) - NEWLINE '\\n' (1, 8) (1, 9) - NL '\\n' (2, 0) (2, 1) - INDENT ' ' (3, 0) (3, 2) - NAME 'def' (3, 2) (3, 5) - NAME 'baz' (3, 6) (3, 9) - OP '(' (3, 9) (3, 10) - OP ')' (3, 10) (3, 11) - OP ':' (3, 11) (3, 12) - NAME 'pass' (3, 13) (3, 17) - NEWLINE '\\n' (3, 17) (3, 18) - NAME 'async' (4, 2) (4, 7) - NAME 'def' (4, 8) (4, 11) - NAME 'bar' (4, 12) (4, 15) - OP '(' (4, 15) (4, 16) - OP ')' (4, 16) (4, 17) - OP ':' (4, 17) (4, 18) - NAME 'pass' (4, 19) (4, 23) - NEWLINE '\\n' (4, 23) (4, 24) - NL '\\n' (5, 0) (5, 1) - NAME 'await' (6, 2) (6, 7) - OP '=' (6, 8) (6, 9) - NUMBER '2' (6, 10) (6, 11) - DEDENT '' (7, 0) (7, 0) - """) - - self.check_tokenize('''\ -async def f(): - - def baz(): pass - async def bar(): pass - - await = 2''', """\ - NAME 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'f' (1, 10) (1, 11) - OP '(' (1, 11) (1, 12) - OP ')' (1, 12) (1, 13) - OP ':' (1, 13) (1, 14) - NEWLINE '\\n' (1, 14) (1, 15) - NL '\\n' (2, 0) (2, 1) - INDENT ' ' (3, 0) (3, 2) - NAME 'def' (3, 2) (3, 5) - NAME 'baz' (3, 6) (3, 9) - OP '(' (3, 9) (3, 10) - OP ')' (3, 10) (3, 11) - OP ':' (3, 11) (3, 12) - NAME 'pass' (3, 13) (3, 17) - NEWLINE '\\n' (3, 17) (3, 18) - NAME 'async' (4, 2) (4, 7) - NAME 'def' (4, 8) (4, 11) - NAME 'bar' (4, 12) (4, 15) - OP '(' (4, 15) (4, 16) - OP ')' (4, 16) (4, 17) - OP ':' (4, 17) (4, 18) - NAME 'pass' (4, 19) (4, 23) - NEWLINE '\\n' (4, 23) (4, 24) - NL '\\n' (5, 0) (5, 1) - NAME 'await' (6, 2) (6, 7) - OP '=' (6, 8) (6, 9) - NUMBER '2' (6, 10) (6, 11) - DEDENT '' (7, 0) (7, 0) - """) - -class GenerateTokensTest(TokenizeTest): - def check_tokenize(self, s, expected): - # Format the tokens in s in a table format. - # The ENDMARKER and final NEWLINE are omitted. - f = StringIO(s) - result = stringify_tokens_from_source(generate_tokens(f.readline), s) - self.assertEqual(result, expected.rstrip().splitlines()) - - -def decistmt(s): - result = [] - g = tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string - for toknum, tokval, _, _, _ in g: - if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens - result.extend([ - (NAME, 'Decimal'), - (OP, '('), - (STRING, repr(tokval)), - (OP, ')') - ]) - else: - result.append((toknum, tokval)) - return untokenize(result).decode('utf-8') - -class TestMisc(TestCase): - - def test_decistmt(self): - # Substitute Decimals for floats in a string of statements. - # This is an example from the docs. - - from decimal import Decimal - s = '+21.3e-5*-.1234/81.7' - self.assertEqual(decistmt(s), - "+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')") - - # The format of the exponent is inherited from the platform C library. - # Known cases are "e-007" (Windows) and "e-07" (not Windows). Since - # we're only showing 11 digits, and the 12th isn't close to 5, the - # rest of the output should be platform-independent. - self.assertRegex(repr(eval(s)), '-3.2171603427[0-9]*e-0+7') - - # Output from calculations with Decimal should be identical across all - # platforms. - self.assertEqual(eval(decistmt(s)), - Decimal('-3.217160342717258261933904529E-7')) - - -class TestTokenizerAdheresToPep0263(TestCase): - """ - Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263. - """ - - def _testFile(self, filename): - path = os.path.join(os.path.dirname(__file__), filename) - TestRoundtrip.check_roundtrip(self, open(path, 'rb')) - - def test_utf8_coding_cookie_and_no_utf8_bom(self): - f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt' - self._testFile(f) - - def test_latin1_coding_cookie_and_utf8_bom(self): - """ - As per PEP 0263, if a file starts with a utf-8 BOM signature, the only - allowed encoding for the comment is 'utf-8'. The text file used in - this test starts with a BOM signature, but specifies latin1 as the - coding, so verify that a SyntaxError is raised, which matches the - behaviour of the interpreter when it encounters a similar condition. - """ - f = 'tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt' - self.assertRaises(SyntaxError, self._testFile, f) - - def test_no_coding_cookie_and_utf8_bom(self): - f = 'tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt' - self._testFile(f) - - def test_utf8_coding_cookie_and_utf8_bom(self): - f = 'tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt' - self._testFile(f) - - def test_bad_coding_cookie(self): - self.assertRaises(SyntaxError, self._testFile, 'bad_coding.py') - self.assertRaises(SyntaxError, self._testFile, 'bad_coding2.py') - - -class Test_Tokenize(TestCase): - - def test__tokenize_decodes_with_specified_encoding(self): - literal = '"ЉЊЈЁЂ"' - line = literal.encode('utf-8') - first = False - def readline(): - nonlocal first - if not first: - first = True - return line - else: - return b'' - - # skip the initial encoding token and the end tokens - tokens = list(_tokenize(readline, encoding='utf-8'))[1:-2] - expected_tokens = [(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')] - self.assertEqual(tokens, expected_tokens, - "bytes not decoded with encoding") - - def test__tokenize_does_not_decode_with_encoding_none(self): - literal = '"ЉЊЈЁЂ"' - first = False - def readline(): - nonlocal first - if not first: - first = True - return literal - else: - return b'' - - # skip the end tokens - tokens = list(_tokenize(readline, encoding=None))[:-2] - expected_tokens = [(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')] - self.assertEqual(tokens, expected_tokens, - "string not tokenized when encoding is None") - - -class TestDetectEncoding(TestCase): - - def get_readline(self, lines): - index = 0 - def readline(): - nonlocal index - if index == len(lines): - raise StopIteration - line = lines[index] - index += 1 - return line - return readline - - def test_no_bom_no_encoding_cookie(self): - lines = ( - b'# something\n', - b'print(something)\n', - b'do_something(else)\n' - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'utf-8') - self.assertEqual(consumed_lines, list(lines[:2])) - - def test_bom_no_cookie(self): - lines = ( - b'\xef\xbb\xbf# something\n', - b'print(something)\n', - b'do_something(else)\n' - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'utf-8-sig') - self.assertEqual(consumed_lines, - [b'# something\n', b'print(something)\n']) - - def test_cookie_first_line_no_bom(self): - lines = ( - b'# -*- coding: latin-1 -*-\n', - b'print(something)\n', - b'do_something(else)\n' - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'iso-8859-1') - self.assertEqual(consumed_lines, [b'# -*- coding: latin-1 -*-\n']) - - def test_matched_bom_and_cookie_first_line(self): - lines = ( - b'\xef\xbb\xbf# coding=utf-8\n', - b'print(something)\n', - b'do_something(else)\n' - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'utf-8-sig') - self.assertEqual(consumed_lines, [b'# coding=utf-8\n']) - - def test_mismatched_bom_and_cookie_first_line_raises_syntaxerror(self): - lines = ( - b'\xef\xbb\xbf# vim: set fileencoding=ascii :\n', - b'print(something)\n', - b'do_something(else)\n' - ) - readline = self.get_readline(lines) - self.assertRaises(SyntaxError, detect_encoding, readline) - - def test_cookie_second_line_no_bom(self): - lines = ( - b'#! something\n', - b'# vim: set fileencoding=ascii :\n', - b'print(something)\n', - b'do_something(else)\n' - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'ascii') - expected = [b'#! something\n', b'# vim: set fileencoding=ascii :\n'] - self.assertEqual(consumed_lines, expected) - - def test_matched_bom_and_cookie_second_line(self): - lines = ( - b'\xef\xbb\xbf#! something\n', - b'f# coding=utf-8\n', - b'print(something)\n', - b'do_something(else)\n' - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'utf-8-sig') - self.assertEqual(consumed_lines, - [b'#! something\n', b'f# coding=utf-8\n']) - - def test_mismatched_bom_and_cookie_second_line_raises_syntaxerror(self): - lines = ( - b'\xef\xbb\xbf#! something\n', - b'# vim: set fileencoding=ascii :\n', - b'print(something)\n', - b'do_something(else)\n' - ) - readline = self.get_readline(lines) - self.assertRaises(SyntaxError, detect_encoding, readline) - - def test_cookie_second_line_noncommented_first_line(self): - lines = ( - b"print('\xc2\xa3')\n", - b'# vim: set fileencoding=iso8859-15 :\n', - b"print('\xe2\x82\xac')\n" - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'utf-8') - expected = [b"print('\xc2\xa3')\n"] - self.assertEqual(consumed_lines, expected) - - def test_cookie_second_line_commented_first_line(self): - lines = ( - b"#print('\xc2\xa3')\n", - b'# vim: set fileencoding=iso8859-15 :\n', - b"print('\xe2\x82\xac')\n" - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'iso8859-15') - expected = [b"#print('\xc2\xa3')\n", b'# vim: set fileencoding=iso8859-15 :\n'] - self.assertEqual(consumed_lines, expected) - - def test_cookie_second_line_empty_first_line(self): - lines = ( - b'\n', - b'# vim: set fileencoding=iso8859-15 :\n', - b"print('\xe2\x82\xac')\n" - ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'iso8859-15') - expected = [b'\n', b'# vim: set fileencoding=iso8859-15 :\n'] - self.assertEqual(consumed_lines, expected) - - def test_latin1_normalization(self): - # See get_normal_name() in tokenizer.c. - encodings = ("latin-1", "iso-8859-1", "iso-latin-1", "latin-1-unix", - "iso-8859-1-unix", "iso-latin-1-mac") - for encoding in encodings: - for rep in ("-", "_"): - enc = encoding.replace("-", rep) - lines = (b"#!/usr/bin/python\n", - b"# coding: " + enc.encode("ascii") + b"\n", - b"print(things)\n", - b"do_something += 4\n") - rl = self.get_readline(lines) - found, consumed_lines = detect_encoding(rl) - self.assertEqual(found, "iso-8859-1") - - def test_syntaxerror_latin1(self): - # Issue 14629: need to raise SyntaxError if the first - # line(s) have non-UTF-8 characters - lines = ( - b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S - ) - readline = self.get_readline(lines) - self.assertRaises(SyntaxError, detect_encoding, readline) - - - def test_utf8_normalization(self): - # See get_normal_name() in tokenizer.c. - encodings = ("utf-8", "utf-8-mac", "utf-8-unix") - for encoding in encodings: - for rep in ("-", "_"): - enc = encoding.replace("-", rep) - lines = (b"#!/usr/bin/python\n", - b"# coding: " + enc.encode("ascii") + b"\n", - b"1 + 3\n") - rl = self.get_readline(lines) - found, consumed_lines = detect_encoding(rl) - self.assertEqual(found, "utf-8") - - def test_short_files(self): - readline = self.get_readline((b'print(something)\n',)) - encoding, consumed_lines = detect_encoding(readline) - self.assertEqual(encoding, 'utf-8') - self.assertEqual(consumed_lines, [b'print(something)\n']) - - encoding, consumed_lines = detect_encoding(self.get_readline(())) - self.assertEqual(encoding, 'utf-8') - self.assertEqual(consumed_lines, []) - - readline = self.get_readline((b'\xef\xbb\xbfprint(something)\n',)) - encoding, consumed_lines = detect_encoding(readline) - self.assertEqual(encoding, 'utf-8-sig') - self.assertEqual(consumed_lines, [b'print(something)\n']) - - readline = self.get_readline((b'\xef\xbb\xbf',)) - encoding, consumed_lines = detect_encoding(readline) - self.assertEqual(encoding, 'utf-8-sig') - self.assertEqual(consumed_lines, []) - - readline = self.get_readline((b'# coding: bad\n',)) - self.assertRaises(SyntaxError, detect_encoding, readline) - - def test_false_encoding(self): - # Issue 18873: "Encoding" detected in non-comment lines - readline = self.get_readline((b'print("#coding=fake")',)) - encoding, consumed_lines = detect_encoding(readline) - self.assertEqual(encoding, 'utf-8') - self.assertEqual(consumed_lines, [b'print("#coding=fake")']) - - def test_open(self): - filename = os_helper.TESTFN + '.py' - self.addCleanup(os_helper.unlink, filename) - - # test coding cookie - for encoding in ('iso-8859-15', 'utf-8'): - with open(filename, 'w', encoding=encoding) as fp: - print("# coding: %s" % encoding, file=fp) - print("print('euro:\u20ac')", file=fp) - with tokenize_open(filename) as fp: - self.assertEqual(fp.encoding, encoding) - self.assertEqual(fp.mode, 'r') - - # test BOM (no coding cookie) - with open(filename, 'w', encoding='utf-8-sig') as fp: - print("print('euro:\u20ac')", file=fp) - with tokenize_open(filename) as fp: - self.assertEqual(fp.encoding, 'utf-8-sig') - self.assertEqual(fp.mode, 'r') - - def test_filename_in_exception(self): - # When possible, include the file name in the exception. - path = 'some_file_path' - lines = ( - b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S - ) - class Bunk: - def __init__(self, lines, path): - self.name = path - self._lines = lines - self._index = 0 - - def readline(self): - if self._index == len(lines): - raise StopIteration - line = lines[self._index] - self._index += 1 - return line - - with self.assertRaises(SyntaxError): - ins = Bunk(lines, path) - # Make sure lacking a name isn't an issue. - del ins.name - detect_encoding(ins.readline) - with self.assertRaisesRegex(SyntaxError, '.*{}'.format(path)): - ins = Bunk(lines, path) - detect_encoding(ins.readline) - - def test_open_error(self): - # Issue #23840: open() must close the binary file on error - m = BytesIO(b'#coding:xxx') - with mock.patch('tokenize._builtin_open', return_value=m): - self.assertRaises(SyntaxError, tokenize_open, 'foobar') - self.assertTrue(m.closed) - - -class TestTokenize(TestCase): - - def test_tokenize(self): - import tokenize as tokenize_module - encoding = object() - encoding_used = None - def mock_detect_encoding(readline): - return encoding, [b'first', b'second'] - - def mock__tokenize(readline, encoding): - nonlocal encoding_used - encoding_used = encoding - out = [] - while True: - next_line = readline() - if next_line: - out.append(next_line) - continue - return out - - counter = 0 - def mock_readline(): - nonlocal counter - counter += 1 - if counter == 5: - return b'' - return str(counter).encode() - - orig_detect_encoding = tokenize_module.detect_encoding - orig__tokenize = tokenize_module._tokenize - tokenize_module.detect_encoding = mock_detect_encoding - tokenize_module._tokenize = mock__tokenize - try: - results = tokenize(mock_readline) - self.assertEqual(list(results), - [b'first', b'second', b'1', b'2', b'3', b'4']) - finally: - tokenize_module.detect_encoding = orig_detect_encoding - tokenize_module._tokenize = orig__tokenize - - self.assertEqual(encoding_used, encoding) - - def test_oneline_defs(self): - buf = [] - for i in range(500): - buf.append('def i{i}(): return {i}'.format(i=i)) - buf.append('OK') - buf = '\n'.join(buf) - - # Test that 500 consequent, one-line defs is OK - toks = list(tokenize(BytesIO(buf.encode('utf-8')).readline)) - self.assertEqual(toks[-3].string, 'OK') # [-1] is always ENDMARKER - # [-2] is always NEWLINE - - def assertExactTypeEqual(self, opstr, *optypes): - tokens = list(tokenize(BytesIO(opstr.encode('utf-8')).readline)) - num_optypes = len(optypes) - self.assertEqual(len(tokens), 3 + num_optypes) - self.assertEqual(tok_name[tokens[0].exact_type], - tok_name[ENCODING]) - for i in range(num_optypes): - self.assertEqual(tok_name[tokens[i + 1].exact_type], - tok_name[optypes[i]]) - self.assertEqual(tok_name[tokens[1 + num_optypes].exact_type], - tok_name[token.NEWLINE]) - self.assertEqual(tok_name[tokens[2 + num_optypes].exact_type], - tok_name[token.ENDMARKER]) - - def test_exact_type(self): - self.assertExactTypeEqual('()', token.LPAR, token.RPAR) - self.assertExactTypeEqual('[]', token.LSQB, token.RSQB) - self.assertExactTypeEqual(':', token.COLON) - self.assertExactTypeEqual(',', token.COMMA) - self.assertExactTypeEqual(';', token.SEMI) - self.assertExactTypeEqual('+', token.PLUS) - self.assertExactTypeEqual('-', token.MINUS) - self.assertExactTypeEqual('*', token.STAR) - self.assertExactTypeEqual('/', token.SLASH) - self.assertExactTypeEqual('|', token.VBAR) - self.assertExactTypeEqual('&', token.AMPER) - self.assertExactTypeEqual('<', token.LESS) - self.assertExactTypeEqual('>', token.GREATER) - self.assertExactTypeEqual('=', token.EQUAL) - self.assertExactTypeEqual('.', token.DOT) - self.assertExactTypeEqual('%', token.PERCENT) - self.assertExactTypeEqual('{}', token.LBRACE, token.RBRACE) - self.assertExactTypeEqual('==', token.EQEQUAL) - self.assertExactTypeEqual('!=', token.NOTEQUAL) - self.assertExactTypeEqual('<=', token.LESSEQUAL) - self.assertExactTypeEqual('>=', token.GREATEREQUAL) - self.assertExactTypeEqual('~', token.TILDE) - self.assertExactTypeEqual('^', token.CIRCUMFLEX) - self.assertExactTypeEqual('<<', token.LEFTSHIFT) - self.assertExactTypeEqual('>>', token.RIGHTSHIFT) - self.assertExactTypeEqual('**', token.DOUBLESTAR) - self.assertExactTypeEqual('+=', token.PLUSEQUAL) - self.assertExactTypeEqual('-=', token.MINEQUAL) - self.assertExactTypeEqual('*=', token.STAREQUAL) - self.assertExactTypeEqual('/=', token.SLASHEQUAL) - self.assertExactTypeEqual('%=', token.PERCENTEQUAL) - self.assertExactTypeEqual('&=', token.AMPEREQUAL) - self.assertExactTypeEqual('|=', token.VBAREQUAL) - self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL) - self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL) - self.assertExactTypeEqual('<<=', token.LEFTSHIFTEQUAL) - self.assertExactTypeEqual('>>=', token.RIGHTSHIFTEQUAL) - self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL) - self.assertExactTypeEqual('//', token.DOUBLESLASH) - self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL) - self.assertExactTypeEqual(':=', token.COLONEQUAL) - self.assertExactTypeEqual('...', token.ELLIPSIS) - self.assertExactTypeEqual('->', token.RARROW) - self.assertExactTypeEqual('@', token.AT) - self.assertExactTypeEqual('@=', token.ATEQUAL) - - self.assertExactTypeEqual('a**2+b**2==c**2', - NAME, token.DOUBLESTAR, NUMBER, - token.PLUS, - NAME, token.DOUBLESTAR, NUMBER, - token.EQEQUAL, - NAME, token.DOUBLESTAR, NUMBER) - self.assertExactTypeEqual('{1, 2, 3}', - token.LBRACE, - token.NUMBER, token.COMMA, - token.NUMBER, token.COMMA, - token.NUMBER, - token.RBRACE) - self.assertExactTypeEqual('^(x & 0x1)', - token.CIRCUMFLEX, - token.LPAR, - token.NAME, token.AMPER, token.NUMBER, - token.RPAR) - - def test_pathological_trailing_whitespace(self): - # See http://bugs.python.org/issue16152 - self.assertExactTypeEqual('@ ', token.AT) - - def test_comment_at_the_end_of_the_source_without_newline(self): - # See http://bugs.python.org/issue44667 - source = 'b = 1\n\n#test' - expected_tokens = [token.NAME, token.EQUAL, token.NUMBER, token.NEWLINE, token.NL, token.COMMENT] - - tokens = list(tokenize(BytesIO(source.encode('utf-8')).readline)) - self.assertEqual(tok_name[tokens[0].exact_type], tok_name[ENCODING]) - for i in range(6): - self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]]) - self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER]) - -class UntokenizeTest(TestCase): - - def test_bad_input_order(self): - # raise if previous row - u = Untokenizer() - u.prev_row = 2 - u.prev_col = 2 - with self.assertRaises(ValueError) as cm: - u.add_whitespace((1,3)) - self.assertEqual(cm.exception.args[0], - 'start (1,3) precedes previous end (2,2)') - # raise if previous column in row - self.assertRaises(ValueError, u.add_whitespace, (2,1)) - - def test_backslash_continuation(self): - # The problem is that \ leaves no token - u = Untokenizer() - u.prev_row = 1 - u.prev_col = 1 - u.tokens = [] - u.add_whitespace((2, 0)) - self.assertEqual(u.tokens, ['\\\n']) - u.prev_row = 2 - u.add_whitespace((4, 4)) - self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', ' ']) - TestRoundtrip.check_roundtrip(self, 'a\n b\n c\n \\\n c\n') - - def test_iter_compat(self): - u = Untokenizer() - token = (NAME, 'Hello') - tokens = [(ENCODING, 'utf-8'), token] - u.compat(token, iter([])) - self.assertEqual(u.tokens, ["Hello "]) - u = Untokenizer() - self.assertEqual(u.untokenize(iter([token])), 'Hello ') - u = Untokenizer() - self.assertEqual(u.untokenize(iter(tokens)), 'Hello ') - self.assertEqual(u.encoding, 'utf-8') - self.assertEqual(untokenize(iter(tokens)), b'Hello ') - - -class TestRoundtrip(TestCase): - - def check_roundtrip(self, f): - """ - Test roundtrip for `untokenize`. `f` is an open file or a string. - The source code in f is tokenized to both 5- and 2-tuples. - Both sequences are converted back to source code via - tokenize.untokenize(), and the latter tokenized again to 2-tuples. - The test fails if the 3 pair tokenizations do not match. - - When untokenize bugs are fixed, untokenize with 5-tuples should - reproduce code that does not contain a backslash continuation - following spaces. A proper test should test this. - """ - # Get source code and original tokenizations - if isinstance(f, str): - code = f.encode('utf-8') - else: - code = f.read() - f.close() - readline = iter(code.splitlines(keepends=True)).__next__ - tokens5 = list(tokenize(readline)) - tokens2 = [tok[:2] for tok in tokens5] - # Reproduce tokens2 from pairs - bytes_from2 = untokenize(tokens2) - readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__ - tokens2_from2 = [tok[:2] for tok in tokenize(readline2)] - self.assertEqual(tokens2_from2, tokens2) - # Reproduce tokens2 from 5-tuples - bytes_from5 = untokenize(tokens5) - readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__ - tokens2_from5 = [tok[:2] for tok in tokenize(readline5)] - self.assertEqual(tokens2_from5, tokens2) - - def test_roundtrip(self): - # There are some standard formatting practices that are easy to get right. - - self.check_roundtrip("if x == 1:\n" - " print(x)\n") - self.check_roundtrip("# This is a comment\n" - "# This also\n") - - # Some people use different formatting conventions, which makes - # untokenize a little trickier. Note that this test involves trailing - # whitespace after the colon. Note that we use hex escapes to make the - # two trailing blanks apparent in the expected output. - - self.check_roundtrip("if x == 1 : \n" - " print(x)\n") - fn = support.findfile("tokenize_tests.txt") - with open(fn, 'rb') as f: - self.check_roundtrip(f) - self.check_roundtrip("if x == 1:\n" - " # A comment by itself.\n" - " print(x) # Comment here, too.\n" - " # Another comment.\n" - "after_if = True\n") - self.check_roundtrip("if (x # The comments need to go in the right place\n" - " == 1):\n" - " print('x==1')\n") - self.check_roundtrip("class Test: # A comment here\n" - " # A comment with weird indent\n" - " after_com = 5\n" - " def x(m): return m*5 # a one liner\n" - " def y(m): # A whitespace after the colon\n" - " return y*4 # 3-space indent\n") - - # Some error-handling code - self.check_roundtrip("try: import somemodule\n" - "except ImportError: # comment\n" - " print('Can not import' # comment2\n)" - "else: print('Loaded')\n") - - def test_continuation(self): - # Balancing continuation - self.check_roundtrip("a = (3,4, \n" - "5,6)\n" - "y = [3, 4,\n" - "5]\n" - "z = {'a': 5,\n" - "'b':15, 'c':True}\n" - "x = len(y) + 5 - a[\n" - "3] - a[2]\n" - "+ len(z) - z[\n" - "'b']\n") - - def test_backslash_continuation(self): - # Backslash means line continuation, except for comments - self.check_roundtrip("x=1+\\\n" - "1\n" - "# This is a comment\\\n" - "# This also\n") - self.check_roundtrip("# Comment \\\n" - "x = 0") - - def test_string_concatenation(self): - # Two string literals on the same line - self.check_roundtrip("'' ''") - - def test_random_files(self): - # Test roundtrip on random python modules. - # pass the '-ucpu' option to process the full directory. - - import glob, random - fn = support.findfile("tokenize_tests.txt") - tempdir = os.path.dirname(fn) or os.curdir - testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py")) - - # Tokenize is broken on test_pep3131.py because regular expressions are - # broken on the obscure unicode identifiers in it. *sigh* - # With roundtrip extended to test the 5-tuple mode of untokenize, - # 7 more testfiles fail. Remove them also until the failure is diagnosed. - - testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py")) - for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'): - testfiles.remove(os.path.join(tempdir, "test_%s.py") % f) - - if not support.is_resource_enabled("cpu"): - testfiles = random.sample(testfiles, 10) - - for testfile in testfiles: - if support.verbose >= 2: - print('tokenize', testfile) - with open(testfile, 'rb') as f: - with self.subTest(file=testfile): - self.check_roundtrip(f) - - - def roundtrip(self, code): - if isinstance(code, str): - code = code.encode('utf-8') - return untokenize(tokenize(BytesIO(code).readline)).decode('utf-8') - - def test_indentation_semantics_retained(self): - """ - Ensure that although whitespace might be mutated in a roundtrip, - the semantic meaning of the indentation remains consistent. - """ - code = "if False:\n\tx=3\n\tx=3\n" - codelines = self.roundtrip(code).split('\n') - self.assertEqual(codelines[1], codelines[2]) - self.check_roundtrip(code) - - -class CTokenizeTest(TestCase): - def check_tokenize(self, s, expected): - # Format the tokens in s in a table format. - # The ENDMARKER and final NEWLINE are omitted. - with self.subTest(source=s): - result = stringify_tokens_from_source( - _generate_tokens_from_c_tokenizer(s), s - ) - self.assertEqual(result, expected.rstrip().splitlines()) - - def test_int(self): - - self.check_tokenize('0xff <= 255', """\ - NUMBER '0xff' (1, 0) (1, 4) - LESSEQUAL '<=' (1, 5) (1, 7) - NUMBER '255' (1, 8) (1, 11) - """) - - self.check_tokenize('0b10 <= 255', """\ - NUMBER '0b10' (1, 0) (1, 4) - LESSEQUAL '<=' (1, 5) (1, 7) - NUMBER '255' (1, 8) (1, 11) - """) - - self.check_tokenize('0o123 <= 0O123', """\ - NUMBER '0o123' (1, 0) (1, 5) - LESSEQUAL '<=' (1, 6) (1, 8) - NUMBER '0O123' (1, 9) (1, 14) - """) - - self.check_tokenize('1234567 > ~0x15', """\ - NUMBER '1234567' (1, 0) (1, 7) - GREATER '>' (1, 8) (1, 9) - TILDE '~' (1, 10) (1, 11) - NUMBER '0x15' (1, 11) (1, 15) - """) - - self.check_tokenize('2134568 != 1231515', """\ - NUMBER '2134568' (1, 0) (1, 7) - NOTEQUAL '!=' (1, 8) (1, 10) - NUMBER '1231515' (1, 11) (1, 18) - """) - - self.check_tokenize('(-124561-1) & 200000000', """\ - LPAR '(' (1, 0) (1, 1) - MINUS '-' (1, 1) (1, 2) - NUMBER '124561' (1, 2) (1, 8) - MINUS '-' (1, 8) (1, 9) - NUMBER '1' (1, 9) (1, 10) - RPAR ')' (1, 10) (1, 11) - AMPER '&' (1, 12) (1, 13) - NUMBER '200000000' (1, 14) (1, 23) - """) - - self.check_tokenize('0xdeadbeef != -1', """\ - NUMBER '0xdeadbeef' (1, 0) (1, 10) - NOTEQUAL '!=' (1, 11) (1, 13) - MINUS '-' (1, 14) (1, 15) - NUMBER '1' (1, 15) (1, 16) - """) - - self.check_tokenize('0xdeadc0de & 12345', """\ - NUMBER '0xdeadc0de' (1, 0) (1, 10) - AMPER '&' (1, 11) (1, 12) - NUMBER '12345' (1, 13) (1, 18) - """) - - self.check_tokenize('0xFF & 0x15 | 1234', """\ - NUMBER '0xFF' (1, 0) (1, 4) - AMPER '&' (1, 5) (1, 6) - NUMBER '0x15' (1, 7) (1, 11) - VBAR '|' (1, 12) (1, 13) - NUMBER '1234' (1, 14) (1, 18) - """) - - def test_float(self): - - self.check_tokenize('x = 3.14159', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - NUMBER '3.14159' (1, 4) (1, 11) - """) - - self.check_tokenize('x = 314159.', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - NUMBER '314159.' (1, 4) (1, 11) - """) - - self.check_tokenize('x = .314159', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - NUMBER '.314159' (1, 4) (1, 11) - """) - - self.check_tokenize('x = 3e14159', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - NUMBER '3e14159' (1, 4) (1, 11) - """) - - self.check_tokenize('x = 3E123', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - NUMBER '3E123' (1, 4) (1, 9) - """) - - self.check_tokenize('x+y = 3e-1230', """\ - NAME 'x' (1, 0) (1, 1) - PLUS '+' (1, 1) (1, 2) - NAME 'y' (1, 2) (1, 3) - EQUAL '=' (1, 4) (1, 5) - NUMBER '3e-1230' (1, 6) (1, 13) - """) - - self.check_tokenize('x = 3.14e159', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - NUMBER '3.14e159' (1, 4) (1, 12) - """) - - def test_string(self): - - self.check_tokenize('x = \'\'; y = ""', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - STRING "''" (1, 4) (1, 6) - SEMI ';' (1, 6) (1, 7) - NAME 'y' (1, 8) (1, 9) - EQUAL '=' (1, 10) (1, 11) - STRING '""' (1, 12) (1, 14) - """) - - self.check_tokenize('x = \'"\'; y = "\'"', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - STRING '\\'"\\'' (1, 4) (1, 7) - SEMI ';' (1, 7) (1, 8) - NAME 'y' (1, 9) (1, 10) - EQUAL '=' (1, 11) (1, 12) - STRING '"\\'"' (1, 13) (1, 16) - """) - - self.check_tokenize('x = "doesn\'t "shrink", does it"', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - STRING '"doesn\\'t "' (1, 4) (1, 14) - NAME 'shrink' (1, 14) (1, 20) - STRING '", does it"' (1, 20) (1, 31) - """) - - self.check_tokenize("x = 'abc' + 'ABC'", """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - STRING "'abc'" (1, 4) (1, 9) - PLUS '+' (1, 10) (1, 11) - STRING "'ABC'" (1, 12) (1, 17) - """) - - self.check_tokenize('y = "ABC" + "ABC"', """\ - NAME 'y' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - STRING '"ABC"' (1, 4) (1, 9) - PLUS '+' (1, 10) (1, 11) - STRING '"ABC"' (1, 12) (1, 17) - """) - - self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - STRING "r'abc'" (1, 4) (1, 10) - PLUS '+' (1, 11) (1, 12) - STRING "r'ABC'" (1, 13) (1, 19) - PLUS '+' (1, 20) (1, 21) - STRING "R'ABC'" (1, 22) (1, 28) - PLUS '+' (1, 29) (1, 30) - STRING "R'ABC'" (1, 31) (1, 37) - """) - - self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\ - NAME 'y' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - STRING 'r"abc"' (1, 4) (1, 10) - PLUS '+' (1, 11) (1, 12) - STRING 'r"ABC"' (1, 13) (1, 19) - PLUS '+' (1, 20) (1, 21) - STRING 'R"ABC"' (1, 22) (1, 28) - PLUS '+' (1, 29) (1, 30) - STRING 'R"ABC"' (1, 31) (1, 37) - """) - - self.check_tokenize("u'abc' + U'abc'", """\ - STRING "u'abc'" (1, 0) (1, 6) - PLUS '+' (1, 7) (1, 8) - STRING "U'abc'" (1, 9) (1, 15) - """) - - self.check_tokenize('u"abc" + U"abc"', """\ - STRING 'u"abc"' (1, 0) (1, 6) - PLUS '+' (1, 7) (1, 8) - STRING 'U"abc"' (1, 9) (1, 15) - """) - - self.check_tokenize("b'abc' + B'abc'", """\ - STRING "b'abc'" (1, 0) (1, 6) - PLUS '+' (1, 7) (1, 8) - STRING "B'abc'" (1, 9) (1, 15) - """) - - self.check_tokenize('b"abc" + B"abc"', """\ - STRING 'b"abc"' (1, 0) (1, 6) - PLUS '+' (1, 7) (1, 8) - STRING 'B"abc"' (1, 9) (1, 15) - """) - - self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\ - STRING "br'abc'" (1, 0) (1, 7) - PLUS '+' (1, 8) (1, 9) - STRING "bR'abc'" (1, 10) (1, 17) - PLUS '+' (1, 18) (1, 19) - STRING "Br'abc'" (1, 20) (1, 27) - PLUS '+' (1, 28) (1, 29) - STRING "BR'abc'" (1, 30) (1, 37) - """) - - self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\ - STRING 'br"abc"' (1, 0) (1, 7) - PLUS '+' (1, 8) (1, 9) - STRING 'bR"abc"' (1, 10) (1, 17) - PLUS '+' (1, 18) (1, 19) - STRING 'Br"abc"' (1, 20) (1, 27) - PLUS '+' (1, 28) (1, 29) - STRING 'BR"abc"' (1, 30) (1, 37) - """) - - self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\ - STRING "rb'abc'" (1, 0) (1, 7) - PLUS '+' (1, 8) (1, 9) - STRING "rB'abc'" (1, 10) (1, 17) - PLUS '+' (1, 18) (1, 19) - STRING "Rb'abc'" (1, 20) (1, 27) - PLUS '+' (1, 28) (1, 29) - STRING "RB'abc'" (1, 30) (1, 37) - """) - - self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\ - STRING 'rb"abc"' (1, 0) (1, 7) - PLUS '+' (1, 8) (1, 9) - STRING 'rB"abc"' (1, 10) (1, 17) - PLUS '+' (1, 18) (1, 19) - STRING 'Rb"abc"' (1, 20) (1, 27) - PLUS '+' (1, 28) (1, 29) - STRING 'RB"abc"' (1, 30) (1, 37) - """) - - self.check_tokenize('"a\\\nde\\\nfg"', """\ - STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3) - """) - - self.check_tokenize('u"a\\\nde"', """\ - STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3) - """) - - self.check_tokenize('rb"a\\\nd"', """\ - STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2) - """) - - self.check_tokenize(r'"""a\ -b"""', """\ - STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) - """) - self.check_tokenize(r'u"""a\ -b"""', """\ - STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) - """) - self.check_tokenize(r'rb"""a\ -b\ -c"""', """\ - STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) - """) - - self.check_tokenize('f"abc"', """\ - STRING 'f"abc"' (1, 0) (1, 6) - """) - - self.check_tokenize('fR"a{b}c"', """\ - STRING 'fR"a{b}c"' (1, 0) (1, 9) - """) - - self.check_tokenize('f"""abc"""', """\ - STRING 'f\"\"\"abc\"\"\"' (1, 0) (1, 10) - """) - - self.check_tokenize(r'f"abc\ -def"', """\ - STRING 'f"abc\\\\\\ndef"' (1, 0) (2, 4) - """) - - self.check_tokenize(r'Rf"abc\ -def"', """\ - STRING 'Rf"abc\\\\\\ndef"' (1, 0) (2, 4) - """) - - def test_function(self): - - self.check_tokenize('def d22(a, b, c=2, d=2, *k): pass', """\ - NAME 'def' (1, 0) (1, 3) - NAME 'd22' (1, 4) (1, 7) - LPAR '(' (1, 7) (1, 8) - NAME 'a' (1, 8) (1, 9) - COMMA ',' (1, 9) (1, 10) - NAME 'b' (1, 11) (1, 12) - COMMA ',' (1, 12) (1, 13) - NAME 'c' (1, 14) (1, 15) - EQUAL '=' (1, 15) (1, 16) - NUMBER '2' (1, 16) (1, 17) - COMMA ',' (1, 17) (1, 18) - NAME 'd' (1, 19) (1, 20) - EQUAL '=' (1, 20) (1, 21) - NUMBER '2' (1, 21) (1, 22) - COMMA ',' (1, 22) (1, 23) - STAR '*' (1, 24) (1, 25) - NAME 'k' (1, 25) (1, 26) - RPAR ')' (1, 26) (1, 27) - COLON ':' (1, 27) (1, 28) - NAME 'pass' (1, 29) (1, 33) - """) - - self.check_tokenize('def d01v_(a=1, *k, **w): pass', """\ - NAME 'def' (1, 0) (1, 3) - NAME 'd01v_' (1, 4) (1, 9) - LPAR '(' (1, 9) (1, 10) - NAME 'a' (1, 10) (1, 11) - EQUAL '=' (1, 11) (1, 12) - NUMBER '1' (1, 12) (1, 13) - COMMA ',' (1, 13) (1, 14) - STAR '*' (1, 15) (1, 16) - NAME 'k' (1, 16) (1, 17) - COMMA ',' (1, 17) (1, 18) - DOUBLESTAR '**' (1, 19) (1, 21) - NAME 'w' (1, 21) (1, 22) - RPAR ')' (1, 22) (1, 23) - COLON ':' (1, 23) (1, 24) - NAME 'pass' (1, 25) (1, 29) - """) - - self.check_tokenize('def d23(a: str, b: int=3) -> int: pass', """\ - NAME 'def' (1, 0) (1, 3) - NAME 'd23' (1, 4) (1, 7) - LPAR '(' (1, 7) (1, 8) - NAME 'a' (1, 8) (1, 9) - COLON ':' (1, 9) (1, 10) - NAME 'str' (1, 11) (1, 14) - COMMA ',' (1, 14) (1, 15) - NAME 'b' (1, 16) (1, 17) - COLON ':' (1, 17) (1, 18) - NAME 'int' (1, 19) (1, 22) - EQUAL '=' (1, 22) (1, 23) - NUMBER '3' (1, 23) (1, 24) - RPAR ')' (1, 24) (1, 25) - RARROW '->' (1, 26) (1, 28) - NAME 'int' (1, 29) (1, 32) - COLON ':' (1, 32) (1, 33) - NAME 'pass' (1, 34) (1, 38) - """) - - def test_comparison(self): - - self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " - "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\ - NAME 'if' (1, 0) (1, 2) - NUMBER '1' (1, 3) (1, 4) - LESS '<' (1, 5) (1, 6) - NUMBER '1' (1, 7) (1, 8) - GREATER '>' (1, 9) (1, 10) - NUMBER '1' (1, 11) (1, 12) - EQEQUAL '==' (1, 13) (1, 15) - NUMBER '1' (1, 16) (1, 17) - GREATEREQUAL '>=' (1, 18) (1, 20) - NUMBER '5' (1, 21) (1, 22) - LESSEQUAL '<=' (1, 23) (1, 25) - NUMBER '0x15' (1, 26) (1, 30) - LESSEQUAL '<=' (1, 31) (1, 33) - NUMBER '0x12' (1, 34) (1, 38) - NOTEQUAL '!=' (1, 39) (1, 41) - NUMBER '1' (1, 42) (1, 43) - NAME 'and' (1, 44) (1, 47) - NUMBER '5' (1, 48) (1, 49) - NAME 'in' (1, 50) (1, 52) - NUMBER '1' (1, 53) (1, 54) - NAME 'not' (1, 55) (1, 58) - NAME 'in' (1, 59) (1, 61) - NUMBER '1' (1, 62) (1, 63) - NAME 'is' (1, 64) (1, 66) - NUMBER '1' (1, 67) (1, 68) - NAME 'or' (1, 69) (1, 71) - NUMBER '5' (1, 72) (1, 73) - NAME 'is' (1, 74) (1, 76) - NAME 'not' (1, 77) (1, 80) - NUMBER '1' (1, 81) (1, 82) - COLON ':' (1, 82) (1, 83) - NAME 'pass' (1, 84) (1, 88) - """) - - def test_additive(self): - - self.check_tokenize('x = 1 - y + 15 - 1 + 0x124 + z + a[5]', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - NUMBER '1' (1, 4) (1, 5) - MINUS '-' (1, 6) (1, 7) - NAME 'y' (1, 8) (1, 9) - PLUS '+' (1, 10) (1, 11) - NUMBER '15' (1, 12) (1, 14) - MINUS '-' (1, 15) (1, 16) - NUMBER '1' (1, 17) (1, 18) - PLUS '+' (1, 19) (1, 20) - NUMBER '0x124' (1, 21) (1, 26) - PLUS '+' (1, 27) (1, 28) - NAME 'z' (1, 29) (1, 30) - PLUS '+' (1, 31) (1, 32) - NAME 'a' (1, 33) (1, 34) - LSQB '[' (1, 34) (1, 35) - NUMBER '5' (1, 35) (1, 36) - RSQB ']' (1, 36) (1, 37) - """) - - def test_multiplicative(self): - - self.check_tokenize('x = 1//1*1/5*12%0x12@42', """\ - NAME 'x' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - NUMBER '1' (1, 4) (1, 5) - DOUBLESLASH '//' (1, 5) (1, 7) - NUMBER '1' (1, 7) (1, 8) - STAR '*' (1, 8) (1, 9) - NUMBER '1' (1, 9) (1, 10) - SLASH '/' (1, 10) (1, 11) - NUMBER '5' (1, 11) (1, 12) - STAR '*' (1, 12) (1, 13) - NUMBER '12' (1, 13) (1, 15) - PERCENT '%' (1, 15) (1, 16) - NUMBER '0x12' (1, 16) (1, 20) - AT '@' (1, 20) (1, 21) - NUMBER '42' (1, 21) (1, 23) - """) - - def test_unary(self): - - self.check_tokenize('~1 ^ 1 & 1 |1 ^ -1', """\ - TILDE '~' (1, 0) (1, 1) - NUMBER '1' (1, 1) (1, 2) - CIRCUMFLEX '^' (1, 3) (1, 4) - NUMBER '1' (1, 5) (1, 6) - AMPER '&' (1, 7) (1, 8) - NUMBER '1' (1, 9) (1, 10) - VBAR '|' (1, 11) (1, 12) - NUMBER '1' (1, 12) (1, 13) - CIRCUMFLEX '^' (1, 14) (1, 15) - MINUS '-' (1, 16) (1, 17) - NUMBER '1' (1, 17) (1, 18) - """) - - self.check_tokenize('-1*1/1+1*1//1 - ---1**1', """\ - MINUS '-' (1, 0) (1, 1) - NUMBER '1' (1, 1) (1, 2) - STAR '*' (1, 2) (1, 3) - NUMBER '1' (1, 3) (1, 4) - SLASH '/' (1, 4) (1, 5) - NUMBER '1' (1, 5) (1, 6) - PLUS '+' (1, 6) (1, 7) - NUMBER '1' (1, 7) (1, 8) - STAR '*' (1, 8) (1, 9) - NUMBER '1' (1, 9) (1, 10) - DOUBLESLASH '//' (1, 10) (1, 12) - NUMBER '1' (1, 12) (1, 13) - MINUS '-' (1, 14) (1, 15) - MINUS '-' (1, 16) (1, 17) - MINUS '-' (1, 17) (1, 18) - MINUS '-' (1, 18) (1, 19) - NUMBER '1' (1, 19) (1, 20) - DOUBLESTAR '**' (1, 20) (1, 22) - NUMBER '1' (1, 22) (1, 23) - """) - - def test_selector(self): - - self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\ - NAME 'import' (1, 0) (1, 6) - NAME 'sys' (1, 7) (1, 10) - COMMA ',' (1, 10) (1, 11) - NAME 'time' (1, 12) (1, 16) - NEWLINE '' (1, 16) (1, 16) - NAME 'x' (2, 0) (2, 1) - EQUAL '=' (2, 2) (2, 3) - NAME 'sys' (2, 4) (2, 7) - DOT '.' (2, 7) (2, 8) - NAME 'modules' (2, 8) (2, 15) - LSQB '[' (2, 15) (2, 16) - STRING "'time'" (2, 16) (2, 22) - RSQB ']' (2, 22) (2, 23) - DOT '.' (2, 23) (2, 24) - NAME 'time' (2, 24) (2, 28) - LPAR '(' (2, 28) (2, 29) - RPAR ')' (2, 29) (2, 30) - """) - - def test_method(self): - - self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\ - AT '@' (1, 0) (1, 1) - NAME 'staticmethod' (1, 1) (1, 13) - NEWLINE '' (1, 13) (1, 13) - NAME 'def' (2, 0) (2, 3) - NAME 'foo' (2, 4) (2, 7) - LPAR '(' (2, 7) (2, 8) - NAME 'x' (2, 8) (2, 9) - COMMA ',' (2, 9) (2, 10) - NAME 'y' (2, 10) (2, 11) - RPAR ')' (2, 11) (2, 12) - COLON ':' (2, 12) (2, 13) - NAME 'pass' (2, 14) (2, 18) - """) - - def test_tabs(self): - - self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\ - AT '@' (1, 0) (1, 1) - NAME 'staticmethod' (1, 1) (1, 13) - NEWLINE '' (1, 13) (1, 13) - NAME 'def' (2, 0) (2, 3) - NAME 'foo' (2, 4) (2, 7) - LPAR '(' (2, 7) (2, 8) - NAME 'x' (2, 8) (2, 9) - COMMA ',' (2, 9) (2, 10) - NAME 'y' (2, 10) (2, 11) - RPAR ')' (2, 11) (2, 12) - COLON ':' (2, 12) (2, 13) - NAME 'pass' (2, 14) (2, 18) - """) - - def test_async(self): - - self.check_tokenize('async = 1', """\ - ASYNC 'async' (1, 0) (1, 5) - EQUAL '=' (1, 6) (1, 7) - NUMBER '1' (1, 8) (1, 9) - """) - - self.check_tokenize('a = (async = 1)', """\ - NAME 'a' (1, 0) (1, 1) - EQUAL '=' (1, 2) (1, 3) - LPAR '(' (1, 4) (1, 5) - ASYNC 'async' (1, 5) (1, 10) - EQUAL '=' (1, 11) (1, 12) - NUMBER '1' (1, 13) (1, 14) - RPAR ')' (1, 14) (1, 15) - """) - - self.check_tokenize('async()', """\ - ASYNC 'async' (1, 0) (1, 5) - LPAR '(' (1, 5) (1, 6) - RPAR ')' (1, 6) (1, 7) - """) - - self.check_tokenize('class async(Bar):pass', """\ - NAME 'class' (1, 0) (1, 5) - ASYNC 'async' (1, 6) (1, 11) - LPAR '(' (1, 11) (1, 12) - NAME 'Bar' (1, 12) (1, 15) - RPAR ')' (1, 15) (1, 16) - COLON ':' (1, 16) (1, 17) - NAME 'pass' (1, 17) (1, 21) - """) - - self.check_tokenize('class async:pass', """\ - NAME 'class' (1, 0) (1, 5) - ASYNC 'async' (1, 6) (1, 11) - COLON ':' (1, 11) (1, 12) - NAME 'pass' (1, 12) (1, 16) - """) - - self.check_tokenize('await = 1', """\ - AWAIT 'await' (1, 0) (1, 5) - EQUAL '=' (1, 6) (1, 7) - NUMBER '1' (1, 8) (1, 9) - """) - - self.check_tokenize('foo.async', """\ - NAME 'foo' (1, 0) (1, 3) - DOT '.' (1, 3) (1, 4) - ASYNC 'async' (1, 4) (1, 9) - """) - - self.check_tokenize('async for a in b: pass', """\ - ASYNC 'async' (1, 0) (1, 5) - NAME 'for' (1, 6) (1, 9) - NAME 'a' (1, 10) (1, 11) - NAME 'in' (1, 12) (1, 14) - NAME 'b' (1, 15) (1, 16) - COLON ':' (1, 16) (1, 17) - NAME 'pass' (1, 18) (1, 22) - """) - - self.check_tokenize('async with a as b: pass', """\ - ASYNC 'async' (1, 0) (1, 5) - NAME 'with' (1, 6) (1, 10) - NAME 'a' (1, 11) (1, 12) - NAME 'as' (1, 13) (1, 15) - NAME 'b' (1, 16) (1, 17) - COLON ':' (1, 17) (1, 18) - NAME 'pass' (1, 19) (1, 23) - """) - - self.check_tokenize('async.foo', """\ - ASYNC 'async' (1, 0) (1, 5) - DOT '.' (1, 5) (1, 6) - NAME 'foo' (1, 6) (1, 9) - """) - - self.check_tokenize('async', """\ - ASYNC 'async' (1, 0) (1, 5) - """) - - self.check_tokenize('async\n#comment\nawait', """\ - ASYNC 'async' (1, 0) (1, 5) - NEWLINE '' (1, 5) (1, 5) - AWAIT 'await' (3, 0) (3, 5) - """) - - self.check_tokenize('async\n...\nawait', """\ - ASYNC 'async' (1, 0) (1, 5) - NEWLINE '' (1, 5) (1, 5) - ELLIPSIS '...' (2, 0) (2, 3) - NEWLINE '' (2, 3) (2, 3) - AWAIT 'await' (3, 0) (3, 5) - """) - - self.check_tokenize('async\nawait', """\ - ASYNC 'async' (1, 0) (1, 5) - NEWLINE '' (1, 5) (1, 5) - AWAIT 'await' (2, 0) (2, 5) - """) - - self.check_tokenize('foo.async + 1', """\ - NAME 'foo' (1, 0) (1, 3) - DOT '.' (1, 3) (1, 4) - ASYNC 'async' (1, 4) (1, 9) - PLUS '+' (1, 10) (1, 11) - NUMBER '1' (1, 12) (1, 13) - """) - - self.check_tokenize('async def foo(): pass', """\ - ASYNC 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'foo' (1, 10) (1, 13) - LPAR '(' (1, 13) (1, 14) - RPAR ')' (1, 14) (1, 15) - COLON ':' (1, 15) (1, 16) - NAME 'pass' (1, 17) (1, 21) - """) - - self.check_tokenize('''\ -async def foo(): - def foo(await): - await = 1 - if 1: - await -async += 1 -''', """\ - ASYNC 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'foo' (1, 10) (1, 13) - LPAR '(' (1, 13) (1, 14) - RPAR ')' (1, 14) (1, 15) - COLON ':' (1, 15) (1, 16) - NEWLINE '' (1, 16) (1, 16) - INDENT '' (2, -1) (2, -1) - NAME 'def' (2, 2) (2, 5) - NAME 'foo' (2, 6) (2, 9) - LPAR '(' (2, 9) (2, 10) - AWAIT 'await' (2, 10) (2, 15) - RPAR ')' (2, 15) (2, 16) - COLON ':' (2, 16) (2, 17) - NEWLINE '' (2, 17) (2, 17) - INDENT '' (3, -1) (3, -1) - AWAIT 'await' (3, 4) (3, 9) - EQUAL '=' (3, 10) (3, 11) - NUMBER '1' (3, 12) (3, 13) - NEWLINE '' (3, 13) (3, 13) - DEDENT '' (4, -1) (4, -1) - NAME 'if' (4, 2) (4, 4) - NUMBER '1' (4, 5) (4, 6) - COLON ':' (4, 6) (4, 7) - NEWLINE '' (4, 7) (4, 7) - INDENT '' (5, -1) (5, -1) - AWAIT 'await' (5, 4) (5, 9) - NEWLINE '' (5, 9) (5, 9) - DEDENT '' (6, -1) (6, -1) - DEDENT '' (6, -1) (6, -1) - ASYNC 'async' (6, 0) (6, 5) - PLUSEQUAL '+=' (6, 6) (6, 8) - NUMBER '1' (6, 9) (6, 10) - NEWLINE '' (6, 10) (6, 10) - """) - - self.check_tokenize('async def foo():\n async for i in 1: pass', """\ - ASYNC 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'foo' (1, 10) (1, 13) - LPAR '(' (1, 13) (1, 14) - RPAR ')' (1, 14) (1, 15) - COLON ':' (1, 15) (1, 16) - NEWLINE '' (1, 16) (1, 16) - INDENT '' (2, -1) (2, -1) - ASYNC 'async' (2, 2) (2, 7) - NAME 'for' (2, 8) (2, 11) - NAME 'i' (2, 12) (2, 13) - NAME 'in' (2, 14) (2, 16) - NUMBER '1' (2, 17) (2, 18) - COLON ':' (2, 18) (2, 19) - NAME 'pass' (2, 20) (2, 24) - DEDENT '' (2, -1) (2, -1) - """) - - self.check_tokenize('async def foo(async): await', """\ - ASYNC 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'foo' (1, 10) (1, 13) - LPAR '(' (1, 13) (1, 14) - ASYNC 'async' (1, 14) (1, 19) - RPAR ')' (1, 19) (1, 20) - COLON ':' (1, 20) (1, 21) - AWAIT 'await' (1, 22) (1, 27) - """) - - self.check_tokenize('''\ -def f(): - - def baz(): pass - async def bar(): pass - - await = 2''', """\ - NAME 'def' (1, 0) (1, 3) - NAME 'f' (1, 4) (1, 5) - LPAR '(' (1, 5) (1, 6) - RPAR ')' (1, 6) (1, 7) - COLON ':' (1, 7) (1, 8) - NEWLINE '' (1, 8) (1, 8) - INDENT '' (3, -1) (3, -1) - NAME 'def' (3, 2) (3, 5) - NAME 'baz' (3, 6) (3, 9) - LPAR '(' (3, 9) (3, 10) - RPAR ')' (3, 10) (3, 11) - COLON ':' (3, 11) (3, 12) - NAME 'pass' (3, 13) (3, 17) - NEWLINE '' (3, 17) (3, 17) - ASYNC 'async' (4, 2) (4, 7) - NAME 'def' (4, 8) (4, 11) - NAME 'bar' (4, 12) (4, 15) - LPAR '(' (4, 15) (4, 16) - RPAR ')' (4, 16) (4, 17) - COLON ':' (4, 17) (4, 18) - NAME 'pass' (4, 19) (4, 23) - NEWLINE '' (4, 23) (4, 23) - AWAIT 'await' (6, 2) (6, 7) - EQUAL '=' (6, 8) (6, 9) - NUMBER '2' (6, 10) (6, 11) - DEDENT '' (6, -1) (6, -1) - """) - - self.check_tokenize('''\ -async def f(): - - def baz(): pass - async def bar(): pass - - await = 2''', """\ - ASYNC 'async' (1, 0) (1, 5) - NAME 'def' (1, 6) (1, 9) - NAME 'f' (1, 10) (1, 11) - LPAR '(' (1, 11) (1, 12) - RPAR ')' (1, 12) (1, 13) - COLON ':' (1, 13) (1, 14) - NEWLINE '' (1, 14) (1, 14) - INDENT '' (3, -1) (3, -1) - NAME 'def' (3, 2) (3, 5) - NAME 'baz' (3, 6) (3, 9) - LPAR '(' (3, 9) (3, 10) - RPAR ')' (3, 10) (3, 11) - COLON ':' (3, 11) (3, 12) - NAME 'pass' (3, 13) (3, 17) - NEWLINE '' (3, 17) (3, 17) - ASYNC 'async' (4, 2) (4, 7) - NAME 'def' (4, 8) (4, 11) - NAME 'bar' (4, 12) (4, 15) - LPAR '(' (4, 15) (4, 16) - RPAR ')' (4, 16) (4, 17) - COLON ':' (4, 17) (4, 18) - NAME 'pass' (4, 19) (4, 23) - NEWLINE '' (4, 23) (4, 23) - AWAIT 'await' (6, 2) (6, 7) - EQUAL '=' (6, 8) (6, 9) - NUMBER '2' (6, 10) (6, 11) - DEDENT '' (6, -1) (6, -1) - """) - - def test_unicode(self): - - self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\ - NAME 'Örter' (1, 0) (1, 6) - EQUAL '=' (1, 7) (1, 8) - STRING "u'places'" (1, 9) (1, 18) - NEWLINE '' (1, 18) (1, 18) - NAME 'grün' (2, 0) (2, 5) - EQUAL '=' (2, 6) (2, 7) - STRING "U'green'" (2, 8) (2, 16) - """) - - def test_invalid_syntax(self): - def get_tokens(string): - return list(_generate_tokens_from_c_tokenizer(string)) - - self.assertRaises(SyntaxError, get_tokens, "(1+2]") - self.assertRaises(SyntaxError, get_tokens, "(1+2}") - self.assertRaises(SyntaxError, get_tokens, "{1+2]") - - self.assertRaises(SyntaxError, get_tokens, "1_") - self.assertRaises(SyntaxError, get_tokens, "1.2_") - self.assertRaises(SyntaxError, get_tokens, "1e2_") - self.assertRaises(SyntaxError, get_tokens, "1e+") - - self.assertRaises(SyntaxError, get_tokens, "\xa0") - self.assertRaises(SyntaxError, get_tokens, "€") - - self.assertRaises(SyntaxError, get_tokens, "0b12") - self.assertRaises(SyntaxError, get_tokens, "0b1_2") - self.assertRaises(SyntaxError, get_tokens, "0b2") - self.assertRaises(SyntaxError, get_tokens, "0b1_") - self.assertRaises(SyntaxError, get_tokens, "0b") - self.assertRaises(SyntaxError, get_tokens, "0o18") - self.assertRaises(SyntaxError, get_tokens, "0o1_8") - self.assertRaises(SyntaxError, get_tokens, "0o8") - self.assertRaises(SyntaxError, get_tokens, "0o1_") - self.assertRaises(SyntaxError, get_tokens, "0o") - self.assertRaises(SyntaxError, get_tokens, "0x1_") - self.assertRaises(SyntaxError, get_tokens, "0x") - self.assertRaises(SyntaxError, get_tokens, "1_") - self.assertRaises(SyntaxError, get_tokens, "012") - self.assertRaises(SyntaxError, get_tokens, "1.2_") - self.assertRaises(SyntaxError, get_tokens, "1e2_") - self.assertRaises(SyntaxError, get_tokens, "1e+") - - self.assertRaises(SyntaxError, get_tokens, "'sdfsdf") - self.assertRaises(SyntaxError, get_tokens, "'''sdfsdf''") - - self.assertRaises(SyntaxError, get_tokens, "("*1000+"a"+")"*1000) - self.assertRaises(SyntaxError, get_tokens, "]") - - def test_max_indent(self): - MAXINDENT = 100 - - def generate_source(indents): - source = ''.join((' ' * x) + 'if True:\n' for x in range(indents)) - source += ' ' * indents + 'pass\n' - return source - - valid = generate_source(MAXINDENT - 1) - tokens = list(_generate_tokens_from_c_tokenizer(valid)) - self.assertEqual(tokens[-1].type, DEDENT) - compile(valid, "", "exec") - - invalid = generate_source(MAXINDENT) - tokens = list(_generate_tokens_from_c_tokenizer(invalid)) - self.assertEqual(tokens[-1].type, NEWLINE) - self.assertRaises( - IndentationError, compile, invalid, "", "exec" - ) - - def test_continuation_lines_indentation(self): - def get_tokens(string): - return [(kind, string) for (kind, string, *_) in _generate_tokens_from_c_tokenizer(string)] - - code = dedent(""" - def fib(n): - \\ - '''Print a Fibonacci series up to n.''' - \\ - a, b = 0, 1 - """) - - self.check_tokenize(code, """\ - NAME 'def' (2, 0) (2, 3) - NAME 'fib' (2, 4) (2, 7) - LPAR '(' (2, 7) (2, 8) - NAME 'n' (2, 8) (2, 9) - RPAR ')' (2, 9) (2, 10) - COLON ':' (2, 10) (2, 11) - NEWLINE '' (2, 11) (2, 11) - INDENT '' (4, -1) (4, -1) - STRING "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39) - NEWLINE '' (4, 39) (4, 39) - NAME 'a' (6, 0) (6, 1) - COMMA ',' (6, 1) (6, 2) - NAME 'b' (6, 3) (6, 4) - EQUAL '=' (6, 5) (6, 6) - NUMBER '0' (6, 7) (6, 8) - COMMA ',' (6, 8) (6, 9) - NUMBER '1' (6, 10) (6, 11) - NEWLINE '' (6, 11) (6, 11) - DEDENT '' (6, -1) (6, -1) - """) - - code_no_cont = dedent(""" - def fib(n): - '''Print a Fibonacci series up to n.''' - a, b = 0, 1 - """) - - self.assertEqual(get_tokens(code), get_tokens(code_no_cont)) - - code = dedent(""" - pass - \\ - - pass - """) - - self.check_tokenize(code, """\ - NAME 'pass' (2, 0) (2, 4) - NEWLINE '' (2, 4) (2, 4) - NAME 'pass' (5, 0) (5, 4) - NEWLINE '' (5, 4) (5, 4) - """) - - code_no_cont = dedent(""" - pass - pass - """) - - self.assertEqual(get_tokens(code), get_tokens(code_no_cont)) - - code = dedent(""" - if x: - y = 1 - \\ - \\ - \\ - \\ - foo = 1 - """) - - self.check_tokenize(code, """\ - NAME 'if' (2, 0) (2, 2) - NAME 'x' (2, 3) (2, 4) - COLON ':' (2, 4) (2, 5) - NEWLINE '' (2, 5) (2, 5) - INDENT '' (3, -1) (3, -1) - NAME 'y' (3, 4) (3, 5) - EQUAL '=' (3, 6) (3, 7) - NUMBER '1' (3, 8) (3, 9) - NEWLINE '' (3, 9) (3, 9) - NAME 'foo' (8, 4) (8, 7) - EQUAL '=' (8, 8) (8, 9) - NUMBER '1' (8, 10) (8, 11) - NEWLINE '' (8, 11) (8, 11) - DEDENT '' (8, -1) (8, -1) - """) - - code_no_cont = dedent(""" - if x: - y = 1 - foo = 1 - """) - - self.assertEqual(get_tokens(code), get_tokens(code_no_cont)) - - -class CTokenizerBufferTests(unittest.TestCase): - def test_newline_at_the_end_of_buffer(self): - # See issue 99581: Make sure that if we need to add a new line at the - # end of the buffer, we have enough space in the buffer, specially when - # the current line is as long as the buffer space available. - test_script = f"""\ - #coding: latin-1 - #{"a"*10000} - #{"a"*10002}""" - with os_helper.temp_dir() as temp_dir: - file_name = make_script(temp_dir, 'foo', test_script) - run_test_script(file_name) - - -if __name__ == "__main__": - unittest.main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a6041ef --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[tool.isort] +py_version=311 +line_length=100 +skip_glob = [ + ".git/*", + ".bazel/*", + "bazel-*", +] diff --git a/scratch/astinterp.py b/scratch/astinterp.py index 560dc52..d79b2d5 100644 --- a/scratch/astinterp.py +++ b/scratch/astinterp.py @@ -7,13 +7,12 @@ # Copyright (c) 2019 Paul Sokolovsky, published under the MIT License import ast +import builtins import logging import os import sys -import builtins from typing import Optional, Type - log = logging.getLogger(__name__) @@ -54,6 +53,7 @@ class ModuleNS(ANamespace): # parent: Optional["ModuleNS"] = None pass + class FunctionNS(ANamespace): pass @@ -181,6 +181,7 @@ class InterpModule: # namespace tree and the some sort of cursor or address into the AST under interpretation # representing where to resume. The logical equivalent of a program counter, but a tree path. + class ModuleInterpreter(StrictNodeVisitor): """An interpreter specific to a single module.""" @@ -717,7 +718,7 @@ class ModuleInterpreter(StrictNodeVisitor): ast.Div: lambda x, y: x / y, ast.FloorDiv: lambda x, y: x // y, ast.Mod: lambda x, y: x % y, - ast.Pow: lambda x, y: x ** y, + ast.Pow: lambda x, y: x**y, ast.LShift: lambda x, y: x << y, ast.RShift: lambda x, y: x >> y, ast.BitAnd: lambda x, y: x & y, diff --git a/scratch/test.py b/scratch/test.py index 567fe3d..ccfe207 100644 --- a/scratch/test.py +++ b/scratch/test.py @@ -19,7 +19,6 @@ print(a.baz) import random - for _ in range(10): print(random.randint(0, 1024)) @@ -30,5 +29,4 @@ def bar(a, b, **bs): import requests - print(len(requests.get("https://pypi.org/pypi/requests/json").text)) diff --git a/tools/build_rules/cp.bzl b/tools/build_rules/cp.bzl deleted file mode 100644 index f006f69..0000000 --- a/tools/build_rules/cp.bzl +++ /dev/null @@ -1,47 +0,0 @@ -load("@bazel_skylib//rules:copy_file.bzl", - "copy_file", -) - -def cp(name, src, **kwargs): - """A slightly more convenient cp() rule. Name and out should always be the same.""" - - rule_name = name.replace(".", "_").replace(":", "/").replace("//", "").replace("/", "_") - copy_file( - name = rule_name, - src = src, - out = name, - **kwargs - ) - return rule_name - - -def _copy_filegroup_impl(ctx): - all_outputs = [] - for t in ctx.attr.deps: - t_prefix = t.label.package - for f in t.files.to_list(): - # Strip out the source prefix... - path = f.short_path.replace(t_prefix + "/", "") - out = ctx.actions.declare_file(path) - print(ctx.attr.name, t.label, f, " => ", path) - all_outputs += [out] - ctx.actions.run_shell( - outputs=[out], - inputs=depset([f]), - arguments=[f.path, out.path], - command="cp $1 $2" - ) - - return [ - DefaultInfo( - files=depset(all_outputs), - runfiles=ctx.runfiles(files=all_outputs)) - ] - - -copy_filegroups = rule( - implementation=_copy_filegroup_impl, - attrs={ - "deps": attr.label_list(), - }, -) diff --git a/tools/build_rules/prelude_bazel b/tools/build_rules/prelude_bazel index 4947b73..d8a1b83 100644 --- a/tools/build_rules/prelude_bazel +++ b/tools/build_rules/prelude_bazel @@ -10,15 +10,6 @@ load("//tools/python:defs.bzl", "py_project", ) -load("@pypa//:requirements.bzl", +load("@pypi//:requirements.bzl", py_requirement="requirement" ) - -load("@bazel_skylib//rules:copy_file.bzl", - "copy_file", -) - -load("//tools/build_rules:cp.bzl", - "cp", - "copy_filegroups" -) diff --git a/tools/python/BUILD b/tools/python/BUILD index 181f52e..f2eb329 100644 --- a/tools/python/BUILD +++ b/tools/python/BUILD @@ -2,7 +2,7 @@ load("@rules_python//python:defs.bzl", "py_runtime_pair", ) -load("@arrdem_source_pypi//:requirements.bzl", "all_requirements") +load("@pypi//:requirements.bzl", "all_requirements") package(default_visibility = ["//visibility:public"]) @@ -13,6 +13,7 @@ exports_files([ "bzl_pytest_shim.py", "bzl_unittest_shim.py", "pythonshim", + "requirements_lock.txt", ]) py_runtime( @@ -40,8 +41,5 @@ py_pytest( srcs = [ "test_licenses.py", ], - data = [ - "requirements.txt", - ], deps = all_requirements, ) diff --git a/tools/python/bzl_pytest_shim.py b/tools/python/bzl_pytest_shim.py index 5a65b98..f331e33 100644 --- a/tools/python/bzl_pytest_shim.py +++ b/tools/python/bzl_pytest_shim.py @@ -4,7 +4,6 @@ import sys import pytest - if __name__ == "__main__": cmdline = ["--ignore=external"] + sys.argv[1:] print(cmdline, file=sys.stderr) diff --git a/tools/python/defs.bzl b/tools/python/defs.bzl index 70bcf57..b25af1d 100644 --- a/tools/python/defs.bzl +++ b/tools/python/defs.bzl @@ -1,4 +1,4 @@ -load("@arrdem_source_pypi//:requirements.bzl", +load("@pypi//:requirements.bzl", _py_requirement = "requirement" ) diff --git a/tools/python/requirements_lock.txt b/tools/python/requirements_lock.txt index a46344f..0c07594 100644 --- a/tools/python/requirements_lock.txt +++ b/tools/python/requirements_lock.txt @@ -1,4 +1,5 @@ attrs==22.2.0 +autoflake8==0.4.0 black==23.1.0 cattrs==22.2.0 click==8.1.3 @@ -6,6 +7,7 @@ coverage==7.2.1 exceptiongroup==1.1.0 hypothesis==6.68.2 iniconfig==2.0.0 +isort==5.12.0 jedi==0.18.2 mypy-extensions==1.0.0 packaging==23.0 @@ -14,6 +16,7 @@ pathspec==0.11.0 platformdirs==3.1.0 pluggy==1.0.0 pudb==2022.1.3 +pyflakes==3.0.1 Pygments==2.14.0 pytest==7.2.2 pytest-cov==4.0.0 diff --git a/tools/python/test_licenses.py b/tools/python/test_licenses.py index 32e7246..21ab266 100644 --- a/tools/python/test_licenses.py +++ b/tools/python/test_licenses.py @@ -4,12 +4,8 @@ Validate 3rdparty library licenses as approved. import re -from pkg_resources import ( - DistInfoDistribution, - working_set, -) import pytest - +from pkg_resources import DistInfoDistribution, working_set # Licenses approved as representing non-copyleft and not precluding commercial usage. # This is all easy, there's a good schema here. @@ -57,11 +53,7 @@ LICENSES_BY_LOWERNAME.update( ) # As a workaround for packages which don"t have correct meadata on PyPi, hand-verified packages -APPROVED_PACKAGES = [ - "yamllint", # WARNING: YAMLLINT IS GLP3"d. - "Flask_Log_Request_ID", # MIT, currently depended on as a git dep. - "anosql", # BSD -] +APPROVED_PACKAGES = [] def bash_license(ln):