Updated mcpyrate to 3.5.4 (last available)
This commit is contained in:
Salvador E. Tropea
parent
34bea23e06
commit
93ee8c3acb
|
|
@ -207,7 +207,7 @@ class KiConf(object):
|
|||
""" Environment vars from KiCad 6 configuration """
|
||||
with open(cfg, 'rt') as f:
|
||||
data = json.load(f)
|
||||
if "environment" in data and 'vars' in data['environment'] and (data['environment']['vars'] != None):
|
||||
if "environment" in data and 'vars' in data['environment'] and (data['environment']['vars'] is not None):
|
||||
for k, v in data['environment']['vars'].items():
|
||||
if GS.debug_level > 1:
|
||||
logger.debug('- KiCad var: {}="{}"'.format(k, v))
|
||||
|
|
|
|||
|
|
@ -5,4 +5,4 @@ from .expander import namemacro, parametricmacro # noqa: F401
|
|||
from .unparser import unparse # noqa: F401
|
||||
from .utils import gensym # noqa: F401
|
||||
|
||||
__version__ = "3.1.0"
|
||||
__version__ = "3.5.4"
|
||||
|
|
|
|||
|
|
@ -11,21 +11,17 @@ OSC = '\033]'
|
|||
BEL = '\a'
|
||||
is_a_tty = sys.stderr.isatty() and os.name == 'posix'
|
||||
|
||||
|
||||
def code_to_chars(code):
|
||||
return CSI + str(code) + 'm' if is_a_tty else ''
|
||||
|
||||
|
||||
def set_title(title):
|
||||
return OSC + '2;' + title + BEL
|
||||
|
||||
def clear_screen(mode=2):
|
||||
return CSI + str(mode) + 'J'
|
||||
|
||||
# def clear_screen(mode=2):
|
||||
# return CSI + str(mode) + 'J'
|
||||
|
||||
|
||||
# def clear_line(mode=2):
|
||||
# return CSI + str(mode) + 'K'
|
||||
def clear_line(mode=2):
|
||||
return CSI + str(mode) + 'K'
|
||||
|
||||
|
||||
class AnsiCodes(object):
|
||||
|
|
@ -39,75 +35,70 @@ class AnsiCodes(object):
|
|||
setattr(self, name, code_to_chars(value))
|
||||
|
||||
|
||||
# class AnsiCursor(object):
|
||||
# def UP(self, n=1):
|
||||
# return CSI + str(n) + 'A'
|
||||
#
|
||||
# def DOWN(self, n=1):
|
||||
# return CSI + str(n) + 'B'
|
||||
#
|
||||
# def FORWARD(self, n=1):
|
||||
# return CSI + str(n) + 'C'
|
||||
#
|
||||
# def BACK(self, n=1):
|
||||
# return CSI + str(n) + 'D'
|
||||
#
|
||||
# def POS(self, x=1, y=1):
|
||||
# return CSI + str(y) + ';' + str(x) + 'H'
|
||||
class AnsiCursor(object):
|
||||
def UP(self, n=1):
|
||||
return CSI + str(n) + 'A'
|
||||
def DOWN(self, n=1):
|
||||
return CSI + str(n) + 'B'
|
||||
def FORWARD(self, n=1):
|
||||
return CSI + str(n) + 'C'
|
||||
def BACK(self, n=1):
|
||||
return CSI + str(n) + 'D'
|
||||
def POS(self, x=1, y=1):
|
||||
return CSI + str(y) + ';' + str(x) + 'H'
|
||||
|
||||
|
||||
class AnsiFore(AnsiCodes):
|
||||
BLACK = 30
|
||||
RED = 31
|
||||
GREEN = 32
|
||||
YELLOW = 33
|
||||
BLUE = 34
|
||||
MAGENTA = 35
|
||||
CYAN = 36
|
||||
WHITE = 37
|
||||
RESET = 39
|
||||
BLACK = 30
|
||||
RED = 31
|
||||
GREEN = 32
|
||||
YELLOW = 33
|
||||
BLUE = 34
|
||||
MAGENTA = 35
|
||||
CYAN = 36
|
||||
WHITE = 37
|
||||
RESET = 39
|
||||
|
||||
# These are fairly well supported, but not part of the standard.
|
||||
LIGHTBLACK_EX = 90
|
||||
LIGHTRED_EX = 91
|
||||
LIGHTGREEN_EX = 92
|
||||
LIGHTYELLOW_EX = 93
|
||||
LIGHTBLUE_EX = 94
|
||||
LIGHTBLACK_EX = 90
|
||||
LIGHTRED_EX = 91
|
||||
LIGHTGREEN_EX = 92
|
||||
LIGHTYELLOW_EX = 93
|
||||
LIGHTBLUE_EX = 94
|
||||
LIGHTMAGENTA_EX = 95
|
||||
LIGHTCYAN_EX = 96
|
||||
LIGHTWHITE_EX = 97
|
||||
LIGHTCYAN_EX = 96
|
||||
LIGHTWHITE_EX = 97
|
||||
|
||||
|
||||
class AnsiBack(AnsiCodes):
|
||||
BLACK = 40
|
||||
RED = 41
|
||||
GREEN = 42
|
||||
YELLOW = 43
|
||||
BLUE = 44
|
||||
MAGENTA = 45
|
||||
CYAN = 46
|
||||
WHITE = 47
|
||||
RESET = 49
|
||||
BLACK = 40
|
||||
RED = 41
|
||||
GREEN = 42
|
||||
YELLOW = 43
|
||||
BLUE = 44
|
||||
MAGENTA = 45
|
||||
CYAN = 46
|
||||
WHITE = 47
|
||||
RESET = 49
|
||||
|
||||
# These are fairly well supported, but not part of the standard.
|
||||
LIGHTBLACK_EX = 100
|
||||
LIGHTRED_EX = 101
|
||||
LIGHTGREEN_EX = 102
|
||||
LIGHTYELLOW_EX = 103
|
||||
LIGHTBLUE_EX = 104
|
||||
LIGHTBLACK_EX = 100
|
||||
LIGHTRED_EX = 101
|
||||
LIGHTGREEN_EX = 102
|
||||
LIGHTYELLOW_EX = 103
|
||||
LIGHTBLUE_EX = 104
|
||||
LIGHTMAGENTA_EX = 105
|
||||
LIGHTCYAN_EX = 106
|
||||
LIGHTWHITE_EX = 107
|
||||
LIGHTCYAN_EX = 106
|
||||
LIGHTWHITE_EX = 107
|
||||
|
||||
|
||||
class AnsiStyle(AnsiCodes):
|
||||
BRIGHT = 1
|
||||
DIM = 2
|
||||
NORMAL = 22
|
||||
BRIGHT = 1
|
||||
DIM = 2
|
||||
NORMAL = 22
|
||||
RESET_ALL = 0
|
||||
|
||||
|
||||
Fore = AnsiFore()
|
||||
Back = AnsiBack()
|
||||
Style = AnsiStyle()
|
||||
# Cursor = AnsiCursor()
|
||||
Fore = AnsiFore()
|
||||
Back = AnsiBack()
|
||||
Style = AnsiStyle()
|
||||
Cursor = AnsiCursor()
|
||||
|
|
|
|||
|
|
@ -3,15 +3,11 @@
|
|||
|
||||
__all__ = ["fix_ctx", "fix_locations"]
|
||||
|
||||
from ast import (Load, Store, Del,
|
||||
Assign, AnnAssign, AugAssign,
|
||||
Attribute, Subscript,
|
||||
comprehension,
|
||||
For, AsyncFor,
|
||||
withitem,
|
||||
Delete,
|
||||
iter_child_nodes)
|
||||
from ast import (AnnAssign, Assign, AsyncFor, Attribute, AugAssign, Del,
|
||||
Delete, For, Load, Store, Subscript, comprehension,
|
||||
iter_child_nodes, withitem)
|
||||
from copy import copy
|
||||
from typing import Type
|
||||
|
||||
from . import walkers
|
||||
|
||||
|
|
@ -20,7 +16,7 @@ try: # Python 3.8+
|
|||
except ImportError:
|
||||
class _NoSuchNodeType:
|
||||
pass
|
||||
NamedExpr = _NoSuchNodeType
|
||||
NamedExpr: Type = _NoSuchNodeType # type: ignore[no-redef]
|
||||
|
||||
|
||||
class _CtxFixer(walkers.ASTTransformer):
|
||||
|
|
@ -71,6 +67,7 @@ class _CtxFixer(walkers.ASTTransformer):
|
|||
# `AugStore` and `AugLoad` are for internal use only, not even
|
||||
# meant to be exposed to the user; the compiler expects `Store`
|
||||
# and `Load` here. https://bugs.python.org/issue39988
|
||||
# Those internal classes are indeed deprecated in Python 3.9.
|
||||
self.withstate(tree.target, ctxclass=Store)
|
||||
self.withstate(tree.value, ctxclass=Load)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ __all__ = ["setcolor", "colorize", "ColorScheme",
|
|||
"Fore", "Back", "Style"]
|
||||
|
||||
try:
|
||||
from colorama import init as colorama_init, Fore, Back, Style
|
||||
from colorama import Back, Fore, Style # type: ignore[import]
|
||||
from colorama import init as colorama_init # type: ignore[import]
|
||||
colorama_init()
|
||||
except ImportError: # pragma: no cover
|
||||
# The `ansi` module is a slightly modified, POSIX-only,
|
||||
|
|
@ -18,6 +19,17 @@ except ImportError: # pragma: no cover
|
|||
# images that don't have the library available.
|
||||
from .ansi import Fore, Back, Style # noqa: F811
|
||||
|
||||
# TODO: Get rid of this hack if Colorama adds these styles later.
|
||||
# Inject some styles missing from Colorama 0.4.4
|
||||
_additional_styles = (("ITALIC", "\33[3m"),
|
||||
("URL", "\33[4m"), # underline plus possibly a special color (depends on terminal app)
|
||||
("BLINK", "\33[5m"),
|
||||
("BLINK2", "\33[6m")) # same effect as BLINK?
|
||||
for _name, _value in _additional_styles:
|
||||
if not hasattr(Style, _name):
|
||||
setattr(Style, _name, _value)
|
||||
del _name, _value
|
||||
|
||||
from .bunch import Bunch
|
||||
|
||||
|
||||
|
|
@ -140,17 +152,15 @@ class ColorScheme(Bunch):
|
|||
# ------------------------------------------------------------
|
||||
# format_bindings, step_expansion, StepExpansion
|
||||
|
||||
self.HEADING = (Style.BRIGHT, Fore.LIGHTBLUE_EX)
|
||||
self.HEADING1 = (Style.BRIGHT, Fore.LIGHTBLUE_EX) # main heading
|
||||
self.HEADING2 = Fore.LIGHTBLUE_EX # subheading (filenames, tree ids, ...)
|
||||
self.SOURCEFILENAME = Style.BRIGHT
|
||||
|
||||
# format_bindings
|
||||
self.GREYEDOUT = Style.DIM # if no bindings
|
||||
|
||||
# step_expansion
|
||||
self.TREEID = Fore.LIGHTBLUE_EX
|
||||
|
||||
# StepExpansion
|
||||
self.ATTENTION = (Style.BRIGHT, Fore.GREEN) # "DialectExpander debug mode"
|
||||
self.ATTENTION = (Style.BRIGHT, Fore.GREEN) # "DialectExpander debug mode", "PHASE 0"
|
||||
self.TRANSFORMERKIND = (Style.BRIGHT, Fore.GREEN) # "source", "AST"
|
||||
self.DIALECTTRANSFORMERNAME = (Style.BRIGHT, Fore.YELLOW)
|
||||
|
||||
|
|
@ -160,4 +170,11 @@ class ColorScheme(Bunch):
|
|||
self.NODETYPE = (Style.BRIGHT, Fore.LIGHTBLUE_EX)
|
||||
self.FIELDNAME = Fore.YELLOW
|
||||
self.BAREVALUE = Fore.GREEN
|
||||
ColorScheme = ColorScheme()
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# runtests
|
||||
self.TESTHEADING = self.HEADING1
|
||||
self.TESTPASS = (Style.BRIGHT, Fore.GREEN)
|
||||
self.TESTFAIL = (Style.BRIGHT, Fore.RED)
|
||||
self.TESTERROR = (Style.BRIGHT, Fore.YELLOW)
|
||||
ColorScheme = ColorScheme() # type: ignore[assignment, misc]
|
||||
|
|
|
|||
|
|
@ -12,18 +12,20 @@ We also provide a public API to compile and run macro-enabled code at run time.
|
|||
|
||||
__all__ = ["expand", "compile",
|
||||
"singlephase_expand",
|
||||
"run", "create_module"]
|
||||
"run", "create_module",
|
||||
"temporary_module"]
|
||||
|
||||
import ast
|
||||
import builtins
|
||||
from contextlib import contextmanager
|
||||
import importlib.util
|
||||
import sys
|
||||
from types import ModuleType, CodeType
|
||||
from types import CodeType, ModuleType
|
||||
|
||||
from .dialects import DialectExpander
|
||||
from .expander import find_macros, expand_macros
|
||||
from .markers import check_no_markers_remaining
|
||||
from . import multiphase
|
||||
from .dialects import DialectExpander
|
||||
from .expander import expand_macros, find_macros
|
||||
from .markers import check_no_markers_remaining
|
||||
from .unparser import unparse
|
||||
from .utils import gensym, getdocstring
|
||||
|
||||
|
|
@ -48,9 +50,8 @@ def expand(source, filename, optimize=-1, self_module=None):
|
|||
|
||||
`filename`: Full path to the `.py` file being compiled.
|
||||
|
||||
`optimize`: Passed to Python's built-in `compile` function, as well as to
|
||||
the multi-phase compiler. The multi-phase compiler uses the
|
||||
`optimize` setting for the temporary higher-phase modules.
|
||||
`optimize`: Passed to the multi-phase compiler. It uses the `optimize`
|
||||
setting for the temporary higher-phase modules.
|
||||
|
||||
`self_module`: Absolute dotted module name of the module being compiled.
|
||||
Needed for modules that request multi-phase compilation.
|
||||
|
|
@ -193,6 +194,8 @@ def compile(source, filename, optimize=-1, self_module=None):
|
|||
The main reason for its existence is to provide a near drop-in replacement for
|
||||
the built-in `compile` for macro-enabled input.
|
||||
|
||||
The `optimize` parameter is passed to Python's built-in `compile` function.
|
||||
|
||||
Currently the API differs from the built-in `compile` in that:
|
||||
|
||||
- `mode` is always `"exec"`,
|
||||
|
|
@ -542,3 +545,42 @@ def create_module(dotted_name=None, filename=None, *, update_parent=True):
|
|||
|
||||
sys.modules[dotted_name] = module
|
||||
return module
|
||||
|
||||
|
||||
@contextmanager
|
||||
def temporary_module(dotted_name=None, filename=None, *, update_parent=True):
|
||||
"""Context manager. Create and destroy a temporary module.
|
||||
|
||||
Usage::
|
||||
|
||||
with temporary_module(name, filename) as module:
|
||||
...
|
||||
|
||||
Arguments are passed to `create_module`. The created module is
|
||||
automatically inserted to `sys.modules`, and then assigned to
|
||||
the as-part.
|
||||
|
||||
When the context exits, the temporary module is removed from
|
||||
`sys.modules`. If the `update_parent` optional kwarg is `True`,
|
||||
it is also removed from the parent package's namespace, for
|
||||
symmetry.
|
||||
|
||||
This is useful for sandboxed testing of macros, because the
|
||||
temporary modules will not pollute `sys.modules` beyond their
|
||||
useful lifetime.
|
||||
"""
|
||||
try:
|
||||
module = create_module(dotted_name, filename, update_parent=update_parent)
|
||||
yield module
|
||||
finally:
|
||||
try:
|
||||
del sys.modules[dotted_name]
|
||||
|
||||
# The standard importer doesn't do this, but it seems nice for symmetry,
|
||||
# considering the use case.
|
||||
if update_parent and dotted_name.find(".") != -1:
|
||||
packagename, finalcomponent = dotted_name.rsplit(".", maxsplit=1)
|
||||
package = sys.modules.get(packagename, None)
|
||||
delattr(package, finalcomponent)
|
||||
except BaseException:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@
|
|||
"""Expander core; essentially, how to apply a macro invocation."""
|
||||
|
||||
__all__ = ["MacroExpansionError", "MacroExpanderMarker", "Done",
|
||||
"BaseMacroExpander", "global_postprocess"]
|
||||
"BaseMacroExpander", "global_postprocess",
|
||||
"add_postprocessor", "remove_postprocessor"]
|
||||
|
||||
from ast import NodeTransformer, AST
|
||||
from contextlib import contextmanager
|
||||
from ast import AST, NodeTransformer
|
||||
from collections import ChainMap
|
||||
from contextlib import contextmanager
|
||||
from copy import deepcopy
|
||||
from typing import Any, Callable, Dict, List
|
||||
|
||||
from .astfixers import fix_ctx, fix_locations
|
||||
from .markers import ASTMarker, delete_markers
|
||||
|
|
@ -14,7 +17,14 @@ from .utils import flatten, format_location
|
|||
|
||||
# Global macro bindings shared across all expanders in the current process.
|
||||
# This is used by `mcpyrate.quotes` for hygienically captured macro functions.
|
||||
global_bindings = {}
|
||||
#
|
||||
# TODO: In reality, a macro function is not just any callable, but it takes certain
|
||||
# (especially named) arguments, and its return type is `Union[AST, List[AST], None]`,
|
||||
# but as of 3.1.0, we skim over this detail.
|
||||
global_bindings: Dict[str, Callable[..., Any]] = {}
|
||||
|
||||
# User function hook for `global_postprocess`.
|
||||
global_postprocessors: List[Callable[..., Any]] = []
|
||||
|
||||
class MacroExpansionError(Exception):
|
||||
"""Base class for errors specific to macro expansion.
|
||||
|
|
@ -83,11 +93,12 @@ class BaseMacroExpander(NodeTransformer):
|
|||
filename: full path to `.py` file being expanded, for error reporting
|
||||
"""
|
||||
|
||||
def __init__(self, bindings, filename):
|
||||
def __init__(self, bindings: Dict[str, Callable[..., Any]], filename: str):
|
||||
self.local_bindings = bindings
|
||||
self.bindings = ChainMap(self.local_bindings, global_bindings)
|
||||
self.filename = filename
|
||||
self.recursive = True
|
||||
self._debughook = None # see `mcpyrate.debug.step_expansion`
|
||||
|
||||
def visit(self, tree):
|
||||
"""Expand macros in `tree`, using current setting for recursive mode.
|
||||
|
|
@ -137,7 +148,7 @@ class BaseMacroExpander(NodeTransformer):
|
|||
with self._recursive_mode(False):
|
||||
return Done(self.visit(tree))
|
||||
|
||||
def _recursive_mode(self, isrecursive):
|
||||
def _recursive_mode(self, isrecursive: bool):
|
||||
"""Context manager. Change recursive mode, restoring the old mode when the context exits."""
|
||||
@contextmanager
|
||||
def recursive_mode():
|
||||
|
|
@ -149,6 +160,30 @@ class BaseMacroExpander(NodeTransformer):
|
|||
self.recursive = wasrecursive
|
||||
return recursive_mode()
|
||||
|
||||
def debughook(self, hook: Callable[..., None]):
|
||||
"""Context manager. Temporarily set a debug hook, restoring the old one when the context exits.
|
||||
|
||||
The debug hook, if one is installed, is called whenever a macro expands.
|
||||
|
||||
The hook receives the following arguments, passed positionally in this order:
|
||||
invocationsubtreeid: int, the `id()` of the *original* macro invocation subtree
|
||||
before anything was done to it.
|
||||
invocationtree: AST, (a deepcopy of) the macro invocation subtree before expansion.
|
||||
expandedtree: AST, the replacement subtree after expanding once. It's the actual live copy.
|
||||
macroname: str, name of the macro that was applied, as it appears in this expander's
|
||||
bindings.
|
||||
macrofunction: callable, the macro function that was applied.
|
||||
"""
|
||||
@contextmanager
|
||||
def debughook_context():
|
||||
oldhook = self._debughook
|
||||
try:
|
||||
self._debughook = hook
|
||||
yield
|
||||
finally:
|
||||
self._debughook = oldhook
|
||||
return debughook_context()
|
||||
|
||||
def expand(self, syntax, target, macroname, tree, sourcecode, kw=None):
|
||||
"""Expand a macro invocation.
|
||||
|
||||
|
|
@ -180,7 +215,21 @@ class BaseMacroExpander(NodeTransformer):
|
|||
|
||||
Very rarely needed; if you need it, you'll know.
|
||||
|
||||
**CAUTION**: not a copy, or at most a shallow copy.
|
||||
**CAUTION**: Not a copy, or at most a shallow copy.
|
||||
|
||||
**CAUTION**: The caller, i.e. the concrete expander's
|
||||
visit method that corresponds to the macro
|
||||
invocation `syntax` being processed,
|
||||
is responsible for providing a complete,
|
||||
unmodified `invocation`.
|
||||
|
||||
So for example, if your concrete expander
|
||||
needs to pop a block macro, be sure to stash
|
||||
an appropriately prepared copy of the invocation
|
||||
(so that it looks like the original one)
|
||||
before you edit the original. This is used for
|
||||
debug purposes, so preserving the appearance
|
||||
of the original invocation is important.
|
||||
|
||||
To send additional named arguments from the actual expander to the
|
||||
macro function, place them in a dictionary and pass that dictionary
|
||||
|
|
@ -200,7 +249,7 @@ class BaseMacroExpander(NodeTransformer):
|
|||
raise MacroApplicationError(f"{loc}\nin {syntax} macro invocation for '{macroname}': the name '{macroname}' is not bound to a macro.")
|
||||
|
||||
# Expand the macro.
|
||||
expansion = _apply_macro(macro, tree, kw)
|
||||
expansion = self._apply_macro(macro, tree, kw, macroname, target)
|
||||
|
||||
# Convert possible iterable result to `list`, then typecheck macro output.
|
||||
try:
|
||||
|
|
@ -254,6 +303,32 @@ class BaseMacroExpander(NodeTransformer):
|
|||
|
||||
return self._visit_expansion(expansion, target)
|
||||
|
||||
def _apply_macro(self, macro, tree, kw, macroname, target):
|
||||
"""Execute `macro` on `tree`, with the dictionary `kw` unpacked into macro's named arguments.
|
||||
|
||||
`macro` is the macro function.
|
||||
|
||||
`tree` is the AST to apply the macro to. This is what the macro gets as its `tree` argument.
|
||||
|
||||
`kw` is a dictionary-like, unpacked into the macro's named arguments.
|
||||
|
||||
`macroname` is the name (str) the macro function `macro` is bound as in this expander.
|
||||
|
||||
`target` is the whole macro invocation AST. A deepcopy is passed to the debug hook (if installed).
|
||||
"""
|
||||
# There's no guarantee whether `macro` edits the AST in-place or produces a new tree,
|
||||
# so to produce reliable debug information, we should take the old `id` and a deep copy
|
||||
# of the macro invocation.
|
||||
if self._debughook:
|
||||
oldid = id(target)
|
||||
oldtree = deepcopy(target) # including the macro invocation itself
|
||||
|
||||
newtree = macro(tree, **kw)
|
||||
|
||||
if self._debughook:
|
||||
self._debughook(oldid, oldtree, newtree, macroname, macro)
|
||||
return newtree
|
||||
|
||||
def _visit_expansion(self, expansion, target):
|
||||
"""Perform local postprocessing.
|
||||
|
||||
|
|
@ -278,10 +353,6 @@ class BaseMacroExpander(NodeTransformer):
|
|||
return bindings[name]
|
||||
return False
|
||||
|
||||
def _apply_macro(macro, tree, kw):
|
||||
"""Execute `macro` on `tree`, with the dictionary `kw` unpacked into macro's named arguments."""
|
||||
return macro(tree, **kw)
|
||||
|
||||
|
||||
# Final postprocessing for the top-level walk can't be done at the end of the
|
||||
# entry points `visit_once` and `visit_recursively`, because it is valid for a
|
||||
|
|
@ -296,8 +367,53 @@ def global_postprocess(tree):
|
|||
to Python's `compile`.
|
||||
"""
|
||||
tree = delete_markers(tree, cls=MacroExpanderMarker)
|
||||
|
||||
# A name macro, appearing as an assignment target, gets the wrong ctx,
|
||||
# because when expanding the name macro, the expander sees only the Name
|
||||
# node, and thus puts an `ast.Load` there as the ctx.
|
||||
tree = fix_ctx(tree, copy_seen_nodes=True)
|
||||
|
||||
# Run the user hooks, if any.
|
||||
for custom_postprocessor in global_postprocessors:
|
||||
tree = custom_postprocessor(tree)
|
||||
|
||||
return tree
|
||||
|
||||
def add_postprocessor(function):
|
||||
"""Add a custom postprocessor for the top-level expansion.
|
||||
|
||||
`function` must accept `AST` and `list` of `AST`, and return
|
||||
the same type.
|
||||
|
||||
Custom postprocessors are automatically called by `global_postprocess`
|
||||
(which is called just after macro expansion for a module ends), in the
|
||||
order they were registered by calling `add_postprocessor`.
|
||||
|
||||
This e.g. allows a macro library to use its own `ASTMarker` subclasses
|
||||
for internal communication between macros, and delete (only) its own
|
||||
markers when done.
|
||||
|
||||
Put something like this somewhere in your library initialization code
|
||||
(preferably somewhere it's easy to discover for someone reading your
|
||||
code, such as in a package `__init__.py`):
|
||||
|
||||
from functools import partial
|
||||
|
||||
import mcpyrate.core
|
||||
from mcpyrate.markers import delete_markers
|
||||
|
||||
_delete_my_markers = partial(delete_markers,
|
||||
cls=MyCustomMarkerBase)
|
||||
mcpyrate.core.add_postprocessor(_delete_my_markers)
|
||||
"""
|
||||
if function not in global_postprocessors:
|
||||
global_postprocessors.append(function)
|
||||
|
||||
def remove_postprocessor(function):
|
||||
"""Remove a previously added custom postprocessor.
|
||||
|
||||
The `function` is the callable object that was registered as a
|
||||
postprocessor using `add_postprocessor`.
|
||||
"""
|
||||
if function in global_postprocessors:
|
||||
global_postprocessors.remove(function)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
__all__ = ["resolve_package", "relativize", "match_syspath",
|
||||
"ismacroimport", "get_macros"]
|
||||
|
||||
from ast import ImportFrom
|
||||
import ast
|
||||
import importlib
|
||||
import importlib.util
|
||||
import os
|
||||
|
|
@ -14,6 +14,7 @@ import sys
|
|||
from .unparser import unparse_with_fallbacks
|
||||
from .utils import format_location
|
||||
|
||||
|
||||
def resolve_package(filename): # TODO: for now, `guess_package`, really. Check the docs again.
|
||||
"""Resolve absolute Python package name for .py source file `filename`.
|
||||
|
||||
|
|
@ -79,7 +80,7 @@ def ismacroimport(statement, magicname='macros'):
|
|||
|
||||
where "macros" is the literal string given as `magicname`.
|
||||
"""
|
||||
if isinstance(statement, ImportFrom):
|
||||
if isinstance(statement, ast.ImportFrom):
|
||||
firstimport = statement.names[0]
|
||||
if firstimport.name == magicname and firstimport.asname is None:
|
||||
return True
|
||||
|
|
@ -146,6 +147,8 @@ def get_macros(macroimport, *, filename, reload=False, allow_asname=True, self_m
|
|||
try:
|
||||
module = sys.modules[module_absname]
|
||||
except KeyError:
|
||||
approx_sourcecode = unparse_with_fallbacks(macroimport, debug=True, color=True)
|
||||
loc = format_location(filename, macroimport, approx_sourcecode)
|
||||
raise ModuleNotFoundError(f"{loc}\nModule {module_absname} not found in `sys.modules`")
|
||||
|
||||
# regular macro-import
|
||||
|
|
@ -165,13 +168,76 @@ def get_macros(macroimport, *, filename, reload=False, allow_asname=True, self_m
|
|||
bindings = {}
|
||||
for name in macroimport.names[1:]:
|
||||
if not allow_asname and name.asname is not None:
|
||||
approx_sourcecode = unparse_with_fallbacks(macroimport, debug=True, color=True)
|
||||
loc = format_location(filename, macroimport, approx_sourcecode)
|
||||
raise ImportError(f"{loc}\nThis expander (see traceback) does not support as-naming macro-imports.")
|
||||
|
||||
try:
|
||||
bindings[name.asname or name.name] = getattr(module, name.name)
|
||||
macro = getattr(module, name.name)
|
||||
except AttributeError as err:
|
||||
approx_sourcecode = unparse_with_fallbacks(macroimport, debug=True, color=True)
|
||||
loc = format_location(filename, macroimport, approx_sourcecode)
|
||||
raise ImportError(f"{loc}\ncannot import name '{name.name}' from module {module_absname}") from err
|
||||
|
||||
if not callable(macro):
|
||||
approx_sourcecode = unparse_with_fallbacks(macroimport, debug=True, color=True)
|
||||
loc = format_location(filename, macroimport, approx_sourcecode)
|
||||
raise ImportError(f"{loc}\nname '{name.name}' in module {module_absname} is not a callable object (got {type(macro)} with value {repr(macro)}), so it cannot be imported as a macro.")
|
||||
|
||||
bindings[name.asname or name.name] = macro
|
||||
|
||||
return module_absname, bindings
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
|
||||
def _mcpyrate_attr(dotted_name, *, force_import=False):
|
||||
"""Create an AST that, when compiled and run, looks up an attribute of `mcpyrate`.
|
||||
|
||||
`dotted_name` is an `str`. Examples::
|
||||
|
||||
_mcpyrate_attr("dump") # -> mcpyrate.dump
|
||||
_mcpyrate_attr("quotes.lookup_value") # -> mcpyrate.quotes.lookup_value
|
||||
|
||||
If `force_import` is `True`, use the builtin `__import__` function to
|
||||
first import the `mcpyrate` module whose attribute will be accessed.
|
||||
This is useful when the eventual use site might not import any `mcpyrate`
|
||||
modules.
|
||||
"""
|
||||
if not isinstance(dotted_name, str):
|
||||
raise TypeError(f"dotted_name name must be str; got {type(dotted_name)} with value {repr(dotted_name)}")
|
||||
|
||||
if dotted_name.find(".") != -1:
|
||||
submodule_dotted_name, _ = dotted_name.rsplit(".", maxsplit=1)
|
||||
else:
|
||||
submodule_dotted_name = None
|
||||
|
||||
# Issue #21: `mcpyrate` might not be in scope at the use site. Fortunately,
|
||||
# `__import__` is a builtin, so we are guaranteed to always have that available.
|
||||
if not force_import:
|
||||
mcpyrate_module = ast.Name(id="mcpyrate")
|
||||
else:
|
||||
globals_call = ast.Call(ast.Name(id="globals"),
|
||||
[],
|
||||
[])
|
||||
|
||||
if submodule_dotted_name:
|
||||
modulename_to_import = f"mcpyrate.{submodule_dotted_name}"
|
||||
else:
|
||||
modulename_to_import = "mcpyrate"
|
||||
|
||||
import_call = ast.Call(ast.Name(id="__import__"),
|
||||
[ast.Constant(value=modulename_to_import),
|
||||
globals_call, # globals (used for determining context)
|
||||
ast.Constant(value=None), # locals (unused)
|
||||
ast.Tuple(elts=[]), # fromlist
|
||||
ast.Constant(value=0)], # level
|
||||
[])
|
||||
# When compiled and run, the import call will evaluate to a reference
|
||||
# to the top-level `mcpyrate` module.
|
||||
mcpyrate_module = import_call
|
||||
|
||||
value = mcpyrate_module
|
||||
for name in dotted_name.split("."):
|
||||
value = ast.Attribute(value=value, attr=name)
|
||||
|
||||
return value
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ __all__ = ["step_expansion", "StepExpansion", "step_phases",
|
|||
import ast
|
||||
import functools
|
||||
import io
|
||||
from sys import stderr
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
from .astdumper import dump
|
||||
|
|
@ -30,10 +30,10 @@ def step_expansion(tree, *, args, syntax, expander, **kw):
|
|||
Syntax::
|
||||
|
||||
step_expansion[...]
|
||||
step_expansion[mode][...]
|
||||
step_expansion[arg, ...][...]
|
||||
with step_expansion:
|
||||
...
|
||||
with step_expansion[mode]:
|
||||
with step_expansion[arg, ...]:
|
||||
...
|
||||
|
||||
This calls `expander.visit_once` in a loop, discarding the `Done` markers,
|
||||
|
|
@ -45,6 +45,9 @@ def step_expansion(tree, *, args, syntax, expander, **kw):
|
|||
before returning control; if it does `expander.visit_recursively(subtree)`,
|
||||
then that subtree will be expanded, in a single step, until no macros remain.
|
||||
|
||||
However, the argument `"detailed"` can be used to report all macro expansions
|
||||
within each step, including those done by `expander.visit(subtree)`.
|
||||
|
||||
Since this is a debugging utility, the source code is rendered in the debug
|
||||
mode of `unparse`, which prints also invisible nodes such as `Module` and
|
||||
`Expr` (as well as any AST markers) in a Python-like pseudocode notation.
|
||||
|
|
@ -52,60 +55,93 @@ def step_expansion(tree, *, args, syntax, expander, **kw):
|
|||
The source code is rendered with syntax highlighting suitable for terminal
|
||||
output.
|
||||
|
||||
The optional macro argument `mode`, if present, sets the renderer mode.
|
||||
It must be one of the strings `"unparse"` (default) or `"dump"`.
|
||||
If `"unparse"`, then at each step, the AST will be shown as source code.
|
||||
If `"dump"`, then at each step, the AST will be shown as a raw AST dump.
|
||||
Supported macro arguments (can be given in any order):
|
||||
|
||||
- One of the strings `"unparse"` (default) or `"dump"` sets the renderer mode.
|
||||
|
||||
If `"unparse"`, then at each step, the AST will be shown as source code.
|
||||
If `"dump"`, then at each step, the AST will be shown as a raw AST dump.
|
||||
|
||||
- One of the strings `"summary"` (default) or `"detailed"` sets the report
|
||||
detail level.
|
||||
|
||||
If `"summary"`, then the whole tree is printed once per step.
|
||||
If `"detailed"`, also each macro expansion in each step is reported,
|
||||
by printing the relevant subtree before and after expansion.
|
||||
|
||||
This macro steps the expansion at macro expansion time. If you have a
|
||||
run-time AST value (such as a quasiquoted tree), and want to step the
|
||||
expansion of that at run time, see the `stepr` macro in `mcpyrate.metatools`.
|
||||
It supports the same arguments as `step_expansion`.
|
||||
"""
|
||||
if syntax not in ("expr", "block"):
|
||||
raise SyntaxError("`step_expansion` is an expr and block macro only")
|
||||
|
||||
supported_args = ("unparse", "dump", "detailed", "summary")
|
||||
formatter = functools.partial(unparse_with_fallbacks, debug=True, color=True,
|
||||
expander=expander)
|
||||
if args:
|
||||
if len(args) != 1:
|
||||
raise SyntaxError("expected `step_expansion` or `step_expansion['mode_str']`")
|
||||
arg = args[0]
|
||||
print_details = False
|
||||
for arg in args:
|
||||
if type(arg) is ast.Constant:
|
||||
mode = arg.value
|
||||
v = arg.value
|
||||
elif type(arg) is ast.Str: # up to Python 3.7
|
||||
mode = arg.s
|
||||
v = arg.s
|
||||
else:
|
||||
raise TypeError(f"expected mode_str, got {repr(arg)} {unparse_with_fallbacks(arg)}")
|
||||
if mode not in ("unparse", "dump"):
|
||||
raise ValueError(f"expected mode_str either 'unparse' or 'dump', got {repr(mode)}")
|
||||
if mode == "dump":
|
||||
raise TypeError(f"expected str argument, got {repr(arg)} {unparse_with_fallbacks(arg)}")
|
||||
|
||||
if v not in supported_args:
|
||||
raise ValueError(f"unknown argument {repr(v)}")
|
||||
|
||||
if v == "dump":
|
||||
formatter = functools.partial(dump, include_attributes=True, color=True)
|
||||
elif v == "detailed":
|
||||
print_details = True
|
||||
|
||||
c, CS = setcolor, ColorScheme
|
||||
|
||||
with _step_expansion_level.changed_by(+1):
|
||||
with _step_expansion_level.changed_by(+1): # The level is used for nested invocations of `step_expansion`.
|
||||
indent = 2 * _step_expansion_level.value
|
||||
stars = indent * '*'
|
||||
codeindent = indent
|
||||
tag = id(tree)
|
||||
print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} ({expander.filename}) {c(CS.HEADING)}before macro expansion:{c()}",
|
||||
file=stderr)
|
||||
print(textwrap.indent(formatter(tree), codeindent * ' '), file=stderr)
|
||||
mc = MacroCollector(expander)
|
||||
mc.visit(tree)
|
||||
|
||||
print(f"{c(CS.HEADING1)}{stars}Tree {c(CS.HEADING2)}0x{tag:x} ({expander.filename}) {c(CS.HEADING1)}before macro expansion:{c()}",
|
||||
file=sys.stderr)
|
||||
print(textwrap.indent(f"{formatter(tree)}", codeindent * ' '), file=sys.stderr)
|
||||
|
||||
step = 0
|
||||
while mc.collected:
|
||||
step += 1
|
||||
tree = expander.visit_once(tree) # -> Done(body=...)
|
||||
tree = tree.body
|
||||
print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} ({expander.filename}) {c(CS.HEADING)}after step {step}:{c()}",
|
||||
file=stderr)
|
||||
print(textwrap.indent(formatter(tree), codeindent * ' '), file=stderr)
|
||||
mc.clear()
|
||||
def doit():
|
||||
nonlocal step
|
||||
nonlocal tree
|
||||
mc = MacroCollector(expander)
|
||||
mc.visit(tree)
|
||||
while mc.collected:
|
||||
step += 1
|
||||
tree = expander.visit_once(tree) # -> Done(body=...)
|
||||
tree = tree.body
|
||||
print(f"{c(CS.HEADING1)}{stars}Tree {c(CS.HEADING2)}0x{tag:x} ({expander.filename}) {c(CS.HEADING1)}after step {step}:{c()}",
|
||||
file=sys.stderr)
|
||||
print(textwrap.indent(formatter(tree), codeindent * ' '), file=sys.stderr)
|
||||
mc.clear()
|
||||
mc.visit(tree)
|
||||
|
||||
if print_details:
|
||||
def print_step(invocationsubtreeid, invocationtree, expandedtree, macroname, macrofunction):
|
||||
print(f"{c(CS.HEADING1)}{stars}Tree {c(CS.HEADING2)}0x{tag:x} ({expander.filename}) {c(CS.HEADING1)}processing step {step}:",
|
||||
file=sys.stderr)
|
||||
print(textwrap.indent(f"{c(CS.HEADING2)}Applying {c(CS.MACRONAME)}{macroname}{c(CS.HEADING2)} at subtree 0x{invocationsubtreeid:x}:{c()}", codeindent * ' '), file=sys.stderr)
|
||||
print(textwrap.indent(f"{formatter(invocationtree)}", (codeindent + 2) * ' '), file=sys.stderr)
|
||||
print(textwrap.indent(f"{c(CS.HEADING2)}Result:{c()}", codeindent * ' '), file=sys.stderr)
|
||||
print(textwrap.indent(f"{formatter(expandedtree)}", (codeindent + 2) * ' '), file=sys.stderr)
|
||||
|
||||
with expander.debughook(print_step):
|
||||
doit()
|
||||
else:
|
||||
doit()
|
||||
|
||||
plural = "s" if step != 1 else ""
|
||||
print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} ({expander.filename}) {c(CS.HEADING)}macro expansion complete after {step} step{plural}.{c()}",
|
||||
file=stderr)
|
||||
print(f"{c(CS.HEADING1)}{stars}Tree {c(CS.HEADING2)}0x{tag:x} ({expander.filename}) {c(CS.HEADING1)}macro expansion complete after {step} step{plural}.{c()}",
|
||||
file=sys.stderr)
|
||||
return tree
|
||||
|
||||
|
||||
|
|
@ -138,7 +174,7 @@ def show_bindings(tree, *, syntax, expander, **kw):
|
|||
"""
|
||||
if syntax != "name":
|
||||
raise SyntaxError("`show_bindings` is an identifier macro only")
|
||||
print(format_bindings(expander, globals_too=False, color=True), file=stderr)
|
||||
print(format_bindings(expander, globals_too=False, color=True), file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -164,7 +200,7 @@ def format_bindings(expander, *, globals_too=False, color=False):
|
|||
|
||||
bindings = expander.bindings if globals_too else expander.local_bindings
|
||||
with io.StringIO() as output:
|
||||
output.write(f"{c(CS.HEADING)}Macro bindings for {c(CS.SOURCEFILENAME)}{expander.filename}{c(CS.HEADING)}:{c()}\n")
|
||||
output.write(f"{c(CS.HEADING1)}Macro bindings for {c(CS.SOURCEFILENAME)}{expander.filename}{c(CS.HEADING1)}:{c()}\n")
|
||||
if not bindings:
|
||||
output.write(maybe_colorize(" <no bindings>\n",
|
||||
ColorScheme.GREYEDOUT))
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ __all__ = ["Dialect", "DialectExpander"]
|
|||
import ast
|
||||
import functools
|
||||
import re
|
||||
from sys import stderr
|
||||
import sys
|
||||
|
||||
from .colorizer import setcolor, colorize, ColorScheme
|
||||
from .coreutils import ismacroimport, get_macros
|
||||
|
|
@ -83,20 +83,17 @@ class Dialect:
|
|||
`mcpyrate.splicing.splice_dialect` (it automatically handles macro-imports,
|
||||
dialect-imports, the magic `__all__`, and the module docstring).
|
||||
|
||||
As an example, for now, until `unpythonic` is ported to `mcpyrate`, see the
|
||||
example dialects in `pydialect`, which are implemented using this exact
|
||||
strategy, but with the older `macropy` macro expander, the older `pydialect`
|
||||
dialect system, and `unpythonic`.
|
||||
As an example, see the `dialects` module in `unpythonic` for example dialects.
|
||||
|
||||
https://github.com/Technologicat/pydialect
|
||||
https://github.com/Technologicat/unpythonic
|
||||
|
||||
To give a flavor; once we get that ported, we'll have *Lispython*, which is
|
||||
essentially Python with TCO, and implicit `return` in tail position::
|
||||
To give a flavor, *Lispython* is essentially Python with TCO, and implicit
|
||||
`return` in tail position::
|
||||
|
||||
# -*- coding: utf-8; -*-
|
||||
'''Lispython example.'''
|
||||
|
||||
from mylibrary import dialects, Lispython
|
||||
from unpythonic.dialects import dialects, Lispython
|
||||
|
||||
def fact(n):
|
||||
def f(k, acc):
|
||||
|
|
@ -114,7 +111,7 @@ class Dialect:
|
|||
return NotImplemented
|
||||
|
||||
|
||||
_message_header = colorize("**StepExpansion: ", ColorScheme.HEADING)
|
||||
_message_header = colorize("**StepExpansion: ", ColorScheme.HEADING1)
|
||||
class StepExpansion(Dialect): # actually part of public API of mcpyrate.debug, for discoverability
|
||||
"""[dialect] Show each step of expansion while dialect-expanding the module.
|
||||
|
||||
|
|
@ -150,8 +147,8 @@ class StepExpansion(Dialect): # actually part of public API of mcpyrate.debug,
|
|||
def _enable_debugmode(self):
|
||||
self.expander.debugmode = True
|
||||
c, CS = setcolor, ColorScheme
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.expander.filename} {c(CS.HEADING)}enabled {c(CS.ATTENTION)}DialectExpander debug mode {c(CS.HEADING)}while taking step {self.expander._step + 1}.{c()}"
|
||||
print(_message_header + msg, file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.expander.filename} {c(CS.HEADING1)}enabled {c(CS.ATTENTION)}DialectExpander debug mode {c(CS.HEADING1)}while taking step {self.expander._step + 1}.{c()}"
|
||||
print(_message_header + msg, file=sys.stderr)
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
|
||||
|
|
@ -210,9 +207,9 @@ class DialectExpander:
|
|||
c, CS = setcolor, ColorScheme
|
||||
if self.debugmode:
|
||||
plural = "s" if self._step != 1 else ""
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}before dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING)}transformers ({self._step} step{plural} total):{c()}\n"
|
||||
print(_message_header + msg, file=stderr)
|
||||
print(format_for_display(content), file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}before dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING1)}transformers ({self._step} step{plural} total):{c()}\n"
|
||||
print(_message_header + msg, file=sys.stderr)
|
||||
print(format_for_display(content), file=sys.stderr)
|
||||
|
||||
# We collect and return the dialect object instances so that both
|
||||
# `transform_ast` and `postprocess_ast` can use the same instances. The
|
||||
|
|
@ -262,14 +259,14 @@ class DialectExpander:
|
|||
self._step += 1
|
||||
|
||||
if self.debugmode:
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}after {c(CS.DIALECTTRANSFORMERNAME)}{module_absname}.{dialectname}.{transform} {c(CS.HEADING)}(step {self._step}):{c()}\n"
|
||||
print(_message_header + msg, file=stderr)
|
||||
print(format_for_display(content), file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}after {c(CS.DIALECTTRANSFORMERNAME)}{module_absname}.{dialectname}.{transform} {c(CS.HEADING1)}(step {self._step}):{c()}\n"
|
||||
print(_message_header + msg, file=sys.stderr)
|
||||
print(format_for_display(content), file=sys.stderr)
|
||||
|
||||
if self.debugmode:
|
||||
plural = "s" if self._step != 1 else ""
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}completed all dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING)}transforms ({self._step} step{plural} total).{c()}"
|
||||
print(_message_header + msg, file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}completed all dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING1)}transforms ({self._step} step{plural} total).{c()}"
|
||||
print(_message_header + msg, file=sys.stderr)
|
||||
|
||||
return content, dialect_instances
|
||||
|
||||
|
|
@ -283,9 +280,9 @@ class DialectExpander:
|
|||
c, CS = setcolor, ColorScheme
|
||||
if self.debugmode:
|
||||
plural = "s" if self._step != 1 else ""
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}before dialect {c(CS.TRANSFORMERKIND)}AST postprocessors {c(CS.HEADING)}({self._step} step{plural} total):{c()}\n"
|
||||
print(_message_header + msg, file=stderr)
|
||||
print(format_for_display(tree), file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}before dialect {c(CS.TRANSFORMERKIND)}AST postprocessors {c(CS.HEADING1)}({self._step} step{plural} total):{c()}\n"
|
||||
print(_message_header + msg, file=sys.stderr)
|
||||
print(format_for_display(tree), file=sys.stderr)
|
||||
|
||||
content = tree
|
||||
for dialect in dialect_instances:
|
||||
|
|
@ -308,14 +305,14 @@ class DialectExpander:
|
|||
self._step += 1
|
||||
|
||||
if self.debugmode:
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}after {c(CS.DIALECTTRANSFORMERNAME)}{format_macrofunction(dialect)}.postprocess_ast {c(CS.HEADING)}(step {self._step}):{c()}\n"
|
||||
print(_message_header + msg, file=stderr)
|
||||
print(format_for_display(content), file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}after {c(CS.DIALECTTRANSFORMERNAME)}{format_macrofunction(dialect)}.postprocess_ast {c(CS.HEADING1)}(step {self._step}):{c()}\n"
|
||||
print(_message_header + msg, file=sys.stderr)
|
||||
print(format_for_display(content), file=sys.stderr)
|
||||
|
||||
if self.debugmode:
|
||||
plural = "s" if self._step != 1 else ""
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}completed all dialect {c(CS.TRANSFORMERKIND)}AST postprocessors {c(CS.HEADING)}({self._step} step{plural} total).{c()}"
|
||||
print(_message_header + msg, file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}completed all dialect {c(CS.TRANSFORMERKIND)}AST postprocessors {c(CS.HEADING1)}({self._step} step{plural} total).{c()}"
|
||||
print(_message_header + msg, file=sys.stderr)
|
||||
|
||||
return content
|
||||
|
||||
|
|
|
|||
|
|
@ -51,15 +51,17 @@ __all__ = ["namemacro", "isnamemacro",
|
|||
"MacroExpander", "MacroCollector",
|
||||
"expand_macros", "find_macros"]
|
||||
|
||||
from ast import (Name, Subscript, Tuple, Import, alias, AST, Assign, Store, Constant,
|
||||
Lambda, arguments, Call, copy_location, iter_fields, NodeVisitor)
|
||||
import sys
|
||||
from ast import (AST, Assign, Call, Starred, Constant, Import, Lambda, Name,
|
||||
NodeVisitor, Store, Subscript, Tuple, alias, arguments,
|
||||
copy_location, iter_fields)
|
||||
from copy import copy
|
||||
from warnings import warn_explicit
|
||||
|
||||
from .core import BaseMacroExpander, global_postprocess, Done
|
||||
from .coreutils import ismacroimport, get_macros
|
||||
from .core import BaseMacroExpander, Done, global_postprocess
|
||||
from .coreutils import get_macros, ismacroimport
|
||||
from .unparser import unparse_with_fallbacks
|
||||
from .utils import format_macrofunction
|
||||
from .utils import format_location, format_macrofunction
|
||||
|
||||
|
||||
def namemacro(function):
|
||||
|
|
@ -94,17 +96,43 @@ def isparametricmacro(function):
|
|||
|
||||
# --------------------------------------------------------------------------------
|
||||
|
||||
def destructure_candidate(tree):
|
||||
def destructure_candidate(tree, *, filename, _validate_call_syntax=True):
|
||||
"""Destructure a macro call candidate AST, `macroname` or `macroname[arg0, ...]`."""
|
||||
if type(tree) is Name:
|
||||
return tree.id, []
|
||||
elif type(tree) is Subscript and type(tree.value) is Name:
|
||||
macroargs = tree.slice.value
|
||||
if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper
|
||||
macroargs = tree.slice
|
||||
else:
|
||||
macroargs = tree.slice.value
|
||||
|
||||
if type(macroargs) is Tuple: # [a0, a1, ...]
|
||||
macroargs = macroargs.elts
|
||||
else: # anything that doesn't have at least one comma at the top level
|
||||
macroargs = [macroargs]
|
||||
return tree.value.id, macroargs
|
||||
# Up to Python 3.8, decorators cannot be subscripted. This is a problem for
|
||||
# decorator macros that would like to have macro arguments.
|
||||
#
|
||||
# To work around this, we allow passing macro arguments also using parentheses
|
||||
# (like in `macropy`). Note macro arguments must still be passed positionally!
|
||||
#
|
||||
# For uniformity, we limit to a subset of the function call syntax that
|
||||
# remains valid if you replace the parentheses with brackets in Python 3.9.
|
||||
elif type(tree) is Call and type(tree.func) is Name and not tree.keywords:
|
||||
# `MacroExpander._detect_macro_items` needs to perform a preliminary check
|
||||
# without full validation when it is trying to detect which `with` items
|
||||
# and decorators are in fact macro invocations.
|
||||
if _validate_call_syntax:
|
||||
if not tree.args: # reject empty args
|
||||
approx_sourcecode = unparse_with_fallbacks(tree, debug=True, color=True)
|
||||
loc = format_location(filename, tree, approx_sourcecode)
|
||||
raise SyntaxError(f"{loc}\nmust provide at least one argument when passing macro arguments")
|
||||
if any(type(arg) is Starred for arg in tree.args): # reject starred items
|
||||
approx_sourcecode = unparse_with_fallbacks(tree, debug=True, color=True)
|
||||
loc = format_location(filename, tree, approx_sourcecode)
|
||||
raise SyntaxError(f"{loc}\nunpacking (splatting) not supported in macro argument position")
|
||||
return tree.func.id, tree.args
|
||||
return None, None # not a macro invocation
|
||||
|
||||
|
||||
|
|
@ -115,13 +143,13 @@ class MacroExpander(BaseMacroExpander):
|
|||
"""Shorthand to check `destructure_candidate` output.
|
||||
|
||||
Return whether that output is a macro call to a macro (of invocation
|
||||
type `syntax`) bound in this expander.
|
||||
type `syntax`) bound in this expander or globally.
|
||||
"""
|
||||
if not (macroname and self.isbound(macroname)):
|
||||
return False
|
||||
if syntax == 'name':
|
||||
return isnamemacro(self.bindings[macroname])
|
||||
return not macroargs or isparametricmacro(self.bindings[macroname])
|
||||
return isnamemacro(self.isbound(macroname))
|
||||
return not macroargs or isparametricmacro(self.isbound(macroname))
|
||||
|
||||
def visit_Subscript(self, subscript):
|
||||
"""Detect an expression (expr) macro invocation.
|
||||
|
|
@ -138,10 +166,16 @@ class MacroExpander(BaseMacroExpander):
|
|||
# because things like `(some_expr_macro[tree])[subscript_expression]` are valid. This
|
||||
# is actually exploited by `h`, as in `q[h[target_macro][tree_for_target_macro]]`.
|
||||
candidate = subscript.value
|
||||
macroname, macroargs = destructure_candidate(candidate)
|
||||
macroname, macroargs = destructure_candidate(candidate, filename=self.filename,
|
||||
_validate_call_syntax=False)
|
||||
if self.ismacrocall(macroname, macroargs, "expr"):
|
||||
# Now we know it's a macro invocation, so we can validate the parenthesis syntax to pass arguments.
|
||||
macroname, macroargs = destructure_candidate(candidate, filename=self.filename)
|
||||
kw = {"args": macroargs}
|
||||
tree = subscript.slice.value
|
||||
if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper
|
||||
tree = subscript.slice
|
||||
else:
|
||||
tree = subscript.slice.value
|
||||
sourcecode = unparse_with_fallbacks(subscript, debug=True, color=True, expander=self)
|
||||
new_tree = self.expand("expr", subscript, macroname, tree, sourcecode=sourcecode, kw=kw)
|
||||
if new_tree is None:
|
||||
|
|
@ -190,13 +224,35 @@ class MacroExpander(BaseMacroExpander):
|
|||
do anything it wants to its input tree. Any remaining block macro
|
||||
invocations are attached to the `With` node, so if that is removed,
|
||||
they will be skipped.
|
||||
|
||||
**NOTE**: At least up to v3.3.0, if there are three or more macro
|
||||
invocations in the same `with` statement::
|
||||
|
||||
with macro1, macro2, macro3:
|
||||
...
|
||||
|
||||
this is equivalent with::
|
||||
|
||||
with macro1:
|
||||
with macro2, macro3:
|
||||
...
|
||||
|
||||
and **not** equivalent with::
|
||||
|
||||
with macro1:
|
||||
with macro2:
|
||||
with macro3:
|
||||
...
|
||||
|
||||
which may be important if `macro1` needs to scan for invocations of
|
||||
`macro2` and `macro3` to work together with them.
|
||||
"""
|
||||
macros, others = self._detect_macro_items(withstmt.items, "block")
|
||||
if not macros:
|
||||
return self.generic_visit(withstmt)
|
||||
with_item = macros[0]
|
||||
candidate = with_item.context_expr
|
||||
macroname, macroargs = destructure_candidate(candidate)
|
||||
macroname, macroargs = destructure_candidate(candidate, filename=self.filename)
|
||||
|
||||
# let the source code and `invocation` see also the withitem we pop away
|
||||
sourcecode = unparse_with_fallbacks(withstmt, debug=True, color=True, expander=self)
|
||||
|
|
@ -250,7 +306,7 @@ class MacroExpander(BaseMacroExpander):
|
|||
if not macros:
|
||||
return self.generic_visit(decorated)
|
||||
innermost_macro = macros[-1]
|
||||
macroname, macroargs = destructure_candidate(innermost_macro)
|
||||
macroname, macroargs = destructure_candidate(innermost_macro, filename=self.filename)
|
||||
|
||||
# let the source code and `invocation` see also the decorator we pop away
|
||||
sourcecode = unparse_with_fallbacks(decorated, debug=True, color=True, expander=self)
|
||||
|
|
@ -279,7 +335,8 @@ class MacroExpander(BaseMacroExpander):
|
|||
candidate = item.context_expr
|
||||
else:
|
||||
candidate = item
|
||||
macroname, macroargs = destructure_candidate(candidate)
|
||||
macroname, macroargs = destructure_candidate(candidate, filename=self.filename,
|
||||
_validate_call_syntax=False)
|
||||
|
||||
# warn about likely mistake
|
||||
if (macroname and self.isbound(macroname) and
|
||||
|
|
@ -350,7 +407,11 @@ class MacroCollector(NodeVisitor):
|
|||
Sister class of the actual `MacroExpander`, mirroring its syntax detection.
|
||||
"""
|
||||
def __init__(self, expander):
|
||||
"""`expander`: a `MacroExpander` instance to query macro bindings from."""
|
||||
"""`expander`: a `MacroExpander` instance to query macro bindings from.
|
||||
|
||||
`filename`: full path to `.py` file being expanded, for error reporting.
|
||||
Only used for errors during `destructure_candidate`.
|
||||
"""
|
||||
self.expander = expander
|
||||
self.clear()
|
||||
|
||||
|
|
@ -379,7 +440,7 @@ class MacroCollector(NodeVisitor):
|
|||
|
||||
def visit_Subscript(self, subscript):
|
||||
candidate = subscript.value
|
||||
macroname, macroargs = destructure_candidate(candidate)
|
||||
macroname, macroargs = destructure_candidate(candidate, filename=self.expander.filename)
|
||||
if self.expander.ismacrocall(macroname, macroargs, "expr"):
|
||||
key = (macroname, "expr")
|
||||
if key not in self._seen:
|
||||
|
|
@ -388,7 +449,10 @@ class MacroCollector(NodeVisitor):
|
|||
self.visit(macroargs)
|
||||
# Don't `self.generic_visit(tree)`; that'll incorrectly detect
|
||||
# the name part as an identifier macro. Recurse only in the expr.
|
||||
self.visit(subscript.slice.value)
|
||||
if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper
|
||||
self.visit(subscript.slice)
|
||||
else:
|
||||
self.visit(subscript.slice.value)
|
||||
else:
|
||||
self.generic_visit(subscript)
|
||||
|
||||
|
|
@ -397,7 +461,7 @@ class MacroCollector(NodeVisitor):
|
|||
if macros:
|
||||
for with_item in macros:
|
||||
candidate = with_item.context_expr
|
||||
macroname, macroargs = destructure_candidate(candidate)
|
||||
macroname, macroargs = destructure_candidate(candidate, filename=self.expander.filename)
|
||||
key = (macroname, "block")
|
||||
if key not in self._seen:
|
||||
self.collected.append(key)
|
||||
|
|
@ -419,7 +483,7 @@ class MacroCollector(NodeVisitor):
|
|||
macros, others = self.expander._detect_macro_items(decorated.decorator_list, "decorator")
|
||||
if macros:
|
||||
for macro in macros:
|
||||
macroname, macroargs = destructure_candidate(macro)
|
||||
macroname, macroargs = destructure_candidate(macro, filename=self.expander.filename)
|
||||
key = (macroname, "decorator")
|
||||
if key not in self._seen:
|
||||
self.collected.append(key)
|
||||
|
|
|
|||
|
|
@ -155,7 +155,7 @@ def path_stats(path, _stats_cache=None):
|
|||
# TODO: Or just document it, that the dialect definition module *must* macro-import those macros
|
||||
# TODO: even if it just injects them in the template?
|
||||
with tokenize.open(path) as sourcefile:
|
||||
tree = ast.parse(sourcefile.read())
|
||||
tree = ast.parse(sourcefile.read(), filename=path)
|
||||
|
||||
macroimports = []
|
||||
dialectimports = []
|
||||
|
|
@ -165,7 +165,7 @@ def path_stats(path, _stats_cache=None):
|
|||
macroimports.append(stmt)
|
||||
elif ismacroimport(stmt, magicname="dialects"):
|
||||
dialectimports.append(stmt)
|
||||
elif iswithphase(stmt): # for multi-phase compilation: scan also inside top-level `with phase`
|
||||
elif iswithphase(stmt, filename=path): # for multi-phase compilation: scan also inside top-level `with phase`
|
||||
scan(stmt)
|
||||
scan(tree)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,9 +9,7 @@ __all__ = ["ASTMarker", "get_markers", "delete_markers", "check_no_markers_remai
|
|||
|
||||
import ast
|
||||
|
||||
from . import core
|
||||
from . import utils
|
||||
from . import walkers
|
||||
from . import core, utils, walkers
|
||||
|
||||
|
||||
class ASTMarker(ast.AST):
|
||||
|
|
@ -36,7 +34,9 @@ class ASTMarker(ast.AST):
|
|||
section. So just before the quote operator exits, it checks that all
|
||||
quasiquote markers within that section have been compiled away.
|
||||
"""
|
||||
def __init__(self, body):
|
||||
# TODO: Silly default `None`, because `copy` and `deepcopy` call `__init__` without arguments,
|
||||
# TODO: though the docs say they behave like `pickle` (and wouldn't thus need to call __init__ at all!).
|
||||
def __init__(self, body=None):
|
||||
"""body: the actual AST that is annotated by this marker"""
|
||||
self.body = body
|
||||
self._fields = ["body"] # support ast.iter_fields
|
||||
|
|
@ -63,7 +63,7 @@ def delete_markers(tree, cls=ASTMarker):
|
|||
class ASTMarkerDeleter(walkers.ASTTransformer):
|
||||
def transform(self, tree):
|
||||
if isinstance(tree, cls):
|
||||
tree = tree.body
|
||||
return self.visit(tree.body)
|
||||
return self.generic_visit(tree)
|
||||
return ASTMarkerDeleter().visit(tree)
|
||||
|
||||
|
|
@ -79,7 +79,6 @@ def check_no_markers_remaining(tree, *, filename, cls=None):
|
|||
`filename` is the full path to the `.py` file, for error reporting.
|
||||
|
||||
Convenience function.
|
||||
|
||||
"""
|
||||
cls = cls or ASTMarker
|
||||
remaining_markers = get_markers(tree, cls)
|
||||
|
|
|
|||
|
|
@ -44,15 +44,15 @@ import ast
|
|||
|
||||
# Note we import some macros as regular functions. We just want their syntax transformers.
|
||||
from .astfixers import fix_locations # noqa: F401, used in macro output.
|
||||
from .coreutils import _mcpyrate_attr
|
||||
from .debug import step_expansion # noqa: F401, used in macro output.
|
||||
from .expander import MacroExpander, namemacro, parametricmacro
|
||||
from .quotes import q, astify, unastify, capture_value
|
||||
from .quotes import astify, capture_value, q, unastify
|
||||
|
||||
|
||||
def _mcpyrate_metatools_attr(attr):
|
||||
"""Create an AST that, when compiled and run, looks up `mcpyrate.metatools.attr`."""
|
||||
mcpyrate_metatools_module = ast.Attribute(value=ast.Name(id="mcpyrate"), attr="metatools")
|
||||
return ast.Attribute(value=mcpyrate_metatools_module, attr=attr)
|
||||
return _mcpyrate_attr(f"metatools.{attr}")
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -21,11 +21,11 @@ because Racket's phase level tower is the system that most resembles this one.
|
|||
__all__ = ["phase", "ismultiphase", "detect_highest_phase", "isdebug", "multiphase_expand"]
|
||||
|
||||
import ast
|
||||
from copy import copy, deepcopy
|
||||
import sys
|
||||
from copy import copy, deepcopy
|
||||
|
||||
from .colorizer import setcolor, ColorScheme
|
||||
from . import compiler
|
||||
from .colorizer import ColorScheme, setcolor
|
||||
from .coreutils import ismacroimport
|
||||
from .expander import destructure_candidate, namemacro, parametricmacro
|
||||
from .unparser import unparse_with_fallbacks
|
||||
|
|
@ -34,7 +34,7 @@ from .utils import getdocstring
|
|||
# --------------------------------------------------------------------------------
|
||||
# Private utilities.
|
||||
|
||||
def iswithphase(stmt):
|
||||
def iswithphase(stmt, *, filename):
|
||||
"""Check if AST node `stmt` is a `with phase[n]`, where `n >= 1` is an integer.
|
||||
|
||||
Return `n`, or `False`.
|
||||
|
|
@ -52,7 +52,7 @@ def iswithphase(stmt):
|
|||
if type(candidate) is not ast.Subscript:
|
||||
return False
|
||||
|
||||
macroname, macroargs = destructure_candidate(candidate)
|
||||
macroname, macroargs = destructure_candidate(candidate, filename=filename)
|
||||
if macroname != "phase":
|
||||
return False
|
||||
if not macroargs or len(macroargs) != 1: # exactly one macro-argument
|
||||
|
|
@ -71,8 +71,11 @@ def iswithphase(stmt):
|
|||
|
||||
return n
|
||||
|
||||
def isfutureimport(tree):
|
||||
"""Return whether `tree` is a `from __future__ import ...`."""
|
||||
return isinstance(tree, ast.ImportFrom) and tree.module == "__future__"
|
||||
|
||||
def extract_phase(tree, *, phase=0):
|
||||
def extract_phase(tree, *, filename, phase=0):
|
||||
"""Split `tree` into given `phase` and remaining parts.
|
||||
|
||||
Primarily meant to be called with `tree` the AST of a module that
|
||||
|
|
@ -88,7 +91,8 @@ def extract_phase(tree, *, phase=0):
|
|||
|
||||
The lifted AST is deep-copied to minimize confusion, since it may get
|
||||
edited by macros during macro expansion. (This guarantees that
|
||||
macro-expanding it, in either phase, gives the same result.)
|
||||
macro-expanding it, in either phase, gives the same result,
|
||||
up to and including any side effects of the macros.)
|
||||
"""
|
||||
if not isinstance(phase, int):
|
||||
raise TypeError(f"`phase` must be `int`, got {type(phase)} with value {repr(phase)}")
|
||||
|
|
@ -99,7 +103,7 @@ def extract_phase(tree, *, phase=0):
|
|||
|
||||
remaining = []
|
||||
def lift(withphase): # Lift a `with phase[n]` code block to phase `n - 1`.
|
||||
original_phase = iswithphase(withphase)
|
||||
original_phase = iswithphase(withphase, filename=filename)
|
||||
assert original_phase
|
||||
if original_phase == 1:
|
||||
# Lifting to phase 0. Drop the `with phase` wrapper.
|
||||
|
|
@ -107,7 +111,11 @@ def extract_phase(tree, *, phase=0):
|
|||
else:
|
||||
# Lifting to phase >= 1. Decrease the `n` in `with phase[n]`
|
||||
# by one, so the block gets processed again in the next phase.
|
||||
macroarg = withphase.items[0].context_expr.slice.value
|
||||
if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper
|
||||
macroarg = withphase.items[0].context_expr.slice
|
||||
else:
|
||||
macroarg = withphase.items[0].context_expr.slice.value
|
||||
|
||||
if type(macroarg) is ast.Constant:
|
||||
macroarg.value -= 1
|
||||
elif type(macroarg) is ast.Num: # TODO: Python 3.8: remove ast.Num
|
||||
|
|
@ -116,10 +124,17 @@ def extract_phase(tree, *, phase=0):
|
|||
|
||||
thisphase = []
|
||||
for stmt in tree.body:
|
||||
if iswithphase(stmt) == phase:
|
||||
if iswithphase(stmt, filename=filename) == phase:
|
||||
thisphase.extend(stmt.body)
|
||||
lift(stmt)
|
||||
else:
|
||||
# Issue #28: `__future__` imports.
|
||||
#
|
||||
# `__future__` imports should affect all phases, because they change
|
||||
# the semantics of the module they appear in. Essentially, they are
|
||||
# a kind of dialect defined by the Python core itself.
|
||||
if isfutureimport(stmt):
|
||||
thisphase.append(stmt)
|
||||
remaining.append(stmt)
|
||||
tree.body[:] = remaining
|
||||
|
||||
|
|
@ -127,6 +142,35 @@ def extract_phase(tree, *, phase=0):
|
|||
newmodule.body = thisphase
|
||||
return newmodule
|
||||
|
||||
def split_futureimports(body):
|
||||
"""Split `body` into `__future__` imports and everything else.
|
||||
|
||||
`body`: list of `ast.stmt`, the suite representing a module top level.
|
||||
|
||||
Returns `[future_imports, the_rest]`.
|
||||
"""
|
||||
k = -1 # ensure `k` gets defined even if `body` is empty
|
||||
for k, bstmt in enumerate(body):
|
||||
if not isfutureimport(bstmt):
|
||||
break
|
||||
if k >= 0:
|
||||
return body[:k], body[k:]
|
||||
return [], body
|
||||
|
||||
def inject_after_futureimports(stmt, body):
|
||||
"""Inject a statement into `body` after `__future__` imports.
|
||||
|
||||
`body`: list of `ast.stmt`, the suite representing a module top level.
|
||||
`stmt`: `ast.stmt`, the statement to inject.
|
||||
"""
|
||||
if getdocstring(body):
|
||||
docstring, *body = body
|
||||
futureimports, body = split_futureimports(body)
|
||||
return [docstring] + futureimports + [stmt] + body
|
||||
else: # no docstring
|
||||
futureimports, body = split_futureimports(body)
|
||||
return futureimports + [stmt] + body
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
# Public utilities.
|
||||
|
||||
|
|
@ -237,7 +281,7 @@ def ismultiphase(tree):
|
|||
return False
|
||||
|
||||
|
||||
def detect_highest_phase(tree):
|
||||
def detect_highest_phase(tree, *, filename):
|
||||
"""Scan a module body for `with phase[n]` statements and return highest `n`, or `None`.
|
||||
|
||||
Primarily meant to be called with `tree` the AST of a module that
|
||||
|
|
@ -248,7 +292,7 @@ def detect_highest_phase(tree):
|
|||
"""
|
||||
maxn = None
|
||||
for stmt in tree.body:
|
||||
n = iswithphase(stmt)
|
||||
n = iswithphase(stmt, filename=filename)
|
||||
if maxn is None or (n is not None and n > maxn):
|
||||
maxn = n
|
||||
return maxn
|
||||
|
|
@ -336,7 +380,7 @@ def multiphase_expand(tree, *, filename, self_module, dexpander=None, _optimize=
|
|||
|
||||
Return value is the final phase-0 `tree`, after macro expansion.
|
||||
"""
|
||||
n = detect_highest_phase(tree)
|
||||
n = detect_highest_phase(tree, filename=filename)
|
||||
debug = isdebug(tree)
|
||||
c, CS = setcolor, ColorScheme
|
||||
|
||||
|
|
@ -350,7 +394,7 @@ def multiphase_expand(tree, *, filename, self_module, dexpander=None, _optimize=
|
|||
original_module = None
|
||||
|
||||
if debug:
|
||||
print(f"{c(CS.HEADING)}**Multi-phase compiling module {c(CS.TREEID)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.TREEID)}){c()}", file=sys.stderr)
|
||||
print(f"{c(CS.HEADING1)}**Multi-phase compiling module {c(CS.HEADING2)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.HEADING2)}){c()}", file=sys.stderr)
|
||||
|
||||
# Inject temporary module into `sys.modules`.
|
||||
#
|
||||
|
|
@ -366,20 +410,18 @@ def multiphase_expand(tree, *, filename, self_module, dexpander=None, _optimize=
|
|||
|
||||
for k in range(n, -1, -1): # phase 0 is what a regular compile would do
|
||||
if debug:
|
||||
print(f"{c(CS.HEADING)}**AST for {c(CS.ATTENTION)}PHASE {k}{c(CS.HEADING)} of module {c(CS.TREEID)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.TREEID)}){c()}", file=sys.stderr)
|
||||
print(f"{c(CS.HEADING1)}**AST for {c(CS.ATTENTION)}PHASE {k}{c(CS.HEADING1)} of module {c(CS.HEADING2)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.HEADING2)}){c()}", file=sys.stderr)
|
||||
|
||||
phase_k_tree = extract_phase(tree, phase=k)
|
||||
phase_k_tree = extract_phase(tree, filename=filename, phase=k)
|
||||
if phase_k_tree.body:
|
||||
# inject `__phase__ = k` for introspection (at run time of the phase being compiled now)
|
||||
tgt = ast.Name(id="__phase__", ctx=ast.Store(), lineno=1, col_offset=1)
|
||||
val = ast.Constant(value=k, lineno=1, col_offset=13)
|
||||
assignment = ast.Assign(targets=[tgt], value=val, lineno=1, col_offset=1)
|
||||
|
||||
if getdocstring(phase_k_tree.body):
|
||||
docstring, *body = phase_k_tree.body
|
||||
phase_k_tree.body = [docstring, assignment] + body
|
||||
else: # no docstring
|
||||
phase_k_tree.body = [assignment] + phase_k_tree.body
|
||||
# Issue #28: `__future__` imports.
|
||||
# They must be the first statements after the module docstring, if any. So we inject after them.
|
||||
phase_k_tree.body = inject_after_futureimports(assignment, phase_k_tree.body)
|
||||
|
||||
if debug:
|
||||
print(unparse_with_fallbacks(phase_k_tree, debug=True, color=True), file=sys.stderr)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Python bytecode cache (`.pyc`) cleaner. Deletes `__pycache__` directories."""
|
||||
|
||||
__all__ = ["getpycachedirs", "deletepycachedirs"]
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def getpycachedirs(path):
|
||||
"""Return a list of all `__pycache__` directories under `path` (str).
|
||||
|
||||
Each of the entries starts with `path`.
|
||||
"""
|
||||
if not os.path.isdir(path):
|
||||
raise OSError(f"No such directory: '{path}'")
|
||||
|
||||
paths = []
|
||||
for root, dirs, files in os.walk(path):
|
||||
if "__pycache__" in dirs:
|
||||
paths.append(os.path.join(root, "__pycache__"))
|
||||
return paths
|
||||
|
||||
|
||||
def deletepycachedirs(path):
|
||||
"""Delete all `__pycache__` directories under `path` (str).
|
||||
|
||||
Ignores `FileNotFoundError`, but other errors raise. If an error occurs,
|
||||
some `.pyc` cache files and their directories may already have been deleted.
|
||||
"""
|
||||
for x in getpycachedirs(path):
|
||||
_delete_directory_recursively(x)
|
||||
|
||||
|
||||
def _delete_directory_recursively(path):
|
||||
"""Delete a directory recursively, like 'rm -rf' in the shell.
|
||||
|
||||
Ignores `FileNotFoundError`, but other errors raise. If an error occurs,
|
||||
some files and directories may already have been deleted.
|
||||
"""
|
||||
for root, dirs, files in os.walk(path, topdown=False, followlinks=False):
|
||||
for x in files:
|
||||
try:
|
||||
os.unlink(os.path.join(root, x))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
for x in dirs:
|
||||
try:
|
||||
os.rmdir(os.path.join(root, x))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
try:
|
||||
os.rmdir(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
|
@ -12,24 +12,33 @@ and uncompiler, respectively.
|
|||
"""
|
||||
|
||||
__all__ = ["capture_value", "capture_macro", "capture_as_macro",
|
||||
"is_captured_value", "is_captured_macro",
|
||||
"astify", "unastify",
|
||||
"q", "u", "n", "a", "s", "t", "h"]
|
||||
|
||||
import ast
|
||||
import copy
|
||||
import pickle
|
||||
import sys
|
||||
|
||||
from .core import global_bindings, Done, MacroExpansionError
|
||||
from .core import Done, MacroExpansionError, global_bindings
|
||||
from .coreutils import _mcpyrate_attr
|
||||
from .expander import MacroExpander, isnamemacro
|
||||
from .markers import ASTMarker, check_no_markers_remaining, delete_markers
|
||||
from .unparser import unparse, unparse_with_fallbacks
|
||||
from .utils import gensym, scrub_uuid, flatten, extract_bindings, NestingLevelTracker
|
||||
from .utils import (NestingLevelTracker, extract_bindings, flatten, gensym,
|
||||
scrub_uuid)
|
||||
|
||||
|
||||
def _mcpyrate_quotes_attr(attr):
|
||||
"""Create an AST that, when compiled and run, looks up `mcpyrate.quotes.attr`."""
|
||||
mcpyrate_quotes_module = ast.Attribute(value=ast.Name(id="mcpyrate"), attr="quotes")
|
||||
return ast.Attribute(value=mcpyrate_quotes_module, attr=attr)
|
||||
def _mcpyrate_quotes_attr(attr, *, force_import=False):
|
||||
"""Create an AST that, when compiled and run, looks up `mcpyrate.quotes.attr`.
|
||||
|
||||
If `force_import` is `True`, use the builtin `__import__` function to
|
||||
first import the `mcpyrate.quotes` module. This is useful for e.g.
|
||||
hygienically unquoted values, whose eventual use site might not import
|
||||
any `mcpyrate` modules.
|
||||
"""
|
||||
return _mcpyrate_attr(f"quotes.{attr}", force_import=force_import)
|
||||
|
||||
|
||||
class QuasiquoteMarker(ASTMarker):
|
||||
|
|
@ -133,13 +142,20 @@ def lift_sourcecode(value, filename="<unknown>"):
|
|||
"""
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"`n[]`: expected an expression that evaluates to str, result was {type(value)} with value {repr(value)}")
|
||||
return ast.parse(value, filename=filename, mode="eval").body
|
||||
return ast.parse(value, filename=f"<invocation of n[] in '{filename}'>", mode="eval").body
|
||||
|
||||
|
||||
def _typecheck(node, cls, macroname):
|
||||
body = node.body if isinstance(node, ASTMarker) else node
|
||||
if not isinstance(body, cls):
|
||||
raise TypeError(f"{macroname}: expected an expression node, got {type(body)} with value {repr(body)}")
|
||||
if isinstance(node, ASTMarker):
|
||||
if isinstance(node.body, list): # statement suite inside a marker
|
||||
for child in node.body:
|
||||
_typecheck(child, cls, macroname)
|
||||
return
|
||||
# single AST node inside a marker
|
||||
_typecheck(node.body, cls, macroname)
|
||||
return
|
||||
if not isinstance(node, cls):
|
||||
raise TypeError(f"{macroname}: expected {cls}, got {type(node)} with value {repr(node)}")
|
||||
|
||||
def _flatten_and_typecheck_iterable(nodes, cls, macroname):
|
||||
try:
|
||||
|
|
@ -207,6 +223,10 @@ def splice_ast_literals(tree, filename):
|
|||
doit(item)
|
||||
newthing.append(item)
|
||||
thing[:] = newthing
|
||||
# As of Python 3.9, `Global` and `Nonlocal` are the only AST node types
|
||||
# where a field contains a `list` of bare strings.
|
||||
elif isinstance(thing, (ast.Global, ast.Nonlocal)):
|
||||
pass
|
||||
elif isinstance(thing, ast.AST):
|
||||
for fieldname, value in ast.iter_fields(thing):
|
||||
if isinstance(value, list):
|
||||
|
|
@ -278,7 +298,7 @@ def capture_value(value, name):
|
|||
# and serialization.
|
||||
#
|
||||
frozen_value = pickle.dumps(value)
|
||||
return ast.Call(_mcpyrate_quotes_attr("lookup_value"),
|
||||
return ast.Call(_mcpyrate_quotes_attr("lookup_value", force_import=True),
|
||||
[ast.Tuple(elts=[ast.Constant(value=name),
|
||||
ast.Constant(value=frozen_value)])],
|
||||
[])
|
||||
|
|
@ -290,8 +310,29 @@ def lookup_value(key):
|
|||
|
||||
Usually there's no need to call this function manually; `capture_value`
|
||||
(and thus also `h[]`) will generate an AST that calls this automatically.
|
||||
|
||||
**NOTE**: For advanced macrology: if your own macros need to detect hygienic
|
||||
captures using `is_captured_value`, and you want to look up the captured
|
||||
value based on a key returned by that function, be aware that `lookup_value`
|
||||
will only succeed if a value has been captured.
|
||||
|
||||
Trying to look up a key that was extracted from a pre-capture AST
|
||||
raises `ValueError`. In terms of the discussion in the docstring of
|
||||
`is_captured_value`, you need a `lookup_value` AST for a value to
|
||||
be present; a `capture_value` AST is too early. The transition occurs
|
||||
when the use site of `q` runs.
|
||||
|
||||
In that scenario, before you call `lookup_value` on your key, check that
|
||||
`frozen_value is not None` (see docstring of `is_captured_value`);
|
||||
that indicates that a value has been captured and can be decoded by
|
||||
this function.
|
||||
"""
|
||||
name, frozen_value = key
|
||||
|
||||
# Trying to look up a result of `is_captured_value` that isn't captured yet.
|
||||
if frozen_value is None:
|
||||
raise ValueError(f"The given key does not (yet) point to a value: {repr(key)}")
|
||||
|
||||
cachekey = (name, id(frozen_value)) # id() so each capture instance behaves independently
|
||||
if cachekey not in _lookup_cache:
|
||||
_lookup_cache[cachekey] = pickle.loads(frozen_value)
|
||||
|
|
@ -333,7 +374,8 @@ def capture_as_macro(macro):
|
|||
|
||||
Like `capture_macro`, but with one less level of delay. This injects the
|
||||
macro into the expander's global bindings table immediately, and returns
|
||||
the uniqified `ast.Name` that can be used to refer to it.
|
||||
the uniqified `ast.Name` that can be used to refer to it hygienically,
|
||||
using `a[]`.
|
||||
|
||||
The name is taken automatically from the name of the macro function.
|
||||
"""
|
||||
|
|
@ -358,6 +400,235 @@ def lookup_macro(key):
|
|||
global_bindings[unique_name] = pickle.loads(frozen_macro)
|
||||
return ast.Name(id=unique_name)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
# Advanced macrology support.
|
||||
|
||||
# TODO: In a future version, do we want to add an ASTMarker for captured values
|
||||
# TODO: that are ready for consumption? We could save the actual AST (which is
|
||||
# TODO: now detected directly) into the `body` attribute of the marker, and make
|
||||
# TODO: the compiler delete `HygienicValue` markers (replacing each by its `.body`)
|
||||
# TODO: as the last step before handing the AST over to Python.
|
||||
|
||||
def is_captured_value(tree):
|
||||
"""Test whether `tree` is a hygienically captured run-time value.
|
||||
|
||||
This function is sometimes useful for advanced macrology. It facilitates
|
||||
user-defined macros to work together in an environment where hygienic
|
||||
captures are present. One macro, using quasiquotes, builds an AST, and
|
||||
another macro analyzes the expanded AST later.
|
||||
|
||||
Consider first, however, if you can arrange things so that the second macro
|
||||
could analyze an *unexpanded* AST; that's often much easier. When the first
|
||||
macro simply must expand first (for whatever reason), that's where this function
|
||||
comes in.
|
||||
|
||||
With this function, you can check (either by name or by value) whether some
|
||||
`q[h[myfunction]]` points to the desired `"myfunction"`, so that e.g. the AST
|
||||
produced by `q[h[myfunction](a0, ...)]` can be recognized as a call to your
|
||||
`myfunction`. This allows your second macro to know it's `myfunction`,
|
||||
so that it'll know how to interpret the args of that call.
|
||||
|
||||
Real-world examples of where this is useful are too unwieldy to explain
|
||||
here, but can be found in `unpythonic.syntax`. Particularly, see any use
|
||||
sites of the helper function `unpythonic.syntax.nameutil.isx`.
|
||||
|
||||
To detect a hygienically captured *macro*, use `is_captured_macro` instead.
|
||||
|
||||
Return value:
|
||||
|
||||
- On no match, return `False`.
|
||||
|
||||
- On match, return a tuple `(name, frozen_value)`, where:
|
||||
|
||||
- `name` (str) is the name of the captured identifier, or when the captured
|
||||
value is from an arbitrary expression, the unparsed source code of that
|
||||
expression. There is no name mangling for identifiers; it's the exact
|
||||
original name that appeared in the source code.
|
||||
|
||||
- `frozen_value` is either a `bytes` object that stores the frozen value
|
||||
as opaque binary data, or `None` if the value has not been captured yet.
|
||||
|
||||
The `bytes` object can be decoded by passing the whole return value as `key`
|
||||
to `lookup_value`. That function will decode the data and return the actual
|
||||
value, just as if the hygienic reference was decoded normally at run time.
|
||||
|
||||
**NOTE**:
|
||||
|
||||
Stages in the life of a hygienically captured *run-time value* in `mcpyrate`:
|
||||
|
||||
1. When the surrounding `q` expands, it first expands any unquotes nested
|
||||
within it, but only those where the quote level hits zero. The `h[]` is
|
||||
converted into a `Capture` AST marker; see the `h` operator for details.
|
||||
|
||||
2. Then, still while the surrounding `q` expands, `q` compiles quasiquote
|
||||
markers. A `Capture` marker, in particular, compiles into a call to
|
||||
the function `capture_value`. This is the output at macro expansion time
|
||||
(of the use site of `q`).
|
||||
|
||||
3. When the use site of `q` reaches run time, the `capture_value` runs
|
||||
(thus actually performing the capture), and replaces itself (in the
|
||||
AST that was produced by `q`) with a call to the function `lookup_value`.
|
||||
That `lookup_value` call is still an AST node.
|
||||
|
||||
4. In typical usage, that use site of `q` is inside the implementation
|
||||
of some user-defined macro. When *that macro's use site* reaches run
|
||||
time, the `lookup_value` runs (each time that expression is executed).
|
||||
|
||||
So in the macro expansion of `q`, we have a call to `capture_value`
|
||||
representing the hygienically captured run-time value. But once the macro
|
||||
that uses `q` has returned its output, then we instead have a call to
|
||||
`lookup_value`. The latter is the most likely scenario for advanced
|
||||
user-defined macros that work together.
|
||||
"""
|
||||
if type(tree) is not ast.Call:
|
||||
return False
|
||||
|
||||
# The format is one of:
|
||||
#
|
||||
# - direct reference: `(mcpyrate.quotes).xxx`
|
||||
# - reference by import: `(__import__("mcpyrate.quotes", ...).quotes).xxx`
|
||||
#
|
||||
# First check the `xxx` part:
|
||||
callee = tree.func
|
||||
if not (type(callee) is ast.Attribute and callee.attr in ("capture_value", "lookup_value")):
|
||||
return False
|
||||
# Then the rest:
|
||||
if not _is_mcpyrate_quotes_reference(callee.value):
|
||||
return False
|
||||
|
||||
# This AST destructuring and constant extraction must match the format
|
||||
# of the argument lists produced by the quasiquote system for calls to
|
||||
# `capture_value` and `lookup_value`.
|
||||
if callee.attr == "capture_value": # the call is `capture_value(..., name)`
|
||||
name_node = tree.args[1]
|
||||
assert type(name_node) is ast.Constant and type(name_node.value) is str
|
||||
return (name_node.value, None) # the value hasn't been captured yet
|
||||
elif callee.attr == "lookup_value": # the call is `lookup_value(key)`
|
||||
key_node = tree.args[0]
|
||||
name_node, frozen_value_node = key_node.elts
|
||||
assert type(name_node) is ast.Constant and type(name_node.value) is str
|
||||
assert type(frozen_value_node) is ast.Constant and type(frozen_value_node.value) is bytes
|
||||
return (name_node.value, frozen_value_node.value)
|
||||
|
||||
assert False # cannot happen
|
||||
|
||||
|
||||
def is_captured_macro(tree):
|
||||
"""Just like `is_captured_value`, but detect a hygienically captured macro instead.
|
||||
|
||||
To detect a hygienically captured *run-time value*, use `is_captured_value` instead.
|
||||
|
||||
Return value:
|
||||
|
||||
- On no match, return `False`.
|
||||
|
||||
- On match, return a tuple `(name, unique_name, frozen_macro)`, where:
|
||||
|
||||
- `name` (str) is the name of the macro, as it appeared in the bindings
|
||||
of the expander instance it was captured from.
|
||||
|
||||
- `unique_name` (str) is `name` with an underscore and UUID appended,
|
||||
to make it unique. This is the name the macro will be injected as
|
||||
into the expander's global bindings table.
|
||||
|
||||
(By unique, we mean "universally unique anywhere for approximately
|
||||
the next one thousand years"; see `mcpyrate.gensym`, which links to
|
||||
the UUID spec used by the implementation.)
|
||||
|
||||
- `frozen_macro` is either `bytes` object that stores a reference to the
|
||||
frozen macro function as opaque binary data.
|
||||
|
||||
The `bytes` object can be decoded by passing the whole return value as `key`
|
||||
to `lookup_macro`. That function will decode the data, inject the macro into
|
||||
the expander's global bindings table (if not already there), and give you an
|
||||
`ast.Name` node whose `id` attribute contains the unique name (str), just as
|
||||
if the hygienic reference was decoded normally at macro expansion time.
|
||||
|
||||
Then, once the injection has taken place, you can obtain the actual macro
|
||||
function object by calling `expander.isbound(id)`.
|
||||
|
||||
**NOTE**:
|
||||
|
||||
Stages in the life of a hygienically captured *macro* in `mcpyrate` are as follows.
|
||||
Note that unlike `capture_value`, a call to `capture_macro` never appears in the AST.
|
||||
|
||||
1. When the surrounding `q` expands, it first expands any unquotes nested
|
||||
within it, but only those where the quote level hits zero. The `h[]` is
|
||||
converted into a `Capture` AST marker; see the `h` operator for details.
|
||||
|
||||
2. Then, still while the surrounding `q` expands, `q` compiles quasiquote
|
||||
markers. A `Capture` marker for a macro, in particular, triggers an
|
||||
immediate call to the function `capture_macro`. The result is an AST
|
||||
representing a call to the function `lookup_macro`. This gets injected
|
||||
into the AST produced by `q`.
|
||||
|
||||
3. When the use site of `q` reaches run time, the `lookup_macro` runs,
|
||||
injecting the macro (under its unique name) into the expander's global
|
||||
bindings table. The `lookup_macro` call replaces itself with an `ast.Name`
|
||||
whose `id` attribute contains the unique name of the macro.
|
||||
|
||||
4. In typical usage, that use site of `q` is inside the implementation
|
||||
of some user-defined macro. Upon further macro expansion of *that macro's
|
||||
use site*, the expander finds the now-bound unique name of the macro, and
|
||||
proceeds to expand that macro.
|
||||
|
||||
So in the macro expansion of `q`, we have a call to `lookup_macro`
|
||||
representing the hygienically captured macro. But this disappears after
|
||||
a very brief window of time, namely when the use site of `q` reaches run
|
||||
time. Thus, this function likely has much fewer use cases than
|
||||
`is_captured_value`, but is provided for completeness.
|
||||
|
||||
(The point of hygienic macro capture is that a macro can safely return a further
|
||||
macro invocation, and guarantee that this will invoke the intended macro - without
|
||||
requiring the user to import that other macro, and without being forced to expand
|
||||
it away before returning from the original macro.)
|
||||
"""
|
||||
if type(tree) is not ast.Call:
|
||||
return False
|
||||
|
||||
callee = tree.func
|
||||
if not (type(callee) is ast.Attribute and callee.attr == "lookup_macro"):
|
||||
return False
|
||||
if not _is_mcpyrate_quotes_reference(callee.value):
|
||||
return False
|
||||
|
||||
# This AST destructuring and constant extraction must match the format
|
||||
# of the argument lists produced by the quasiquote system for calls to
|
||||
# `lookup_macro`.
|
||||
key_node = tree.args[0] # the call is `lookup_macro(key)`
|
||||
name_node, unique_name_node, frozen_macro_node = key_node.elts
|
||||
assert type(name_node) is ast.Constant and type(name_node.value) is str
|
||||
assert type(unique_name_node) is ast.Constant and type(unique_name_node.value) is str
|
||||
assert type(frozen_macro_node) is ast.Constant and type(frozen_macro_node.value) is bytes
|
||||
return (name_node.value, unique_name_node.value, frozen_macro_node.value)
|
||||
|
||||
|
||||
def _is_mcpyrate_quotes_reference(tree):
|
||||
"""Detect whether `tree` is a reference to `mcpyrate.quotes`.
|
||||
|
||||
This matches the ASTs corresponding to:
|
||||
- direct reference: `mcpyrate.quotes`
|
||||
- reference by import: `__import__("mcpyrate.quotes", ...).quotes`
|
||||
|
||||
Note `__import__` of a dotted module name returns the top-level module,
|
||||
so we have the name `quotes` appear twice in different places.
|
||||
|
||||
See `_mcpyrate_quotes_attr` and `mcpyrate.coreutils._mcpyrate_attr`.
|
||||
"""
|
||||
if not (type(tree) is ast.Attribute and tree.attr == "quotes"):
|
||||
return False
|
||||
moduleref = tree.value
|
||||
if type(moduleref) is ast.Name and moduleref.id == "mcpyrate":
|
||||
return "direct" # ok, direct reference
|
||||
elif (type(moduleref) is ast.Call and type(moduleref.func) is ast.Name and
|
||||
moduleref.func.id == "__import__" and type(moduleref.args[0]) is ast.Constant and
|
||||
moduleref.args[0].value == "mcpyrate.quotes"):
|
||||
return "import" # ok, reference by import
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
# The quasiquote compiler and uncompiler.
|
||||
|
||||
|
|
@ -443,7 +714,7 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr`
|
|||
|
||||
# Builtin types. Mainly support for `u[]`, but also used by the
|
||||
# general case for AST node fields that contain bare values.
|
||||
elif T in (int, float, str, bytes, bool, type(None)):
|
||||
elif T in (int, float, str, bytes, bool, type(None), type(...)):
|
||||
return ast.Constant(value=x)
|
||||
|
||||
elif T is list:
|
||||
|
|
@ -461,6 +732,7 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr`
|
|||
# (Note we support only exactly `Done`, not arbitrary descendants.)
|
||||
elif T is Done:
|
||||
fields = [ast.keyword(a, recurse(b)) for a, b in ast.iter_fields(x)]
|
||||
# We have imported `Done`, so we can refer to it as `mcpyrate.quotes.Done`.
|
||||
node = ast.Call(_mcpyrate_quotes_attr("Done"),
|
||||
[],
|
||||
fields)
|
||||
|
|
@ -682,8 +954,11 @@ def _replace_tree_in_macro_invocation(invocation, newtree):
|
|||
"""
|
||||
new_invocation = copy.copy(invocation)
|
||||
if type(new_invocation) is ast.Subscript:
|
||||
new_invocation.slice = copy.copy(invocation.slice)
|
||||
new_invocation.slice.value = newtree
|
||||
if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper
|
||||
new_invocation.slice = newtree
|
||||
else:
|
||||
new_invocation.slice = copy.copy(invocation.slice)
|
||||
new_invocation.slice.value = newtree
|
||||
elif type(new_invocation) is ast.With:
|
||||
new_invocation.body = newtree
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -19,9 +19,12 @@ __all__ = ["MacroConsole"]
|
|||
|
||||
import ast
|
||||
import code
|
||||
import copy
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
from .. import __version__ as mcpyrate_version
|
||||
from ..compiler import create_module
|
||||
from ..core import MacroExpansionError
|
||||
from ..debug import format_bindings
|
||||
from ..expander import find_macros, MacroExpander, global_postprocess
|
||||
|
|
@ -31,6 +34,8 @@ from .utils import get_makemacro_sourcecode
|
|||
# Despite the meta-levels, there's just one global importer for the Python process.
|
||||
from .. import activate # noqa: F401
|
||||
|
||||
_magic_module_name = "__mcpyrate_repl_self__"
|
||||
|
||||
class MacroConsole(code.InteractiveConsole):
|
||||
def __init__(self, locals=None, filename="<interactive input>"):
|
||||
"""Parameters like in `code.InteractiveConsole`."""
|
||||
|
|
@ -40,7 +45,51 @@ class MacroConsole(code.InteractiveConsole):
|
|||
if locals is None:
|
||||
locals = {}
|
||||
# Lucky that both meta-levels speak the same language, eh?
|
||||
locals['__macro_expander__'] = self.expander
|
||||
locals["__macro_expander__"] = self.expander
|
||||
|
||||
# support `from __self__ import macros, ...`
|
||||
#
|
||||
# To do this, we need the top-level variables in the REPL to be stored
|
||||
# in the namespace of a module, so that `find_macros` can look them up.
|
||||
#
|
||||
# We create a module dynamically. The one obvious way would be to then
|
||||
# replace the `__dict__` of that module with the given `locals`, but
|
||||
# `types.ModuleType` is special in that the `__dict__` binding itself
|
||||
# is read-only:
|
||||
# https://docs.python.org/3/library/stdtypes.html#modules
|
||||
#
|
||||
# This leaves two possible solutions:
|
||||
#
|
||||
# a. Keep our magic module separate from `locals`, and shallow-copy
|
||||
# the user namespace into its `__dict__` after each REPL input.
|
||||
# The magic module's namespace is only used for macro lookups.
|
||||
#
|
||||
# + Behaves exactly like `code.InteractiveConsole` in that the
|
||||
# user-given `locals` *is* the dict where the top-level variables
|
||||
# are stored. So the user can keep a reference to that dict, and
|
||||
# they will see any changes made by assigning to top-level
|
||||
# variables in the REPL.
|
||||
#
|
||||
# - May slow down the REPL when a lot of top-level variables exist.
|
||||
# Likely not a problem in practice.
|
||||
#
|
||||
# b. Use the module's `__dict__` as the local namespace of the REPL
|
||||
# session. At startup, update it from `locals`, to install any
|
||||
# user-provided initial variable bindings for the session.
|
||||
#
|
||||
# + Elegant. No extra copying.
|
||||
#
|
||||
# - Behavior does not match `code.InteractiveConsole`. The
|
||||
# `locals` argument only provides initial values; any updates
|
||||
# made during the REPL session are stored in the module's
|
||||
# `__dict__`, which is a different dict instance. Hence the user
|
||||
# will not see any changes made by assigning to top-level
|
||||
# variables in the REPL.
|
||||
#
|
||||
# We currently use strategy a., for least astonishment.
|
||||
#
|
||||
magic_module = create_module(dotted_name=_magic_module_name, filename=filename)
|
||||
self.magic_module_metadata = copy.copy(magic_module.__dict__)
|
||||
|
||||
super().__init__(locals, filename)
|
||||
|
||||
|
|
@ -61,7 +110,7 @@ class MacroConsole(code.InteractiveConsole):
|
|||
This bypasses `runsource`, so it too can use this function.
|
||||
"""
|
||||
source = textwrap.dedent(source)
|
||||
tree = ast.parse(source)
|
||||
tree = ast.parse(source, self.filename)
|
||||
tree = ast.Interactive(tree.body)
|
||||
code = compile(tree, "<console internal>", "single", self.compile.compiler.flags, 1)
|
||||
self.runcode(code)
|
||||
|
|
@ -84,12 +133,13 @@ class MacroConsole(code.InteractiveConsole):
|
|||
def runsource(self, source, filename="<interactive input>", symbol="single"):
|
||||
# Special REPL commands.
|
||||
if source == "macros?":
|
||||
self.write(format_bindings(self.expander))
|
||||
self.write(format_bindings(self.expander, color=True))
|
||||
return False # complete input
|
||||
elif source.endswith("??"):
|
||||
return self.runsource(f'mcpyrate.repl.utils.sourcecode({source[:-2]})')
|
||||
# Use `_internal_execute` instead of `runsource` to prevent expansion of name macros.
|
||||
return self._internal_execute(f"mcpyrate.repl.utils.sourcecode({source[:-2]})")
|
||||
elif source.endswith("?"):
|
||||
return self.runsource(f"mcpyrate.repl.utils.doc({source[:-1]})")
|
||||
return self._internal_execute(f"mcpyrate.repl.utils.doc({source[:-1]})")
|
||||
|
||||
try:
|
||||
code = self.compile(source, filename, symbol)
|
||||
|
|
@ -100,9 +150,18 @@ class MacroConsole(code.InteractiveConsole):
|
|||
|
||||
try:
|
||||
# TODO: If we want to support dialects in the REPL, this is where to do it.
|
||||
tree = ast.parse(source)
|
||||
# Look at `mcpyrate.compiler.compile`.
|
||||
tree = ast.parse(source, self.filename)
|
||||
|
||||
bindings = find_macros(tree, filename=self.expander.filename, reload=True) # macro-imports (this will import the modules)
|
||||
# macro-imports (this will import the modules)
|
||||
sys.modules[_magic_module_name].__dict__.clear()
|
||||
sys.modules[_magic_module_name].__dict__.update(self.locals) # for self-macro-imports
|
||||
# We treat the initial magic module metadata as write-protected: even if the user
|
||||
# defines a variable of the same name in the user namespace, the metadata fields
|
||||
# in the magic module won't be overwritten.
|
||||
sys.modules[_magic_module_name].__dict__.update(self.magic_module_metadata)
|
||||
bindings = find_macros(tree, filename=self.expander.filename,
|
||||
reload=True, self_module=_magic_module_name)
|
||||
if bindings:
|
||||
self._macro_bindings_changed = True
|
||||
self.expander.bindings.update(bindings)
|
||||
|
|
@ -118,6 +177,7 @@ class MacroConsole(code.InteractiveConsole):
|
|||
# In this case, the standard stack trace is long and points only to our code and the stdlib,
|
||||
# not the erroneous input that's the actual culprit. Better ignore it, and emulate showsyntaxerror.
|
||||
# TODO: support sys.excepthook.
|
||||
# TODO: Look at `code.InteractiveConsole.showsyntaxerror` for how to do that.
|
||||
self.write(f"{err.__class__.__name__}: {str(err)}\n")
|
||||
return False # erroneous input
|
||||
except ImportError as err: # during macro lookup in a successfully imported module
|
||||
|
|
@ -139,6 +199,10 @@ class MacroConsole(code.InteractiveConsole):
|
|||
self._macro_bindings_changed = False
|
||||
|
||||
for asname, function in self.expander.bindings.items():
|
||||
# Catch broken bindings due to erroneous imports in user code
|
||||
# (e.g. accidentally to a module object instead of to a function object)
|
||||
if not (hasattr(function, "__module__") and hasattr(function, "__qualname__")):
|
||||
continue
|
||||
if not function.__module__: # Macros defined in the REPL have `__module__=None`.
|
||||
continue
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -22,7 +22,9 @@ line magic.
|
|||
"""
|
||||
|
||||
import ast
|
||||
import copy
|
||||
from functools import partial
|
||||
import sys
|
||||
|
||||
from IPython.core import magic_arguments
|
||||
from IPython.core.error import InputRejected
|
||||
|
|
@ -30,6 +32,7 @@ from IPython.core.magic import Magics, magics_class, cell_magic, line_magic
|
|||
|
||||
from .. import __version__ as mcpyrate_version
|
||||
from ..astdumper import dump
|
||||
from ..compiler import create_module
|
||||
from ..debug import format_bindings
|
||||
from ..expander import find_macros, MacroExpander, global_postprocess
|
||||
from .utils import get_makemacro_sourcecode
|
||||
|
|
@ -38,6 +41,7 @@ from .utils import get_makemacro_sourcecode
|
|||
# Despite the meta-levels, there's just one global importer for the Python process.
|
||||
from .. import activate # noqa: F401
|
||||
|
||||
_magic_module_name = "__mcpyrate_repl_self__"
|
||||
_placeholder = "<ipython-session>"
|
||||
_instance = None
|
||||
|
||||
|
|
@ -85,21 +89,21 @@ class AstMagics(Magics):
|
|||
@line_magic
|
||||
def macros(self, line):
|
||||
"""Print a human-readable list of macros currently imported into the session."""
|
||||
# Line magics print `\n\n` at the end automatically, so remove our final `\n`.
|
||||
print(format_bindings(_instance.macro_transformer.expander).rstrip())
|
||||
# Line magics print an extra `\n` at the end automatically, so remove our final `\n`.
|
||||
print(format_bindings(_instance.macro_transformer.expander, color=True), end="")
|
||||
|
||||
# I don't know if this is useful - one can use the `mcpyrate.debug.step_expansion`
|
||||
# macro also in the REPL - but let's put it in for now.
|
||||
# http://alexleone.blogspot.co.uk/2010/01/python-ast-pretty-printer.html
|
||||
@magic_arguments.magic_arguments()
|
||||
@magic_arguments.argument(
|
||||
'-m', '--mode', default='exec',
|
||||
"-m", "--mode", default="exec",
|
||||
help="The mode in which to parse the code. Can be exec (default), "
|
||||
"eval or single."
|
||||
)
|
||||
# TODO: add support for expand-once
|
||||
@magic_arguments.argument(
|
||||
'-e', '--expand', default='no',
|
||||
"-e", "--expand", default="no",
|
||||
help="Whether to expand macros before dumping the AST. Can be yes "
|
||||
"or no (default)."
|
||||
)
|
||||
|
|
@ -107,7 +111,7 @@ class AstMagics(Magics):
|
|||
def dump_ast(self, line, cell):
|
||||
"""Parse the code in the cell, and pretty-print the AST."""
|
||||
args = magic_arguments.parse_argstring(self.dump_ast, line)
|
||||
tree = ast.parse(cell, mode=args.mode)
|
||||
tree = ast.parse(cell, filename=_placeholder, mode=args.mode)
|
||||
if args.expand != "no":
|
||||
tree = _instance.macro_transformer.visit(tree)
|
||||
print(dump(tree))
|
||||
|
|
@ -119,18 +123,27 @@ class InteractiveMacroTransformer(ast.NodeTransformer):
|
|||
def __init__(self, extension_instance, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._ipyextension = extension_instance
|
||||
self.expander = MacroExpander(bindings={}, filename="<ipython-session>")
|
||||
self.expander = MacroExpander(bindings={}, filename=_placeholder)
|
||||
|
||||
def visit(self, tree):
|
||||
try:
|
||||
bindings = find_macros(tree, filename=self.expander.filename, reload=True) # macro-imports (this will import the modules)
|
||||
sys.modules[_magic_module_name].__dict__.clear()
|
||||
sys.modules[_magic_module_name].__dict__.update(self._ipyextension.shell.user_ns) # for self-macro-imports
|
||||
# We treat the initial magic module metadata as write-protected: even if the user
|
||||
# defines a variable of the same name in the user namespace, the metadata fields
|
||||
# in the magic module won't be overwritten. (IPython actually defines e.g. `__name__`
|
||||
# in `user_ns` even if the user explicitly doesn't.)
|
||||
sys.modules[_magic_module_name].__dict__.update(self._ipyextension.magic_module_metadata)
|
||||
# macro-imports (this will import the modules)
|
||||
bindings = find_macros(tree, filename=self.expander.filename,
|
||||
reload=True, self_module=_magic_module_name)
|
||||
if bindings:
|
||||
self._ipyextension._macro_bindings_changed = True
|
||||
self.expander.bindings.update(bindings)
|
||||
newtree = self.expander.visit(tree)
|
||||
newtree = global_postprocess(newtree)
|
||||
new_tree = self.expander.visit(tree)
|
||||
new_tree = global_postprocess(new_tree)
|
||||
self._ipyextension.src = _placeholder
|
||||
return newtree
|
||||
return new_tree
|
||||
except Exception as err:
|
||||
# see IPython.core.interactiveshell.InteractiveShell.transform_ast()
|
||||
raise InputRejected(*err.args)
|
||||
|
|
@ -149,7 +162,11 @@ class IMcpyrateExtension:
|
|||
self.macro_transformer = InteractiveMacroTransformer(extension_instance=self)
|
||||
self.shell.ast_transformers.append(self.macro_transformer) # TODO: last or first?
|
||||
# Lucky that both meta-levels speak the same language, eh?
|
||||
shell.user_ns['__macro_expander__'] = self.macro_transformer.expander
|
||||
shell.user_ns["__macro_expander__"] = self.macro_transformer.expander
|
||||
|
||||
# support `from __self__ import macros, ...`
|
||||
magic_module = create_module(dotted_name=_magic_module_name, filename=_placeholder)
|
||||
self.magic_module_metadata = copy.copy(magic_module.__dict__)
|
||||
|
||||
self.shell.run_cell(get_makemacro_sourcecode(),
|
||||
store_history=False,
|
||||
|
|
@ -158,15 +175,15 @@ class IMcpyrateExtension:
|
|||
# TODO: If we want to support dialects in the REPL, we need to install
|
||||
# a string transformer here to call the dialect system's source transformer,
|
||||
# and then modify `InteractiveMacroTransformer` to run the dialect system's
|
||||
# AST transformer before it runs the macro expander.
|
||||
# AST transformer before it runs the macro expander. Look at `mcpyrate.compiler.compile`.
|
||||
|
||||
ipy = self.shell.get_ipython()
|
||||
ipy.events.register('post_run_cell', self._refresh_macro_functions)
|
||||
ipy.events.register("post_run_cell", self._refresh_macro_functions)
|
||||
|
||||
def __del__(self):
|
||||
ipy = self.shell.get_ipython()
|
||||
ipy.events.unregister('post_run_cell', self._refresh_macro_functions)
|
||||
del self.shell.user_ns['__macro_expander__']
|
||||
ipy.events.unregister("post_run_cell", self._refresh_macro_functions)
|
||||
del self.shell.user_ns["__macro_expander__"]
|
||||
self.shell.ast_transformers.remove(self.macro_transformer)
|
||||
self.shell.input_transformers_post.remove(self._get_source_code)
|
||||
|
||||
|
|
@ -194,7 +211,11 @@ class IMcpyrateExtension:
|
|||
silent=True)
|
||||
|
||||
for asname, function in self.macro_transformer.expander.bindings.items():
|
||||
if not function.__module__:
|
||||
# Catch broken bindings due to erroneous imports in user code
|
||||
# (e.g. accidentally to a module object instead of to a function object)
|
||||
if not (hasattr(function, "__module__") and hasattr(function, "__qualname__")):
|
||||
continue
|
||||
if not function.__module__: # Macros defined in the REPL have `__module__=None`.
|
||||
continue
|
||||
commands = ["%%ignore_importerror",
|
||||
f"from {function.__module__} import {function.__qualname__} as {asname}"]
|
||||
|
|
|
|||
|
|
@ -6,21 +6,25 @@
|
|||
|
||||
import argparse
|
||||
import atexit
|
||||
from importlib import import_module
|
||||
from importlib.util import resolve_name, module_from_spec
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
from importlib import import_module
|
||||
from importlib.util import module_from_spec, resolve_name
|
||||
|
||||
from ..coreutils import relativize
|
||||
|
||||
from .. import __version__ as __mcpyrate_version__
|
||||
from .. import activate # noqa: F401
|
||||
from ..core import MacroApplicationError
|
||||
from ..coreutils import relativize
|
||||
from ..pycachecleaner import deletepycachedirs, getpycachedirs
|
||||
|
||||
__version__ = "3.0.0"
|
||||
__version__ = "3.1.0"
|
||||
|
||||
_config_dir = "~/.config/mcpyrate"
|
||||
_macropython_module = None # sys.modules doesn't always seem to keep it, so stash it locally too.
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
|
||||
def import_module_as_main(name, script_mode):
|
||||
"""Import a module, pretending it's __main__.
|
||||
|
||||
|
|
@ -98,8 +102,19 @@ def import_module_as_main(name, script_mode):
|
|||
sys.modules["__main__"] = module # replace this bootstrapper with the new __main__
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
except Exception:
|
||||
except Exception as err:
|
||||
sys.modules["__main__"] = _macropython_module
|
||||
if isinstance(err, MacroApplicationError):
|
||||
# To avoid noise, discard most of the traceback of the chained
|
||||
# macro-expansion errors emitted by the expander core. The
|
||||
# linked (`__cause__`) exceptions have the actual tracebacks.
|
||||
#
|
||||
# Keep just the last entry, which should state that this
|
||||
# exception came from `expand` in `core.py`.
|
||||
tb = err.__traceback__
|
||||
while tb.tb_next:
|
||||
tb = tb.tb_next
|
||||
raise err.with_traceback(tb)
|
||||
raise
|
||||
# # __main__ has no parent module so we don't need to do this.
|
||||
# if path is not None:
|
||||
|
|
@ -128,7 +143,7 @@ def main():
|
|||
parser = argparse.ArgumentParser(description="""Run a Python program or an interactive interpreter with mcpyrate enabled.""",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
|
||||
parser.add_argument('-v', '--version', action='version', version=('%(prog)s (mcpyrate ' + __version__ + ')'))
|
||||
parser.add_argument('-v', '--version', action='version', version=('%(prog)s ' + __version__ + ' (mcpyrate ' + __mcpyrate_version__ + ')'))
|
||||
parser.add_argument(dest='filename', nargs='?', default=None, type=str, metavar='file',
|
||||
help='script to run')
|
||||
parser.add_argument('-m', '--module', dest='module', default=None, type=str, metavar='mod',
|
||||
|
|
@ -140,16 +155,37 @@ def main():
|
|||
help='For use together with "-i". Automatically "import numpy as np", '
|
||||
'"import matplotlib.pyplot as plt", and enable mpl\'s interactive '
|
||||
'mode (somewhat like IPython\'s pylab mode).')
|
||||
parser.add_argument('-c', '--clean', dest='path_to_clean', default=None, type=str, metavar='dir',
|
||||
help='Delete Python bytecode (`.pyc`) caches inside given directory, recursively, '
|
||||
'and then exit. This removes the `__pycache__` directories, too. '
|
||||
'The purpose is to facilitate testing `mcpyrate` and programs that use it, '
|
||||
'because the expander only runs when the `.pyc` cache for the module being '
|
||||
'imported is out of date or does not exist. For any given module, once the '
|
||||
'expander is done, Python\'s import system automatically writes the `.pyc` '
|
||||
'cache for that module.')
|
||||
parser.add_argument("-n", "--dry-run", dest="dry_run", action="store_true", default=False,
|
||||
help='For use together with "-c". Just scan for and print the .pyc cache '
|
||||
'directory paths, don\'t actually clean them.')
|
||||
opts = parser.parse_args()
|
||||
|
||||
if opts.path_to_clean:
|
||||
# If an error occurs during cleaning, we just let it produce a standard stack trace.
|
||||
if not opts.dry_run:
|
||||
deletepycachedirs(opts.path_to_clean)
|
||||
else:
|
||||
for x in getpycachedirs(opts.path_to_clean):
|
||||
print(x)
|
||||
sys.exit(0) # only reached if cleaning (or dry run) successful
|
||||
|
||||
if opts.interactive:
|
||||
from .console import MacroConsole
|
||||
import readline # noqa: F401, side effect: enable GNU readline in input()
|
||||
import rlcompleter # noqa: F401, side effects: readline tab completion
|
||||
|
||||
from .console import MacroConsole
|
||||
repl_locals = {}
|
||||
if opts.pylab: # like IPython's pylab mode, but we keep things in separate namespaces.
|
||||
import numpy
|
||||
import matplotlib.pyplot
|
||||
import numpy
|
||||
repl_locals["np"] = numpy
|
||||
repl_locals["plt"] = matplotlib.pyplot
|
||||
matplotlib.pyplot.ion()
|
||||
|
|
@ -206,6 +242,13 @@ def main():
|
|||
if sys.path[0] != "":
|
||||
sys.path.insert(0, "")
|
||||
|
||||
# https://www.python.org/dev/peps/pep-0582/
|
||||
if sys.version_info >= (3, 8, 0):
|
||||
local_pypackages_dir = pathlib.Path.cwd().expanduser().resolve() / "__pypackages__"
|
||||
if local_pypackages_dir.is_dir():
|
||||
# TODO: figure out correct insert index (spec: "after cwd, just before site packages")
|
||||
sys.path.insert(1, str(local_pypackages_dir))
|
||||
|
||||
import_module_as_main(opts.module, script_mode=False)
|
||||
|
||||
else: # no module, so use opts.filename
|
||||
|
|
@ -217,6 +260,13 @@ def main():
|
|||
if sys.path[0] != containing_directory:
|
||||
sys.path.insert(0, containing_directory)
|
||||
|
||||
# https://www.python.org/dev/peps/pep-0582/
|
||||
if sys.version_info >= (3, 8, 0):
|
||||
local_pypackages_dir = fullpath.parent / "__pypackages__"
|
||||
if local_pypackages_dir.is_dir():
|
||||
# TODO: figure out correct insert index (spec: "after cwd, just before site packages")
|
||||
sys.path.insert(1, str(local_pypackages_dir))
|
||||
|
||||
# This approach finds the standard loader even for macro-enabled scripts.
|
||||
# TODO: For mcpyrate, that could be ok, since we monkey-patch just that.
|
||||
# TODO: But maybe better to leave the option to replace the whole loader later.
|
||||
|
|
@ -237,6 +287,6 @@ def main():
|
|||
|
||||
import_module_as_main(module_name, script_mode=True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
_macropython_module = sys.modules["__macropython__"] = sys.modules["__main__"]
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,41 +1,81 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
'''Utilities for building REPLs.'''
|
||||
"""Utilities for building REPLs."""
|
||||
|
||||
__all__ = ["doc", "sourcecode", "get_makemacro_sourcecode"]
|
||||
|
||||
from functools import partial
|
||||
import inspect
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
def doc(obj):
|
||||
"""Print an object's docstring, non-interactively.
|
||||
from ..colorizer import colorize, ColorScheme
|
||||
|
||||
|
||||
def _get_source(obj):
|
||||
# `inspect.getsourcefile` accepts "a module, class, method, function,
|
||||
# traceback, frame, or code object" (the error message says this if
|
||||
# we try it on something invalid).
|
||||
#
|
||||
# So if `obj` is an instance, we need to try again with its `__class__`.
|
||||
for x in (obj, obj.__class__): # TODO: other places to fall back to?
|
||||
try:
|
||||
filename = inspect.getsourcefile(x)
|
||||
source, firstlineno = inspect.getsourcelines(x)
|
||||
return filename, source, firstlineno
|
||||
except (TypeError, OSError):
|
||||
continue
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def doc(obj, *, file=None, end="\n"):
|
||||
"""Print an object's docstring non-interactively.
|
||||
|
||||
If available, print also the filename and the starting line number
|
||||
of the definition of `obj`.
|
||||
|
||||
The default is to print to `sys.stderr` (resolved at call time, to respect
|
||||
possible overrides); to override, use the `file` parameter, which works
|
||||
like in the builtin `print`.
|
||||
|
||||
The `end` parameter is passed to `print` as-is.
|
||||
"""
|
||||
file = file or sys.stderr
|
||||
printer = partial(print, file=file, end=end)
|
||||
try:
|
||||
filename = inspect.getsourcefile(obj)
|
||||
source, firstlineno = inspect.getsourcelines(obj)
|
||||
print(f"{filename}:{firstlineno}")
|
||||
except (TypeError, OSError):
|
||||
filename, source, firstlineno = _get_source(obj)
|
||||
printer(colorize(f"{filename}:{firstlineno}", ColorScheme.SOURCEFILENAME))
|
||||
except NotImplementedError:
|
||||
pass
|
||||
if not hasattr(obj, "__doc__") or not obj.__doc__:
|
||||
print("<no docstring>")
|
||||
printer(colorize("<no docstring>", ColorScheme.GREYEDOUT))
|
||||
return
|
||||
print(inspect.cleandoc(obj.__doc__))
|
||||
print(inspect.cleandoc(obj.__doc__), file=file)
|
||||
|
||||
def sourcecode(obj):
|
||||
"""Print an object's source code, non-interactively.
|
||||
|
||||
def sourcecode(obj, *, file=None, end=None):
|
||||
"""Print an object's source code to `sys.stderr`, non-interactively.
|
||||
|
||||
If available, print also the filename and the starting line number
|
||||
of the definition of `obj`.
|
||||
|
||||
Default is to print to `sys.stderr` (resolved at call time, to respect
|
||||
possible overrides); to override, use the `file` argument, which works
|
||||
like in the builtin `print`.
|
||||
|
||||
The `end` parameter is passed to `print` as-is.
|
||||
"""
|
||||
file = file or sys.stderr
|
||||
printer = partial(print, file=file, end=end)
|
||||
try:
|
||||
filename = inspect.getsourcefile(obj)
|
||||
source, firstlineno = inspect.getsourcelines(obj)
|
||||
print(f"{filename}:{firstlineno}")
|
||||
filename, source, firstlineno = _get_source(obj)
|
||||
printer(colorize(f"{filename}:{firstlineno}", ColorScheme.SOURCEFILENAME))
|
||||
# TODO: No syntax highlighting for now, because we'd have to parse and unparse,
|
||||
# TODO: which loses the original formatting and comments.
|
||||
for line in source:
|
||||
print(line.rstrip("\n"))
|
||||
except (TypeError, OSError):
|
||||
print("<no source code available>")
|
||||
printer(line.rstrip("\n"))
|
||||
except NotImplementedError:
|
||||
printer(colorize("<no source code available>", ColorScheme.GREYEDOUT))
|
||||
|
||||
|
||||
def get_makemacro_sourcecode():
|
||||
"""Return source code for the REPL's `macro` magic function.
|
||||
|
|
@ -45,13 +85,13 @@ def get_makemacro_sourcecode():
|
|||
We assume the expander instance has been bound to the global variable
|
||||
`__macro_expander__` inside the REPL session.
|
||||
"""
|
||||
return """
|
||||
return textwrap.dedent('''
|
||||
def macro(function):
|
||||
'''[mcpyrate] `macro(f)`: bind function `f` as a macro. Works also as a decorator. REPL only.'''
|
||||
"""[mcpyrate] `macro(f)`: bind function `f` as a macro. Works also as a decorator. REPL only."""
|
||||
if not callable(function):
|
||||
raise TypeError(f'`function` must be callable, got {type(function)} with value {repr(function)}')
|
||||
if function.__name__ == '<lambda>':
|
||||
raise TypeError('`function` must be a named function, got a lambda.')
|
||||
raise TypeError(f"`function` must be callable, got {type(function)} with value {repr(function)}")
|
||||
if function.__name__ == "<lambda>":
|
||||
raise TypeError("`function` must be a named function, got a lambda.")
|
||||
__macro_expander__.bindings[function.__name__] = function
|
||||
return function
|
||||
"""
|
||||
''')
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ def splice_expression(expr, template, tag="__paste_here__"):
|
|||
|
||||
Returns `template` with `expr` spliced in. Note `template` is **not** copied,
|
||||
and will be mutated in-place.
|
||||
|
||||
"""
|
||||
if not template:
|
||||
return expr
|
||||
|
|
@ -58,7 +57,7 @@ def splice_expression(expr, template, tag="__paste_here__"):
|
|||
raise TypeError(f"`template` must be an AST or `list`; got {type(template)} with value {repr(template)}")
|
||||
|
||||
def ispastehere(tree):
|
||||
return type(tree) is ast.Expr and type(tree.value) is ast.Name and tree.value.id == tag
|
||||
return type(tree) is ast.Name and tree.id == tag
|
||||
|
||||
class ExpressionSplicer(ASTTransformer):
|
||||
def __init__(self):
|
||||
|
|
|
|||
|
|
@ -1,17 +1,22 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
"""Back-convert a Python AST into source code. Original formatting is disregarded."""
|
||||
"""Back-convert a Python AST into source code. Original formatting is disregarded.
|
||||
|
||||
Python 3.9+ provides `ast.unparse`, but ours comes with some additional features,
|
||||
notably syntax highlighting and debug rendering of invisible AST nodes.
|
||||
"""
|
||||
|
||||
__all__ = ["UnparserError", "unparse", "unparse_with_fallbacks"]
|
||||
|
||||
import ast
|
||||
import builtins
|
||||
from contextlib import contextmanager
|
||||
import io
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .astdumper import dump # fallback
|
||||
from .colorizer import setcolor, colorize, ColorScheme
|
||||
from . import markers
|
||||
from .astdumper import dump # fallback
|
||||
from .colorizer import ColorScheme, colorize, setcolor
|
||||
quotes = None # HACK: avoid circular import
|
||||
|
||||
# Large float and imaginary literals get turned into infinities in the AST.
|
||||
# We unparse those infinities to INFSTR.
|
||||
|
|
@ -78,6 +83,10 @@ class Unparser:
|
|||
`expander`: optional `BaseMacroExpander` instance. If provided,
|
||||
used for syntax highlighting macro names.
|
||||
"""
|
||||
# HACK: avoid circular import
|
||||
global quotes
|
||||
from . import quotes
|
||||
|
||||
self.debug = debug
|
||||
self.color = color
|
||||
self._color_override = False # for syntax highlighting of decorators
|
||||
|
|
@ -162,6 +171,12 @@ class Unparser:
|
|||
for t in tree:
|
||||
self.dispatch(t)
|
||||
return
|
||||
if self.debug and quotes.is_captured_value(tree):
|
||||
self.captured_value(tree)
|
||||
return
|
||||
if self.debug and quotes.is_captured_macro(tree):
|
||||
self.captured_macro(tree)
|
||||
return
|
||||
if isinstance(tree, markers.ASTMarker): # mcpyrate and macro communication internal
|
||||
self.astmarker(tree)
|
||||
return
|
||||
|
|
@ -184,7 +199,10 @@ class Unparser:
|
|||
|
||||
def astmarker(self, tree):
|
||||
def write_astmarker_field_value(v):
|
||||
if isinstance(v, ast.AST):
|
||||
if isinstance(v, list): # statement suite
|
||||
for item in v:
|
||||
self.dispatch(item)
|
||||
elif isinstance(v, ast.AST):
|
||||
self.dispatch(v)
|
||||
else:
|
||||
self.write(repr(v))
|
||||
|
|
@ -194,8 +212,8 @@ class Unparser:
|
|||
# that "source code" containing AST markers cannot be eval'd.
|
||||
# If you need to get rid of them, see `mcpyrate.markers.delete_markers`.
|
||||
|
||||
header = self.maybe_colorize(f"$ASTMarker", ColorScheme.ASTMARKER)
|
||||
if isinstance(tree.body, ast.stmt):
|
||||
header = self.maybe_colorize("$ASTMarker", ColorScheme.ASTMARKER)
|
||||
if isinstance(tree.body, (ast.stmt, list)):
|
||||
print_mode = "stmt"
|
||||
self.fill(header, lineno_node=tree)
|
||||
else:
|
||||
|
|
@ -242,6 +260,20 @@ class Unparser:
|
|||
if print_mode == "expr":
|
||||
self.write(")")
|
||||
|
||||
def captured_value(self, t): # hygienic capture; output of `mcpyrate.quotes.h`; only emitted in debug mode
|
||||
name, _ignored_value = quotes.is_captured_value(t)
|
||||
self.write(self.maybe_colorize("$h", ColorScheme.ASTMARKER))
|
||||
self.write(self.maybe_colorize("[", ColorScheme.ASTMARKER))
|
||||
self.write(name)
|
||||
self.write(self.maybe_colorize("]", ColorScheme.ASTMARKER))
|
||||
|
||||
def captured_macro(self, t): # hygienic capture; output of `mcpyrate.quotes.h`; only emitted in debug mode
|
||||
name, _ignored_unique_name, _ignored_value = quotes.is_captured_macro(t)
|
||||
self.write(self.maybe_colorize("$h", ColorScheme.ASTMARKER))
|
||||
self.write(self.maybe_colorize("[", ColorScheme.ASTMARKER))
|
||||
self.write(self.maybe_colorize(name, ColorScheme.MACRONAME))
|
||||
self.write(self.maybe_colorize("]", ColorScheme.ASTMARKER))
|
||||
|
||||
# top level nodes
|
||||
def _Module(self, t): # ast.parse(..., mode="exec")
|
||||
self.toplevelnode(t)
|
||||
|
|
@ -584,6 +616,8 @@ class Unparser:
|
|||
v = self.maybe_colorize(v, ColorScheme.NAMECONSTANT)
|
||||
elif type(t.value) in (str, bytes):
|
||||
v = self.maybe_colorize(v, ColorScheme.STRING)
|
||||
else: # pragma: no cover
|
||||
raise UnparserError(f"Don't know how to unparse Constant with value of type {type(t.value)}, got {repr(t.value)}")
|
||||
self.write(v)
|
||||
|
||||
def _Bytes(self, t): # up to Python 3.7
|
||||
|
|
@ -681,25 +715,33 @@ class Unparser:
|
|||
interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values))
|
||||
self.write("}")
|
||||
|
||||
# Python 3.9+: we must emit the parentheses separate from the main logic,
|
||||
# because a Tuple directly inside a Subscript slice should be rendered
|
||||
# without parentheses; so our `_Subscript` method special-cases that.
|
||||
# This is important, because the notation `a[1,2:5]` is fine, but
|
||||
# `a[(1,2:5)]` is a syntax error. See https://bugs.python.org/issue34822
|
||||
def _Tuple(self, t):
|
||||
self.write("(")
|
||||
self.__Tuple_helper(t)
|
||||
self.write(")")
|
||||
|
||||
def __Tuple_helper(self, t):
|
||||
if len(t.elts) == 1:
|
||||
(elt,) = t.elts
|
||||
self.dispatch(elt)
|
||||
self.write(",")
|
||||
else:
|
||||
interleave(lambda: self.write(", "), self.dispatch, t.elts)
|
||||
self.write(")")
|
||||
|
||||
unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"}
|
||||
def _UnaryOp(self, t):
|
||||
self.write("(")
|
||||
# if it's an English keyword, highlight it.
|
||||
# If it's an English keyword, highlight it, and add a space.
|
||||
if t.op.__class__.__name__ == "Not":
|
||||
self.write(self.maybe_colorize_python_keyword(self.unop[t.op.__class__.__name__]))
|
||||
self.write(" ")
|
||||
else:
|
||||
self.write(self.unop[t.op.__class__.__name__])
|
||||
self.write(" ")
|
||||
self.dispatch(t.operand)
|
||||
self.write(")")
|
||||
|
||||
|
|
@ -815,7 +857,12 @@ class Unparser:
|
|||
def _Subscript(self, t):
|
||||
self.dispatch(t.value)
|
||||
self.write("[")
|
||||
self.dispatch(t.slice)
|
||||
# Python 3.9+: Omit parentheses for a tuple directly inside a Subscript slice.
|
||||
# See https://bugs.python.org/issue34822
|
||||
if type(t.slice) is ast.Tuple:
|
||||
self.__Tuple_helper(t.slice)
|
||||
else:
|
||||
self.dispatch(t.slice)
|
||||
self.write("]")
|
||||
|
||||
def _Starred(self, t):
|
||||
|
|
@ -826,7 +873,7 @@ class Unparser:
|
|||
def _Ellipsis(self, t): # up to Python 3.7
|
||||
self.write("...")
|
||||
|
||||
def _Index(self, t):
|
||||
def _Index(self, t): # up to Python 3.8; the Index wrapper is gone in Python 3.9
|
||||
self.dispatch(t.value)
|
||||
|
||||
def _Slice(self, t):
|
||||
|
|
@ -839,7 +886,7 @@ class Unparser:
|
|||
self.write(":")
|
||||
self.dispatch(t.step)
|
||||
|
||||
def _ExtSlice(self, t):
|
||||
def _ExtSlice(self, t): # up to Python 3.8; Python 3.9 uses a Tuple instead
|
||||
interleave(lambda: self.write(", "), self.dispatch, t.dims)
|
||||
|
||||
# argument
|
||||
|
|
|
|||
|
|
@ -216,6 +216,10 @@ def format_location(filename, tree, sourcecode):
|
|||
|
||||
def format_macrofunction(function):
|
||||
"""Format the fully qualified name of a macro function, for error messages."""
|
||||
# Catch broken bindings due to erroneous imports in user code
|
||||
# (e.g. accidentally to a module object instead of to a function object)
|
||||
if not (hasattr(function, "__module__") and hasattr(function, "__qualname__")):
|
||||
return repr(function)
|
||||
if not function.__module__: # Macros defined in the REPL have `__module__=None`.
|
||||
return function.__qualname__
|
||||
return f"{function.__module__}.{function.__qualname__}"
|
||||
|
|
|
|||
|
|
@ -1,9 +1,42 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
"""AST walkers.
|
||||
|
||||
These have a state stack and a node collector, and accept also a `list` of statements.
|
||||
Otherwise they work like `ast.NodeVisitor` and `ast.NodeTransformer`.
|
||||
|
||||
Basic usage summary::
|
||||
|
||||
def kittify(mytree):
|
||||
class Kittifier(ASTTransformer):
|
||||
def transform(self, tree):
|
||||
if type(tree) is ast.Constant:
|
||||
self.collect(tree.value)
|
||||
tree.value = "meow!" if self.state.meows % 2 == 0 else "miaow!"
|
||||
self.state.meows += 1
|
||||
return self.generic_visit(tree) # recurse
|
||||
w = Kittifier(meows=0) # set the initial state here
|
||||
mytree = w.visit(mytree) # it's basically an ast.NodeTransformer
|
||||
print(w.collected) # collected values, in the order visited
|
||||
return mytree
|
||||
|
||||
def getmeows(mytree):
|
||||
class MeowCollector(ASTVisitor):
|
||||
def examine(self, tree):
|
||||
if type(tree) is ast.Constant and tree.value in ("meow!", "miaow!"):
|
||||
self.collect(tree)
|
||||
self.generic_visit(tree)
|
||||
w = MeowCollector()
|
||||
w.visit(mytree)
|
||||
print(w.collected)
|
||||
return w.collected
|
||||
|
||||
For an example of `withstate`, see `mcpyrate.astfixers`.
|
||||
"""
|
||||
|
||||
__all__ = ["ASTVisitor", "ASTTransformer"]
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from ast import NodeVisitor, NodeTransformer
|
||||
from ast import NodeVisitor, NodeTransformer, iter_child_nodes
|
||||
|
||||
from .bunch import Bunch
|
||||
from . import utils
|
||||
|
|
@ -34,6 +67,19 @@ class BaseASTWalker:
|
|||
`tree` can be an AST node or a statement suite (`list` of AST nodes).
|
||||
It is identified by `id(tree)` at enter time. Bindings update a copy
|
||||
of `self.state`.
|
||||
|
||||
If several `withstate` calls are made for the same `tree`, the last one
|
||||
overrides.
|
||||
|
||||
Generally speaking:
|
||||
|
||||
`withstate(subtree, ...)` should be used if you then intend to
|
||||
`visit(subtree)`, which recurses into that node (or suite) only.
|
||||
|
||||
`generic_withstate(tree, ...)` should be used if you then intend to
|
||||
`generic_visit(tree)`, which recurses into the children of `tree`.
|
||||
|
||||
(It is possible to mix and match, but think through what you're doing.)
|
||||
"""
|
||||
newstate = self.state.copy()
|
||||
newstate.update(**bindings)
|
||||
|
|
@ -47,6 +93,33 @@ class BaseASTWalker:
|
|||
else:
|
||||
self._subtree_overrides[id(tree)] = newstate
|
||||
|
||||
def generic_withstate(self, tree, **bindings):
|
||||
"""Like `withstate`, but set up the new state for all children of `tree`.
|
||||
|
||||
The same state instance is shared between the child nodes.
|
||||
|
||||
If several `generic_withstate` calls are made for the same `tree`, the
|
||||
last one overrides (assuming the list of children has not changed in between).
|
||||
|
||||
The silly name is because this relates to `withstate` as `generic_visit`
|
||||
relates to `visit`.
|
||||
|
||||
Generally speaking:
|
||||
|
||||
`generic_withstate(tree, ...)` should be used if you then intend to
|
||||
`generic_visit(tree)`, which recurses into the children of `tree`.
|
||||
|
||||
`withstate(subtree, ...)` should be used if you then intend to
|
||||
`visit(subtree)`, which recurses into that node (or suite) only.
|
||||
|
||||
(It is possible to mix and match, but think through what you're doing.)
|
||||
"""
|
||||
newstate = self.state.copy()
|
||||
newstate.update(**bindings)
|
||||
|
||||
for node in iter_child_nodes(tree):
|
||||
self._subtree_overrides[id(node)] = newstate
|
||||
|
||||
def collect(self, value):
|
||||
"""Collect a value. The values are placed in the list `self.collected`."""
|
||||
self.collected.append(value)
|
||||
|
|
@ -111,10 +184,10 @@ class ASTTransformer(BaseASTWalker, NodeTransformer, metaclass=ABCMeta):
|
|||
try:
|
||||
if isinstance(tree, list):
|
||||
new_tree = utils.flatten(self.visit(elt) for elt in tree)
|
||||
if new_tree:
|
||||
tree[:] = new_tree
|
||||
return tree
|
||||
return None
|
||||
if not new_tree:
|
||||
new_tree = [] # preserve the type of `tree`; an empty list shouldn't turn into `None`
|
||||
tree[:] = new_tree
|
||||
return tree
|
||||
return self.transform(tree)
|
||||
finally:
|
||||
if newstate:
|
||||
|
|
|
|||
Loading…
Reference in New Issue