From 93ee8c3acb8c02ffd0f253711683a36d1eb248ce Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Thu, 6 Jan 2022 12:10:06 -0300 Subject: [PATCH] Updated mcpyrate to 3.5.4 (last available) --- kibot/kicad/config.py | 2 +- kibot/mcpyrate/__init__.py | 2 +- kibot/mcpyrate/ansi.py | 117 +++++------ kibot/mcpyrate/astfixers.py | 15 +- kibot/mcpyrate/colorizer.py | 31 ++- kibot/mcpyrate/compiler.py | 58 +++++- kibot/mcpyrate/core.py | 140 +++++++++++-- kibot/mcpyrate/coreutils.py | 72 ++++++- kibot/mcpyrate/debug.py | 106 ++++++---- kibot/mcpyrate/dialects.py | 53 +++-- kibot/mcpyrate/expander.py | 104 ++++++++-- kibot/mcpyrate/importer.py | 4 +- kibot/mcpyrate/markers.py | 11 +- kibot/mcpyrate/metatools.py | 6 +- kibot/mcpyrate/multiphase.py | 82 ++++++-- kibot/mcpyrate/pycachecleaner.py | 57 ++++++ kibot/mcpyrate/quotes.py | 305 +++++++++++++++++++++++++++-- kibot/mcpyrate/repl/console.py | 78 +++++++- kibot/mcpyrate/repl/iconsole.py | 53 +++-- kibot/mcpyrate/repl/macropython.py | 70 ++++++- kibot/mcpyrate/repl/utils.py | 86 +++++--- kibot/mcpyrate/splicing.py | 3 +- kibot/mcpyrate/unparser.py | 73 +++++-- kibot/mcpyrate/utils.py | 4 + kibot/mcpyrate/walkers.py | 83 +++++++- 25 files changed, 1306 insertions(+), 309 deletions(-) create mode 100644 kibot/mcpyrate/pycachecleaner.py diff --git a/kibot/kicad/config.py b/kibot/kicad/config.py index 70f1c791..2d5f9cfe 100644 --- a/kibot/kicad/config.py +++ b/kibot/kicad/config.py @@ -207,7 +207,7 @@ class KiConf(object): """ Environment vars from KiCad 6 configuration """ with open(cfg, 'rt') as f: data = json.load(f) - if "environment" in data and 'vars' in data['environment'] and (data['environment']['vars'] != None): + if "environment" in data and 'vars' in data['environment'] and (data['environment']['vars'] is not None): for k, v in data['environment']['vars'].items(): if GS.debug_level > 1: logger.debug('- KiCad var: {}="{}"'.format(k, v)) diff --git a/kibot/mcpyrate/__init__.py b/kibot/mcpyrate/__init__.py index 428b4ab7..f677b11d 100644 --- a/kibot/mcpyrate/__init__.py +++ b/kibot/mcpyrate/__init__.py @@ -5,4 +5,4 @@ from .expander import namemacro, parametricmacro # noqa: F401 from .unparser import unparse # noqa: F401 from .utils import gensym # noqa: F401 -__version__ = "3.1.0" +__version__ = "3.5.4" diff --git a/kibot/mcpyrate/ansi.py b/kibot/mcpyrate/ansi.py index 1ae9f911..e97ba2fa 100644 --- a/kibot/mcpyrate/ansi.py +++ b/kibot/mcpyrate/ansi.py @@ -11,21 +11,17 @@ OSC = '\033]' BEL = '\a' is_a_tty = sys.stderr.isatty() and os.name == 'posix' - def code_to_chars(code): return CSI + str(code) + 'm' if is_a_tty else '' - def set_title(title): return OSC + '2;' + title + BEL +def clear_screen(mode=2): + return CSI + str(mode) + 'J' -# def clear_screen(mode=2): -# return CSI + str(mode) + 'J' - - -# def clear_line(mode=2): -# return CSI + str(mode) + 'K' +def clear_line(mode=2): + return CSI + str(mode) + 'K' class AnsiCodes(object): @@ -39,75 +35,70 @@ class AnsiCodes(object): setattr(self, name, code_to_chars(value)) -# class AnsiCursor(object): -# def UP(self, n=1): -# return CSI + str(n) + 'A' -# -# def DOWN(self, n=1): -# return CSI + str(n) + 'B' -# -# def FORWARD(self, n=1): -# return CSI + str(n) + 'C' -# -# def BACK(self, n=1): -# return CSI + str(n) + 'D' -# -# def POS(self, x=1, y=1): -# return CSI + str(y) + ';' + str(x) + 'H' +class AnsiCursor(object): + def UP(self, n=1): + return CSI + str(n) + 'A' + def DOWN(self, n=1): + return CSI + str(n) + 'B' + def FORWARD(self, n=1): + return CSI + str(n) + 'C' + def BACK(self, n=1): + return CSI + str(n) + 'D' + def POS(self, x=1, y=1): + return CSI + str(y) + ';' + str(x) + 'H' class AnsiFore(AnsiCodes): - BLACK = 30 - RED = 31 - GREEN = 32 - YELLOW = 33 - BLUE = 34 - MAGENTA = 35 - CYAN = 36 - WHITE = 37 - RESET = 39 + BLACK = 30 + RED = 31 + GREEN = 32 + YELLOW = 33 + BLUE = 34 + MAGENTA = 35 + CYAN = 36 + WHITE = 37 + RESET = 39 # These are fairly well supported, but not part of the standard. - LIGHTBLACK_EX = 90 - LIGHTRED_EX = 91 - LIGHTGREEN_EX = 92 - LIGHTYELLOW_EX = 93 - LIGHTBLUE_EX = 94 + LIGHTBLACK_EX = 90 + LIGHTRED_EX = 91 + LIGHTGREEN_EX = 92 + LIGHTYELLOW_EX = 93 + LIGHTBLUE_EX = 94 LIGHTMAGENTA_EX = 95 - LIGHTCYAN_EX = 96 - LIGHTWHITE_EX = 97 + LIGHTCYAN_EX = 96 + LIGHTWHITE_EX = 97 class AnsiBack(AnsiCodes): - BLACK = 40 - RED = 41 - GREEN = 42 - YELLOW = 43 - BLUE = 44 - MAGENTA = 45 - CYAN = 46 - WHITE = 47 - RESET = 49 + BLACK = 40 + RED = 41 + GREEN = 42 + YELLOW = 43 + BLUE = 44 + MAGENTA = 45 + CYAN = 46 + WHITE = 47 + RESET = 49 # These are fairly well supported, but not part of the standard. - LIGHTBLACK_EX = 100 - LIGHTRED_EX = 101 - LIGHTGREEN_EX = 102 - LIGHTYELLOW_EX = 103 - LIGHTBLUE_EX = 104 + LIGHTBLACK_EX = 100 + LIGHTRED_EX = 101 + LIGHTGREEN_EX = 102 + LIGHTYELLOW_EX = 103 + LIGHTBLUE_EX = 104 LIGHTMAGENTA_EX = 105 - LIGHTCYAN_EX = 106 - LIGHTWHITE_EX = 107 + LIGHTCYAN_EX = 106 + LIGHTWHITE_EX = 107 class AnsiStyle(AnsiCodes): - BRIGHT = 1 - DIM = 2 - NORMAL = 22 + BRIGHT = 1 + DIM = 2 + NORMAL = 22 RESET_ALL = 0 - -Fore = AnsiFore() -Back = AnsiBack() -Style = AnsiStyle() -# Cursor = AnsiCursor() +Fore = AnsiFore() +Back = AnsiBack() +Style = AnsiStyle() +Cursor = AnsiCursor() diff --git a/kibot/mcpyrate/astfixers.py b/kibot/mcpyrate/astfixers.py index ca224d21..439ca59f 100644 --- a/kibot/mcpyrate/astfixers.py +++ b/kibot/mcpyrate/astfixers.py @@ -3,15 +3,11 @@ __all__ = ["fix_ctx", "fix_locations"] -from ast import (Load, Store, Del, - Assign, AnnAssign, AugAssign, - Attribute, Subscript, - comprehension, - For, AsyncFor, - withitem, - Delete, - iter_child_nodes) +from ast import (AnnAssign, Assign, AsyncFor, Attribute, AugAssign, Del, + Delete, For, Load, Store, Subscript, comprehension, + iter_child_nodes, withitem) from copy import copy +from typing import Type from . import walkers @@ -20,7 +16,7 @@ try: # Python 3.8+ except ImportError: class _NoSuchNodeType: pass - NamedExpr = _NoSuchNodeType + NamedExpr: Type = _NoSuchNodeType # type: ignore[no-redef] class _CtxFixer(walkers.ASTTransformer): @@ -71,6 +67,7 @@ class _CtxFixer(walkers.ASTTransformer): # `AugStore` and `AugLoad` are for internal use only, not even # meant to be exposed to the user; the compiler expects `Store` # and `Load` here. https://bugs.python.org/issue39988 + # Those internal classes are indeed deprecated in Python 3.9. self.withstate(tree.target, ctxclass=Store) self.withstate(tree.value, ctxclass=Load) diff --git a/kibot/mcpyrate/colorizer.py b/kibot/mcpyrate/colorizer.py index 7005a8c4..ec0f517b 100644 --- a/kibot/mcpyrate/colorizer.py +++ b/kibot/mcpyrate/colorizer.py @@ -10,7 +10,8 @@ __all__ = ["setcolor", "colorize", "ColorScheme", "Fore", "Back", "Style"] try: - from colorama import init as colorama_init, Fore, Back, Style + from colorama import Back, Fore, Style # type: ignore[import] + from colorama import init as colorama_init # type: ignore[import] colorama_init() except ImportError: # pragma: no cover # The `ansi` module is a slightly modified, POSIX-only, @@ -18,6 +19,17 @@ except ImportError: # pragma: no cover # images that don't have the library available. from .ansi import Fore, Back, Style # noqa: F811 +# TODO: Get rid of this hack if Colorama adds these styles later. +# Inject some styles missing from Colorama 0.4.4 +_additional_styles = (("ITALIC", "\33[3m"), + ("URL", "\33[4m"), # underline plus possibly a special color (depends on terminal app) + ("BLINK", "\33[5m"), + ("BLINK2", "\33[6m")) # same effect as BLINK? +for _name, _value in _additional_styles: + if not hasattr(Style, _name): + setattr(Style, _name, _value) +del _name, _value + from .bunch import Bunch @@ -140,17 +152,15 @@ class ColorScheme(Bunch): # ------------------------------------------------------------ # format_bindings, step_expansion, StepExpansion - self.HEADING = (Style.BRIGHT, Fore.LIGHTBLUE_EX) + self.HEADING1 = (Style.BRIGHT, Fore.LIGHTBLUE_EX) # main heading + self.HEADING2 = Fore.LIGHTBLUE_EX # subheading (filenames, tree ids, ...) self.SOURCEFILENAME = Style.BRIGHT # format_bindings self.GREYEDOUT = Style.DIM # if no bindings - # step_expansion - self.TREEID = Fore.LIGHTBLUE_EX - # StepExpansion - self.ATTENTION = (Style.BRIGHT, Fore.GREEN) # "DialectExpander debug mode" + self.ATTENTION = (Style.BRIGHT, Fore.GREEN) # "DialectExpander debug mode", "PHASE 0" self.TRANSFORMERKIND = (Style.BRIGHT, Fore.GREEN) # "source", "AST" self.DIALECTTRANSFORMERNAME = (Style.BRIGHT, Fore.YELLOW) @@ -160,4 +170,11 @@ class ColorScheme(Bunch): self.NODETYPE = (Style.BRIGHT, Fore.LIGHTBLUE_EX) self.FIELDNAME = Fore.YELLOW self.BAREVALUE = Fore.GREEN -ColorScheme = ColorScheme() + + # ------------------------------------------------------------ + # runtests + self.TESTHEADING = self.HEADING1 + self.TESTPASS = (Style.BRIGHT, Fore.GREEN) + self.TESTFAIL = (Style.BRIGHT, Fore.RED) + self.TESTERROR = (Style.BRIGHT, Fore.YELLOW) +ColorScheme = ColorScheme() # type: ignore[assignment, misc] diff --git a/kibot/mcpyrate/compiler.py b/kibot/mcpyrate/compiler.py index 5ad6e909..47b027e6 100644 --- a/kibot/mcpyrate/compiler.py +++ b/kibot/mcpyrate/compiler.py @@ -12,18 +12,20 @@ We also provide a public API to compile and run macro-enabled code at run time. __all__ = ["expand", "compile", "singlephase_expand", - "run", "create_module"] + "run", "create_module", + "temporary_module"] import ast import builtins +from contextlib import contextmanager import importlib.util import sys -from types import ModuleType, CodeType +from types import CodeType, ModuleType -from .dialects import DialectExpander -from .expander import find_macros, expand_macros -from .markers import check_no_markers_remaining from . import multiphase +from .dialects import DialectExpander +from .expander import expand_macros, find_macros +from .markers import check_no_markers_remaining from .unparser import unparse from .utils import gensym, getdocstring @@ -48,9 +50,8 @@ def expand(source, filename, optimize=-1, self_module=None): `filename`: Full path to the `.py` file being compiled. - `optimize`: Passed to Python's built-in `compile` function, as well as to - the multi-phase compiler. The multi-phase compiler uses the - `optimize` setting for the temporary higher-phase modules. + `optimize`: Passed to the multi-phase compiler. It uses the `optimize` + setting for the temporary higher-phase modules. `self_module`: Absolute dotted module name of the module being compiled. Needed for modules that request multi-phase compilation. @@ -193,6 +194,8 @@ def compile(source, filename, optimize=-1, self_module=None): The main reason for its existence is to provide a near drop-in replacement for the built-in `compile` for macro-enabled input. + The `optimize` parameter is passed to Python's built-in `compile` function. + Currently the API differs from the built-in `compile` in that: - `mode` is always `"exec"`, @@ -542,3 +545,42 @@ def create_module(dotted_name=None, filename=None, *, update_parent=True): sys.modules[dotted_name] = module return module + + +@contextmanager +def temporary_module(dotted_name=None, filename=None, *, update_parent=True): + """Context manager. Create and destroy a temporary module. + + Usage:: + + with temporary_module(name, filename) as module: + ... + + Arguments are passed to `create_module`. The created module is + automatically inserted to `sys.modules`, and then assigned to + the as-part. + + When the context exits, the temporary module is removed from + `sys.modules`. If the `update_parent` optional kwarg is `True`, + it is also removed from the parent package's namespace, for + symmetry. + + This is useful for sandboxed testing of macros, because the + temporary modules will not pollute `sys.modules` beyond their + useful lifetime. + """ + try: + module = create_module(dotted_name, filename, update_parent=update_parent) + yield module + finally: + try: + del sys.modules[dotted_name] + + # The standard importer doesn't do this, but it seems nice for symmetry, + # considering the use case. + if update_parent and dotted_name.find(".") != -1: + packagename, finalcomponent = dotted_name.rsplit(".", maxsplit=1) + package = sys.modules.get(packagename, None) + delattr(package, finalcomponent) + except BaseException: + pass diff --git a/kibot/mcpyrate/core.py b/kibot/mcpyrate/core.py index aebe081a..ceaf1171 100644 --- a/kibot/mcpyrate/core.py +++ b/kibot/mcpyrate/core.py @@ -2,11 +2,14 @@ """Expander core; essentially, how to apply a macro invocation.""" __all__ = ["MacroExpansionError", "MacroExpanderMarker", "Done", - "BaseMacroExpander", "global_postprocess"] + "BaseMacroExpander", "global_postprocess", + "add_postprocessor", "remove_postprocessor"] -from ast import NodeTransformer, AST -from contextlib import contextmanager +from ast import AST, NodeTransformer from collections import ChainMap +from contextlib import contextmanager +from copy import deepcopy +from typing import Any, Callable, Dict, List from .astfixers import fix_ctx, fix_locations from .markers import ASTMarker, delete_markers @@ -14,7 +17,14 @@ from .utils import flatten, format_location # Global macro bindings shared across all expanders in the current process. # This is used by `mcpyrate.quotes` for hygienically captured macro functions. -global_bindings = {} +# +# TODO: In reality, a macro function is not just any callable, but it takes certain +# (especially named) arguments, and its return type is `Union[AST, List[AST], None]`, +# but as of 3.1.0, we skim over this detail. +global_bindings: Dict[str, Callable[..., Any]] = {} + +# User function hook for `global_postprocess`. +global_postprocessors: List[Callable[..., Any]] = [] class MacroExpansionError(Exception): """Base class for errors specific to macro expansion. @@ -83,11 +93,12 @@ class BaseMacroExpander(NodeTransformer): filename: full path to `.py` file being expanded, for error reporting """ - def __init__(self, bindings, filename): + def __init__(self, bindings: Dict[str, Callable[..., Any]], filename: str): self.local_bindings = bindings self.bindings = ChainMap(self.local_bindings, global_bindings) self.filename = filename self.recursive = True + self._debughook = None # see `mcpyrate.debug.step_expansion` def visit(self, tree): """Expand macros in `tree`, using current setting for recursive mode. @@ -137,7 +148,7 @@ class BaseMacroExpander(NodeTransformer): with self._recursive_mode(False): return Done(self.visit(tree)) - def _recursive_mode(self, isrecursive): + def _recursive_mode(self, isrecursive: bool): """Context manager. Change recursive mode, restoring the old mode when the context exits.""" @contextmanager def recursive_mode(): @@ -149,6 +160,30 @@ class BaseMacroExpander(NodeTransformer): self.recursive = wasrecursive return recursive_mode() + def debughook(self, hook: Callable[..., None]): + """Context manager. Temporarily set a debug hook, restoring the old one when the context exits. + + The debug hook, if one is installed, is called whenever a macro expands. + + The hook receives the following arguments, passed positionally in this order: + invocationsubtreeid: int, the `id()` of the *original* macro invocation subtree + before anything was done to it. + invocationtree: AST, (a deepcopy of) the macro invocation subtree before expansion. + expandedtree: AST, the replacement subtree after expanding once. It's the actual live copy. + macroname: str, name of the macro that was applied, as it appears in this expander's + bindings. + macrofunction: callable, the macro function that was applied. + """ + @contextmanager + def debughook_context(): + oldhook = self._debughook + try: + self._debughook = hook + yield + finally: + self._debughook = oldhook + return debughook_context() + def expand(self, syntax, target, macroname, tree, sourcecode, kw=None): """Expand a macro invocation. @@ -180,7 +215,21 @@ class BaseMacroExpander(NodeTransformer): Very rarely needed; if you need it, you'll know. - **CAUTION**: not a copy, or at most a shallow copy. + **CAUTION**: Not a copy, or at most a shallow copy. + + **CAUTION**: The caller, i.e. the concrete expander's + visit method that corresponds to the macro + invocation `syntax` being processed, + is responsible for providing a complete, + unmodified `invocation`. + + So for example, if your concrete expander + needs to pop a block macro, be sure to stash + an appropriately prepared copy of the invocation + (so that it looks like the original one) + before you edit the original. This is used for + debug purposes, so preserving the appearance + of the original invocation is important. To send additional named arguments from the actual expander to the macro function, place them in a dictionary and pass that dictionary @@ -200,7 +249,7 @@ class BaseMacroExpander(NodeTransformer): raise MacroApplicationError(f"{loc}\nin {syntax} macro invocation for '{macroname}': the name '{macroname}' is not bound to a macro.") # Expand the macro. - expansion = _apply_macro(macro, tree, kw) + expansion = self._apply_macro(macro, tree, kw, macroname, target) # Convert possible iterable result to `list`, then typecheck macro output. try: @@ -254,6 +303,32 @@ class BaseMacroExpander(NodeTransformer): return self._visit_expansion(expansion, target) + def _apply_macro(self, macro, tree, kw, macroname, target): + """Execute `macro` on `tree`, with the dictionary `kw` unpacked into macro's named arguments. + + `macro` is the macro function. + + `tree` is the AST to apply the macro to. This is what the macro gets as its `tree` argument. + + `kw` is a dictionary-like, unpacked into the macro's named arguments. + + `macroname` is the name (str) the macro function `macro` is bound as in this expander. + + `target` is the whole macro invocation AST. A deepcopy is passed to the debug hook (if installed). + """ + # There's no guarantee whether `macro` edits the AST in-place or produces a new tree, + # so to produce reliable debug information, we should take the old `id` and a deep copy + # of the macro invocation. + if self._debughook: + oldid = id(target) + oldtree = deepcopy(target) # including the macro invocation itself + + newtree = macro(tree, **kw) + + if self._debughook: + self._debughook(oldid, oldtree, newtree, macroname, macro) + return newtree + def _visit_expansion(self, expansion, target): """Perform local postprocessing. @@ -278,10 +353,6 @@ class BaseMacroExpander(NodeTransformer): return bindings[name] return False -def _apply_macro(macro, tree, kw): - """Execute `macro` on `tree`, with the dictionary `kw` unpacked into macro's named arguments.""" - return macro(tree, **kw) - # Final postprocessing for the top-level walk can't be done at the end of the # entry points `visit_once` and `visit_recursively`, because it is valid for a @@ -296,8 +367,53 @@ def global_postprocess(tree): to Python's `compile`. """ tree = delete_markers(tree, cls=MacroExpanderMarker) + # A name macro, appearing as an assignment target, gets the wrong ctx, # because when expanding the name macro, the expander sees only the Name # node, and thus puts an `ast.Load` there as the ctx. tree = fix_ctx(tree, copy_seen_nodes=True) + + # Run the user hooks, if any. + for custom_postprocessor in global_postprocessors: + tree = custom_postprocessor(tree) + return tree + +def add_postprocessor(function): + """Add a custom postprocessor for the top-level expansion. + + `function` must accept `AST` and `list` of `AST`, and return + the same type. + + Custom postprocessors are automatically called by `global_postprocess` + (which is called just after macro expansion for a module ends), in the + order they were registered by calling `add_postprocessor`. + + This e.g. allows a macro library to use its own `ASTMarker` subclasses + for internal communication between macros, and delete (only) its own + markers when done. + + Put something like this somewhere in your library initialization code + (preferably somewhere it's easy to discover for someone reading your + code, such as in a package `__init__.py`): + + from functools import partial + + import mcpyrate.core + from mcpyrate.markers import delete_markers + + _delete_my_markers = partial(delete_markers, + cls=MyCustomMarkerBase) + mcpyrate.core.add_postprocessor(_delete_my_markers) + """ + if function not in global_postprocessors: + global_postprocessors.append(function) + +def remove_postprocessor(function): + """Remove a previously added custom postprocessor. + + The `function` is the callable object that was registered as a + postprocessor using `add_postprocessor`. + """ + if function in global_postprocessors: + global_postprocessors.remove(function) diff --git a/kibot/mcpyrate/coreutils.py b/kibot/mcpyrate/coreutils.py index a9396c42..824f71be 100644 --- a/kibot/mcpyrate/coreutils.py +++ b/kibot/mcpyrate/coreutils.py @@ -4,7 +4,7 @@ __all__ = ["resolve_package", "relativize", "match_syspath", "ismacroimport", "get_macros"] -from ast import ImportFrom +import ast import importlib import importlib.util import os @@ -14,6 +14,7 @@ import sys from .unparser import unparse_with_fallbacks from .utils import format_location + def resolve_package(filename): # TODO: for now, `guess_package`, really. Check the docs again. """Resolve absolute Python package name for .py source file `filename`. @@ -79,7 +80,7 @@ def ismacroimport(statement, magicname='macros'): where "macros" is the literal string given as `magicname`. """ - if isinstance(statement, ImportFrom): + if isinstance(statement, ast.ImportFrom): firstimport = statement.names[0] if firstimport.name == magicname and firstimport.asname is None: return True @@ -146,6 +147,8 @@ def get_macros(macroimport, *, filename, reload=False, allow_asname=True, self_m try: module = sys.modules[module_absname] except KeyError: + approx_sourcecode = unparse_with_fallbacks(macroimport, debug=True, color=True) + loc = format_location(filename, macroimport, approx_sourcecode) raise ModuleNotFoundError(f"{loc}\nModule {module_absname} not found in `sys.modules`") # regular macro-import @@ -165,13 +168,76 @@ def get_macros(macroimport, *, filename, reload=False, allow_asname=True, self_m bindings = {} for name in macroimport.names[1:]: if not allow_asname and name.asname is not None: + approx_sourcecode = unparse_with_fallbacks(macroimport, debug=True, color=True) + loc = format_location(filename, macroimport, approx_sourcecode) raise ImportError(f"{loc}\nThis expander (see traceback) does not support as-naming macro-imports.") try: - bindings[name.asname or name.name] = getattr(module, name.name) + macro = getattr(module, name.name) except AttributeError as err: approx_sourcecode = unparse_with_fallbacks(macroimport, debug=True, color=True) loc = format_location(filename, macroimport, approx_sourcecode) raise ImportError(f"{loc}\ncannot import name '{name.name}' from module {module_absname}") from err + if not callable(macro): + approx_sourcecode = unparse_with_fallbacks(macroimport, debug=True, color=True) + loc = format_location(filename, macroimport, approx_sourcecode) + raise ImportError(f"{loc}\nname '{name.name}' in module {module_absname} is not a callable object (got {type(macro)} with value {repr(macro)}), so it cannot be imported as a macro.") + + bindings[name.asname or name.name] = macro + return module_absname, bindings + +# -------------------------------------------------------------------------------- + +def _mcpyrate_attr(dotted_name, *, force_import=False): + """Create an AST that, when compiled and run, looks up an attribute of `mcpyrate`. + + `dotted_name` is an `str`. Examples:: + + _mcpyrate_attr("dump") # -> mcpyrate.dump + _mcpyrate_attr("quotes.lookup_value") # -> mcpyrate.quotes.lookup_value + + If `force_import` is `True`, use the builtin `__import__` function to + first import the `mcpyrate` module whose attribute will be accessed. + This is useful when the eventual use site might not import any `mcpyrate` + modules. + """ + if not isinstance(dotted_name, str): + raise TypeError(f"dotted_name name must be str; got {type(dotted_name)} with value {repr(dotted_name)}") + + if dotted_name.find(".") != -1: + submodule_dotted_name, _ = dotted_name.rsplit(".", maxsplit=1) + else: + submodule_dotted_name = None + + # Issue #21: `mcpyrate` might not be in scope at the use site. Fortunately, + # `__import__` is a builtin, so we are guaranteed to always have that available. + if not force_import: + mcpyrate_module = ast.Name(id="mcpyrate") + else: + globals_call = ast.Call(ast.Name(id="globals"), + [], + []) + + if submodule_dotted_name: + modulename_to_import = f"mcpyrate.{submodule_dotted_name}" + else: + modulename_to_import = "mcpyrate" + + import_call = ast.Call(ast.Name(id="__import__"), + [ast.Constant(value=modulename_to_import), + globals_call, # globals (used for determining context) + ast.Constant(value=None), # locals (unused) + ast.Tuple(elts=[]), # fromlist + ast.Constant(value=0)], # level + []) + # When compiled and run, the import call will evaluate to a reference + # to the top-level `mcpyrate` module. + mcpyrate_module = import_call + + value = mcpyrate_module + for name in dotted_name.split("."): + value = ast.Attribute(value=value, attr=name) + + return value diff --git a/kibot/mcpyrate/debug.py b/kibot/mcpyrate/debug.py index 526c1553..1a008344 100644 --- a/kibot/mcpyrate/debug.py +++ b/kibot/mcpyrate/debug.py @@ -8,7 +8,7 @@ __all__ = ["step_expansion", "StepExpansion", "step_phases", import ast import functools import io -from sys import stderr +import sys import textwrap from .astdumper import dump @@ -30,10 +30,10 @@ def step_expansion(tree, *, args, syntax, expander, **kw): Syntax:: step_expansion[...] - step_expansion[mode][...] + step_expansion[arg, ...][...] with step_expansion: ... - with step_expansion[mode]: + with step_expansion[arg, ...]: ... This calls `expander.visit_once` in a loop, discarding the `Done` markers, @@ -45,6 +45,9 @@ def step_expansion(tree, *, args, syntax, expander, **kw): before returning control; if it does `expander.visit_recursively(subtree)`, then that subtree will be expanded, in a single step, until no macros remain. + However, the argument `"detailed"` can be used to report all macro expansions + within each step, including those done by `expander.visit(subtree)`. + Since this is a debugging utility, the source code is rendered in the debug mode of `unparse`, which prints also invisible nodes such as `Module` and `Expr` (as well as any AST markers) in a Python-like pseudocode notation. @@ -52,60 +55,93 @@ def step_expansion(tree, *, args, syntax, expander, **kw): The source code is rendered with syntax highlighting suitable for terminal output. - The optional macro argument `mode`, if present, sets the renderer mode. - It must be one of the strings `"unparse"` (default) or `"dump"`. - If `"unparse"`, then at each step, the AST will be shown as source code. - If `"dump"`, then at each step, the AST will be shown as a raw AST dump. + Supported macro arguments (can be given in any order): + + - One of the strings `"unparse"` (default) or `"dump"` sets the renderer mode. + + If `"unparse"`, then at each step, the AST will be shown as source code. + If `"dump"`, then at each step, the AST will be shown as a raw AST dump. + + - One of the strings `"summary"` (default) or `"detailed"` sets the report + detail level. + + If `"summary"`, then the whole tree is printed once per step. + If `"detailed"`, also each macro expansion in each step is reported, + by printing the relevant subtree before and after expansion. This macro steps the expansion at macro expansion time. If you have a run-time AST value (such as a quasiquoted tree), and want to step the expansion of that at run time, see the `stepr` macro in `mcpyrate.metatools`. + It supports the same arguments as `step_expansion`. """ if syntax not in ("expr", "block"): raise SyntaxError("`step_expansion` is an expr and block macro only") + supported_args = ("unparse", "dump", "detailed", "summary") formatter = functools.partial(unparse_with_fallbacks, debug=True, color=True, expander=expander) - if args: - if len(args) != 1: - raise SyntaxError("expected `step_expansion` or `step_expansion['mode_str']`") - arg = args[0] + print_details = False + for arg in args: if type(arg) is ast.Constant: - mode = arg.value + v = arg.value elif type(arg) is ast.Str: # up to Python 3.7 - mode = arg.s + v = arg.s else: - raise TypeError(f"expected mode_str, got {repr(arg)} {unparse_with_fallbacks(arg)}") - if mode not in ("unparse", "dump"): - raise ValueError(f"expected mode_str either 'unparse' or 'dump', got {repr(mode)}") - if mode == "dump": + raise TypeError(f"expected str argument, got {repr(arg)} {unparse_with_fallbacks(arg)}") + + if v not in supported_args: + raise ValueError(f"unknown argument {repr(v)}") + + if v == "dump": formatter = functools.partial(dump, include_attributes=True, color=True) + elif v == "detailed": + print_details = True c, CS = setcolor, ColorScheme - with _step_expansion_level.changed_by(+1): + with _step_expansion_level.changed_by(+1): # The level is used for nested invocations of `step_expansion`. indent = 2 * _step_expansion_level.value stars = indent * '*' codeindent = indent tag = id(tree) - print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} ({expander.filename}) {c(CS.HEADING)}before macro expansion:{c()}", - file=stderr) - print(textwrap.indent(formatter(tree), codeindent * ' '), file=stderr) - mc = MacroCollector(expander) - mc.visit(tree) + + print(f"{c(CS.HEADING1)}{stars}Tree {c(CS.HEADING2)}0x{tag:x} ({expander.filename}) {c(CS.HEADING1)}before macro expansion:{c()}", + file=sys.stderr) + print(textwrap.indent(f"{formatter(tree)}", codeindent * ' '), file=sys.stderr) + step = 0 - while mc.collected: - step += 1 - tree = expander.visit_once(tree) # -> Done(body=...) - tree = tree.body - print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} ({expander.filename}) {c(CS.HEADING)}after step {step}:{c()}", - file=stderr) - print(textwrap.indent(formatter(tree), codeindent * ' '), file=stderr) - mc.clear() + def doit(): + nonlocal step + nonlocal tree + mc = MacroCollector(expander) mc.visit(tree) + while mc.collected: + step += 1 + tree = expander.visit_once(tree) # -> Done(body=...) + tree = tree.body + print(f"{c(CS.HEADING1)}{stars}Tree {c(CS.HEADING2)}0x{tag:x} ({expander.filename}) {c(CS.HEADING1)}after step {step}:{c()}", + file=sys.stderr) + print(textwrap.indent(formatter(tree), codeindent * ' '), file=sys.stderr) + mc.clear() + mc.visit(tree) + + if print_details: + def print_step(invocationsubtreeid, invocationtree, expandedtree, macroname, macrofunction): + print(f"{c(CS.HEADING1)}{stars}Tree {c(CS.HEADING2)}0x{tag:x} ({expander.filename}) {c(CS.HEADING1)}processing step {step}:", + file=sys.stderr) + print(textwrap.indent(f"{c(CS.HEADING2)}Applying {c(CS.MACRONAME)}{macroname}{c(CS.HEADING2)} at subtree 0x{invocationsubtreeid:x}:{c()}", codeindent * ' '), file=sys.stderr) + print(textwrap.indent(f"{formatter(invocationtree)}", (codeindent + 2) * ' '), file=sys.stderr) + print(textwrap.indent(f"{c(CS.HEADING2)}Result:{c()}", codeindent * ' '), file=sys.stderr) + print(textwrap.indent(f"{formatter(expandedtree)}", (codeindent + 2) * ' '), file=sys.stderr) + + with expander.debughook(print_step): + doit() + else: + doit() + plural = "s" if step != 1 else "" - print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} ({expander.filename}) {c(CS.HEADING)}macro expansion complete after {step} step{plural}.{c()}", - file=stderr) + print(f"{c(CS.HEADING1)}{stars}Tree {c(CS.HEADING2)}0x{tag:x} ({expander.filename}) {c(CS.HEADING1)}macro expansion complete after {step} step{plural}.{c()}", + file=sys.stderr) return tree @@ -138,7 +174,7 @@ def show_bindings(tree, *, syntax, expander, **kw): """ if syntax != "name": raise SyntaxError("`show_bindings` is an identifier macro only") - print(format_bindings(expander, globals_too=False, color=True), file=stderr) + print(format_bindings(expander, globals_too=False, color=True), file=sys.stderr) return None @@ -164,7 +200,7 @@ def format_bindings(expander, *, globals_too=False, color=False): bindings = expander.bindings if globals_too else expander.local_bindings with io.StringIO() as output: - output.write(f"{c(CS.HEADING)}Macro bindings for {c(CS.SOURCEFILENAME)}{expander.filename}{c(CS.HEADING)}:{c()}\n") + output.write(f"{c(CS.HEADING1)}Macro bindings for {c(CS.SOURCEFILENAME)}{expander.filename}{c(CS.HEADING1)}:{c()}\n") if not bindings: output.write(maybe_colorize(" \n", ColorScheme.GREYEDOUT)) diff --git a/kibot/mcpyrate/dialects.py b/kibot/mcpyrate/dialects.py index 940f8595..8dd69da5 100644 --- a/kibot/mcpyrate/dialects.py +++ b/kibot/mcpyrate/dialects.py @@ -6,7 +6,7 @@ __all__ = ["Dialect", "DialectExpander"] import ast import functools import re -from sys import stderr +import sys from .colorizer import setcolor, colorize, ColorScheme from .coreutils import ismacroimport, get_macros @@ -83,20 +83,17 @@ class Dialect: `mcpyrate.splicing.splice_dialect` (it automatically handles macro-imports, dialect-imports, the magic `__all__`, and the module docstring). - As an example, for now, until `unpythonic` is ported to `mcpyrate`, see the - example dialects in `pydialect`, which are implemented using this exact - strategy, but with the older `macropy` macro expander, the older `pydialect` - dialect system, and `unpythonic`. + As an example, see the `dialects` module in `unpythonic` for example dialects. - https://github.com/Technologicat/pydialect + https://github.com/Technologicat/unpythonic - To give a flavor; once we get that ported, we'll have *Lispython*, which is - essentially Python with TCO, and implicit `return` in tail position:: + To give a flavor, *Lispython* is essentially Python with TCO, and implicit + `return` in tail position:: # -*- coding: utf-8; -*- '''Lispython example.''' - from mylibrary import dialects, Lispython + from unpythonic.dialects import dialects, Lispython def fact(n): def f(k, acc): @@ -114,7 +111,7 @@ class Dialect: return NotImplemented -_message_header = colorize("**StepExpansion: ", ColorScheme.HEADING) +_message_header = colorize("**StepExpansion: ", ColorScheme.HEADING1) class StepExpansion(Dialect): # actually part of public API of mcpyrate.debug, for discoverability """[dialect] Show each step of expansion while dialect-expanding the module. @@ -150,8 +147,8 @@ class StepExpansion(Dialect): # actually part of public API of mcpyrate.debug, def _enable_debugmode(self): self.expander.debugmode = True c, CS = setcolor, ColorScheme - msg = f"{c(CS.SOURCEFILENAME)}{self.expander.filename} {c(CS.HEADING)}enabled {c(CS.ATTENTION)}DialectExpander debug mode {c(CS.HEADING)}while taking step {self.expander._step + 1}.{c()}" - print(_message_header + msg, file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.expander.filename} {c(CS.HEADING1)}enabled {c(CS.ATTENTION)}DialectExpander debug mode {c(CS.HEADING1)}while taking step {self.expander._step + 1}.{c()}" + print(_message_header + msg, file=sys.stderr) # -------------------------------------------------------------------------------- @@ -210,9 +207,9 @@ class DialectExpander: c, CS = setcolor, ColorScheme if self.debugmode: plural = "s" if self._step != 1 else "" - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}before dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING)}transformers ({self._step} step{plural} total):{c()}\n" - print(_message_header + msg, file=stderr) - print(format_for_display(content), file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}before dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING1)}transformers ({self._step} step{plural} total):{c()}\n" + print(_message_header + msg, file=sys.stderr) + print(format_for_display(content), file=sys.stderr) # We collect and return the dialect object instances so that both # `transform_ast` and `postprocess_ast` can use the same instances. The @@ -262,14 +259,14 @@ class DialectExpander: self._step += 1 if self.debugmode: - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}after {c(CS.DIALECTTRANSFORMERNAME)}{module_absname}.{dialectname}.{transform} {c(CS.HEADING)}(step {self._step}):{c()}\n" - print(_message_header + msg, file=stderr) - print(format_for_display(content), file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}after {c(CS.DIALECTTRANSFORMERNAME)}{module_absname}.{dialectname}.{transform} {c(CS.HEADING1)}(step {self._step}):{c()}\n" + print(_message_header + msg, file=sys.stderr) + print(format_for_display(content), file=sys.stderr) if self.debugmode: plural = "s" if self._step != 1 else "" - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}completed all dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING)}transforms ({self._step} step{plural} total).{c()}" - print(_message_header + msg, file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}completed all dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING1)}transforms ({self._step} step{plural} total).{c()}" + print(_message_header + msg, file=sys.stderr) return content, dialect_instances @@ -283,9 +280,9 @@ class DialectExpander: c, CS = setcolor, ColorScheme if self.debugmode: plural = "s" if self._step != 1 else "" - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}before dialect {c(CS.TRANSFORMERKIND)}AST postprocessors {c(CS.HEADING)}({self._step} step{plural} total):{c()}\n" - print(_message_header + msg, file=stderr) - print(format_for_display(tree), file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}before dialect {c(CS.TRANSFORMERKIND)}AST postprocessors {c(CS.HEADING1)}({self._step} step{plural} total):{c()}\n" + print(_message_header + msg, file=sys.stderr) + print(format_for_display(tree), file=sys.stderr) content = tree for dialect in dialect_instances: @@ -308,14 +305,14 @@ class DialectExpander: self._step += 1 if self.debugmode: - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}after {c(CS.DIALECTTRANSFORMERNAME)}{format_macrofunction(dialect)}.postprocess_ast {c(CS.HEADING)}(step {self._step}):{c()}\n" - print(_message_header + msg, file=stderr) - print(format_for_display(content), file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}after {c(CS.DIALECTTRANSFORMERNAME)}{format_macrofunction(dialect)}.postprocess_ast {c(CS.HEADING1)}(step {self._step}):{c()}\n" + print(_message_header + msg, file=sys.stderr) + print(format_for_display(content), file=sys.stderr) if self.debugmode: plural = "s" if self._step != 1 else "" - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}completed all dialect {c(CS.TRANSFORMERKIND)}AST postprocessors {c(CS.HEADING)}({self._step} step{plural} total).{c()}" - print(_message_header + msg, file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING1)}completed all dialect {c(CS.TRANSFORMERKIND)}AST postprocessors {c(CS.HEADING1)}({self._step} step{plural} total).{c()}" + print(_message_header + msg, file=sys.stderr) return content diff --git a/kibot/mcpyrate/expander.py b/kibot/mcpyrate/expander.py index a9717662..e760f45e 100644 --- a/kibot/mcpyrate/expander.py +++ b/kibot/mcpyrate/expander.py @@ -51,15 +51,17 @@ __all__ = ["namemacro", "isnamemacro", "MacroExpander", "MacroCollector", "expand_macros", "find_macros"] -from ast import (Name, Subscript, Tuple, Import, alias, AST, Assign, Store, Constant, - Lambda, arguments, Call, copy_location, iter_fields, NodeVisitor) +import sys +from ast import (AST, Assign, Call, Starred, Constant, Import, Lambda, Name, + NodeVisitor, Store, Subscript, Tuple, alias, arguments, + copy_location, iter_fields) from copy import copy from warnings import warn_explicit -from .core import BaseMacroExpander, global_postprocess, Done -from .coreutils import ismacroimport, get_macros +from .core import BaseMacroExpander, Done, global_postprocess +from .coreutils import get_macros, ismacroimport from .unparser import unparse_with_fallbacks -from .utils import format_macrofunction +from .utils import format_location, format_macrofunction def namemacro(function): @@ -94,17 +96,43 @@ def isparametricmacro(function): # -------------------------------------------------------------------------------- -def destructure_candidate(tree): +def destructure_candidate(tree, *, filename, _validate_call_syntax=True): """Destructure a macro call candidate AST, `macroname` or `macroname[arg0, ...]`.""" if type(tree) is Name: return tree.id, [] elif type(tree) is Subscript and type(tree.value) is Name: - macroargs = tree.slice.value + if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper + macroargs = tree.slice + else: + macroargs = tree.slice.value + if type(macroargs) is Tuple: # [a0, a1, ...] macroargs = macroargs.elts else: # anything that doesn't have at least one comma at the top level macroargs = [macroargs] return tree.value.id, macroargs + # Up to Python 3.8, decorators cannot be subscripted. This is a problem for + # decorator macros that would like to have macro arguments. + # + # To work around this, we allow passing macro arguments also using parentheses + # (like in `macropy`). Note macro arguments must still be passed positionally! + # + # For uniformity, we limit to a subset of the function call syntax that + # remains valid if you replace the parentheses with brackets in Python 3.9. + elif type(tree) is Call and type(tree.func) is Name and not tree.keywords: + # `MacroExpander._detect_macro_items` needs to perform a preliminary check + # without full validation when it is trying to detect which `with` items + # and decorators are in fact macro invocations. + if _validate_call_syntax: + if not tree.args: # reject empty args + approx_sourcecode = unparse_with_fallbacks(tree, debug=True, color=True) + loc = format_location(filename, tree, approx_sourcecode) + raise SyntaxError(f"{loc}\nmust provide at least one argument when passing macro arguments") + if any(type(arg) is Starred for arg in tree.args): # reject starred items + approx_sourcecode = unparse_with_fallbacks(tree, debug=True, color=True) + loc = format_location(filename, tree, approx_sourcecode) + raise SyntaxError(f"{loc}\nunpacking (splatting) not supported in macro argument position") + return tree.func.id, tree.args return None, None # not a macro invocation @@ -115,13 +143,13 @@ class MacroExpander(BaseMacroExpander): """Shorthand to check `destructure_candidate` output. Return whether that output is a macro call to a macro (of invocation - type `syntax`) bound in this expander. + type `syntax`) bound in this expander or globally. """ if not (macroname and self.isbound(macroname)): return False if syntax == 'name': - return isnamemacro(self.bindings[macroname]) - return not macroargs or isparametricmacro(self.bindings[macroname]) + return isnamemacro(self.isbound(macroname)) + return not macroargs or isparametricmacro(self.isbound(macroname)) def visit_Subscript(self, subscript): """Detect an expression (expr) macro invocation. @@ -138,10 +166,16 @@ class MacroExpander(BaseMacroExpander): # because things like `(some_expr_macro[tree])[subscript_expression]` are valid. This # is actually exploited by `h`, as in `q[h[target_macro][tree_for_target_macro]]`. candidate = subscript.value - macroname, macroargs = destructure_candidate(candidate) + macroname, macroargs = destructure_candidate(candidate, filename=self.filename, + _validate_call_syntax=False) if self.ismacrocall(macroname, macroargs, "expr"): + # Now we know it's a macro invocation, so we can validate the parenthesis syntax to pass arguments. + macroname, macroargs = destructure_candidate(candidate, filename=self.filename) kw = {"args": macroargs} - tree = subscript.slice.value + if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper + tree = subscript.slice + else: + tree = subscript.slice.value sourcecode = unparse_with_fallbacks(subscript, debug=True, color=True, expander=self) new_tree = self.expand("expr", subscript, macroname, tree, sourcecode=sourcecode, kw=kw) if new_tree is None: @@ -190,13 +224,35 @@ class MacroExpander(BaseMacroExpander): do anything it wants to its input tree. Any remaining block macro invocations are attached to the `With` node, so if that is removed, they will be skipped. + + **NOTE**: At least up to v3.3.0, if there are three or more macro + invocations in the same `with` statement:: + + with macro1, macro2, macro3: + ... + + this is equivalent with:: + + with macro1: + with macro2, macro3: + ... + + and **not** equivalent with:: + + with macro1: + with macro2: + with macro3: + ... + + which may be important if `macro1` needs to scan for invocations of + `macro2` and `macro3` to work together with them. """ macros, others = self._detect_macro_items(withstmt.items, "block") if not macros: return self.generic_visit(withstmt) with_item = macros[0] candidate = with_item.context_expr - macroname, macroargs = destructure_candidate(candidate) + macroname, macroargs = destructure_candidate(candidate, filename=self.filename) # let the source code and `invocation` see also the withitem we pop away sourcecode = unparse_with_fallbacks(withstmt, debug=True, color=True, expander=self) @@ -250,7 +306,7 @@ class MacroExpander(BaseMacroExpander): if not macros: return self.generic_visit(decorated) innermost_macro = macros[-1] - macroname, macroargs = destructure_candidate(innermost_macro) + macroname, macroargs = destructure_candidate(innermost_macro, filename=self.filename) # let the source code and `invocation` see also the decorator we pop away sourcecode = unparse_with_fallbacks(decorated, debug=True, color=True, expander=self) @@ -279,7 +335,8 @@ class MacroExpander(BaseMacroExpander): candidate = item.context_expr else: candidate = item - macroname, macroargs = destructure_candidate(candidate) + macroname, macroargs = destructure_candidate(candidate, filename=self.filename, + _validate_call_syntax=False) # warn about likely mistake if (macroname and self.isbound(macroname) and @@ -350,7 +407,11 @@ class MacroCollector(NodeVisitor): Sister class of the actual `MacroExpander`, mirroring its syntax detection. """ def __init__(self, expander): - """`expander`: a `MacroExpander` instance to query macro bindings from.""" + """`expander`: a `MacroExpander` instance to query macro bindings from. + + `filename`: full path to `.py` file being expanded, for error reporting. + Only used for errors during `destructure_candidate`. + """ self.expander = expander self.clear() @@ -379,7 +440,7 @@ class MacroCollector(NodeVisitor): def visit_Subscript(self, subscript): candidate = subscript.value - macroname, macroargs = destructure_candidate(candidate) + macroname, macroargs = destructure_candidate(candidate, filename=self.expander.filename) if self.expander.ismacrocall(macroname, macroargs, "expr"): key = (macroname, "expr") if key not in self._seen: @@ -388,7 +449,10 @@ class MacroCollector(NodeVisitor): self.visit(macroargs) # Don't `self.generic_visit(tree)`; that'll incorrectly detect # the name part as an identifier macro. Recurse only in the expr. - self.visit(subscript.slice.value) + if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper + self.visit(subscript.slice) + else: + self.visit(subscript.slice.value) else: self.generic_visit(subscript) @@ -397,7 +461,7 @@ class MacroCollector(NodeVisitor): if macros: for with_item in macros: candidate = with_item.context_expr - macroname, macroargs = destructure_candidate(candidate) + macroname, macroargs = destructure_candidate(candidate, filename=self.expander.filename) key = (macroname, "block") if key not in self._seen: self.collected.append(key) @@ -419,7 +483,7 @@ class MacroCollector(NodeVisitor): macros, others = self.expander._detect_macro_items(decorated.decorator_list, "decorator") if macros: for macro in macros: - macroname, macroargs = destructure_candidate(macro) + macroname, macroargs = destructure_candidate(macro, filename=self.expander.filename) key = (macroname, "decorator") if key not in self._seen: self.collected.append(key) diff --git a/kibot/mcpyrate/importer.py b/kibot/mcpyrate/importer.py index befcca5b..230381be 100644 --- a/kibot/mcpyrate/importer.py +++ b/kibot/mcpyrate/importer.py @@ -155,7 +155,7 @@ def path_stats(path, _stats_cache=None): # TODO: Or just document it, that the dialect definition module *must* macro-import those macros # TODO: even if it just injects them in the template? with tokenize.open(path) as sourcefile: - tree = ast.parse(sourcefile.read()) + tree = ast.parse(sourcefile.read(), filename=path) macroimports = [] dialectimports = [] @@ -165,7 +165,7 @@ def path_stats(path, _stats_cache=None): macroimports.append(stmt) elif ismacroimport(stmt, magicname="dialects"): dialectimports.append(stmt) - elif iswithphase(stmt): # for multi-phase compilation: scan also inside top-level `with phase` + elif iswithphase(stmt, filename=path): # for multi-phase compilation: scan also inside top-level `with phase` scan(stmt) scan(tree) diff --git a/kibot/mcpyrate/markers.py b/kibot/mcpyrate/markers.py index 11ca81f2..6f073d83 100644 --- a/kibot/mcpyrate/markers.py +++ b/kibot/mcpyrate/markers.py @@ -9,9 +9,7 @@ __all__ = ["ASTMarker", "get_markers", "delete_markers", "check_no_markers_remai import ast -from . import core -from . import utils -from . import walkers +from . import core, utils, walkers class ASTMarker(ast.AST): @@ -36,7 +34,9 @@ class ASTMarker(ast.AST): section. So just before the quote operator exits, it checks that all quasiquote markers within that section have been compiled away. """ - def __init__(self, body): + # TODO: Silly default `None`, because `copy` and `deepcopy` call `__init__` without arguments, + # TODO: though the docs say they behave like `pickle` (and wouldn't thus need to call __init__ at all!). + def __init__(self, body=None): """body: the actual AST that is annotated by this marker""" self.body = body self._fields = ["body"] # support ast.iter_fields @@ -63,7 +63,7 @@ def delete_markers(tree, cls=ASTMarker): class ASTMarkerDeleter(walkers.ASTTransformer): def transform(self, tree): if isinstance(tree, cls): - tree = tree.body + return self.visit(tree.body) return self.generic_visit(tree) return ASTMarkerDeleter().visit(tree) @@ -79,7 +79,6 @@ def check_no_markers_remaining(tree, *, filename, cls=None): `filename` is the full path to the `.py` file, for error reporting. Convenience function. - """ cls = cls or ASTMarker remaining_markers = get_markers(tree, cls) diff --git a/kibot/mcpyrate/metatools.py b/kibot/mcpyrate/metatools.py index fb198202..9ae4711f 100644 --- a/kibot/mcpyrate/metatools.py +++ b/kibot/mcpyrate/metatools.py @@ -44,15 +44,15 @@ import ast # Note we import some macros as regular functions. We just want their syntax transformers. from .astfixers import fix_locations # noqa: F401, used in macro output. +from .coreutils import _mcpyrate_attr from .debug import step_expansion # noqa: F401, used in macro output. from .expander import MacroExpander, namemacro, parametricmacro -from .quotes import q, astify, unastify, capture_value +from .quotes import astify, capture_value, q, unastify def _mcpyrate_metatools_attr(attr): """Create an AST that, when compiled and run, looks up `mcpyrate.metatools.attr`.""" - mcpyrate_metatools_module = ast.Attribute(value=ast.Name(id="mcpyrate"), attr="metatools") - return ast.Attribute(value=mcpyrate_metatools_module, attr=attr) + return _mcpyrate_attr(f"metatools.{attr}") # -------------------------------------------------------------------------------- diff --git a/kibot/mcpyrate/multiphase.py b/kibot/mcpyrate/multiphase.py index 68bc4140..fbb73d61 100644 --- a/kibot/mcpyrate/multiphase.py +++ b/kibot/mcpyrate/multiphase.py @@ -21,11 +21,11 @@ because Racket's phase level tower is the system that most resembles this one. __all__ = ["phase", "ismultiphase", "detect_highest_phase", "isdebug", "multiphase_expand"] import ast -from copy import copy, deepcopy import sys +from copy import copy, deepcopy -from .colorizer import setcolor, ColorScheme from . import compiler +from .colorizer import ColorScheme, setcolor from .coreutils import ismacroimport from .expander import destructure_candidate, namemacro, parametricmacro from .unparser import unparse_with_fallbacks @@ -34,7 +34,7 @@ from .utils import getdocstring # -------------------------------------------------------------------------------- # Private utilities. -def iswithphase(stmt): +def iswithphase(stmt, *, filename): """Check if AST node `stmt` is a `with phase[n]`, where `n >= 1` is an integer. Return `n`, or `False`. @@ -52,7 +52,7 @@ def iswithphase(stmt): if type(candidate) is not ast.Subscript: return False - macroname, macroargs = destructure_candidate(candidate) + macroname, macroargs = destructure_candidate(candidate, filename=filename) if macroname != "phase": return False if not macroargs or len(macroargs) != 1: # exactly one macro-argument @@ -71,8 +71,11 @@ def iswithphase(stmt): return n +def isfutureimport(tree): + """Return whether `tree` is a `from __future__ import ...`.""" + return isinstance(tree, ast.ImportFrom) and tree.module == "__future__" -def extract_phase(tree, *, phase=0): +def extract_phase(tree, *, filename, phase=0): """Split `tree` into given `phase` and remaining parts. Primarily meant to be called with `tree` the AST of a module that @@ -88,7 +91,8 @@ def extract_phase(tree, *, phase=0): The lifted AST is deep-copied to minimize confusion, since it may get edited by macros during macro expansion. (This guarantees that - macro-expanding it, in either phase, gives the same result.) + macro-expanding it, in either phase, gives the same result, + up to and including any side effects of the macros.) """ if not isinstance(phase, int): raise TypeError(f"`phase` must be `int`, got {type(phase)} with value {repr(phase)}") @@ -99,7 +103,7 @@ def extract_phase(tree, *, phase=0): remaining = [] def lift(withphase): # Lift a `with phase[n]` code block to phase `n - 1`. - original_phase = iswithphase(withphase) + original_phase = iswithphase(withphase, filename=filename) assert original_phase if original_phase == 1: # Lifting to phase 0. Drop the `with phase` wrapper. @@ -107,7 +111,11 @@ def extract_phase(tree, *, phase=0): else: # Lifting to phase >= 1. Decrease the `n` in `with phase[n]` # by one, so the block gets processed again in the next phase. - macroarg = withphase.items[0].context_expr.slice.value + if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper + macroarg = withphase.items[0].context_expr.slice + else: + macroarg = withphase.items[0].context_expr.slice.value + if type(macroarg) is ast.Constant: macroarg.value -= 1 elif type(macroarg) is ast.Num: # TODO: Python 3.8: remove ast.Num @@ -116,10 +124,17 @@ def extract_phase(tree, *, phase=0): thisphase = [] for stmt in tree.body: - if iswithphase(stmt) == phase: + if iswithphase(stmt, filename=filename) == phase: thisphase.extend(stmt.body) lift(stmt) else: + # Issue #28: `__future__` imports. + # + # `__future__` imports should affect all phases, because they change + # the semantics of the module they appear in. Essentially, they are + # a kind of dialect defined by the Python core itself. + if isfutureimport(stmt): + thisphase.append(stmt) remaining.append(stmt) tree.body[:] = remaining @@ -127,6 +142,35 @@ def extract_phase(tree, *, phase=0): newmodule.body = thisphase return newmodule +def split_futureimports(body): + """Split `body` into `__future__` imports and everything else. + + `body`: list of `ast.stmt`, the suite representing a module top level. + + Returns `[future_imports, the_rest]`. + """ + k = -1 # ensure `k` gets defined even if `body` is empty + for k, bstmt in enumerate(body): + if not isfutureimport(bstmt): + break + if k >= 0: + return body[:k], body[k:] + return [], body + +def inject_after_futureimports(stmt, body): + """Inject a statement into `body` after `__future__` imports. + + `body`: list of `ast.stmt`, the suite representing a module top level. + `stmt`: `ast.stmt`, the statement to inject. + """ + if getdocstring(body): + docstring, *body = body + futureimports, body = split_futureimports(body) + return [docstring] + futureimports + [stmt] + body + else: # no docstring + futureimports, body = split_futureimports(body) + return futureimports + [stmt] + body + # -------------------------------------------------------------------------------- # Public utilities. @@ -237,7 +281,7 @@ def ismultiphase(tree): return False -def detect_highest_phase(tree): +def detect_highest_phase(tree, *, filename): """Scan a module body for `with phase[n]` statements and return highest `n`, or `None`. Primarily meant to be called with `tree` the AST of a module that @@ -248,7 +292,7 @@ def detect_highest_phase(tree): """ maxn = None for stmt in tree.body: - n = iswithphase(stmt) + n = iswithphase(stmt, filename=filename) if maxn is None or (n is not None and n > maxn): maxn = n return maxn @@ -336,7 +380,7 @@ def multiphase_expand(tree, *, filename, self_module, dexpander=None, _optimize= Return value is the final phase-0 `tree`, after macro expansion. """ - n = detect_highest_phase(tree) + n = detect_highest_phase(tree, filename=filename) debug = isdebug(tree) c, CS = setcolor, ColorScheme @@ -350,7 +394,7 @@ def multiphase_expand(tree, *, filename, self_module, dexpander=None, _optimize= original_module = None if debug: - print(f"{c(CS.HEADING)}**Multi-phase compiling module {c(CS.TREEID)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.TREEID)}){c()}", file=sys.stderr) + print(f"{c(CS.HEADING1)}**Multi-phase compiling module {c(CS.HEADING2)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.HEADING2)}){c()}", file=sys.stderr) # Inject temporary module into `sys.modules`. # @@ -366,20 +410,18 @@ def multiphase_expand(tree, *, filename, self_module, dexpander=None, _optimize= for k in range(n, -1, -1): # phase 0 is what a regular compile would do if debug: - print(f"{c(CS.HEADING)}**AST for {c(CS.ATTENTION)}PHASE {k}{c(CS.HEADING)} of module {c(CS.TREEID)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.TREEID)}){c()}", file=sys.stderr) + print(f"{c(CS.HEADING1)}**AST for {c(CS.ATTENTION)}PHASE {k}{c(CS.HEADING1)} of module {c(CS.HEADING2)}'{self_module}' ({c(CS.SOURCEFILENAME)}{filename}{c(CS.HEADING2)}){c()}", file=sys.stderr) - phase_k_tree = extract_phase(tree, phase=k) + phase_k_tree = extract_phase(tree, filename=filename, phase=k) if phase_k_tree.body: # inject `__phase__ = k` for introspection (at run time of the phase being compiled now) tgt = ast.Name(id="__phase__", ctx=ast.Store(), lineno=1, col_offset=1) val = ast.Constant(value=k, lineno=1, col_offset=13) assignment = ast.Assign(targets=[tgt], value=val, lineno=1, col_offset=1) - if getdocstring(phase_k_tree.body): - docstring, *body = phase_k_tree.body - phase_k_tree.body = [docstring, assignment] + body - else: # no docstring - phase_k_tree.body = [assignment] + phase_k_tree.body + # Issue #28: `__future__` imports. + # They must be the first statements after the module docstring, if any. So we inject after them. + phase_k_tree.body = inject_after_futureimports(assignment, phase_k_tree.body) if debug: print(unparse_with_fallbacks(phase_k_tree, debug=True, color=True), file=sys.stderr) diff --git a/kibot/mcpyrate/pycachecleaner.py b/kibot/mcpyrate/pycachecleaner.py new file mode 100644 index 00000000..15cf21be --- /dev/null +++ b/kibot/mcpyrate/pycachecleaner.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Python bytecode cache (`.pyc`) cleaner. Deletes `__pycache__` directories.""" + +__all__ = ["getpycachedirs", "deletepycachedirs"] + +import os + + +def getpycachedirs(path): + """Return a list of all `__pycache__` directories under `path` (str). + + Each of the entries starts with `path`. + """ + if not os.path.isdir(path): + raise OSError(f"No such directory: '{path}'") + + paths = [] + for root, dirs, files in os.walk(path): + if "__pycache__" in dirs: + paths.append(os.path.join(root, "__pycache__")) + return paths + + +def deletepycachedirs(path): + """Delete all `__pycache__` directories under `path` (str). + + Ignores `FileNotFoundError`, but other errors raise. If an error occurs, + some `.pyc` cache files and their directories may already have been deleted. + """ + for x in getpycachedirs(path): + _delete_directory_recursively(x) + + +def _delete_directory_recursively(path): + """Delete a directory recursively, like 'rm -rf' in the shell. + + Ignores `FileNotFoundError`, but other errors raise. If an error occurs, + some files and directories may already have been deleted. + """ + for root, dirs, files in os.walk(path, topdown=False, followlinks=False): + for x in files: + try: + os.unlink(os.path.join(root, x)) + except FileNotFoundError: + pass + + for x in dirs: + try: + os.rmdir(os.path.join(root, x)) + except FileNotFoundError: + pass + + try: + os.rmdir(path) + except FileNotFoundError: + pass diff --git a/kibot/mcpyrate/quotes.py b/kibot/mcpyrate/quotes.py index 0a5b6fe1..8e1278e8 100644 --- a/kibot/mcpyrate/quotes.py +++ b/kibot/mcpyrate/quotes.py @@ -12,24 +12,33 @@ and uncompiler, respectively. """ __all__ = ["capture_value", "capture_macro", "capture_as_macro", + "is_captured_value", "is_captured_macro", "astify", "unastify", "q", "u", "n", "a", "s", "t", "h"] import ast import copy import pickle +import sys -from .core import global_bindings, Done, MacroExpansionError +from .core import Done, MacroExpansionError, global_bindings +from .coreutils import _mcpyrate_attr from .expander import MacroExpander, isnamemacro from .markers import ASTMarker, check_no_markers_remaining, delete_markers from .unparser import unparse, unparse_with_fallbacks -from .utils import gensym, scrub_uuid, flatten, extract_bindings, NestingLevelTracker +from .utils import (NestingLevelTracker, extract_bindings, flatten, gensym, + scrub_uuid) -def _mcpyrate_quotes_attr(attr): - """Create an AST that, when compiled and run, looks up `mcpyrate.quotes.attr`.""" - mcpyrate_quotes_module = ast.Attribute(value=ast.Name(id="mcpyrate"), attr="quotes") - return ast.Attribute(value=mcpyrate_quotes_module, attr=attr) +def _mcpyrate_quotes_attr(attr, *, force_import=False): + """Create an AST that, when compiled and run, looks up `mcpyrate.quotes.attr`. + + If `force_import` is `True`, use the builtin `__import__` function to + first import the `mcpyrate.quotes` module. This is useful for e.g. + hygienically unquoted values, whose eventual use site might not import + any `mcpyrate` modules. + """ + return _mcpyrate_attr(f"quotes.{attr}", force_import=force_import) class QuasiquoteMarker(ASTMarker): @@ -133,13 +142,20 @@ def lift_sourcecode(value, filename=""): """ if not isinstance(value, str): raise TypeError(f"`n[]`: expected an expression that evaluates to str, result was {type(value)} with value {repr(value)}") - return ast.parse(value, filename=filename, mode="eval").body + return ast.parse(value, filename=f"", mode="eval").body def _typecheck(node, cls, macroname): - body = node.body if isinstance(node, ASTMarker) else node - if not isinstance(body, cls): - raise TypeError(f"{macroname}: expected an expression node, got {type(body)} with value {repr(body)}") + if isinstance(node, ASTMarker): + if isinstance(node.body, list): # statement suite inside a marker + for child in node.body: + _typecheck(child, cls, macroname) + return + # single AST node inside a marker + _typecheck(node.body, cls, macroname) + return + if not isinstance(node, cls): + raise TypeError(f"{macroname}: expected {cls}, got {type(node)} with value {repr(node)}") def _flatten_and_typecheck_iterable(nodes, cls, macroname): try: @@ -207,6 +223,10 @@ def splice_ast_literals(tree, filename): doit(item) newthing.append(item) thing[:] = newthing + # As of Python 3.9, `Global` and `Nonlocal` are the only AST node types + # where a field contains a `list` of bare strings. + elif isinstance(thing, (ast.Global, ast.Nonlocal)): + pass elif isinstance(thing, ast.AST): for fieldname, value in ast.iter_fields(thing): if isinstance(value, list): @@ -278,7 +298,7 @@ def capture_value(value, name): # and serialization. # frozen_value = pickle.dumps(value) - return ast.Call(_mcpyrate_quotes_attr("lookup_value"), + return ast.Call(_mcpyrate_quotes_attr("lookup_value", force_import=True), [ast.Tuple(elts=[ast.Constant(value=name), ast.Constant(value=frozen_value)])], []) @@ -290,8 +310,29 @@ def lookup_value(key): Usually there's no need to call this function manually; `capture_value` (and thus also `h[]`) will generate an AST that calls this automatically. + + **NOTE**: For advanced macrology: if your own macros need to detect hygienic + captures using `is_captured_value`, and you want to look up the captured + value based on a key returned by that function, be aware that `lookup_value` + will only succeed if a value has been captured. + + Trying to look up a key that was extracted from a pre-capture AST + raises `ValueError`. In terms of the discussion in the docstring of + `is_captured_value`, you need a `lookup_value` AST for a value to + be present; a `capture_value` AST is too early. The transition occurs + when the use site of `q` runs. + + In that scenario, before you call `lookup_value` on your key, check that + `frozen_value is not None` (see docstring of `is_captured_value`); + that indicates that a value has been captured and can be decoded by + this function. """ name, frozen_value = key + + # Trying to look up a result of `is_captured_value` that isn't captured yet. + if frozen_value is None: + raise ValueError(f"The given key does not (yet) point to a value: {repr(key)}") + cachekey = (name, id(frozen_value)) # id() so each capture instance behaves independently if cachekey not in _lookup_cache: _lookup_cache[cachekey] = pickle.loads(frozen_value) @@ -333,7 +374,8 @@ def capture_as_macro(macro): Like `capture_macro`, but with one less level of delay. This injects the macro into the expander's global bindings table immediately, and returns - the uniqified `ast.Name` that can be used to refer to it. + the uniqified `ast.Name` that can be used to refer to it hygienically, + using `a[]`. The name is taken automatically from the name of the macro function. """ @@ -358,6 +400,235 @@ def lookup_macro(key): global_bindings[unique_name] = pickle.loads(frozen_macro) return ast.Name(id=unique_name) + +# -------------------------------------------------------------------------------- +# Advanced macrology support. + +# TODO: In a future version, do we want to add an ASTMarker for captured values +# TODO: that are ready for consumption? We could save the actual AST (which is +# TODO: now detected directly) into the `body` attribute of the marker, and make +# TODO: the compiler delete `HygienicValue` markers (replacing each by its `.body`) +# TODO: as the last step before handing the AST over to Python. + +def is_captured_value(tree): + """Test whether `tree` is a hygienically captured run-time value. + + This function is sometimes useful for advanced macrology. It facilitates + user-defined macros to work together in an environment where hygienic + captures are present. One macro, using quasiquotes, builds an AST, and + another macro analyzes the expanded AST later. + + Consider first, however, if you can arrange things so that the second macro + could analyze an *unexpanded* AST; that's often much easier. When the first + macro simply must expand first (for whatever reason), that's where this function + comes in. + + With this function, you can check (either by name or by value) whether some + `q[h[myfunction]]` points to the desired `"myfunction"`, so that e.g. the AST + produced by `q[h[myfunction](a0, ...)]` can be recognized as a call to your + `myfunction`. This allows your second macro to know it's `myfunction`, + so that it'll know how to interpret the args of that call. + + Real-world examples of where this is useful are too unwieldy to explain + here, but can be found in `unpythonic.syntax`. Particularly, see any use + sites of the helper function `unpythonic.syntax.nameutil.isx`. + + To detect a hygienically captured *macro*, use `is_captured_macro` instead. + + Return value: + + - On no match, return `False`. + + - On match, return a tuple `(name, frozen_value)`, where: + + - `name` (str) is the name of the captured identifier, or when the captured + value is from an arbitrary expression, the unparsed source code of that + expression. There is no name mangling for identifiers; it's the exact + original name that appeared in the source code. + + - `frozen_value` is either a `bytes` object that stores the frozen value + as opaque binary data, or `None` if the value has not been captured yet. + + The `bytes` object can be decoded by passing the whole return value as `key` + to `lookup_value`. That function will decode the data and return the actual + value, just as if the hygienic reference was decoded normally at run time. + + **NOTE**: + + Stages in the life of a hygienically captured *run-time value* in `mcpyrate`: + + 1. When the surrounding `q` expands, it first expands any unquotes nested + within it, but only those where the quote level hits zero. The `h[]` is + converted into a `Capture` AST marker; see the `h` operator for details. + + 2. Then, still while the surrounding `q` expands, `q` compiles quasiquote + markers. A `Capture` marker, in particular, compiles into a call to + the function `capture_value`. This is the output at macro expansion time + (of the use site of `q`). + + 3. When the use site of `q` reaches run time, the `capture_value` runs + (thus actually performing the capture), and replaces itself (in the + AST that was produced by `q`) with a call to the function `lookup_value`. + That `lookup_value` call is still an AST node. + + 4. In typical usage, that use site of `q` is inside the implementation + of some user-defined macro. When *that macro's use site* reaches run + time, the `lookup_value` runs (each time that expression is executed). + + So in the macro expansion of `q`, we have a call to `capture_value` + representing the hygienically captured run-time value. But once the macro + that uses `q` has returned its output, then we instead have a call to + `lookup_value`. The latter is the most likely scenario for advanced + user-defined macros that work together. + """ + if type(tree) is not ast.Call: + return False + + # The format is one of: + # + # - direct reference: `(mcpyrate.quotes).xxx` + # - reference by import: `(__import__("mcpyrate.quotes", ...).quotes).xxx` + # + # First check the `xxx` part: + callee = tree.func + if not (type(callee) is ast.Attribute and callee.attr in ("capture_value", "lookup_value")): + return False + # Then the rest: + if not _is_mcpyrate_quotes_reference(callee.value): + return False + + # This AST destructuring and constant extraction must match the format + # of the argument lists produced by the quasiquote system for calls to + # `capture_value` and `lookup_value`. + if callee.attr == "capture_value": # the call is `capture_value(..., name)` + name_node = tree.args[1] + assert type(name_node) is ast.Constant and type(name_node.value) is str + return (name_node.value, None) # the value hasn't been captured yet + elif callee.attr == "lookup_value": # the call is `lookup_value(key)` + key_node = tree.args[0] + name_node, frozen_value_node = key_node.elts + assert type(name_node) is ast.Constant and type(name_node.value) is str + assert type(frozen_value_node) is ast.Constant and type(frozen_value_node.value) is bytes + return (name_node.value, frozen_value_node.value) + + assert False # cannot happen + + +def is_captured_macro(tree): + """Just like `is_captured_value`, but detect a hygienically captured macro instead. + + To detect a hygienically captured *run-time value*, use `is_captured_value` instead. + + Return value: + + - On no match, return `False`. + + - On match, return a tuple `(name, unique_name, frozen_macro)`, where: + + - `name` (str) is the name of the macro, as it appeared in the bindings + of the expander instance it was captured from. + + - `unique_name` (str) is `name` with an underscore and UUID appended, + to make it unique. This is the name the macro will be injected as + into the expander's global bindings table. + + (By unique, we mean "universally unique anywhere for approximately + the next one thousand years"; see `mcpyrate.gensym`, which links to + the UUID spec used by the implementation.) + + - `frozen_macro` is either `bytes` object that stores a reference to the + frozen macro function as opaque binary data. + + The `bytes` object can be decoded by passing the whole return value as `key` + to `lookup_macro`. That function will decode the data, inject the macro into + the expander's global bindings table (if not already there), and give you an + `ast.Name` node whose `id` attribute contains the unique name (str), just as + if the hygienic reference was decoded normally at macro expansion time. + + Then, once the injection has taken place, you can obtain the actual macro + function object by calling `expander.isbound(id)`. + + **NOTE**: + + Stages in the life of a hygienically captured *macro* in `mcpyrate` are as follows. + Note that unlike `capture_value`, a call to `capture_macro` never appears in the AST. + + 1. When the surrounding `q` expands, it first expands any unquotes nested + within it, but only those where the quote level hits zero. The `h[]` is + converted into a `Capture` AST marker; see the `h` operator for details. + + 2. Then, still while the surrounding `q` expands, `q` compiles quasiquote + markers. A `Capture` marker for a macro, in particular, triggers an + immediate call to the function `capture_macro`. The result is an AST + representing a call to the function `lookup_macro`. This gets injected + into the AST produced by `q`. + + 3. When the use site of `q` reaches run time, the `lookup_macro` runs, + injecting the macro (under its unique name) into the expander's global + bindings table. The `lookup_macro` call replaces itself with an `ast.Name` + whose `id` attribute contains the unique name of the macro. + + 4. In typical usage, that use site of `q` is inside the implementation + of some user-defined macro. Upon further macro expansion of *that macro's + use site*, the expander finds the now-bound unique name of the macro, and + proceeds to expand that macro. + + So in the macro expansion of `q`, we have a call to `lookup_macro` + representing the hygienically captured macro. But this disappears after + a very brief window of time, namely when the use site of `q` reaches run + time. Thus, this function likely has much fewer use cases than + `is_captured_value`, but is provided for completeness. + + (The point of hygienic macro capture is that a macro can safely return a further + macro invocation, and guarantee that this will invoke the intended macro - without + requiring the user to import that other macro, and without being forced to expand + it away before returning from the original macro.) + """ + if type(tree) is not ast.Call: + return False + + callee = tree.func + if not (type(callee) is ast.Attribute and callee.attr == "lookup_macro"): + return False + if not _is_mcpyrate_quotes_reference(callee.value): + return False + + # This AST destructuring and constant extraction must match the format + # of the argument lists produced by the quasiquote system for calls to + # `lookup_macro`. + key_node = tree.args[0] # the call is `lookup_macro(key)` + name_node, unique_name_node, frozen_macro_node = key_node.elts + assert type(name_node) is ast.Constant and type(name_node.value) is str + assert type(unique_name_node) is ast.Constant and type(unique_name_node.value) is str + assert type(frozen_macro_node) is ast.Constant and type(frozen_macro_node.value) is bytes + return (name_node.value, unique_name_node.value, frozen_macro_node.value) + + +def _is_mcpyrate_quotes_reference(tree): + """Detect whether `tree` is a reference to `mcpyrate.quotes`. + + This matches the ASTs corresponding to: + - direct reference: `mcpyrate.quotes` + - reference by import: `__import__("mcpyrate.quotes", ...).quotes` + + Note `__import__` of a dotted module name returns the top-level module, + so we have the name `quotes` appear twice in different places. + + See `_mcpyrate_quotes_attr` and `mcpyrate.coreutils._mcpyrate_attr`. + """ + if not (type(tree) is ast.Attribute and tree.attr == "quotes"): + return False + moduleref = tree.value + if type(moduleref) is ast.Name and moduleref.id == "mcpyrate": + return "direct" # ok, direct reference + elif (type(moduleref) is ast.Call and type(moduleref.func) is ast.Name and + moduleref.func.id == "__import__" and type(moduleref.args[0]) is ast.Constant and + moduleref.args[0].value == "mcpyrate.quotes"): + return "import" # ok, reference by import + else: + return False + + # -------------------------------------------------------------------------------- # The quasiquote compiler and uncompiler. @@ -443,7 +714,7 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr` # Builtin types. Mainly support for `u[]`, but also used by the # general case for AST node fields that contain bare values. - elif T in (int, float, str, bytes, bool, type(None)): + elif T in (int, float, str, bytes, bool, type(None), type(...)): return ast.Constant(value=x) elif T is list: @@ -461,6 +732,7 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr` # (Note we support only exactly `Done`, not arbitrary descendants.) elif T is Done: fields = [ast.keyword(a, recurse(b)) for a, b in ast.iter_fields(x)] + # We have imported `Done`, so we can refer to it as `mcpyrate.quotes.Done`. node = ast.Call(_mcpyrate_quotes_attr("Done"), [], fields) @@ -682,8 +954,11 @@ def _replace_tree_in_macro_invocation(invocation, newtree): """ new_invocation = copy.copy(invocation) if type(new_invocation) is ast.Subscript: - new_invocation.slice = copy.copy(invocation.slice) - new_invocation.slice.value = newtree + if sys.version_info >= (3, 9, 0): # Python 3.9+: no ast.Index wrapper + new_invocation.slice = newtree + else: + new_invocation.slice = copy.copy(invocation.slice) + new_invocation.slice.value = newtree elif type(new_invocation) is ast.With: new_invocation.body = newtree else: diff --git a/kibot/mcpyrate/repl/console.py b/kibot/mcpyrate/repl/console.py index 6dc0abc4..e3c5cc76 100644 --- a/kibot/mcpyrate/repl/console.py +++ b/kibot/mcpyrate/repl/console.py @@ -19,9 +19,12 @@ __all__ = ["MacroConsole"] import ast import code +import copy +import sys import textwrap from .. import __version__ as mcpyrate_version +from ..compiler import create_module from ..core import MacroExpansionError from ..debug import format_bindings from ..expander import find_macros, MacroExpander, global_postprocess @@ -31,6 +34,8 @@ from .utils import get_makemacro_sourcecode # Despite the meta-levels, there's just one global importer for the Python process. from .. import activate # noqa: F401 +_magic_module_name = "__mcpyrate_repl_self__" + class MacroConsole(code.InteractiveConsole): def __init__(self, locals=None, filename=""): """Parameters like in `code.InteractiveConsole`.""" @@ -40,7 +45,51 @@ class MacroConsole(code.InteractiveConsole): if locals is None: locals = {} # Lucky that both meta-levels speak the same language, eh? - locals['__macro_expander__'] = self.expander + locals["__macro_expander__"] = self.expander + + # support `from __self__ import macros, ...` + # + # To do this, we need the top-level variables in the REPL to be stored + # in the namespace of a module, so that `find_macros` can look them up. + # + # We create a module dynamically. The one obvious way would be to then + # replace the `__dict__` of that module with the given `locals`, but + # `types.ModuleType` is special in that the `__dict__` binding itself + # is read-only: + # https://docs.python.org/3/library/stdtypes.html#modules + # + # This leaves two possible solutions: + # + # a. Keep our magic module separate from `locals`, and shallow-copy + # the user namespace into its `__dict__` after each REPL input. + # The magic module's namespace is only used for macro lookups. + # + # + Behaves exactly like `code.InteractiveConsole` in that the + # user-given `locals` *is* the dict where the top-level variables + # are stored. So the user can keep a reference to that dict, and + # they will see any changes made by assigning to top-level + # variables in the REPL. + # + # - May slow down the REPL when a lot of top-level variables exist. + # Likely not a problem in practice. + # + # b. Use the module's `__dict__` as the local namespace of the REPL + # session. At startup, update it from `locals`, to install any + # user-provided initial variable bindings for the session. + # + # + Elegant. No extra copying. + # + # - Behavior does not match `code.InteractiveConsole`. The + # `locals` argument only provides initial values; any updates + # made during the REPL session are stored in the module's + # `__dict__`, which is a different dict instance. Hence the user + # will not see any changes made by assigning to top-level + # variables in the REPL. + # + # We currently use strategy a., for least astonishment. + # + magic_module = create_module(dotted_name=_magic_module_name, filename=filename) + self.magic_module_metadata = copy.copy(magic_module.__dict__) super().__init__(locals, filename) @@ -61,7 +110,7 @@ class MacroConsole(code.InteractiveConsole): This bypasses `runsource`, so it too can use this function. """ source = textwrap.dedent(source) - tree = ast.parse(source) + tree = ast.parse(source, self.filename) tree = ast.Interactive(tree.body) code = compile(tree, "", "single", self.compile.compiler.flags, 1) self.runcode(code) @@ -84,12 +133,13 @@ class MacroConsole(code.InteractiveConsole): def runsource(self, source, filename="", symbol="single"): # Special REPL commands. if source == "macros?": - self.write(format_bindings(self.expander)) + self.write(format_bindings(self.expander, color=True)) return False # complete input elif source.endswith("??"): - return self.runsource(f'mcpyrate.repl.utils.sourcecode({source[:-2]})') + # Use `_internal_execute` instead of `runsource` to prevent expansion of name macros. + return self._internal_execute(f"mcpyrate.repl.utils.sourcecode({source[:-2]})") elif source.endswith("?"): - return self.runsource(f"mcpyrate.repl.utils.doc({source[:-1]})") + return self._internal_execute(f"mcpyrate.repl.utils.doc({source[:-1]})") try: code = self.compile(source, filename, symbol) @@ -100,9 +150,18 @@ class MacroConsole(code.InteractiveConsole): try: # TODO: If we want to support dialects in the REPL, this is where to do it. - tree = ast.parse(source) + # Look at `mcpyrate.compiler.compile`. + tree = ast.parse(source, self.filename) - bindings = find_macros(tree, filename=self.expander.filename, reload=True) # macro-imports (this will import the modules) + # macro-imports (this will import the modules) + sys.modules[_magic_module_name].__dict__.clear() + sys.modules[_magic_module_name].__dict__.update(self.locals) # for self-macro-imports + # We treat the initial magic module metadata as write-protected: even if the user + # defines a variable of the same name in the user namespace, the metadata fields + # in the magic module won't be overwritten. + sys.modules[_magic_module_name].__dict__.update(self.magic_module_metadata) + bindings = find_macros(tree, filename=self.expander.filename, + reload=True, self_module=_magic_module_name) if bindings: self._macro_bindings_changed = True self.expander.bindings.update(bindings) @@ -118,6 +177,7 @@ class MacroConsole(code.InteractiveConsole): # In this case, the standard stack trace is long and points only to our code and the stdlib, # not the erroneous input that's the actual culprit. Better ignore it, and emulate showsyntaxerror. # TODO: support sys.excepthook. + # TODO: Look at `code.InteractiveConsole.showsyntaxerror` for how to do that. self.write(f"{err.__class__.__name__}: {str(err)}\n") return False # erroneous input except ImportError as err: # during macro lookup in a successfully imported module @@ -139,6 +199,10 @@ class MacroConsole(code.InteractiveConsole): self._macro_bindings_changed = False for asname, function in self.expander.bindings.items(): + # Catch broken bindings due to erroneous imports in user code + # (e.g. accidentally to a module object instead of to a function object) + if not (hasattr(function, "__module__") and hasattr(function, "__qualname__")): + continue if not function.__module__: # Macros defined in the REPL have `__module__=None`. continue try: diff --git a/kibot/mcpyrate/repl/iconsole.py b/kibot/mcpyrate/repl/iconsole.py index 79733e7e..b744c368 100644 --- a/kibot/mcpyrate/repl/iconsole.py +++ b/kibot/mcpyrate/repl/iconsole.py @@ -22,7 +22,9 @@ line magic. """ import ast +import copy from functools import partial +import sys from IPython.core import magic_arguments from IPython.core.error import InputRejected @@ -30,6 +32,7 @@ from IPython.core.magic import Magics, magics_class, cell_magic, line_magic from .. import __version__ as mcpyrate_version from ..astdumper import dump +from ..compiler import create_module from ..debug import format_bindings from ..expander import find_macros, MacroExpander, global_postprocess from .utils import get_makemacro_sourcecode @@ -38,6 +41,7 @@ from .utils import get_makemacro_sourcecode # Despite the meta-levels, there's just one global importer for the Python process. from .. import activate # noqa: F401 +_magic_module_name = "__mcpyrate_repl_self__" _placeholder = "" _instance = None @@ -85,21 +89,21 @@ class AstMagics(Magics): @line_magic def macros(self, line): """Print a human-readable list of macros currently imported into the session.""" - # Line magics print `\n\n` at the end automatically, so remove our final `\n`. - print(format_bindings(_instance.macro_transformer.expander).rstrip()) + # Line magics print an extra `\n` at the end automatically, so remove our final `\n`. + print(format_bindings(_instance.macro_transformer.expander, color=True), end="") # I don't know if this is useful - one can use the `mcpyrate.debug.step_expansion` # macro also in the REPL - but let's put it in for now. # http://alexleone.blogspot.co.uk/2010/01/python-ast-pretty-printer.html @magic_arguments.magic_arguments() @magic_arguments.argument( - '-m', '--mode', default='exec', + "-m", "--mode", default="exec", help="The mode in which to parse the code. Can be exec (default), " "eval or single." ) # TODO: add support for expand-once @magic_arguments.argument( - '-e', '--expand', default='no', + "-e", "--expand", default="no", help="Whether to expand macros before dumping the AST. Can be yes " "or no (default)." ) @@ -107,7 +111,7 @@ class AstMagics(Magics): def dump_ast(self, line, cell): """Parse the code in the cell, and pretty-print the AST.""" args = magic_arguments.parse_argstring(self.dump_ast, line) - tree = ast.parse(cell, mode=args.mode) + tree = ast.parse(cell, filename=_placeholder, mode=args.mode) if args.expand != "no": tree = _instance.macro_transformer.visit(tree) print(dump(tree)) @@ -119,18 +123,27 @@ class InteractiveMacroTransformer(ast.NodeTransformer): def __init__(self, extension_instance, *args, **kwargs): super().__init__(*args, **kwargs) self._ipyextension = extension_instance - self.expander = MacroExpander(bindings={}, filename="") + self.expander = MacroExpander(bindings={}, filename=_placeholder) def visit(self, tree): try: - bindings = find_macros(tree, filename=self.expander.filename, reload=True) # macro-imports (this will import the modules) + sys.modules[_magic_module_name].__dict__.clear() + sys.modules[_magic_module_name].__dict__.update(self._ipyextension.shell.user_ns) # for self-macro-imports + # We treat the initial magic module metadata as write-protected: even if the user + # defines a variable of the same name in the user namespace, the metadata fields + # in the magic module won't be overwritten. (IPython actually defines e.g. `__name__` + # in `user_ns` even if the user explicitly doesn't.) + sys.modules[_magic_module_name].__dict__.update(self._ipyextension.magic_module_metadata) + # macro-imports (this will import the modules) + bindings = find_macros(tree, filename=self.expander.filename, + reload=True, self_module=_magic_module_name) if bindings: self._ipyextension._macro_bindings_changed = True self.expander.bindings.update(bindings) - newtree = self.expander.visit(tree) - newtree = global_postprocess(newtree) + new_tree = self.expander.visit(tree) + new_tree = global_postprocess(new_tree) self._ipyextension.src = _placeholder - return newtree + return new_tree except Exception as err: # see IPython.core.interactiveshell.InteractiveShell.transform_ast() raise InputRejected(*err.args) @@ -149,7 +162,11 @@ class IMcpyrateExtension: self.macro_transformer = InteractiveMacroTransformer(extension_instance=self) self.shell.ast_transformers.append(self.macro_transformer) # TODO: last or first? # Lucky that both meta-levels speak the same language, eh? - shell.user_ns['__macro_expander__'] = self.macro_transformer.expander + shell.user_ns["__macro_expander__"] = self.macro_transformer.expander + + # support `from __self__ import macros, ...` + magic_module = create_module(dotted_name=_magic_module_name, filename=_placeholder) + self.magic_module_metadata = copy.copy(magic_module.__dict__) self.shell.run_cell(get_makemacro_sourcecode(), store_history=False, @@ -158,15 +175,15 @@ class IMcpyrateExtension: # TODO: If we want to support dialects in the REPL, we need to install # a string transformer here to call the dialect system's source transformer, # and then modify `InteractiveMacroTransformer` to run the dialect system's - # AST transformer before it runs the macro expander. + # AST transformer before it runs the macro expander. Look at `mcpyrate.compiler.compile`. ipy = self.shell.get_ipython() - ipy.events.register('post_run_cell', self._refresh_macro_functions) + ipy.events.register("post_run_cell", self._refresh_macro_functions) def __del__(self): ipy = self.shell.get_ipython() - ipy.events.unregister('post_run_cell', self._refresh_macro_functions) - del self.shell.user_ns['__macro_expander__'] + ipy.events.unregister("post_run_cell", self._refresh_macro_functions) + del self.shell.user_ns["__macro_expander__"] self.shell.ast_transformers.remove(self.macro_transformer) self.shell.input_transformers_post.remove(self._get_source_code) @@ -194,7 +211,11 @@ class IMcpyrateExtension: silent=True) for asname, function in self.macro_transformer.expander.bindings.items(): - if not function.__module__: + # Catch broken bindings due to erroneous imports in user code + # (e.g. accidentally to a module object instead of to a function object) + if not (hasattr(function, "__module__") and hasattr(function, "__qualname__")): + continue + if not function.__module__: # Macros defined in the REPL have `__module__=None`. continue commands = ["%%ignore_importerror", f"from {function.__module__} import {function.__qualname__} as {asname}"] diff --git a/kibot/mcpyrate/repl/macropython.py b/kibot/mcpyrate/repl/macropython.py index 7cd07028..87a0973e 100644 --- a/kibot/mcpyrate/repl/macropython.py +++ b/kibot/mcpyrate/repl/macropython.py @@ -6,21 +6,25 @@ import argparse import atexit -from importlib import import_module -from importlib.util import resolve_name, module_from_spec import os import pathlib import sys +from importlib import import_module +from importlib.util import module_from_spec, resolve_name -from ..coreutils import relativize - +from .. import __version__ as __mcpyrate_version__ from .. import activate # noqa: F401 +from ..core import MacroApplicationError +from ..coreutils import relativize +from ..pycachecleaner import deletepycachedirs, getpycachedirs -__version__ = "3.0.0" +__version__ = "3.1.0" _config_dir = "~/.config/mcpyrate" _macropython_module = None # sys.modules doesn't always seem to keep it, so stash it locally too. +# -------------------------------------------------------------------------------- + def import_module_as_main(name, script_mode): """Import a module, pretending it's __main__. @@ -98,8 +102,19 @@ def import_module_as_main(name, script_mode): sys.modules["__main__"] = module # replace this bootstrapper with the new __main__ try: spec.loader.exec_module(module) - except Exception: + except Exception as err: sys.modules["__main__"] = _macropython_module + if isinstance(err, MacroApplicationError): + # To avoid noise, discard most of the traceback of the chained + # macro-expansion errors emitted by the expander core. The + # linked (`__cause__`) exceptions have the actual tracebacks. + # + # Keep just the last entry, which should state that this + # exception came from `expand` in `core.py`. + tb = err.__traceback__ + while tb.tb_next: + tb = tb.tb_next + raise err.with_traceback(tb) raise # # __main__ has no parent module so we don't need to do this. # if path is not None: @@ -128,7 +143,7 @@ def main(): parser = argparse.ArgumentParser(description="""Run a Python program or an interactive interpreter with mcpyrate enabled.""", formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('-v', '--version', action='version', version=('%(prog)s (mcpyrate ' + __version__ + ')')) + parser.add_argument('-v', '--version', action='version', version=('%(prog)s ' + __version__ + ' (mcpyrate ' + __mcpyrate_version__ + ')')) parser.add_argument(dest='filename', nargs='?', default=None, type=str, metavar='file', help='script to run') parser.add_argument('-m', '--module', dest='module', default=None, type=str, metavar='mod', @@ -140,16 +155,37 @@ def main(): help='For use together with "-i". Automatically "import numpy as np", ' '"import matplotlib.pyplot as plt", and enable mpl\'s interactive ' 'mode (somewhat like IPython\'s pylab mode).') + parser.add_argument('-c', '--clean', dest='path_to_clean', default=None, type=str, metavar='dir', + help='Delete Python bytecode (`.pyc`) caches inside given directory, recursively, ' + 'and then exit. This removes the `__pycache__` directories, too. ' + 'The purpose is to facilitate testing `mcpyrate` and programs that use it, ' + 'because the expander only runs when the `.pyc` cache for the module being ' + 'imported is out of date or does not exist. For any given module, once the ' + 'expander is done, Python\'s import system automatically writes the `.pyc` ' + 'cache for that module.') + parser.add_argument("-n", "--dry-run", dest="dry_run", action="store_true", default=False, + help='For use together with "-c". Just scan for and print the .pyc cache ' + 'directory paths, don\'t actually clean them.') opts = parser.parse_args() + if opts.path_to_clean: + # If an error occurs during cleaning, we just let it produce a standard stack trace. + if not opts.dry_run: + deletepycachedirs(opts.path_to_clean) + else: + for x in getpycachedirs(opts.path_to_clean): + print(x) + sys.exit(0) # only reached if cleaning (or dry run) successful + if opts.interactive: - from .console import MacroConsole import readline # noqa: F401, side effect: enable GNU readline in input() import rlcompleter # noqa: F401, side effects: readline tab completion + + from .console import MacroConsole repl_locals = {} if opts.pylab: # like IPython's pylab mode, but we keep things in separate namespaces. - import numpy import matplotlib.pyplot + import numpy repl_locals["np"] = numpy repl_locals["plt"] = matplotlib.pyplot matplotlib.pyplot.ion() @@ -206,6 +242,13 @@ def main(): if sys.path[0] != "": sys.path.insert(0, "") + # https://www.python.org/dev/peps/pep-0582/ + if sys.version_info >= (3, 8, 0): + local_pypackages_dir = pathlib.Path.cwd().expanduser().resolve() / "__pypackages__" + if local_pypackages_dir.is_dir(): + # TODO: figure out correct insert index (spec: "after cwd, just before site packages") + sys.path.insert(1, str(local_pypackages_dir)) + import_module_as_main(opts.module, script_mode=False) else: # no module, so use opts.filename @@ -217,6 +260,13 @@ def main(): if sys.path[0] != containing_directory: sys.path.insert(0, containing_directory) + # https://www.python.org/dev/peps/pep-0582/ + if sys.version_info >= (3, 8, 0): + local_pypackages_dir = fullpath.parent / "__pypackages__" + if local_pypackages_dir.is_dir(): + # TODO: figure out correct insert index (spec: "after cwd, just before site packages") + sys.path.insert(1, str(local_pypackages_dir)) + # This approach finds the standard loader even for macro-enabled scripts. # TODO: For mcpyrate, that could be ok, since we monkey-patch just that. # TODO: But maybe better to leave the option to replace the whole loader later. @@ -237,6 +287,6 @@ def main(): import_module_as_main(module_name, script_mode=True) -if __name__ == '__main__': +if __name__ == "__main__": _macropython_module = sys.modules["__macropython__"] = sys.modules["__main__"] main() diff --git a/kibot/mcpyrate/repl/utils.py b/kibot/mcpyrate/repl/utils.py index 3228e967..5f5fb7f9 100644 --- a/kibot/mcpyrate/repl/utils.py +++ b/kibot/mcpyrate/repl/utils.py @@ -1,41 +1,81 @@ # -*- coding: utf-8; -*- -'''Utilities for building REPLs.''' +"""Utilities for building REPLs.""" __all__ = ["doc", "sourcecode", "get_makemacro_sourcecode"] +from functools import partial import inspect +import sys +import textwrap -def doc(obj): - """Print an object's docstring, non-interactively. +from ..colorizer import colorize, ColorScheme + + +def _get_source(obj): + # `inspect.getsourcefile` accepts "a module, class, method, function, + # traceback, frame, or code object" (the error message says this if + # we try it on something invalid). + # + # So if `obj` is an instance, we need to try again with its `__class__`. + for x in (obj, obj.__class__): # TODO: other places to fall back to? + try: + filename = inspect.getsourcefile(x) + source, firstlineno = inspect.getsourcelines(x) + return filename, source, firstlineno + except (TypeError, OSError): + continue + raise NotImplementedError + + +def doc(obj, *, file=None, end="\n"): + """Print an object's docstring non-interactively. If available, print also the filename and the starting line number of the definition of `obj`. + + The default is to print to `sys.stderr` (resolved at call time, to respect + possible overrides); to override, use the `file` parameter, which works + like in the builtin `print`. + + The `end` parameter is passed to `print` as-is. """ + file = file or sys.stderr + printer = partial(print, file=file, end=end) try: - filename = inspect.getsourcefile(obj) - source, firstlineno = inspect.getsourcelines(obj) - print(f"{filename}:{firstlineno}") - except (TypeError, OSError): + filename, source, firstlineno = _get_source(obj) + printer(colorize(f"{filename}:{firstlineno}", ColorScheme.SOURCEFILENAME)) + except NotImplementedError: pass if not hasattr(obj, "__doc__") or not obj.__doc__: - print("") + printer(colorize("", ColorScheme.GREYEDOUT)) return - print(inspect.cleandoc(obj.__doc__)) + print(inspect.cleandoc(obj.__doc__), file=file) -def sourcecode(obj): - """Print an object's source code, non-interactively. + +def sourcecode(obj, *, file=None, end=None): + """Print an object's source code to `sys.stderr`, non-interactively. If available, print also the filename and the starting line number of the definition of `obj`. + + Default is to print to `sys.stderr` (resolved at call time, to respect + possible overrides); to override, use the `file` argument, which works + like in the builtin `print`. + + The `end` parameter is passed to `print` as-is. """ + file = file or sys.stderr + printer = partial(print, file=file, end=end) try: - filename = inspect.getsourcefile(obj) - source, firstlineno = inspect.getsourcelines(obj) - print(f"{filename}:{firstlineno}") + filename, source, firstlineno = _get_source(obj) + printer(colorize(f"{filename}:{firstlineno}", ColorScheme.SOURCEFILENAME)) + # TODO: No syntax highlighting for now, because we'd have to parse and unparse, + # TODO: which loses the original formatting and comments. for line in source: - print(line.rstrip("\n")) - except (TypeError, OSError): - print("") + printer(line.rstrip("\n")) + except NotImplementedError: + printer(colorize("", ColorScheme.GREYEDOUT)) + def get_makemacro_sourcecode(): """Return source code for the REPL's `macro` magic function. @@ -45,13 +85,13 @@ def get_makemacro_sourcecode(): We assume the expander instance has been bound to the global variable `__macro_expander__` inside the REPL session. """ - return """ + return textwrap.dedent(''' def macro(function): - '''[mcpyrate] `macro(f)`: bind function `f` as a macro. Works also as a decorator. REPL only.''' + """[mcpyrate] `macro(f)`: bind function `f` as a macro. Works also as a decorator. REPL only.""" if not callable(function): - raise TypeError(f'`function` must be callable, got {type(function)} with value {repr(function)}') - if function.__name__ == '': - raise TypeError('`function` must be a named function, got a lambda.') + raise TypeError(f"`function` must be callable, got {type(function)} with value {repr(function)}") + if function.__name__ == "": + raise TypeError("`function` must be a named function, got a lambda.") __macro_expander__.bindings[function.__name__] = function return function - """ + ''') diff --git a/kibot/mcpyrate/splicing.py b/kibot/mcpyrate/splicing.py index 35d855ca..06c28fe5 100644 --- a/kibot/mcpyrate/splicing.py +++ b/kibot/mcpyrate/splicing.py @@ -46,7 +46,6 @@ def splice_expression(expr, template, tag="__paste_here__"): Returns `template` with `expr` spliced in. Note `template` is **not** copied, and will be mutated in-place. - """ if not template: return expr @@ -58,7 +57,7 @@ def splice_expression(expr, template, tag="__paste_here__"): raise TypeError(f"`template` must be an AST or `list`; got {type(template)} with value {repr(template)}") def ispastehere(tree): - return type(tree) is ast.Expr and type(tree.value) is ast.Name and tree.value.id == tag + return type(tree) is ast.Name and tree.id == tag class ExpressionSplicer(ASTTransformer): def __init__(self): diff --git a/kibot/mcpyrate/unparser.py b/kibot/mcpyrate/unparser.py index bd43b00c..2b607dc0 100644 --- a/kibot/mcpyrate/unparser.py +++ b/kibot/mcpyrate/unparser.py @@ -1,17 +1,22 @@ # -*- coding: utf-8; -*- -"""Back-convert a Python AST into source code. Original formatting is disregarded.""" +"""Back-convert a Python AST into source code. Original formatting is disregarded. + +Python 3.9+ provides `ast.unparse`, but ours comes with some additional features, +notably syntax highlighting and debug rendering of invisible AST nodes. +""" __all__ = ["UnparserError", "unparse", "unparse_with_fallbacks"] import ast import builtins -from contextlib import contextmanager import io import sys +from contextlib import contextmanager -from .astdumper import dump # fallback -from .colorizer import setcolor, colorize, ColorScheme from . import markers +from .astdumper import dump # fallback +from .colorizer import ColorScheme, colorize, setcolor +quotes = None # HACK: avoid circular import # Large float and imaginary literals get turned into infinities in the AST. # We unparse those infinities to INFSTR. @@ -78,6 +83,10 @@ class Unparser: `expander`: optional `BaseMacroExpander` instance. If provided, used for syntax highlighting macro names. """ + # HACK: avoid circular import + global quotes + from . import quotes + self.debug = debug self.color = color self._color_override = False # for syntax highlighting of decorators @@ -162,6 +171,12 @@ class Unparser: for t in tree: self.dispatch(t) return + if self.debug and quotes.is_captured_value(tree): + self.captured_value(tree) + return + if self.debug and quotes.is_captured_macro(tree): + self.captured_macro(tree) + return if isinstance(tree, markers.ASTMarker): # mcpyrate and macro communication internal self.astmarker(tree) return @@ -184,7 +199,10 @@ class Unparser: def astmarker(self, tree): def write_astmarker_field_value(v): - if isinstance(v, ast.AST): + if isinstance(v, list): # statement suite + for item in v: + self.dispatch(item) + elif isinstance(v, ast.AST): self.dispatch(v) else: self.write(repr(v)) @@ -194,8 +212,8 @@ class Unparser: # that "source code" containing AST markers cannot be eval'd. # If you need to get rid of them, see `mcpyrate.markers.delete_markers`. - header = self.maybe_colorize(f"$ASTMarker", ColorScheme.ASTMARKER) - if isinstance(tree.body, ast.stmt): + header = self.maybe_colorize("$ASTMarker", ColorScheme.ASTMARKER) + if isinstance(tree.body, (ast.stmt, list)): print_mode = "stmt" self.fill(header, lineno_node=tree) else: @@ -242,6 +260,20 @@ class Unparser: if print_mode == "expr": self.write(")") + def captured_value(self, t): # hygienic capture; output of `mcpyrate.quotes.h`; only emitted in debug mode + name, _ignored_value = quotes.is_captured_value(t) + self.write(self.maybe_colorize("$h", ColorScheme.ASTMARKER)) + self.write(self.maybe_colorize("[", ColorScheme.ASTMARKER)) + self.write(name) + self.write(self.maybe_colorize("]", ColorScheme.ASTMARKER)) + + def captured_macro(self, t): # hygienic capture; output of `mcpyrate.quotes.h`; only emitted in debug mode + name, _ignored_unique_name, _ignored_value = quotes.is_captured_macro(t) + self.write(self.maybe_colorize("$h", ColorScheme.ASTMARKER)) + self.write(self.maybe_colorize("[", ColorScheme.ASTMARKER)) + self.write(self.maybe_colorize(name, ColorScheme.MACRONAME)) + self.write(self.maybe_colorize("]", ColorScheme.ASTMARKER)) + # top level nodes def _Module(self, t): # ast.parse(..., mode="exec") self.toplevelnode(t) @@ -584,6 +616,8 @@ class Unparser: v = self.maybe_colorize(v, ColorScheme.NAMECONSTANT) elif type(t.value) in (str, bytes): v = self.maybe_colorize(v, ColorScheme.STRING) + else: # pragma: no cover + raise UnparserError(f"Don't know how to unparse Constant with value of type {type(t.value)}, got {repr(t.value)}") self.write(v) def _Bytes(self, t): # up to Python 3.7 @@ -681,25 +715,33 @@ class Unparser: interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values)) self.write("}") + # Python 3.9+: we must emit the parentheses separate from the main logic, + # because a Tuple directly inside a Subscript slice should be rendered + # without parentheses; so our `_Subscript` method special-cases that. + # This is important, because the notation `a[1,2:5]` is fine, but + # `a[(1,2:5)]` is a syntax error. See https://bugs.python.org/issue34822 def _Tuple(self, t): self.write("(") + self.__Tuple_helper(t) + self.write(")") + + def __Tuple_helper(self, t): if len(t.elts) == 1: (elt,) = t.elts self.dispatch(elt) self.write(",") else: interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write(")") unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} def _UnaryOp(self, t): self.write("(") - # if it's an English keyword, highlight it. + # If it's an English keyword, highlight it, and add a space. if t.op.__class__.__name__ == "Not": self.write(self.maybe_colorize_python_keyword(self.unop[t.op.__class__.__name__])) + self.write(" ") else: self.write(self.unop[t.op.__class__.__name__]) - self.write(" ") self.dispatch(t.operand) self.write(")") @@ -815,7 +857,12 @@ class Unparser: def _Subscript(self, t): self.dispatch(t.value) self.write("[") - self.dispatch(t.slice) + # Python 3.9+: Omit parentheses for a tuple directly inside a Subscript slice. + # See https://bugs.python.org/issue34822 + if type(t.slice) is ast.Tuple: + self.__Tuple_helper(t.slice) + else: + self.dispatch(t.slice) self.write("]") def _Starred(self, t): @@ -826,7 +873,7 @@ class Unparser: def _Ellipsis(self, t): # up to Python 3.7 self.write("...") - def _Index(self, t): + def _Index(self, t): # up to Python 3.8; the Index wrapper is gone in Python 3.9 self.dispatch(t.value) def _Slice(self, t): @@ -839,7 +886,7 @@ class Unparser: self.write(":") self.dispatch(t.step) - def _ExtSlice(self, t): + def _ExtSlice(self, t): # up to Python 3.8; Python 3.9 uses a Tuple instead interleave(lambda: self.write(", "), self.dispatch, t.dims) # argument diff --git a/kibot/mcpyrate/utils.py b/kibot/mcpyrate/utils.py index e78b66dd..688cf91d 100644 --- a/kibot/mcpyrate/utils.py +++ b/kibot/mcpyrate/utils.py @@ -216,6 +216,10 @@ def format_location(filename, tree, sourcecode): def format_macrofunction(function): """Format the fully qualified name of a macro function, for error messages.""" + # Catch broken bindings due to erroneous imports in user code + # (e.g. accidentally to a module object instead of to a function object) + if not (hasattr(function, "__module__") and hasattr(function, "__qualname__")): + return repr(function) if not function.__module__: # Macros defined in the REPL have `__module__=None`. return function.__qualname__ return f"{function.__module__}.{function.__qualname__}" diff --git a/kibot/mcpyrate/walkers.py b/kibot/mcpyrate/walkers.py index e413e48b..248653f2 100644 --- a/kibot/mcpyrate/walkers.py +++ b/kibot/mcpyrate/walkers.py @@ -1,9 +1,42 @@ # -*- coding: utf-8; -*- +"""AST walkers. + +These have a state stack and a node collector, and accept also a `list` of statements. +Otherwise they work like `ast.NodeVisitor` and `ast.NodeTransformer`. + +Basic usage summary:: + + def kittify(mytree): + class Kittifier(ASTTransformer): + def transform(self, tree): + if type(tree) is ast.Constant: + self.collect(tree.value) + tree.value = "meow!" if self.state.meows % 2 == 0 else "miaow!" + self.state.meows += 1 + return self.generic_visit(tree) # recurse + w = Kittifier(meows=0) # set the initial state here + mytree = w.visit(mytree) # it's basically an ast.NodeTransformer + print(w.collected) # collected values, in the order visited + return mytree + + def getmeows(mytree): + class MeowCollector(ASTVisitor): + def examine(self, tree): + if type(tree) is ast.Constant and tree.value in ("meow!", "miaow!"): + self.collect(tree) + self.generic_visit(tree) + w = MeowCollector() + w.visit(mytree) + print(w.collected) + return w.collected + +For an example of `withstate`, see `mcpyrate.astfixers`. +""" __all__ = ["ASTVisitor", "ASTTransformer"] from abc import ABCMeta, abstractmethod -from ast import NodeVisitor, NodeTransformer +from ast import NodeVisitor, NodeTransformer, iter_child_nodes from .bunch import Bunch from . import utils @@ -34,6 +67,19 @@ class BaseASTWalker: `tree` can be an AST node or a statement suite (`list` of AST nodes). It is identified by `id(tree)` at enter time. Bindings update a copy of `self.state`. + + If several `withstate` calls are made for the same `tree`, the last one + overrides. + + Generally speaking: + + `withstate(subtree, ...)` should be used if you then intend to + `visit(subtree)`, which recurses into that node (or suite) only. + + `generic_withstate(tree, ...)` should be used if you then intend to + `generic_visit(tree)`, which recurses into the children of `tree`. + + (It is possible to mix and match, but think through what you're doing.) """ newstate = self.state.copy() newstate.update(**bindings) @@ -47,6 +93,33 @@ class BaseASTWalker: else: self._subtree_overrides[id(tree)] = newstate + def generic_withstate(self, tree, **bindings): + """Like `withstate`, but set up the new state for all children of `tree`. + + The same state instance is shared between the child nodes. + + If several `generic_withstate` calls are made for the same `tree`, the + last one overrides (assuming the list of children has not changed in between). + + The silly name is because this relates to `withstate` as `generic_visit` + relates to `visit`. + + Generally speaking: + + `generic_withstate(tree, ...)` should be used if you then intend to + `generic_visit(tree)`, which recurses into the children of `tree`. + + `withstate(subtree, ...)` should be used if you then intend to + `visit(subtree)`, which recurses into that node (or suite) only. + + (It is possible to mix and match, but think through what you're doing.) + """ + newstate = self.state.copy() + newstate.update(**bindings) + + for node in iter_child_nodes(tree): + self._subtree_overrides[id(node)] = newstate + def collect(self, value): """Collect a value. The values are placed in the list `self.collected`.""" self.collected.append(value) @@ -111,10 +184,10 @@ class ASTTransformer(BaseASTWalker, NodeTransformer, metaclass=ABCMeta): try: if isinstance(tree, list): new_tree = utils.flatten(self.visit(elt) for elt in tree) - if new_tree: - tree[:] = new_tree - return tree - return None + if not new_tree: + new_tree = [] # preserve the type of `tree`; an empty list shouldn't turn into `None` + tree[:] = new_tree + return tree return self.transform(tree) finally: if newstate: