diff --git a/kibot/mcpyrate/activate.py b/kibot/mcpyrate/activate.py index 1ffe2612..acfa440c 100644 --- a/kibot/mcpyrate/activate.py +++ b/kibot/mcpyrate/activate.py @@ -25,24 +25,27 @@ the `PYTHONDONTWRITEBYTECODE` environment variable, and the attribute https://www.python.org/dev/peps/pep-0552/ ''' +__all__ = ["activate", "deactivate"] + from importlib.machinery import SourceFileLoader, FileFinder from .importer import source_to_xcode, path_xstats, invalidate_xcaches def activate(): SourceFileLoader.source_to_code = source_to_xcode - # we could replace SourceFileLoader.set_data with a no-op to force-disable pyc caching. + # Bytecode caching (`.pyc`) support. If you need to force-disable `.pyc` + # caching, replace `SourceFileLoader.set_data` with a no-op, like `mcpy` does. SourceFileLoader.path_stats = path_xstats FileFinder.invalidate_caches = invalidate_xcaches -def de_activate(): - SourceFileLoader.source_to_code = old_source_to_code - SourceFileLoader.path_stats = old_path_stats - FileFinder.invalidate_caches = old_invalidate_caches +def deactivate(): + SourceFileLoader.source_to_code = stdlib_source_to_code + SourceFileLoader.path_stats = stdlib_path_stats + FileFinder.invalidate_caches = stdlib_invalidate_caches -old_source_to_code = SourceFileLoader.source_to_code -old_path_stats = SourceFileLoader.path_stats -old_invalidate_caches = FileFinder.invalidate_caches +stdlib_source_to_code = SourceFileLoader.source_to_code +stdlib_path_stats = SourceFileLoader.path_stats +stdlib_invalidate_caches = FileFinder.invalidate_caches activate() diff --git a/kibot/mcpyrate/astdumper.py b/kibot/mcpyrate/astdumper.py index 2e4a4d3d..9c75ac09 100644 --- a/kibot/mcpyrate/astdumper.py +++ b/kibot/mcpyrate/astdumper.py @@ -8,7 +8,11 @@ __all__ = ["dump"] from ast import AST, iter_fields -def dump(tree, *, include_attributes=False, multiline=True): +from .colorizer import colorize, ColorScheme + +NoneType = type(None) + +def dump(tree, *, include_attributes=False, multiline=True, color=False): """Return a formatted dump of `tree`, as a string. `tree` can be an AST node or a statement suite (`list` of AST nodes). @@ -21,26 +25,47 @@ def dump(tree, *, include_attributes=False, multiline=True): To put everything on one line, use `multiline=False`. + If you're printing the result into a terminal, consider `color=True`. + Similar to `macropy`'s `real_repr`, but with indentation. The method `ast.AST.__repr__` itself can't be monkey-patched, because `ast.AST` is a built-in/extension type. """ + def maybe_colorize(text, *colors): + if not color: + return text + return colorize(text, *colors) + + def maybe_colorize_value(value): + if type(value) in (str, bytes, NoneType, bool, int, float, complex): + # Pass through an already formatted list-as-a-string from an inner level. + if isinstance(value, str) and value.startswith("["): + return value + return maybe_colorize(str(value), ColorScheme.BAREVALUE) + return str(value) + def recurse(tree, previndent=0): def separator(): if multiline: return f",\n{(previndent + moreindent) * ' '}" return ", " + if isinstance(tree, AST): moreindent = len(f"{tree.__class__.__name__}(") fields = [(k, recurse(v, previndent + moreindent + len(f"{k}="))) for k, v in iter_fields(tree)] if include_attributes and tree._attributes: - fields.extend([(k, recurse(getattr(tree, k, None), previndent + moreindent + len(f"{k}="))) + fields.extend([(k, recurse(getattr(tree, k, None), + previndent + moreindent + len(f"{k}="))) for k in tree._attributes]) + colorized_fields = [(maybe_colorize(k, ColorScheme.FIELDNAME), + maybe_colorize_value(v)) + for k, v in fields] return ''.join([ - tree.__class__.__name__, + maybe_colorize(tree.__class__.__name__, ColorScheme.NODETYPE), '(', - separator().join((f'{k}={v}' for k, v in fields)), + separator().join((f'{k}={v}' for k, v in colorized_fields)), ')']) + elif isinstance(tree, list): moreindent = len("[") items = [recurse(elt, previndent + moreindent) for elt in tree] @@ -49,6 +74,7 @@ def dump(tree, *, include_attributes=False, multiline=True): items[-1] = items[-1] + ']' return separator().join(items) return '[]' + return repr(tree) if not isinstance(tree, (AST, list)): diff --git a/kibot/mcpyrate/astfixers.py b/kibot/mcpyrate/astfixers.py index 6ee9d222..3ff3bea0 100644 --- a/kibot/mcpyrate/astfixers.py +++ b/kibot/mcpyrate/astfixers.py @@ -1,7 +1,7 @@ # -*- coding: utf-8; -*- -'''Fix missing `ctx` attributes and source location info in an AST.''' +'''Fix `ctx` attributes and source location info in an AST.''' -__all__ = ['fix_missing_ctx', 'fix_missing_locations'] +__all__ = ['fix_ctx', 'fix_locations'] from ast import (Load, Store, Del, Assign, AnnAssign, AugAssign, @@ -32,8 +32,8 @@ class _CtxFixer(Walker): return self.generic_visit(tree) def _fix_one(self, tree): - '''Fix one missing `ctx` attribute, using the currently active ctx class.''' - if ("ctx" in type(tree)._fields and (not hasattr(tree, "ctx") or tree.ctx is None)): + '''Fix one `ctx` attribute, using the currently active ctx class.''' + if "ctx" in type(tree)._fields: tree.ctx = self.state.ctxclass() def _setup_subtree_contexts(self, tree): @@ -84,29 +84,30 @@ class _CtxFixer(Walker): self.withstate(tree.targets, ctxclass=Del) -def fix_missing_ctx(tree): - '''Fix any missing `ctx` attributes in `tree`. +def fix_ctx(tree): + '''Fix `ctx` attributes in `tree`. Modifies `tree` in-place. For convenience, returns the modified `tree`. ''' return _CtxFixer().visit(tree) -def fix_missing_locations(tree, reference_node, *, mode): +def fix_locations(tree, reference_node, *, mode): '''Like `ast.fix_missing_locations`, but customized for a macro expander. Differences: - - If `reference_node` has no location info, return immediately (no-op). + - If `reference_node` has source no location info, return immediately (no-op). - If `tree is None`, return immediately (no-op). - If `tree` is a `list` of AST nodes, loop over it. The `mode` parameter: - If `mode="reference"`, populate any missing location info by - copying it from `reference_node`. Always use the same values. + copying it from `reference_node`. Always use the same reference info. - Good for a macro expander. + Good when expanding a macro invocation, to set the source location + of any macro-generated nodes to that of the macro invocation node. - If `mode="update"`, behave exactly like `ast.fix_missing_locations`, except that at the top level of `tree`, initialize `lineno` and @@ -123,7 +124,7 @@ def fix_missing_locations(tree, reference_node, *, mode): Good when `tree` is a code template that comes from another file, so that any line numbers already in the AST would be misleading - at the use site. + at the use site (because they point to lines in that other file). Modifies `tree` in-place. For convenience, returns the modified `tree`. ''' diff --git a/kibot/mcpyrate/colorizer.py b/kibot/mcpyrate/colorizer.py new file mode 100644 index 00000000..b876292a --- /dev/null +++ b/kibot/mcpyrate/colorizer.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8; -*- +"""Colorize terminal output using Colorama.""" + +__all__ = ["setcolor", "colorize", "ColorScheme", + "Fore", "Back", "Style"] + +from colorama import init as colorama_init, Fore, Back, Style +colorama_init() + +def setcolor(*colors): + """Set color for ANSI terminal display. + + For available `colors`, see `Fore`, `Back` and `Style`. + + Each entry can also be a tuple (arbitrarily nested), which is useful + for defining compound styles. + """ + def _setcolor(color): + if isinstance(color, (list, tuple)): + return "".join(_setcolor(elt) for elt in color) + return color + return _setcolor(colors) + +def colorize(text, *colors, reset=True): + """Colorize string `text` for ANSI terminal display. Reset color at end of `text`. + + For available `colors`, see `Fore`, `Back` and `Style`. + These are imported from `colorama`. + + Usage:: + + colorize("I'm new here", Fore.GREEN) + colorize("I'm bold and bluetiful", Style.BRIGHT, Fore.BLUE) + + Each entry can also be a tuple (arbitrarily nested), which is useful + for defining compound styles:: + + BRIGHT_BLUE = (Style.BRIGHT, Fore.BLUE) + ... + colorize("I'm bold and bluetiful, too", BRIGHT_BLUE) + + **CAUTION**: Does not nest. Style resets after the colorized text. + """ + return "{}{}{}".format(setcolor(colors), + text, + setcolor(Style.RESET_ALL)) + + +class ColorScheme: + """The color scheme for debug utilities. + + See `Fore`, `Back`, `Style` in `colorama` for valid values. To make a + compound style, place the values into a tuple. + + The defaults are designed to fit the "Solarized" (Zenburn-like) theme + of `gnome-terminal`, with "Show bold text in bright colors" set to OFF. + But they should work with most color schemes. + """ + _RESET = Style.RESET_ALL + + # ------------------------------------------------------------ + # unparse + + LINENUMBER = Style.DIM + + LANGUAGEKEYWORD = (Style.BRIGHT, Fore.YELLOW) # for, if, import, ... + + DEFNAME = (Style.BRIGHT, Fore.CYAN) # name of a function or class being defined + DECORATOR = Fore.LIGHTBLUE_EX + + STRING = Fore.GREEN + NUMBER = Fore.GREEN + NAMECONSTANT = Fore.GREEN # True, False, None + + BUILTINEXCEPTION = Fore.CYAN + BUILTINOTHER = Style.BRIGHT # str, property, print, ... + + INVISIBLENODE = Style.DIM # AST node with no surface syntax repr (Module, Expr) + + # AST markers for data-driven communication within the macro expander + ASTMARKER = Style.DIM # the "$AstMarker" title + ASTMARKERCLASS = Fore.YELLOW # the actual marker type name + + # ------------------------------------------------------------ + # format_bindings, step_expansion, StepExpansion + + HEADING = (Style.BRIGHT, Fore.LIGHTBLUE_EX) + SOURCEFILENAME = (Style.BRIGHT, Fore.RESET) + + # format_bindings + MACROBINDING = (Style.BRIGHT, Fore.RESET) + GREYEDOUT = Style.DIM + + # step_expansion + TREEID = (Style.NORMAL, Fore.LIGHTBLUE_EX) + + # StepExpansion + ATTENTION = (Style.BRIGHT, Fore.GREEN) # the string "DialectExpander debug mode" + DIALECTTRANSFORMER = (Style.BRIGHT, Fore.YELLOW) + + # ------------------------------------------------------------ + # dump + + NODETYPE = (Style.BRIGHT, Fore.LIGHTBLUE_EX) + FIELDNAME = Fore.YELLOW + BAREVALUE = Fore.GREEN diff --git a/kibot/mcpyrate/core.py b/kibot/mcpyrate/core.py index c6ac4365..69b11efc 100644 --- a/kibot/mcpyrate/core.py +++ b/kibot/mcpyrate/core.py @@ -8,7 +8,7 @@ from ast import NodeTransformer, AST from contextlib import contextmanager from collections import ChainMap -from .astfixers import fix_missing_ctx, fix_missing_locations +from .astfixers import fix_ctx, fix_locations from .markers import ASTMarker, delete_markers from .utils import flatten_suite, format_location @@ -147,15 +147,18 @@ class BaseMacroExpander(NodeTransformer): macro function, place them in a dictionary and pass that dictionary as `kw`. ''' - macro = self.bindings[macroname] + loc = format_location(self.filename, target, sourcecode) + + macro = self.isbound(macroname) + if not macro: + raise MacroExpansionError(f"{loc}\nThe name '{macroname}' is not bound to a macro.") + kw = kw or {} kw.update({ 'syntax': syntax, 'expander': self, 'invocation': target}) - loc = format_location(self.filename, target, sourcecode) - # Expand the macro. try: expansion = _apply_macro(macro, tree, kw) @@ -200,8 +203,8 @@ class BaseMacroExpander(NodeTransformer): if it detects any more macro invocations. ''' if expansion is not None: - expansion = fix_missing_locations(expansion, target, mode="reference") - expansion = fix_missing_ctx(expansion) + expansion = fix_locations(expansion, target, mode="reference") + expansion = fix_ctx(expansion) if self.recursive: expansion = self.visit(expansion) @@ -231,4 +234,9 @@ def global_postprocess(tree): Call this after macro expansion is otherwise done, before sending `tree` to Python's `compile`. ''' - return delete_markers(tree, cls=MacroExpanderMarker) + tree = delete_markers(tree, cls=MacroExpanderMarker) + # A name macro, appearing as an assignment target, gets the wrong ctx, + # because when expanding the name macro, the expander sees only the Name + # node, and thus puts an `ast.Load` there as the ctx. + tree = fix_ctx(tree) + return tree diff --git a/kibot/mcpyrate/debug.py b/kibot/mcpyrate/debug.py index 5490016d..ddf4b701 100644 --- a/kibot/mcpyrate/debug.py +++ b/kibot/mcpyrate/debug.py @@ -12,6 +12,7 @@ from sys import stderr import textwrap from .astdumper import dump +from .colorizer import setcolor, colorize, ColorScheme from .dialects import StepExpansion # re-export for discoverability, it's a debug feature from .expander import MacroCollector, namemacro, parametricmacro from .unparser import unparse_with_fallbacks @@ -54,7 +55,7 @@ def step_expansion(tree, *, args, syntax, expander, **kw): if syntax not in ("expr", "block"): raise SyntaxError("`step_expansion` is an expr and block macro only") - formatter = functools.partial(unparse_with_fallbacks, debug=True) + formatter = functools.partial(unparse_with_fallbacks, debug=True, color=True) if args: if len(args) != 1: raise SyntaxError("expected `step_expansion` or `step_expansion['mode_str']`") @@ -68,14 +69,17 @@ def step_expansion(tree, *, args, syntax, expander, **kw): if mode not in ("unparse", "dump"): raise ValueError(f"expected mode_str either 'unparse' or 'dump', got {repr(mode)}") if mode == "dump": - formatter = functools.partial(dump, include_attributes=True) + formatter = functools.partial(dump, include_attributes=True, color=True) + + c, CS = setcolor, ColorScheme with _step_expansion_level.changed_by(+1): indent = 2 * _step_expansion_level.value stars = indent * '*' codeindent = indent tag = id(tree) - print(f"{stars}Tree 0x{tag:x} before macro expansion:", file=stderr) + print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} {c(CS.HEADING)}before macro expansion:{c(CS._RESET)}", + file=stderr) print(textwrap.indent(formatter(tree), codeindent * ' '), file=stderr) mc = MacroCollector(expander) mc.visit(tree) @@ -84,12 +88,14 @@ def step_expansion(tree, *, args, syntax, expander, **kw): step += 1 tree = expander.visit_once(tree) # -> Done(body=...) tree = tree.body - print(f"{stars}Tree 0x{tag:x} after step {step}:", file=stderr) + print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} {c(CS.HEADING)}after step {step}:{c(CS._RESET)}", + file=stderr) print(textwrap.indent(formatter(tree), codeindent * ' '), file=stderr) mc.clear() mc.visit(tree) plural = "s" if step != 1 else "" - print(f"{stars}Tree 0x{tag:x} macro expansion complete after {step} step{plural}.", file=stderr) + print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} {c(CS.HEADING)}macro expansion complete after {step} step{plural}.{c(CS._RESET)}", + file=stderr) return tree @@ -103,16 +109,23 @@ def show_bindings(tree, *, syntax, expander, **kw): This can appear in any expression position, and at run-time evaluates to `None`. - At macro expansion time, for each macro binding, this prints to `sys.stderr` - the macro name, and the fully qualified name of the corresponding macro function. + At macro expansion time, when the macro expander reaches the `show_bindings` + expression, bindings *that are in effect at that point in time* are shown. + + (That disclaimer is important, because hygienically unquoted macros may add + new bindings as those expressions are reached, and true to the dynamic nature + of Python, when a macro runs, it is allowed to edit the expander's bindings.) + + For each macro binding, we print to `sys.stderr` the macro name, and the + fully qualified name of the corresponding macro function. Any bindings that have an uuid as part of the name are hygienically - unquoted macros. Those make a per-process global binding across all modules - and all expander instances. + unquoted macros. Those bindings are global across all modules and + all expander instances. """ if syntax != "name": raise SyntaxError("`show_bindings` is an identifier macro only") - print(format_bindings(expander), file=stderr) + print(format_bindings(expander, color=True), file=stderr) # Can't just delete the node (return None) if it's in an Expr(value=...). # # For correct coverage reporting, we can't return a `Constant`, because CPython @@ -125,19 +138,34 @@ def show_bindings(tree, *, syntax, expander, **kw): return ast.Call(func=lam, args=[], keywords=[]) -def format_bindings(expander): +def format_bindings(expander, *, color=False): """Return a human-readable report of the macro bindings currently seen by `expander`. Global bindings (across all expanders) are also included. + If `color=True`, use `colorama` to colorize the output. (For terminals.) + If you want to access them programmatically, just access `expander.bindings` directly. """ + def maybe_setcolor(*colors): + if not color: + return "" + return setcolor(*colors) + def maybe_colorize(text, *colors): + if not color: + return text + return colorize(text, *colors) + + c, CS = maybe_setcolor, ColorScheme + with io.StringIO() as output: - output.write(f"Macro bindings for {expander.filename}:\n") + output.write(f"{c(CS.HEADING)}Macro bindings for {c(CS.SOURCEFILENAME)}{expander.filename}{c(CS.HEADING)}:{c(CS._RESET)}\n") if not expander.bindings: - output.write(" \n") + output.write(maybe_colorize(" \n", + ColorScheme.GREYEDOUT)) else: for k, v in sorted(expander.bindings.items()): + k = maybe_colorize(k, ColorScheme.MACROBINDING) output.write(f" {k}: {format_macrofunction(v)}\n") return output.getvalue() diff --git a/kibot/mcpyrate/dialects.py b/kibot/mcpyrate/dialects.py index 289e5a00..3cee53cc 100644 --- a/kibot/mcpyrate/dialects.py +++ b/kibot/mcpyrate/dialects.py @@ -5,11 +5,13 @@ __all__ = ["Dialect", "expand_dialects"] import ast +import functools import importlib import importlib.util import re from sys import stderr +from .colorizer import setcolor, colorize, ColorScheme from .coreutils import ismacroimport, get_macros from .unparser import unparse_with_fallbacks @@ -108,7 +110,7 @@ class Dialect: return NotImplemented -_message_header = "**StepExpansion: " +_message_header = colorize("**StepExpansion: ", ColorScheme.HEADING) class StepExpansion(Dialect): # actually part of public API of mcpyrate.debug, for discoverability """[dialect] Show each step of expansion while dialect-expanding the module. @@ -125,7 +127,9 @@ class StepExpansion(Dialect): # actually part of public API of mcpyrate.debug, """ def transform_source(self, text): self.expander.debugmode = True - print(f"{_message_header}{self.expander.filename} enabled DialectExpander debug mode while taking step {self.expander._step + 1}.", file=stderr) + c, CS = setcolor, ColorScheme + msg = f"{c(CS.SOURCEFILENAME)}{self.expander.filename} {c(CS.HEADING)}enabled {c(CS.ATTENTION)}DialectExpander debug mode {c(CS.HEADING)}while taking step {self.expander._step + 1}.{c(CS._RESET)}" + print(_message_header + msg, file=stderr) # Pass through the input (instead of returning `NotImplemented`) to # consider this as having taken a step, thus triggering the debug mode # output printer. (If this was the first dialect applied, our output is @@ -171,17 +175,19 @@ class DialectExpander: def transform_ast(self, tree): '''Apply all whole-module AST transformers.''' + formatter = functools.partial(unparse_with_fallbacks, debug=True, color=True) return self._transform(tree, kind="AST", find_dialectimport=self.find_dialectimport_ast, transform="transform_ast", - format_for_display=unparse_with_fallbacks) + format_for_display=formatter) def _transform(self, content, *, kind, find_dialectimport, transform, format_for_display): + c, CS = setcolor, ColorScheme if self.debugmode: plural = "s" if self._step != 1 else "" - print(f"{_message_header}{self.filename} before dialect {kind} transformers ({self._step} step{plural} total):\n", file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}before dialect {c(CS.DIALECTTRANSFORMER)}{kind} {c(CS.HEADING)}transformers ({self._step} step{plural} total):{c(CS._RESET)}\n" + print(_message_header + msg, file=stderr) print(format_for_display(content), file=stderr) - print("-" * 79, file=stderr) while True: module_absname, bindings = find_dialectimport(content) @@ -218,14 +224,14 @@ class DialectExpander: self._step += 1 if self.debugmode: - print(f"{_message_header}{self.filename} after {module_absname}.{dialectname}.{transform} (step {self._step}):\n", file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}after {c(CS.DIALECTTRANSFORMER)}{module_absname}.{dialectname}.{transform} {c(CS.HEADING)}(step {self._step}):{c(CS._RESET)}\n" + print(_message_header + msg, file=stderr) print(format_for_display(content), file=stderr) - print("-" * 79, file=stderr) if self.debugmode: plural = "s" if self._step != 1 else "" - print(f"{_message_header}All dialect {kind} transformers completed for {self.filename} ({self._step} step{plural} total).", file=stderr) - print("-" * 79, file=stderr) + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}completed all dialect {c(CS.DIALECTTRANSFORMER)}{kind} {c(CS.HEADING)}transforms ({self._step} step{plural} total).{c(CS._RESET)}" + print(_message_header + msg, file=stderr) return content diff --git a/kibot/mcpyrate/importer.py b/kibot/mcpyrate/importer.py index 02bc6327..8b87e6ea 100644 --- a/kibot/mcpyrate/importer.py +++ b/kibot/mcpyrate/importer.py @@ -4,6 +4,7 @@ __all__ = ['source_to_xcode', 'path_xstats', 'invalidate_xcaches'] import ast +import distutils.sysconfig import importlib.util from importlib.machinery import FileFinder, SourceFileLoader import tokenize @@ -32,14 +33,14 @@ def source_to_xcode(self, data, path, *, _optimize=-1): remaining_markers = get_markers(expansion) if remaining_markers: - raise MacroExpansionError("{path}: AST markers remaining after expansion: {remaining_markers}") + raise MacroExpansionError(f"{path}: AST markers remaining after expansion: {remaining_markers}") return compile(expansion, path, 'exec', dont_inherit=True, optimize=_optimize) # TODO: Support PEP552 (Deterministic pycs). Need to intercept source file hashing, too. # TODO: https://www.python.org/dev/peps/pep-0552/ -_path_stats = SourceFileLoader.path_stats +_stdlib_path_stats = SourceFileLoader.path_stats _xstats_cache = {} def path_xstats(self, path): '''[mcpyrate] Compute a `.py` source file's mtime, accounting for macro-imports. @@ -56,8 +57,11 @@ def path_xstats(self, path): If `path` does not end in `.py`, delegate to the standard implementation of `SourceFileLoader.path_stats`. ''' - if not path.endswith(".py"): - return _path_stats(path) + # Ignore stdlib, it's big and doesn't use macros. Allows faster error + # exits, because an uncaught exception causes Python to load a ton of + # .py based stdlib modules. Also makes `macropython -i` start faster. + if path in _stdlib_sourcefile_paths or not path.endswith(".py"): + return _stdlib_path_stats(self, path) if path in _xstats_cache: return _xstats_cache[path] @@ -162,11 +166,25 @@ def path_xstats(self, path): return result -_invalidate_caches = FileFinder.invalidate_caches +_stdlib_invalidate_caches = FileFinder.invalidate_caches def invalidate_xcaches(self): '''[mcpyrate] Clear the macro dependency tree cache. Then delegate to the standard implementation of `FileFinder.invalidate_caches`. ''' _xstats_cache.clear() - return _invalidate_caches(self) + return _stdlib_invalidate_caches(self) + + +def _detect_stdlib_sourcefile_paths(): + '''Return a set of full paths of `.py` files that are part of Python's standard library.''' + # Adapted from StackOverflow answer by Adam Spiers, https://stackoverflow.com/a/8992937 + # Note we don't want to get module names, but full paths to `.py` files. + stdlib_dir = distutils.sysconfig.get_python_lib(standard_lib=True) + paths = set() + for root, dirs, files in os.walk(stdlib_dir): + for nm in files: + if nm[-3:] == '.py': + paths.add(os.path.join(root, nm)) + return paths +_stdlib_sourcefile_paths = _detect_stdlib_sourcefile_paths() diff --git a/kibot/mcpyrate/quotes.py b/kibot/mcpyrate/quotes.py index 0c53a677..b79b73a3 100644 --- a/kibot/mcpyrate/quotes.py +++ b/kibot/mcpyrate/quotes.py @@ -46,16 +46,6 @@ def _mcpyrate_quotes_attr(attr): attr=attr, ctx=ast.Load()) -def _capture_into(mapping, value, basename): - for k, v in mapping.items(): - if v is value: - key = k - break - else: - key = gensym(basename) - mapping[key] = value - return key - def capture(value, name): """Hygienically capture a run-time value. @@ -64,19 +54,11 @@ def capture(value, name): The return value is an AST that, when compiled and run, returns the captured value (even in another Python process later). - - Hygienically captured macro invocations are treated using a different - mechanism; see `mcpyrate.core.global_bindings`. """ # If we didn't need to consider bytecode caching, we could just store the # value in a registry that is populated at macro expansion time. Each # unique value (by `id`) could be stored only once. # - # key = _capture_into(_hygienic_registry, value, name) - # return ast.Call(_mcpyrate_quotes_attr("lookup"), - # [ast.Constant(value=key)], - # []) - # But we want to support bytecode caching. To avoid introducing hard-to-find # bugs into user code, we must provide consistent semantics, regardless of # whether updating of the bytecode cache is actually enabled or not (see @@ -101,24 +83,49 @@ def capture(value, name): # frozen_value = pickle.dumps(value) return ast.Call(_mcpyrate_quotes_attr("lookup"), - [ast.Tuple(elts=[ast.Constant(value=name), # for human-readability of expanded code + [ast.Tuple(elts=[ast.Constant(value=name), ast.Constant(value=frozen_value)])], []) _lookup_cache = {} def lookup(key): """Look up a hygienically captured run-time value.""" - # if type(key) is str: # captured in sys.dont_write_bytecode mode, in this process - # return _hygienic_registry[key] - # else: # frozen into macro-expanded code - # name, frozen_value = key - # return pickle.loads(frozen_value) name, frozen_value = key cachekey = (name, id(frozen_value)) # id() so each capture instance behaves independently if cachekey not in _lookup_cache: _lookup_cache[cachekey] = pickle.loads(frozen_value) return _lookup_cache[cachekey] +def capture_macro(macro, name): + """Hygienically capture a macro. + + `macro`: A macro function. Must be picklable. + `name`: The name of the macro, as it appeared in the bindings of + the expander it was captured from. For human-readability. + + The return value is an AST that, when compiled and run, injects the + macro into the expander's global macro bindings table (even in another + Python process later), and then evaluates to that macro name as an + `ast.Name` (so further expansion of that AST will invoke the macro). + """ + frozen_macro = pickle.dumps(macro) + unique_name = gensym(name) + return ast.Call(_mcpyrate_quotes_attr("lookup_macro"), + [ast.Tuple(elts=[ast.Constant(value=unique_name), + ast.Constant(value=frozen_macro)])], + []) + +def lookup_macro(key): + """Look up a hygienically captured macro. + + This will inject the macro to the global macro bindings table, + and then evaluate to that macro name, as an `ast.Name`. + """ + unique_name, frozen_macro = key + if unique_name not in global_bindings: + global_bindings[unique_name] = pickle.loads(frozen_macro) + return ast.Name(id=unique_name) + # -------------------------------------------------------------------------------- def astify(x, expander=None): # like `macropy`'s `ast_repr` @@ -154,15 +161,17 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr` if expander and type(x.body) is ast.Name: function = expander.isbound(x.body.id) if function: - # Hygienically capture a macro name. We do this immediately, - # during the expansion of `q`. This allows macros in scope at - # the use site of `q` to be hygienically propagated out to the - # use site of the macro that used `q`. So you can write macros - # that `q[h[macroname][...]]` and `macroname` doesn't have to be - # macro-imported wherever that code gets spliced in. - macroname = x.body.id - uniquename = _capture_into(global_bindings, function, macroname) - return recurse(ast.Name(id=uniquename)) + # Hygienically capture a macro. We do this immediately, + # during the expansion of `q`, because the value we want to + # store, i.e. the macro function, is available only at + # macro-expansion time. + # + # This allows macros in scope at the use site of `q` to be + # hygienically propagated out to the use site of the macro + # that used `q`. So you can write macros that `q[h[macroname][...]]`, + # and `macroname` doesn't have to be macro-imported wherever + # that code gets spliced in. + return capture_macro(function, x.body.id) # Hygienically capture a garden variety run-time value. # At the use site of q[], this captures the value, and rewrites itself # into a lookup. At the use site of the macro that used q[], that @@ -373,10 +382,6 @@ def n(tree, *, syntax, **kw): if _quotelevel.value < 1: raise SyntaxError("`n` encountered while quotelevel < 1") with _quotelevel.changed_by(-1): - ret = ASTLiteral(astify(ast.Name(id=ASTLiteral(tree)))) - print(tree.__dict__) - print(ret.body.func.value.value.value.__dict__) - print(unparse(ret)) return ASTLiteral(astify(ast.Name(id=ASTLiteral(tree)))) diff --git a/kibot/mcpyrate/splicing.py b/kibot/mcpyrate/splicing.py index 995e765a..85bcd9af 100644 --- a/kibot/mcpyrate/splicing.py +++ b/kibot/mcpyrate/splicing.py @@ -6,7 +6,7 @@ __all__ = ["splice_statements", "splice_dialect"] import ast from copy import deepcopy -from .astfixers import fix_missing_locations +from .astfixers import fix_locations from .coreutils import ismacroimport from .walker import Walker @@ -145,7 +145,7 @@ def splice_dialect(body, template, tag="__paste_here__"): # # Pretend the template code appears at the beginning of the user module. for stmt in template: - fix_missing_locations(stmt, body[0], mode="overwrite") + fix_locations(stmt, body[0], mode="overwrite") # TODO: remove ast.Str once we bump minimum language version to Python 3.8 if type(body[0]) is ast.Expr and type(body[0].value) in (ast.Constant, ast.Str): diff --git a/kibot/mcpyrate/unparser.py b/kibot/mcpyrate/unparser.py index 82364f29..4133c82c 100644 --- a/kibot/mcpyrate/unparser.py +++ b/kibot/mcpyrate/unparser.py @@ -4,16 +4,26 @@ __all__ = ['UnparserError', 'unparse', 'unparse_with_fallbacks'] import ast +import builtins +from contextlib import contextmanager import io import sys from .astdumper import dump # fallback +from .colorizer import colorize, ColorScheme from .markers import ASTMarker # Large float and imaginary literals get turned into infinities in the AST. # We unparse those infinities to INFSTR. INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) +# for syntax highlighting +_all_public_builtins = {x for x in dir(builtins) if not x.startswith("_")} +builtin_exceptions = {x for x in _all_public_builtins if x.endswith("Error")} +builtin_warnings = {x for x in _all_public_builtins if x.endswith("Warning")} +builtin_exceptions_and_warnings = builtin_exceptions | builtin_warnings +builtin_others = _all_public_builtins - builtin_exceptions_and_warnings + class UnparserError(SyntaxError): """Failed to unparse the given AST.""" @@ -39,22 +49,56 @@ class Unparser: for the abstract syntax. Original formatting is disregarded. """ - def __init__(self, tree, *, file=sys.stdout, debug=False): + def __init__(self, tree, *, file=sys.stdout, debug=False, color=False): """Print the source for `tree` to `file`. `debug`: bool, print invisible nodes (`Module`, `Expr`). The output is then not valid Python, but may better show the problem when code produced by a macro mysteriously - fails to compile (even though the unparse looks ok). + fails to compile (even though a non-debug unparse looks ok). + + `color`: bool, use Colorama to color output. For syntax highlighting + when printing into a terminal. """ self.debug = debug + self.color = color + self._color_override = False # to syntax highlight decorators self.f = file self._indent = 0 self.dispatch(tree) print("", file=self.f) self.f.flush() + def maybe_colorize(self, text, *colors): + "Colorize text if color is enabled." + if self._color_override: + return text + if not self.color: + return text + return colorize(text, *colors) + + def python_keyword(self, text): + "Shorthand to colorize a language keyword such as `def`, `for`, ..." + return self.maybe_colorize(text, ColorScheme.LANGUAGEKEYWORD) + + def nocolor(self): + """Context manager. Temporarily prevent coloring. + + Useful for syntax highlighting decorators (so that the method rendering + the decorator may force a particular color, instead of allowing + auto-coloring based on the data in the decorator AST node). + """ + @contextmanager + def _nocolor(): + old_color_override = self._color_override + self._color_override = True + try: + yield + finally: + self._color_override = old_color_override + return _nocolor() + def fill(self, text="", *, lineno_node=None): "Indent a piece of text, according to the current indentation level." self.write("\n") @@ -66,7 +110,8 @@ class Unparser: # # Assume line numbers usually have at most 4 digits, but # degrade gracefully for those crazy 5-digit source files. - self.write(f"L{lineno:5d} " if lineno else "L ---- ") + self.write(self.maybe_colorize(f"L{lineno:5d} " if lineno else "L ---- ", + ColorScheme.LINENUMBER)) self.write(" " * self._indent + text) def write(self, text): @@ -111,7 +156,11 @@ class Unparser: self.dispatch(v) else: self.write(repr(v)) - self.fill(f"$ASTMarker<{tree.__class__.__name__}>", lineno_node=tree) # markers cannot be eval'd + self.fill(self.maybe_colorize(f"$ASTMarker", ColorScheme.ASTMARKER), + lineno_node=tree) # markers cannot be eval'd + clsname = self.maybe_colorize(tree.__class__.__name__, + ColorScheme.ASTMARKERCLASS) + self.write(f"<{clsname}>") self.enter() self.write(" ") if len(tree._fields) == 1 and tree._fields[0] == "body": @@ -128,7 +177,8 @@ class Unparser: def _Module(self, t): # TODO: Python 3.8 type_ignores. Since we don't store the source text, maybe ignore that? if self.debug: - self.fill("$Module", lineno_node=t) + self.fill(self.maybe_colorize("$Module", ColorScheme.INVISIBLENODE), + lineno_node=t) self.enter() for stmt in t.body: self.dispatch(stmt) @@ -140,7 +190,8 @@ class Unparser: # stmt def _Expr(self, t): if self.debug: - self.fill("$Expr", lineno_node=t) + self.fill(self.maybe_colorize("$Expr", ColorScheme.INVISIBLENODE), + lineno_node=t) self.enter() self.write(" ") self.dispatch(t.value) @@ -150,15 +201,15 @@ class Unparser: self.dispatch(t.value) def _Import(self, t): - self.fill("import ", lineno_node=t) + self.fill(self.python_keyword("import "), lineno_node=t) interleave(lambda: self.write(", "), self.dispatch, t.names) def _ImportFrom(self, t): - self.fill("from ", lineno_node=t) + self.fill(self.python_keyword("from "), lineno_node=t) self.write("." * t.level) if t.module: self.write(t.module) - self.write(" import ") + self.write(self.python_keyword(" import ")) interleave(lambda: self.write(", "), self.dispatch, t.names) def _Assign(self, t): @@ -189,42 +240,42 @@ class Unparser: self.dispatch(t.value) def _Return(self, t): - self.fill("return", lineno_node=t) + self.fill(self.python_keyword("return"), lineno_node=t) if t.value: self.write(" ") self.dispatch(t.value) def _Pass(self, t): - self.fill("pass", lineno_node=t) + self.fill(self.python_keyword("pass"), lineno_node=t) def _Break(self, t): - self.fill("break", lineno_node=t) + self.fill(self.python_keyword("break"), lineno_node=t) def _Continue(self, t): - self.fill("continue", lineno_node=t) + self.fill(self.python_keyword("continue"), lineno_node=t) def _Delete(self, t): - self.fill("del ", lineno_node=t) + self.fill(self.python_keyword("del "), lineno_node=t) interleave(lambda: self.write(", "), self.dispatch, t.targets) def _Assert(self, t): - self.fill("assert ", lineno_node=t) + self.fill(self.python_keyword("assert "), lineno_node=t) self.dispatch(t.test) if t.msg: self.write(", ") self.dispatch(t.msg) def _Global(self, t): - self.fill("global ", lineno_node=t) + self.fill(self.python_keyword("global "), lineno_node=t) interleave(lambda: self.write(", "), self.write, t.names) def _Nonlocal(self, t): - self.fill("nonlocal ", lineno_node=t) + self.fill(self.python_keyword("nonlocal "), lineno_node=t) interleave(lambda: self.write(", "), self.write, t.names) def _Await(self, t): # expr self.write("(") - self.write("await") + self.write(self.python_keyword("await")) if t.value: self.write(" ") self.dispatch(t.value) @@ -232,7 +283,7 @@ class Unparser: def _Yield(self, t): # expr self.write("(") - self.write("yield") + self.write(self.python_keyword("yield")) if t.value: self.write(" ") self.dispatch(t.value) @@ -240,48 +291,48 @@ class Unparser: def _YieldFrom(self, t): # expr self.write("(") - self.write("yield from") + self.write(self.python_keyword("yield from")) if t.value: self.write(" ") self.dispatch(t.value) self.write(")") def _Raise(self, t): - self.fill("raise", lineno_node=t) + self.fill(self.python_keyword("raise"), lineno_node=t) if not t.exc: assert not t.cause return self.write(" ") self.dispatch(t.exc) if t.cause: - self.write(" from ") + self.write(self.python_keyword(" from ")) self.dispatch(t.cause) def _Try(self, t): - self.fill("try", lineno_node=t) + self.fill(self.python_keyword("try"), lineno_node=t) self.enter() self.dispatch(t.body) self.leave() for ex in t.handlers: self.dispatch(ex) if t.orelse: - self.fill("else") + self.fill(self.python_keyword("else")) self.enter() self.dispatch(t.orelse) self.leave() if t.finalbody: - self.fill("finally") + self.fill(self.python_keyword("finally")) self.enter() self.dispatch(t.finalbody) self.leave() def _ExceptHandler(self, t): - self.fill("except", lineno_node=t) + self.fill(self.python_keyword("except"), lineno_node=t) if t.type: self.write(" ") self.dispatch(t.type) if t.name: - self.write(" as ") + self.write(self.python_keyword(" as ")) self.write(t.name) self.enter() self.dispatch(t.body) @@ -289,10 +340,18 @@ class Unparser: def _ClassDef(self, t): self.write("\n") + for deco in t.decorator_list: - self.fill("@", lineno_node=deco) - self.dispatch(deco) - self.fill("class " + t.name, lineno_node=t) + self.fill(self.maybe_colorize("@", ColorScheme.DECORATOR), + lineno_node=deco) + self.write(ColorScheme.DECORATOR) + with self.nocolor(): + self.dispatch(deco) + self.write(ColorScheme._RESET) + + class_str = (self.python_keyword("class ") + + self.maybe_colorize(t.name, ColorScheme.DEFNAME)) + self.fill(class_str, lineno_node=t) self.write("(") comma = False for e in t.bases: @@ -321,10 +380,17 @@ class Unparser: def __FunctionDef_helper(self, t, fill_suffix): self.write("\n") + for deco in t.decorator_list: - self.fill("@", lineno_node=deco) - self.dispatch(deco) - def_str = fill_suffix + " " + t.name + "(" + self.fill(self.maybe_colorize("@", ColorScheme.DECORATOR), + lineno_node=deco) + self.write(ColorScheme.DECORATOR) + with self.nocolor(): + self.dispatch(deco) + self.write(ColorScheme._RESET) + + def_str = (self.python_keyword(fill_suffix) + + " " + self.maybe_colorize(t.name, ColorScheme.DEFNAME) + "(") self.fill(def_str, lineno_node=t) self.dispatch(t.args) self.write(")") @@ -337,28 +403,28 @@ class Unparser: # TODO: Python 3.8 type_comment, ignore it? def _For(self, t): - self.__For_helper("for ", t) + self.__For_helper(self.python_keyword("for "), t) def _AsyncFor(self, t): - self.__For_helper("async for ", t) + self.__For_helper(self.python_keyword("async for "), t) def __For_helper(self, fill, t): self.fill(fill, lineno_node=t) self.dispatch(t.target) - self.write(" in ") + self.write(self.python_keyword(" in ")) self.dispatch(t.iter) self.enter() self.dispatch(t.body) self.leave() if t.orelse: - self.fill("else") + self.fill(self.python_keyword("else")) self.enter() self.dispatch(t.orelse) self.leave() # TODO: Python 3.8 type_comment, ignore it? def _If(self, t): - self.fill("if ", lineno_node=t) + self.fill(self.python_keyword("if "), lineno_node=t) self.dispatch(t.test) self.enter() self.dispatch(t.body) @@ -367,32 +433,32 @@ class Unparser: while (t.orelse and len(t.orelse) == 1 and isinstance(t.orelse[0], ast.If)): t = t.orelse[0] - self.fill("elif ") + self.fill(self.python_keyword("elif ")) self.dispatch(t.test) self.enter() self.dispatch(t.body) self.leave() # final else if t.orelse: - self.fill("else") + self.fill(self.python_keyword("else")) self.enter() self.dispatch(t.orelse) self.leave() def _While(self, t): - self.fill("while ", lineno_node=t) + self.fill(self.python_keyword("while "), lineno_node=t) self.dispatch(t.test) self.enter() self.dispatch(t.body) self.leave() if t.orelse: - self.fill("else") + self.fill(self.python_keyword("else")) self.enter() self.dispatch(t.orelse) self.leave() def _With(self, t): - self.fill("with ", lineno_node=t) + self.fill(self.python_keyword("with "), lineno_node=t) interleave(lambda: self.write(", "), self.dispatch, t.items) self.enter() self.dispatch(t.body) @@ -400,7 +466,7 @@ class Unparser: # TODO: Python 3.8 type_comment, ignore it? def _AsyncWith(self, t): - self.fill("async with ", lineno_node=t) + self.fill(self.python_keyword("async with "), lineno_node=t) interleave(lambda: self.write(", "), self.dispatch, t.items) self.enter() self.dispatch(t.body) @@ -422,27 +488,38 @@ class Unparser: if type(t.value) in (int, float, complex): # Represent AST infinity as an overflowing decimal literal. v = repr(t.value).replace("inf", INFSTR) + v = self.maybe_colorize(v, ColorScheme.NUMBER) elif t.value is Ellipsis: v = "..." else: v = repr(t.value) + if t.value in (True, False, None): + v = self.maybe_colorize(v, ColorScheme.NAMECONSTANT) + elif type(t.value) in (str, bytes): + v = self.maybe_colorize(v, ColorScheme.STRING) self.write(v) def _Bytes(self, t): # up to Python 3.7 - self.write(repr(t.s)) + self.write(self.maybe_colorize(repr(t.s), ColorScheme.STRING)) def _Str(self, tree): # up to Python 3.7 - self.write(repr(tree.s)) + self.write(self.maybe_colorize(repr(tree.s), ColorScheme.STRING)) def _Name(self, t): - self.write(t.id) + v = t.id + if v in builtin_exceptions_and_warnings: + v = self.maybe_colorize(v, ColorScheme.BUILTINEXCEPTION) + elif v in builtin_others: + v = self.maybe_colorize(v, ColorScheme.BUILTINOTHER) + self.write(v) def _NameConstant(self, t): # up to Python 3.7 - self.write(repr(t.value)) + self.write(self.maybe_colorize(repr(t.value), ColorScheme.NAMECONSTANT)) def _Num(self, t): # up to Python 3.7 # Represent AST infinity as an overflowing decimal literal. - self.write(repr(t.n).replace("inf", INFSTR)) + v = repr(t.n).replace("inf", INFSTR) + self.write(self.maybe_colorize(v, ColorScheme.NUMBER)) def _List(self, t): self.write("[") @@ -481,21 +558,21 @@ class Unparser: def _comprehension(self, t): if t.is_async: - self.write(" async") - self.write(" for ") + self.write(self.python_keyword(" async")) + self.write(self.python_keyword(" for ")) self.dispatch(t.target) - self.write(" in ") + self.write(self.python_keyword(" in ")) self.dispatch(t.iter) for if_clause in t.ifs: - self.write(" if ") + self.write(self.python_keyword(" if ")) self.dispatch(if_clause) def _IfExp(self, t): self.write("(") self.dispatch(t.body) - self.write(" if ") + self.write(self.python_keyword(" if ")) self.dispatch(t.test) - self.write(" else ") + self.write(self.python_keyword(" else ")) self.dispatch(t.orelse) self.write(")") @@ -556,7 +633,8 @@ class Unparser: boolops = {ast.And: 'and', ast.Or: 'or'} def _BoolOp(self, t): self.write("(") - s = " %s " % self.boolops[t.op.__class__] + s = self.python_keyword(self.boolops[t.op.__class__]) + s = f" {s} " interleave(lambda: self.write(s), self.dispatch, t.values) self.write(")") @@ -599,27 +677,29 @@ class Unparser: self.write("'") def _FormattedValue_helper(self, t): - self.write("{") + def c(text): + return self.maybe_colorize(text, ColorScheme.STRING) + self.write(c("{")) self.dispatch(t.value) if t.conversion == 115: - self.write("!s") + self.write(c("!s")) elif t.conversion == 114: - self.write("!r") + self.write(c("!r")) elif t.conversion == 97: - self.write("!a") + self.write(c("!a")) elif t.conversion == -1: # no formatting pass else: raise ValueError(f"Don't know how to unparse conversion code {t.conversion}") if t.format_spec: - self.write(":") + self.write(c(":")) self._JoinedStr_helper(t.format_spec) - self.write("}") + self.write(c("}")) def _JoinedStr(self, t): - self.write("f'") + self.write("f" + self.maybe_colorize("'", ColorScheme.STRING)) self._JoinedStr_helper(t) - self.write("'") + self.write(self.maybe_colorize("'", ColorScheme.STRING)) def _JoinedStr_helper(self, t): def escape(s): @@ -627,9 +707,9 @@ class Unparser: for v in t.values: # Omit the surrounding quotes in string snippets if type(v) is ast.Constant: - self.write(escape(v.value)) + self.write(self.maybe_colorize(escape(v.value), ColorScheme.STRING)) elif type(v) is ast.Str: # up to Python 3.7 - self.write(escape(v.s)) + self.write(self.maybe_colorize(escape(v.s), ColorScheme.STRING)) elif type(v) is ast.FormattedValue: self._FormattedValue_helper(v) else: @@ -750,7 +830,7 @@ class Unparser: def _Lambda(self, t): self.write("(") - self.write("lambda ") + self.write(self.python_keyword("lambda ")) self.dispatch(t.args) self.write(": ") self.dispatch(t.body) @@ -759,16 +839,16 @@ class Unparser: def _alias(self, t): self.write(t.name) if t.asname: - self.write(" as " + t.asname) + self.write(self.python_keyword(" as ") + t.asname) def _withitem(self, t): self.dispatch(t.context_expr) if t.optional_vars: - self.write(" as ") + self.write(self.python_keyword(" as ")) self.dispatch(t.optional_vars) -def unparse(tree, *, debug=False): +def unparse(tree, *, debug=False, color=False): """Convert the AST `tree` into source code. Return the code as a string. `debug`: bool, print invisible nodes (`Module`, `Expr`). @@ -781,7 +861,7 @@ def unparse(tree, *, debug=False): """ try: with io.StringIO() as output: - Unparser(tree, file=output, debug=debug) + Unparser(tree, file=output, debug=debug, color=color) code = output.getvalue().strip() return code except UnparserError as err: # fall back to an AST dump @@ -797,7 +877,7 @@ def unparse(tree, *, debug=False): raise UnparserError(msg) from err -def unparse_with_fallbacks(tree, *, debug=False): +def unparse_with_fallbacks(tree, *, debug=False, color=False): """Like `unparse`, but upon error, don't raise; return the error message. Usually you'll want the exception to be raised. This is mainly useful to @@ -806,7 +886,7 @@ def unparse_with_fallbacks(tree, *, debug=False): at the receiving end. """ try: - text = unparse(tree, debug=debug) + text = unparse(tree, debug=debug, color=color) except UnparserError as err: text = err.args[0] except Exception as err: