Updated mcpyrate
This commit is contained in:
Salvador E. Tropea
parent
ae8da88539
commit
5ce56b1a05
|
|
@ -25,24 +25,27 @@ the `PYTHONDONTWRITEBYTECODE` environment variable, and the attribute
|
|||
https://www.python.org/dev/peps/pep-0552/
|
||||
'''
|
||||
|
||||
__all__ = ["activate", "deactivate"]
|
||||
|
||||
from importlib.machinery import SourceFileLoader, FileFinder
|
||||
from .importer import source_to_xcode, path_xstats, invalidate_xcaches
|
||||
|
||||
|
||||
def activate():
|
||||
SourceFileLoader.source_to_code = source_to_xcode
|
||||
# we could replace SourceFileLoader.set_data with a no-op to force-disable pyc caching.
|
||||
# Bytecode caching (`.pyc`) support. If you need to force-disable `.pyc`
|
||||
# caching, replace `SourceFileLoader.set_data` with a no-op, like `mcpy` does.
|
||||
SourceFileLoader.path_stats = path_xstats
|
||||
FileFinder.invalidate_caches = invalidate_xcaches
|
||||
|
||||
|
||||
def de_activate():
|
||||
SourceFileLoader.source_to_code = old_source_to_code
|
||||
SourceFileLoader.path_stats = old_path_stats
|
||||
FileFinder.invalidate_caches = old_invalidate_caches
|
||||
def deactivate():
|
||||
SourceFileLoader.source_to_code = stdlib_source_to_code
|
||||
SourceFileLoader.path_stats = stdlib_path_stats
|
||||
FileFinder.invalidate_caches = stdlib_invalidate_caches
|
||||
|
||||
|
||||
old_source_to_code = SourceFileLoader.source_to_code
|
||||
old_path_stats = SourceFileLoader.path_stats
|
||||
old_invalidate_caches = FileFinder.invalidate_caches
|
||||
stdlib_source_to_code = SourceFileLoader.source_to_code
|
||||
stdlib_path_stats = SourceFileLoader.path_stats
|
||||
stdlib_invalidate_caches = FileFinder.invalidate_caches
|
||||
activate()
|
||||
|
|
|
|||
|
|
@ -8,7 +8,11 @@ __all__ = ["dump"]
|
|||
|
||||
from ast import AST, iter_fields
|
||||
|
||||
def dump(tree, *, include_attributes=False, multiline=True):
|
||||
from .colorizer import colorize, ColorScheme
|
||||
|
||||
NoneType = type(None)
|
||||
|
||||
def dump(tree, *, include_attributes=False, multiline=True, color=False):
|
||||
"""Return a formatted dump of `tree`, as a string.
|
||||
|
||||
`tree` can be an AST node or a statement suite (`list` of AST nodes).
|
||||
|
|
@ -21,26 +25,47 @@ def dump(tree, *, include_attributes=False, multiline=True):
|
|||
|
||||
To put everything on one line, use `multiline=False`.
|
||||
|
||||
If you're printing the result into a terminal, consider `color=True`.
|
||||
|
||||
Similar to `macropy`'s `real_repr`, but with indentation. The method
|
||||
`ast.AST.__repr__` itself can't be monkey-patched, because `ast.AST`
|
||||
is a built-in/extension type.
|
||||
"""
|
||||
def maybe_colorize(text, *colors):
|
||||
if not color:
|
||||
return text
|
||||
return colorize(text, *colors)
|
||||
|
||||
def maybe_colorize_value(value):
|
||||
if type(value) in (str, bytes, NoneType, bool, int, float, complex):
|
||||
# Pass through an already formatted list-as-a-string from an inner level.
|
||||
if isinstance(value, str) and value.startswith("["):
|
||||
return value
|
||||
return maybe_colorize(str(value), ColorScheme.BAREVALUE)
|
||||
return str(value)
|
||||
|
||||
def recurse(tree, previndent=0):
|
||||
def separator():
|
||||
if multiline:
|
||||
return f",\n{(previndent + moreindent) * ' '}"
|
||||
return ", "
|
||||
|
||||
if isinstance(tree, AST):
|
||||
moreindent = len(f"{tree.__class__.__name__}(")
|
||||
fields = [(k, recurse(v, previndent + moreindent + len(f"{k}="))) for k, v in iter_fields(tree)]
|
||||
if include_attributes and tree._attributes:
|
||||
fields.extend([(k, recurse(getattr(tree, k, None), previndent + moreindent + len(f"{k}=")))
|
||||
fields.extend([(k, recurse(getattr(tree, k, None),
|
||||
previndent + moreindent + len(f"{k}=")))
|
||||
for k in tree._attributes])
|
||||
colorized_fields = [(maybe_colorize(k, ColorScheme.FIELDNAME),
|
||||
maybe_colorize_value(v))
|
||||
for k, v in fields]
|
||||
return ''.join([
|
||||
tree.__class__.__name__,
|
||||
maybe_colorize(tree.__class__.__name__, ColorScheme.NODETYPE),
|
||||
'(',
|
||||
separator().join((f'{k}={v}' for k, v in fields)),
|
||||
separator().join((f'{k}={v}' for k, v in colorized_fields)),
|
||||
')'])
|
||||
|
||||
elif isinstance(tree, list):
|
||||
moreindent = len("[")
|
||||
items = [recurse(elt, previndent + moreindent) for elt in tree]
|
||||
|
|
@ -49,6 +74,7 @@ def dump(tree, *, include_attributes=False, multiline=True):
|
|||
items[-1] = items[-1] + ']'
|
||||
return separator().join(items)
|
||||
return '[]'
|
||||
|
||||
return repr(tree)
|
||||
|
||||
if not isinstance(tree, (AST, list)):
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
'''Fix missing `ctx` attributes and source location info in an AST.'''
|
||||
'''Fix `ctx` attributes and source location info in an AST.'''
|
||||
|
||||
__all__ = ['fix_missing_ctx', 'fix_missing_locations']
|
||||
__all__ = ['fix_ctx', 'fix_locations']
|
||||
|
||||
from ast import (Load, Store, Del,
|
||||
Assign, AnnAssign, AugAssign,
|
||||
|
|
@ -32,8 +32,8 @@ class _CtxFixer(Walker):
|
|||
return self.generic_visit(tree)
|
||||
|
||||
def _fix_one(self, tree):
|
||||
'''Fix one missing `ctx` attribute, using the currently active ctx class.'''
|
||||
if ("ctx" in type(tree)._fields and (not hasattr(tree, "ctx") or tree.ctx is None)):
|
||||
'''Fix one `ctx` attribute, using the currently active ctx class.'''
|
||||
if "ctx" in type(tree)._fields:
|
||||
tree.ctx = self.state.ctxclass()
|
||||
|
||||
def _setup_subtree_contexts(self, tree):
|
||||
|
|
@ -84,29 +84,30 @@ class _CtxFixer(Walker):
|
|||
self.withstate(tree.targets, ctxclass=Del)
|
||||
|
||||
|
||||
def fix_missing_ctx(tree):
|
||||
'''Fix any missing `ctx` attributes in `tree`.
|
||||
def fix_ctx(tree):
|
||||
'''Fix `ctx` attributes in `tree`.
|
||||
|
||||
Modifies `tree` in-place. For convenience, returns the modified `tree`.
|
||||
'''
|
||||
return _CtxFixer().visit(tree)
|
||||
|
||||
|
||||
def fix_missing_locations(tree, reference_node, *, mode):
|
||||
def fix_locations(tree, reference_node, *, mode):
|
||||
'''Like `ast.fix_missing_locations`, but customized for a macro expander.
|
||||
|
||||
Differences:
|
||||
|
||||
- If `reference_node` has no location info, return immediately (no-op).
|
||||
- If `reference_node` has source no location info, return immediately (no-op).
|
||||
- If `tree is None`, return immediately (no-op).
|
||||
- If `tree` is a `list` of AST nodes, loop over it.
|
||||
|
||||
The `mode` parameter:
|
||||
|
||||
- If `mode="reference"`, populate any missing location info by
|
||||
copying it from `reference_node`. Always use the same values.
|
||||
copying it from `reference_node`. Always use the same reference info.
|
||||
|
||||
Good for a macro expander.
|
||||
Good when expanding a macro invocation, to set the source location
|
||||
of any macro-generated nodes to that of the macro invocation node.
|
||||
|
||||
- If `mode="update"`, behave exactly like `ast.fix_missing_locations`,
|
||||
except that at the top level of `tree`, initialize `lineno` and
|
||||
|
|
@ -123,7 +124,7 @@ def fix_missing_locations(tree, reference_node, *, mode):
|
|||
|
||||
Good when `tree` is a code template that comes from another file,
|
||||
so that any line numbers already in the AST would be misleading
|
||||
at the use site.
|
||||
at the use site (because they point to lines in that other file).
|
||||
|
||||
Modifies `tree` in-place. For convenience, returns the modified `tree`.
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
"""Colorize terminal output using Colorama."""
|
||||
|
||||
__all__ = ["setcolor", "colorize", "ColorScheme",
|
||||
"Fore", "Back", "Style"]
|
||||
|
||||
from colorama import init as colorama_init, Fore, Back, Style
|
||||
colorama_init()
|
||||
|
||||
def setcolor(*colors):
|
||||
"""Set color for ANSI terminal display.
|
||||
|
||||
For available `colors`, see `Fore`, `Back` and `Style`.
|
||||
|
||||
Each entry can also be a tuple (arbitrarily nested), which is useful
|
||||
for defining compound styles.
|
||||
"""
|
||||
def _setcolor(color):
|
||||
if isinstance(color, (list, tuple)):
|
||||
return "".join(_setcolor(elt) for elt in color)
|
||||
return color
|
||||
return _setcolor(colors)
|
||||
|
||||
def colorize(text, *colors, reset=True):
|
||||
"""Colorize string `text` for ANSI terminal display. Reset color at end of `text`.
|
||||
|
||||
For available `colors`, see `Fore`, `Back` and `Style`.
|
||||
These are imported from `colorama`.
|
||||
|
||||
Usage::
|
||||
|
||||
colorize("I'm new here", Fore.GREEN)
|
||||
colorize("I'm bold and bluetiful", Style.BRIGHT, Fore.BLUE)
|
||||
|
||||
Each entry can also be a tuple (arbitrarily nested), which is useful
|
||||
for defining compound styles::
|
||||
|
||||
BRIGHT_BLUE = (Style.BRIGHT, Fore.BLUE)
|
||||
...
|
||||
colorize("I'm bold and bluetiful, too", BRIGHT_BLUE)
|
||||
|
||||
**CAUTION**: Does not nest. Style resets after the colorized text.
|
||||
"""
|
||||
return "{}{}{}".format(setcolor(colors),
|
||||
text,
|
||||
setcolor(Style.RESET_ALL))
|
||||
|
||||
|
||||
class ColorScheme:
|
||||
"""The color scheme for debug utilities.
|
||||
|
||||
See `Fore`, `Back`, `Style` in `colorama` for valid values. To make a
|
||||
compound style, place the values into a tuple.
|
||||
|
||||
The defaults are designed to fit the "Solarized" (Zenburn-like) theme
|
||||
of `gnome-terminal`, with "Show bold text in bright colors" set to OFF.
|
||||
But they should work with most color schemes.
|
||||
"""
|
||||
_RESET = Style.RESET_ALL
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# unparse
|
||||
|
||||
LINENUMBER = Style.DIM
|
||||
|
||||
LANGUAGEKEYWORD = (Style.BRIGHT, Fore.YELLOW) # for, if, import, ...
|
||||
|
||||
DEFNAME = (Style.BRIGHT, Fore.CYAN) # name of a function or class being defined
|
||||
DECORATOR = Fore.LIGHTBLUE_EX
|
||||
|
||||
STRING = Fore.GREEN
|
||||
NUMBER = Fore.GREEN
|
||||
NAMECONSTANT = Fore.GREEN # True, False, None
|
||||
|
||||
BUILTINEXCEPTION = Fore.CYAN
|
||||
BUILTINOTHER = Style.BRIGHT # str, property, print, ...
|
||||
|
||||
INVISIBLENODE = Style.DIM # AST node with no surface syntax repr (Module, Expr)
|
||||
|
||||
# AST markers for data-driven communication within the macro expander
|
||||
ASTMARKER = Style.DIM # the "$AstMarker" title
|
||||
ASTMARKERCLASS = Fore.YELLOW # the actual marker type name
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# format_bindings, step_expansion, StepExpansion
|
||||
|
||||
HEADING = (Style.BRIGHT, Fore.LIGHTBLUE_EX)
|
||||
SOURCEFILENAME = (Style.BRIGHT, Fore.RESET)
|
||||
|
||||
# format_bindings
|
||||
MACROBINDING = (Style.BRIGHT, Fore.RESET)
|
||||
GREYEDOUT = Style.DIM
|
||||
|
||||
# step_expansion
|
||||
TREEID = (Style.NORMAL, Fore.LIGHTBLUE_EX)
|
||||
|
||||
# StepExpansion
|
||||
ATTENTION = (Style.BRIGHT, Fore.GREEN) # the string "DialectExpander debug mode"
|
||||
DIALECTTRANSFORMER = (Style.BRIGHT, Fore.YELLOW)
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# dump
|
||||
|
||||
NODETYPE = (Style.BRIGHT, Fore.LIGHTBLUE_EX)
|
||||
FIELDNAME = Fore.YELLOW
|
||||
BAREVALUE = Fore.GREEN
|
||||
|
|
@ -8,7 +8,7 @@ from ast import NodeTransformer, AST
|
|||
from contextlib import contextmanager
|
||||
from collections import ChainMap
|
||||
|
||||
from .astfixers import fix_missing_ctx, fix_missing_locations
|
||||
from .astfixers import fix_ctx, fix_locations
|
||||
from .markers import ASTMarker, delete_markers
|
||||
from .utils import flatten_suite, format_location
|
||||
|
||||
|
|
@ -147,15 +147,18 @@ class BaseMacroExpander(NodeTransformer):
|
|||
macro function, place them in a dictionary and pass that dictionary
|
||||
as `kw`.
|
||||
'''
|
||||
macro = self.bindings[macroname]
|
||||
loc = format_location(self.filename, target, sourcecode)
|
||||
|
||||
macro = self.isbound(macroname)
|
||||
if not macro:
|
||||
raise MacroExpansionError(f"{loc}\nThe name '{macroname}' is not bound to a macro.")
|
||||
|
||||
kw = kw or {}
|
||||
kw.update({
|
||||
'syntax': syntax,
|
||||
'expander': self,
|
||||
'invocation': target})
|
||||
|
||||
loc = format_location(self.filename, target, sourcecode)
|
||||
|
||||
# Expand the macro.
|
||||
try:
|
||||
expansion = _apply_macro(macro, tree, kw)
|
||||
|
|
@ -200,8 +203,8 @@ class BaseMacroExpander(NodeTransformer):
|
|||
if it detects any more macro invocations.
|
||||
'''
|
||||
if expansion is not None:
|
||||
expansion = fix_missing_locations(expansion, target, mode="reference")
|
||||
expansion = fix_missing_ctx(expansion)
|
||||
expansion = fix_locations(expansion, target, mode="reference")
|
||||
expansion = fix_ctx(expansion)
|
||||
if self.recursive:
|
||||
expansion = self.visit(expansion)
|
||||
|
||||
|
|
@ -231,4 +234,9 @@ def global_postprocess(tree):
|
|||
Call this after macro expansion is otherwise done, before sending `tree`
|
||||
to Python's `compile`.
|
||||
'''
|
||||
return delete_markers(tree, cls=MacroExpanderMarker)
|
||||
tree = delete_markers(tree, cls=MacroExpanderMarker)
|
||||
# A name macro, appearing as an assignment target, gets the wrong ctx,
|
||||
# because when expanding the name macro, the expander sees only the Name
|
||||
# node, and thus puts an `ast.Load` there as the ctx.
|
||||
tree = fix_ctx(tree)
|
||||
return tree
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from sys import stderr
|
|||
import textwrap
|
||||
|
||||
from .astdumper import dump
|
||||
from .colorizer import setcolor, colorize, ColorScheme
|
||||
from .dialects import StepExpansion # re-export for discoverability, it's a debug feature
|
||||
from .expander import MacroCollector, namemacro, parametricmacro
|
||||
from .unparser import unparse_with_fallbacks
|
||||
|
|
@ -54,7 +55,7 @@ def step_expansion(tree, *, args, syntax, expander, **kw):
|
|||
if syntax not in ("expr", "block"):
|
||||
raise SyntaxError("`step_expansion` is an expr and block macro only")
|
||||
|
||||
formatter = functools.partial(unparse_with_fallbacks, debug=True)
|
||||
formatter = functools.partial(unparse_with_fallbacks, debug=True, color=True)
|
||||
if args:
|
||||
if len(args) != 1:
|
||||
raise SyntaxError("expected `step_expansion` or `step_expansion['mode_str']`")
|
||||
|
|
@ -68,14 +69,17 @@ def step_expansion(tree, *, args, syntax, expander, **kw):
|
|||
if mode not in ("unparse", "dump"):
|
||||
raise ValueError(f"expected mode_str either 'unparse' or 'dump', got {repr(mode)}")
|
||||
if mode == "dump":
|
||||
formatter = functools.partial(dump, include_attributes=True)
|
||||
formatter = functools.partial(dump, include_attributes=True, color=True)
|
||||
|
||||
c, CS = setcolor, ColorScheme
|
||||
|
||||
with _step_expansion_level.changed_by(+1):
|
||||
indent = 2 * _step_expansion_level.value
|
||||
stars = indent * '*'
|
||||
codeindent = indent
|
||||
tag = id(tree)
|
||||
print(f"{stars}Tree 0x{tag:x} before macro expansion:", file=stderr)
|
||||
print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} {c(CS.HEADING)}before macro expansion:{c(CS._RESET)}",
|
||||
file=stderr)
|
||||
print(textwrap.indent(formatter(tree), codeindent * ' '), file=stderr)
|
||||
mc = MacroCollector(expander)
|
||||
mc.visit(tree)
|
||||
|
|
@ -84,12 +88,14 @@ def step_expansion(tree, *, args, syntax, expander, **kw):
|
|||
step += 1
|
||||
tree = expander.visit_once(tree) # -> Done(body=...)
|
||||
tree = tree.body
|
||||
print(f"{stars}Tree 0x{tag:x} after step {step}:", file=stderr)
|
||||
print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} {c(CS.HEADING)}after step {step}:{c(CS._RESET)}",
|
||||
file=stderr)
|
||||
print(textwrap.indent(formatter(tree), codeindent * ' '), file=stderr)
|
||||
mc.clear()
|
||||
mc.visit(tree)
|
||||
plural = "s" if step != 1 else ""
|
||||
print(f"{stars}Tree 0x{tag:x} macro expansion complete after {step} step{plural}.", file=stderr)
|
||||
print(f"{c(CS.HEADING)}{stars}Tree {c(CS.TREEID)}0x{tag:x} {c(CS.HEADING)}macro expansion complete after {step} step{plural}.{c(CS._RESET)}",
|
||||
file=stderr)
|
||||
return tree
|
||||
|
||||
|
||||
|
|
@ -103,16 +109,23 @@ def show_bindings(tree, *, syntax, expander, **kw):
|
|||
|
||||
This can appear in any expression position, and at run-time evaluates to `None`.
|
||||
|
||||
At macro expansion time, for each macro binding, this prints to `sys.stderr`
|
||||
the macro name, and the fully qualified name of the corresponding macro function.
|
||||
At macro expansion time, when the macro expander reaches the `show_bindings`
|
||||
expression, bindings *that are in effect at that point in time* are shown.
|
||||
|
||||
(That disclaimer is important, because hygienically unquoted macros may add
|
||||
new bindings as those expressions are reached, and true to the dynamic nature
|
||||
of Python, when a macro runs, it is allowed to edit the expander's bindings.)
|
||||
|
||||
For each macro binding, we print to `sys.stderr` the macro name, and the
|
||||
fully qualified name of the corresponding macro function.
|
||||
|
||||
Any bindings that have an uuid as part of the name are hygienically
|
||||
unquoted macros. Those make a per-process global binding across all modules
|
||||
and all expander instances.
|
||||
unquoted macros. Those bindings are global across all modules and
|
||||
all expander instances.
|
||||
"""
|
||||
if syntax != "name":
|
||||
raise SyntaxError("`show_bindings` is an identifier macro only")
|
||||
print(format_bindings(expander), file=stderr)
|
||||
print(format_bindings(expander, color=True), file=stderr)
|
||||
# Can't just delete the node (return None) if it's in an Expr(value=...).
|
||||
#
|
||||
# For correct coverage reporting, we can't return a `Constant`, because CPython
|
||||
|
|
@ -125,19 +138,34 @@ def show_bindings(tree, *, syntax, expander, **kw):
|
|||
return ast.Call(func=lam, args=[], keywords=[])
|
||||
|
||||
|
||||
def format_bindings(expander):
|
||||
def format_bindings(expander, *, color=False):
|
||||
"""Return a human-readable report of the macro bindings currently seen by `expander`.
|
||||
|
||||
Global bindings (across all expanders) are also included.
|
||||
|
||||
If `color=True`, use `colorama` to colorize the output. (For terminals.)
|
||||
|
||||
If you want to access them programmatically, just access `expander.bindings` directly.
|
||||
"""
|
||||
def maybe_setcolor(*colors):
|
||||
if not color:
|
||||
return ""
|
||||
return setcolor(*colors)
|
||||
def maybe_colorize(text, *colors):
|
||||
if not color:
|
||||
return text
|
||||
return colorize(text, *colors)
|
||||
|
||||
c, CS = maybe_setcolor, ColorScheme
|
||||
|
||||
with io.StringIO() as output:
|
||||
output.write(f"Macro bindings for {expander.filename}:\n")
|
||||
output.write(f"{c(CS.HEADING)}Macro bindings for {c(CS.SOURCEFILENAME)}{expander.filename}{c(CS.HEADING)}:{c(CS._RESET)}\n")
|
||||
if not expander.bindings:
|
||||
output.write(" <no bindings>\n")
|
||||
output.write(maybe_colorize(" <no bindings>\n",
|
||||
ColorScheme.GREYEDOUT))
|
||||
else:
|
||||
for k, v in sorted(expander.bindings.items()):
|
||||
k = maybe_colorize(k, ColorScheme.MACROBINDING)
|
||||
output.write(f" {k}: {format_macrofunction(v)}\n")
|
||||
return output.getvalue()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,11 +5,13 @@ __all__ = ["Dialect",
|
|||
"expand_dialects"]
|
||||
|
||||
import ast
|
||||
import functools
|
||||
import importlib
|
||||
import importlib.util
|
||||
import re
|
||||
from sys import stderr
|
||||
|
||||
from .colorizer import setcolor, colorize, ColorScheme
|
||||
from .coreutils import ismacroimport, get_macros
|
||||
from .unparser import unparse_with_fallbacks
|
||||
|
||||
|
|
@ -108,7 +110,7 @@ class Dialect:
|
|||
return NotImplemented
|
||||
|
||||
|
||||
_message_header = "**StepExpansion: "
|
||||
_message_header = colorize("**StepExpansion: ", ColorScheme.HEADING)
|
||||
class StepExpansion(Dialect): # actually part of public API of mcpyrate.debug, for discoverability
|
||||
"""[dialect] Show each step of expansion while dialect-expanding the module.
|
||||
|
||||
|
|
@ -125,7 +127,9 @@ class StepExpansion(Dialect): # actually part of public API of mcpyrate.debug,
|
|||
"""
|
||||
def transform_source(self, text):
|
||||
self.expander.debugmode = True
|
||||
print(f"{_message_header}{self.expander.filename} enabled DialectExpander debug mode while taking step {self.expander._step + 1}.", file=stderr)
|
||||
c, CS = setcolor, ColorScheme
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.expander.filename} {c(CS.HEADING)}enabled {c(CS.ATTENTION)}DialectExpander debug mode {c(CS.HEADING)}while taking step {self.expander._step + 1}.{c(CS._RESET)}"
|
||||
print(_message_header + msg, file=stderr)
|
||||
# Pass through the input (instead of returning `NotImplemented`) to
|
||||
# consider this as having taken a step, thus triggering the debug mode
|
||||
# output printer. (If this was the first dialect applied, our output is
|
||||
|
|
@ -171,17 +175,19 @@ class DialectExpander:
|
|||
|
||||
def transform_ast(self, tree):
|
||||
'''Apply all whole-module AST transformers.'''
|
||||
formatter = functools.partial(unparse_with_fallbacks, debug=True, color=True)
|
||||
return self._transform(tree, kind="AST",
|
||||
find_dialectimport=self.find_dialectimport_ast,
|
||||
transform="transform_ast",
|
||||
format_for_display=unparse_with_fallbacks)
|
||||
format_for_display=formatter)
|
||||
|
||||
def _transform(self, content, *, kind, find_dialectimport, transform, format_for_display):
|
||||
c, CS = setcolor, ColorScheme
|
||||
if self.debugmode:
|
||||
plural = "s" if self._step != 1 else ""
|
||||
print(f"{_message_header}{self.filename} before dialect {kind} transformers ({self._step} step{plural} total):\n", file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}before dialect {c(CS.DIALECTTRANSFORMER)}{kind} {c(CS.HEADING)}transformers ({self._step} step{plural} total):{c(CS._RESET)}\n"
|
||||
print(_message_header + msg, file=stderr)
|
||||
print(format_for_display(content), file=stderr)
|
||||
print("-" * 79, file=stderr)
|
||||
|
||||
while True:
|
||||
module_absname, bindings = find_dialectimport(content)
|
||||
|
|
@ -218,14 +224,14 @@ class DialectExpander:
|
|||
self._step += 1
|
||||
|
||||
if self.debugmode:
|
||||
print(f"{_message_header}{self.filename} after {module_absname}.{dialectname}.{transform} (step {self._step}):\n", file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}after {c(CS.DIALECTTRANSFORMER)}{module_absname}.{dialectname}.{transform} {c(CS.HEADING)}(step {self._step}):{c(CS._RESET)}\n"
|
||||
print(_message_header + msg, file=stderr)
|
||||
print(format_for_display(content), file=stderr)
|
||||
print("-" * 79, file=stderr)
|
||||
|
||||
if self.debugmode:
|
||||
plural = "s" if self._step != 1 else ""
|
||||
print(f"{_message_header}All dialect {kind} transformers completed for {self.filename} ({self._step} step{plural} total).", file=stderr)
|
||||
print("-" * 79, file=stderr)
|
||||
msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}completed all dialect {c(CS.DIALECTTRANSFORMER)}{kind} {c(CS.HEADING)}transforms ({self._step} step{plural} total).{c(CS._RESET)}"
|
||||
print(_message_header + msg, file=stderr)
|
||||
|
||||
return content
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
__all__ = ['source_to_xcode', 'path_xstats', 'invalidate_xcaches']
|
||||
|
||||
import ast
|
||||
import distutils.sysconfig
|
||||
import importlib.util
|
||||
from importlib.machinery import FileFinder, SourceFileLoader
|
||||
import tokenize
|
||||
|
|
@ -32,14 +33,14 @@ def source_to_xcode(self, data, path, *, _optimize=-1):
|
|||
|
||||
remaining_markers = get_markers(expansion)
|
||||
if remaining_markers:
|
||||
raise MacroExpansionError("{path}: AST markers remaining after expansion: {remaining_markers}")
|
||||
raise MacroExpansionError(f"{path}: AST markers remaining after expansion: {remaining_markers}")
|
||||
|
||||
return compile(expansion, path, 'exec', dont_inherit=True, optimize=_optimize)
|
||||
|
||||
|
||||
# TODO: Support PEP552 (Deterministic pycs). Need to intercept source file hashing, too.
|
||||
# TODO: https://www.python.org/dev/peps/pep-0552/
|
||||
_path_stats = SourceFileLoader.path_stats
|
||||
_stdlib_path_stats = SourceFileLoader.path_stats
|
||||
_xstats_cache = {}
|
||||
def path_xstats(self, path):
|
||||
'''[mcpyrate] Compute a `.py` source file's mtime, accounting for macro-imports.
|
||||
|
|
@ -56,8 +57,11 @@ def path_xstats(self, path):
|
|||
If `path` does not end in `.py`, delegate to the standard implementation
|
||||
of `SourceFileLoader.path_stats`.
|
||||
'''
|
||||
if not path.endswith(".py"):
|
||||
return _path_stats(path)
|
||||
# Ignore stdlib, it's big and doesn't use macros. Allows faster error
|
||||
# exits, because an uncaught exception causes Python to load a ton of
|
||||
# .py based stdlib modules. Also makes `macropython -i` start faster.
|
||||
if path in _stdlib_sourcefile_paths or not path.endswith(".py"):
|
||||
return _stdlib_path_stats(self, path)
|
||||
if path in _xstats_cache:
|
||||
return _xstats_cache[path]
|
||||
|
||||
|
|
@ -162,11 +166,25 @@ def path_xstats(self, path):
|
|||
return result
|
||||
|
||||
|
||||
_invalidate_caches = FileFinder.invalidate_caches
|
||||
_stdlib_invalidate_caches = FileFinder.invalidate_caches
|
||||
def invalidate_xcaches(self):
|
||||
'''[mcpyrate] Clear the macro dependency tree cache.
|
||||
|
||||
Then delegate to the standard implementation of `FileFinder.invalidate_caches`.
|
||||
'''
|
||||
_xstats_cache.clear()
|
||||
return _invalidate_caches(self)
|
||||
return _stdlib_invalidate_caches(self)
|
||||
|
||||
|
||||
def _detect_stdlib_sourcefile_paths():
|
||||
'''Return a set of full paths of `.py` files that are part of Python's standard library.'''
|
||||
# Adapted from StackOverflow answer by Adam Spiers, https://stackoverflow.com/a/8992937
|
||||
# Note we don't want to get module names, but full paths to `.py` files.
|
||||
stdlib_dir = distutils.sysconfig.get_python_lib(standard_lib=True)
|
||||
paths = set()
|
||||
for root, dirs, files in os.walk(stdlib_dir):
|
||||
for nm in files:
|
||||
if nm[-3:] == '.py':
|
||||
paths.add(os.path.join(root, nm))
|
||||
return paths
|
||||
_stdlib_sourcefile_paths = _detect_stdlib_sourcefile_paths()
|
||||
|
|
|
|||
|
|
@ -46,16 +46,6 @@ def _mcpyrate_quotes_attr(attr):
|
|||
attr=attr,
|
||||
ctx=ast.Load())
|
||||
|
||||
def _capture_into(mapping, value, basename):
|
||||
for k, v in mapping.items():
|
||||
if v is value:
|
||||
key = k
|
||||
break
|
||||
else:
|
||||
key = gensym(basename)
|
||||
mapping[key] = value
|
||||
return key
|
||||
|
||||
def capture(value, name):
|
||||
"""Hygienically capture a run-time value.
|
||||
|
||||
|
|
@ -64,19 +54,11 @@ def capture(value, name):
|
|||
|
||||
The return value is an AST that, when compiled and run, returns the
|
||||
captured value (even in another Python process later).
|
||||
|
||||
Hygienically captured macro invocations are treated using a different
|
||||
mechanism; see `mcpyrate.core.global_bindings`.
|
||||
"""
|
||||
# If we didn't need to consider bytecode caching, we could just store the
|
||||
# value in a registry that is populated at macro expansion time. Each
|
||||
# unique value (by `id`) could be stored only once.
|
||||
#
|
||||
# key = _capture_into(_hygienic_registry, value, name)
|
||||
# return ast.Call(_mcpyrate_quotes_attr("lookup"),
|
||||
# [ast.Constant(value=key)],
|
||||
# [])
|
||||
|
||||
# But we want to support bytecode caching. To avoid introducing hard-to-find
|
||||
# bugs into user code, we must provide consistent semantics, regardless of
|
||||
# whether updating of the bytecode cache is actually enabled or not (see
|
||||
|
|
@ -101,24 +83,49 @@ def capture(value, name):
|
|||
#
|
||||
frozen_value = pickle.dumps(value)
|
||||
return ast.Call(_mcpyrate_quotes_attr("lookup"),
|
||||
[ast.Tuple(elts=[ast.Constant(value=name), # for human-readability of expanded code
|
||||
[ast.Tuple(elts=[ast.Constant(value=name),
|
||||
ast.Constant(value=frozen_value)])],
|
||||
[])
|
||||
|
||||
_lookup_cache = {}
|
||||
def lookup(key):
|
||||
"""Look up a hygienically captured run-time value."""
|
||||
# if type(key) is str: # captured in sys.dont_write_bytecode mode, in this process
|
||||
# return _hygienic_registry[key]
|
||||
# else: # frozen into macro-expanded code
|
||||
# name, frozen_value = key
|
||||
# return pickle.loads(frozen_value)
|
||||
name, frozen_value = key
|
||||
cachekey = (name, id(frozen_value)) # id() so each capture instance behaves independently
|
||||
if cachekey not in _lookup_cache:
|
||||
_lookup_cache[cachekey] = pickle.loads(frozen_value)
|
||||
return _lookup_cache[cachekey]
|
||||
|
||||
def capture_macro(macro, name):
|
||||
"""Hygienically capture a macro.
|
||||
|
||||
`macro`: A macro function. Must be picklable.
|
||||
`name`: The name of the macro, as it appeared in the bindings of
|
||||
the expander it was captured from. For human-readability.
|
||||
|
||||
The return value is an AST that, when compiled and run, injects the
|
||||
macro into the expander's global macro bindings table (even in another
|
||||
Python process later), and then evaluates to that macro name as an
|
||||
`ast.Name` (so further expansion of that AST will invoke the macro).
|
||||
"""
|
||||
frozen_macro = pickle.dumps(macro)
|
||||
unique_name = gensym(name)
|
||||
return ast.Call(_mcpyrate_quotes_attr("lookup_macro"),
|
||||
[ast.Tuple(elts=[ast.Constant(value=unique_name),
|
||||
ast.Constant(value=frozen_macro)])],
|
||||
[])
|
||||
|
||||
def lookup_macro(key):
|
||||
"""Look up a hygienically captured macro.
|
||||
|
||||
This will inject the macro to the global macro bindings table,
|
||||
and then evaluate to that macro name, as an `ast.Name`.
|
||||
"""
|
||||
unique_name, frozen_macro = key
|
||||
if unique_name not in global_bindings:
|
||||
global_bindings[unique_name] = pickle.loads(frozen_macro)
|
||||
return ast.Name(id=unique_name)
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
|
||||
def astify(x, expander=None): # like `macropy`'s `ast_repr`
|
||||
|
|
@ -154,15 +161,17 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr`
|
|||
if expander and type(x.body) is ast.Name:
|
||||
function = expander.isbound(x.body.id)
|
||||
if function:
|
||||
# Hygienically capture a macro name. We do this immediately,
|
||||
# during the expansion of `q`. This allows macros in scope at
|
||||
# the use site of `q` to be hygienically propagated out to the
|
||||
# use site of the macro that used `q`. So you can write macros
|
||||
# that `q[h[macroname][...]]` and `macroname` doesn't have to be
|
||||
# macro-imported wherever that code gets spliced in.
|
||||
macroname = x.body.id
|
||||
uniquename = _capture_into(global_bindings, function, macroname)
|
||||
return recurse(ast.Name(id=uniquename))
|
||||
# Hygienically capture a macro. We do this immediately,
|
||||
# during the expansion of `q`, because the value we want to
|
||||
# store, i.e. the macro function, is available only at
|
||||
# macro-expansion time.
|
||||
#
|
||||
# This allows macros in scope at the use site of `q` to be
|
||||
# hygienically propagated out to the use site of the macro
|
||||
# that used `q`. So you can write macros that `q[h[macroname][...]]`,
|
||||
# and `macroname` doesn't have to be macro-imported wherever
|
||||
# that code gets spliced in.
|
||||
return capture_macro(function, x.body.id)
|
||||
# Hygienically capture a garden variety run-time value.
|
||||
# At the use site of q[], this captures the value, and rewrites itself
|
||||
# into a lookup. At the use site of the macro that used q[], that
|
||||
|
|
@ -373,10 +382,6 @@ def n(tree, *, syntax, **kw):
|
|||
if _quotelevel.value < 1:
|
||||
raise SyntaxError("`n` encountered while quotelevel < 1")
|
||||
with _quotelevel.changed_by(-1):
|
||||
ret = ASTLiteral(astify(ast.Name(id=ASTLiteral(tree))))
|
||||
print(tree.__dict__)
|
||||
print(ret.body.func.value.value.value.__dict__)
|
||||
print(unparse(ret))
|
||||
return ASTLiteral(astify(ast.Name(id=ASTLiteral(tree))))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ __all__ = ["splice_statements", "splice_dialect"]
|
|||
import ast
|
||||
from copy import deepcopy
|
||||
|
||||
from .astfixers import fix_missing_locations
|
||||
from .astfixers import fix_locations
|
||||
from .coreutils import ismacroimport
|
||||
from .walker import Walker
|
||||
|
||||
|
|
@ -145,7 +145,7 @@ def splice_dialect(body, template, tag="__paste_here__"):
|
|||
#
|
||||
# Pretend the template code appears at the beginning of the user module.
|
||||
for stmt in template:
|
||||
fix_missing_locations(stmt, body[0], mode="overwrite")
|
||||
fix_locations(stmt, body[0], mode="overwrite")
|
||||
|
||||
# TODO: remove ast.Str once we bump minimum language version to Python 3.8
|
||||
if type(body[0]) is ast.Expr and type(body[0].value) in (ast.Constant, ast.Str):
|
||||
|
|
|
|||
|
|
@ -4,16 +4,26 @@
|
|||
__all__ = ['UnparserError', 'unparse', 'unparse_with_fallbacks']
|
||||
|
||||
import ast
|
||||
import builtins
|
||||
from contextlib import contextmanager
|
||||
import io
|
||||
import sys
|
||||
|
||||
from .astdumper import dump # fallback
|
||||
from .colorizer import colorize, ColorScheme
|
||||
from .markers import ASTMarker
|
||||
|
||||
# Large float and imaginary literals get turned into infinities in the AST.
|
||||
# We unparse those infinities to INFSTR.
|
||||
INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
|
||||
|
||||
# for syntax highlighting
|
||||
_all_public_builtins = {x for x in dir(builtins) if not x.startswith("_")}
|
||||
builtin_exceptions = {x for x in _all_public_builtins if x.endswith("Error")}
|
||||
builtin_warnings = {x for x in _all_public_builtins if x.endswith("Warning")}
|
||||
builtin_exceptions_and_warnings = builtin_exceptions | builtin_warnings
|
||||
builtin_others = _all_public_builtins - builtin_exceptions_and_warnings
|
||||
|
||||
|
||||
class UnparserError(SyntaxError):
|
||||
"""Failed to unparse the given AST."""
|
||||
|
|
@ -39,22 +49,56 @@ class Unparser:
|
|||
for the abstract syntax. Original formatting is disregarded.
|
||||
"""
|
||||
|
||||
def __init__(self, tree, *, file=sys.stdout, debug=False):
|
||||
def __init__(self, tree, *, file=sys.stdout, debug=False, color=False):
|
||||
"""Print the source for `tree` to `file`.
|
||||
|
||||
`debug`: bool, print invisible nodes (`Module`, `Expr`).
|
||||
|
||||
The output is then not valid Python, but may better show
|
||||
the problem when code produced by a macro mysteriously
|
||||
fails to compile (even though the unparse looks ok).
|
||||
fails to compile (even though a non-debug unparse looks ok).
|
||||
|
||||
`color`: bool, use Colorama to color output. For syntax highlighting
|
||||
when printing into a terminal.
|
||||
"""
|
||||
self.debug = debug
|
||||
self.color = color
|
||||
self._color_override = False # to syntax highlight decorators
|
||||
self.f = file
|
||||
self._indent = 0
|
||||
self.dispatch(tree)
|
||||
print("", file=self.f)
|
||||
self.f.flush()
|
||||
|
||||
def maybe_colorize(self, text, *colors):
|
||||
"Colorize text if color is enabled."
|
||||
if self._color_override:
|
||||
return text
|
||||
if not self.color:
|
||||
return text
|
||||
return colorize(text, *colors)
|
||||
|
||||
def python_keyword(self, text):
|
||||
"Shorthand to colorize a language keyword such as `def`, `for`, ..."
|
||||
return self.maybe_colorize(text, ColorScheme.LANGUAGEKEYWORD)
|
||||
|
||||
def nocolor(self):
|
||||
"""Context manager. Temporarily prevent coloring.
|
||||
|
||||
Useful for syntax highlighting decorators (so that the method rendering
|
||||
the decorator may force a particular color, instead of allowing
|
||||
auto-coloring based on the data in the decorator AST node).
|
||||
"""
|
||||
@contextmanager
|
||||
def _nocolor():
|
||||
old_color_override = self._color_override
|
||||
self._color_override = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._color_override = old_color_override
|
||||
return _nocolor()
|
||||
|
||||
def fill(self, text="", *, lineno_node=None):
|
||||
"Indent a piece of text, according to the current indentation level."
|
||||
self.write("\n")
|
||||
|
|
@ -66,7 +110,8 @@ class Unparser:
|
|||
#
|
||||
# Assume line numbers usually have at most 4 digits, but
|
||||
# degrade gracefully for those crazy 5-digit source files.
|
||||
self.write(f"L{lineno:5d} " if lineno else "L ---- ")
|
||||
self.write(self.maybe_colorize(f"L{lineno:5d} " if lineno else "L ---- ",
|
||||
ColorScheme.LINENUMBER))
|
||||
self.write(" " * self._indent + text)
|
||||
|
||||
def write(self, text):
|
||||
|
|
@ -111,7 +156,11 @@ class Unparser:
|
|||
self.dispatch(v)
|
||||
else:
|
||||
self.write(repr(v))
|
||||
self.fill(f"$ASTMarker<{tree.__class__.__name__}>", lineno_node=tree) # markers cannot be eval'd
|
||||
self.fill(self.maybe_colorize(f"$ASTMarker", ColorScheme.ASTMARKER),
|
||||
lineno_node=tree) # markers cannot be eval'd
|
||||
clsname = self.maybe_colorize(tree.__class__.__name__,
|
||||
ColorScheme.ASTMARKERCLASS)
|
||||
self.write(f"<{clsname}>")
|
||||
self.enter()
|
||||
self.write(" ")
|
||||
if len(tree._fields) == 1 and tree._fields[0] == "body":
|
||||
|
|
@ -128,7 +177,8 @@ class Unparser:
|
|||
def _Module(self, t):
|
||||
# TODO: Python 3.8 type_ignores. Since we don't store the source text, maybe ignore that?
|
||||
if self.debug:
|
||||
self.fill("$Module", lineno_node=t)
|
||||
self.fill(self.maybe_colorize("$Module", ColorScheme.INVISIBLENODE),
|
||||
lineno_node=t)
|
||||
self.enter()
|
||||
for stmt in t.body:
|
||||
self.dispatch(stmt)
|
||||
|
|
@ -140,7 +190,8 @@ class Unparser:
|
|||
# stmt
|
||||
def _Expr(self, t):
|
||||
if self.debug:
|
||||
self.fill("$Expr", lineno_node=t)
|
||||
self.fill(self.maybe_colorize("$Expr", ColorScheme.INVISIBLENODE),
|
||||
lineno_node=t)
|
||||
self.enter()
|
||||
self.write(" ")
|
||||
self.dispatch(t.value)
|
||||
|
|
@ -150,15 +201,15 @@ class Unparser:
|
|||
self.dispatch(t.value)
|
||||
|
||||
def _Import(self, t):
|
||||
self.fill("import ", lineno_node=t)
|
||||
self.fill(self.python_keyword("import "), lineno_node=t)
|
||||
interleave(lambda: self.write(", "), self.dispatch, t.names)
|
||||
|
||||
def _ImportFrom(self, t):
|
||||
self.fill("from ", lineno_node=t)
|
||||
self.fill(self.python_keyword("from "), lineno_node=t)
|
||||
self.write("." * t.level)
|
||||
if t.module:
|
||||
self.write(t.module)
|
||||
self.write(" import ")
|
||||
self.write(self.python_keyword(" import "))
|
||||
interleave(lambda: self.write(", "), self.dispatch, t.names)
|
||||
|
||||
def _Assign(self, t):
|
||||
|
|
@ -189,42 +240,42 @@ class Unparser:
|
|||
self.dispatch(t.value)
|
||||
|
||||
def _Return(self, t):
|
||||
self.fill("return", lineno_node=t)
|
||||
self.fill(self.python_keyword("return"), lineno_node=t)
|
||||
if t.value:
|
||||
self.write(" ")
|
||||
self.dispatch(t.value)
|
||||
|
||||
def _Pass(self, t):
|
||||
self.fill("pass", lineno_node=t)
|
||||
self.fill(self.python_keyword("pass"), lineno_node=t)
|
||||
|
||||
def _Break(self, t):
|
||||
self.fill("break", lineno_node=t)
|
||||
self.fill(self.python_keyword("break"), lineno_node=t)
|
||||
|
||||
def _Continue(self, t):
|
||||
self.fill("continue", lineno_node=t)
|
||||
self.fill(self.python_keyword("continue"), lineno_node=t)
|
||||
|
||||
def _Delete(self, t):
|
||||
self.fill("del ", lineno_node=t)
|
||||
self.fill(self.python_keyword("del "), lineno_node=t)
|
||||
interleave(lambda: self.write(", "), self.dispatch, t.targets)
|
||||
|
||||
def _Assert(self, t):
|
||||
self.fill("assert ", lineno_node=t)
|
||||
self.fill(self.python_keyword("assert "), lineno_node=t)
|
||||
self.dispatch(t.test)
|
||||
if t.msg:
|
||||
self.write(", ")
|
||||
self.dispatch(t.msg)
|
||||
|
||||
def _Global(self, t):
|
||||
self.fill("global ", lineno_node=t)
|
||||
self.fill(self.python_keyword("global "), lineno_node=t)
|
||||
interleave(lambda: self.write(", "), self.write, t.names)
|
||||
|
||||
def _Nonlocal(self, t):
|
||||
self.fill("nonlocal ", lineno_node=t)
|
||||
self.fill(self.python_keyword("nonlocal "), lineno_node=t)
|
||||
interleave(lambda: self.write(", "), self.write, t.names)
|
||||
|
||||
def _Await(self, t): # expr
|
||||
self.write("(")
|
||||
self.write("await")
|
||||
self.write(self.python_keyword("await"))
|
||||
if t.value:
|
||||
self.write(" ")
|
||||
self.dispatch(t.value)
|
||||
|
|
@ -232,7 +283,7 @@ class Unparser:
|
|||
|
||||
def _Yield(self, t): # expr
|
||||
self.write("(")
|
||||
self.write("yield")
|
||||
self.write(self.python_keyword("yield"))
|
||||
if t.value:
|
||||
self.write(" ")
|
||||
self.dispatch(t.value)
|
||||
|
|
@ -240,48 +291,48 @@ class Unparser:
|
|||
|
||||
def _YieldFrom(self, t): # expr
|
||||
self.write("(")
|
||||
self.write("yield from")
|
||||
self.write(self.python_keyword("yield from"))
|
||||
if t.value:
|
||||
self.write(" ")
|
||||
self.dispatch(t.value)
|
||||
self.write(")")
|
||||
|
||||
def _Raise(self, t):
|
||||
self.fill("raise", lineno_node=t)
|
||||
self.fill(self.python_keyword("raise"), lineno_node=t)
|
||||
if not t.exc:
|
||||
assert not t.cause
|
||||
return
|
||||
self.write(" ")
|
||||
self.dispatch(t.exc)
|
||||
if t.cause:
|
||||
self.write(" from ")
|
||||
self.write(self.python_keyword(" from "))
|
||||
self.dispatch(t.cause)
|
||||
|
||||
def _Try(self, t):
|
||||
self.fill("try", lineno_node=t)
|
||||
self.fill(self.python_keyword("try"), lineno_node=t)
|
||||
self.enter()
|
||||
self.dispatch(t.body)
|
||||
self.leave()
|
||||
for ex in t.handlers:
|
||||
self.dispatch(ex)
|
||||
if t.orelse:
|
||||
self.fill("else")
|
||||
self.fill(self.python_keyword("else"))
|
||||
self.enter()
|
||||
self.dispatch(t.orelse)
|
||||
self.leave()
|
||||
if t.finalbody:
|
||||
self.fill("finally")
|
||||
self.fill(self.python_keyword("finally"))
|
||||
self.enter()
|
||||
self.dispatch(t.finalbody)
|
||||
self.leave()
|
||||
|
||||
def _ExceptHandler(self, t):
|
||||
self.fill("except", lineno_node=t)
|
||||
self.fill(self.python_keyword("except"), lineno_node=t)
|
||||
if t.type:
|
||||
self.write(" ")
|
||||
self.dispatch(t.type)
|
||||
if t.name:
|
||||
self.write(" as ")
|
||||
self.write(self.python_keyword(" as "))
|
||||
self.write(t.name)
|
||||
self.enter()
|
||||
self.dispatch(t.body)
|
||||
|
|
@ -289,10 +340,18 @@ class Unparser:
|
|||
|
||||
def _ClassDef(self, t):
|
||||
self.write("\n")
|
||||
|
||||
for deco in t.decorator_list:
|
||||
self.fill("@", lineno_node=deco)
|
||||
self.dispatch(deco)
|
||||
self.fill("class " + t.name, lineno_node=t)
|
||||
self.fill(self.maybe_colorize("@", ColorScheme.DECORATOR),
|
||||
lineno_node=deco)
|
||||
self.write(ColorScheme.DECORATOR)
|
||||
with self.nocolor():
|
||||
self.dispatch(deco)
|
||||
self.write(ColorScheme._RESET)
|
||||
|
||||
class_str = (self.python_keyword("class ") +
|
||||
self.maybe_colorize(t.name, ColorScheme.DEFNAME))
|
||||
self.fill(class_str, lineno_node=t)
|
||||
self.write("(")
|
||||
comma = False
|
||||
for e in t.bases:
|
||||
|
|
@ -321,10 +380,17 @@ class Unparser:
|
|||
|
||||
def __FunctionDef_helper(self, t, fill_suffix):
|
||||
self.write("\n")
|
||||
|
||||
for deco in t.decorator_list:
|
||||
self.fill("@", lineno_node=deco)
|
||||
self.dispatch(deco)
|
||||
def_str = fill_suffix + " " + t.name + "("
|
||||
self.fill(self.maybe_colorize("@", ColorScheme.DECORATOR),
|
||||
lineno_node=deco)
|
||||
self.write(ColorScheme.DECORATOR)
|
||||
with self.nocolor():
|
||||
self.dispatch(deco)
|
||||
self.write(ColorScheme._RESET)
|
||||
|
||||
def_str = (self.python_keyword(fill_suffix) +
|
||||
" " + self.maybe_colorize(t.name, ColorScheme.DEFNAME) + "(")
|
||||
self.fill(def_str, lineno_node=t)
|
||||
self.dispatch(t.args)
|
||||
self.write(")")
|
||||
|
|
@ -337,28 +403,28 @@ class Unparser:
|
|||
# TODO: Python 3.8 type_comment, ignore it?
|
||||
|
||||
def _For(self, t):
|
||||
self.__For_helper("for ", t)
|
||||
self.__For_helper(self.python_keyword("for "), t)
|
||||
|
||||
def _AsyncFor(self, t):
|
||||
self.__For_helper("async for ", t)
|
||||
self.__For_helper(self.python_keyword("async for "), t)
|
||||
|
||||
def __For_helper(self, fill, t):
|
||||
self.fill(fill, lineno_node=t)
|
||||
self.dispatch(t.target)
|
||||
self.write(" in ")
|
||||
self.write(self.python_keyword(" in "))
|
||||
self.dispatch(t.iter)
|
||||
self.enter()
|
||||
self.dispatch(t.body)
|
||||
self.leave()
|
||||
if t.orelse:
|
||||
self.fill("else")
|
||||
self.fill(self.python_keyword("else"))
|
||||
self.enter()
|
||||
self.dispatch(t.orelse)
|
||||
self.leave()
|
||||
# TODO: Python 3.8 type_comment, ignore it?
|
||||
|
||||
def _If(self, t):
|
||||
self.fill("if ", lineno_node=t)
|
||||
self.fill(self.python_keyword("if "), lineno_node=t)
|
||||
self.dispatch(t.test)
|
||||
self.enter()
|
||||
self.dispatch(t.body)
|
||||
|
|
@ -367,32 +433,32 @@ class Unparser:
|
|||
while (t.orelse and len(t.orelse) == 1 and
|
||||
isinstance(t.orelse[0], ast.If)):
|
||||
t = t.orelse[0]
|
||||
self.fill("elif ")
|
||||
self.fill(self.python_keyword("elif "))
|
||||
self.dispatch(t.test)
|
||||
self.enter()
|
||||
self.dispatch(t.body)
|
||||
self.leave()
|
||||
# final else
|
||||
if t.orelse:
|
||||
self.fill("else")
|
||||
self.fill(self.python_keyword("else"))
|
||||
self.enter()
|
||||
self.dispatch(t.orelse)
|
||||
self.leave()
|
||||
|
||||
def _While(self, t):
|
||||
self.fill("while ", lineno_node=t)
|
||||
self.fill(self.python_keyword("while "), lineno_node=t)
|
||||
self.dispatch(t.test)
|
||||
self.enter()
|
||||
self.dispatch(t.body)
|
||||
self.leave()
|
||||
if t.orelse:
|
||||
self.fill("else")
|
||||
self.fill(self.python_keyword("else"))
|
||||
self.enter()
|
||||
self.dispatch(t.orelse)
|
||||
self.leave()
|
||||
|
||||
def _With(self, t):
|
||||
self.fill("with ", lineno_node=t)
|
||||
self.fill(self.python_keyword("with "), lineno_node=t)
|
||||
interleave(lambda: self.write(", "), self.dispatch, t.items)
|
||||
self.enter()
|
||||
self.dispatch(t.body)
|
||||
|
|
@ -400,7 +466,7 @@ class Unparser:
|
|||
# TODO: Python 3.8 type_comment, ignore it?
|
||||
|
||||
def _AsyncWith(self, t):
|
||||
self.fill("async with ", lineno_node=t)
|
||||
self.fill(self.python_keyword("async with "), lineno_node=t)
|
||||
interleave(lambda: self.write(", "), self.dispatch, t.items)
|
||||
self.enter()
|
||||
self.dispatch(t.body)
|
||||
|
|
@ -422,27 +488,38 @@ class Unparser:
|
|||
if type(t.value) in (int, float, complex):
|
||||
# Represent AST infinity as an overflowing decimal literal.
|
||||
v = repr(t.value).replace("inf", INFSTR)
|
||||
v = self.maybe_colorize(v, ColorScheme.NUMBER)
|
||||
elif t.value is Ellipsis:
|
||||
v = "..."
|
||||
else:
|
||||
v = repr(t.value)
|
||||
if t.value in (True, False, None):
|
||||
v = self.maybe_colorize(v, ColorScheme.NAMECONSTANT)
|
||||
elif type(t.value) in (str, bytes):
|
||||
v = self.maybe_colorize(v, ColorScheme.STRING)
|
||||
self.write(v)
|
||||
|
||||
def _Bytes(self, t): # up to Python 3.7
|
||||
self.write(repr(t.s))
|
||||
self.write(self.maybe_colorize(repr(t.s), ColorScheme.STRING))
|
||||
|
||||
def _Str(self, tree): # up to Python 3.7
|
||||
self.write(repr(tree.s))
|
||||
self.write(self.maybe_colorize(repr(tree.s), ColorScheme.STRING))
|
||||
|
||||
def _Name(self, t):
|
||||
self.write(t.id)
|
||||
v = t.id
|
||||
if v in builtin_exceptions_and_warnings:
|
||||
v = self.maybe_colorize(v, ColorScheme.BUILTINEXCEPTION)
|
||||
elif v in builtin_others:
|
||||
v = self.maybe_colorize(v, ColorScheme.BUILTINOTHER)
|
||||
self.write(v)
|
||||
|
||||
def _NameConstant(self, t): # up to Python 3.7
|
||||
self.write(repr(t.value))
|
||||
self.write(self.maybe_colorize(repr(t.value), ColorScheme.NAMECONSTANT))
|
||||
|
||||
def _Num(self, t): # up to Python 3.7
|
||||
# Represent AST infinity as an overflowing decimal literal.
|
||||
self.write(repr(t.n).replace("inf", INFSTR))
|
||||
v = repr(t.n).replace("inf", INFSTR)
|
||||
self.write(self.maybe_colorize(v, ColorScheme.NUMBER))
|
||||
|
||||
def _List(self, t):
|
||||
self.write("[")
|
||||
|
|
@ -481,21 +558,21 @@ class Unparser:
|
|||
|
||||
def _comprehension(self, t):
|
||||
if t.is_async:
|
||||
self.write(" async")
|
||||
self.write(" for ")
|
||||
self.write(self.python_keyword(" async"))
|
||||
self.write(self.python_keyword(" for "))
|
||||
self.dispatch(t.target)
|
||||
self.write(" in ")
|
||||
self.write(self.python_keyword(" in "))
|
||||
self.dispatch(t.iter)
|
||||
for if_clause in t.ifs:
|
||||
self.write(" if ")
|
||||
self.write(self.python_keyword(" if "))
|
||||
self.dispatch(if_clause)
|
||||
|
||||
def _IfExp(self, t):
|
||||
self.write("(")
|
||||
self.dispatch(t.body)
|
||||
self.write(" if ")
|
||||
self.write(self.python_keyword(" if "))
|
||||
self.dispatch(t.test)
|
||||
self.write(" else ")
|
||||
self.write(self.python_keyword(" else "))
|
||||
self.dispatch(t.orelse)
|
||||
self.write(")")
|
||||
|
||||
|
|
@ -556,7 +633,8 @@ class Unparser:
|
|||
boolops = {ast.And: 'and', ast.Or: 'or'}
|
||||
def _BoolOp(self, t):
|
||||
self.write("(")
|
||||
s = " %s " % self.boolops[t.op.__class__]
|
||||
s = self.python_keyword(self.boolops[t.op.__class__])
|
||||
s = f" {s} "
|
||||
interleave(lambda: self.write(s), self.dispatch, t.values)
|
||||
self.write(")")
|
||||
|
||||
|
|
@ -599,27 +677,29 @@ class Unparser:
|
|||
self.write("'")
|
||||
|
||||
def _FormattedValue_helper(self, t):
|
||||
self.write("{")
|
||||
def c(text):
|
||||
return self.maybe_colorize(text, ColorScheme.STRING)
|
||||
self.write(c("{"))
|
||||
self.dispatch(t.value)
|
||||
if t.conversion == 115:
|
||||
self.write("!s")
|
||||
self.write(c("!s"))
|
||||
elif t.conversion == 114:
|
||||
self.write("!r")
|
||||
self.write(c("!r"))
|
||||
elif t.conversion == 97:
|
||||
self.write("!a")
|
||||
self.write(c("!a"))
|
||||
elif t.conversion == -1: # no formatting
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"Don't know how to unparse conversion code {t.conversion}")
|
||||
if t.format_spec:
|
||||
self.write(":")
|
||||
self.write(c(":"))
|
||||
self._JoinedStr_helper(t.format_spec)
|
||||
self.write("}")
|
||||
self.write(c("}"))
|
||||
|
||||
def _JoinedStr(self, t):
|
||||
self.write("f'")
|
||||
self.write("f" + self.maybe_colorize("'", ColorScheme.STRING))
|
||||
self._JoinedStr_helper(t)
|
||||
self.write("'")
|
||||
self.write(self.maybe_colorize("'", ColorScheme.STRING))
|
||||
|
||||
def _JoinedStr_helper(self, t):
|
||||
def escape(s):
|
||||
|
|
@ -627,9 +707,9 @@ class Unparser:
|
|||
for v in t.values:
|
||||
# Omit the surrounding quotes in string snippets
|
||||
if type(v) is ast.Constant:
|
||||
self.write(escape(v.value))
|
||||
self.write(self.maybe_colorize(escape(v.value), ColorScheme.STRING))
|
||||
elif type(v) is ast.Str: # up to Python 3.7
|
||||
self.write(escape(v.s))
|
||||
self.write(self.maybe_colorize(escape(v.s), ColorScheme.STRING))
|
||||
elif type(v) is ast.FormattedValue:
|
||||
self._FormattedValue_helper(v)
|
||||
else:
|
||||
|
|
@ -750,7 +830,7 @@ class Unparser:
|
|||
|
||||
def _Lambda(self, t):
|
||||
self.write("(")
|
||||
self.write("lambda ")
|
||||
self.write(self.python_keyword("lambda "))
|
||||
self.dispatch(t.args)
|
||||
self.write(": ")
|
||||
self.dispatch(t.body)
|
||||
|
|
@ -759,16 +839,16 @@ class Unparser:
|
|||
def _alias(self, t):
|
||||
self.write(t.name)
|
||||
if t.asname:
|
||||
self.write(" as " + t.asname)
|
||||
self.write(self.python_keyword(" as ") + t.asname)
|
||||
|
||||
def _withitem(self, t):
|
||||
self.dispatch(t.context_expr)
|
||||
if t.optional_vars:
|
||||
self.write(" as ")
|
||||
self.write(self.python_keyword(" as "))
|
||||
self.dispatch(t.optional_vars)
|
||||
|
||||
|
||||
def unparse(tree, *, debug=False):
|
||||
def unparse(tree, *, debug=False, color=False):
|
||||
"""Convert the AST `tree` into source code. Return the code as a string.
|
||||
|
||||
`debug`: bool, print invisible nodes (`Module`, `Expr`).
|
||||
|
|
@ -781,7 +861,7 @@ def unparse(tree, *, debug=False):
|
|||
"""
|
||||
try:
|
||||
with io.StringIO() as output:
|
||||
Unparser(tree, file=output, debug=debug)
|
||||
Unparser(tree, file=output, debug=debug, color=color)
|
||||
code = output.getvalue().strip()
|
||||
return code
|
||||
except UnparserError as err: # fall back to an AST dump
|
||||
|
|
@ -797,7 +877,7 @@ def unparse(tree, *, debug=False):
|
|||
raise UnparserError(msg) from err
|
||||
|
||||
|
||||
def unparse_with_fallbacks(tree, *, debug=False):
|
||||
def unparse_with_fallbacks(tree, *, debug=False, color=False):
|
||||
"""Like `unparse`, but upon error, don't raise; return the error message.
|
||||
|
||||
Usually you'll want the exception to be raised. This is mainly useful to
|
||||
|
|
@ -806,7 +886,7 @@ def unparse_with_fallbacks(tree, *, debug=False):
|
|||
at the receiving end.
|
||||
"""
|
||||
try:
|
||||
text = unparse(tree, debug=debug)
|
||||
text = unparse(tree, debug=debug, color=color)
|
||||
except UnparserError as err:
|
||||
text = err.args[0]
|
||||
except Exception as err:
|
||||
|
|
|
|||
Loading…
Reference in New Issue