diff --git a/README.md b/README.md index d5ea7a5f..73380b78 100644 --- a/README.md +++ b/README.md @@ -640,7 +640,7 @@ Next time you need this list just use an alias, like this: A short-cut to use for simple cases where a variant is an overkill. - `drill_marks`: [string='full'] what to use to indicate the drill places, can be none, small or full (for real scale). - `exclude_edge_layer`: [boolean=true] do not include the PCB edge layer. - - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen. + - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only). - `force_plot_invisible_refs_vals`: [boolean=false] include references and values even when they are marked as invisible. - `metric_units`: [boolean=false] use mm instead of inches. - `output`: [string='%f-%i%v.%x'] output file name, the default KiCad name if empty. Affected by global options. @@ -724,7 +724,7 @@ Next time you need this list just use an alias, like this: - `dnf_filter`: [string|list(string)=''] Name of the filter to mark components as not fitted. A short-cut to use for simple cases where a variant is an overkill. - `exclude_edge_layer`: [boolean=true] do not include the PCB edge layer. - - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen. + - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only). - `force_plot_invisible_refs_vals`: [boolean=false] include references and values even when they are marked as invisible. - `gerber_job_file`: [string='%f-%i%v.%x'] name for the gerber job file (%i='job', %x='gbrjob'). Affected by global options. - `gerber_precision`: [number=4.6] this the gerber coordinate format, can be 4.5 or 4.6. @@ -761,7 +761,7 @@ Next time you need this list just use an alias, like this: A short-cut to use for simple cases where a variant is an overkill. - `drill_marks`: [string='full'] what to use to indicate the drill places, can be none, small or full (for real scale). - `exclude_edge_layer`: [boolean=true] do not include the PCB edge layer. - - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen. + - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only). - `force_plot_invisible_refs_vals`: [boolean=false] include references and values even when they are marked as invisible. - `mirror_plot`: [boolean=false] plot mirrored. - `output`: [string='%f-%i%v.%x'] output file name, the default KiCad name if empty. Affected by global options. @@ -981,7 +981,7 @@ Next time you need this list just use an alias, like this: A short-cut to use for simple cases where a variant is an overkill. - `drill_marks`: [string='full'] what to use to indicate the drill places, can be none, small or full (for real scale). - `exclude_edge_layer`: [boolean=true] do not include the PCB edge layer. - - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen. + - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only). - `force_plot_invisible_refs_vals`: [boolean=false] include references and values even when they are marked as invisible. - `layers`: [list(dict)|list(string)|string] [all,selected,copper,technical,user] List of PCB layers to plot. @@ -996,7 +996,7 @@ Next time you need this list just use an alias, like this: A short-cut to use for simple cases where a variant is an overkill. - `drill_marks`: [string='full'] what to use to indicate the drill places, can be none, small or full (for real scale). - `exclude_edge_layer`: [boolean=true] do not include the PCB edge layer. - - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen. + - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only). - `force_plot_invisible_refs_vals`: [boolean=false] include references and values even when they are marked as invisible. - `line_width`: [number=0.1] [0.02,2] for objects without width [mm]. - `mirror_plot`: [boolean=false] plot mirrored. @@ -1100,7 +1100,7 @@ Next time you need this list just use an alias, like this: A short-cut to use for simple cases where a variant is an overkill. - `drill_marks`: [string='full'] what to use to indicate the drill places, can be none, small or full (for real scale). - `exclude_edge_layer`: [boolean=true] do not include the PCB edge layer. - - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen. + - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only). - `force_plot_invisible_refs_vals`: [boolean=false] include references and values even when they are marked as invisible. - `line_width`: [number=0.15] [0.02,2] for objects without width [mm]. - `mirror_plot`: [boolean=false] plot mirrored. @@ -1176,7 +1176,7 @@ Next time you need this list just use an alias, like this: A short-cut to use for simple cases where a variant is an overkill. - `drill_marks`: [string='full'] what to use to indicate the drill places, can be none, small or full (for real scale). - `exclude_edge_layer`: [boolean=true] do not include the PCB edge layer. - - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen. + - `exclude_pads_from_silkscreen`: [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only). - `force_plot_invisible_refs_vals`: [boolean=false] include references and values even when they are marked as invisible. - `line_width`: [number=0.25] [0.02,2] for objects without width [mm]. - `mirror_plot`: [boolean=false] plot mirrored. diff --git a/docs/samples/generic_plot.kibot.yaml b/docs/samples/generic_plot.kibot.yaml index 44076921..8aaf188b 100644 --- a/docs/samples/generic_plot.kibot.yaml +++ b/docs/samples/generic_plot.kibot.yaml @@ -165,7 +165,7 @@ outputs: drill_marks: 'full' # [boolean=true] do not include the PCB edge layer exclude_edge_layer: true - # [boolean=false] do not plot the component pads in the silk screen + # [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only) exclude_pads_from_silkscreen: false # [boolean=false] include references and values even when they are marked as invisible force_plot_invisible_refs_vals: false @@ -264,7 +264,7 @@ outputs: dnf_filter: '' # [boolean=true] do not include the PCB edge layer exclude_edge_layer: true - # [boolean=false] do not plot the component pads in the silk screen + # [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only) exclude_pads_from_silkscreen: false # [boolean=false] include references and values even when they are marked as invisible force_plot_invisible_refs_vals: false @@ -312,7 +312,7 @@ outputs: drill_marks: 'full' # [boolean=true] do not include the PCB edge layer exclude_edge_layer: true - # [boolean=false] do not plot the component pads in the silk screen + # [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only) exclude_pads_from_silkscreen: false # [boolean=false] include references and values even when they are marked as invisible force_plot_invisible_refs_vals: false @@ -620,7 +620,7 @@ outputs: drill_marks: 'full' # [boolean=true] do not include the PCB edge layer exclude_edge_layer: true - # [boolean=false] do not plot the component pads in the silk screen + # [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only) exclude_pads_from_silkscreen: false # [boolean=false] include references and values even when they are marked as invisible force_plot_invisible_refs_vals: false @@ -730,7 +730,7 @@ outputs: drill_marks: 'full' # [boolean=true] do not include the PCB edge layer exclude_edge_layer: true - # [boolean=false] do not plot the component pads in the silk screen + # [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only) exclude_pads_from_silkscreen: false # [boolean=false] include references and values even when they are marked as invisible force_plot_invisible_refs_vals: false @@ -820,7 +820,7 @@ outputs: drill_marks: 'full' # [boolean=true] do not include the PCB edge layer exclude_edge_layer: true - # [boolean=false] do not plot the component pads in the silk screen + # [boolean=false] do not plot the component pads in the silk screen (KiCad 5.x only) exclude_pads_from_silkscreen: false # [boolean=false] include references and values even when they are marked as invisible force_plot_invisible_refs_vals: false diff --git a/experiments/__doc__/coverage_mcpyrate/application.py b/experiments/__doc__/coverage_mcpyrate/application.py index 94b26d1e..e8744f04 100644 --- a/experiments/__doc__/coverage_mcpyrate/application.py +++ b/experiments/__doc__/coverage_mcpyrate/application.py @@ -1,6 +1,5 @@ from mymacros import macros, document # noqa: F401 -from mcpyrate.debug import macros, step_expansion # noqa: F401,F811 - +from mcpyrate.debug import macros, step_expansion, show_bindings # noqa: F401,F811 with step_expansion["dump"]: with document: # <--- Not covered? @@ -20,6 +19,7 @@ class d(object): """ documenting d.at1 """ # <--- Not covered? +show_bindings print("a = "+str(a)+" # "+_help_a) # noqa: F821 print("b = "+str(b)+" # "+_help_b) # noqa: F821 print("c = "+str(c)+" # "+_help_c) # noqa: F821 diff --git a/experiments/__doc__/coverage_mcpyrate/mymacros.py b/experiments/__doc__/coverage_mcpyrate/mymacros.py index 1a2b9dd3..8d4bcf7b 100644 --- a/experiments/__doc__/coverage_mcpyrate/mymacros.py +++ b/experiments/__doc__/coverage_mcpyrate/mymacros.py @@ -1,4 +1,6 @@ -from ast import (Assign, Name, Attribute, Expr, Num, Str, NameConstant, Load, Store, copy_location) +from ast import (Assign, Name, Attribute, Expr, Num, Str, NameConstant, copy_location, walk) +from mcpyrate.quotes import macros, q, u, n, a # noqa: F401 +import mcpyrate # noqa: F401 def document(tree, **kw): @@ -11,9 +13,9 @@ def document(tree, **kw): # Simplify it just to show the problem isn't related to the content of the macro # Note: This triggers another issue, Expr nodes can be optimized out if not assigned to a target # return tree - for n in range(len(tree)): - s = tree[n] - if not n: + for index in range(len(tree)): + s = tree[index] + if not index: prev = s continue # The whole sentence is a string? @@ -42,16 +44,12 @@ def document(tree, **kw): elif isinstance(value, NameConstant) and isinstance(value.value, bool): type_hint = '[boolean={}]'.format(str(value.value).lower()) # Transform the string into an assign for _help_ID - if is_attr: - target = Attribute(value=Name(id='self', ctx=Load()), attr=doc_id, ctx=Store()) - else: - target = Name(id=doc_id, ctx=Store()) - help_str = s.value - help_str.s = type_hint+s.value.s - tree[n] = Assign(targets=[target], value=help_str) - # Copy the line number from the original docstring - copy_location(target, s) - copy_location(tree[n], s) + name = 'self.'+doc_id if is_attr else doc_id + with q as quoted: + n[name] = u[type_hint + s.value.s.rstrip()] + tree[index] = quoted[0] + for node in walk(tree[index]): + copy_location(node, s) prev = s # Return the modified AST return tree diff --git a/kibot/macros.py b/kibot/macros.py index 222e57bd..7d177637 100644 --- a/kibot/macros.py +++ b/kibot/macros.py @@ -8,11 +8,11 @@ Macros to make the output plug-ins cleaner. """ from .gs import GS # noqa: F401 from ast import (Assign, Name, Attribute, Expr, Num, Str, NameConstant, Load, Store, UnaryOp, USub, - ClassDef, Call, ImportFrom, copy_location, alias) + ClassDef, Call, ImportFrom, copy_location, alias, walk) from .mcpyrate import unparse -from .mcpyrate.quotes import macros, q, u, n, a -from .mcpyrate.utils import rename -from . import mcpyrate +from .mcpyrate.quotes import macros, q, u, n, a # noqa: F401 +from . import mcpyrate # noqa: F401 + def document(sentences, **kw): """ This macro takes literal strings and converts them into: @@ -75,13 +75,12 @@ def document(sentences, **kw): post_hint += '. Affected by global options' if True: # Transform the string into an assign for _help_ID - target = q[self._] if is_attr else q[_] - copy_location(target, s) + name = 'self.'+doc_id if is_attr else doc_id with q as quoted: - a[target] = u[type_hint + s.value.s.rstrip() + post_hint] - rename("_", doc_id, quoted) + n[name] = u[type_hint+s.value.s.rstrip()+post_hint] + for node in walk(quoted[0]): + copy_location(node, s) sentences[index] = quoted[0] - copy_location(sentences[index], s) else: # Transform the string into an assign for _help_ID if is_attr: diff --git a/kibot/mcpyrate/colorizer.py b/kibot/mcpyrate/colorizer.py index b876292a..433cb278 100644 --- a/kibot/mcpyrate/colorizer.py +++ b/kibot/mcpyrate/colorizer.py @@ -5,15 +5,27 @@ __all__ = ["setcolor", "colorize", "ColorScheme", "Fore", "Back", "Style"] from colorama import init as colorama_init, Fore, Back, Style + + colorama_init() + def setcolor(*colors): - """Set color for ANSI terminal display. + """Set color for terminal display. + + Returns a string that, when printed into a terminal, sets the color + and style. We use `colorama`, so this works on any OS. For available `colors`, see `Fore`, `Back` and `Style`. + These are imported from `colorama`. Each entry can also be a tuple (arbitrarily nested), which is useful for defining compound styles. + + **CAUTION**: The specified style and color remain in effect until another + explicit call to `setcolor`. To reset, use `setcolor(Style.RESET_ALL)`. + If you want to colorize a piece of text so that the color and style + auto-reset after your text, use `colorize` instead. """ def _setcolor(color): if isinstance(color, (list, tuple)): @@ -21,40 +33,53 @@ def setcolor(*colors): return color return _setcolor(colors) + def colorize(text, *colors, reset=True): - """Colorize string `text` for ANSI terminal display. Reset color at end of `text`. + """Colorize string `text` for terminal display. + + Returns `text`, augmented with color and style commands for terminals. + We use `colorama`, so this works on any OS. For available `colors`, see `Fore`, `Back` and `Style`. These are imported from `colorama`. Usage:: - colorize("I'm new here", Fore.GREEN) - colorize("I'm bold and bluetiful", Style.BRIGHT, Fore.BLUE) + print(colorize("I'm new here", Fore.GREEN)) + print(colorize("I'm bold and bluetiful", Style.BRIGHT, Fore.BLUE)) Each entry can also be a tuple (arbitrarily nested), which is useful for defining compound styles:: BRIGHT_BLUE = (Style.BRIGHT, Fore.BLUE) ... - colorize("I'm bold and bluetiful, too", BRIGHT_BLUE) + print(colorize("I'm bold and bluetiful, too", BRIGHT_BLUE)) - **CAUTION**: Does not nest. Style resets after the colorized text. + **CAUTION**: Does not nest. Style and color reset after the colorized text. + If you want to set a color and style until further notice, use `setcolor` + instead. """ return "{}{}{}".format(setcolor(colors), text, setcolor(Style.RESET_ALL)) +# TODO: use a Bunch to support `clear` and `update`? class ColorScheme: - """The color scheme for debug utilities. + """The color scheme for terminal output in `mcpyrate`'s debug utilities. + + This is just a bunch of constants. To change the colors, simply assign new + values to them. Changes take effect immediately for any new output. + + (Don't replace the `ColorScheme` class itself, though; all the use sites + from-import it.) See `Fore`, `Back`, `Style` in `colorama` for valid values. To make a compound style, place the values into a tuple. The defaults are designed to fit the "Solarized" (Zenburn-like) theme of `gnome-terminal`, with "Show bold text in bright colors" set to OFF. - But they should work with most color schemes. + But they work also with "Tango", and indeed with most themes. """ _RESET = Style.RESET_ALL @@ -64,18 +89,26 @@ class ColorScheme: LINENUMBER = Style.DIM LANGUAGEKEYWORD = (Style.BRIGHT, Fore.YELLOW) # for, if, import, ... + BUILTINEXCEPTION = Fore.CYAN # TypeError, ValueError, Warning, ... + BUILTINOTHER = Style.BRIGHT # str, property, print, ... DEFNAME = (Style.BRIGHT, Fore.CYAN) # name of a function or class being defined DECORATOR = Fore.LIGHTBLUE_EX + # These can be highlighted differently although Python 3.8+ uses `Constant` for all. STRING = Fore.GREEN NUMBER = Fore.GREEN NAMECONSTANT = Fore.GREEN # True, False, None - BUILTINEXCEPTION = Fore.CYAN - BUILTINOTHER = Style.BRIGHT # str, property, print, ... + # Macro names are syntax-highlighted when a macro expander instance is + # running and is provided to `unparse`, so it can query for bindings. + # `step_expansion` does that automatically. + # + # So they won't yet be highlighted during dialect AST transforms, + # because at that point, there is no *macro* expander. + MACRONAME = Fore.BLUE - INVISIBLENODE = Style.DIM # AST node with no surface syntax repr (Module, Expr) + INVISIBLENODE = Style.DIM # AST node with no surface syntax repr (`Module`, `Expr`) # AST markers for data-driven communication within the macro expander ASTMARKER = Style.DIM # the "$AstMarker" title @@ -84,19 +117,23 @@ class ColorScheme: # ------------------------------------------------------------ # format_bindings, step_expansion, StepExpansion + # TODO: Clean the implementations to use `_RESET` at the appropriate points + # TODO: so we don't need to specify things `Fore.RESET` or `Style.NORMAL` here. + HEADING = (Style.BRIGHT, Fore.LIGHTBLUE_EX) SOURCEFILENAME = (Style.BRIGHT, Fore.RESET) # format_bindings - MACROBINDING = (Style.BRIGHT, Fore.RESET) - GREYEDOUT = Style.DIM + MACROBINDING = MACRONAME + GREYEDOUT = Style.DIM # if no bindings # step_expansion TREEID = (Style.NORMAL, Fore.LIGHTBLUE_EX) # StepExpansion - ATTENTION = (Style.BRIGHT, Fore.GREEN) # the string "DialectExpander debug mode" - DIALECTTRANSFORMER = (Style.BRIGHT, Fore.YELLOW) + ATTENTION = (Style.BRIGHT, Fore.GREEN) # "DialectExpander debug mode" + TRANSFORMERKIND = (Style.BRIGHT, Fore.GREEN) # source, AST + DIALECTTRANSFORMERNAME = (Style.BRIGHT, Fore.YELLOW) # ------------------------------------------------------------ # dump diff --git a/kibot/mcpyrate/core.py b/kibot/mcpyrate/core.py index 69b11efc..75a1c4be 100644 --- a/kibot/mcpyrate/core.py +++ b/kibot/mcpyrate/core.py @@ -45,8 +45,8 @@ class BaseMacroExpander(NodeTransformer): ''' def __init__(self, bindings, filename): - self._bindings = bindings - self.bindings = ChainMap(self._bindings, global_bindings) + self.local_bindings = bindings + self.bindings = ChainMap(self.local_bindings, global_bindings) self.filename = filename self.recursive = True diff --git a/kibot/mcpyrate/debug.py b/kibot/mcpyrate/debug.py index ddf4b701..7214bdd1 100644 --- a/kibot/mcpyrate/debug.py +++ b/kibot/mcpyrate/debug.py @@ -55,7 +55,8 @@ def step_expansion(tree, *, args, syntax, expander, **kw): if syntax not in ("expr", "block"): raise SyntaxError("`step_expansion` is an expr and block macro only") - formatter = functools.partial(unparse_with_fallbacks, debug=True, color=True) + formatter = functools.partial(unparse_with_fallbacks, debug=True, color=True, + expander=expander) if args: if len(args) != 1: raise SyntaxError("expected `step_expansion` or `step_expansion['mode_str']`") @@ -125,25 +126,16 @@ def show_bindings(tree, *, syntax, expander, **kw): """ if syntax != "name": raise SyntaxError("`show_bindings` is an identifier macro only") - print(format_bindings(expander, color=True), file=stderr) - # Can't just delete the node (return None) if it's in an Expr(value=...). - # - # For correct coverage reporting, we can't return a `Constant`, because CPython - # optimizes away do-nothing constants. So trick the compiler into thinking - # this is important, by making the expansion result call a no-op function. - lam = ast.Lambda(args=ast.arguments(posonlyargs=[], args=[], vararg=None, - kwonlyargs=[], kw_defaults=[], kwarg=None, - defaults=[]), - body=ast.Constant(value=None)) - return ast.Call(func=lam, args=[], keywords=[]) + print(format_bindings(expander, globals_too=False, color=True), file=stderr) + return None -def format_bindings(expander, *, color=False): +def format_bindings(expander, *, globals_too=False, color=False): """Return a human-readable report of the macro bindings currently seen by `expander`. - Global bindings (across all expanders) are also included. + If `globals_too=True`, global bindings (across all expanders) are also included. - If `color=True`, use `colorama` to colorize the output. (For terminals.) + If `color=True`, colorize the output for printing into a terminal. If you want to access them programmatically, just access `expander.bindings` directly. """ @@ -158,13 +150,14 @@ def format_bindings(expander, *, color=False): c, CS = maybe_setcolor, ColorScheme + bindings = expander.bindings if globals_too else expander.local_bindings with io.StringIO() as output: output.write(f"{c(CS.HEADING)}Macro bindings for {c(CS.SOURCEFILENAME)}{expander.filename}{c(CS.HEADING)}:{c(CS._RESET)}\n") - if not expander.bindings: + if not bindings: output.write(maybe_colorize(" \n", ColorScheme.GREYEDOUT)) else: - for k, v in sorted(expander.bindings.items()): + for k, v in sorted(bindings.items()): k = maybe_colorize(k, ColorScheme.MACROBINDING) output.write(f" {k}: {format_macrofunction(v)}\n") return output.getvalue() diff --git a/kibot/mcpyrate/dialects.py b/kibot/mcpyrate/dialects.py index 3cee53cc..181d7c00 100644 --- a/kibot/mcpyrate/dialects.py +++ b/kibot/mcpyrate/dialects.py @@ -185,7 +185,7 @@ class DialectExpander: c, CS = setcolor, ColorScheme if self.debugmode: plural = "s" if self._step != 1 else "" - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}before dialect {c(CS.DIALECTTRANSFORMER)}{kind} {c(CS.HEADING)}transformers ({self._step} step{plural} total):{c(CS._RESET)}\n" + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}before dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING)}transformers ({self._step} step{plural} total):{c(CS._RESET)}\n" print(_message_header + msg, file=stderr) print(format_for_display(content), file=stderr) @@ -224,13 +224,13 @@ class DialectExpander: self._step += 1 if self.debugmode: - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}after {c(CS.DIALECTTRANSFORMER)}{module_absname}.{dialectname}.{transform} {c(CS.HEADING)}(step {self._step}):{c(CS._RESET)}\n" + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}after {c(CS.DIALECTTRANSFORMERNAME)}{module_absname}.{dialectname}.{transform} {c(CS.HEADING)}(step {self._step}):{c(CS._RESET)}\n" print(_message_header + msg, file=stderr) print(format_for_display(content), file=stderr) if self.debugmode: plural = "s" if self._step != 1 else "" - msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}completed all dialect {c(CS.DIALECTTRANSFORMER)}{kind} {c(CS.HEADING)}transforms ({self._step} step{plural} total).{c(CS._RESET)}" + msg = f"{c(CS.SOURCEFILENAME)}{self.filename} {c(CS.HEADING)}completed all dialect {c(CS.TRANSFORMERKIND)}{kind} {c(CS.HEADING)}transforms ({self._step} step{plural} total).{c(CS._RESET)}" print(_message_header + msg, file=stderr) return content diff --git a/kibot/mcpyrate/expander.py b/kibot/mcpyrate/expander.py index ba99ebd2..ec9b5726 100644 --- a/kibot/mcpyrate/expander.py +++ b/kibot/mcpyrate/expander.py @@ -52,7 +52,7 @@ __all__ = ['namemacro', 'isnamemacro', 'expand_macros', 'find_macros'] from ast import (Name, Subscript, Tuple, Import, alias, AST, Assign, Store, Constant, - copy_location, iter_fields, NodeVisitor) + Lambda, arguments, Call, copy_location, iter_fields, NodeVisitor) from copy import copy from warnings import warn_explicit @@ -145,6 +145,11 @@ class MacroExpander(BaseMacroExpander): sourcecode = unparse_with_fallbacks(subscript) new_tree = self.expand('expr', subscript, macroname, tree, sourcecode=sourcecode, kw=kw) + if new_tree is None: + # Expression slots in the AST cannot be empty, but we can make + # something that evaluates to `None` at run-time, and get + # correct coverage while at it. + new_tree = _make_coverage_dummy_expr(subscript) else: new_tree = self.generic_visit(subscript) return new_tree @@ -205,7 +210,7 @@ class MacroExpander(BaseMacroExpander): tree = withstmt.body if not withstmt.items else [withstmt] new_tree = self.expand('block', original_withstmt, macroname, tree, sourcecode=sourcecode, kw=kw) - new_tree = _add_coverage_dummy_node(new_tree, withstmt, macroname) + new_tree = _insert_coverage_dummy_stmt(new_tree, withstmt, macroname) return new_tree def visit_ClassDef(self, classdef): @@ -258,7 +263,7 @@ class MacroExpander(BaseMacroExpander): kw = {'args': macroargs} new_tree = self.expand('decorator', original_decorated, macroname, decorated, sourcecode=sourcecode, kw=kw) - new_tree = _add_coverage_dummy_node(new_tree, innermost_macro, macroname) + new_tree = _insert_coverage_dummy_stmt(new_tree, innermost_macro, macroname) return new_tree def _detect_macro_items(self, items, syntax): @@ -317,7 +322,12 @@ class MacroExpander(BaseMacroExpander): kw = {'args': None} sourcecode = unparse_with_fallbacks(name) new_tree = self.expand('name', name, macroname, name, sourcecode=sourcecode, kw=kw) - if new_tree is not None: + if new_tree is None: + # Expression slots in the AST cannot be empty, but we can make + # something that evaluates to `None` at run-time, and get + # correct coverage while at it. + new_tree = _make_coverage_dummy_expr(name) + else: if not ismodified(new_tree): new_tree = Done(new_tree) elif self.recursive: # and modified @@ -436,11 +446,11 @@ class MacroCollector(NodeVisitor): self._seen.add(key) -def _add_coverage_dummy_node(tree, macronode, macroname): - '''Force `macronode` to be reported as covered by coverage tools. +def _insert_coverage_dummy_stmt(tree, macronode, macroname): + '''Force statement `macronode` to be reported as covered by coverage tools. - The dummy node will be injected to `tree`. The `tree` must appear in a - position where `ast.NodeTransformer.visit` may return a list of nodes. + A dummy node will be injected to `tree`. The `tree` must appear in a + statement position, so `ast.NodeTransformer.visit` may return a list of nodes. `macronode` is the macro invocation node to copy source location info from. `macroname` is included in the dummy node, to ease debugging. @@ -463,6 +473,30 @@ def _add_coverage_dummy_node(tree, macronode, macroname): tree.insert(0, Done(dummy)) # mark as Done so any expansions further out won't mess this up. return tree + +def _make_coverage_dummy_expr(macronode): + '''Force expression `macronode` to be reported as covered by coverage tools. + + This facilitates "deleting" expression nodes by `return None` from a macro. + Since an expression slot in the AST cannot be empty, we inject a dummy node + that evaluates to `None`. + + `macronode` is the macro invocation node to copy source location info from. + ''' + # TODO: inject the macro name for human-readability + # We inject a lambda and an immediate call to it, because a constant `None`, + # if it appears alone in an `ast.Expr`, is optimized away by CPython. + # We must set location info manually, because we run after `expand`. + non = copy_location(Constant(value=None), macronode) + lam = copy_location(Lambda(args=arguments(posonlyargs=[], args=[], vararg=None, + kwonlyargs=[], kw_defaults=[], kwarg=None, + defaults=[]), + body=non), + macronode) + call = copy_location(Call(func=lam, args=[], keywords=[]), macronode) + return Done(call) + + # -------------------------------------------------------------------------------- def expand_macros(tree, bindings, *, filename): diff --git a/kibot/mcpyrate/quotes.py b/kibot/mcpyrate/quotes.py index b79b73a3..7e33446c 100644 --- a/kibot/mcpyrate/quotes.py +++ b/kibot/mcpyrate/quotes.py @@ -1,7 +1,9 @@ # -*- coding: utf-8; -*- """Quasiquotes. Build ASTs in your macros, using syntax that mostly looks like regular code.""" -__all__ = ['capture', 'lookup', 'astify', 'unastify', +__all__ = ['lift_identifier', + 'capture_value', 'capture_macro', + 'astify', 'unastify', 'q', 'u', 'n', 'a', 's', 'h', 'expand1q', 'expandq', 'expand1', 'expand'] @@ -15,64 +17,137 @@ from .markers import ASTMarker, get_markers from .unparser import unparse from .utils import gensym, NestingLevelTracker -# -------------------------------------------------------------------------------- + +def _mcpyrate_quotes_attr(attr): + """Create an AST that, when compiled and run, looks up `mcpyrate.quotes.attr` in `Load` context.""" + mcpyrate_quotes_module = ast.Attribute(value=ast.Name(id="mcpyrate"), attr="quotes") + return ast.Attribute(value=mcpyrate_quotes_module, attr=attr) + class QuasiquoteMarker(ASTMarker): """Base class for AST markers used by quasiquotes. Compiled away by `astify`.""" pass -class ASTLiteral(QuasiquoteMarker): # like `macropy`'s `Literal` - """Keep the given subtree as-is.""" +# -------------------------------------------------------------------------------- +# Unquote commands for `astify`. Each type corresponds to an unquote macro. + +class Unquote(QuasiquoteMarker): + """Interpolate the value of the given subtree into the quoted tree. Emitted by `u[]`.""" pass -class CaptureLater(QuasiquoteMarker): # like `macropy`'s `Captured` - """Capture the value the given subtree evaluates to at the use site of `q`.""" + +class LiftIdentifier(QuasiquoteMarker): + """Perform string to variable access conversion on given subtree. Emitted by `n[]`. + + Details: convert the string the given subtree evaluates to, at the use site + of `q`, into the variable access the text of the string represents, when it + is interpreted as Python source code. + + (This allows computing the name to be accessed.) + """ + pass + + +class ASTLiteral(QuasiquoteMarker): # like `macropy`'s `Literal` + """Keep the given subtree as-is. Emitted by `a[]`. + + Although the effect is similar, this is semantically different from + `mcpyrate.core.Done`. This controls AST unquoting in the quasiquote + subsystem, whereas `Done` tells the expander to stop expanding that + subtree. + """ + pass + + +class ASTList(QuasiquoteMarker): + """Interpolate the given `list` of AST nodes as an `ast.List` node. Emitted by `s[]`.""" + pass + + +class Capture(QuasiquoteMarker): # like `macropy`'s `Captured` + """Capture given subtree hygienically. Emitted by `h[]`. + + Details: capture the value or macro name the given subtree evaluates to, + at the use site of `q`. The value or macro reference is frozen (by pickle) + so that it can be restored also in another Python process later. + """ def __init__(self, body, name): super().__init__(body) self.name = name self._fields += ["name"] # -------------------------------------------------------------------------------- +# Run-time parts of the unquote operators. -# Hygienically captured run-time values... but to support `.pyc` caching, we can't use a per-process dictionary. -# _hygienic_registry = {} +# Unquote doesn't have its own function here, because it's a special case of `astify`. -def _mcpyrate_quotes_attr(attr): - """Create an AST that, when compiled and run, looks up `mcpyrate.quotes.attr` in `Load` context.""" - mcpyrate_quotes_module = ast.Attribute(value=ast.Name(id="mcpyrate", ctx=ast.Load()), - attr="quotes", - ctx=ast.Load()) - return ast.Attribute(value=mcpyrate_quotes_module, - attr=attr, - ctx=ast.Load()) +def lift_identifier(value, filename=""): + """Lift a string into a variable access. Run-time part of `n[]`. -def capture(value, name): - """Hygienically capture a run-time value. + Examples:: + + lift_identifier("kitty") -> Name(id='kitty') + lift_identifier("kitty.tail") -> Attribute(value=Name(id='kitty'), + attr='tail') + lift_identifier("kitty.tail.color") -> Attribute(value=Attribute(value=Name(id='kitty'), + attr='tail'), + attr='color') + + Works with subscript expressions, too:: + + lift_identifier("kitties[3].paws[2].claws") + """ + if not isinstance(value, str): + raise TypeError(f"n[]: expected an expression that evaluates to str, result was {type(value)} with value {repr(value)}") + return ast.parse(value, filename=filename, mode="eval").body + + +def ast_literal(tree): + """Interpolate an AST node. Run-time part of `a[]`.""" + if not isinstance(tree, ast.AST): + raise TypeError(f"a[]: expected an AST node, got {type(tree)} with value {repr(tree)}") + return tree + + +def ast_list(nodes): + """Interpolate a `list` of AST nodes as an `ast.List` node. Run-time part of `s[]`.""" + if not isinstance(nodes, list): + raise TypeError(f"s[]: expected an expression that evaluates to list, result was {type(nodes)} with value {repr(nodes)}") + if not all(isinstance(tree, ast.AST) for tree in nodes): + raise ValueError(f"s[]: expected a list of AST nodes, got {repr(nodes)}") + return ast.List(elts=nodes) + + +def capture_value(value, name): + """Hygienically capture a run-time value. Used by `h[]`. `value`: A run-time value. Must be picklable. - `name`: A human-readable name. + `name`: For human-readability. The return value is an AST that, when compiled and run, returns the captured value (even in another Python process later). """ # If we didn't need to consider bytecode caching, we could just store the - # value in a registry that is populated at macro expansion time. Each - # unique value (by `id`) could be stored only once. + # value in a dictionary (that lives at the top level of `mcpyrate.quotes`) + # that is populated at macro expansion time. Each unique value (by `id`) + # could be stored only once. # # But we want to support bytecode caching. To avoid introducing hard-to-find # bugs into user code, we must provide consistent semantics, regardless of # whether updating of the bytecode cache is actually enabled or not (see - # `sys.dont_write_bytecode`). + # `sys.dont_write_bytecode`). So we must do the same thing regardless of + # whether the captured value is used in the current process, or in another + # Python process later. # - # If the macro expansion result is to be re-used from a `.pyc`, we must - # serialize and store the captured value to disk, so that values from - # "macro expansion time last week" remain available when the `.pyc` is - # loaded in another Python process, much later. + # If the macro expansion result is to remain available for re-use from a + # `.pyc`, we must serialize and store the captured value to disk, so that + # values from "macro expansion time last week" are still available when the + # `.pyc` is loaded in another Python process later. # # Modules are macro-expanded independently (no global finalization for the # whole codebase), and a `.pyc` may indeed later get loaded into some other # codebase that imports the same module, so we can't make a centralized - # registry, like we could without bytecode caching (for the current process). + # registry, like we could without bytecode caching. # # So really pretty much the only thing we can do reliably and simply is to # store a fresh serialized copy of the value at the capture location in the @@ -82,31 +157,41 @@ def capture(value, name): # and serialization. # frozen_value = pickle.dumps(value) - return ast.Call(_mcpyrate_quotes_attr("lookup"), + return ast.Call(_mcpyrate_quotes_attr("lookup_value"), [ast.Tuple(elts=[ast.Constant(value=name), ast.Constant(value=frozen_value)])], []) + _lookup_cache = {} -def lookup(key): - """Look up a hygienically captured run-time value.""" +def lookup_value(key): + """Look up a hygienically captured run-time value. Used by `h[]`. + + Usually there's no need to call this function manually; `capture_value` + (and thus also `h[]`) will generate an AST that calls this automatically. + """ name, frozen_value = key cachekey = (name, id(frozen_value)) # id() so each capture instance behaves independently if cachekey not in _lookup_cache: _lookup_cache[cachekey] = pickle.loads(frozen_value) return _lookup_cache[cachekey] + def capture_macro(macro, name): - """Hygienically capture a macro. + """Hygienically capture a macro. Used by `h[]`. `macro`: A macro function. Must be picklable. - `name`: The name of the macro, as it appeared in the bindings of - the expander it was captured from. For human-readability. + `name`: For human-readability. The recommended value is the name of + the macro, as it appeared in the bindings of the expander + it was captured from. - The return value is an AST that, when compiled and run, injects the - macro into the expander's global macro bindings table (even in another - Python process later), and then evaluates to that macro name as an - `ast.Name` (so further expansion of that AST will invoke the macro). + The name of the captured macro is automatically uniqified using + `gensym(name)`. + + The return value is an AST that, when compiled and run, injects the macro + into the expander's global macro bindings table (even in another Python + process later), and then evaluates to the uniqified macro name as an + `ast.Name`, so that macro-expanding that AST will invoke the macro. """ frozen_macro = pickle.dumps(macro) unique_name = gensym(name) @@ -115,11 +200,15 @@ def capture_macro(macro, name): ast.Constant(value=frozen_macro)])], []) -def lookup_macro(key): - """Look up a hygienically captured macro. - This will inject the macro to the global macro bindings table, - and then evaluate to that macro name, as an `ast.Name`. +def lookup_macro(key): + """Look up a hygienically captured macro. Used by `h[]`. + + This injects the macro to the expander's global macro bindings table, + and then returns the macro name, as an `ast.Name`. + + Usually there's no need to call this function manually; `capture_macro` + (and thus also `h[]`) will generate an AST that calls this automatically. """ unique_name, frozen_macro = key if unique_name not in global_bindings: @@ -127,37 +216,66 @@ def lookup_macro(key): return ast.Name(id=unique_name) # -------------------------------------------------------------------------------- +# The quasiquote compiler and uncompiler. def astify(x, expander=None): # like `macropy`'s `ast_repr` - """Lift a value into its AST representation, if possible. + """Quasiquote compiler. Lift a value into its AST representation, if possible. When the AST is compiled and run, it will evaluate to `x`. - If `x` itself is an AST, then produce an AST that, when compiled and run, - will generate the AST `x`. + Note the above implies that if `x` itself is an AST, then this produces + an AST that, when compiled and run, will generate the AST `x`. This is + the mechanism that `q` uses to produce the quoted AST. - If the input is a `list` of ASTs for a statement suite, the return value + If the input is a `list` of ASTs (e.g. a statement suite), the return value is a single `ast.List` node, with its `elts` taken from the input list. However, most of the time it's not used this way, because `BaseMacroExpander` already translates a `visit` to a statement suite into visits to individual - nodes, because otherwise `ast.NodeTransformer` chokes on the input. (The only - exception is `q` in block mode; it'll produce a `List` this way.) + nodes, because `ast.NodeTransformer` requires that. The only exception is + `q` in block mode; it'll produce a `List` this way. - `expander` is a `BaseMacroExpander` instance, used for detecting macro names - inside `CaptureLater` markers. If no `expander` is provided, macros cannot be - hygienically captured. + `expander` is a `BaseMacroExpander` instance, used for detecting macros + inside `Capture` markers. Macros can be hygienically captured only if + an `expander` is provided. - Raises `TypeError` when the lifting fails. + Raises `TypeError` if the lifting fails. """ def recurse(x): # second layer just to auto-pass `expander` by closure. T = type(x) - # Drop the ASTLiteral wrapper; it only tells us to pass through this subtree as-is. - if T is ASTLiteral: - return x.body + # Compile the unquote commands. + # + # Minimally, `astify` must support `ASTLiteral`; the others could be + # implemented inside the unquote operators, as `ASTLiteral(ast.Call(...))`. + # + # But maybe this approach is cleaner. We can do almost everything here, + # in a regular function, and each unquote macro is just a thin wrapper + # on top of the corresponding marker type. + if T is Unquote: # `u[]` + # We want to generate an AST that compiles to the *value* of `x.body`, + # evaluated at the use site of `q`. But when the `q` expands, it is + # too early. We must `astify` *at the use site* of `q`. So use an + # `ast.Call` to delay until run-time, and pass in `x.body` as-is. + return ast.Call(_mcpyrate_quotes_attr("astify"), [x.body], []) - # This is the magic part of q[h[]]. - elif T is CaptureLater: + elif T is LiftIdentifier: # `n[]` + # Delay the identifier lifting, so it runs at the use site of `q`, + # where the actual value of `x.body` becomes available. + filename = expander.filename if expander else "" + return ast.Call(_mcpyrate_quotes_attr('lift_identifier'), + [x.body, + ast.Constant(value=filename)], + []) + + elif T is ASTLiteral: # `a[]` + # Pass through this subtree as-is, but typecheck the argument + # at the use site of `q`. + return ast.Call(_mcpyrate_quotes_attr('ast_literal'), [x.body], []) + + elif T is ASTList: # `s[]` + return ast.Call(_mcpyrate_quotes_attr('ast_list'), [x.body], []) + + elif T is Capture: # `h[]` if expander and type(x.body) is ast.Name: function = expander.isbound(x.body.id) if function: @@ -174,13 +292,15 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr` return capture_macro(function, x.body.id) # Hygienically capture a garden variety run-time value. # At the use site of q[], this captures the value, and rewrites itself - # into a lookup. At the use site of the macro that used q[], that - # rewritten code looks up the captured value. - return ast.Call(_mcpyrate_quotes_attr('capture'), + # into an AST that represents a lookup. At the use site of the macro + # that used q[], that code runs, and looks up the captured value. + return ast.Call(_mcpyrate_quotes_attr('capture_value'), [x.body, ast.Constant(value=x.name)], []) + # Builtin types. Mainly support for `u[]`, but also used by the + # general case for AST node fields that contain bare values. elif T in (int, float, str, bytes, bool, type(None)): return ast.Constant(value=x) @@ -194,10 +314,11 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr` elif T is set: return ast.Set(elts=list(recurse(elt) for elt in x)) + # General case. elif isinstance(x, ast.AST): # TODO: Add support for astifying ASTMarkers? - # TODO: Otherwise the same as regular AST node, but need to refer to the - # TODO: module it is defined in, and we don't have everything in scope here. + # Otherwise the same as regular AST node, but need to refer to the + # module it is defined in, and we don't have everything in scope here. if isinstance(x, ASTMarker): raise TypeError(f"Cannot astify internal AST markers, got {unparse(x)}") @@ -208,8 +329,7 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr` # name conflicts at the use site of `q[]`. fields = [ast.keyword(a, recurse(b)) for a, b in ast.iter_fields(x)] node = ast.Call(ast.Attribute(value=_mcpyrate_quotes_attr('ast'), - attr=x.__class__.__name__, - ctx=ast.Load()), + attr=x.__class__.__name__), [], fields) # Copy source location info for correct coverage reporting of a quoted block. @@ -233,7 +353,7 @@ def astify(x, expander=None): # like `macropy`'s `ast_repr` def unastify(tree): - """Inverse of `astify`. + """Quasiquote uncompiler. Inverse of `astify`. `tree` must have been produced by `astify`. Otherwise raises `TypeError`. @@ -245,31 +365,27 @@ def unastify(tree): a value from outside the quote context into the quoted representation - so that the value actually becomes quoted! - whereas `unastify` inverts the quote operation. + + Note also that `astify` compiles unquote commands into ASTs for calls to + the run-time parts of the unquote operators. That's what `unastify` sees. + We *could* detect those calls and uncompile them into AST markers, but we + currently don't. + + The use case of `unastify` is second-order macros, to transform a quoted + AST at macro expansion time when the extra AST layer added by `astify` is + still present. The recipe is `unastify`, process just like any AST, then + quote again. + + If you just want to macro-expand a quoted AST, see `expand` and `expand1`. """ # CAUTION: in `unastify`, we implement only what we minimally need. - def attr_ast_to_dotted_name(tree): - # Input is like: - # (mcpyrate.quotes).thing - # ((mcpyrate.quotes).ast).thing - if type(tree) is not ast.Attribute: - raise TypeError - acc = [] - def recurse(tree): - acc.append(tree.attr) - if type(tree.value) is ast.Attribute: - recurse(tree.value) - elif type(tree.value) is ast.Name: - acc.append(tree.value.id) - else: - raise NotImplementedError - recurse(tree) - return ".".join(reversed(acc)) - our_module_globals = globals() def lookup_thing(dotted_name): if not dotted_name.startswith("mcpyrate.quotes"): raise NotImplementedError path = dotted_name.split(".") + if not all(component.isidentifier() for component in path): + raise NotImplementedError if len(path) < 3: raise NotImplementedError name_of_thing = path[2] @@ -302,7 +418,7 @@ def unastify(tree): return {unastify(elt) for elt in tree.elts} elif T is ast.Call: - dotted_name = attr_ast_to_dotted_name(tree.func) + dotted_name = unparse(tree.func) callee = lookup_thing(dotted_name) args = unastify(tree.args) kwargs = {k: v for k, v in unastify(tree.keywords)} @@ -322,7 +438,7 @@ _quotelevel = NestingLevelTracker() def _unquote_expand(tree, expander): """Expand quasiquote macros in `tree`. If quotelevel is zero, expand all macros in `tree`.""" if _quotelevel.value == 0: - tree = expander.visit_recursively(tree) # result should be runnable, so always use recursive mode. + tree = expander.visit(tree) else: tree = _expand_quasiquotes(tree, expander) @@ -342,7 +458,7 @@ def q(tree, *, syntax, expander, **kw): raise SyntaxError("`q` is an expr and block macro only") with _quotelevel.changed_by(+1): tree = _expand_quasiquotes(tree, expander) # expand any inner quotes and unquotes first - tree = astify(tree, expander=expander) # Magic part of `q`. Supply `expander` for `h[macro]` detection. + tree = astify(tree, expander) # Magic part of `q`. Supply `expander` for `h[macro]` detection. ps = get_markers(tree, QuasiquoteMarker) # postcondition: no remaining QuasiquoteMarkers if ps: assert False, f"QuasiquoteMarker instances remaining in output: {ps}" @@ -350,7 +466,7 @@ def q(tree, *, syntax, expander, **kw): target = kw['optional_vars'] # List, Tuple, Name if type(target) is not ast.Name: raise SyntaxError(f"expected a single asname, got {unparse(target)}") - # Note this `Assign` runs at the use site of `q`, it's not part of the quoted code section. + # Note this `Assign` runs at the use site of `q`, it's not part of the quoted code block. tree = ast.Assign([target], tree) # Here `tree` is a List. return tree @@ -366,45 +482,58 @@ def u(tree, *, syntax, expander, **kw): raise SyntaxError("`u` encountered while quotelevel < 1") with _quotelevel.changed_by(-1): _unquote_expand(tree, expander) - # We want to generate an AST that compiles to the *value* of `v`. But when - # this runs, it is too early. We must astify *at the use site*. So use an - # `ast.Call` to delay, and in there, splice in `tree` as-is. - return ASTLiteral(ast.Call(_mcpyrate_quotes_attr("astify"), [tree], [])) + return Unquote(tree) -def n(tree, *, syntax, **kw): - """[syntax, expr] name-unquote. Splice a string, lifted into a lexical identifier, into a quasiquote. +def n(tree, *, syntax, expander, **kw): + """[syntax, expr] name-unquote. In a quasiquote, lift a string into a variable access. - The resulting node's `ctx` is filled in automatically by the macro expander later. + Examples:: + + `n["kitty"]` refers to the variable `kitty`, + `n[x]` refers to the variable whose name is taken from the variable `x` (at the use site of `q`), + `n["kitty.tail"]` refers to the attribute `tail` of the variable `kitty`, + `n["kitty." + x]` refers to an attribute of the variable `kitty`, where the attribute + is determined by the value of the variable `x` at the use site of `q`. + + Works with subscript expressions, too:: + + `n[f"kitties[{j}].paws[{k}].claws"]` + + Any expression can be used, as long as it evaluates to a string containing + only valid identifiers and dots. This is checked when the use site of `q` runs. + + The correct `ctx` for the use site is filled in automatically by the macro expander later. """ if syntax != "expr": raise SyntaxError("`n` is an expr macro only") if _quotelevel.value < 1: raise SyntaxError("`n` encountered while quotelevel < 1") with _quotelevel.changed_by(-1): - return ASTLiteral(astify(ast.Name(id=ASTLiteral(tree)))) + _unquote_expand(tree, expander) + return LiftIdentifier(tree) -def a(tree, *, syntax, **kw): +def a(tree, *, syntax, expander, **kw): """[syntax, expr] AST-unquote. Splice an AST into a quasiquote.""" if syntax != "expr": raise SyntaxError("`a` is an expr macro only") if _quotelevel.value < 1: raise SyntaxError("`a` encountered while quotelevel < 1") with _quotelevel.changed_by(-1): + _unquote_expand(tree, expander) return ASTLiteral(tree) -def s(tree, *, syntax, **kw): +def s(tree, *, syntax, expander, **kw): """[syntax, expr] list-unquote. Splice a `list` of ASTs, as an `ast.List`, into a quasiquote.""" if syntax != "expr": raise SyntaxError("`s` is an expr macro only") if _quotelevel.value < 1: raise SyntaxError("`s` encountered while quotelevel < 1") - return ASTLiteral(ast.Call(ast.Attribute(value=_mcpyrate_quotes_attr('ast'), - attr='List'), - [], - [ast.keyword("elts", tree)])) + with _quotelevel.changed_by(-1): + _unquote_expand(tree, expander) + return ASTList(tree) def h(tree, *, syntax, expander, **kw): @@ -422,10 +551,8 @@ def h(tree, *, syntax, expander, **kw): Python process. (In other words, values from "macro expansion time last week" would not otherwise be available.) - Supports also macros. To hygienically splice a macro invocation, `h[]` only - the macro name. Macro captures are not pickled; they simply extend the bindings - of the expander (with a uniqified macro name) that is expanding the use site of - the surrounding `q`. + Supports also macros. To hygienically splice a macro invocation, + `h[]` only the macro name. """ if syntax != "expr": raise SyntaxError("`h` is an expr macro only") @@ -434,7 +561,7 @@ def h(tree, *, syntax, expander, **kw): with _quotelevel.changed_by(-1): name = unparse(tree) _unquote_expand(tree, expander) - return CaptureLater(tree, name) + return Capture(tree, name) # -------------------------------------------------------------------------------- @@ -518,6 +645,6 @@ def expand(tree, *, syntax, expander, **kw): tree = expander.visit_once(tree) # make the quotes inside this invocation expand first; -> Done(body=...) # Always use recursive mode, because `expand[...]` may appear inside # another macro invocation that uses `visit_once` (which sets the expander - # mode to non-recursive for the dynamic extent of the visit). + # mode to non-recursive for the dynamic extent of the `visit_once`). tree = expander.visit_recursively(unastify(tree.body)) # On wrong kind of input, `unastify` will `TypeError` for us. return q(tree, syntax=syntax, expander=expander, **kw) diff --git a/kibot/mcpyrate/unparser.py b/kibot/mcpyrate/unparser.py index 4133c82c..293cffe7 100644 --- a/kibot/mcpyrate/unparser.py +++ b/kibot/mcpyrate/unparser.py @@ -49,27 +49,36 @@ class Unparser: for the abstract syntax. Original formatting is disregarded. """ - def __init__(self, tree, *, file=sys.stdout, debug=False, color=False): + def __init__(self, tree, *, file=sys.stdout, + debug=False, color=False, expander=None): """Print the source for `tree` to `file`. `debug`: bool, print invisible nodes (`Module`, `Expr`). + For statement nodes, print also line numbers (`lineno` + attribute from the AST node). + The output is then not valid Python, but may better show the problem when code produced by a macro mysteriously fails to compile (even though a non-debug unparse looks ok). - `color`: bool, use Colorama to color output. For syntax highlighting - when printing into a terminal. + `color`: bool, whether to use syntax highlighting. For terminal output. + + `expander`: optional `BaseMacroExpander` instance. If provided, + used for syntax highlighting macro names. """ self.debug = debug self.color = color - self._color_override = False # to syntax highlight decorators + self._color_override = False # for syntax highlighting of decorators + self.expander = expander self.f = file self._indent = 0 self.dispatch(tree) print("", file=self.f) self.f.flush() + # -------------------------------------------------------------------------------- + def maybe_colorize(self, text, *colors): "Colorize text if color is enabled." if self._color_override: @@ -78,7 +87,7 @@ class Unparser: return text return colorize(text, *colors) - def python_keyword(self, text): + def maybe_colorize_python_keyword(self, text): "Shorthand to colorize a language keyword such as `def`, `for`, ..." return self.maybe_colorize(text, ColorScheme.LANGUAGEKEYWORD) @@ -87,7 +96,7 @@ class Unparser: Useful for syntax highlighting decorators (so that the method rendering the decorator may force a particular color, instead of allowing - auto-coloring based on the data in the decorator AST node). + auto-coloring based on the AST data). """ @contextmanager def _nocolor(): @@ -99,8 +108,14 @@ class Unparser: self._color_override = old_color_override return _nocolor() + # -------------------------------------------------------------------------------- + def fill(self, text="", *, lineno_node=None): - "Indent a piece of text, according to the current indentation level." + """Begin a new line, indent to the current level, then write `text`. + + If in debug mode, then from `lineno_node`, get the `lineno` attribute + for printing the line number. Print `----` if `lineno` missing. + """ self.write("\n") if self.debug and isinstance(lineno_node, ast.AST): lineno = lineno_node.lineno if hasattr(lineno_node, "lineno") else None @@ -145,34 +160,73 @@ class Unparser: # -------------------------------------------------------------------------------- # Unparsing methods # - # There should be one method per concrete grammar type. - # Constructors should be grouped by sum type. Ideally, - # this would follow the order in the grammar, but - # currently doesn't. + # Beside `astmarker`, which is a macro expander data-driven communication + # thing, there should be one method per concrete grammar type. Constructors + # should be grouped by sum type. Ideally, this would follow the order in + # the grammar, but currently doesn't. + # + # https://docs.python.org/3/library/ast.html#abstract-grammar + # https://greentreesnakes.readthedocs.io/en/latest/nodes.html def astmarker(self, tree): - def write_field_value(v): + def write_astmarker_field_value(v): if isinstance(v, ast.AST): self.dispatch(v) else: self.write(repr(v)) - self.fill(self.maybe_colorize(f"$ASTMarker", ColorScheme.ASTMARKER), - lineno_node=tree) # markers cannot be eval'd + + # Print like a statement or like an expression, depending on the + # content. The marker itself is neither. Prefix by a "$" to indicate + # that "source code" containing AST markers cannot be eval'd. + # If you need to get rid of them, see `mcpyrate.markers.delete_markers`. + + header = self.maybe_colorize(f"$ASTMarker", ColorScheme.ASTMARKER) + if isinstance(tree.body, ast.stmt): + print_mode = "stmt" + self.fill(header, lineno_node=tree) + else: + print_mode = "expr" + self.write("(") + self.write(header) + clsname = self.maybe_colorize(tree.__class__.__name__, ColorScheme.ASTMARKERCLASS) self.write(f"<{clsname}>") + self.enter() self.write(" ") if len(tree._fields) == 1 and tree._fields[0] == "body": - write_field_value(tree.body) + # If there is just a single "body" field, don't bother with field names. + write_astmarker_field_value(tree.body) else: - for k, v in ast.iter_fields(tree): - self.fill(k) - self.enter() - self.write(" ") - write_field_value(v) - self.leave() + # The following is just a notation. There's no particular reason to + # use "k: v" syntax in statement mode and "k=v" in expression mode, + # except that it meshes well with the indentation rules and looks + # somewhat pythonic. + # + # In statement mode, a field may contain a statement; dispatching + # to a statement will begin a new line. + if print_mode == "stmt": + for k, v in ast.iter_fields(tree): + self.fill(k, lineno_node=tree) + self.enter() + self.write(" ") + write_astmarker_field_value(v) + self.leave() + else: # "expr" + first = True + for k, v in ast.iter_fields(tree): + if first: + first = False + else: + self.write(", ") + self.write(f"{k}=") + self.write("(") + write_astmarker_field_value(v) + self.write(")") self.leave() + if print_mode == "expr": + self.write(")") def _Module(self, t): # TODO: Python 3.8 type_ignores. Since we don't store the source text, maybe ignore that? @@ -201,15 +255,15 @@ class Unparser: self.dispatch(t.value) def _Import(self, t): - self.fill(self.python_keyword("import "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("import "), lineno_node=t) interleave(lambda: self.write(", "), self.dispatch, t.names) def _ImportFrom(self, t): - self.fill(self.python_keyword("from "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("from "), lineno_node=t) self.write("." * t.level) if t.module: self.write(t.module) - self.write(self.python_keyword(" import ")) + self.write(self.maybe_colorize_python_keyword(" import ")) interleave(lambda: self.write(", "), self.dispatch, t.names) def _Assign(self, t): @@ -240,42 +294,42 @@ class Unparser: self.dispatch(t.value) def _Return(self, t): - self.fill(self.python_keyword("return"), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("return"), lineno_node=t) if t.value: self.write(" ") self.dispatch(t.value) def _Pass(self, t): - self.fill(self.python_keyword("pass"), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("pass"), lineno_node=t) def _Break(self, t): - self.fill(self.python_keyword("break"), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("break"), lineno_node=t) def _Continue(self, t): - self.fill(self.python_keyword("continue"), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("continue"), lineno_node=t) def _Delete(self, t): - self.fill(self.python_keyword("del "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("del "), lineno_node=t) interleave(lambda: self.write(", "), self.dispatch, t.targets) def _Assert(self, t): - self.fill(self.python_keyword("assert "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("assert "), lineno_node=t) self.dispatch(t.test) if t.msg: self.write(", ") self.dispatch(t.msg) def _Global(self, t): - self.fill(self.python_keyword("global "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("global "), lineno_node=t) interleave(lambda: self.write(", "), self.write, t.names) def _Nonlocal(self, t): - self.fill(self.python_keyword("nonlocal "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("nonlocal "), lineno_node=t) interleave(lambda: self.write(", "), self.write, t.names) def _Await(self, t): # expr self.write("(") - self.write(self.python_keyword("await")) + self.write(self.maybe_colorize_python_keyword("await")) if t.value: self.write(" ") self.dispatch(t.value) @@ -283,7 +337,7 @@ class Unparser: def _Yield(self, t): # expr self.write("(") - self.write(self.python_keyword("yield")) + self.write(self.maybe_colorize_python_keyword("yield")) if t.value: self.write(" ") self.dispatch(t.value) @@ -291,48 +345,48 @@ class Unparser: def _YieldFrom(self, t): # expr self.write("(") - self.write(self.python_keyword("yield from")) + self.write(self.maybe_colorize_python_keyword("yield from")) if t.value: self.write(" ") self.dispatch(t.value) self.write(")") def _Raise(self, t): - self.fill(self.python_keyword("raise"), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("raise"), lineno_node=t) if not t.exc: assert not t.cause return self.write(" ") self.dispatch(t.exc) if t.cause: - self.write(self.python_keyword(" from ")) + self.write(self.maybe_colorize_python_keyword(" from ")) self.dispatch(t.cause) def _Try(self, t): - self.fill(self.python_keyword("try"), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("try"), lineno_node=t) self.enter() self.dispatch(t.body) self.leave() for ex in t.handlers: self.dispatch(ex) if t.orelse: - self.fill(self.python_keyword("else")) + self.fill(self.maybe_colorize_python_keyword("else")) self.enter() self.dispatch(t.orelse) self.leave() if t.finalbody: - self.fill(self.python_keyword("finally")) + self.fill(self.maybe_colorize_python_keyword("finally")) self.enter() self.dispatch(t.finalbody) self.leave() def _ExceptHandler(self, t): - self.fill(self.python_keyword("except"), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("except"), lineno_node=t) if t.type: self.write(" ") self.dispatch(t.type) if t.name: - self.write(self.python_keyword(" as ")) + self.write(self.maybe_colorize_python_keyword(" as ")) self.write(t.name) self.enter() self.dispatch(t.body) @@ -349,7 +403,7 @@ class Unparser: self.dispatch(deco) self.write(ColorScheme._RESET) - class_str = (self.python_keyword("class ") + + class_str = (self.maybe_colorize_python_keyword("class ") + self.maybe_colorize(t.name, ColorScheme.DEFNAME)) self.fill(class_str, lineno_node=t) self.write("(") @@ -389,7 +443,7 @@ class Unparser: self.dispatch(deco) self.write(ColorScheme._RESET) - def_str = (self.python_keyword(fill_suffix) + + def_str = (self.maybe_colorize_python_keyword(fill_suffix) + " " + self.maybe_colorize(t.name, ColorScheme.DEFNAME) + "(") self.fill(def_str, lineno_node=t) self.dispatch(t.args) @@ -403,28 +457,28 @@ class Unparser: # TODO: Python 3.8 type_comment, ignore it? def _For(self, t): - self.__For_helper(self.python_keyword("for "), t) + self.__For_helper(self.maybe_colorize_python_keyword("for "), t) def _AsyncFor(self, t): - self.__For_helper(self.python_keyword("async for "), t) + self.__For_helper(self.maybe_colorize_python_keyword("async for "), t) def __For_helper(self, fill, t): self.fill(fill, lineno_node=t) self.dispatch(t.target) - self.write(self.python_keyword(" in ")) + self.write(self.maybe_colorize_python_keyword(" in ")) self.dispatch(t.iter) self.enter() self.dispatch(t.body) self.leave() if t.orelse: - self.fill(self.python_keyword("else")) + self.fill(self.maybe_colorize_python_keyword("else")) self.enter() self.dispatch(t.orelse) self.leave() # TODO: Python 3.8 type_comment, ignore it? def _If(self, t): - self.fill(self.python_keyword("if "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("if "), lineno_node=t) self.dispatch(t.test) self.enter() self.dispatch(t.body) @@ -433,32 +487,32 @@ class Unparser: while (t.orelse and len(t.orelse) == 1 and isinstance(t.orelse[0], ast.If)): t = t.orelse[0] - self.fill(self.python_keyword("elif ")) + self.fill(self.maybe_colorize_python_keyword("elif ")) self.dispatch(t.test) self.enter() self.dispatch(t.body) self.leave() # final else if t.orelse: - self.fill(self.python_keyword("else")) + self.fill(self.maybe_colorize_python_keyword("else")) self.enter() self.dispatch(t.orelse) self.leave() def _While(self, t): - self.fill(self.python_keyword("while "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("while "), lineno_node=t) self.dispatch(t.test) self.enter() self.dispatch(t.body) self.leave() if t.orelse: - self.fill(self.python_keyword("else")) + self.fill(self.maybe_colorize_python_keyword("else")) self.enter() self.dispatch(t.orelse) self.leave() def _With(self, t): - self.fill(self.python_keyword("with "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("with "), lineno_node=t) interleave(lambda: self.write(", "), self.dispatch, t.items) self.enter() self.dispatch(t.body) @@ -466,7 +520,7 @@ class Unparser: # TODO: Python 3.8 type_comment, ignore it? def _AsyncWith(self, t): - self.fill(self.python_keyword("async with "), lineno_node=t) + self.fill(self.maybe_colorize_python_keyword("async with "), lineno_node=t) interleave(lambda: self.write(", "), self.dispatch, t.items) self.enter() self.dispatch(t.body) @@ -511,6 +565,8 @@ class Unparser: v = self.maybe_colorize(v, ColorScheme.BUILTINEXCEPTION) elif v in builtin_others: v = self.maybe_colorize(v, ColorScheme.BUILTINOTHER) + elif self.expander and self.expander.isbound(v): + v = self.maybe_colorize(v, ColorScheme.MACRONAME) self.write(v) def _NameConstant(self, t): # up to Python 3.7 @@ -558,21 +614,21 @@ class Unparser: def _comprehension(self, t): if t.is_async: - self.write(self.python_keyword(" async")) - self.write(self.python_keyword(" for ")) + self.write(self.maybe_colorize_python_keyword(" async")) + self.write(self.maybe_colorize_python_keyword(" for ")) self.dispatch(t.target) - self.write(self.python_keyword(" in ")) + self.write(self.maybe_colorize_python_keyword(" in ")) self.dispatch(t.iter) for if_clause in t.ifs: - self.write(self.python_keyword(" if ")) + self.write(self.maybe_colorize_python_keyword(" if ")) self.dispatch(if_clause) def _IfExp(self, t): self.write("(") self.dispatch(t.body) - self.write(self.python_keyword(" if ")) + self.write(self.maybe_colorize_python_keyword(" if ")) self.dispatch(t.test) - self.write(self.python_keyword(" else ")) + self.write(self.maybe_colorize_python_keyword(" else ")) self.dispatch(t.orelse) self.write(")") @@ -633,7 +689,7 @@ class Unparser: boolops = {ast.And: 'and', ast.Or: 'or'} def _BoolOp(self, t): self.write("(") - s = self.python_keyword(self.boolops[t.op.__class__]) + s = self.maybe_colorize_python_keyword(self.boolops[t.op.__class__]) s = f" {s} " interleave(lambda: self.write(s), self.dispatch, t.values) self.write(")") @@ -766,13 +822,13 @@ class Unparser: if hasattr(t, "posonlyargs"): args_sets = [t.posonlyargs, t.args] defaults_sets = [defaults[:nposonlyargs], defaults[nposonlyargs:]] - set_separator = ', /' else: args_sets = [t.args] defaults_sets = [defaults] - set_separator = '' - for args, defaults in zip(args_sets, defaults_sets): + def write_arg_default_pairs(data): + nonlocal first + args, defaults = data for a, d in zip(args, defaults): if first: first = False @@ -782,7 +838,14 @@ class Unparser: if d: self.write("=") self.dispatch(d) - self.write(set_separator) + + def maybe_separate_positional_only_args(): + if not first: + self.write(', /') + + interleave(maybe_separate_positional_only_args, + write_arg_default_pairs, + zip(args_sets, defaults_sets)) # varargs, or bare '*' if no varargs but keyword-only arguments present if t.vararg or t.kwonlyargs: @@ -830,7 +893,16 @@ class Unparser: def _Lambda(self, t): self.write("(") - self.write(self.python_keyword("lambda ")) + self.write(self.maybe_colorize_python_keyword("lambda")) + + def takes_arguments(lam): + a = lam.args + if hasattr(a, "posonlyargs") and a.posonlyargs: + return True + return a.args or a.vararg or a.kwonlyargs or a.kwarg + if takes_arguments(t): + self.write(" ") + self.dispatch(t.args) self.write(": ") self.dispatch(t.body) @@ -839,29 +911,29 @@ class Unparser: def _alias(self, t): self.write(t.name) if t.asname: - self.write(self.python_keyword(" as ") + t.asname) + self.write(self.maybe_colorize_python_keyword(" as ") + t.asname) def _withitem(self, t): self.dispatch(t.context_expr) if t.optional_vars: - self.write(self.python_keyword(" as ")) + self.write(self.maybe_colorize_python_keyword(" as ")) self.dispatch(t.optional_vars) -def unparse(tree, *, debug=False, color=False): +def unparse(tree, *, debug=False, color=False, expander=None): """Convert the AST `tree` into source code. Return the code as a string. `debug`: bool, print invisible nodes (`Module`, `Expr`). The output is then not valid Python, but may better show the problem when code produced by a macro mysteriously - fails to compile (even though the unparse looks ok). + fails to compile (even though a non-debug unparse looks ok). Upon invalid input, raises `UnparserError`. """ try: with io.StringIO() as output: - Unparser(tree, file=output, debug=debug, color=color) + Unparser(tree, file=output, debug=debug, color=color, expander=expander) code = output.getvalue().strip() return code except UnparserError as err: # fall back to an AST dump @@ -877,7 +949,7 @@ def unparse(tree, *, debug=False, color=False): raise UnparserError(msg) from err -def unparse_with_fallbacks(tree, *, debug=False, color=False): +def unparse_with_fallbacks(tree, *, debug=False, color=False, expander=None): """Like `unparse`, but upon error, don't raise; return the error message. Usually you'll want the exception to be raised. This is mainly useful to @@ -886,9 +958,9 @@ def unparse_with_fallbacks(tree, *, debug=False, color=False): at the receiving end. """ try: - text = unparse(tree, debug=debug, color=color) + text = unparse(tree, debug=debug, color=color, expander=expander) except UnparserError as err: - text = err.args[0] + text = str(err) except Exception as err: # This can only happen if there is a bug in the unparser, but we don't # want to lose the macro use site filename and line number if this diff --git a/kibot/out_pcbdraw.py b/kibot/out_pcbdraw.py index b282303b..2e56de27 100644 --- a/kibot/out_pcbdraw.py +++ b/kibot/out_pcbdraw.py @@ -17,7 +17,7 @@ from .optionable import Optionable from .out_base import VariantOptions from .macros import macros, document, output_class # noqa: F401 from . import log -from .mcpyrate.debug import macros, step_expansion +from .mcpyrate.debug import macros, step_expansion # noqa: F811,F401 logger = log.get_logger(__name__) @@ -305,4 +305,3 @@ class PcbDraw(BaseOutput): # noqa: F821 with document: self.options = PcbDrawOptions """ [dict] Options for the `pcbdraw` output """ -