diff --git a/.coveragerc b/.coveragerc index 9af10502..68167f59 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,7 @@ source = kibot src omit = */kibot/docopt.py + */kibot/mcpy/* [report] exclude_lines = diff --git a/debian/control b/debian/control index 93871a07..3719eb90 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ X-Python3-Version: >= 3.2 Package: kibot Architecture: all Multi-Arch: foreign -Depends: ${misc:Depends}, ${python3:Depends}, python3-yaml, kicad (>= 5.1.0), python3-wxgtk4.0, python3-mcpy +Depends: ${misc:Depends}, ${python3:Depends}, python3-yaml, kicad (>= 5.1.0), python3-wxgtk4.0 Recommends: kibom.inti-cmnb, kicad-automation-scripts.inti-cmnb (>= 1.1.2), interactivehtmlbom.inti-cmnb, pcbdraw, python3-xlsxwriter Description: KiCad Bot KiBot is a program which helps you to automate the generation of KiCad diff --git a/kibot/kiplot.py b/kibot/kiplot.py index 5dee9cd3..7c699a86 100644 --- a/kibot/kiplot.py +++ b/kibot/kiplot.py @@ -60,7 +60,7 @@ def _load_actions(path): def load_actions(): """ Load all the available ouputs and preflights """ - from mcpy import activate + from kibot.mcpy import activate # activate.activate() _load_actions(os.path.abspath(os.path.dirname(__file__))) home = os.environ.get('HOME') diff --git a/kibot/mcpy/__init__.py b/kibot/mcpy/__init__.py new file mode 100644 index 00000000..a084bf11 --- /dev/null +++ b/kibot/mcpy/__init__.py @@ -0,0 +1 @@ +from .unparse import unparse # noqa: F401 diff --git a/kibot/mcpy/activate.py b/kibot/mcpy/activate.py new file mode 100644 index 00000000..4ce0f254 --- /dev/null +++ b/kibot/mcpy/activate.py @@ -0,0 +1,24 @@ +"""Install mcpy hooks to preprocess source files. + +Actually, the library monkey-patches SourceFileLoader to compile the code +in a different way, providing the macro-expansion for the AST before compiling +into real code. +""" +from .importhooks import source_to_xcode, nop +from importlib.machinery import SourceFileLoader + +old_source_to_code = SourceFileLoader.source_to_code +old_set_data = SourceFileLoader.set_data + + +def activate(): + SourceFileLoader.source_to_code = source_to_xcode + SourceFileLoader.set_data = nop + + +def de_activate(): + SourceFileLoader.source_to_code = old_source_to_code + SourceFileLoader.set_data = old_set_data + + +activate() diff --git a/kibot/mcpy/core.py b/kibot/mcpy/core.py new file mode 100644 index 00000000..86be296a --- /dev/null +++ b/kibot/mcpy/core.py @@ -0,0 +1,146 @@ +""" Provide the functionality to find and expand macros. """ +import sys +from ast import Name, copy_location, Import, alias, ImportFrom +from .visitors import BaseMacroExpander + + +class _MacroExpander(BaseMacroExpander): + + def visit_With(self, withstmt): + """ + Check for a with macro as + + with macroname: + # with's body is the argument + + It replaces the with node with the result of the macro. + """ + with_item = withstmt.items[0] + candidate = with_item.context_expr + if isinstance(candidate, Name) and self._ismacro(candidate.id): + macro = candidate.id + tree = withstmt.body + kw = {'optional_vars': with_item.optional_vars} + new_tree = self._expand('block', withstmt, macro, tree, kw) + else: + new_tree = self.generic_visit(withstmt) + + return new_tree + + def visit_Subscript(self, subscript): + """ + Check for a expression macro as + + macroname['index expression is the argument'] + + It replaces the expression node with the result of the macro. + """ + candidate = subscript.value + if isinstance(candidate, Name) and self._ismacro(candidate.id): + macro = candidate.id + tree = subscript.slice.value + new_tree = self._expand('expr', subscript, macro, tree) + else: + new_tree = self.generic_visit(subscript) + + return new_tree + + def visit_ClassDef(self, classdef): + return self._visit_Decorated(classdef) + + def visit_FunctionDef(self, functiondef): + return self._visit_Decorated(functiondef) + + def _visit_Decorated(self, decorated): + """ + Check for a expression macro as + + @macroname + def f(): + # The whole function is the target of the macro + + Or + + @macroname + class C(): + # The whole class is the target of the macro + + It replaces the whole decorated node with the result of the macro. + """ + macros, decorators = self._filter_out_macros(decorated.decorator_list) + decorated.decorator_list = decorators + if macros: + for macro in reversed(macros): + new_tree = self._expand('decorator', decorated, macro, decorated) + else: + new_tree = self.generic_visit(decorated) + + return new_tree + + def _filter_out_macros(self, decorators): + """ + Identify macro names inside a decorator list, and return a pair with + macro names and the decorators not identified as macros. + """ + macros, remaining = [], [] + for d in decorators: + if isinstance(d, Name) and self._ismacro(d.id): + macros.append(d.id) + else: + remaining.append(d) + + return macros, remaining + + +def expand_macros(tree, bindings): + """ + Return an expanded version of tree with macros applied. + """ + expansion = _MacroExpander(bindings).visit(tree) + return expansion + + +def find_macros(tree): + """ + Looks for `from ... import macros, ...` statements in the module body and + returns a dict with names and implementations for found macros or an empty + dict if no macros are used. + """ + bindings = {} + for index, statement in enumerate(tree.body): + if _is_macro_import(statement): + bindings.update(_get_macros(statement)) + # Remove all names to prevent macro names to be used + module = statement.module + tree.body[index] = copy_location( + Import(names=[alias(name=module, asname=None)]), + statement + ) + + return bindings + + +def _is_macro_import(statement): + """ + A "macro import" is a statement with the form of + + from ... import macros, ... + + """ + is_macro_import = False + if isinstance(statement, ImportFrom): + firstimport = statement.names[0] + if firstimport.name == 'macros' and firstimport.asname is None: + is_macro_import = True + + return is_macro_import + + +def _get_macros(macroimport): + """ + Returns a map with names and macros from the macro import statement. + """ + modulename = macroimport.module + __import__(modulename) + module = sys.modules[modulename] + return {name.asname or name.name: getattr(module, name.name) for name in macroimport.names[1:]} diff --git a/kibot/mcpy/importhooks.py b/kibot/mcpy/importhooks.py new file mode 100644 index 00000000..2f8ee943 --- /dev/null +++ b/kibot/mcpy/importhooks.py @@ -0,0 +1,16 @@ +import ast +from .core import find_macros, expand_macros + + +def nop(*args, **kw): + pass + + +def source_to_xcode(self, data, path, *, _optimize=-1): + '''Intercepts the source to code transformation and expand the macros + before compiling to actual code.''' + tree = ast.parse(data) + module_macro_bindings = find_macros(tree) + expansion = expand_macros(tree, bindings=module_macro_bindings) + return compile(expansion, path, 'exec', dont_inherit=True, + optimize=_optimize) diff --git a/kibot/mcpy/unparse.py b/kibot/mcpy/unparse.py new file mode 100644 index 00000000..070d619f --- /dev/null +++ b/kibot/mcpy/unparse.py @@ -0,0 +1,608 @@ +"""Usage: unparse.py """ +import sys +import ast +# import tokenize +import io +# import os + +# Large float and imaginary literals get turned into infinities in the AST. +# We unparse those infinities to INFSTR. +INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) + + +def interleave(inter, f, seq): + """Call f on each item in seq, calling inter() in between. + """ + seq = iter(seq) + try: + f(next(seq)) + except StopIteration: + pass + else: + for x in seq: + inter() + f(x) + + +class Unparser: + """Methods in this class recursively traverse an AST and + output source code for the abstract syntax; original formatting + is disregarded. """ + + def __init__(self, tree, file=sys.stdout): + """Unparser(tree, file=sys.stdout) -> None. + Print the source for tree to file.""" + self.f = file + self._indent = 0 + self.dispatch(tree) + print("", file=self.f) + self.f.flush() + + def fill(self, text=""): + "Indent a piece of text, according to the current indentation level" + self.f.write("\n"+" "*self._indent + text) + + def write(self, text): + "Append a piece of text to the current line." + self.f.write(text) + + def enter(self): + "Print ':', and increase the indentation." + self.write(":") + self._indent += 1 + + def leave(self): + "Decrease the indentation level." + self._indent -= 1 + + def dispatch(self, tree): + "Dispatcher function, dispatching tree type T to method _T." + if isinstance(tree, list): + for t in tree: + self.dispatch(t) + return + meth = getattr(self, "_"+tree.__class__.__name__) + meth(tree) + + # ############# Unparsing methods ###################### + # There should be one method per concrete grammar type # + # Constructors should be grouped by sum type. Ideally, # + # this would follow the order in the grammar, but # + # currently doesn't. # + # ###################################################### + + def _Module(self, tree): + for stmt in tree.body: + self.dispatch(stmt) + + # stmt + def _Expr(self, tree): + self.fill() + self.dispatch(tree.value) + + def _Import(self, t): + self.fill("import ") + interleave(lambda: self.write(", "), self.dispatch, t.names) + + def _ImportFrom(self, t): + self.fill("from ") + self.write("." * t.level) + if t.module: + self.write(t.module) + self.write(" import ") + interleave(lambda: self.write(", "), self.dispatch, t.names) + + def _Assign(self, t): + self.fill() + for target in t.targets: + self.dispatch(target) + self.write(" = ") + self.dispatch(t.value) + + def _AugAssign(self, t): + self.fill() + self.dispatch(t.target) + self.write(" "+self.binop[t.op.__class__.__name__]+"= ") + self.dispatch(t.value) + + def _Return(self, t): + self.fill("return") + if t.value: + self.write(" ") + self.dispatch(t.value) + + def _Pass(self, t): + self.fill("pass") + + def _Break(self, t): + self.fill("break") + + def _Continue(self, t): + self.fill("continue") + + def _Delete(self, t): + self.fill("del ") + interleave(lambda: self.write(", "), self.dispatch, t.targets) + + def _Assert(self, t): + self.fill("assert ") + self.dispatch(t.test) + if t.msg: + self.write(", ") + self.dispatch(t.msg) + + def _Global(self, t): + self.fill("global ") + interleave(lambda: self.write(", "), self.write, t.names) + + def _Nonlocal(self, t): + self.fill("nonlocal ") + interleave(lambda: self.write(", "), self.write, t.names) + + def _Await(self, t): + self.write("(") + self.write("await") + if t.value: + self.write(" ") + self.dispatch(t.value) + self.write(")") + + def _Yield(self, t): + self.write("(") + self.write("yield") + if t.value: + self.write(" ") + self.dispatch(t.value) + self.write(")") + + def _YieldFrom(self, t): + self.write("(") + self.write("yield from") + if t.value: + self.write(" ") + self.dispatch(t.value) + self.write(")") + + def _Raise(self, t): + self.fill("raise") + if not t.exc: + assert not t.cause + return + self.write(" ") + self.dispatch(t.exc) + if t.cause: + self.write(" from ") + self.dispatch(t.cause) + + def _Try(self, t): + self.fill("try") + self.enter() + self.dispatch(t.body) + self.leave() + for ex in t.handlers: + self.dispatch(ex) + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + if t.finalbody: + self.fill("finally") + self.enter() + self.dispatch(t.finalbody) + self.leave() + + def _ExceptHandler(self, t): + self.fill("except") + if t.type: + self.write(" ") + self.dispatch(t.type) + if t.name: + self.write(" as ") + self.write(t.name) + self.enter() + self.dispatch(t.body) + self.leave() + + def _ClassDef(self, t): + self.write("\n") + for deco in t.decorator_list: + self.fill("@") + self.dispatch(deco) + self.fill("class "+t.name) + self.write("(") + comma = False + for e in t.bases: + if comma: + self.write(", ") + else: + comma = True + self.dispatch(e) + for e in t.keywords: + if comma: + self.write(", ") + else: + comma = True + self.dispatch(e) + self.write(")") + + self.enter() + self.dispatch(t.body) + self.leave() + + def _FunctionDef(self, t): + self.__FunctionDef_helper(t, "def") + + def _AsyncFunctionDef(self, t): + self.__FunctionDef_helper(t, "async def") + + def __FunctionDef_helper(self, t, fill_suffix): + self.write("\n") + for deco in t.decorator_list: + self.fill("@") + self.dispatch(deco) + def_str = fill_suffix+" "+t.name + "(" + self.fill(def_str) + self.dispatch(t.args) + self.write(")") + if t.returns: + self.write(" -> ") + self.dispatch(t.returns) + self.enter() + self.dispatch(t.body) + self.leave() + + def _For(self, t): + self.__For_helper("for ", t) + + def _AsyncFor(self, t): + self.__For_helper("async for ", t) + + def __For_helper(self, fill, t): + self.fill(fill) + self.dispatch(t.target) + self.write(" in ") + self.dispatch(t.iter) + self.enter() + self.dispatch(t.body) + self.leave() + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _If(self, t): + self.fill("if ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + # collapse nested ifs into equivalent elifs. + while (t.orelse and len(t.orelse) == 1 and + isinstance(t.orelse[0], ast.If)): + t = t.orelse[0] + self.fill("elif ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + # final else + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _While(self, t): + self.fill("while ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _With(self, t): + self.fill("with ") + interleave(lambda: self.write(", "), self.dispatch, t.items) + self.enter() + self.dispatch(t.body) + self.leave() + + def _AsyncWith(self, t): + self.fill("async with ") + interleave(lambda: self.write(", "), self.dispatch, t.items) + self.enter() + self.dispatch(t.body) + self.leave() + + # expr + def _Bytes(self, t): + self.write(repr(t.s)) + + def _Str(self, tree): + self.write(repr(tree.s)) + + def _Name(self, t): + self.write(t.id) + + def _NameConstant(self, t): + self.write(repr(t.value)) + + def _Num(self, t): + # Substitute overflowing decimal literal for AST infinities. + self.write(repr(t.n).replace("inf", INFSTR)) + + def _List(self, t): + self.write("[") + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write("]") + + def _ListComp(self, t): + self.write("[") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write("]") + + def _GeneratorExp(self, t): + self.write("(") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write(")") + + def _SetComp(self, t): + self.write("{") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write("}") + + def _DictComp(self, t): + self.write("{") + self.dispatch(t.key) + self.write(": ") + self.dispatch(t.value) + for gen in t.generators: + self.dispatch(gen) + self.write("}") + + def _comprehension(self, t): + self.write(" for ") + self.dispatch(t.target) + self.write(" in ") + self.dispatch(t.iter) + for if_clause in t.ifs: + self.write(" if ") + self.dispatch(if_clause) + + def _IfExp(self, t): + self.write("(") + self.dispatch(t.body) + self.write(" if ") + self.dispatch(t.test) + self.write(" else ") + self.dispatch(t.orelse) + self.write(")") + + def _Set(self, t): + assert(t.elts) # should be at least one element + self.write("{") + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write("}") + + def _Dict(self, t): + self.write("{") + + def write_pair(pair): + (k, v) = pair + self.dispatch(k) + self.write(": ") + self.dispatch(v) + interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values)) + self.write("}") + + def _Tuple(self, t): + self.write("(") + if len(t.elts) == 1: + (elt,) = t.elts + self.dispatch(elt) + self.write(",") + else: + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write(")") + + unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} + + def _UnaryOp(self, t): + self.write("(") + self.write(self.unop[t.op.__class__.__name__]) + self.write(" ") + self.dispatch(t.operand) + self.write(")") + + binop = {"Add": "+", "Sub": "-", "Mult": "*", "MatMult": "@", "Div": "/", "Mod": "%", + "LShift": "<<", "RShift": ">>", "BitOr": "|", "BitXor": "^", "BitAnd": "&", + "FloorDiv": "//", "Pow": "**"} + + def _BinOp(self, t): + self.write("(") + self.dispatch(t.left) + self.write(" " + self.binop[t.op.__class__.__name__] + " ") + self.dispatch(t.right) + self.write(")") + + cmpops = {"Eq": "==", "NotEq": "!=", "Lt": "<", "LtE": "<=", "Gt": ">", "GtE": ">=", + "Is": "is", "IsNot": "is not", "In": "in", "NotIn": "not in"} + + def _Compare(self, t): + self.write("(") + self.dispatch(t.left) + for o, e in zip(t.ops, t.comparators): + self.write(" " + self.cmpops[o.__class__.__name__] + " ") + self.dispatch(e) + self.write(")") + + boolops = {ast.And: 'and', ast.Or: 'or'} + + def _BoolOp(self, t): + self.write("(") + s = " %s " % self.boolops[t.op.__class__] + interleave(lambda: self.write(s), self.dispatch, t.values) + self.write(")") + + def _Attribute(self, t): + self.dispatch(t.value) + # Special case: 3.__abs__() is a syntax error, so if t.value + # is an integer literal then we need to either parenthesize + # it or add an extra space to get 3 .__abs__(). + if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): + self.write(" ") + self.write(".") + self.write(t.attr) + + def _Call(self, t): + self.dispatch(t.func) + self.write("(") + comma = False + for e in t.args: + if comma: + self.write(", ") + else: + comma = True + self.dispatch(e) + for e in t.keywords: + if comma: + self.write(", ") + else: + comma = True + self.dispatch(e) + self.write(")") + + def _Subscript(self, t): + self.dispatch(t.value) + self.write("[") + self.dispatch(t.slice) + self.write("]") + + def _Starred(self, t): + self.write("*") + self.dispatch(t.value) + + # slice + def _Ellipsis(self, t): + self.write("...") + + def _Index(self, t): + self.dispatch(t.value) + + def _Slice(self, t): + if t.lower: + self.dispatch(t.lower) + self.write(":") + if t.upper: + self.dispatch(t.upper) + if t.step: + self.write(":") + self.dispatch(t.step) + + def _ExtSlice(self, t): + interleave(lambda: self.write(', '), self.dispatch, t.dims) + + # argument + def _arg(self, t): + self.write(t.arg) + if t.annotation: + self.write(": ") + self.dispatch(t.annotation) + + # others + def _arguments(self, t): + first = True + # normal arguments + defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults + for a, d in zip(t.args, defaults): + if first: + first = False + else: + self.write(", ") + self.dispatch(a) + if d: + self.write("=") + self.dispatch(d) + + # varargs, or bare '*' if no varargs but keyword-only arguments present + if t.vararg or t.kwonlyargs: + if first: + first = False + else: + self.write(", ") + self.write("*") + if t.vararg: + self.write(t.vararg.arg) + if t.vararg.annotation: + self.write(": ") + self.dispatch(t.vararg.annotation) + + # keyword-only arguments + if t.kwonlyargs: + for a, d in zip(t.kwonlyargs, t.kw_defaults): + if first: + first = False + else: + self.write(", ") + self.dispatch(a), + if d: + self.write("=") + self.dispatch(d) + + # kwargs + if t.kwarg: + if first: + first = False + else: + self.write(", ") + self.write("**"+t.kwarg.arg) + if t.kwarg.annotation: + self.write(": ") + self.dispatch(t.kwarg.annotation) + + def _keyword(self, t): + if t.arg is None: + self.write("**") + else: + self.write(t.arg) + self.write("=") + self.dispatch(t.value) + + def _Lambda(self, t): + self.write("(") + self.write("lambda ") + self.dispatch(t.args) + self.write(": ") + self.dispatch(t.body) + self.write(")") + + def _alias(self, t): + self.write(t.name) + if t.asname: + self.write(" as "+t.asname) + + def _withitem(self, t): + self.dispatch(t.context_expr) + if t.optional_vars: + self.write(" as ") + self.dispatch(t.optional_vars) + + +def unparse(tree): + output = io.StringIO() + Unparser(tree, output) + code = output.getvalue().strip() + output.close() + return code diff --git a/kibot/mcpy/visitors.py b/kibot/mcpy/visitors.py new file mode 100644 index 00000000..8095b90e --- /dev/null +++ b/kibot/mcpy/visitors.py @@ -0,0 +1,58 @@ +# from functools import wraps +from ast import NodeTransformer, AST, copy_location, fix_missing_locations +from .unparse import unparse + + +class BaseMacroExpander(NodeTransformer): + """ + A base class for macro expander visitors. After identifying valid macro + syntax, the actual expander should return the result of calling `_expand()` + method with the proper arguments. + """ + + def __init__(self, bindings): + self.bindings = bindings + + def visit(self, tree): + """Short-circuit visit() to avoid expansions if no macros.""" + return super().visit(tree) if self.bindings else tree + + def _expand(self, syntax, target, macroname, tree, kw=None): + """ + Transform `target` node, replacing it with the expansion result of + aplying the named macro on the proper node and recursively treat the + expansion as well. + """ + macro = self.bindings[macroname] + kw = kw or {} + kw.update({ + 'syntax': syntax, + 'to_source': unparse, + 'expand_macros': self.visit + }) + expansion = _apply_macro(macro, tree, kw) + + return self._visit_expansion(expansion, target) + + def _visit_expansion(self, expansion, target): + """ + Ensures the macro expansions into None (deletions), other nodes or + list of nodes are expanded too. + """ + if expansion is not None: + is_node = isinstance(expansion, AST) + expansion = [expansion] if is_node else expansion + expansion = map(lambda n: copy_location(n, target), expansion) + expansion = map(fix_missing_locations, expansion) + expansion = map(self.visit, expansion) + expansion = list(expansion).pop() if is_node else list(expansion) + + return expansion + + def _ismacro(self, name): + return name in self.bindings + + +def _apply_macro(macro, tree, kw): + """ Executes the macro on tree passing extra kwargs. """ + return macro(tree, **kw) diff --git a/setup.py b/setup.py index 1c3749ca..9ec96e9d 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup(name='kibot', # Packages are marked using __init__.py packages=find_packages(), scripts=['src/kibot', 'src/kiplot'], - install_requires=['pyyaml', 'mcpy', 'xlsxwriter'], + install_requires=['pyyaml', 'xlsxwriter'], classifiers=['Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers',