Updated mcpyrate, now the time is 260 ms, just 13% over mcpy

kibot/mcpyrate/dialects.py

@@ -5,10 +5,10 @@ __all__ = ["Dialect",
            "expand_dialects"]
 
 import ast
-from collections import deque
+import importlib
+import importlib.util
 import re
 from sys import stderr
-import tokenize
 
 from .coreutils import ismacroimport, get_macros
 from .unparser import unparse_with_fallbacks
@@ -154,7 +154,7 @@ class DialectExpander:
 
         Return value is an AST for the module.
         '''
-        text = _decode_source_content(data)
+        text = importlib.util.decode_source(data)
         text = self.transform_source(text)
         try:
             tree = ast.parse(data, filename=self.filename, mode="exec")
@@ -276,9 +276,11 @@ class DialectExpander:
         '''Find the first dialect-import statement by scanning the AST `tree`.
 
         Transform the dialect-import into `import ...`, where `...` is the absolute
-        module name the dialects are being imported from.
+        module name the dialects are being imported from. As a side effect, import
+        the dialect definition module.
 
-        As a side effect, import the dialect definition module.
+        Primarily meant to be called with `tree` the AST of a module that
+        uses dialects, but works with any `tree` that has a `body` attribute.
 
         A dialect-import is a statement of the form::
 
@@ -302,15 +304,6 @@ class DialectExpander:
                                              statement)
         return module_absname, bindings
 
-
-def _decode_source_content(data):
-    '''Decode a .py source file from bytes to string, parsing the encoding tag like `tokenize`.'''
-    lines = deque(data.split(b"\n"))
-    def readline():
-        return lines.popleft()
-    encoding, lines_read = tokenize.detect_encoding(readline)
-    return data.decode(encoding)
-
 # --------------------------------------------------------------------------------
 
 def expand_dialects(data, *, filename):

kibot/mcpyrate/importer.py

@@ -8,6 +8,7 @@ import importlib.util
 from importlib.machinery import FileFinder, SourceFileLoader
 import tokenize
 import os
+import pickle
 import sys
 
 from .core import MacroExpansionError
@@ -25,7 +26,6 @@ def source_to_xcode(self, data, path, *, _optimize=-1):
     Intercepts the source to bytecode transformation.
     '''
     tree = expand_dialects(data, filename=path)
-    # tree = ast.parse(data)
 
     module_macro_bindings = find_macros(tree, filename=path)
     expansion = expand_macros(tree, bindings=module_macro_bindings, filename=path)
@@ -61,20 +61,69 @@ def path_xstats(self, path):
     if path in _xstats_cache:
         return _xstats_cache[path]
 
-    # TODO: This can be slow, the point of `.pyc` is to avoid the parse-and-compile cost.
+    stat_result = os.stat(path)
-    # TODO: We do save the macro-expansion cost, though, and that's likely much more expensive.
+
+    # Try for cached macro-import statements for `path` to avoid the parse cost.
     #
-    # If that becomes an issue, maybe make our own cache file storing the
+    # This is a single node in the dependency graph; the result depends only
-    # macro-imports found in source file `path`, store it in the pyc
+    # on the content of the source file `path` itself. So we invalidate the
-    # directory, and invalidate it based on the mtime of `path` (only)?
+    # macro-import statement cache for `path` based on the mtime of `path` only.
+    #
+    # For a given source file `path`, the `.pyc` sometimes becomes newer than
+    # the macro-dependency cache. This is normal. Unlike the bytecode, the
+    # macro-dependency cache only needs to be refreshed when the text of the
+    # source file `path` changes.
+    #
+    # So if some of the macro-dependency source files have changed (so `path`
+    # must be re-expanded and recompiled), but `path` itself hasn't, the text
+    # of the source file `path` will still have the same macro-imports it did
+    # last time.
+    #
+    pycpath = importlib.util.cache_from_source(path)
+    if pycpath.endswith(".pyc"):
+        pycpath = pycpath[:-4]
+    importcachepath = pycpath + ".mcpyrate.pickle"
+    try:
+        cache_valid = False
+        with open(importcachepath, "rb") as importcachefile:
+            data = pickle.load(importcachefile)
+        if data["st_mtime_ns"] == stat_result.st_mtime_ns:
+            cache_valid = True
+    except Exception:
+        pass
+
+    if cache_valid:
+        macro_and_dialect_imports = data["macroimports"] + data["dialectimports"]
+        has_relative_macroimports = data["has_relative_macroimports"]
+    else:
+        # This can be slow, the point of `.pyc` is to avoid the parse-and-compile cost.
+        # We do save the macro-expansion cost, though, and that's likely much more expensive.
+        #
+        # TODO: Dialects may inject imports in the template that the dialect transformer itself
+        # TODO: doesn't need. How to detect those? Regex-search the source text?
         with tokenize.open(path) as sourcefile:
             tree = ast.parse(sourcefile.read())
-
         macroimports = [stmt for stmt in tree.body if ismacroimport(stmt)]
         dialectimports = [stmt for stmt in tree.body if ismacroimport(stmt, magicname="dialects")]
+
         macro_and_dialect_imports = macroimports + dialectimports
         has_relative_macroimports = any(macroimport.level for macroimport in macro_and_dialect_imports)
 
+        # macro-import statement cache goes with the .pyc
+        if not sys.dont_write_bytecode:
+            data = {"st_mtime_ns": stat_result.st_mtime_ns,
+                    "macroimports": macroimports,
+                    "dialectimports": dialectimports,
+                    "has_relative_macroimports": has_relative_macroimports}
+            try:
+                with open(importcachepath, "wb") as importcachefile:
+                    pickle.dump(data, importcachefile)
+            except Exception:
+                pass
+
+    # The rest of the lookup process depends on the configuration of the currently
+    # running Python, particularly its `sys.path`, so we do it dynamically.
+    #
     # TODO: some duplication with code in mcpyrate.coreutils.get_macros, including the error messages.
     package_absname = None
     if has_relative_macroimports:
@@ -96,7 +145,6 @@ def path_xstats(self, path):
         stats = path_xstats(self, origin)
         mtimes.append(stats['mtime'])
 
-    stat_result = os.stat(path)
     mtime = stat_result.st_mtime_ns * 1e-9
     # size = stat_result.st_size
     mtimes.append(mtime)

kibot/mcpyrate/repl/console.py

@@ -22,6 +22,7 @@ import code
 import textwrap
 
 from .. import __version__ as mcpyrate_version
+from ..core import MacroExpansionError
 from ..debug import format_bindings
 from ..expander import find_macros, MacroExpander, global_postprocess
 from .utils import get_makemacro_sourcecode
@@ -110,7 +111,7 @@ class MacroConsole(code.InteractiveConsole):
 
             tree = ast.Interactive(tree.body)
             code = compile(tree, filename, symbol, self.compile.compiler.flags, 1)
-        except (OverflowError, SyntaxError, ValueError):
+        except (OverflowError, SyntaxError, ValueError, MacroExpansionError):
             self.showsyntaxerror(filename)
             return False  # erroneous input
         except ModuleNotFoundError as err:  # during macro module lookup
@@ -138,5 +139,10 @@ class MacroConsole(code.InteractiveConsole):
         self._macro_bindings_changed = False
 
         for asname, function in self.expander.bindings.items():
+            if not function.__module__:    # Macros defined in the REPL have `__module__=None`.
+                continue
+            try:
                 source = f"from {function.__module__} import {function.__qualname__} as {asname}"
                 self._internal_execute(source)
+            except (ModuleNotFoundError, ImportError):
+                pass

kibot/mcpyrate/repl/iconsole.py

@@ -194,6 +194,8 @@ class IMcpyrateExtension:
                                    silent=True)
 
         for asname, function in self.macro_transformer.expander.bindings.items():
+            if not function.__module__:
+                continue
             commands = ["%%ignore_importerror",
                         f"from {function.__module__} import {function.__qualname__} as {asname}"]
             internal_execute("\n".join(commands))

kibot/mcpyrate/repl/macropython.py

@@ -4,18 +4,21 @@
 
 # TODO: Currently tested in CPython 3.6, and PyPy3 7.3.0 (Python 3.6). Test in 3.7+.
 
+import argparse
+import atexit
 from importlib import import_module
 from importlib.util import resolve_name, module_from_spec
-import pathlib
 import os
+import pathlib
 import sys
-import argparse
 
 from ..coreutils import relativize
 
 from .. import activate  # noqa: F401
 
 __version__ = "3.0.0"
+
+_config_dir = "~/.config/mcpyrate"
 _macropython_module = None  # sys.modules doesn't always seem to keep it, so stash it locally too.
 
 def import_module_as_main(name, script_mode):
@@ -153,6 +156,18 @@ def main():
         readline.set_completer(rlcompleter.Completer(namespace=repl_locals).complete)
         readline.parse_and_bind("tab: complete")  # PyPy ignores this, but not needed there.
 
+        config_dir = pathlib.Path(_config_dir).expanduser().resolve()
+        try:
+            readline.read_history_file(config_dir / "macropython_history")
+        except FileNotFoundError:
+            pass
+
+        def save_history():
+            config_dir.mkdir(parents=True, exist_ok=True)
+            readline.set_history_length(1000)
+            readline.write_history_file(config_dir / "macropython_history")
+        atexit.register(save_history)
+
         # Add CWD to import path like the builtin interactive console does.
         if sys.path[0] != "":
             sys.path.insert(0, "")
@@ -209,6 +224,8 @@ def main():
         # if not spec:
         #     raise ImportError(f"Not a Python module: '{opts.filename}'"
         # module = module_from_spec(spec)
+        # TODO: if we use this approach, we should first initialize parent packages.
+        # sys.modules[module.__name__] = module
         # spec.loader.exec_module(module)
 
         root_path, relative_path = relativize(opts.filename)