Updated mcpyrate, now the cache can be enabled.

This compensates most of the speed penalty:
macropy:            ~650 ms (has cache)
mcpyrate no cache:   440 ms
mcpyrate:            290 ms
mcpy:                230 ms (no cache)

kibot/__main__.py

@@ -274,7 +274,7 @@ def main():
         var = redef.split('=')[0]
         GS.global_from_cli[var] = redef[len(var)+1:]
 
-    clean_cache()
+    # clean_cache()
 
     # Output dir: relative to CWD (absolute path overrides)
     GS.out_dir = os.path.join(os.getcwd(), args.out_dir)

kibot/mcpyrate/activate.py

@@ -25,21 +25,11 @@ the `PYTHONDONTWRITEBYTECODE` environment variable, and the attribute
     https://www.python.org/dev/peps/pep-0552/
 '''
 
-# from importlib.machinery import SourceFileLoader, FileFinder
+from importlib.machinery import SourceFileLoader, FileFinder
-# 
-# from .importer import source_to_xcode, path_xstats, invalidate_xcaches
-# 
-# SourceFileLoader.source_to_code = source_to_xcode
-# # we could replace SourceFileLoader.set_data with a no-op to force-disable pyc caching.
-# SourceFileLoader.path_stats = path_xstats
-# FileFinder.invalidate_caches = invalidate_xcaches
 
-from importlib.machinery import SourceFileLoader
+from .importer import source_to_xcode, path_xstats, invalidate_xcaches
-
-from .importer import source_to_xcode
-
-def nop(*args, **kwargs):
-    pass
 
 SourceFileLoader.source_to_code = source_to_xcode
-SourceFileLoader.set_data = nop
+# we could replace SourceFileLoader.set_data with a no-op to force-disable pyc caching.
+SourceFileLoader.path_stats = path_xstats
+FileFinder.invalidate_caches = invalidate_xcaches

kibot/mcpyrate/importer.py

@@ -4,10 +4,11 @@
 __all__ = ['source_to_xcode', 'path_xstats', 'invalidate_xcaches']
 
 import ast
-import tokenize
-import os
 import importlib.util
 from importlib.machinery import FileFinder, SourceFileLoader
+import tokenize
+import os
+import sys
 
 from .core import MacroExpansionError
 from .dialects import expand_dialects
@@ -24,6 +25,7 @@ def source_to_xcode(self, data, path, *, _optimize=-1):
     Intercepts the source to bytecode transformation.
     '''
     tree = expand_dialects(data, filename=path)
+    # tree = ast.parse(data)
 
     module_macro_bindings = find_macros(tree, filename=path)
     expansion = expand_macros(tree, bindings=module_macro_bindings, filename=path)
@@ -100,6 +102,14 @@ def path_xstats(self, path):
     mtimes.append(mtime)
 
     result = {'mtime': max(mtimes)}  # and sum(sizes)? OTOH, as of Python 3.8, only 'mtime' is mandatory.
+    if sys.version_info >= (3, 7, 0):
+        # Docs say `size` is optional, and this is correct in 3.6 (and in PyPy3 7.3.0):
+        # https://docs.python.org/3/library/importlib.html#importlib.abc.SourceLoader.path_stats
+        #
+        # but in 3.7 and later, the implementation is expecting at least a `None` there,
+        # if the `size` is not used. See `get_code` in:
+        # https://github.com/python/cpython/blob/master/Lib/importlib/_bootstrap_external.py
+        result['size'] = None
     _xstats_cache[path] = result
     return result