More robust behavior when the Python cache interferes.

Files using macros can't be cached. Now if we find caches for them we
try to delete the caches (pypi installation for example). If we can't
remove the files a warning is generated.
If we fail to load a plug-in an error explaining it is generated.
Also added a test to reproduce it.

kibot/__main__.py

@@ -71,6 +71,35 @@ from .misc import (NO_PCB_FILE, NO_SCH_FILE, EXIT_BAD_ARGS)
 from .docopt import docopt
 
 logger = None
+has_macro = ['pre_filters',
+             'out_any_layer',
+             'out_svg_sch_print',
+             'pre_erc',
+             'out_gerb_drill',
+             'drill_marks',
+             'pre_update_xml',
+             'out_pdf_sch_print',
+             'out_hpgl',
+             'out_dxf',
+             'out_pdf_pcb_print',
+             'out_pdf',
+             'out_svg',
+             'out_pcbdraw',
+             'pre_ignore_unconnected',
+             'pre_check_zone_fills',
+             'out_gerber',
+             'out_any_drill',
+             'out_step',
+             'out_ps',
+             'pre_drc',
+             'out_excellon',
+             'out_bom',
+             'out_base',
+             'out_ibom',
+             'out_kibom',
+             'out_position',
+             'layer',
+             ]
 
 
 def list_pre_and_outs(logger, outputs):
@@ -186,6 +215,25 @@ def set_locale():
         pass
 
 
+def clean_cache():
+    """ Files that expands macros can't be pre-compiled """
+    here = os.path.abspath(os.path.dirname(__file__))
+    cache = os.path.join(here, '__pycache__')
+    logger.debug('Python cache dir: '+cache)
+    try:
+        if os.path.isdir(cache):
+            for f in has_macro:
+                fnames = glob(os.path.join(cache, f+'.*'))
+                for fname in fnames:
+                    if os.path.isfile(fname):
+                        logger.debug('Removing '+fname)
+                        os.remove(fname)
+    except PermissionError:
+        logger.warning('Wrong installation, avoid creating Python cache files\n'
+                       'If you are using `pip` to install use the `--no-compile` option:\n'
+                       '$ pip install --no-compile kibot\n')
+
+
 def main():
     set_locale()
     ver = 'KiBot '+__version__+' - '+__copyright__+' - License: '+__license__
@@ -197,6 +245,8 @@ def main():
     GS.debug_enabled = logger.getEffectiveLevel() <= DEBUG
     GS.debug_level = args.verbose
 
+    clean_cache()
+
     # Output dir: relative to CWD (absolute path overrides)
     GS.out_dir = os.path.join(os.getcwd(), args.out_dir)
 

kibot/kiplot.py

@@ -20,7 +20,7 @@ from importlib.util import (spec_from_file_location, module_from_spec)
 
 from .gs import (GS)
 from .misc import (PLOT_ERROR, NO_PCBNEW_MODULE, MISSING_TOOL, CMD_EESCHEMA_DO, URL_EESCHEMA_DO, CORRUPTED_PCB,
-                   EXIT_BAD_ARGS, CORRUPTED_SCH, EXIT_BAD_CONFIG)
+                   EXIT_BAD_ARGS, CORRUPTED_SCH, EXIT_BAD_CONFIG, WRONG_INSTALL)
 from .error import PlotError, KiPlotConfigurationError, config_error, trace_dump
 from .pre_base import BasePreFlight
 from .kicad.v5_sch import Schematic, SchFileError
@@ -46,7 +46,12 @@ def _import(name, path):
     # Python 3.4+ import mechanism
     spec = spec_from_file_location("kibot."+name, path)
     mod = module_from_spec(spec)
-    spec.loader.exec_module(mod)
+    try:
+        spec.loader.exec_module(mod)
+    except ImportError:
+        trace_dump()
+        logger.error('Unable to import plug-ins')
+        exit(WRONG_INSTALL)
 
 
 def _load_actions(path):

kibot/misc.py

@@ -28,6 +28,7 @@ WONT_OVERWRITE = 19
 PCBDRAW_ERR = 20
 SVG_SCH_PRINT = 21
 CORRUPTED_SCH = 22
+WRONG_INSTALL = 23
 
 CMD_EESCHEMA_DO = 'eeschema_do'
 URL_EESCHEMA_DO = 'https://github.com/INTI-CMNB/kicad-automation-scripts'

tests/test_plot/test_misc.py

@@ -29,9 +29,12 @@ pytest-3 --log-cli-level debug
 """
 
 import os
+import stat
 import sys
 import shutil
 import logging
+from subprocess import call
+from glob import glob
 # Look for the 'utils' module from where the script is running
 prev_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if prev_dir not in sys.path:
@@ -42,7 +45,7 @@ prev_dir = os.path.dirname(prev_dir)
 if prev_dir not in sys.path:
     sys.path.insert(0, prev_dir)
 from kibot.misc import (EXIT_BAD_ARGS, EXIT_BAD_CONFIG, NO_PCB_FILE, NO_SCH_FILE, EXAMPLE_CFG, WONT_OVERWRITE, CORRUPTED_PCB,
-                        PCBDRAW_ERR)
+                        PCBDRAW_ERR, WRONG_INSTALL)
 
 
 POS_DIR = 'positiondir'
@@ -349,8 +352,10 @@ def test_help_output_plugin_1():
     home = os.environ['HOME']
     os.environ['HOME'] = os.path.join(ctx.get_board_dir(), '..')
     logging.debug('HOME='+os.environ['HOME'])
-    ctx.run(extra=['--help-output', 'test'], no_verbose=True, no_out_dir=True, no_yaml_file=True, no_board_file=True)
-    os.environ['HOME'] = home
+    try:
+        ctx.run(extra=['--help-output', 'test'], no_verbose=True, no_out_dir=True, no_yaml_file=True, no_board_file=True)
+    finally:
+        os.environ['HOME'] = home
     assert ctx.search_out('Test for plugin')
     assert ctx.search_out('Type: .?test.?')
     assert ctx.search_out('nothing')
@@ -363,8 +368,10 @@ def test_help_output_plugin_2():
     home = os.environ['HOME']
     os.environ['HOME'] = os.path.join(ctx.get_board_dir(), '..')
     logging.debug('HOME='+os.environ['HOME'])
-    ctx.run(extra=['--help-output', 'test2'], no_verbose=True, no_out_dir=True, no_yaml_file=True, no_board_file=True)
-    os.environ['HOME'] = home
+    try:
+        ctx.run(extra=['--help-output', 'test2'], no_verbose=True, no_out_dir=True, no_yaml_file=True, no_board_file=True)
+    finally:
+        os.environ['HOME'] = home
     assert ctx.search_out('Test for plugin')
     assert ctx.search_out('Type: .?test2.?')
     assert ctx.search_out('todo')
@@ -456,3 +463,22 @@ def test_pcbdraw_fail():
     ctx.run(PCBDRAW_ERR)
     assert ctx.search_err('Failed to run')
     ctx.clean_up()
+
+
+def test_import_fail():
+    ctx = context.TestContext('test_help_output_plugin_1', '3Rs', 'pre_and_position', POS_DIR)
+    # Create a read only cache entry that we should delete
+    call(['py3compile', 'kibot/out_any_layer.py'])
+    cache_dir = os.path.join('kibot', '__pycache__')
+    cache_file = glob(os.path.join(cache_dir, 'out_any_layer.*'))[0]
+    os.chmod(cache_file, stat.S_IREAD)
+    os.chmod(cache_dir, stat.S_IREAD | stat.S_IEXEC)
+    try:
+        # Run the command
+        ctx.run(WRONG_INSTALL, extra=['--help-list-outputs'], no_out_dir=True, no_yaml_file=True, no_board_file=True)
+    finally:
+        os.chmod(cache_dir, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+        os.remove(cache_file)
+    assert ctx.search_err('Wrong installation')
+    assert ctx.search_err('Unable to import plug-ins')
+    ctx.clean_up()