The step output now can download missing 3D models.

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support to field overwrite according to variant.
 - Support to generate negative X positions for the bottom layer.
 - A filter to rotate footprints in the position file (#28).
+- The step output now can download missing 3D models.
 
 ### Changed
 - Now position files are naturally sorted (R10 after R9, not after R1)

README.md

@@ -1173,6 +1173,8 @@ Next time you need this list just use an alias, like this:
       * Valid keys:
         - `dnf_filter`: [string|list(string)=''] Name of the filter to mark components as not fitted.
                         A short-cut to use for simple cases where a variant is an overkill.
+        - `download`: [boolean=true] downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD.
+        - `kicad_3d_url`: [string='https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/'] base URL for the KiCad 3D models.
         - `metric_units`: [boolean=true] use metric units instead of inches.
         - `min_distance`: [number=-1] the minimum distance between points to treat them as separate ones (-1 is KiCad default: 0.01 mm).
         - `no_virtual`: [boolean=false] used to exclude 3D models for components with 'virtual' attribute.

debian/control

@@ -11,7 +11,7 @@ Package: kibot
 Architecture: all
 Multi-Arch: foreign
 Depends: ${misc:Depends}, ${python3:Depends}, python3-yaml, kicad (>= 5.1.0), python3-wxgtk4.0
-Recommends: kibom.inti-cmnb (>= 1.8.0), kicad-automation-scripts.inti-cmnb (>= 1.1.2), interactivehtmlbom.inti-cmnb, pcbdraw, imagemagick, librsvg2-bin, python3-xlsxwriter
+Recommends: kibom.inti-cmnb (>= 1.8.0), interactivehtmlbom.inti-cmnb, pcbdraw, imagemagick, librsvg2-bin, python3-xlsxwriter
 Description: KiCad Bot
  KiBot is a program which helps you to automate the generation of KiCad
  output documents easily, repeatable, and most of all, scriptably.

docs/samples/generic_plot.kibot.yaml

@@ -800,6 +800,10 @@ outputs:
       # [string|list(string)=''] Name of the filter to mark components as not fitted.
       # A short-cut to use for simple cases where a variant is an overkill
       dnf_filter: ''
+      # [boolean=true] downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD
+      download: true
+      # [string='https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/'] base URL for the KiCad 3D models
+      kicad_3d_url: 'https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/'
       # [boolean=true] use metric units instead of inches
       metric_units: true
       # [number=-1] the minimum distance between points to treat them as separate ones (-1 is KiCad default: 0.01 mm)

kibot/kicad/config.py

@@ -288,3 +288,6 @@ class KiConf(object):
                     KiConf.lib_aliases[alias.name] = alias
         # Load the project's table
         KiConf.load_lib_aliases(os.path.join(KiConf.dirname, SYM_LIB_TABLE))
+
+    def expand_env(name):
+        return os.path.abspath(expand_env(un_quote(name), KiConf.kicad_env))

kibot/misc.py

@@ -159,6 +159,8 @@ W_MISSCMP = '(W043) '
 W_VARSCH = '(W044) '
 W_WRONGPASTE = '(W045) '
 W_MISFLDNAME = '(W046) '
+W_MISS3D = '(W047) '
+W_FAILDL = '(W048) '
 
 
 class Rect(object):

kibot/out_step.py

@@ -5,12 +5,16 @@
 # Project: KiBot (formerly KiPlot)
 import re
 import os
+import requests
+import tempfile
 from subprocess import (check_output, STDOUT, CalledProcessError)
 from tempfile import NamedTemporaryFile
+from shutil import rmtree
 from .error import KiPlotConfigurationError
-from .misc import (KICAD2STEP, KICAD2STEP_ERR)
+from .misc import KICAD2STEP, KICAD2STEP_ERR, W_MISS3D, W_FAILDL
 from .gs import (GS)
 from .out_base import VariantOptions
+from .kicad.config import KiConf
 from .macros import macros, document, output_class  # noqa: F401
 from . import log
 
@@ -32,6 +36,12 @@ class STEPOptions(VariantOptions):
             """ the minimum distance between points to treat them as separate ones (-1 is KiCad default: 0.01 mm) """
             self.output = GS.def_global_output
             """ name for the generated STEP file (%i='3D' %x='step') """
+            self.download = True
+            """ downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD """
+            self.kicad_3d_url = 'https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/'
+            """ base URL for the KiCad 3D models """
+        # Temporal dir used to store the downloaded files
+        self._tmp_dir = None
         super().__init__()
 
     @property
@@ -44,8 +54,110 @@ class STEPOptions(VariantOptions):
             raise KiPlotConfigurationError('Origin must be `grid` or `drill` or `X,Y`')
         self._origin = val
 
+    def download_model(self, url, fname):
+        """ Download the 3D model from the provided URL """
+        logger.debug('Downloading `{}`'.format(url))
+        r = requests.get(url, allow_redirects=True)
+        if r.status_code != 200:
+            logger.warning(W_FAILDL+'Failed to download `{}`'.format(url))
+            return None
+        if self._tmp_dir is None:
+            self._tmp_dir = tempfile.mkdtemp()
+            logger.debug('Using `{}` as temporal dir for downloaded files'.format(self._tmp_dir))
+        dest = os.path.join(self._tmp_dir, fname)
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        with open(dest, 'wb') as f:
+            f.write(r.content)
+        return dest
+
+    def undo_3d_models_rename(self):
+        """ Restores the file name for any renamed 3D module """
+        if not self.undo_3d_models:
+            return
+        for m in GS.board.GetModules():
+            # Get the model references
+            models = m.Models()
+            models_l = []
+            while not models.empty():
+                models_l.append(models.pop())
+            # Fix any changed path
+            for m3d in models_l:
+                if m3d.m_Filename in self.undo_3d_models:
+                    m3d.m_Filename = self.undo_3d_models[m3d.m_Filename]
+            # Push the models back
+            for model in models_l:
+                models.push_front(model)
+
+    def list_components(self):
+        """ Check we have the 3D models.
+            Inform missing models.
+            Try to download the missing models """
+        models_replaced = False
+        # Load KiCad configuration so we can expand the 3D models path
+        KiConf.init(GS.pcb_file)
+        # List of models we already downloaded
+        downloaded = set()
+        self.undo_3d_models = {}
+        # Look for all the footprints
+        for m in GS.board.GetModules():
+            ref = m.GetReference()
+            # Extract the models (the iterator returns copies)
+            models = m.Models()
+            models_l = []
+            while not models.empty():
+                models_l.append(models.pop())
+            # Look for all the 3D models for this footprint
+            for m3d in models_l:
+                full_name = KiConf.expand_env(m3d.m_Filename)
+                if not os.path.isfile(full_name):
+                    # Missing 3D model
+                    if full_name not in downloaded:
+                        logger.warning(W_MISS3D+'Missing 3D model for {}: `{}`'.format(ref, full_name))
+                    if self.download and m3d.m_Filename.startswith('${KISYS3DMOD}/'):
+                        # This is a model from KiCad, try to download it
+                        fname = m3d.m_Filename[14:]
+                        replace = None
+                        if full_name in downloaded:
+                            # Already downloaded
+                            replace = os.path.join(self._tmp_dir, fname)
+                        else:
+                            # Download the model
+                            url = self.kicad_3d_url+fname
+                            replace = self.download_model(url, fname)
+                            if replace:
+                                # Successfully downloaded
+                                downloaded.add(full_name)
+                                self.undo_3d_models[replace] = m3d.m_Filename
+                                # If this is a .wrl also download the .step
+                                if url.endswith('.wrl'):
+                                    url = url[:-4]+'.step'
+                                    fname = fname[:-4]+'.step'
+                                    self.download_model(url, fname)
+                        if replace:
+                            m3d.m_Filename = replace
+                            models_replaced = True
+            # Push the models back
+            for model in models_l:
+                models.push_front(model)
+        return models_replaced
+
+    def save_board(self, dir):
+        """ Save the PCB to a temporal file """
+        with NamedTemporaryFile(mode='w', suffix='.kicad_pcb', delete=False, dir=dir) as f:
+            fname = f.name
+        logger.debug('Storing modified PCB to `{}`'.format(fname))
+        GS.board.Save(fname)
+        return fname
+
     def filter_components(self, dir):
         if not self._comps:
+            if self.list_components():
+                # Some missing components found and we downloaded them
+                # Save the fixed board
+                ret = self.save_board(dir)
+                # Undo the changes
+                self.undo_3d_models_rename()
+                return ret
             return GS.pcb_file
         comps_hash = self.get_refs_hash()
         # Remove the 3D models for not fitted components
@@ -59,11 +171,9 @@ class STEPOptions(VariantOptions):
                 while not models.empty():
                     rem_m_models.append(models.pop())
                 rem_models.append(rem_m_models)
-        # Save the PCB to a temporal file
-        with NamedTemporaryFile(mode='w', suffix='.kicad_pcb', delete=False, dir=dir) as f:
-            fname = f.name
-        logger.debug('Storing filtered PCB to `{}`'.format(fname))
-        GS.board.Save(fname)
+        self.list_components()
+        fname = self.save_board(dir)
+        self.undo_3d_models_rename()
         # Undo the removing
         for m in GS.board.GetModules():
             ref = m.GetReference()
@@ -115,6 +225,9 @@ class STEPOptions(VariantOptions):
             # Remove the temporal PCB
             if board_name != GS.pcb_file:
                 os.remove(board_name)
+            # Remove the downloaded 3D models
+            if self._tmp_dir:
+                rmtree(self._tmp_dir)
         logger.debug('Output from command:\n'+cmd_output.decode())
 
 

setup.py

@@ -20,7 +20,7 @@ setup(name='kibot',
       # Packages are marked using __init__.py
       packages=find_packages(),
       scripts=['src/kibot', 'src/kiplot'],
-      install_requires=['kiauto', 'pyyaml', 'xlsxwriter', 'colorama'],
+      install_requires=['kiauto', 'pyyaml', 'xlsxwriter', 'colorama', 'requests'],
       classifiers=['Development Status :: 5 - Production/Stable',
                    'Environment :: Console',
                    'Intended Audience :: Developers',