From c626f864f9771aa679e38f104e238cfceccc1d43 Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Wed, 30 Dec 2020 15:22:00 -0300 Subject: [PATCH] The step output now can download missing 3D models. --- CHANGELOG.md | 1 + README.md | 2 + debian/control | 2 +- docs/samples/generic_plot.kibot.yaml | 4 + kibot/kicad/config.py | 3 + kibot/misc.py | 2 + kibot/out_step.py | 125 +++++++++++++++++++++++++-- setup.py | 2 +- 8 files changed, 133 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f88f70..9737e20f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support to field overwrite according to variant. - Support to generate negative X positions for the bottom layer. - A filter to rotate footprints in the position file (#28). +- The step output now can download missing 3D models. ### Changed - Now position files are naturally sorted (R10 after R9, not after R1) diff --git a/README.md b/README.md index 0e1dcf74..e71e15ad 100644 --- a/README.md +++ b/README.md @@ -1173,6 +1173,8 @@ Next time you need this list just use an alias, like this: * Valid keys: - `dnf_filter`: [string|list(string)=''] Name of the filter to mark components as not fitted. A short-cut to use for simple cases where a variant is an overkill. + - `download`: [boolean=true] downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD. + - `kicad_3d_url`: [string='https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/'] base URL for the KiCad 3D models. - `metric_units`: [boolean=true] use metric units instead of inches. - `min_distance`: [number=-1] the minimum distance between points to treat them as separate ones (-1 is KiCad default: 0.01 mm). - `no_virtual`: [boolean=false] used to exclude 3D models for components with 'virtual' attribute. diff --git a/debian/control b/debian/control index e35d57ef..49cdb53e 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: kibot Architecture: all Multi-Arch: foreign Depends: ${misc:Depends}, ${python3:Depends}, python3-yaml, kicad (>= 5.1.0), python3-wxgtk4.0 -Recommends: kibom.inti-cmnb (>= 1.8.0), kicad-automation-scripts.inti-cmnb (>= 1.1.2), interactivehtmlbom.inti-cmnb, pcbdraw, imagemagick, librsvg2-bin, python3-xlsxwriter +Recommends: kibom.inti-cmnb (>= 1.8.0), interactivehtmlbom.inti-cmnb, pcbdraw, imagemagick, librsvg2-bin, python3-xlsxwriter Description: KiCad Bot KiBot is a program which helps you to automate the generation of KiCad output documents easily, repeatable, and most of all, scriptably. diff --git a/docs/samples/generic_plot.kibot.yaml b/docs/samples/generic_plot.kibot.yaml index 173677d7..978776a9 100644 --- a/docs/samples/generic_plot.kibot.yaml +++ b/docs/samples/generic_plot.kibot.yaml @@ -800,6 +800,10 @@ outputs: # [string|list(string)=''] Name of the filter to mark components as not fitted. # A short-cut to use for simple cases where a variant is an overkill dnf_filter: '' + # [boolean=true] downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD + download: true + # [string='https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/'] base URL for the KiCad 3D models + kicad_3d_url: 'https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/' # [boolean=true] use metric units instead of inches metric_units: true # [number=-1] the minimum distance between points to treat them as separate ones (-1 is KiCad default: 0.01 mm) diff --git a/kibot/kicad/config.py b/kibot/kicad/config.py index 075e40a8..fac847f3 100644 --- a/kibot/kicad/config.py +++ b/kibot/kicad/config.py @@ -288,3 +288,6 @@ class KiConf(object): KiConf.lib_aliases[alias.name] = alias # Load the project's table KiConf.load_lib_aliases(os.path.join(KiConf.dirname, SYM_LIB_TABLE)) + + def expand_env(name): + return os.path.abspath(expand_env(un_quote(name), KiConf.kicad_env)) diff --git a/kibot/misc.py b/kibot/misc.py index 8aa48638..4275b6e7 100644 --- a/kibot/misc.py +++ b/kibot/misc.py @@ -159,6 +159,8 @@ W_MISSCMP = '(W043) ' W_VARSCH = '(W044) ' W_WRONGPASTE = '(W045) ' W_MISFLDNAME = '(W046) ' +W_MISS3D = '(W047) ' +W_FAILDL = '(W048) ' class Rect(object): diff --git a/kibot/out_step.py b/kibot/out_step.py index fb618a11..ae17aeb8 100644 --- a/kibot/out_step.py +++ b/kibot/out_step.py @@ -5,12 +5,16 @@ # Project: KiBot (formerly KiPlot) import re import os +import requests +import tempfile from subprocess import (check_output, STDOUT, CalledProcessError) from tempfile import NamedTemporaryFile +from shutil import rmtree from .error import KiPlotConfigurationError -from .misc import (KICAD2STEP, KICAD2STEP_ERR) +from .misc import KICAD2STEP, KICAD2STEP_ERR, W_MISS3D, W_FAILDL from .gs import (GS) from .out_base import VariantOptions +from .kicad.config import KiConf from .macros import macros, document, output_class # noqa: F401 from . import log @@ -32,6 +36,12 @@ class STEPOptions(VariantOptions): """ the minimum distance between points to treat them as separate ones (-1 is KiCad default: 0.01 mm) """ self.output = GS.def_global_output """ name for the generated STEP file (%i='3D' %x='step') """ + self.download = True + """ downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD """ + self.kicad_3d_url = 'https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/' + """ base URL for the KiCad 3D models """ + # Temporal dir used to store the downloaded files + self._tmp_dir = None super().__init__() @property @@ -44,8 +54,110 @@ class STEPOptions(VariantOptions): raise KiPlotConfigurationError('Origin must be `grid` or `drill` or `X,Y`') self._origin = val + def download_model(self, url, fname): + """ Download the 3D model from the provided URL """ + logger.debug('Downloading `{}`'.format(url)) + r = requests.get(url, allow_redirects=True) + if r.status_code != 200: + logger.warning(W_FAILDL+'Failed to download `{}`'.format(url)) + return None + if self._tmp_dir is None: + self._tmp_dir = tempfile.mkdtemp() + logger.debug('Using `{}` as temporal dir for downloaded files'.format(self._tmp_dir)) + dest = os.path.join(self._tmp_dir, fname) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, 'wb') as f: + f.write(r.content) + return dest + + def undo_3d_models_rename(self): + """ Restores the file name for any renamed 3D module """ + if not self.undo_3d_models: + return + for m in GS.board.GetModules(): + # Get the model references + models = m.Models() + models_l = [] + while not models.empty(): + models_l.append(models.pop()) + # Fix any changed path + for m3d in models_l: + if m3d.m_Filename in self.undo_3d_models: + m3d.m_Filename = self.undo_3d_models[m3d.m_Filename] + # Push the models back + for model in models_l: + models.push_front(model) + + def list_components(self): + """ Check we have the 3D models. + Inform missing models. + Try to download the missing models """ + models_replaced = False + # Load KiCad configuration so we can expand the 3D models path + KiConf.init(GS.pcb_file) + # List of models we already downloaded + downloaded = set() + self.undo_3d_models = {} + # Look for all the footprints + for m in GS.board.GetModules(): + ref = m.GetReference() + # Extract the models (the iterator returns copies) + models = m.Models() + models_l = [] + while not models.empty(): + models_l.append(models.pop()) + # Look for all the 3D models for this footprint + for m3d in models_l: + full_name = KiConf.expand_env(m3d.m_Filename) + if not os.path.isfile(full_name): + # Missing 3D model + if full_name not in downloaded: + logger.warning(W_MISS3D+'Missing 3D model for {}: `{}`'.format(ref, full_name)) + if self.download and m3d.m_Filename.startswith('${KISYS3DMOD}/'): + # This is a model from KiCad, try to download it + fname = m3d.m_Filename[14:] + replace = None + if full_name in downloaded: + # Already downloaded + replace = os.path.join(self._tmp_dir, fname) + else: + # Download the model + url = self.kicad_3d_url+fname + replace = self.download_model(url, fname) + if replace: + # Successfully downloaded + downloaded.add(full_name) + self.undo_3d_models[replace] = m3d.m_Filename + # If this is a .wrl also download the .step + if url.endswith('.wrl'): + url = url[:-4]+'.step' + fname = fname[:-4]+'.step' + self.download_model(url, fname) + if replace: + m3d.m_Filename = replace + models_replaced = True + # Push the models back + for model in models_l: + models.push_front(model) + return models_replaced + + def save_board(self, dir): + """ Save the PCB to a temporal file """ + with NamedTemporaryFile(mode='w', suffix='.kicad_pcb', delete=False, dir=dir) as f: + fname = f.name + logger.debug('Storing modified PCB to `{}`'.format(fname)) + GS.board.Save(fname) + return fname + def filter_components(self, dir): if not self._comps: + if self.list_components(): + # Some missing components found and we downloaded them + # Save the fixed board + ret = self.save_board(dir) + # Undo the changes + self.undo_3d_models_rename() + return ret return GS.pcb_file comps_hash = self.get_refs_hash() # Remove the 3D models for not fitted components @@ -59,11 +171,9 @@ class STEPOptions(VariantOptions): while not models.empty(): rem_m_models.append(models.pop()) rem_models.append(rem_m_models) - # Save the PCB to a temporal file - with NamedTemporaryFile(mode='w', suffix='.kicad_pcb', delete=False, dir=dir) as f: - fname = f.name - logger.debug('Storing filtered PCB to `{}`'.format(fname)) - GS.board.Save(fname) + self.list_components() + fname = self.save_board(dir) + self.undo_3d_models_rename() # Undo the removing for m in GS.board.GetModules(): ref = m.GetReference() @@ -115,6 +225,9 @@ class STEPOptions(VariantOptions): # Remove the temporal PCB if board_name != GS.pcb_file: os.remove(board_name) + # Remove the downloaded 3D models + if self._tmp_dir: + rmtree(self._tmp_dir) logger.debug('Output from command:\n'+cmd_output.decode()) diff --git a/setup.py b/setup.py index 908e6374..d461a09a 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup(name='kibot', # Packages are marked using __init__.py packages=find_packages(), scripts=['src/kibot', 'src/kiplot'], - install_requires=['kiauto', 'pyyaml', 'xlsxwriter', 'colorama'], + install_requires=['kiauto', 'pyyaml', 'xlsxwriter', 'colorama', 'requests'], classifiers=['Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers',