From 96822923b4de7b5e72b27f6a09741f1a666c701c Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Fri, 27 Jan 2023 12:12:16 -0300 Subject: [PATCH] [3D][Added] A mechanism to cache downloaded 3D models --- CHANGELOG.md | 1 + README.md | 32 ++++++++++++++++++++++------ docs/README.in | 2 +- docs/samples/generic_plot.kibot.yaml | 30 +++++++++++++++++++++----- kibot/out_base_3d.py | 29 ++++++++++++++++++------- 5 files changed, 74 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 579df4c7..32b4466b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for `groups` of `outputs` - Internal templates import - Better support for wrong pre-flight options (#360) + - A mechanism to cache downloaded 3D models - New outputs: - `vrml` export the 3D model in Virtual Reality Modeling Language (#349) - `ps_sch_print`, `dxf_sch_print` and `hpgl_sch_print` variants of diff --git a/README.md b/README.md index b9eede0b..e8953d82 100644 --- a/README.md +++ b/README.md @@ -1581,7 +1581,11 @@ Notes: You can also specify the name of the output that generates the PCB3D file. See the `PCB2Blender_2_1` and `PCB2Blender_2_1_haschtl` templates. * Valid keys: - - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD. + - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. + Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + They are downloaded to a temporal directory and discarded. + If you want to cache the downloaded files specify a directory using the + KIBOT_3D_MODELS environment variable. - **`no_virtual`**: [boolean=false] Used to exclude 3D models for components with 'virtual' attribute. - **`show_components`**: [list(string)|string=all] [none,all] List of components to draw, can be also a string for `none` or `all`. Unlike the `pcbdraw` output, the default is `all`. @@ -1994,7 +1998,11 @@ Notes: Avoid using `_` as first character. These names are reserved for KiBot. - **`options`**: [dict] Options for the `copy_files` output. * Valid keys: - - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD. + - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. + Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + They are downloaded to a temporal directory and discarded. + If you want to cache the downloaded files specify a directory using the + KIBOT_3D_MODELS environment variable. - **`files`**: [list(dict)] Which files will be included. * Valid keys: - **`source`**: [string='*'] File names to add, wildcards allowed. Use ** for recursive match. @@ -4151,7 +4159,11 @@ Notes: Avoid using `_` as first character. These names are reserved for KiBot. - **`options`**: [dict] Options for the `render_3d` output. * Valid keys: - - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD. + - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. + Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + They are downloaded to a temporal directory and discarded. + If you want to cache the downloaded files specify a directory using the + KIBOT_3D_MODELS environment variable. - **`move_x`**: [number=0] Steps to move in the X axis, positive is to the right. Just like pressing the right arrow in the 3D viewer. - **`move_y`**: [number=0] Steps to move in the Y axis, positive is up. @@ -4423,7 +4435,11 @@ Notes: Avoid using `_` as first character. These names are reserved for KiBot. - **`options`**: [dict] Options for the `step` output. * Valid keys: - - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD. + - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. + Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + They are downloaded to a temporal directory and discarded. + If you want to cache the downloaded files specify a directory using the + KIBOT_3D_MODELS environment variable. - **`no_virtual`**: [boolean=false] Used to exclude 3D models for components with 'virtual' attribute. - **`origin`**: [string='grid'] Determines the coordinates origin. Using grid the coordinates are the same as you have in the design sheet. The drill option uses the auxiliary reference defined by the user. @@ -4644,7 +4660,11 @@ Notes: Avoid using `_` as first character. These names are reserved for KiBot. - **`options`**: [dict] Options for the `vrml` output. * Valid keys: - - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD. + - **`download`**: [boolean=true] Downloads missing 3D models from KiCad git. + Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + They are downloaded to a temporal directory and discarded. + If you want to cache the downloaded files specify a directory using the + KIBOT_3D_MODELS environment variable. - **`no_virtual`**: [boolean=false] Used to exclude 3D models for components with 'virtual' attribute. - **`output`**: [string='%f-%i%I%v.%x'] Filename for the output (%i=vrml, %x=wrl). Affected by global options. - **`show_components`**: [list(string)|string=all] [none,all] List of components to draw, can be also a string for `none` or `all`. @@ -5860,7 +5880,7 @@ If you have any suggestion don't hesitate in contacting me to add them. ### 3D models and docker images The default KiCad 3D models aren't included in the KiBot docker images. -This is because the 3D models currently needs around 5 GB and the current docker images are between 1 and 1.6 GB. +This is because the 3D models currently needs around 5 GB and the current docker images are between 1 and 2.8 GB. So adding them means a huge increase in size. This is not a big problem because KiBot will download any missing 3D model from KiCad's repo. diff --git a/docs/README.in b/docs/README.in index c0017679..82f9faa4 100644 --- a/docs/README.in +++ b/docs/README.in @@ -2021,7 +2021,7 @@ If you have any suggestion don't hesitate in contacting me to add them. ### 3D models and docker images The default KiCad 3D models aren't included in the KiBot docker images. -This is because the 3D models currently needs around 5 GB and the current docker images are between 1 and 1.6 GB. +This is because the 3D models currently needs around 5 GB and the current docker images are between 1 and 2.8 GB. So adding them means a huge increase in size. This is not a big problem because KiBot will download any missing 3D model from KiCad's repo. diff --git a/docs/samples/generic_plot.kibot.yaml b/docs/samples/generic_plot.kibot.yaml index 20ec18ae..b7f491d6 100644 --- a/docs/samples/generic_plot.kibot.yaml +++ b/docs/samples/generic_plot.kibot.yaml @@ -152,7 +152,11 @@ outputs: # [string|list(string)='_none'] Name of the filter to mark components as not fitted. # A short-cut to use for simple cases where a variant is an overkill dnf_filter: '_none' - # [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD + # [boolean=true] Downloads missing 3D models from KiCad git. + # Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + # They are downloaded to a temporal directory and discarded. + # If you want to cache the downloaded files specify a directory using the + # KIBOT_3D_MODELS environment variable download: true # [list(string)=[]] List of components to highlight highlight: [] @@ -622,7 +626,11 @@ outputs: # [string|list(string)='_none'] Name of the filter to mark components as not fitted. # A short-cut to use for simple cases where a variant is an overkill dnf_filter: '_none' - # [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD + # [boolean=true] Downloads missing 3D models from KiCad git. + # Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + # They are downloaded to a temporal directory and discarded. + # If you want to cache the downloaded files specify a directory using the + # KIBOT_3D_MODELS environment variable download: true # [list(dict)] Which files will be included files: @@ -2841,7 +2849,11 @@ outputs: # [string|list(string)='_none'] Name of the filter to mark components as not fitted. # A short-cut to use for simple cases where a variant is an overkill dnf_filter: '_none' - # [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD + # [boolean=true] Downloads missing 3D models from KiCad git. + # Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + # They are downloaded to a temporal directory and discarded. + # If you want to cache the downloaded files specify a directory using the + # KIBOT_3D_MODELS environment variable download: true # [number=720] Image height (aprox.) height: 720 @@ -3093,7 +3105,11 @@ outputs: # [string|list(string)='_none'] Name of the filter to mark components as not fitted. # A short-cut to use for simple cases where a variant is an overkill dnf_filter: '_none' - # [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD + # [boolean=true] Downloads missing 3D models from KiCad git. + # Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + # They are downloaded to a temporal directory and discarded. + # If you want to cache the downloaded files specify a directory using the + # KIBOT_3D_MODELS environment variable download: true # [string='https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/'] Base URL for the KiCad 3D models kicad_3d_url: 'https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/' @@ -3301,7 +3317,11 @@ outputs: # [string|list(string)='_none'] Name of the filter to mark components as not fitted. # A short-cut to use for simple cases where a variant is an overkill dnf_filter: '_none' - # [boolean=true] Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD + # [boolean=true] Downloads missing 3D models from KiCad git. + # Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + # They are downloaded to a temporal directory and discarded. + # If you want to cache the downloaded files specify a directory using the + # KIBOT_3D_MODELS environment variable download: true # [list(string)=[]] List of components to highlight highlight: [] diff --git a/kibot/out_base_3d.py b/kibot/out_base_3d.py index ded0208f..3abab4fa 100644 --- a/kibot/out_base_3d.py +++ b/kibot/out_base_3d.py @@ -62,7 +62,11 @@ class Base3DOptions(VariantOptions): self.no_virtual = False """ *Used to exclude 3D models for components with 'virtual' attribute """ self.download = True - """ *Downloads missing 3D models from KiCad git. Only applies to models in KISYS3DMOD """ + """ *Downloads missing 3D models from KiCad git. + Only applies to models in KISYS3DMOD and KICAD6_3DMODEL_DIR. + They are downloaded to a temporal directory and discarded. + If you want to cache the downloaded files specify a directory using the + KIBOT_3D_MODELS environment variable """ self.kicad_3d_url = 'https://gitlab.com/kicad/libraries/kicad-packages3D/-/raw/master/' """ Base URL for the KiCad 3D models """ # Temporal dir used to store the downloaded files @@ -78,6 +82,22 @@ class Base3DOptions(VariantOptions): def download_model(self, url, fname, rel_dirs): """ Download the 3D model from the provided URL """ + # Find a place to store the downloaded model + if self._tmp_dir is None: + self._tmp_dir = os.environ.get('KIBOT_3D_MODELS') + if self._tmp_dir is None: + self._tmp_dir = tempfile.mkdtemp() + self._files_to_remove.append(self._tmp_dir) + else: + self._tmp_dir = os.path.abspath(self._tmp_dir) + rel_dirs.append(self._tmp_dir) + logger.debug('Using `{}` as temporal dir for downloaded files'.format(self._tmp_dir)) + dest = os.path.join(self._tmp_dir, fname) + os.makedirs(os.path.dirname(dest), exist_ok=True) + # Is already there? + if os.path.isfile(dest): + logger.debug('Using cached model `{}`'.format(dest)) + return dest logger.debug('Downloading `{}`'.format(url)) failed = False try: @@ -87,13 +107,6 @@ class Base3DOptions(VariantOptions): if failed or r.status_code != 200: logger.warning(W_FAILDL+'Failed to download `{}`'.format(url)) return None - if self._tmp_dir is None: - self._tmp_dir = tempfile.mkdtemp() - self._files_to_remove.append(self._tmp_dir) - rel_dirs.append(self._tmp_dir) - logger.debug('Using `{}` as temporal dir for downloaded files'.format(self._tmp_dir)) - dest = os.path.join(self._tmp_dir, fname) - os.makedirs(os.path.dirname(dest), exist_ok=True) with open(dest, 'wb') as f: f.write(r.content) return dest