Added datasheets downloader

Closes #119
2021-12-29 15:20:54 -03:00 · 2021-12-29 15:20:54 -03:00 · 6d939bbdbe
parent 167bdcd4e9
commit 6d939bbdbe
9 changed files with 339 additions and 1 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -52,6 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  So you can create a compressed file containing the source schematic and
  PCB files. (#93)
 - Support for new KiCost options `split_extra_fields` and `board_qty`. (#120)
+- Datasheet downloader. (#119)

 ### Changed
 - Internal BoM: now components with different Tolerance, Voltage, Current
--- a/README.md
+++ b/README.md
@ -915,6 +915,32 @@ Next time you need this list just use an alias, like this:
    - `output_id`: [string=''] Text to use for the %I expansion content. To differentiate variations of this output.
    - `run_by_default`: [boolean=true] When enabled this output will be created when no specific outputs are requested.

+* Datasheets downloader
+  * Type: `download_datasheets`
+  * Description: Downloads the datasheets for the project
+  * Valid keys:
+    - `comment`: [string=''] A comment for documentation purposes.
+    - `dir`: [string='./'] Output directory for the generated files. If it starts with `+` the rest is concatenated to the default dir.
+    - `disable_run_by_default`: [string|boolean] Use it to disable the `run_by_default` status of other output.
+                                Useful when this output extends another and you don't want to generate the original.
+                                Use the boolean true value to disable the output you are extending.
+    - `extends`: [string=''] Copy the `options` section from the indicated output.
+    - `name`: [string=''] Used to identify this particular output definition.
+    - `options`: [dict] Options for the `download_datasheets` output.
+      * Valid keys:
+        - `dnf`: [boolean=false] Include the DNF components.
+        - `dnf_filter`: [string|list(string)='_none'] Name of the filter to mark components as not fitted.
+                        A short-cut to use for simple cases where a variant is an overkill.
+        - `field`: [string='Datasheet'] Name of the field containing the URL.
+        - `link_repeated`: [boolean=true] Instead of download things we already downloaded use symlinks.
+        - `output`: [string='${VALUE}.pdf'] Name used for the downloaded datasheet.
+                    ${FIELD} will be replaced by the FIELD content.
+        - `repeated`: [boolean=false] Download URLs that we already downloaded.
+                      It only makes sense if the `output` field makes their output different.
+        - `variant`: [string=''] Board variant to apply.
+    - `output_id`: [string=''] Text to use for the %I expansion content. To differentiate variations of this output.
+    - `run_by_default`: [boolean=true] When enabled this output will be created when no specific outputs are requested.
+
 * DXF (Drawing Exchange Format)
  * Type: `dxf`
  * Description: Exports the PCB to 2D mechanical EDA tools (like AutoCAD).
--- a/docs/samples/generic_plot.kibot.yaml
+++ b/docs/samples/generic_plot.kibot.yaml
@ -301,6 +301,30 @@ outputs:
      # [string='%f-%i%v.%x'] Name for the generated archive (%i=name of the output %x=according to format). Affected by global options
      output: '%f-%i%v.%x'

+  # Datasheets downloader:
+  - name: 'download_datasheets_example'
+    comment: 'Downloads the datasheets for the project'
+    type: 'download_datasheets'
+    dir: 'Example/download_datasheets_dir'
+    options:
+      # [boolean=false] Include the DNF components
+      dnf: false
+      # [string|list(string)='_none'] Name of the filter to mark components as not fitted.
+      # A short-cut to use for simple cases where a variant is an overkill
+      dnf_filter: '_none'
+      # [string='Datasheet'] Name of the field containing the URL
+      field: 'Datasheet'
+      # [boolean=true] Instead of download things we already downloaded use symlinks
+      link_repeated: true
+      # [string='${VALUE}.pdf'] Name used for the downloaded datasheet.
+      # ${FIELD} will be replaced by the FIELD content
+      output: '${VALUE}.pdf'
+      # [boolean=false] Download URLs that we already downloaded.
+      # It only makes sense if the `output` field makes their output different
+      repeated: false
+      # [string=''] Board variant to apply
+      variant: ''
+
  # DXF (Drawing Exchange Format):
  # This output is what you get from the File/Plot menu in pcbnew.
  - name: 'dxf_example'
--- a/kibot/misc.py
+++ b/kibot/misc.py
@ -215,6 +215,7 @@ W_EMPTREP = '(W073) '
 W_BADCHARS = '(W074) '
 W_DATEFORMAT = '(W075) '
 W_UNKFLD = '(W076) '
+W_ALRDOWN = '(W077) '


 class Rect(object):
--- a/kibot/out_download_datasheets.py
+++ b/kibot/out_download_datasheets.py
@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 Salvador E. Tropea
+# Copyright (c) 2021 Instituto Nacional de Tecnología Industrial
+# License: GPL-3.0
+# Project: KiBot (formerly KiPlot)
+import os
+import re
+import requests
+from .out_base import VariantOptions
+from .fil_base import DummyFilter
+from .error import KiPlotConfigurationError
+from .misc import W_UNKFLD, W_ALRDOWN, W_FAILDL
+from .gs import GS
+from .macros import macros, document, output_class  # noqa: F401
+from . import log
+logger = log.get_logger(__name__)
+USER_AGENT = 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'
+
+
+class Download_Datasheets_Options(VariantOptions):
+    _vars_regex = re.compile(r'\$\{([^\}]+)\}')
+
+    def __init__(self):
+        super().__init__()
+        with document:
+            self.field = 'Datasheet'
+            """ Name of the field containing the URL """
+            self.output = '${VALUE}.pdf'
+            """ Name used for the downloaded datasheet.
+                ${FIELD} will be replaced by the FIELD content """
+            self.dnf = False
+            """ Include the DNF components """
+            self.repeated = False
+            """ Download URLs that we already downloaded.
+                It only makes sense if the `output` field makes their output different """
+            self.link_repeated = True
+            """ Instead of download things we already downloaded use symlinks """
+        # Used to collect the targets
+        self._dry = False
+
+    def config(self, parent):
+        super().config(parent)
+        if not self.field:
+            raise KiPlotConfigurationError("Empty `field` ({})".format(str(self._tree)))
+        if not self.output:
+            raise KiPlotConfigurationError("Empty `output` ({})".format(str(self._tree)))
+        self.field = self.field.lower()
+
+    def download(self, c, ds, dir, name, known):
+        dest = os.path.join(dir, name)
+        logger.debug('To download: {} -> {}'.format(ds, dest))
+        if name in self._downloaded:
+            logger.warning(W_ALRDOWN+'Datasheet `{}` already downloaded'.format(name))
+            return None
+        elif known is not None and self.link_repeated:
+            # We already downloaded this URL, but stored it with a different name
+            if not self._dry:
+                os.symlink(known, dest)
+            self._created.append(os.path.relpath(dest))
+        elif not os.path.isfile(dest):
+            # Download
+            if not self._dry:
+                r = requests.get(ds, allow_redirects=True, headers={'User-Agent': USER_AGENT})
+                if r.status_code != 200:
+                    logger.warning(W_FAILDL+'Failed to download `{}`'.format(ds))
+                    return None
+                with open(dest, 'wb') as f:
+                    f.write(r.content)
+            self._downloaded.add(name)
+            self._created.append(os.path.relpath(dest))
+        elif self._dry:
+            self._created.append(os.path.relpath(dest))
+        return name
+
+    def out_name(self, c):
+        """ Compute the name of the output file.
+            Replaces ${FIELD} and %X. """
+        out = ''
+        last = 0
+        pattern = self.output
+        pattern_l = len(pattern)
+        for match in Download_Datasheets_Options._vars_regex.finditer(pattern):
+            fname = match.group(1).lower()
+            value = c.get_field_value(fname)
+            if value is None:
+                value = 'Unknown'
+                logger.warning(W_UNKFLD+"In datasheets download output file name:"
+                               " Field `{}` not defined for {}, using `Unknown`".format(fname, c.ref))
+            if match.start():
+                out += pattern[last:match.start()]
+            out += value
+            last = match.end()
+        if last < pattern_l:
+            out += pattern[last:pattern_l]
+        out = self.expand_filename_sch(out)
+        return out.replace('/', '_')
+
+    def run(self, output_dir):
+        if not self.dnf_filter and not self.variant:
+            # Add a dummy filter to force the creation of a components list
+            self.dnf_filter = DummyFilter()
+        super().run(output_dir)
+        self._urls = {}
+        self._downloaded = set()
+        self._created = []
+        field_used = False
+        for c in self._comps:
+            ds = c.get_field_value(self.field)
+            if ds is not None:
+                field_used = True
+            if not c.included or (not c.fitted and not self.dnf):
+                continue
+            if ds:
+                known = self._urls.get(ds, None)
+                if known is None or self.repeated:
+                    name = self.out_name(c)
+                    name = self.download(c, ds, output_dir, name, known)
+                    if known is None:
+                        self._urls[ds] = name
+                else:
+                    logger.debug('Already downloaded: '+ds)
+        if not field_used:
+            known_fields = GS.sch.get_field_names({})
+            if self.field not in known_fields:
+                logger.warning(W_UNKFLD+"The field used for datasheets ({}) doesn't seem to be used".format(self.field))
+        else:
+            logger.debug('Unique URLs: '+str(len(self._urls)))
+            logger.debug('Downloaded: '+str(len(self._downloaded)))
+            logger.debug('Created: '+str(len(self._created)))
+
+    def get_targets(self, out_dir):
+        # Do a dry run to collect the output names
+        self._dry = True
+        self.run(out_dir)
+        self._dry = False
+        return self._created
+
+
+@output_class
+class Download_Datasheets(BaseOutput):  # noqa: F821
+    """ Datasheets downloader
+        Downloads the datasheets for the project """
+    def __init__(self):
+        super().__init__()
+        with document:
+            self.options = Download_Datasheets_Options
+            """ [dict] Options for the `download_datasheets` output """
+        self._sch_related = True
+
+    def run(self, output_dir):
+        # No output member, just a dir
+        self.options.run(output_dir)
--- a/tests/board_samples/kicad_5/kibom-variant_2ds.sch
+++ b/tests/board_samples/kicad_5/kibom-variant_2ds.sch
@ -0,0 +1,72 @@
+EESchema Schematic File Version 4
+EELAYER 30 0
+EELAYER END
+$Descr A4 11693 8268
+encoding utf-8
+Sheet 1 1
+Title "KiBom Test Schematic"
+Date "2020-03-12"
+Rev "A"
+Comp "https://github.com/SchrodingersGat/KiBom"
+Comment1 ""
+Comment2 ""
+Comment3 ""
+Comment4 ""
+$EndDescr
+Text Notes 500  600  0    79   ~ 0
+This schematic serves as a test-file for the KiBot export script.\n
+Text Notes 5950 2600 0    118  ~ 0
+The test tests the following \nvariants matrix:\n        production   test   default\nC1                     X\nC2                     X\nR1          X          X       X\nR2          X                  X\n
+$Comp
+L Device:C C1
+U 1 1 5F43BEC2
+P 1000 1700
+F 0 "C1" H 1115 1746 50  0000 L CNN
+F 1 "1nF" H 1115 1655 50  0000 L CNN
+F 2 "" H 1038 1550 50  0001 C CNN
+F 3 "http://localhost:8000/c.pdf" H 1000 1700 50  0001 C CNN
+F 4 "-production,+test" H 1000 1700 50  0001 C CNN "Config"
+F 5 "C0805C102J4GAC7800" H 1000 1700 50  0001 C CNN "manf#"
+	1    1000 1700
+	1    0    0    -1  
+$EndComp
+$Comp
+L Device:C C2
+U 1 1 5F43CE1C
+P 1450 1700
+F 0 "C2" H 1565 1746 50  0000 L CNN
+F 1 "1000 pF" H 1565 1655 50  0000 L CNN
+F 2 "" H 1488 1550 50  0001 C CNN
+F 3 "http://localhost:8000/c.pdf" H 1450 1700 50  0001 C CNN
+F 4 "+test" H 1450 1700 50  0001 C CNN "Config"
+F 5 "C0805C102J4GAC7800" H 1000 1700 50  0001 C CNN "manf#"
+	1    1450 1700
+	1    0    0    -1  
+$EndComp
+$Comp
+L Device:R R1
+U 1 1 5F43D144
+P 2100 1700
+F 0 "R1" H 2170 1746 50  0000 L CNN
+F 1 "1k" H 2170 1655 50  0000 L CNN
+F 2 "" V 2030 1700 50  0001 C CNN
+F 3 "http://localhost:8000/r.pdf" H 2100 1700 50  0001 C CNN
+F 4 "3k3" H 2100 1700 50  0001 C CNN "test:Value"
+F 5 "CR0805-JW-102ELF" H 1000 1700 50  0001 C CNN "manf#"
+	1    2100 1700
+	1    0    0    -1  
+$EndComp
+$Comp
+L Device:R R2
+U 1 1 5F43D4BB
+P 2500 1700
+F 0 "R2" H 2570 1746 50  0000 L CNN
+F 1 "1000" H 2570 1655 50  0000 L CNN
+F 2 "" V 2430 1700 50  0001 C CNN
+F 3 "http://localhost:8000/r.pdf" H 2500 1700 50  0001 C CNN
+F 4 "-test" H 2500 1700 50  0001 C CNN "Config"
+F 5 "CR0805-JW-102ELF" H 1000 1700 50  0001 C CNN "manf#"
+	1    2500 1700
+	1    0    0    -1  
+$EndComp
+$EndSCHEMATC
--- a/tests/test_plot/test_misc.py
+++ b/tests/test_plot/test_misc.py
@ -894,3 +894,17 @@ def test_date_format_2(test_dir):
    ctx.expect_out_file(POS_DIR+'/bom_13_07_2020.csv')
    assert ctx.search_err('Trying to reformat SCH time, but not in ISO format')
    ctx.clean_up()
+
+
+def test_download_datasheets_1(test_dir):
+    prj = 'kibom-variant_2ds'
+    ctx = context.TestContextSCH(test_dir, 'test_download_datasheets_1', prj, 'download_datasheets_1', '')
+    # We use a fake server to avoid needing good URLs and reliable internet connection
+    ctx.run(kicost=True)
+    ctx.expect_out_file('DS/C0805C102J4GAC7800.pdf')
+    ctx.expect_out_file('DS/CR0805-JW-102ELF.pdf')
+    ctx.expect_out_file('DS_production/CR0805-JW-102ELF.pdf')
+    ctx.expect_out_file('DS_test/C0805C102J4GAC7800-1000 pF__test.pdf')
+    ctx.expect_out_file('DS_test/C0805C102J4GAC7800-1nF__test.pdf')
+    ctx.expect_out_file('DS_test/CR0805-JW-102ELF-3k3__test.pdf')
+    ctx.clean_up()
--- a/tests/utils/dummy-web-server.py
+++ b/tests/utils/dummy-web-server.py
@ -66,7 +66,7 @@ class S(BaseHTTPRequestHandler):

    def do_GET(self):
        self._set_headers()
-        self.wfile.write(self._html("hi!"))
+        self.wfile.write(self._html(self.path))

    def do_HEAD(self):
        self._set_headers()
--- a/tests/yaml_samples/download_datasheets_1.kibot.yaml
+++ b/tests/yaml_samples/download_datasheets_1.kibot.yaml
@ -0,0 +1,48 @@
+# Example KiBot config file
+kibot:
+  version: 1
+
+filters:
+  - name: 'Variant rename'
+    type: var_rename
+    separator: ':'
+
+variants:
+  - name: 'production'
+    comment: 'Production variant'
+    type: kibom
+    file_id: '_(production)'
+    variant: production
+
+  - name: 'test'
+    comment: 'Test variant'
+    type: kibom
+    file_id: '_(test)'
+    variant: test
+    pre_transform: 'Variant rename'
+
+outputs:
+  - name: 'down_ds'
+    comment: "Datasheets"
+    type: download_datasheets
+    dir: DS
+    options:
+      output: '${manf#}.pdf'
+
+  - name: 'down_ds_production'
+    comment: "Datasheets for production"
+    type: download_datasheets
+    dir: DS_%V
+    options:
+      variant: production
+      output: '${manf#}.pdf'
+
+  - name: 'down_ds_test'
+    comment: "Datasheets for test"
+    type: download_datasheets
+    dir: DS_%V
+    options:
+      variant: test
+      output: '${manf#}-${VALUE}:/%V.pdf'
+      repeated: true
+