diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cbce17d..89dbee26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 So you can create a compressed file containing the source schematic and PCB files. (#93) - Support for new KiCost options `split_extra_fields` and `board_qty`. (#120) +- Datasheet downloader. (#119) ### Changed - Internal BoM: now components with different Tolerance, Voltage, Current diff --git a/README.md b/README.md index 1319e80c..956e2a82 100644 --- a/README.md +++ b/README.md @@ -915,6 +915,32 @@ Next time you need this list just use an alias, like this: - `output_id`: [string=''] Text to use for the %I expansion content. To differentiate variations of this output. - `run_by_default`: [boolean=true] When enabled this output will be created when no specific outputs are requested. +* Datasheets downloader + * Type: `download_datasheets` + * Description: Downloads the datasheets for the project + * Valid keys: + - `comment`: [string=''] A comment for documentation purposes. + - `dir`: [string='./'] Output directory for the generated files. If it starts with `+` the rest is concatenated to the default dir. + - `disable_run_by_default`: [string|boolean] Use it to disable the `run_by_default` status of other output. + Useful when this output extends another and you don't want to generate the original. + Use the boolean true value to disable the output you are extending. + - `extends`: [string=''] Copy the `options` section from the indicated output. + - `name`: [string=''] Used to identify this particular output definition. + - `options`: [dict] Options for the `download_datasheets` output. + * Valid keys: + - `dnf`: [boolean=false] Include the DNF components. + - `dnf_filter`: [string|list(string)='_none'] Name of the filter to mark components as not fitted. + A short-cut to use for simple cases where a variant is an overkill. + - `field`: [string='Datasheet'] Name of the field containing the URL. + - `link_repeated`: [boolean=true] Instead of download things we already downloaded use symlinks. + - `output`: [string='${VALUE}.pdf'] Name used for the downloaded datasheet. + ${FIELD} will be replaced by the FIELD content. + - `repeated`: [boolean=false] Download URLs that we already downloaded. + It only makes sense if the `output` field makes their output different. + - `variant`: [string=''] Board variant to apply. + - `output_id`: [string=''] Text to use for the %I expansion content. To differentiate variations of this output. + - `run_by_default`: [boolean=true] When enabled this output will be created when no specific outputs are requested. + * DXF (Drawing Exchange Format) * Type: `dxf` * Description: Exports the PCB to 2D mechanical EDA tools (like AutoCAD). diff --git a/docs/samples/generic_plot.kibot.yaml b/docs/samples/generic_plot.kibot.yaml index e3a0c016..e956927e 100644 --- a/docs/samples/generic_plot.kibot.yaml +++ b/docs/samples/generic_plot.kibot.yaml @@ -301,6 +301,30 @@ outputs: # [string='%f-%i%v.%x'] Name for the generated archive (%i=name of the output %x=according to format). Affected by global options output: '%f-%i%v.%x' + # Datasheets downloader: + - name: 'download_datasheets_example' + comment: 'Downloads the datasheets for the project' + type: 'download_datasheets' + dir: 'Example/download_datasheets_dir' + options: + # [boolean=false] Include the DNF components + dnf: false + # [string|list(string)='_none'] Name of the filter to mark components as not fitted. + # A short-cut to use for simple cases where a variant is an overkill + dnf_filter: '_none' + # [string='Datasheet'] Name of the field containing the URL + field: 'Datasheet' + # [boolean=true] Instead of download things we already downloaded use symlinks + link_repeated: true + # [string='${VALUE}.pdf'] Name used for the downloaded datasheet. + # ${FIELD} will be replaced by the FIELD content + output: '${VALUE}.pdf' + # [boolean=false] Download URLs that we already downloaded. + # It only makes sense if the `output` field makes their output different + repeated: false + # [string=''] Board variant to apply + variant: '' + # DXF (Drawing Exchange Format): # This output is what you get from the File/Plot menu in pcbnew. - name: 'dxf_example' diff --git a/kibot/misc.py b/kibot/misc.py index e717d2ba..ceeedb6b 100644 --- a/kibot/misc.py +++ b/kibot/misc.py @@ -215,6 +215,7 @@ W_EMPTREP = '(W073) ' W_BADCHARS = '(W074) ' W_DATEFORMAT = '(W075) ' W_UNKFLD = '(W076) ' +W_ALRDOWN = '(W077) ' class Rect(object): diff --git a/kibot/out_download_datasheets.py b/kibot/out_download_datasheets.py new file mode 100644 index 00000000..2760edcc --- /dev/null +++ b/kibot/out_download_datasheets.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2021 Salvador E. Tropea +# Copyright (c) 2021 Instituto Nacional de TecnologĂ­a Industrial +# License: GPL-3.0 +# Project: KiBot (formerly KiPlot) +import os +import re +import requests +from .out_base import VariantOptions +from .fil_base import DummyFilter +from .error import KiPlotConfigurationError +from .misc import W_UNKFLD, W_ALRDOWN, W_FAILDL +from .gs import GS +from .macros import macros, document, output_class # noqa: F401 +from . import log +logger = log.get_logger(__name__) +USER_AGENT = 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1' + + +class Download_Datasheets_Options(VariantOptions): + _vars_regex = re.compile(r'\$\{([^\}]+)\}') + + def __init__(self): + super().__init__() + with document: + self.field = 'Datasheet' + """ Name of the field containing the URL """ + self.output = '${VALUE}.pdf' + """ Name used for the downloaded datasheet. + ${FIELD} will be replaced by the FIELD content """ + self.dnf = False + """ Include the DNF components """ + self.repeated = False + """ Download URLs that we already downloaded. + It only makes sense if the `output` field makes their output different """ + self.link_repeated = True + """ Instead of download things we already downloaded use symlinks """ + # Used to collect the targets + self._dry = False + + def config(self, parent): + super().config(parent) + if not self.field: + raise KiPlotConfigurationError("Empty `field` ({})".format(str(self._tree))) + if not self.output: + raise KiPlotConfigurationError("Empty `output` ({})".format(str(self._tree))) + self.field = self.field.lower() + + def download(self, c, ds, dir, name, known): + dest = os.path.join(dir, name) + logger.debug('To download: {} -> {}'.format(ds, dest)) + if name in self._downloaded: + logger.warning(W_ALRDOWN+'Datasheet `{}` already downloaded'.format(name)) + return None + elif known is not None and self.link_repeated: + # We already downloaded this URL, but stored it with a different name + if not self._dry: + os.symlink(known, dest) + self._created.append(os.path.relpath(dest)) + elif not os.path.isfile(dest): + # Download + if not self._dry: + r = requests.get(ds, allow_redirects=True, headers={'User-Agent': USER_AGENT}) + if r.status_code != 200: + logger.warning(W_FAILDL+'Failed to download `{}`'.format(ds)) + return None + with open(dest, 'wb') as f: + f.write(r.content) + self._downloaded.add(name) + self._created.append(os.path.relpath(dest)) + elif self._dry: + self._created.append(os.path.relpath(dest)) + return name + + def out_name(self, c): + """ Compute the name of the output file. + Replaces ${FIELD} and %X. """ + out = '' + last = 0 + pattern = self.output + pattern_l = len(pattern) + for match in Download_Datasheets_Options._vars_regex.finditer(pattern): + fname = match.group(1).lower() + value = c.get_field_value(fname) + if value is None: + value = 'Unknown' + logger.warning(W_UNKFLD+"In datasheets download output file name:" + " Field `{}` not defined for {}, using `Unknown`".format(fname, c.ref)) + if match.start(): + out += pattern[last:match.start()] + out += value + last = match.end() + if last < pattern_l: + out += pattern[last:pattern_l] + out = self.expand_filename_sch(out) + return out.replace('/', '_') + + def run(self, output_dir): + if not self.dnf_filter and not self.variant: + # Add a dummy filter to force the creation of a components list + self.dnf_filter = DummyFilter() + super().run(output_dir) + self._urls = {} + self._downloaded = set() + self._created = [] + field_used = False + for c in self._comps: + ds = c.get_field_value(self.field) + if ds is not None: + field_used = True + if not c.included or (not c.fitted and not self.dnf): + continue + if ds: + known = self._urls.get(ds, None) + if known is None or self.repeated: + name = self.out_name(c) + name = self.download(c, ds, output_dir, name, known) + if known is None: + self._urls[ds] = name + else: + logger.debug('Already downloaded: '+ds) + if not field_used: + known_fields = GS.sch.get_field_names({}) + if self.field not in known_fields: + logger.warning(W_UNKFLD+"The field used for datasheets ({}) doesn't seem to be used".format(self.field)) + else: + logger.debug('Unique URLs: '+str(len(self._urls))) + logger.debug('Downloaded: '+str(len(self._downloaded))) + logger.debug('Created: '+str(len(self._created))) + + def get_targets(self, out_dir): + # Do a dry run to collect the output names + self._dry = True + self.run(out_dir) + self._dry = False + return self._created + + +@output_class +class Download_Datasheets(BaseOutput): # noqa: F821 + """ Datasheets downloader + Downloads the datasheets for the project """ + def __init__(self): + super().__init__() + with document: + self.options = Download_Datasheets_Options + """ [dict] Options for the `download_datasheets` output """ + self._sch_related = True + + def run(self, output_dir): + # No output member, just a dir + self.options.run(output_dir) diff --git a/tests/board_samples/kicad_5/kibom-variant_2ds.sch b/tests/board_samples/kicad_5/kibom-variant_2ds.sch new file mode 100644 index 00000000..99ad952f --- /dev/null +++ b/tests/board_samples/kicad_5/kibom-variant_2ds.sch @@ -0,0 +1,72 @@ +EESchema Schematic File Version 4 +EELAYER 30 0 +EELAYER END +$Descr A4 11693 8268 +encoding utf-8 +Sheet 1 1 +Title "KiBom Test Schematic" +Date "2020-03-12" +Rev "A" +Comp "https://github.com/SchrodingersGat/KiBom" +Comment1 "" +Comment2 "" +Comment3 "" +Comment4 "" +$EndDescr +Text Notes 500 600 0 79 ~ 0 +This schematic serves as a test-file for the KiBot export script.\n +Text Notes 5950 2600 0 118 ~ 0 +The test tests the following \nvariants matrix:\n production test default\nC1 X\nC2 X\nR1 X X X\nR2 X X\n +$Comp +L Device:C C1 +U 1 1 5F43BEC2 +P 1000 1700 +F 0 "C1" H 1115 1746 50 0000 L CNN +F 1 "1nF" H 1115 1655 50 0000 L CNN +F 2 "" H 1038 1550 50 0001 C CNN +F 3 "http://localhost:8000/c.pdf" H 1000 1700 50 0001 C CNN +F 4 "-production,+test" H 1000 1700 50 0001 C CNN "Config" +F 5 "C0805C102J4GAC7800" H 1000 1700 50 0001 C CNN "manf#" + 1 1000 1700 + 1 0 0 -1 +$EndComp +$Comp +L Device:C C2 +U 1 1 5F43CE1C +P 1450 1700 +F 0 "C2" H 1565 1746 50 0000 L CNN +F 1 "1000 pF" H 1565 1655 50 0000 L CNN +F 2 "" H 1488 1550 50 0001 C CNN +F 3 "http://localhost:8000/c.pdf" H 1450 1700 50 0001 C CNN +F 4 "+test" H 1450 1700 50 0001 C CNN "Config" +F 5 "C0805C102J4GAC7800" H 1000 1700 50 0001 C CNN "manf#" + 1 1450 1700 + 1 0 0 -1 +$EndComp +$Comp +L Device:R R1 +U 1 1 5F43D144 +P 2100 1700 +F 0 "R1" H 2170 1746 50 0000 L CNN +F 1 "1k" H 2170 1655 50 0000 L CNN +F 2 "" V 2030 1700 50 0001 C CNN +F 3 "http://localhost:8000/r.pdf" H 2100 1700 50 0001 C CNN +F 4 "3k3" H 2100 1700 50 0001 C CNN "test:Value" +F 5 "CR0805-JW-102ELF" H 1000 1700 50 0001 C CNN "manf#" + 1 2100 1700 + 1 0 0 -1 +$EndComp +$Comp +L Device:R R2 +U 1 1 5F43D4BB +P 2500 1700 +F 0 "R2" H 2570 1746 50 0000 L CNN +F 1 "1000" H 2570 1655 50 0000 L CNN +F 2 "" V 2430 1700 50 0001 C CNN +F 3 "http://localhost:8000/r.pdf" H 2500 1700 50 0001 C CNN +F 4 "-test" H 2500 1700 50 0001 C CNN "Config" +F 5 "CR0805-JW-102ELF" H 1000 1700 50 0001 C CNN "manf#" + 1 2500 1700 + 1 0 0 -1 +$EndComp +$EndSCHEMATC diff --git a/tests/test_plot/test_misc.py b/tests/test_plot/test_misc.py index 07a69e2f..54546977 100644 --- a/tests/test_plot/test_misc.py +++ b/tests/test_plot/test_misc.py @@ -894,3 +894,17 @@ def test_date_format_2(test_dir): ctx.expect_out_file(POS_DIR+'/bom_13_07_2020.csv') assert ctx.search_err('Trying to reformat SCH time, but not in ISO format') ctx.clean_up() + + +def test_download_datasheets_1(test_dir): + prj = 'kibom-variant_2ds' + ctx = context.TestContextSCH(test_dir, 'test_download_datasheets_1', prj, 'download_datasheets_1', '') + # We use a fake server to avoid needing good URLs and reliable internet connection + ctx.run(kicost=True) + ctx.expect_out_file('DS/C0805C102J4GAC7800.pdf') + ctx.expect_out_file('DS/CR0805-JW-102ELF.pdf') + ctx.expect_out_file('DS_production/CR0805-JW-102ELF.pdf') + ctx.expect_out_file('DS_test/C0805C102J4GAC7800-1000 pF__test.pdf') + ctx.expect_out_file('DS_test/C0805C102J4GAC7800-1nF__test.pdf') + ctx.expect_out_file('DS_test/CR0805-JW-102ELF-3k3__test.pdf') + ctx.clean_up() diff --git a/tests/utils/dummy-web-server.py b/tests/utils/dummy-web-server.py index 11499ce9..d3195111 100755 --- a/tests/utils/dummy-web-server.py +++ b/tests/utils/dummy-web-server.py @@ -66,7 +66,7 @@ class S(BaseHTTPRequestHandler): def do_GET(self): self._set_headers() - self.wfile.write(self._html("hi!")) + self.wfile.write(self._html(self.path)) def do_HEAD(self): self._set_headers() diff --git a/tests/yaml_samples/download_datasheets_1.kibot.yaml b/tests/yaml_samples/download_datasheets_1.kibot.yaml new file mode 100644 index 00000000..1b7330ab --- /dev/null +++ b/tests/yaml_samples/download_datasheets_1.kibot.yaml @@ -0,0 +1,48 @@ +# Example KiBot config file +kibot: + version: 1 + +filters: + - name: 'Variant rename' + type: var_rename + separator: ':' + +variants: + - name: 'production' + comment: 'Production variant' + type: kibom + file_id: '_(production)' + variant: production + + - name: 'test' + comment: 'Test variant' + type: kibom + file_id: '_(test)' + variant: test + pre_transform: 'Variant rename' + +outputs: + - name: 'down_ds' + comment: "Datasheets" + type: download_datasheets + dir: DS + options: + output: '${manf#}.pdf' + + - name: 'down_ds_production' + comment: "Datasheets for production" + type: download_datasheets + dir: DS_%V + options: + variant: production + output: '${manf#}.pdf' + + - name: 'down_ds_test' + comment: "Datasheets for test" + type: download_datasheets + dir: DS_%V + options: + variant: test + output: '${manf#}-${VALUE}:/%V.pdf' + repeated: true +