diff --git a/CHANGELOG.md b/CHANGELOG.md index e97e58e1..18fc853e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Filters: - New `value_split` to extract information from the Value field and put it in separated fields. I.e. tolerance, voltage, etc. + - New `spec_to_field` to extract information from the distributors specs and + put in fields. I.e. RoHS status. - New `generic` options `exclude_not_in_bom` and `exclude_not_on_board` to - use KiCad 6+ flags. + use KiCad 6+ flags. (See #429) - New internal filters: - `_value_split` splits the Value field but the field remains and the extra data is not visible diff --git a/README.md b/README.md index e5a4db9d..6f71e809 100644 --- a/README.md +++ b/README.md @@ -1021,6 +1021,29 @@ filters: Components matching the regular expression will be rotated the indicated angle. - `skip_bottom`: [boolean=false] Do not rotate components on the bottom. - `skip_top`: [boolean=false] Do not rotate components on the top. +- spec_to_field: Spec_to_Field + This filter extracts information from the specs obtained from component distributors + and fills fields. + I.e. create a field with the RoHS status of a component. + In order to make it work you must be able to get prices using the KiCost options of + the `bom` output. Make sure you can do this before trying to use this filter. + Usage [example](https://inti-cmnb.github.io/kibot-examples-1/spec_to_field/). + * Valid keys: + - **`from_output`**: [string=''] Name of the output used to collect the specs. + Currently this must be a `bom` output with KiCost enabled and a distributor that returns specs. + - `comment`: [string=''] A comment for documentation purposes. + - `name`: [string=''] Used to identify this particular filter definition. + - `specs`: [list(dict)|dict] *One or more specs to be copied. + * Valid keys: + - **`field`**: [string=''] Name of the destination field. + - `collision`: [string='warning'] [warning,error,ignore] How to report a collision between the current value and the new value. + - `compare`: [string='plain'] [plain,smart] How we compare the current value to determine a collision. + `plain` is a strict comparison. `smart` tries to extract any number and compare it. + - `policy`: [string='overwrite'] [overwrite,update,new] Controls the behavior of the copy mechanism. + `overwrite` always copy the spec value, + `update` copy only if the field already exist, + `new` copy only if the field doesn't exist.. + - `spec`: [string|list(string)=''] *Name/s of the source spec/s. - subparts: Subparts This filter implements the KiCost subparts mechanism. * Valid keys: diff --git a/kibot/bom/xlsx_writer.py b/kibot/bom/xlsx_writer.py index bd731b6f..11164aa9 100644 --- a/kibot/bom/xlsx_writer.py +++ b/kibot/bom/xlsx_writer.py @@ -561,6 +561,15 @@ def do_title(cfg, worksheet, col1, length, fmt_title, fmt_info): worksheet.merge_range(c+r_extra, col1, c+r_extra, length, text, fmt_info) +def copy_specs_to_components(parts, groups): + """ Link the KiCost information in the components. + So we can access to the specs for the components. + This can be used by filters. """ + for p in parts: + for c in p.kibot_group.components: + c.kicost_part = p + + def _create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_subtitle, fmt_head, fmt_cols, cfg): if not KICOST_SUPPORT: logger.warning(W_NOKICOST+'KiCost sheet requested but failed to load KiCost support') @@ -642,6 +651,8 @@ def _create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_ part.refs = [c.ref for c in g.components] part.fields = g.fields part.fields['manf#_qty'] = compute_qtys(cfg, g) + # Internally used to make copy_specs_to_components simpler + part.kibot_group = g parts.append(part) # Process any "join" request apply_join_requests(cfg.join_ce, part.fields, g.fields) @@ -651,6 +662,8 @@ def _create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_ dist_list = solve_distributors(cfg) # Get the prices query_part_info(parts, dist_list) + # Put the specs in the components + copy_specs_to_components(parts, groups) # Distributors again. During `query_part_info` user defined distributors could be added solve_distributors(cfg, silent=False) # Create a class to hold the spreadsheet parameters diff --git a/kibot/config_reader.py b/kibot/config_reader.py index 612ea62c..d9dfce83 100644 --- a/kibot/config_reader.py +++ b/kibot/config_reader.py @@ -16,7 +16,7 @@ import json from sys import (exit, maxsize) from collections import OrderedDict -from .error import KiPlotConfigurationError +from .error import KiPlotConfigurationError, config_error from .misc import (NO_YAML_MODULE, EXIT_BAD_ARGS, EXAMPLE_CFG, WONT_OVERWRITE, W_NOOUTPUTS, W_UNKOUT, W_NOFILTERS, W_NOVARIANTS, W_NOGLOBALS, TRY_INSTALL_CHECK, W_NOPREFLIGHTS, W_NOGROUPS) from .gs import GS @@ -446,7 +446,11 @@ class CfgYamlReader(object): return sel_globals def configure_variant_or_filter(self, o_var): - o_var.config(None) + try: + o_var.config(None) + except KiPlotConfigurationError as e: + msg = "In filter/variant '"+o_var.name+"' ("+o_var.type+"): "+str(e) + config_error(msg) def configure_variants(self, variants): logger.debug('Configuring variants') diff --git a/kibot/fil_spec_to_field.py b/kibot/fil_spec_to_field.py new file mode 100644 index 00000000..2fa7c6e2 --- /dev/null +++ b/kibot/fil_spec_to_field.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2023 Salvador E. Tropea +# Copyright (c) 2023 Instituto Nacional de Tecnología Industrial +# License: GPL-3.0 +# Project: KiBot (formerly KiPlot) +# Description: Extracts information from the distributor spec and fills fields +import re +from .bom.xlsx_writer import get_spec +from .error import KiPlotConfigurationError +from .kiplot import look_for_output, run_output +from .misc import W_FLDCOLLISION +# from .gs import GS +from .optionable import Optionable +from .macros import macros, document, filter_class # noqa: F401 +from . import log + +logger = log.get_logger() + + +class SpecOptions(Optionable): + """ A spec to copy """ + def __init__(self): + super().__init__() + self._unknown_is_error = True + with document: + self.spec = Optionable + """ [string|list(string)=''] *Name/s of the source spec/s """ + self.field = '' + """ *Name of the destination field """ + self.policy = 'overwrite' + """ [overwrite,update,new] Controls the behavior of the copy mechanism. + `overwrite` always copy the spec value, + `update` copy only if the field already exist, + `new` copy only if the field doesn't exist. """ + self.collision = 'warning' + """ [warning,error,ignore] How to report a collision between the current value and the new value """ + self.compare = 'plain' + """ [plain,smart] How we compare the current value to determine a collision. + `plain` is a strict comparison. `smart` tries to extract any number and compare it """ + self._field_example = 'RoHS' + self._spec_example = 'rohs_status' + + def config(self, parent): + super().config(parent) + if not self.field: + raise KiPlotConfigurationError("Missing or empty `field` in spec_to_field filter ({})".format(str(self._tree))) + if not self.spec: + raise KiPlotConfigurationError("Missing or empty `spec` in spec_to_field filter ({})".format(str(self._tree))) + self.spec = self.force_list(self.spec) + + +@filter_class +class Spec_to_Field(BaseFilter): # noqa: F821 + """ Spec_to_Field + This filter extracts information from the specs obtained from component distributors + and fills fields. + I.e. create a field with the RoHS status of a component. + In order to make it work you must be able to get prices using the KiCost options of + the `bom` output. Make sure you can do this before trying to use this filter. + Usage [example](https://inti-cmnb.github.io/kibot-examples-1/spec_to_field/) """ + def __init__(self): + super().__init__() + self._is_transform = True + with document: + self.from_output = '' + """ *Name of the output used to collect the specs. + Currently this must be a `bom` output with KiCost enabled and a distributor that returns specs """ + self.specs = SpecOptions + """ [list(dict)|dict] *One or more specs to be copied """ + self._from = None + + def config(self, parent): + super().config(parent) + if not self.from_output: + raise KiPlotConfigurationError("You must specify an output that collected the specs") + if isinstance(self.specs, type): + raise KiPlotConfigurationError("At least one spec must be provided ({})".format(str(self._tree))) + if isinstance(self.specs, SpecOptions): + self.specs = [self.specs] + + def compare(self, cur_val, spec_val, how): + cur_val = cur_val.lower().strip() + spec_val = spec_val.lower().strip() + if how == 'plain': + logger.debugl(3, f" - Compare {cur_val} == {spec_val}") + return cur_val == spec_val + # smart + cur_match = re.match(r'(.*?)(\d+)(.*?)', cur_val) + if cur_match: + spec_match = re.match(r'(.*?)(\d+)(.*?)', spec_val) + if spec_match: + logger.debugl(3, f" - Compare {int(cur_match.group(2))} == {int(spec_match.group(2))}") + return int(cur_match.group(2)) == int(spec_match.group(2)) + logger.debugl(3, f" - Compare {cur_val} == {spec_val}") + return cur_val == spec_val + + def solve_from(self): + if self._from is not None: + return + # Check the renderer output is valid + out = look_for_output(self.from_output, 'from_output', self._parent, {'bom'}) + if not out._done: + run_output(out) + self._from = out + + def filter(self, comp): + self.solve_from() + for d, dd in comp.kicost_part.dd.items(): + logger.error(f"{d} {dd.extra_info}") + for s in self.specs: + field = s.field.lower() + spec_name = [] + spec_val = [] + for sp in s.spec: + name, val = get_spec(comp.kicost_part, sp) + if name: + spec_name.append(name) + if val: + spec_val.append(val) + spec_name = ','.join(spec_name) + spec_val = ' '.join(spec_val) + if not spec_name or not spec_val: + # No info + continue + has_field = comp.is_field(field) + cur_val = comp.get_field_value(field) if has_field else None + if cur_val: + if cur_val == spec_val: + # Already there + continue + if not self.compare(cur_val, spec_val, s.compare): + # Collision + desc = "{} field `{}` collision, has `{}`, found `{}`".format(comp.ref, s.field, cur_val, spec_val) + if s.collision == 'warning': + logger.warning(W_FLDCOLLISION+desc) + elif s.collision == 'error': + raise KiPlotConfigurationError(desc) + if s.policy == 'overwrite' or (self.p == 'update' and has_field) or (s.policy == 'new' and not has_field): + comp.set_field(s.field, spec_val) + logger.debugl(2, "- {} {}: {} ({})".format(comp.ref, s.field, spec_val, spec_name)) diff --git a/kibot/kiplot.py b/kibot/kiplot.py index 16313f54..495c6d11 100644 --- a/kibot/kiplot.py +++ b/kibot/kiplot.py @@ -447,6 +447,16 @@ def configure_and_run(tree, out_dir, msg): out.run(out_dir) +def look_for_output(name, op_name, parent, valids): + out = RegOutput.get_output(name) + if out is None: + raise KiPlotConfigurationError('Unknown output `{}` selected in {}'.format(name, parent)) + config_output(out) + if out.type not in valids: + raise KiPlotConfigurationError('`{}` must be {} type, not {}'.format(op_name, valids, out.type)) + return out + + def _generate_outputs(outputs, targets, invert, skip_pre, cli_order, no_priority, dont_stop): logger.debug("Starting outputs for board {}".format(GS.pcb_file)) # Make a list of target outputs diff --git a/kibot/misc.py b/kibot/misc.py index 306f755f..c9cec855 100644 --- a/kibot/misc.py +++ b/kibot/misc.py @@ -274,6 +274,7 @@ W_RES3DNAME = '(W125) ' W_ESCINV = '(W126) ' W_BADVAL4 = '(W127) ' W_ENVEXIST = '(W128) ' +W_FLDCOLLISION = '(W129) ' # Somehow arbitrary, the colors are real, but can be different PCB_MAT_COLORS = {'fr1': "937042", 'fr2': "949d70", 'fr3': "adacb4", 'fr4': "332B16", 'fr5': "6cc290"} PCB_FINISH_COLORS = {'hal': "8b898c", 'hasl': "8b898c", 'imag': "8b898c", 'enig': "cfb96e", 'enepig': "cfb96e", diff --git a/kibot/out_populate.py b/kibot/out_populate.py index 324807f8..7fa1001a 100644 --- a/kibot/out_populate.py +++ b/kibot/out_populate.py @@ -17,10 +17,9 @@ from tempfile import NamedTemporaryFile from .error import KiPlotConfigurationError from .misc import W_PCBDRAW, RENDERERS from .gs import GS -from .kiplot import config_output, run_output +from .kiplot import run_output, look_for_output from .optionable import Optionable from .out_base import VariantOptions -from .registrable import RegOutput from .macros import macros, document, output_class # noqa: F401 from . import log @@ -122,13 +121,7 @@ class PopulateOptions(VariantOptions): is_html = self.format == 'html' # Check the renderer output is valid - out = RegOutput.get_output(self.renderer) - if out is None: - raise KiPlotConfigurationError('Unknown output `{}` selected in {}'.format(self.renderer, self._parent)) - config_output(out) - if out.type not in RENDERERS: - raise KiPlotConfigurationError('The `renderer` must be {} type, not {}'.format(RENDERERS, out.type)) - self._renderer = out + self._renderer = look_for_output(self.renderer, 'renderer', self._parent, RENDERERS) # Load the input content try: _, content = load_content(self.input) diff --git a/tests/board_samples/kicad_5/kibom-variant_2c.sch b/tests/board_samples/kicad_5/kibom-variant_2c.sch index 78633405..e7b63a5b 100644 --- a/tests/board_samples/kicad_5/kibom-variant_2c.sch +++ b/tests/board_samples/kicad_5/kibom-variant_2c.sch @@ -83,7 +83,8 @@ F 4 "-test" H 2500 1700 50 0001 C CNN "Config" F 5 "Bourns" H 1000 1700 50 0001 C CNN "manf" F 6 "CR0603-JW-102ELF" H 1000 1700 50 0001 C CNN "manf#" F 7 "CR0603-JW-102ELFCT-ND" H 1000 1700 50 0001 C CNN "digikey#" -F 8 "1000" H 2500 1700 50 0001 C CNN "Resistance" +F 8 "5%" H 1000 1700 50 0001 C CNN "tolerance" +F 9 "1000" H 2500 1700 50 0001 C CNN "Resistance" 1 2500 1700 1 0 0 -1 $EndComp diff --git a/tests/board_samples/kicad_6/kibom-variant_2c.kicad_sch b/tests/board_samples/kicad_6/kibom-variant_2c.kicad_sch index dc38df46..5ab56ca9 100644 --- a/tests/board_samples/kicad_6/kibom-variant_2c.kicad_sch +++ b/tests/board_samples/kicad_6/kibom-variant_2c.kicad_sch @@ -263,6 +263,9 @@ (property "digikey#" "CR0603-JW-102ELFCT-ND" (id 7) (at 25.4 43.18 0) (effects (font (size 1.27 1.27)) hide) ) + (property "tolerance" "5%" (at 53.34 43.18 0) + (effects (font (size 1.27 1.27)) hide) + ) (property "Resistance" "1000" (id 8) (at 63.5 43.18 0) (effects (font (size 1.27 1.27)) hide) ) diff --git a/tests/board_samples/kicad_7/kibom-variant_2c.kicad_sch b/tests/board_samples/kicad_7/kibom-variant_2c.kicad_sch index 3a49dbd1..f5ce4c52 100644 --- a/tests/board_samples/kicad_7/kibom-variant_2c.kicad_sch +++ b/tests/board_samples/kicad_7/kibom-variant_2c.kicad_sch @@ -284,6 +284,9 @@ (property "digikey#" "CR0603-JW-102ELFCT-ND" (at 25.4 43.18 0) (effects (font (size 1.27 1.27)) hide) ) + (property "tolerance" "5%" (at 53.34 43.18 0) + (effects (font (size 1.27 1.27)) hide) + ) (property "Resistance" "1000" (at 63.5 43.18 0) (effects (font (size 1.27 1.27)) hide) ) diff --git a/tests/board_samples/kicad_8/kibom-variant_2c.kicad_sch b/tests/board_samples/kicad_8/kibom-variant_2c.kicad_sch index 3a49dbd1..f5ce4c52 100644 --- a/tests/board_samples/kicad_8/kibom-variant_2c.kicad_sch +++ b/tests/board_samples/kicad_8/kibom-variant_2c.kicad_sch @@ -284,6 +284,9 @@ (property "digikey#" "CR0603-JW-102ELFCT-ND" (at 25.4 43.18 0) (effects (font (size 1.27 1.27)) hide) ) + (property "tolerance" "5%" (at 53.34 43.18 0) + (effects (font (size 1.27 1.27)) hide) + ) (property "Resistance" "1000" (at 63.5 43.18 0) (effects (font (size 1.27 1.27)) hide) ) diff --git a/tests/test_plot/test_kicost.py b/tests/test_plot/test_kicost.py index 9b0fdc2d..2ad6b77d 100644 --- a/tests/test_plot/test_kicost.py +++ b/tests/test_plot/test_kicost.py @@ -124,3 +124,17 @@ def test_kicost_bom_merge_1(test_dir): convert2csv(ctx, output, sheet='Costs') csv = output[:-4]+'csv' ctx.compare_txt_d2(csv) + + +def test_kicost_spec_to_field_1(test_dir): + """ Internal BoM + KiCost, select distributors (Mouser+Digi-Key). With DNF sheet. + Then copy the RoHS spec to a variant schematic """ + prj = 'kibom-variant_2c' + ctx = context.TestContextSCH(test_dir, prj, 'spec_to_field_1', OUT_DIR) + ctx.run(kicost=True, extra_debug=True) + output = prj+'-bom.xlsx' + ctx.expect_out_file_d(output) + ctx.search_err([r'WARNING:\(.*\) C1 field `Tolerance` collision, has `20%`, found `.10%`', + r'WARNING:\(.*\) R1 field `Tolerance` collision, has `1%`, found `.5%`', + 'C1 RoHS: Compliant', 'R2 Tolerance: .5%']) + ctx.clean_up() diff --git a/tests/yaml_samples/int_bom_kicost_sel_dist_1_xlsx.kibot.yaml b/tests/yaml_samples/int_bom_kicost_sel_dist_1_xlsx.kibot.yaml index bd111005..0818ca2b 100644 --- a/tests/yaml_samples/int_bom_kicost_sel_dist_1_xlsx.kibot.yaml +++ b/tests/yaml_samples/int_bom_kicost_sel_dist_1_xlsx.kibot.yaml @@ -4,7 +4,7 @@ kibot: outputs: - name: 'bom_internal' - comment: "Bill of Materials in HTML format" + comment: "Bill of Materials in XLSX format w/prices" type: bom dir: KiCost options: diff --git a/tests/yaml_samples/spec_to_field_1.kibot.yaml b/tests/yaml_samples/spec_to_field_1.kibot.yaml new file mode 100644 index 00000000..8a34cd7e --- /dev/null +++ b/tests/yaml_samples/spec_to_field_1.kibot.yaml @@ -0,0 +1,49 @@ +kibot: + version: 1 + +filters: + - name: spec_to_field + type: spec_to_field + comment: 'Copy the RoHS status' + from_output: 'bom_internal' + specs: + - spec: rohs_status + field: RoHS + - spec: [resistance_tolerance, capacitance_tolerance] + field: Tolerance + compare: smart + +outputs: + - name: create_sch + comment: "Apply the filter to the Schematic" + type: sch_variant + dir: Modified + options: + pre_transform: spec_to_field + copy_project: true + + - name: 'bom_internal' + comment: "BoM with prices, here used to get the specs" + type: bom + dir: KiCost + options: + group_fields: ['digikey#'] + columns: + - References + - Part + - Value + - Quantity Per PCB + - field: manf + name: Manufacturer + - field: manf# + name: Manufacturer P/N + - field: digikey# + level: 1 + comment: 'Code used to buy the part at Digi-Key' + distributors: + - Mouser + - Digi-Key + xlsx: + kicost: true + specs: true + kicost_config: tests/data/kicost_default_config.yaml