[Filters][Added] `spec_to_field`

- To extract information from the distributors specs and put in fields.
  I.e. RoHS status.

CHANGELOG.md

@@ -19,8 +19,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Filters:
   - New `value_split` to extract information from the Value field and put it in
     separated fields. I.e. tolerance, voltage, etc.
+  - New `spec_to_field` to extract information from the distributors specs and
+    put in fields. I.e. RoHS status.
   - New `generic` options `exclude_not_in_bom` and `exclude_not_on_board` to
-    use KiCad 6+ flags.
+    use KiCad 6+ flags. (See #429)
 - New internal filters:
   - `_value_split` splits the Value field but the field remains and the extra
     data is not visible

README.md

@@ -1021,6 +1021,29 @@ filters:
                    Components matching the regular expression will be rotated the indicated angle.
     - `skip_bottom`: [boolean=false] Do not rotate components on the bottom.
     - `skip_top`: [boolean=false] Do not rotate components on the top.
+- spec_to_field: Spec_to_Field
+        This filter extracts information from the specs obtained from component distributors
+        and fills fields.
+        I.e. create a field with the RoHS status of a component.
+        In order to make it work you must be able to get prices using the KiCost options of
+        the `bom` output. Make sure you can do this before trying to use this filter.
+        Usage [example](https://inti-cmnb.github.io/kibot-examples-1/spec_to_field/).
+  * Valid keys:
+    - **`from_output`**: [string=''] Name of the output used to collect the specs.
+                         Currently this must be a `bom` output with KiCost enabled and a distributor that returns specs.
+    - `comment`: [string=''] A comment for documentation purposes.
+    - `name`: [string=''] Used to identify this particular filter definition.
+    - `specs`: [list(dict)|dict] *One or more specs to be copied.
+      * Valid keys:
+        - **`field`**: [string=''] Name of the destination field.
+        - `collision`: [string='warning'] [warning,error,ignore] How to report a collision between the current value and the new value.
+        - `compare`: [string='plain'] [plain,smart] How we compare the current value to determine a collision.
+                     `plain` is a strict comparison. `smart` tries to extract any number and compare it.
+        - `policy`: [string='overwrite'] [overwrite,update,new] Controls the behavior of the copy mechanism.
+                    `overwrite` always copy the spec value,
+                    `update` copy only if the field already exist,
+                    `new` copy only if the field doesn't exist..
+        - `spec`: [string|list(string)=''] *Name/s of the source spec/s.
 - subparts: Subparts
         This filter implements the KiCost subparts mechanism.
   * Valid keys:

kibot/bom/xlsx_writer.py

@@ -561,6 +561,15 @@ def do_title(cfg, worksheet, col1, length, fmt_title, fmt_info):
         worksheet.merge_range(c+r_extra, col1, c+r_extra, length, text, fmt_info)
 
 
+def copy_specs_to_components(parts, groups):
+    """ Link the KiCost information in the components.
+        So we can access to the specs for the components.
+        This can be used by filters. """
+    for p in parts:
+        for c in p.kibot_group.components:
+            c.kicost_part = p
+
+
 def _create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_subtitle, fmt_head, fmt_cols, cfg):
     if not KICOST_SUPPORT:
         logger.warning(W_NOKICOST+'KiCost sheet requested but failed to load KiCost support')
@@ -642,6 +651,8 @@ def _create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_
             part.refs = [c.ref for c in g.components]
             part.fields = g.fields
             part.fields['manf#_qty'] = compute_qtys(cfg, g)
+            # Internally used to make copy_specs_to_components simpler
+            part.kibot_group = g
             parts.append(part)
             # Process any "join" request
             apply_join_requests(cfg.join_ce, part.fields, g.fields)
@@ -651,6 +662,8 @@ def _create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_
         dist_list = solve_distributors(cfg)
         # Get the prices
         query_part_info(parts, dist_list)
+        # Put the specs in the components
+        copy_specs_to_components(parts, groups)
         # Distributors again. During `query_part_info` user defined distributors could be added
         solve_distributors(cfg, silent=False)
         # Create a class to hold the spreadsheet parameters

kibot/config_reader.py

@@ -16,7 +16,7 @@ import json
 from sys import (exit, maxsize)
 from collections import OrderedDict
 
-from .error import KiPlotConfigurationError
+from .error import KiPlotConfigurationError, config_error
 from .misc import (NO_YAML_MODULE, EXIT_BAD_ARGS, EXAMPLE_CFG, WONT_OVERWRITE, W_NOOUTPUTS, W_UNKOUT, W_NOFILTERS,
                    W_NOVARIANTS, W_NOGLOBALS, TRY_INSTALL_CHECK, W_NOPREFLIGHTS, W_NOGROUPS)
 from .gs import GS
@@ -446,7 +446,11 @@ class CfgYamlReader(object):
         return sel_globals
 
     def configure_variant_or_filter(self, o_var):
-        o_var.config(None)
+        try:
+            o_var.config(None)
+        except KiPlotConfigurationError as e:
+            msg = "In filter/variant '"+o_var.name+"' ("+o_var.type+"): "+str(e)
+            config_error(msg)
 
     def configure_variants(self, variants):
         logger.debug('Configuring variants')

kibot/fil_spec_to_field.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Salvador E. Tropea
+# Copyright (c) 2023 Instituto Nacional de Tecnología Industrial
+# License: GPL-3.0
+# Project: KiBot (formerly KiPlot)
+# Description: Extracts information from the distributor spec and fills fields
+import re
+from .bom.xlsx_writer import get_spec
+from .error import KiPlotConfigurationError
+from .kiplot import look_for_output, run_output
+from .misc import W_FLDCOLLISION
+# from .gs import GS
+from .optionable import Optionable
+from .macros import macros, document, filter_class  # noqa: F401
+from . import log
+
+logger = log.get_logger()
+
+
+class SpecOptions(Optionable):
+    """ A spec to copy """
+    def __init__(self):
+        super().__init__()
+        self._unknown_is_error = True
+        with document:
+            self.spec = Optionable
+            """ [string|list(string)=''] *Name/s of the source spec/s """
+            self.field = ''
+            """ *Name of the destination field """
+            self.policy = 'overwrite'
+            """ [overwrite,update,new] Controls the behavior of the copy mechanism.
+                `overwrite` always copy the spec value,
+                `update` copy only if the field already exist,
+                `new` copy only if the field doesn't exist. """
+            self.collision = 'warning'
+            """ [warning,error,ignore] How to report a collision between the current value and the new value """
+            self.compare = 'plain'
+            """ [plain,smart] How we compare the current value to determine a collision.
+                `plain` is a strict comparison. `smart` tries to extract any number and compare it """
+        self._field_example = 'RoHS'
+        self._spec_example = 'rohs_status'
+
+    def config(self, parent):
+        super().config(parent)
+        if not self.field:
+            raise KiPlotConfigurationError("Missing or empty `field` in spec_to_field filter ({})".format(str(self._tree)))
+        if not self.spec:
+            raise KiPlotConfigurationError("Missing or empty `spec` in spec_to_field filter ({})".format(str(self._tree)))
+        self.spec = self.force_list(self.spec)
+
+
+@filter_class
+class Spec_to_Field(BaseFilter):  # noqa: F821
+    """ Spec_to_Field
+        This filter extracts information from the specs obtained from component distributors
+        and fills fields.
+        I.e. create a field with the RoHS status of a component.
+        In order to make it work you must be able to get prices using the KiCost options of
+        the `bom` output. Make sure you can do this before trying to use this filter.
+        Usage [example](https://inti-cmnb.github.io/kibot-examples-1/spec_to_field/) """
+    def __init__(self):
+        super().__init__()
+        self._is_transform = True
+        with document:
+            self.from_output = ''
+            """ *Name of the output used to collect the specs.
+                Currently this must be a `bom` output with KiCost enabled and a distributor that returns specs """
+            self.specs = SpecOptions
+            """ [list(dict)|dict] *One or more specs to be copied """
+        self._from = None
+
+    def config(self, parent):
+        super().config(parent)
+        if not self.from_output:
+            raise KiPlotConfigurationError("You must specify an output that collected the specs")
+        if isinstance(self.specs, type):
+            raise KiPlotConfigurationError("At least one spec must be provided ({})".format(str(self._tree)))
+        if isinstance(self.specs, SpecOptions):
+            self.specs = [self.specs]
+
+    def compare(self, cur_val, spec_val, how):
+        cur_val = cur_val.lower().strip()
+        spec_val = spec_val.lower().strip()
+        if how == 'plain':
+            logger.debugl(3, f"  - Compare {cur_val} == {spec_val}")
+            return cur_val == spec_val
+        # smart
+        cur_match = re.match(r'(.*?)(\d+)(.*?)', cur_val)
+        if cur_match:
+            spec_match = re.match(r'(.*?)(\d+)(.*?)', spec_val)
+            if spec_match:
+                logger.debugl(3, f"  - Compare {int(cur_match.group(2))} == {int(spec_match.group(2))}")
+                return int(cur_match.group(2)) == int(spec_match.group(2))
+        logger.debugl(3, f"  - Compare {cur_val} == {spec_val}")
+        return cur_val == spec_val
+
+    def solve_from(self):
+        if self._from is not None:
+            return
+        # Check the renderer output is valid
+        out = look_for_output(self.from_output, 'from_output', self._parent, {'bom'})
+        if not out._done:
+            run_output(out)
+        self._from = out
+
+    def filter(self, comp):
+        self.solve_from()
+        for d, dd in comp.kicost_part.dd.items():
+            logger.error(f"{d} {dd.extra_info}")
+        for s in self.specs:
+            field = s.field.lower()
+            spec_name = []
+            spec_val = []
+            for sp in s.spec:
+                name, val = get_spec(comp.kicost_part, sp)
+                if name:
+                    spec_name.append(name)
+                if val:
+                    spec_val.append(val)
+            spec_name = ','.join(spec_name)
+            spec_val = ' '.join(spec_val)
+            if not spec_name or not spec_val:
+                # No info
+                continue
+            has_field = comp.is_field(field)
+            cur_val = comp.get_field_value(field) if has_field else None
+            if cur_val:
+                if cur_val == spec_val:
+                    # Already there
+                    continue
+                if not self.compare(cur_val, spec_val, s.compare):
+                    # Collision
+                    desc = "{} field `{}` collision, has `{}`, found `{}`".format(comp.ref, s.field, cur_val, spec_val)
+                    if s.collision == 'warning':
+                        logger.warning(W_FLDCOLLISION+desc)
+                    elif s.collision == 'error':
+                        raise KiPlotConfigurationError(desc)
+            if s.policy == 'overwrite' or (self.p == 'update' and has_field) or (s.policy == 'new' and not has_field):
+                comp.set_field(s.field, spec_val)
+                logger.debugl(2, "- {} {}: {} ({})".format(comp.ref, s.field, spec_val, spec_name))

kibot/kiplot.py

@@ -447,6 +447,16 @@ def configure_and_run(tree, out_dir, msg):
     out.run(out_dir)
 
 
+def look_for_output(name, op_name, parent, valids):
+    out = RegOutput.get_output(name)
+    if out is None:
+        raise KiPlotConfigurationError('Unknown output `{}` selected in {}'.format(name, parent))
+    config_output(out)
+    if out.type not in valids:
+        raise KiPlotConfigurationError('`{}` must be {} type, not {}'.format(op_name, valids, out.type))
+    return out
+
+
 def _generate_outputs(outputs, targets, invert, skip_pre, cli_order, no_priority, dont_stop):
     logger.debug("Starting outputs for board {}".format(GS.pcb_file))
     # Make a list of target outputs

kibot/misc.py

@@ -274,6 +274,7 @@ W_RES3DNAME = '(W125) '
 W_ESCINV = '(W126) '
 W_BADVAL4 = '(W127) '
 W_ENVEXIST = '(W128) '
+W_FLDCOLLISION = '(W129) '
 # Somehow arbitrary, the colors are real, but can be different
 PCB_MAT_COLORS = {'fr1': "937042", 'fr2': "949d70", 'fr3': "adacb4", 'fr4': "332B16", 'fr5': "6cc290"}
 PCB_FINISH_COLORS = {'hal': "8b898c", 'hasl': "8b898c", 'imag': "8b898c", 'enig': "cfb96e", 'enepig': "cfb96e",

kibot/out_populate.py

@@ -17,10 +17,9 @@ from tempfile import NamedTemporaryFile
 from .error import KiPlotConfigurationError
 from .misc import W_PCBDRAW, RENDERERS
 from .gs import GS
-from .kiplot import config_output, run_output
+from .kiplot import run_output, look_for_output
 from .optionable import Optionable
 from .out_base import VariantOptions
-from .registrable import RegOutput
 from .macros import macros, document, output_class  # noqa: F401
 from . import log
 
@@ -122,13 +121,7 @@ class PopulateOptions(VariantOptions):
 
         is_html = self.format == 'html'
         # Check the renderer output is valid
-        out = RegOutput.get_output(self.renderer)
+        self._renderer = look_for_output(self.renderer, 'renderer', self._parent, RENDERERS)
-        if out is None:
-            raise KiPlotConfigurationError('Unknown output `{}` selected in {}'.format(self.renderer, self._parent))
-        config_output(out)
-        if out.type not in RENDERERS:
-            raise KiPlotConfigurationError('The `renderer` must be {} type, not {}'.format(RENDERERS, out.type))
-        self._renderer = out
         # Load the input content
         try:
             _, content = load_content(self.input)

tests/board_samples/kicad_5/kibom-variant_2c.sch

@@ -83,7 +83,8 @@ F 4 "-test" H 2500 1700 50  0001 C CNN "Config"
 F 5 "Bourns" H 1000 1700 50  0001 C CNN "manf"
 F 6 "CR0603-JW-102ELF" H 1000 1700 50  0001 C CNN "manf#"
 F 7 "CR0603-JW-102ELFCT-ND" H 1000 1700 50  0001 C CNN "digikey#"
-F 8 "1000" H 2500 1700 50  0001 C CNN "Resistance"
+F 8 "5%" H 1000 1700 50  0001 C CNN "tolerance"
+F 9 "1000" H 2500 1700 50  0001 C CNN "Resistance"
 	1    2500 1700
 	1    0    0    -1  
 $EndComp

tests/board_samples/kicad_6/kibom-variant_2c.kicad_sch

@@ -263,6 +263,9 @@
     (property "digikey#" "CR0603-JW-102ELFCT-ND" (id 7) (at 25.4 43.18 0)
       (effects (font (size 1.27 1.27)) hide)
     )
+    (property "tolerance" "5%" (at 53.34 43.18 0)
+      (effects (font (size 1.27 1.27)) hide)
+    )
     (property "Resistance" "1000" (id 8) (at 63.5 43.18 0)
       (effects (font (size 1.27 1.27)) hide)
     )

tests/board_samples/kicad_7/kibom-variant_2c.kicad_sch

@@ -284,6 +284,9 @@
     (property "digikey#" "CR0603-JW-102ELFCT-ND" (at 25.4 43.18 0)
       (effects (font (size 1.27 1.27)) hide)
     )
+    (property "tolerance" "5%" (at 53.34 43.18 0)
+      (effects (font (size 1.27 1.27)) hide)
+    )
     (property "Resistance" "1000" (at 63.5 43.18 0)
       (effects (font (size 1.27 1.27)) hide)
     )

tests/board_samples/kicad_8/kibom-variant_2c.kicad_sch

@@ -284,6 +284,9 @@
     (property "digikey#" "CR0603-JW-102ELFCT-ND" (at 25.4 43.18 0)
       (effects (font (size 1.27 1.27)) hide)
     )
+    (property "tolerance" "5%" (at 53.34 43.18 0)
+      (effects (font (size 1.27 1.27)) hide)
+    )
     (property "Resistance" "1000" (at 63.5 43.18 0)
       (effects (font (size 1.27 1.27)) hide)
     )

tests/test_plot/test_kicost.py

@@ -124,3 +124,17 @@ def test_kicost_bom_merge_1(test_dir):
     convert2csv(ctx, output, sheet='Costs')
     csv = output[:-4]+'csv'
     ctx.compare_txt_d2(csv)
+
+
+def test_kicost_spec_to_field_1(test_dir):
+    """ Internal BoM + KiCost, select distributors (Mouser+Digi-Key). With DNF sheet.
+        Then copy the RoHS spec to a variant schematic """
+    prj = 'kibom-variant_2c'
+    ctx = context.TestContextSCH(test_dir, prj, 'spec_to_field_1', OUT_DIR)
+    ctx.run(kicost=True, extra_debug=True)
+    output = prj+'-bom.xlsx'
+    ctx.expect_out_file_d(output)
+    ctx.search_err([r'WARNING:\(.*\) C1 field `Tolerance` collision, has `20%`, found `.10%`',
+                    r'WARNING:\(.*\) R1 field `Tolerance` collision, has `1%`, found `.5%`',
+                    'C1 RoHS: Compliant', 'R2 Tolerance: .5%'])
+    ctx.clean_up()

tests/yaml_samples/int_bom_kicost_sel_dist_1_xlsx.kibot.yaml

@@ -4,7 +4,7 @@ kibot:
 
 outputs:
   - name: 'bom_internal'
-    comment: "Bill of Materials in HTML format"
+    comment: "Bill of Materials in XLSX format w/prices"
     type: bom
     dir: KiCost
     options:

tests/yaml_samples/spec_to_field_1.kibot.yaml

@@ -0,0 +1,49 @@
+kibot:
+  version: 1
+
+filters:
+  - name: spec_to_field
+    type: spec_to_field
+    comment: 'Copy the RoHS status'
+    from_output: 'bom_internal'
+    specs:
+      - spec: rohs_status
+        field: RoHS
+      - spec: [resistance_tolerance, capacitance_tolerance]
+        field: Tolerance
+        compare: smart
+
+outputs:
+  - name: create_sch
+    comment: "Apply the filter to the Schematic"
+    type: sch_variant
+    dir: Modified
+    options:
+      pre_transform: spec_to_field
+      copy_project: true
+
+  - name: 'bom_internal'
+    comment: "BoM with prices, here used to get the specs"
+    type: bom
+    dir: KiCost
+    options:
+      group_fields: ['digikey#']
+      columns:
+        - References
+        - Part
+        - Value
+        - Quantity Per PCB
+        - field: manf
+          name: Manufacturer
+        - field: manf#
+          name: Manufacturer P/N
+        - field: digikey#
+          level: 1
+          comment: 'Code used to buy the part at Digi-Key'
+      distributors:
+        - Mouser
+        - Digi-Key
+      xlsx:
+        kicost: true
+        specs: true
+        kicost_config: tests/data/kicost_default_config.yaml