Added distributors selection to KiCost+KiBot

README.md

@@ -678,6 +678,7 @@ Next time you need this list just use an alias, like this:
             - `hide_stats_info`: [boolean=false] Hide statistics information.
             - `quote_all`: [boolean=false] Enclose all values using double quotes.
             - `separator`: [string=','] CSV Separator. TXT and TSV always use tab as delimiter.
+        - `distributors`: [string|list(string)] Include this distributors list. Default is all the available.
         - `dnc_filter`: [string|list(string)='_kibom_dnc'] Name of the filter to mark components as 'Do Not Change'.
                         The default filter marks components with a DNC value or DNC in the Config field.
         - `dnf_filter`: [string|list(string)='_kibom_dnf'] Name of the filter to mark components as 'Do Not Fit'.
@@ -713,6 +714,7 @@ Next time you need this list just use an alias, like this:
         - `no_conflict`: [list(string)] List of fields where we tolerate conflicts.
                          Use it to avoid undesired warnings.
                          By default the field indicated in `fit_field` and the field `part` are excluded.
+        - `no_distributors`: [string|list(string)] Exclude this distributors list. They are removed after computing `distributors`.
         - `normalize_locale`: [boolean=false] When normalizing values use the locale decimal point.
         - `normalize_values`: [boolean=false] Try to normalize the R, L and C values, producing uniform units and prefixes.
         - `number`: [number=1] Number of boards to build (components multiplier).
@@ -1123,8 +1125,7 @@ Next time you need this list just use an alias, like this:
             - `file`: [string=''] Name of the XML to aggregate.
             - `variant`: [string=' '] Variant for this project.
         - `currency`: [string|list(string)=USD] Currency priority. Use ISO4217 codes (i.e. USD, EUR).
-        - `distributors`: [string|list(string)] Use only this distributors list. Default is all the available.
+        - `distributors`: [string|list(string)] Include this distributors list. Default is all the available.
-                          Not compatible with `no_distributors` option.
         - `dnf_filter`: [string|list(string)=''] Name of the filter to mark components as not fitted.
                         A short-cut to use for simple cases where a variant is an overkill.
                         Internal variants and filters are currently ignored.
@@ -1134,8 +1135,7 @@ Next time you need this list just use an alias, like this:
         - `ignore_fields`: [string|list(string)] List of fields to be ignored.
         - `kicost_variant`: [string=''] Regular expression to match the variant field (KiCost option, not internal variants).
         - `no_collapse`: [boolean=false] Do not collapse the part references (collapse=R1-R4).
-        - `no_distributors`: [string|list(string)] Use all but this distributors list. Default is use all the available.
+        - `no_distributors`: [string|list(string)] Exclude this distributors list. They are removed after computing `distributors`.
-                             Not compatible with `distributors` option.
         - `no_price`: [boolean=false] Do not look for components price. For testing purposes.
         - `output`: [string='%f-%i%v.%x'] Filename for the output (%i=kicost, %x=xlsx). Affected by global options.
         - `show_cat_url`: [boolean=false] Include the catalogue links in the catalogue code.

docs/samples/generic_plot.kibot.yaml

@@ -109,6 +109,8 @@ outputs:
         quote_all: false
         # [string=','] CSV Separator. TXT and TSV always use tab as delimiter
         separator: ','
+      # [string|list(string)] Include this distributors list. Default is all the available
+      distributors:
       # [string|list(string)='_kibom_dnc'] Name of the filter to mark components as 'Do Not Change'.
       # The default filter marks components with a DNC value or DNC in the Config field
       dnc_filter: '_kibom_dnc'
@@ -166,6 +168,8 @@ outputs:
       # Use it to avoid undesired warnings.
       # By default the field indicated in `fit_field` and the field `part` are excluded
       no_conflict: ['Config', 'Part']
+      # [string|list(string)] Exclude this distributors list. They are removed after computing `distributors`
+      no_distributors:
       # [boolean=false] When normalizing values use the locale decimal point
       normalize_locale: false
       # [boolean=false] Try to normalize the R, L and C values, producing uniform units and prefixes
@@ -707,8 +711,7 @@ outputs:
           variant: ' '
       # [string|list(string)=USD] Currency priority. Use ISO4217 codes (i.e. USD, EUR)
       currency: USD
-      # [string|list(string)] Use only this distributors list. Default is all the available.
+      # [string|list(string)] Include this distributors list. Default is all the available
-      # Not compatible with `no_distributors` option
       distributors:
       # [string|list(string)=''] Name of the filter to mark components as not fitted.
       # A short-cut to use for simple cases where a variant is an overkill.
@@ -725,8 +728,7 @@ outputs:
       kicost_variant: ''
       # [boolean=false] Do not collapse the part references (collapse=R1-R4)
       no_collapse: false
-      # [string|list(string)] Use all but this distributors list. Default is use all the available.
+      # [string|list(string)] Exclude this distributors list. They are removed after computing `distributors`
-      # Not compatible with `distributors` option
       no_distributors:
       # [boolean=false] Do not look for components price. For testing purposes
       no_price: false

kibot/bom/xlsx_writer.py

@@ -17,7 +17,7 @@ from base64 import b64decode
 from .columnlist import ColumnList
 from .kibot_logo import KIBOT_LOGO
 from .. import log
-from ..misc import W_NOKICOST
+from ..misc import W_NOKICOST, W_UNKDIST
 from ..__main__ import __version__
 try:
     from xlsxwriter import Workbook
@@ -37,6 +37,7 @@ try:
             sys.path.insert(0, rel_path)
     from kicost.kicost import query_part_info
     from kicost.spreadsheet import create_worksheet, Spreadsheet
+    from kicost.distributors import init_distributor_dict, set_distributors_logger, get_distributors_list
     import kicost.global_vars as kvar
     KICOST_SUPPORT = True
 except ModuleNotFoundError:
@@ -319,6 +320,8 @@ def write_info(cfg, r_info_start, worksheet, column_widths, col1, fmt_info, fmt_
 class Part(object):
     def __init__(self):
         super().__init__()
+        self.datasheet = None
+        self.lifecycle = None
 
 
 def adapt_extra_cost_columns(cfg):
@@ -361,9 +364,44 @@ def apply_join_requests(join, adapted, original):
             adapted[key] = val + append
 
 
+def remove_unknown_distributors(distributors, available, silent):
+    new_distributors = []
+    for d in distributors:
+        d = d.lower()
+        if d not in available:
+            if not silent:
+                logger.warning(W_UNKDIST+'Unknown distributor `{}`'.format(d))
+        else:
+            new_distributors.append(d)
+    return new_distributors
+
+
+def solve_distributors(cfg, silent=True):
+    # List of distributors to scrape
+    available = get_distributors_list()
+    include = remove_unknown_distributors(cfg.distributors, available, silent)
+    exclude = remove_unknown_distributors(cfg.no_distributors, available, silent)
+    # Default is to sort the entries
+    Spreadsheet.SORT_DISTRIBUTORS = True
+    if not include:
+        # All by default
+        dist_list = available
+    else:
+        # Requested to be included
+        dist_list = include
+        # Keep user sorting
+        Spreadsheet.SORT_DISTRIBUTORS = False
+    # Requested to be excluded
+    for d in exclude:
+        dist_list.remove(d)
+    Spreadsheet.DISTRIBUTORS = dist_list
+    return dist_list
+
+
 def create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_subtitle, cfg):
     if not KICOST_SUPPORT:
         logger.warning(W_NOKICOST, 'KiCost sheet requested but failed to load KiCost support')
+        return
     if cfg.debug_level > 2:
         logger.debug("Groups exported to KiCost:")
         for g in groups:
@@ -373,6 +411,9 @@ def create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_s
                 logger.debug(pprint.pformat(c.__dict__))
     # Force KiCost to use our logger
     kvar.logger = logger
+    set_distributors_logger(logger)
+    # Start with a clean list of available distributors
+    init_distributor_dict()
     # Create the projects information structure
     prj_info = [{'title': p.name, 'company': p.sch.company, 'date': p.sch.date} for p in cfg.aggregate]
     # Create the worksheets
@@ -437,8 +478,12 @@ def create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_s
             parts.append(part)
             # Process any "join" request
             apply_join_requests(cfg.join_ce, part.fields, g.fields)
+        # Distributors
+        dist_list = solve_distributors(cfg)
         # Get the prices
-        query_part_info(parts)
+        query_part_info(parts, dist_list)
+        # Distributors again. During `query_part_info` user defined distributors could be added
+        solve_distributors(cfg, silent=False)
         # Create a class to hold the spreadsheet parameters
         ss = Spreadsheet(workbook, ws_names[ws], prj_info)
         wks = ss.wks

kibot/kiplot.py

@@ -12,6 +12,7 @@ Main KiBot code
 import os
 import re
 from sys import exit
+from sys import path as sys_path
 from shutil import which
 from subprocess import run, PIPE
 from glob import glob
@@ -46,6 +47,7 @@ def _import(name, path):
         trace_dump()
         logger.error('Unable to import plug-ins: '+str(e))
         logger.error('Make sure you used `--no-compile` if you used pip for installation')
+        logger.error('Python path: '+str(sys_path))
         exit(WRONG_INSTALL)
 
 

kibot/misc.py

@@ -124,7 +124,7 @@ DNC = {
     "fixed",
 }
 # KiCost distributors
-DISTRIBUTORS = ['digikey', 'farnell', 'mouser', 'newark', 'rs', 'arrow', 'tme', 'lcsc']
+DISTRIBUTORS = ['arrow', 'digikey', 'farnell', 'lcsc', 'mouser', 'newark', 'rs', 'tme']
 DISTRIBUTORS_F = [d+'#' for d in DISTRIBUTORS]
 # ISO ISO4217 currency codes
 # Not all, but the ones we get from the European Central Bank (march 2021)

kibot/out_bom.py

@@ -299,6 +299,10 @@ class BoMOptions(BaseOptions):
             """ Generate the `Source BoM` column using the reference ID instead of the project name """
             self.int_qtys = True
             """ Component quantities are always expressed as integers. Using the ceil() function """
+            self.distributors = Optionable
+            """ [string|list(string)] Include this distributors list. Default is all the available """
+            self.no_distributors = Optionable
+            """ [string|list(string)] Exclude this distributors list. They are removed after computing `distributors` """
         self._format_example = 'CSV'
         super().__init__()
 
@@ -455,6 +459,9 @@ class BoMOptions(BaseOptions):
         # Make sure aggregate is a list
         if isinstance(self.aggregate, type):
             self.aggregate = []
+        # List of distributors
+        self.distributors = Optionable.force_list(self.distributors)
+        self.no_distributors = Optionable.force_list(self.no_distributors)
         # Columns
         valid_columns = self._get_columns()
         (self.columns, self.column_levels, self.column_comments, self.column_rename,

kibot/out_kicost.py

@@ -46,11 +46,9 @@ class KiCostOptions(VariantOptions):
             self.show_cat_url = False
             """ Include the catalogue links in the catalogue code """
             self.distributors = Optionable
-            """ [string|list(string)] Use only this distributors list. Default is all the available.
+            """ [string|list(string)] Include this distributors list. Default is all the available """
-                Not compatible with `no_distributors` option """
             self.no_distributors = Optionable
-            """ [string|list(string)] Use all but this distributors list. Default is use all the available.
+            """ [string|list(string)] Exclude this distributors list. They are removed after computing `distributors` """
-                Not compatible with `distributors` option """
             self.currency = Optionable
             """ [string|list(string)=USD] Currency priority. Use ISO4217 codes (i.e. USD, EUR) """
             self.group_fields = Optionable
@@ -95,8 +93,6 @@ class KiCostOptions(VariantOptions):
             self.output = '%f.%x'
         self.distributors = self._validate_dis(self.distributors)
         self.no_distributors = self._validate_dis(self.no_distributors)
-        if self.distributors and self.no_distributors:
-            raise KiPlotConfigurationError('`distributors` and `no_distributors` are incompatible, choose one')
         self.currency = self._validate_cur(self.currency)
         self.group_fields = Optionable.force_list(self.group_fields)
         self.ignore_fields = Optionable.force_list(self.ignore_fields)

submodules/KiCost

@@ -1 +1 @@
-Subproject commit 81634342361ba5965a3eeab80f7f945a76faa637
+Subproject commit d9aa22aeb2373f5671467887c89d793b6d453e80

tests/reference/5_1_6/KiCost/kibom-variant_2c-bom_dk_mou.csv

@@ -0,0 +1,14 @@
+,,,,KiBot Bill of Materials,,,,,,,,,,,,,,,,
+,,,,Schematic:,kibom-variant_2c,,,,Board Qty:,50,,,,,,,,,,
+,,,,Variant:,default,,,,Unit Cost:,0,,,,,,,,,,
+,,,,Revision:,A,,,,Total Cost:,0,,,,0,0,,,,0,0
+,,,,Date:,2021-04-06,,,,,,,,,,,,,,,
+,,,,KiCad Version:,,,,,,,,,,,,,,,,
+
+Global Part Info,,,,,,,,,,,Mouser,,,,,Digi-Key,,,,
+References,Value,Real value,Tolerancia,Voltage,Footprint,Manufacturer,Manufacturer P/N,Build Quantity,Unit$,Ext$,Avail,Purch,Unit$,Ext$,Cat#,Avail,Purch,Unit$,Ext$,Cat#
+R1 R2,1k,1000,1%,,,Bourns,CR0603-JW-102ELF,100,0,0,110042,,0,0,652CR0603JW102ELF,38625,,0,0,CR0603-JW-102ELFCT-ND
+
+,,,,,,,,,Total Purchase:,0,Buy here,0,,0,,Buy here,0,,0,
+Created:,,,,,,,,,Purchase description:,,,0,,,,,0,,,
+KiCost,,,,,,,,,,,,,,,,,,,,

tests/reference/5_1_6/KiCost/kibom-variant_2c-bom_dk_mou_dnf.csv

@@ -0,0 +1,14 @@
+,,,,KiBot Bill of Materials,,,,,,,,,,,,,,,,
+,,,,Schematic:,kibom-variant_2c,,,,Board Qty:,50,,,,,,,,,,
+,,,,Variant:,default,,,,Unit Cost:,0,,,,,,,,,,
+,,,,Revision:,A,,,,Total Cost:,0,,,,0,0,,,,0,0
+,,,,Date:,2021-04-06,,,,,,,,,,,,,,,
+,,,,KiCad Version:,,,,,,,,,,,,,,,,
+
+Global Part Info,,,,,,,,,,,Mouser,,,,,Digi-Key,,,,
+References,Value,Real value,Tolerancia,Voltage,Footprint,Manufacturer,Manufacturer P/N,Build Quantity,Unit$,Ext$,Avail,Purch,Unit$,Ext$,Cat#,Avail,Purch,Unit$,Ext$,Cat#
+C1 C2,1nF, 1000pF,20%,50 V 100 V,,Samsung,CL10B102KC8NNNC,100,0,0,NonStk,,0,0,187CL10B102KC8NNNC,NonStk,,0,0,1276-1131-1-ND
+
+,,,,,,,,,Total Purchase:,0,Buy here,0,,0,,Buy here,0,,0,
+Created:,,,,,,,,,Purchase description:,,,0,,,,,0,,,
+KiCost,,,,,,,,,,,,,,,,,,,,

tests/test_plot/test_kicost.py

@@ -75,3 +75,17 @@ def test_kicost_bom_simple(test_dir):
     convert2csv(ctx.get_out_path(output), sheet='Costs (DNF)')
     ctx.compare_txt(csv, output[:-5]+'_dnf.csv')
     ctx.clean_up()
+
+
+def test_kicost_bom_sel_dist_1(test_dir):
+    prj = 'kibom-variant_2c'
+    ctx = context.TestContextSCH(test_dir, 'test_kicost_bom_sel_dist_1', prj, 'int_bom_kicost_sel_dist_1_xlsx', OUT_DIR)
+    ctx.run(kicost=True)  # , extra_debug=True
+    output = op.join(OUT_DIR, prj+'-bom.xlsx')
+    ctx.expect_out_file(output)
+    convert2csv(ctx.get_out_path(output), sheet='Costs')
+    csv = output[:-4]+'csv'
+    ctx.compare_txt(csv, output[:-5]+'_dk_mou.csv')
+    convert2csv(ctx.get_out_path(output), sheet='Costs (DNF)')
+    ctx.compare_txt(csv, output[:-5]+'_dk_mou_dnf.csv')
+    ctx.clean_up()

tests/yaml_samples/int_bom_kicost_sel_dist_1_xlsx.kibot.yaml

@@ -0,0 +1,40 @@
+# Example KiBot config file
+kibot:
+  version: 1
+
+outputs:
+  - name: 'bom_internal'
+    comment: "Bill of Materials in HTML format"
+    type: bom
+    dir: KiCost
+    options:
+      use_alt: true
+      number: 50
+      columns:
+        - References
+        - Part
+        - Value
+        - Quantity Per PCB
+        - field: manf
+          name: Manufacturer
+        - field: manf#
+          name: Manufacturer P/N
+        - field: digikey#
+          level: 1
+          comment: 'Code used to buy the part at Digi-Key'
+      cost_extra_columns:
+         - field: 'Resistance'
+           name: 'Real value'
+           comment: 'Verdadero valor'
+           join: 'Capacitance'
+           level: 1
+         - field: 'Tolerance'
+           name: Tolerancia
+           comment: 'La tolerancia'
+           level: 1
+         - 'Voltage'
+      distributors:
+         - Mouser
+         - digikey
+      xlsx:
+        kicost: true