Added distributors selection to KiCost+KiBot

2021-04-28 12:13:12 -03:00 · 2021-04-28 12:13:12 -03:00 · 7c23381adb
parent af18022c2d
commit 7c23381adb
13 changed files with 157 additions and 18 deletions
--- a/README.md
+++ b/README.md
@ -678,6 +678,7 @@ Next time you need this list just use an alias, like this:
            - `hide_stats_info`: [boolean=false] Hide statistics information.
            - `quote_all`: [boolean=false] Enclose all values using double quotes.
            - `separator`: [string=','] CSV Separator. TXT and TSV always use tab as delimiter.
+        - `distributors`: [string|list(string)] Include this distributors list. Default is all the available.
        - `dnc_filter`: [string|list(string)='_kibom_dnc'] Name of the filter to mark components as 'Do Not Change'.
                        The default filter marks components with a DNC value or DNC in the Config field.
        - `dnf_filter`: [string|list(string)='_kibom_dnf'] Name of the filter to mark components as 'Do Not Fit'.
@ -713,6 +714,7 @@ Next time you need this list just use an alias, like this:
        - `no_conflict`: [list(string)] List of fields where we tolerate conflicts.
                         Use it to avoid undesired warnings.
                         By default the field indicated in `fit_field` and the field `part` are excluded.
+        - `no_distributors`: [string|list(string)] Exclude this distributors list. They are removed after computing `distributors`.
        - `normalize_locale`: [boolean=false] When normalizing values use the locale decimal point.
        - `normalize_values`: [boolean=false] Try to normalize the R, L and C values, producing uniform units and prefixes.
        - `number`: [number=1] Number of boards to build (components multiplier).
@ -1123,8 +1125,7 @@ Next time you need this list just use an alias, like this:
            - `file`: [string=''] Name of the XML to aggregate.
            - `variant`: [string=' '] Variant for this project.
        - `currency`: [string|list(string)=USD] Currency priority. Use ISO4217 codes (i.e. USD, EUR).
-        - `distributors`: [string|list(string)] Use only this distributors list. Default is all the available.
-                          Not compatible with `no_distributors` option.
+        - `distributors`: [string|list(string)] Include this distributors list. Default is all the available.
        - `dnf_filter`: [string|list(string)=''] Name of the filter to mark components as not fitted.
                        A short-cut to use for simple cases where a variant is an overkill.
                        Internal variants and filters are currently ignored.
@ -1134,8 +1135,7 @@ Next time you need this list just use an alias, like this:
        - `ignore_fields`: [string|list(string)] List of fields to be ignored.
        - `kicost_variant`: [string=''] Regular expression to match the variant field (KiCost option, not internal variants).
        - `no_collapse`: [boolean=false] Do not collapse the part references (collapse=R1-R4).
-        - `no_distributors`: [string|list(string)] Use all but this distributors list. Default is use all the available.
-                             Not compatible with `distributors` option.
+        - `no_distributors`: [string|list(string)] Exclude this distributors list. They are removed after computing `distributors`.
        - `no_price`: [boolean=false] Do not look for components price. For testing purposes.
        - `output`: [string='%f-%i%v.%x'] Filename for the output (%i=kicost, %x=xlsx). Affected by global options.
        - `show_cat_url`: [boolean=false] Include the catalogue links in the catalogue code.
--- a/docs/samples/generic_plot.kibot.yaml
+++ b/docs/samples/generic_plot.kibot.yaml
@ -109,6 +109,8 @@ outputs:
        quote_all: false
        # [string=','] CSV Separator. TXT and TSV always use tab as delimiter
        separator: ','
+      # [string|list(string)] Include this distributors list. Default is all the available
+      distributors:
      # [string|list(string)='_kibom_dnc'] Name of the filter to mark components as 'Do Not Change'.
      # The default filter marks components with a DNC value or DNC in the Config field
      dnc_filter: '_kibom_dnc'
@ -166,6 +168,8 @@ outputs:
      # Use it to avoid undesired warnings.
      # By default the field indicated in `fit_field` and the field `part` are excluded
      no_conflict: ['Config', 'Part']
+      # [string|list(string)] Exclude this distributors list. They are removed after computing `distributors`
+      no_distributors:
      # [boolean=false] When normalizing values use the locale decimal point
      normalize_locale: false
      # [boolean=false] Try to normalize the R, L and C values, producing uniform units and prefixes
@ -707,8 +711,7 @@ outputs:
          variant: ' '
      # [string|list(string)=USD] Currency priority. Use ISO4217 codes (i.e. USD, EUR)
      currency: USD
-      # [string|list(string)] Use only this distributors list. Default is all the available.
-      # Not compatible with `no_distributors` option
+      # [string|list(string)] Include this distributors list. Default is all the available
      distributors:
      # [string|list(string)=''] Name of the filter to mark components as not fitted.
      # A short-cut to use for simple cases where a variant is an overkill.
@ -725,8 +728,7 @@ outputs:
      kicost_variant: ''
      # [boolean=false] Do not collapse the part references (collapse=R1-R4)
      no_collapse: false
-      # [string|list(string)] Use all but this distributors list. Default is use all the available.
-      # Not compatible with `distributors` option
+      # [string|list(string)] Exclude this distributors list. They are removed after computing `distributors`
      no_distributors:
      # [boolean=false] Do not look for components price. For testing purposes
      no_price: false
--- a/kibot/bom/xlsx_writer.py
+++ b/kibot/bom/xlsx_writer.py
@ -17,7 +17,7 @@ from base64 import b64decode
 from .columnlist import ColumnList
 from .kibot_logo import KIBOT_LOGO
 from .. import log
-from ..misc import W_NOKICOST
+from ..misc import W_NOKICOST, W_UNKDIST
 from ..__main__ import __version__
 try:
    from xlsxwriter import Workbook
@ -37,6 +37,7 @@ try:
            sys.path.insert(0, rel_path)
    from kicost.kicost import query_part_info
    from kicost.spreadsheet import create_worksheet, Spreadsheet
+    from kicost.distributors import init_distributor_dict, set_distributors_logger, get_distributors_list
    import kicost.global_vars as kvar
    KICOST_SUPPORT = True
 except ModuleNotFoundError:
@ -319,6 +320,8 @@ def write_info(cfg, r_info_start, worksheet, column_widths, col1, fmt_info, fmt_
 class Part(object):
    def __init__(self):
        super().__init__()
+        self.datasheet = None
+        self.lifecycle = None


 def adapt_extra_cost_columns(cfg):
@ -361,9 +364,44 @@ def apply_join_requests(join, adapted, original):
            adapted[key] = val + append


+def remove_unknown_distributors(distributors, available, silent):
+    new_distributors = []
+    for d in distributors:
+        d = d.lower()
+        if d not in available:
+            if not silent:
+                logger.warning(W_UNKDIST+'Unknown distributor `{}`'.format(d))
+        else:
+            new_distributors.append(d)
+    return new_distributors
+
+
+def solve_distributors(cfg, silent=True):
+    # List of distributors to scrape
+    available = get_distributors_list()
+    include = remove_unknown_distributors(cfg.distributors, available, silent)
+    exclude = remove_unknown_distributors(cfg.no_distributors, available, silent)
+    # Default is to sort the entries
+    Spreadsheet.SORT_DISTRIBUTORS = True
+    if not include:
+        # All by default
+        dist_list = available
+    else:
+        # Requested to be included
+        dist_list = include
+        # Keep user sorting
+        Spreadsheet.SORT_DISTRIBUTORS = False
+    # Requested to be excluded
+    for d in exclude:
+        dist_list.remove(d)
+    Spreadsheet.DISTRIBUTORS = dist_list
+    return dist_list
+
+
 def create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_subtitle, cfg):
    if not KICOST_SUPPORT:
        logger.warning(W_NOKICOST, 'KiCost sheet requested but failed to load KiCost support')
+        return
    if cfg.debug_level > 2:
        logger.debug("Groups exported to KiCost:")
        for g in groups:
@ -373,6 +411,9 @@ def create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_s
                logger.debug(pprint.pformat(c.__dict__))
    # Force KiCost to use our logger
    kvar.logger = logger
+    set_distributors_logger(logger)
+    # Start with a clean list of available distributors
+    init_distributor_dict()
    # Create the projects information structure
    prj_info = [{'title': p.name, 'company': p.sch.company, 'date': p.sch.date} for p in cfg.aggregate]
    # Create the worksheets
@ -437,8 +478,12 @@ def create_kicost_sheet(workbook, groups, image_data, fmt_title, fmt_info, fmt_s
            parts.append(part)
            # Process any "join" request
            apply_join_requests(cfg.join_ce, part.fields, g.fields)
+        # Distributors
+        dist_list = solve_distributors(cfg)
        # Get the prices
-        query_part_info(parts)
+        query_part_info(parts, dist_list)
+        # Distributors again. During `query_part_info` user defined distributors could be added
+        solve_distributors(cfg, silent=False)
        # Create a class to hold the spreadsheet parameters
        ss = Spreadsheet(workbook, ws_names[ws], prj_info)
        wks = ss.wks
--- a/kibot/kiplot.py
+++ b/kibot/kiplot.py
@ -12,6 +12,7 @@ Main KiBot code
 import os
 import re
 from sys import exit
+from sys import path as sys_path
 from shutil import which
 from subprocess import run, PIPE
 from glob import glob
@ -46,6 +47,7 @@ def _import(name, path):
        trace_dump()
        logger.error('Unable to import plug-ins: '+str(e))
        logger.error('Make sure you used `--no-compile` if you used pip for installation')
+        logger.error('Python path: '+str(sys_path))
        exit(WRONG_INSTALL)


--- a/kibot/misc.py
+++ b/kibot/misc.py
@ -124,7 +124,7 @@ DNC = {
    "fixed",
 }
 # KiCost distributors
-DISTRIBUTORS = ['digikey', 'farnell', 'mouser', 'newark', 'rs', 'arrow', 'tme', 'lcsc']
+DISTRIBUTORS = ['arrow', 'digikey', 'farnell', 'lcsc', 'mouser', 'newark', 'rs', 'tme']
 DISTRIBUTORS_F = [d+'#' for d in DISTRIBUTORS]
 # ISO ISO4217 currency codes
 # Not all, but the ones we get from the European Central Bank (march 2021)
--- a/kibot/out_bom.py
+++ b/kibot/out_bom.py
@ -299,6 +299,10 @@ class BoMOptions(BaseOptions):
            """ Generate the `Source BoM` column using the reference ID instead of the project name """
            self.int_qtys = True
            """ Component quantities are always expressed as integers. Using the ceil() function """
+            self.distributors = Optionable
+            """ [string|list(string)] Include this distributors list. Default is all the available """
+            self.no_distributors = Optionable
+            """ [string|list(string)] Exclude this distributors list. They are removed after computing `distributors` """
        self._format_example = 'CSV'
        super().__init__()

@ -455,6 +459,9 @@ class BoMOptions(BaseOptions):
        # Make sure aggregate is a list
        if isinstance(self.aggregate, type):
            self.aggregate = []
+        # List of distributors
+        self.distributors = Optionable.force_list(self.distributors)
+        self.no_distributors = Optionable.force_list(self.no_distributors)
        # Columns
        valid_columns = self._get_columns()
        (self.columns, self.column_levels, self.column_comments, self.column_rename,
--- a/kibot/out_kicost.py
+++ b/kibot/out_kicost.py
@ -46,11 +46,9 @@ class KiCostOptions(VariantOptions):
            self.show_cat_url = False
            """ Include the catalogue links in the catalogue code """
            self.distributors = Optionable
-            """ [string|list(string)] Use only this distributors list. Default is all the available.
-                Not compatible with `no_distributors` option """
+            """ [string|list(string)] Include this distributors list. Default is all the available """
            self.no_distributors = Optionable
-            """ [string|list(string)] Use all but this distributors list. Default is use all the available.
-                Not compatible with `distributors` option """
+            """ [string|list(string)] Exclude this distributors list. They are removed after computing `distributors` """
            self.currency = Optionable
            """ [string|list(string)=USD] Currency priority. Use ISO4217 codes (i.e. USD, EUR) """
            self.group_fields = Optionable
@ -95,8 +93,6 @@ class KiCostOptions(VariantOptions):
            self.output = '%f.%x'
        self.distributors = self._validate_dis(self.distributors)
        self.no_distributors = self._validate_dis(self.no_distributors)
-        if self.distributors and self.no_distributors:
-            raise KiPlotConfigurationError('`distributors` and `no_distributors` are incompatible, choose one')
        self.currency = self._validate_cur(self.currency)
        self.group_fields = Optionable.force_list(self.group_fields)
        self.ignore_fields = Optionable.force_list(self.ignore_fields)
--- a/submodules/KiCost
+++ b/submodules/KiCost
@ -1 +1 @@
-Subproject commit 81634342361ba5965a3eeab80f7f945a76faa637
+Subproject commit d9aa22aeb2373f5671467887c89d793b6d453e80
--- a/tests/data/kitspace_queries.txt
+++ b/tests/data/kitspace_queries.txt
--- a/tests/reference/5_1_6/KiCost/kibom-variant_2c-bom_dk_mou.csv
+++ b/tests/reference/5_1_6/KiCost/kibom-variant_2c-bom_dk_mou.csv
@ -0,0 +1,14 @@
+,,,,KiBot Bill of Materials,,,,,,,,,,,,,,,,
+,,,,Schematic:,kibom-variant_2c,,,,Board Qty:,50,,,,,,,,,,
+,,,,Variant:,default,,,,Unit Cost:,0,,,,,,,,,,
+,,,,Revision:,A,,,,Total Cost:,0,,,,0,0,,,,0,0
+,,,,Date:,2021-04-06,,,,,,,,,,,,,,,
+,,,,KiCad Version:,,,,,,,,,,,,,,,,
+
+Global Part Info,,,,,,,,,,,Mouser,,,,,Digi-Key,,,,
+References,Value,Real value,Tolerancia,Voltage,Footprint,Manufacturer,Manufacturer P/N,Build Quantity,Unit$,Ext$,Avail,Purch,Unit$,Ext$,Cat#,Avail,Purch,Unit$,Ext$,Cat#
+R1 R2,1k,1000,1%,,,Bourns,CR0603-JW-102ELF,100,0,0,110042,,0,0,652CR0603JW102ELF,38625,,0,0,CR0603-JW-102ELFCT-ND
+
+,,,,,,,,,Total Purchase:,0,Buy here,0,,0,,Buy here,0,,0,
+Created:,,,,,,,,,Purchase description:,,,0,,,,,0,,,
+KiCost,,,,,,,,,,,,,,,,,,,,
--- a/tests/reference/5_1_6/KiCost/kibom-variant_2c-bom_dk_mou_dnf.csv
+++ b/tests/reference/5_1_6/KiCost/kibom-variant_2c-bom_dk_mou_dnf.csv
@ -0,0 +1,14 @@
+,,,,KiBot Bill of Materials,,,,,,,,,,,,,,,,
+,,,,Schematic:,kibom-variant_2c,,,,Board Qty:,50,,,,,,,,,,
+,,,,Variant:,default,,,,Unit Cost:,0,,,,,,,,,,
+,,,,Revision:,A,,,,Total Cost:,0,,,,0,0,,,,0,0
+,,,,Date:,2021-04-06,,,,,,,,,,,,,,,
+,,,,KiCad Version:,,,,,,,,,,,,,,,,
+
+Global Part Info,,,,,,,,,,,Mouser,,,,,Digi-Key,,,,
+References,Value,Real value,Tolerancia,Voltage,Footprint,Manufacturer,Manufacturer P/N,Build Quantity,Unit$,Ext$,Avail,Purch,Unit$,Ext$,Cat#,Avail,Purch,Unit$,Ext$,Cat#
+C1 C2,1nF, 1000pF,20%,50 V 100 V,,Samsung,CL10B102KC8NNNC,100,0,0,NonStk,,0,0,187CL10B102KC8NNNC,NonStk,,0,0,1276-1131-1-ND
+
+,,,,,,,,,Total Purchase:,0,Buy here,0,,0,,Buy here,0,,0,
+Created:,,,,,,,,,Purchase description:,,,0,,,,,0,,,
+KiCost,,,,,,,,,,,,,,,,,,,,
--- a/tests/test_plot/test_kicost.py
+++ b/tests/test_plot/test_kicost.py
@ -75,3 +75,17 @@ def test_kicost_bom_simple(test_dir):
    convert2csv(ctx.get_out_path(output), sheet='Costs (DNF)')
    ctx.compare_txt(csv, output[:-5]+'_dnf.csv')
    ctx.clean_up()
+
+
+def test_kicost_bom_sel_dist_1(test_dir):
+    prj = 'kibom-variant_2c'
+    ctx = context.TestContextSCH(test_dir, 'test_kicost_bom_sel_dist_1', prj, 'int_bom_kicost_sel_dist_1_xlsx', OUT_DIR)
+    ctx.run(kicost=True)  # , extra_debug=True
+    output = op.join(OUT_DIR, prj+'-bom.xlsx')
+    ctx.expect_out_file(output)
+    convert2csv(ctx.get_out_path(output), sheet='Costs')
+    csv = output[:-4]+'csv'
+    ctx.compare_txt(csv, output[:-5]+'_dk_mou.csv')
+    convert2csv(ctx.get_out_path(output), sheet='Costs (DNF)')
+    ctx.compare_txt(csv, output[:-5]+'_dk_mou_dnf.csv')
+    ctx.clean_up()
--- a/tests/yaml_samples/int_bom_kicost_sel_dist_1_xlsx.kibot.yaml
+++ b/tests/yaml_samples/int_bom_kicost_sel_dist_1_xlsx.kibot.yaml
@ -0,0 +1,40 @@
+# Example KiBot config file
+kibot:
+  version: 1
+
+outputs:
+  - name: 'bom_internal'
+    comment: "Bill of Materials in HTML format"
+    type: bom
+    dir: KiCost
+    options:
+      use_alt: true
+      number: 50
+      columns:
+        - References
+        - Part
+        - Value
+        - Quantity Per PCB
+        - field: manf
+          name: Manufacturer
+        - field: manf#
+          name: Manufacturer P/N
+        - field: digikey#
+          level: 1
+          comment: 'Code used to buy the part at Digi-Key'
+      cost_extra_columns:
+         - field: 'Resistance'
+           name: 'Real value'
+           comment: 'Verdadero valor'
+           join: 'Capacitance'
+           level: 1
+         - field: 'Tolerance'
+           name: Tolerancia
+           comment: 'La tolerancia'
+           level: 1
+         - 'Voltage'
+      distributors:
+         - Mouser
+         - digikey
+      xlsx:
+        kicost: true