diff --git a/kibot/bom/bom.py b/kibot/bom/bom.py index 72a457e7..1a4f1f14 100644 --- a/kibot/bom/bom.py +++ b/kibot/bom/bom.py @@ -263,37 +263,6 @@ class ComponentGroup(object): return row -def test_reg_exclude(cfg, c): - """ Test if this part should be included, based on any regex expressions provided in the preferences """ - for reg in cfg.exclude_any: - field_value = c.get_field_value(reg.column) - if reg.regex.search(field_value): - if cfg.debug_level > 1: - logger.debug("Excluding '{ref}': Field '{field}' ({value}) matched '{re}'".format( - ref=c.ref, field=reg.column, value=field_value, re=reg.regex)) - # Found a match - return True - # Default, could not find any matches - return False - - -def test_reg_include(cfg, c): - """ Reject components that doesn't match the provided regex. - So we include only the components that matches any of the regexs. """ - if not cfg.include_only: # Nothing to match against, means include all - return True - for reg in cfg.include_only: - field_value = c.get_field_value(reg.column) - if reg.regex.search(field_value): - if cfg.debug_level > 1: - logger.debug("Including '{ref}': Field '{field}' ({value}) matched '{re}'".format( - ref=c.ref, field=reg.column, value=field_value, re=reg.regex)) - # Found a match - return True - # Default, could not find a match - return False - - def get_value_sort(comp): """ Try to better sort R, L and C components """ res = comp.value_sort @@ -325,12 +294,8 @@ def group_components(cfg, components): groups = [] # Iterate through each component, and test whether a group for these already exists for c in components: - if cfg.test_regex: - # Skip components if they do not meet regex requirements - if not test_reg_include(cfg, c): - continue - if test_reg_exclude(cfg, c): - continue + if not c.in_bom: # Skip components marked as excluded from BoM + continue # Cache the value used to sort if c.ref_prefix in RLC_PREFIX and c.value.lower() not in DNF: c.value_sort = comp_match(c.value, c.ref_prefix) @@ -390,7 +355,12 @@ def group_components(cfg, components): def do_bom(file_name, ext, comps, cfg): - # Solve `fixed` and `fitted` attributes for all components + # Apply all the filters + for c in comps: + c.in_bom = cfg.exclude_filter.filter(c) + c.fitted = cfg.dnf_filter.filter(c) + c.fixed = cfg.dnc_filter.filter(c) + # Apply the variant cfg.variant.filter(comps) # Group components according to group_fields groups = group_components(cfg, comps) diff --git a/kibot/out_bom.py b/kibot/out_bom.py index 615e3866..1c557fb2 100644 --- a/kibot/out_bom.py +++ b/kibot/out_bom.py @@ -8,7 +8,6 @@ Internal BoM (Bill of Materials) output for KiBot. This is somehow compatible with KiBoM. """ import os -from re import compile, IGNORECASE from .gs import GS from .optionable import Optionable, BaseOptions from .registrable import RegOutput @@ -17,6 +16,7 @@ from .macros import macros, document, output_class # noqa: F401 from .bom.columnlist import ColumnList, BoMError from .bom.bom import do_bom from .var_kibom import KiBoM +from .fil_generic import Generic from . import log logger = log.get_logger(__name__) @@ -30,26 +30,6 @@ DEFAULT_ALIASES = [['r', 'r_small', 'res', 'resistor'], ] -# String matches for marking a component as "do not fit" -class BoMRegex(Optionable): - """ Implements the pair column/regex """ - def __init__(self): - super().__init__() - self._unkown_is_error = True - with document: - self.column = '' - """ Name of the column to apply the regular expression """ - self.regex = '' - """ Regular expression to match """ - self.field = None - """ {column} """ - self.regexp = None - """ {regex} """ - -# def __str__(self): -# return self.column+'\t'+self.regex - - class BoMColumns(Optionable): """ Information for the BoM columns """ def __init__(self): @@ -194,15 +174,15 @@ class GroupFields(Optionable): class BoMOptions(BaseOptions): - DEFAULT_EXCLUDE = [[ColumnList.COL_REFERENCE, '^TP[0-9]*'], - [ColumnList.COL_REFERENCE, '^FID'], - [ColumnList.COL_PART, 'mount.*hole'], - [ColumnList.COL_PART, 'solder.*bridge'], - [ColumnList.COL_PART, 'solder.*jump'], - [ColumnList.COL_PART, 'test.*point'], - [ColumnList.COL_FP, 'test.*point'], - [ColumnList.COL_FP, 'mount.*hole'], - [ColumnList.COL_FP, 'fiducial'], + DEFAULT_EXCLUDE = [{'column': ColumnList.COL_REFERENCE, 'regex': '^TP[0-9]*'}, + {'column': ColumnList.COL_REFERENCE, 'regex': '^FID'}, + {'column': ColumnList.COL_PART, 'regex': 'mount.*hole'}, + {'column': ColumnList.COL_PART, 'regex': 'solder.*bridge'}, + {'column': ColumnList.COL_PART, 'regex': 'solder.*jump'}, + {'column': ColumnList.COL_PART, 'regex': 'test.*point'}, + {'column': ColumnList.COL_FP, 'regex': 'test.*point'}, + {'column': ColumnList.COL_FP, 'regex': 'mount.*hole'}, + {'column': ColumnList.COL_FP, 'regex': 'fiducial'}, ] def __init__(self): @@ -220,18 +200,38 @@ class BoMOptions(BaseOptions): # Equivalent to KiBoM INI: self.ignore_dnf = True """ Exclude DNF (Do Not Fit) components """ + self.fit_field = 'Config' + """ Field name used for internal filters """ self.use_alt = False """ Print grouped references in the alternate compressed style eg: R1-R7,R18 """ + self.columns = BoMColumns + """ [list(dict)|list(string)] List of columns to display. + Can be just the name of the field """ + self.normalize_values = False + """ Try to normalize the R, L and C values, producing uniform units and prefixes """ + self.normalize_locale = False + """ When normalizing values use the locale decimal point """ + self.html = BoMHTML + """ [dict] Options for the HTML format """ + self.xlsx = BoMXLSX + """ [dict] Options for the XLSX format """ + self.csv = BoMCSV + """ [dict] Options for the CSV, TXT and TSV formats """ + # * Filters + self.exclude_filter = '_mechanical' + """ Name of the filter to exclude components from BoM processing. + The default filter excludes test points, fiducial marks, mounting holes, etc """ + self.dnf_filter = '_kibom_dnf' + """ Name of the filter to mark components as 'Do Not Fit'. + The default filter marks components with a DNF value or DNF in the Config field """ + self.dnc_filter = '_kibom_dnc' + """ Name of the filter to mark components as 'Do Not Change'. + The default filter marks components with a DNC value or DNC in the Config field """ + # * Grouping criteria self.group_connectors = True """ Connectors with the same footprints will be grouped together, independent of the name of the connector """ - self.test_regex = True - """ Each component group will be tested against a number of regular-expressions - (see `include_only` and `exclude_any`) """ self.merge_blank_fields = True """ Component groups with blank fields will be merged into the most compatible group, where possible """ - self.fit_field = 'Config' - """ Field name used to determine if a particular part is to be fitted (also DNC, not for variants). - This value is used only when no variants are specified """ self.group_fields = GroupFields """ [list(string)] List of fields used for sorting individual components into groups. Components which match (comparing *all* fields) will be grouped together. @@ -248,47 +248,6 @@ class BoMOptions(BaseOptions): - ['sw', 'switch'] - ['zener', 'zenersmall'] - ['d', 'diode', 'd_small'] """ - self.include_only = BoMRegex - """ [list(dict)] A series of regular expressions used to select included parts. - If there are any regex defined here, only components that match against ANY of them will be included. - Column names are case-insensitive. - If empty all the components are included """ - self.exclude_any = BoMRegex - """ [list(dict)] A series of regular expressions used to exclude parts. - If a component matches ANY of these, it will be excluded. - Column names are case-insensitive. - If empty the following list is used: - - column: References - ..regex: '^TP[0-9]*' - - column: References - ..regex: '^FID' - - column: Part - ..regex: 'mount.*hole' - - column: Part - ..regex: 'solder.*bridge' - - column: Part - ..regex: 'solder.*jump' - - column: Part - ..regex: 'test.*point' - - column: Footprint - ..regex: 'test.*point' - - column: Footprint - ..regex: 'mount.*hole' - - column: Footprint - ..regex: 'fiducial' """ - self.columns = BoMColumns - """ [list(dict)|list(string)] List of columns to display. - Can be just the name of the field """ - self.normalize_values = False - """ Try to normalize the R, L and C values, producing uniform units and prefixes """ - self.normalize_locale = False - """ When normalizing values use the locale decimal point """ - self.html = BoMHTML - """ [dict] Options for the HTML format """ - self.xlsx = BoMXLSX - """ [dict] Options for the XLSX format """ - self.csv = BoMCSV - """ [dict] Options for the CSV, TXT and TSV formats """ super().__init__() @staticmethod @@ -311,20 +270,12 @@ class BoMOptions(BaseOptions): # Explicit selection return self.format.lower() - @staticmethod - def _fix_ref_field(field): - """ References -> Reference """ - col = field.lower() - if col == ColumnList.COL_REFERENCE_L: - col = col[:-1] - return col - def _normalize_variant(self): """ Replaces the name of the variant by an object handling it. """ if self.variant: - if self.variant not in RegOutput._def_variants: + if not RegOutput.is_variant(self.variant): raise KiPlotConfigurationError("Unknown variant name `{}`".format(self.variant)) - self.variant = RegOutput._def_variants[self.variant] + self.variant = RegOutput.get_variant(self.variant) else: # If no variant is specified use the KiBoM variant class with basic functionality self.variant = KiBoM() @@ -332,6 +283,44 @@ class BoMOptions(BaseOptions): self.variant.variant = [] self.variant.name = 'default' + def _solve_exclude_filter(self): + """ Check we have a valid exclude filter. Create it if needed. """ + if not RegOutput.is_filter(self.exclude_filter): + if self.exclude_filter == '_mechanical': + o = Generic() + o_tree = {'name': '_mechanical', 'type': 'generic', 'comment': 'Internal default mechanical filter'} + o_tree['exclude_any'] = BoMOptions.DEFAULT_EXCLUDE + o.set_tree(o_tree) + o.config() + RegOutput.add_filter(o) + self.exclude_filter = o + return + raise KiPlotConfigurationError("Unknown filter `{}` used for `exclude_filter`".format(self.exclude_filter)) + self.exclude_filter = RegOutput.get_filter(self.exclude_filter) + + def _solve_dnx_filter(self, name, type, invert=False): + real_name = name + if real_name == '_kibom_'+type: + # Allow different internal filters using different config fields + real_name += '_'+self.fit_field + if not RegOutput.is_filter(real_name): + o = Generic() + o_tree = {'name': real_name, 'type': 'generic'} + o_tree['comment'] = 'Internal KiBoM '+type.upper()+' filter ('+self.fit_field+')' + o_tree['config_field'] = self.fit_field + o_tree['exclude_value'] = True + o_tree['exclude_config'] = True + o_tree['keys'] = type+'_list' + if invert: + o_tree['invert'] = True + o.set_tree(o_tree) + o.config() + RegOutput.add_filter(o) + return o + if not RegOutput.is_filter(real_name): + raise KiPlotConfigurationError("Unknown filter `{}` used for `{}_filter`".format(real_name, type)) + return RegOutput.get_filter(real_name) + def config(self): super().config() self.format = self._guess_format() @@ -359,28 +348,16 @@ class BoMOptions(BaseOptions): # component_aliases if isinstance(self.component_aliases, type): self.component_aliases = DEFAULT_ALIASES - # include_only - if isinstance(self.include_only, type): - self.include_only = None - else: - for r in self.include_only: - r.regex = compile(r.regex, flags=IGNORECASE) - # exclude_any - if isinstance(self.exclude_any, type): - self.exclude_any = [] - for r in BoMOptions.DEFAULT_EXCLUDE: - o = BoMRegex() - o.column = self._fix_ref_field(r[0]) - o.regex = compile(r[1], flags=IGNORECASE) - self.exclude_any.append(o) - else: - for r in self.exclude_any: - r.column = self._fix_ref_field(r.column) - r.regex = compile(r.regex, flags=IGNORECASE) - # Make the config field name lowercase - self.fit_field = self.fit_field.lower() + # exclude_filter + self._solve_exclude_filter() + # dnf_filter + self.dnf_filter = self._solve_dnx_filter(self.dnf_filter, 'dnf') + # dnc_filter + self.dnc_filter = self._solve_dnx_filter(self.dnc_filter, 'dnc', True) # Variants, make it an object self._normalize_variant() + # Field names are handled in lowercase + self.fit_field = self.fit_field.lower() # Columns self.column_rename = {} self.join = [] diff --git a/tests/test_plot/test_int_bom.py b/tests/test_plot/test_int_bom.py index 2706dd87..8ff12597 100644 --- a/tests/test_plot/test_int_bom.py +++ b/tests/test_plot/test_int_bom.py @@ -915,19 +915,19 @@ def test_int_bom_include_only(): ctx.clean_up() -def test_int_bom_no_test_regex(): - prj = 'kibom-test' - ext = 'csv' - ctx = context.TestContextSCH('test_int_bom_simple_csv', prj, 'int_bom_no_include_only', BOM_DIR) - ctx.run() - out = prj + '-bom.' + ext - rows, header, info = ctx.load_csv(out) - assert header == KIBOM_TEST_HEAD - ref_column = header.index(REF_COLUMN_NAME) - qty_column = header.index(QTY_COLUMN_NAME) - check_kibom_test_netlist(rows, ref_column, KIBOM_TEST_GROUPS, KIBOM_TEST_EXCLUDE, KIBOM_TEST_COMPONENTS) - check_dnc(rows, 'R7', ref_column, qty_column) - ctx.clean_up() +# def test_int_bom_no_test_regex(): +# prj = 'kibom-test' +# ext = 'csv' +# ctx = context.TestContextSCH('test_int_bom_simple_csv', prj, 'int_bom_no_include_only', BOM_DIR) +# ctx.run() +# out = prj + '-bom.' + ext +# rows, header, info = ctx.load_csv(out) +# assert header == KIBOM_TEST_HEAD +# ref_column = header.index(REF_COLUMN_NAME) +# qty_column = header.index(QTY_COLUMN_NAME) +# check_kibom_test_netlist(rows, ref_column, KIBOM_TEST_GROUPS, KIBOM_TEST_EXCLUDE, KIBOM_TEST_COMPONENTS) +# check_dnc(rows, 'R7', ref_column, qty_column) +# ctx.clean_up() def test_int_bom_sub_sheet_alt(): diff --git a/tests/yaml_samples/int_bom_exclude_any.kibot.yaml b/tests/yaml_samples/int_bom_exclude_any.kibot.yaml index 24c0fd8b..d5e3706d 100644 --- a/tests/yaml_samples/int_bom_exclude_any.kibot.yaml +++ b/tests/yaml_samples/int_bom_exclude_any.kibot.yaml @@ -2,26 +2,34 @@ kibot: version: 1 +filters: + - name: 'exclude_any' + type: 'generic' + comment: 'Almost same as KiBoM, no fiducial' + exclude_any: + - column: References + regex: '^TP[0-9]*' + - column: References + regex: '^FID' + - column: Part + regex: 'mount.*hole' + - column: Part + regex: 'solder.*bridge' + - column: Part + regex: 'solder.*jump' + - column: Part + regex: 'test.*point' + - column: Footprint + regex: 'test.*point' + - column: Footprint + regex: 'mount.*hole' + + outputs: - name: 'bom_internal' comment: "Bill of Materials in CSV format" type: bom dir: BoM options: - exclude_any: - - column: References - regex: '^TP[0-9]*' - - column: References - regex: '^FID' - - column: Part - regex: 'mount.*hole' - - column: Part - regex: 'solder.*bridge' - - column: Part - regex: 'solder.*jump' - - column: Part - regex: 'test.*point' - - column: Footprint - regex: 'test.*point' - - column: Footprint - regex: 'mount.*hole' + exclude_filter: 'exclude_any' + diff --git a/tests/yaml_samples/int_bom_include_only.kibot.yaml b/tests/yaml_samples/int_bom_include_only.kibot.yaml index b1515fec..b8520ab2 100644 --- a/tests/yaml_samples/int_bom_include_only.kibot.yaml +++ b/tests/yaml_samples/int_bom_include_only.kibot.yaml @@ -2,12 +2,18 @@ kibot: version: 1 +filters: + - name: 'include_only' + type: 'generic' + comment: 'Test for include_only' + include_only: + - column: 'Footprint' + regex: '0805' + outputs: - name: 'bom_internal' comment: "Bill of Materials in CSV format" type: bom dir: BoM options: - include_only: - - column: 'Footprint' - regex: '0805' + exclude_filter: 'include_only' diff --git a/tests/yaml_samples/int_bom_no_include_only.kibot.yaml b/tests/yaml_samples/int_bom_no_include_only.kibot.yaml deleted file mode 100644 index 049c479b..00000000 --- a/tests/yaml_samples/int_bom_no_include_only.kibot.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Example KiBot config file -kibot: - version: 1 - -outputs: - - name: 'bom_internal' - comment: "Bill of Materials in CSV format" - type: bom - dir: BoM - options: - test_regex: false - include_only: - - column: 'Footprint' - regex: '0805'