From 6af9faf9092b0605c29e971c27a2012dcdaa37a3 Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Sat, 29 Aug 2020 17:39:56 -0300 Subject: [PATCH] Created the base filter class and the generic filter. Moved all the KiBoM and IBoM filter functionality that was in their variants to this generic mechanism. --- kibot/fil_base.py | 20 +++++ kibot/fil_generic.py | 181 +++++++++++++++++++++++++++++++++++++++++++ kibot/optionable.py | 4 + kibot/out_base.py | 20 ++++- kibot/var_ibom.py | 31 ++------ kibot/var_kibom.py | 74 ++---------------- 6 files changed, 232 insertions(+), 98 deletions(-) create mode 100644 kibot/fil_base.py create mode 100644 kibot/fil_generic.py diff --git a/kibot/fil_base.py b/kibot/fil_base.py new file mode 100644 index 00000000..7c2c3a6c --- /dev/null +++ b/kibot/fil_base.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 Salvador E. Tropea +# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial +# License: GPL-3.0 +# Project: KiBot (formerly KiPlot) +from .registrable import RegFilter +from .macros import macros, document # noqa: F401 + + +class BaseFilter(RegFilter): + def __init__(self): + super().__init__() + self._unkown_is_error = True + with document: + self.name = '' + """ Used to identify this particular filter definition """ + self.type = '' + """ Type of filter """ + self.comment = '' + """ A comment for documentation purposes """ diff --git a/kibot/fil_generic.py b/kibot/fil_generic.py new file mode 100644 index 00000000..1d173dd0 --- /dev/null +++ b/kibot/fil_generic.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 Salvador E. Tropea +# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial +# License: GPL-3.0 +# Project: KiBot (formerly KiPlot) +""" +Implements the KiBoM and IBoM filters. +""" +from re import compile, IGNORECASE +from .optionable import Optionable +from .bom.columnlist import ColumnList +from .gs import GS +from .misc import DNF, DNC +from .macros import macros, document, filter_class # noqa: F401 +from .out_base import BoMRegex +from . import log + +logger = log.get_logger(__name__) + + +class DNFList(Optionable): + _default = DNF + + def __init__(self): + super().__init__() + + +@filter_class +class Generic(BaseFilter): # noqa: F821 + """ Generic filter + This filter is based on regular exressions. + It also provides some shortcuts for common situations. + Note that matches aren't case sensitive and spaces at the beggining and the end are removed. """ + def __init__(self): + super().__init__() + with document: + self.invert = False + """ Invert the result of the filter """ + self.include_only = BoMRegex + """ [list(dict)] A series of regular expressions used to include parts. + If there are any regex defined here, only components that match against ANY of them will be included. + Column/field names are case-insensitive. + If empty this rule is ignored """ + self.exclude_any = BoMRegex + """ [list(dict)] A series of regular expressions used to exclude parts. + If a component matches ANY of these, it will be excluded. + Column names are case-insensitive """ + self.keys = DNFList + """ [string|list(string)=dnf_list] [dnc_list,dnf_list] List of keys to match. + The `dnf_list` and `dnc_list` internal lists can be specified as strings """ + self.exclude_value = False + """ Exclude components if their 'Value' is any of the keys """ + self.config_field = 'Config' + """ Name of the field used to clasify components """ + self.config_separators = ' ,' + """ Characters used to separate options inside the config field """ + self.exclude_config = False + """ Exclude components containing a key value in the config field. + Separators are applied """ + self.exclude_field = False + """ Exclude components if a field is named as any of the keys """ + self.exclude_empty_val = False + """ Exclude components with empty 'Value' """ + self.exclude_refs = Optionable + """ [list(string)] List of references to be excluded. + Use R* for all references with R prefix """ + # Skip virtual components if needed + # TODO: We currently lack this information + # if config.blacklist_virtual and m.attr == 'Virtual': + # return True + self.add_to_doc('keys', 'Use `dnf_list` for '+str(DNF)) + self.add_to_doc('keys', 'Use `dnc_list` for '+str(DNC)) + + @staticmethod + def _fix_field(field): + """ References -> Reference """ + col = field.lower() + if col == ColumnList.COL_REFERENCE_L: + col = col[:-1] + return col + + def config(self): + super().config() + # include_only + if isinstance(self.include_only, type): + self.include_only = None + else: + for r in self.include_only: + r.column = self._fix_field(r.column) + r.regex = compile(r.regex, flags=IGNORECASE) + # exclude_any + if isinstance(self.exclude_any, type): + self.exclude_any = None + else: + for r in self.exclude_any: + r.column = self._fix_field(r.column) + r.regex = compile(r.regex, flags=IGNORECASE) + # keys + if isinstance(self.keys, type): + self.keys = DNF + elif isinstance(self.keys, str): + self.keys = DNF if self.keys == 'dnf_list' else DNC + else: + # Ensure lowercase + self.keys = [v.lower() for v in self.keys] + # Config field must be lowercase + self.config_field = self.config_field.lower() + # exclude_refs + if isinstance(self.exclude_refs, type): + self.exclude_refs = None + + def test_reg_include(self, c): + """ Reject components that doesn't match the provided regex. + So we include only the components that matches any of the regexs. """ + if not self.include_only: # Nothing to match against, means include all + return True + for reg in self.include_only: + field_value = c.get_field_value(reg.column) + if reg.regex.search(field_value): + if GS.debug_level > 1: + logger.debug("Including '{ref}': Field '{field}' ({value}) matched '{re}'".format( + ref=c.ref, field=reg.column, value=field_value, re=reg.regex)) + # Found a match + return True + # Default, could not find a match + return False + + def test_reg_exclude(self, c): + """ Test if this part should be included, based on any regex expressions provided in the preferences """ + if not self.exclude_any: # Nothing to match against, means don't exclude any + return False + for reg in self.exclude_any: + field_value = c.get_field_value(reg.column) + if reg.regex.search(field_value): + if GS.debug_level > 1: + logger.debug("Excluding '{ref}': Field '{field}' ({value}) matched '{re}'".format( + ref=c.ref, field=reg.column, value=field_value, re=reg.regex)) + # Found a match + return True + # Default, could not find any matches + return False + + def filter(self, comp): + exclude = self.invert + value = comp.value.strip().lower() + # Exclude components with empty 'Value' + if self.exclude_empty_val and (value == '' or value == '~'): + return exclude + # List of references to be excluded + if self.exclude_refs and (comp.ref in self.exclude_refs or comp.ref_prefix+'*' in self.exclude_refs): + return exclude + # All stuff where keys are involved + if self.keys: + # Exclude components if their 'Value' is any of the keys + if self.exclude_value and value in self.keys: + return exclude + # Exclude components if a field is named as any of the keys + if self.exclude_field: + for k in self.keys: + if k in comp.dfields: + return exclude + # Exclude components containing a key value in the config field. + if self.exclude_config: + config = comp.get_field_value(self.config_field).strip().lower() + if self.config_separators: + # Try with all the separators + for sep in self.config_separators: + opts = config.split(sep) + # Try with all the extracted values + for opt in opts: + if opt.strip() in self.keys: + return exclude + else: # No separator + if config in self.keys: + return exclude + # Regular expressions + if not self.test_reg_include(comp): + return exclude + if self.test_reg_exclude(comp): + return exclude + return not exclude diff --git a/kibot/optionable.py b/kibot/optionable.py index 6b2059fa..e795a684 100644 --- a/kibot/optionable.py +++ b/kibot/optionable.py @@ -67,6 +67,10 @@ class Optionable(object): return getattr(self, '_help_'+alias).strip(), alias, True return doc, name, False + def add_to_doc(self, name, text): + doc = getattr(self, '_help_'+name).strip() + setattr(self, '_help_'+name, doc+'.\n'+text) + @staticmethod def _typeof(v): if isinstance(v, bool): diff --git a/kibot/out_base.py b/kibot/out_base.py index 86751e9a..c3a30b59 100644 --- a/kibot/out_base.py +++ b/kibot/out_base.py @@ -4,6 +4,7 @@ # License: GPL-3.0 # Project: KiBot (formerly KiPlot) from .registrable import RegOutput +from .optionable import Optionable from .macros import macros, document # noqa: F401 from . import log @@ -29,9 +30,6 @@ class BaseOutput(RegOutput): def attr2longopt(attr): return '--'+attr.replace('_', '-') - def __str__(self): - return "'{}' ({}) [{}]".format(self.comment, self.name, self.type) - def is_sch(self): """ True for outputs that works on the schematic """ return self._sch_related @@ -50,3 +48,19 @@ class BaseOutput(RegOutput): def run(self, output_dir, board): self.options.run(output_dir, board) + + +class BoMRegex(Optionable): + """ Implements the pair column/regex """ + def __init__(self): + super().__init__() + self._unkown_is_error = True + with document: + self.column = '' + """ Name of the column to apply the regular expression """ + self.regex = '' + """ Regular expression to match """ + self.field = None + """ {column} """ + self.regexp = None + """ {regex} """ diff --git a/kibot/var_ibom.py b/kibot/var_ibom.py index 2e1673f8..4c0e31f1 100644 --- a/kibot/var_ibom.py +++ b/kibot/var_ibom.py @@ -29,13 +29,6 @@ class IBoM(BaseVariant): # noqa: F821 """ [string|list(string)=''] List of board variants to exclude from the BOM """ self.variants_whitelist = Optionable """ [string|list(string)=''] List of board variants to include in the BOM """ - self.blacklist = Optionable - """ [string|list(string)=''] List of comma separated blacklisted components or prefixes with *. E.g. 'X1,MH*' """ - self.blacklist_empty_val = False - """ Blacklist components with empty value """ - self.dnp_field = '' - """ Name of the extra field that indicates do not populate status. - Components with this field not empty will be blacklisted """ @staticmethod def _force_list(val): @@ -55,25 +48,9 @@ class IBoM(BaseVariant): # noqa: F821 super().config() self.variants_blacklist = self._force_list(self.variants_blacklist) self.variants_whitelist = self._force_list(self.variants_whitelist) - self.blacklist = self._force_list(self.blacklist) def skip_component(self, c): - """ Skip blacklisted components. - This is what IBoM does internally """ - if c.ref in self.blacklist: - return True - if c.ref_prefix + '*' in self.blacklist: - return True - # Remove components with empty value - if self.blacklist_empty_val and c.value in ['', '~']: - return True - # Skip virtual components if needed - # TODO: We currently lack this information - # if config.blacklist_virtual and m.attr == 'Virtual': - # return True - # Skip components with dnp field not empty - if self.dnp_field and c.get_field_value(self.dnp_field): - return True + """ Skip components that doesn't belong to this variant. """ # Apply variants white/black lists if self.variant_field: ref_variant = c.get_field_value(self.variant_field).lower() @@ -85,13 +62,15 @@ class IBoM(BaseVariant): # noqa: F821 return False def filter(self, comps): - logger.debug("Applying IBoM style filter `{}`".format(self.name)) + logger.debug("Applying IBoM style variants `{}`".format(self.name)) # Make black/white lists case insensitive self.variants_whitelist = [v.lower() for v in self.variants_whitelist] self.variants_blacklist = [v.lower() for v in self.variants_blacklist] # Apply to all the components for c in comps: + if not (c.fitted and c.in_bom): + # Don't check if we already discarded it + continue c.fitted = not self.skip_component(c) - c.fixed = False if not c.fitted and GS.debug_level > 2: logger.debug('ref: {} value: {} -> False'.format(c.ref, c.value)) diff --git a/kibot/var_kibom.py b/kibot/var_kibom.py index 2b606c4a..069c88e7 100644 --- a/kibot/var_kibom.py +++ b/kibot/var_kibom.py @@ -8,7 +8,6 @@ Implements the KiBoM variants mechanism. """ from .optionable import Optionable from .gs import GS -from .misc import DNF, DNC from .macros import macros, document, variant_class # noqa: F401 from . import log @@ -43,68 +42,7 @@ class KiBoM(BaseVariant): # noqa: F821 # Config field must be lowercase self.config_field = self.config_field.lower() - @staticmethod - def basic_comp_is_fitted(value, config): - """ Basic `fitted` criteria, no variants. - value: component value (lowercase). - config: content of the 'Config' field (lowercase). """ - # Check the value field first - if value in DNF: - return False - # Empty value means part is fitted - if not config: - return True - # Also support space separated list (simple cases) - opts = config.split(" ") - for opt in opts: - if opt in DNF: - return False - # Normal separator is "," - opts = config.split(",") - for opt in opts: - if opt in DNF: - return False - return True - - @staticmethod - def basic_comp_is_fixed(value, config): - """ Basic `fixed` criteria, no variants - Fixed components shouldn't be replaced without express authorization. - value: component value (lowercase). - config: content of the 'Config' field (lowercase). """ - # Check the value field first - if value in DNC: - return True - # Empty is not fixed - if not config: - return False - # Also support space separated list (simple cases) - opts = config.split(" ") - for opt in opts: - if opt in DNC: - return True - # Normal separator is "," - opts = config.split(",") - for opt in opts: - if opt in DNC: - return True - return False - - @staticmethod - def _base_filter(comps, f_config): - """ Fill the `fixed` and `fitted` using the basic criteria. - No variant is applied in this step. """ - logger.debug("- Generic KiBoM rules") - for c in comps: - value = c.value.lower() - config = c.get_field_value(f_config).lower() - c.fitted = KiBoM.basic_comp_is_fitted(value, config) - if GS.debug_level > 2: - logger.debug('ref: {} value: {} config: {} -> fitted {}'. - format(c.ref, value, config, c.fitted)) - c.fixed = KiBoM.basic_comp_is_fixed(value, config) - - def variant_comp_is_fitted(self, value, config): + def _variant_comp_is_fitted(self, value, config): """ Apply the variants to determine if this component will be fitted. value: component value (lowercase). config: content of the 'Config' field (lowercase). """ @@ -126,16 +64,14 @@ class KiBoM(BaseVariant): # noqa: F821 return not exclusive def filter(self, comps): - logger.debug("Applying KiBoM style filter `{}`".format(self.name)) - self._base_filter(comps, self.config_field) - logger.debug("- Variant specific rules") + logger.debug("Applying KiBoM style variants `{}`".format(self.name)) for c in comps: - if not c.fitted: - # Don't check if we already discarded it during the basic test + if not (c.fitted and c.in_bom): + # Don't check if we already discarded it continue value = c.value.lower() config = c.get_field_value(self.config_field).lower() - c.fitted = self.variant_comp_is_fitted(value, config) + c.fitted = self._variant_comp_is_fitted(value, config) if not c.fitted and GS.debug_level > 2: logger.debug('ref: {} value: {} config: {} variant: {} -> False'. format(c.ref, value, config, self.variant))