Created the base filter class and the generic filter.

Moved all the KiBoM and IBoM filter functionality that was in their
variants to this generic mechanism.

kibot/fil_base.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 Salvador E. Tropea
+# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial
+# License: GPL-3.0
+# Project: KiBot (formerly KiPlot)
+from .registrable import RegFilter
+from .macros import macros, document  # noqa: F401
+
+
+class BaseFilter(RegFilter):
+    def __init__(self):
+        super().__init__()
+        self._unkown_is_error = True
+        with document:
+            self.name = ''
+            """ Used to identify this particular filter definition """
+            self.type = ''
+            """ Type of filter """
+            self.comment = ''
+            """ A comment for documentation purposes """

kibot/fil_generic.py

@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 Salvador E. Tropea
+# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial
+# License: GPL-3.0
+# Project: KiBot (formerly KiPlot)
+"""
+Implements the KiBoM and IBoM filters.
+"""
+from re import compile, IGNORECASE
+from .optionable import Optionable
+from .bom.columnlist import ColumnList
+from .gs import GS
+from .misc import DNF, DNC
+from .macros import macros, document, filter_class  # noqa: F401
+from .out_base import BoMRegex
+from . import log
+
+logger = log.get_logger(__name__)
+
+
+class DNFList(Optionable):
+    _default = DNF
+
+    def __init__(self):
+        super().__init__()
+
+
+@filter_class
+class Generic(BaseFilter):  # noqa: F821
+    """ Generic filter
+        This filter is based on regular exressions.
+        It also provides some shortcuts for common situations.
+        Note that matches aren't case sensitive and spaces at the beggining and the end are removed. """
+    def __init__(self):
+        super().__init__()
+        with document:
+            self.invert = False
+            """ Invert the result of the filter """
+            self.include_only = BoMRegex
+            """ [list(dict)] A series of regular expressions used to include parts.
+                If there are any regex defined here, only components that match against ANY of them will be included.
+                Column/field names are case-insensitive.
+                If empty this rule is ignored """
+            self.exclude_any = BoMRegex
+            """ [list(dict)] A series of regular expressions used to exclude parts.
+                If a component matches ANY of these, it will be excluded.
+                Column names are case-insensitive  """
+            self.keys = DNFList
+            """ [string|list(string)=dnf_list] [dnc_list,dnf_list] List of keys to match.
+                The `dnf_list` and `dnc_list` internal lists can be specified as strings """
+            self.exclude_value = False
+            """ Exclude components if their 'Value' is any of the keys """
+            self.config_field = 'Config'
+            """ Name of the field used to clasify components """
+            self.config_separators = ' ,'
+            """ Characters used to separate options inside the config field """
+            self.exclude_config = False
+            """ Exclude components containing a key value in the config field.
+                Separators are applied """
+            self.exclude_field = False
+            """ Exclude components if a field is named as any of the keys """
+            self.exclude_empty_val = False
+            """ Exclude components with empty 'Value' """
+            self.exclude_refs = Optionable
+            """ [list(string)] List of references to be excluded.
+                Use R* for all references with R prefix """
+            # Skip virtual components if needed
+            # TODO: We currently lack this information
+            # if config.blacklist_virtual and m.attr == 'Virtual':
+            #     return True
+        self.add_to_doc('keys', 'Use `dnf_list` for '+str(DNF))
+        self.add_to_doc('keys', 'Use `dnc_list` for '+str(DNC))
+
+    @staticmethod
+    def _fix_field(field):
+        """ References -> Reference """
+        col = field.lower()
+        if col == ColumnList.COL_REFERENCE_L:
+            col = col[:-1]
+        return col
+
+    def config(self):
+        super().config()
+        # include_only
+        if isinstance(self.include_only, type):
+            self.include_only = None
+        else:
+            for r in self.include_only:
+                r.column = self._fix_field(r.column)
+                r.regex = compile(r.regex, flags=IGNORECASE)
+        # exclude_any
+        if isinstance(self.exclude_any, type):
+            self.exclude_any = None
+        else:
+            for r in self.exclude_any:
+                r.column = self._fix_field(r.column)
+                r.regex = compile(r.regex, flags=IGNORECASE)
+        # keys
+        if isinstance(self.keys, type):
+            self.keys = DNF
+        elif isinstance(self.keys, str):
+            self.keys = DNF if self.keys == 'dnf_list' else DNC
+        else:
+            # Ensure lowercase
+            self.keys = [v.lower() for v in self.keys]
+        # Config field must be lowercase
+        self.config_field = self.config_field.lower()
+        # exclude_refs
+        if isinstance(self.exclude_refs, type):
+            self.exclude_refs = None
+
+    def test_reg_include(self, c):
+        """ Reject components that doesn't match the provided regex.
+            So we include only the components that matches any of the regexs. """
+        if not self.include_only:  # Nothing to match against, means include all
+            return True
+        for reg in self.include_only:
+            field_value = c.get_field_value(reg.column)
+            if reg.regex.search(field_value):
+                if GS.debug_level > 1:
+                    logger.debug("Including '{ref}': Field '{field}' ({value}) matched '{re}'".format(
+                                 ref=c.ref, field=reg.column, value=field_value, re=reg.regex))
+                    # Found a match
+                    return True
+        # Default, could not find a match
+        return False
+
+    def test_reg_exclude(self, c):
+        """ Test if this part should be included, based on any regex expressions provided in the preferences """
+        if not self.exclude_any:  # Nothing to match against, means don't exclude any
+            return False
+        for reg in self.exclude_any:
+            field_value = c.get_field_value(reg.column)
+            if reg.regex.search(field_value):
+                if GS.debug_level > 1:
+                    logger.debug("Excluding '{ref}': Field '{field}' ({value}) matched '{re}'".format(
+                                 ref=c.ref, field=reg.column, value=field_value, re=reg.regex))
+                # Found a match
+                return True
+        # Default, could not find any matches
+        return False
+
+    def filter(self, comp):
+        exclude = self.invert
+        value = comp.value.strip().lower()
+        # Exclude components with empty 'Value'
+        if self.exclude_empty_val and (value == '' or value == '~'):
+            return exclude
+        # List of references to be excluded
+        if self.exclude_refs and (comp.ref in self.exclude_refs or comp.ref_prefix+'*' in self.exclude_refs):
+            return exclude
+        # All stuff where keys are involved
+        if self.keys:
+            # Exclude components if their 'Value' is any of the keys
+            if self.exclude_value and value in self.keys:
+                return exclude
+            # Exclude components if a field is named as any of the keys
+            if self.exclude_field:
+                for k in self.keys:
+                    if k in comp.dfields:
+                        return exclude
+            # Exclude components containing a key value in the config field.
+            if self.exclude_config:
+                config = comp.get_field_value(self.config_field).strip().lower()
+                if self.config_separators:
+                    # Try with all the separators
+                    for sep in self.config_separators:
+                        opts = config.split(sep)
+                        # Try with all the extracted values
+                        for opt in opts:
+                            if opt.strip() in self.keys:
+                                return exclude
+                else:  # No separator
+                    if config in self.keys:
+                        return exclude
+        # Regular expressions
+        if not self.test_reg_include(comp):
+            return exclude
+        if self.test_reg_exclude(comp):
+            return exclude
+        return not exclude

kibot/optionable.py

@@ -67,6 +67,10 @@ class Optionable(object):
             return getattr(self, '_help_'+alias).strip(), alias, True
         return doc, name, False
 
+    def add_to_doc(self, name, text):
+        doc = getattr(self, '_help_'+name).strip()
+        setattr(self, '_help_'+name, doc+'.\n'+text)
+
     @staticmethod
     def _typeof(v):
         if isinstance(v, bool):

kibot/out_base.py

@@ -4,6 +4,7 @@
 # License: GPL-3.0
 # Project: KiBot (formerly KiPlot)
 from .registrable import RegOutput
+from .optionable import Optionable
 from .macros import macros, document  # noqa: F401
 from . import log
 
@@ -29,9 +30,6 @@ class BaseOutput(RegOutput):
     def attr2longopt(attr):
         return '--'+attr.replace('_', '-')
 
-    def __str__(self):
-        return "'{}' ({}) [{}]".format(self.comment, self.name, self.type)
-
     def is_sch(self):
         """ True for outputs that works on the schematic """
         return self._sch_related
@@ -50,3 +48,19 @@ class BaseOutput(RegOutput):
 
     def run(self, output_dir, board):
         self.options.run(output_dir, board)
+
+
+class BoMRegex(Optionable):
+    """ Implements the pair column/regex """
+    def __init__(self):
+        super().__init__()
+        self._unkown_is_error = True
+        with document:
+            self.column = ''
+            """ Name of the column to apply the regular expression """
+            self.regex = ''
+            """ Regular expression to match """
+            self.field = None
+            """ {column} """
+            self.regexp = None
+            """ {regex} """

kibot/var_ibom.py

@@ -29,13 +29,6 @@ class IBoM(BaseVariant):  # noqa: F821
             """ [string|list(string)=''] List of board variants to exclude from the BOM """
             self.variants_whitelist = Optionable
             """ [string|list(string)=''] List of board variants to include in the BOM """
-            self.blacklist = Optionable
-            """ [string|list(string)=''] List of comma separated blacklisted components or prefixes with *. E.g. 'X1,MH*' """
-            self.blacklist_empty_val = False
-            """ Blacklist components with empty value """
-            self.dnp_field = ''
-            """ Name of the extra field that indicates do not populate status.
-                Components with this field not empty will be blacklisted """
 
     @staticmethod
     def _force_list(val):
@@ -55,25 +48,9 @@ class IBoM(BaseVariant):  # noqa: F821
         super().config()
         self.variants_blacklist = self._force_list(self.variants_blacklist)
         self.variants_whitelist = self._force_list(self.variants_whitelist)
-        self.blacklist = self._force_list(self.blacklist)
 
     def skip_component(self, c):
-        """ Skip blacklisted components.
-            This is what IBoM does internally """
-        if c.ref in self.blacklist:
-            return True
-        if c.ref_prefix + '*' in self.blacklist:
-            return True
-        # Remove components with empty value
-        if self.blacklist_empty_val and c.value in ['', '~']:
-            return True
-        # Skip virtual components if needed
-        # TODO: We currently lack this information
-        # if config.blacklist_virtual and m.attr == 'Virtual':
-        #     return True
-        # Skip components with dnp field not empty
-        if self.dnp_field and c.get_field_value(self.dnp_field):
-            return True
+        """ Skip components that doesn't belong to this variant. """
         # Apply variants white/black lists
         if self.variant_field:
             ref_variant = c.get_field_value(self.variant_field).lower()
@@ -85,13 +62,15 @@ class IBoM(BaseVariant):  # noqa: F821
         return False
 
     def filter(self, comps):
-        logger.debug("Applying IBoM style filter `{}`".format(self.name))
+        logger.debug("Applying IBoM style variants `{}`".format(self.name))
         # Make black/white lists case insensitive
         self.variants_whitelist = [v.lower() for v in self.variants_whitelist]
         self.variants_blacklist = [v.lower() for v in self.variants_blacklist]
         # Apply to all the components
         for c in comps:
+            if not (c.fitted and c.in_bom):
+                # Don't check if we already discarded it
+                continue
             c.fitted = not self.skip_component(c)
-            c.fixed = False
             if not c.fitted and GS.debug_level > 2:
                 logger.debug('ref: {} value: {} -> False'.format(c.ref, c.value))

kibot/var_kibom.py

@@ -8,7 +8,6 @@ Implements the KiBoM variants mechanism.
 """
 from .optionable import Optionable
 from .gs import GS
-from .misc import DNF, DNC
 from .macros import macros, document, variant_class  # noqa: F401
 from . import log
 
@@ -43,68 +42,7 @@ class KiBoM(BaseVariant):  # noqa: F821
         # Config field must be lowercase
         self.config_field = self.config_field.lower()
 
-    @staticmethod
-    def basic_comp_is_fitted(value, config):
-        """ Basic `fitted` criteria, no variants.
-            value: component value (lowercase).
-            config: content of the 'Config' field (lowercase). """
-        # Check the value field first
-        if value in DNF:
-            return False
-        # Empty value means part is fitted
-        if not config:
-            return True
-        # Also support space separated list (simple cases)
-        opts = config.split(" ")
-        for opt in opts:
-            if opt in DNF:
-                return False
-        # Normal separator is ","
-        opts = config.split(",")
-        for opt in opts:
-            if opt in DNF:
-                return False
-        return True
-
-    @staticmethod
-    def basic_comp_is_fixed(value, config):
-        """ Basic `fixed` criteria, no variants
-            Fixed components shouldn't be replaced without express authorization.
-            value: component value (lowercase).
-            config: content of the 'Config' field (lowercase). """
-        # Check the value field first
-        if value in DNC:
-            return True
-        # Empty is not fixed
-        if not config:
-            return False
-        # Also support space separated list (simple cases)
-        opts = config.split(" ")
-        for opt in opts:
-            if opt in DNC:
-                return True
-        # Normal separator is ","
-        opts = config.split(",")
-        for opt in opts:
-            if opt in DNC:
-                return True
-        return False
-
-    @staticmethod
-    def _base_filter(comps, f_config):
-        """ Fill the `fixed` and `fitted` using the basic criteria.
-            No variant is applied in this step. """
-        logger.debug("- Generic KiBoM rules")
-        for c in comps:
-            value = c.value.lower()
-            config = c.get_field_value(f_config).lower()
-            c.fitted = KiBoM.basic_comp_is_fitted(value, config)
-            if GS.debug_level > 2:
-                logger.debug('ref: {} value: {} config: {} -> fitted {}'.
-                             format(c.ref, value, config, c.fitted))
-            c.fixed = KiBoM.basic_comp_is_fixed(value, config)
-
-    def variant_comp_is_fitted(self, value, config):
+    def _variant_comp_is_fitted(self, value, config):
         """ Apply the variants to determine if this component will be fitted.
             value: component value (lowercase).
             config: content of the 'Config' field (lowercase). """
@@ -126,16 +64,14 @@ class KiBoM(BaseVariant):  # noqa: F821
         return not exclusive
 
     def filter(self, comps):
-        logger.debug("Applying KiBoM style filter `{}`".format(self.name))
-        self._base_filter(comps, self.config_field)
-        logger.debug("- Variant specific rules")
+        logger.debug("Applying KiBoM style variants `{}`".format(self.name))
         for c in comps:
-            if not c.fitted:
-                # Don't check if we already discarded it during the basic test
+            if not (c.fitted and c.in_bom):
+                # Don't check if we already discarded it
                 continue
             value = c.value.lower()
             config = c.get_field_value(self.config_field).lower()
-            c.fitted = self.variant_comp_is_fitted(value, config)
+            c.fitted = self._variant_comp_is_fitted(value, config)
             if not c.fitted and GS.debug_level > 2:
                 logger.debug('ref: {} value: {} config: {} variant: {} -> False'.
                              format(c.ref, value, config, self.variant))