Major improvements to regex parsing

KiBOM/component.py

@@ -1,10 +1,8 @@
 from columns import ColumnList
-
 from preferences import BomPref
-
 import units
-
 from sort import natural_sort
+import re
 
 DNF = ["dnf", "do not fit", "nofit", "no stuff", "nostuff", "noload", "do not load"]
 
@@ -269,6 +267,37 @@ class ComponentGroup():
             self.fields[ColumnList.COL_FP] = ""
             self.fields[ColumnList.COL_FP_LIB] = ""
             
+    #run test against all available regex exclusions in the preference file
+    #return True if none match (i.e. this group is OK)
+    #retunr False if any match
+    def testRegex(self):
+        
+        for key in self.prefs.regex.keys():
+            reg = self.prefs.regex[key]
+            if not type(reg) in [str, list]: continue #regex must be a string, or a list of strings
+            #does this group have a column that matches this regex?
+            if not self.getField(key): continue
+            
+            #list of regex to compare against
+            if type(reg) is str:
+                regex = [reg]
+            else:
+                regex = reg
+                
+            #test each regex
+            for r in regex:
+                field = self.getField(key)
+                if re.search(r, field, flags=re.IGNORECASE) is not None:
+                    print("'{col}' value '{val}' matched regex '{reg}'".format(
+                        col = key,
+                        val = self.getField(key),
+                        reg = r
+                        ))
+                    return False
+                    
+        return True
+                
+
     #return a dict of the KiCAD data based on the supplied columns
     def getRow(self, columns):
         row = [self.getField(key) for key in columns]

KiBOM/netlist_reader.py

@@ -289,11 +289,6 @@ class netlist():
             
         self.prefs = prefs
         
-        # component blacklist regexs, made from exluded_* above.
-        self.excluded_references = []
-        self.excluded_values = []
-        self.excluded_footprints = []
-
         if fname != "":
             self.load(fname)
 
@@ -390,58 +385,9 @@ class netlist():
         return sheet.get("rev")
 
     def getInterestingComponents(self):
-        """Return a subset of all components, those that should show up in the BOM.
-        Omit those that should not, by consulting the blacklists:
-        excluded_values, excluded_refs, and excluded_footprints, which hold one
-        or more regular expressions.  If any of the the regular expressions match
-        the corresponding field's value in a component, then the component is exluded.
-        """
         
-        # pre-compile all the regex expressions:
+        #copy out the components
-        del self.excluded_references[:]
+        ret = [c for c in self.components]
-        del self.excluded_values[:]
-        del self.excluded_footprints[:]
-
-        for rex in self.prefs.excluded_references:
-            self.excluded_references.append( re.compile( rex ) )
-
-        for rex in self.prefs.excluded_values:
-            self.excluded_values.append( re.compile( rex ) )
-
-        for rex in self.prefs.excluded_footprints:
-            self.excluded_footprints.append( re.compile( rex ) )
-
-        # the subset of components to return, considered as "interesting".
-        ret = []
-
-        # run each component thru a series of tests, if it passes all, then add it
-        # to the interesting list 'ret'.
-        for c in self.components:
-            exclude = False
-            if not exclude:
-                for refs in self.excluded_references:
-                    if refs.match(c.getRef()):
-                        exclude = True
-                        break;
-            if not exclude:
-                for vals in self.excluded_values:
-                    if vals.match(c.getValue()):
-                        exclude = True
-                        break;
-            if not exclude:
-                for mods in self.excluded_footprints:
-                    if mods.match(c.getFootprint()):
-                        exclude = True
-                        break;
-
-            if not exclude:
-                # This is a fairly personal way to flag DNS (Do Not Stuff).  NU for
-                # me means Normally Uninstalled.  You can 'or in' another expression here.
-                if c.getField( "Installed" ) == 'NU':
-                    exclude = True
-
-            if not exclude:
-                ret.append(c)
 
         # Sort first by ref as this makes for easier to read BOM's
         ret.sort(key=lambda g: g.getRef())
@@ -452,9 +398,7 @@ class netlist():
     
         groups = []
         
-        """
+        # Iterate through each component, and test whether a group for these already exists
-        Iterate through each component, and test whether a group for these already exists
-        """
         for c in components:
             found = False
             
@@ -465,7 +409,7 @@ class netlist():
                     break
             
             if not found:
-                g = ComponentGroup()
+                g = ComponentGroup(prefs=self.prefs) #pass down the preferences
                 g.addComponent(c)
                 groups.append(g)
             
@@ -474,6 +418,10 @@ class netlist():
             g.sortComponents()
             g.updateFields()
             
+        if self.prefs.useRegex:
+            #remove any that don't match regex
+            groups = [g for g in groups if g.testRegex()]
+
         #sort the groups
         #first priority is the Type of component (e.g. R?, U?, L?)
         groups = sorted(groups, key=lambda g: [g.components[0].getPrefix(), g.components[0].getValue()])

KiBOM/preferences.py

@@ -15,11 +15,25 @@ class BomPref:
     SECTION_EXCLUDE_VALUES = "EXCLUDE_COMPONENT_VALUES"
     SECTION_EXCLUDE_REFS = "EXCLUDE_COMPONENT_REFS"
     SECTION_EXCLUDE_FP = "EXCLUDE_COMPONENT_FP"
+    SECTION_EXCLUDE_PART = "EXCLUDE_COMPONENT_PART"
+    SECTION_EXCLUDE_DESC = "EXCLUDE_COMPONENT_DESC"
     SECTION_ALIASES = "COMPONENT_ALIASES"
     
     OPT_IGNORE_DNF = "ignore_dnf"
     OPT_NUMBER_ROWS = "number_rows"
     OPT_GROUP_CONN = "group_connectors"
+    OPT_USE_REGEX = "test_regex"
+    
+    #list of columns which we can use reg-ex on
+    COL_REG_EX = [
+        ColumnList.COL_REFERENCE,
+        ColumnList.COL_DESCRIPTION,
+        ColumnList.COL_VALUE,
+        ColumnList.COL_FP,
+        ColumnList.COL_FP_LIB,
+        ColumnList.COL_PART,
+        ColumnList.COL_PART_LIB
+        ]
 
     def __init__(self):
         self.ignore = [
@@ -29,6 +43,7 @@ class BomPref:
         self.ignoreDNF = False #ignore rows for do-not-fit parts
         self.numberRows = True #add row-numbers to BoM output
         self.groupConnectors = True #group connectors and ignore component value
+        self.useRegex = True #Test various columns with regex
         
         #default reference exclusions
         self.excluded_references = [
@@ -41,7 +56,8 @@ class BomPref:
             'SCOPETEST',
             'MOUNT_HOLE',
             'MOUNTING_HOLE',
-            'SOLDER_BRIDGE.*'
+            'SOLDER_BRIDGE.*',
+            'test'
             ]
             
         #default footprint exclusions
@@ -56,6 +72,29 @@ class BomPref:
             ["l", "l_small", "inductor"]
             ]
             
+        #dictionary of possible regex expressions for ignoring component row(s)
+        self.regex = dict.fromkeys(self.COL_REG_EX)
+        
+        #default regex values
+        self.regex[ColumnList.COL_REFERENCE] = [
+            'TP[0-9]+',
+            ]
+            
+        self.regex[ColumnList.COL_PART] = [
+            'mounthole',
+            'scopetest',
+            'mount_hole',
+            'solder_bridge',
+            'test_point',
+            ]
+            
+        self.regex[ColumnList.COL_FP] = [
+            'mounthole'
+            ]
+        
+    def columnToGroup(self, col):
+        return "REGEXCLUDE_" + col.upper().replace(" ","_")
+        
     #read KiBOM preferences from file
     def Read(self, file, verbose=False):
         file = os.path.abspath(file)
@@ -76,27 +115,24 @@ class BomPref:
                     self.numberRows = cf.get(self.SECTION_GENERAL, self.OPT_NUMBER_ROWS) == "1"
                 if cf.has_option(self.SECTION_GENERAL, self.OPT_GROUP_CONN):
                     self.groupConnectors = cf.get(self.SECTION_GENERAL, self.OPT_GROUP_CONN) == "1"
+                if cf.has_option(self.SECTION_GENERAL, self.OPT_USE_REGEX):
+                    self.useRegex = cf.get(self.SECTION_GENERAL, self.OPT_USE_REGEX) == "1"
                     
             #read out ignored-rows
             if self.SECTION_IGNORE in cf.sections():
                 self.ignore = [i for i in cf.options(self.SECTION_IGNORE)]
             
-            #read out excluded values
-            if self.SECTION_EXCLUDE_VALUES in cf.sections():
-                self.excludedValues = [e for e in cf.options(self.SECTION_EXCLUDE_VALUES)]
-                
-            #read out excluded values
-            if self.SECTION_EXCLUDE_REFS in cf.sections():
-                self.excludedValues = [e for e in cf.options(self.SECTION_EXCLUDE_REFS)]
-                
-            #read out excluded values
-            if self.SECTION_EXCLUDE_FP in cf.sections():
-                self.excludedValues = [e for e in cf.options(self.SECTION_EXCLUDE_FP)]
-                
             #read out component aliases
             if self.SECTION_ALIASES in cf.sections():
                 self.aliases = [a.split(" ") for a in cf.options(self.SECTION_ALIASES)]
                 
+            #read out the regex
+            for key in self.regex.keys():
+                section = self.columnToGroup(key)
+                if section in cf.sections():
+                    self.regex[key] = [r for r in cf.options(section)]
+            
+                
             
     #write KiBOM preferences to file
     def Write(self, file):
@@ -106,12 +142,14 @@ class BomPref:
         
         cf.add_section(self.SECTION_GENERAL)
         cf.set(self.SECTION_GENERAL, "; General BoM options here")
-        cf.set(self.SECTION_GENERAL, "; If {opt} option is set to 1, rows that are not to be fitted on the PCB will not be written to the BoM file".format(opt=self.OPT_IGNORE_DNF))
+        cf.set(self.SECTION_GENERAL, "; If '{opt}' option is set to 1, rows that are not to be fitted on the PCB will not be written to the BoM file".format(opt=self.OPT_IGNORE_DNF))
         cf.set(self.SECTION_GENERAL, self.OPT_IGNORE_DNF, 1 if self.ignoreDNF else 0)
-        cf.set(self.SECTION_GENERAL, "; If {opt} option is set to 1, each row in the BoM will be prepended with an incrementing row number".format(opt=self.OPT_NUMBER_ROWS))
+        cf.set(self.SECTION_GENERAL, "; If '{opt}' option is set to 1, each row in the BoM will be prepended with an incrementing row number".format(opt=self.OPT_NUMBER_ROWS))
         cf.set(self.SECTION_GENERAL, self.OPT_NUMBER_ROWS, 1 if self.numberRows else 0)
-        cf.set(self.SECTION_GENERAL, "; If {opt} option is set to 1, connectors with the same footprints will be grouped together, independent of the name of the connector".format(opt=self.OPT_GROUP_CONN))
+        cf.set(self.SECTION_GENERAL, "; If '{opt}' option is set to 1, connectors with the same footprints will be grouped together, independent of the name of the connector".format(opt=self.OPT_GROUP_CONN))
         cf.set(self.SECTION_GENERAL, self.OPT_GROUP_CONN, 1 if self.groupConnectors else 0)
+        cf.set(self.SECTION_GENERAL, "; If '{opt}' option is set to 1, each component group will be tested against a number of regular-expressions (specified, per column, below). If any matches are found, the row is ignored in the output file".format(opt=self.OPT_USE_REGEX))
+        cf.set(self.SECTION_GENERAL, self.OPT_USE_REGEX, 1 if self.useRegex else 0)
         
         cf.add_section(self.SECTION_IGNORE)
         cf.set(self.SECTION_IGNORE, "; Any column heading that appears here will be excluded from the Generated BoM")
@@ -120,24 +158,6 @@ class BomPref:
         for i in self.ignore:
             cf.set(self.SECTION_IGNORE, i)
             
-        cf.add_section(self.SECTION_EXCLUDE_VALUES)
-        cf.set(self.SECTION_EXCLUDE_VALUES, "; A series of reg-ex strings for ignoring component values")
-        cf.set(self.SECTION_EXCLUDE_VALUES, "; Any components with values that match any of these reg-ex strings will be ignored")
-        for e in self.excluded_values:
-            cf.set(self.SECTION_EXCLUDE_VALUES, e)
-            
-        cf.add_section(self.SECTION_EXCLUDE_REFS)
-        cf.set(self.SECTION_EXCLUDE_REFS, "; A series of reg-ex strings for ignoring component references")
-        cf.set(self.SECTION_EXCLUDE_REFS, "; Any components with references that match any of these reg-ex strings will be ignored")
-        for e in self.excluded_references:
-            cf.set(self.SECTION_EXCLUDE_REFS, e)
-            
-        cf.add_section(self.SECTION_EXCLUDE_FP)
-        cf.set(self.SECTION_EXCLUDE_FP, "; A series of reg-ex strings for ignoring component footprints")
-        cf.set(self.SECTION_EXCLUDE_FP, "; Any components with footprints that match any of these reg-ex strings will be ignored")
-        for e in self.excluded_footprints:
-            cf.set(self.SECTION_EXCLUDE_FP, e)
-            
         cf.add_section(self.SECTION_ALIASES)
         cf.set(self.SECTION_ALIASES, "; A series of values which are considered to be equivalent for the part name")
         cf.set(self.SECTION_ALIASES, "; Each line represents a space-separated list of equivalent component name values")
@@ -145,5 +165,24 @@ class BomPref:
         for a in self.aliases:
             cf.set(self.SECTION_ALIASES, " ".join(a))
             
+        for col in self.regex.keys():
+            
+            reg = self.regex[col]
+            
+            section = self.columnToGroup(col)
+            cf.add_section(section)
+            #comments
+            cf.set(section, "; A list of regex to compare against the '{col}' column".format(col=col))
+            cf.set(section, "; If the value in the '{col}' column matches any of these expressions, the row will be excluded from the BoM".format(col=col))
+            
+            if type(reg) == str:
+                cf.set(section, reg)
+                
+            elif type(reg) == list:
+                for r in reg:
+                    cf.set(section, r)
+            
+
+            
         with open(file, 'wb') as configfile:
             cf.write(configfile)