diff --git a/kibot/bom/electro_grammar.py b/kibot/bom/electro_grammar.py index 6a134e16..2999d664 100644 --- a/kibot/bom/electro_grammar.py +++ b/kibot/bom/electro_grammar.py @@ -201,8 +201,11 @@ def initialize(): parser = Lark(g, start='main') # , debug=DEBUG) -def parse(text, with_extra=False): +def parse(text, with_extra=False, stronger=False): initialize() + if stronger: + text = text.replace('+/-', ' +/-') + text = text.replace(' - ', ' ') try: tree = parser.parse(text) except Exception as e: diff --git a/kibot/bom/units.py b/kibot/bom/units.py index dafe3e78..a761c428 100644 --- a/kibot/bom/units.py +++ b/kibot/bom/units.py @@ -163,7 +163,7 @@ def value_from_grammar(r): return parsed -def comp_match(component, ref_prefix, ref=None): +def comp_match(component, ref_prefix, ref=None, relax_severity=False, stronger=False): """ Return a normalized value and units for a given component value string Also tries to separate extra data, i.e. tolerance, using a complex parser @@ -199,18 +199,29 @@ def comp_match(component, ref_prefix, ref=None): # Ignore case match = re.compile(match_string(), flags=re.IGNORECASE) + log_func_warn = logger.debug if relax_severity else logger.warning where = ' in {}'.format(ref) if ref is not None else '' result = match.match(component) + if not result: + # This is used to parse things like "1/8 W", but we get "1/8" here + result = re.match(r'(\d+)\/(\d+)', component) + if result: + val = int(result.group(1))/int(result.group(2)) + val, pow = get_prefix(val, '') + parsed = ParsedValue(val, pow, get_unit('', ref_prefix)) + # Cache the result + parser_cache[original+ref_prefix] = parsed + return parsed if not result: # Failed with the regex, try with the parser - result = parse(ref_prefix[0]+' '+with_commas, with_extra=True) + result = parse(ref_prefix[0]+' '+with_commas, with_extra=True, stronger=stronger) if result: result = value_from_grammar(result) if result and result.get_extra('discarded'): discarded = " ".join(list(map(lambda x: '`'+x+'`', result.get_extra('discarded')))) - logger.warning(W_BADVAL4 + "Malformed value: `{}` (discarded: {}{})".format(original, discarded, where)) + log_func_warn(W_BADVAL4+"Malformed value: `{}` (discarded: {}{})".format(original, discarded, where)) if not result: - logger.warning(W_BADVAL1 + "Malformed value: `{}` (no match{})".format(original, where)) + log_func_warn(W_BADVAL1+"Malformed value: `{}` (no match{})".format(original, where)) return None # Cache the result parser_cache[original+ref_prefix] = result @@ -218,7 +229,7 @@ def comp_match(component, ref_prefix, ref=None): value, prefix, units, post = result.groups() if value == '.': - logger.warning(W_BADVAL2 + "Malformed value: `{}` (reduced to decimal point{})".format(original, where)) + log_func_warn(W_BADVAL2+"Malformed value: `{}` (reduced to decimal point{})".format(original, where)) return None if value == '': value = '0' @@ -229,12 +240,11 @@ def comp_match(component, ref_prefix, ref=None): # We will also have a trailing number if post: if "." in value: - logger.warning(W_BADVAL3 + "Malformed value: `{}` (unit split, but contains decimal point{})". - format(original, where)) + log_func_warn(W_BADVAL3+"Malformed value: `{}` (unit split, but contains decimal point{})".format(original, where)) return None value = float(value) - postValue = float(post) / (10 ** len(post)) - val = value * 1.0 + postValue + postValue = float(post)/(10**len(post)) + val = value*1.0+postValue else: val = float(value) diff --git a/kibot/fil_spec_to_field.py b/kibot/fil_spec_to_field.py index 2fa7c6e2..8746f4e3 100644 --- a/kibot/fil_spec_to_field.py +++ b/kibot/fil_spec_to_field.py @@ -5,6 +5,7 @@ # Project: KiBot (formerly KiPlot) # Description: Extracts information from the distributor spec and fills fields import re +from .bom.units import comp_match, get_prefix, ParsedValue from .bom.xlsx_writer import get_spec from .error import KiPlotConfigurationError from .kiplot import look_for_output, run_output @@ -15,6 +16,10 @@ from .macros import macros, document, filter_class # noqa: F401 from . import log logger = log.get_logger() +UNITS = {'voltage': 'V', 'power': 'W', 'current': 'A'} +EI_TYPES = {'value': 'value', 'tolerance': 'percent', 'footprint': 'string', 'power': 'power', 'current': 'current', + 'voltage': 'voltage', 'frequency': 'string', 'temp_coeff': 'string', 'manf': 'string', 'size': 'string'} +DEF_CHECK = ['_value', '_tolerance', '_power', '_current', '_voltage', '_temp_coeff'] class SpecOptions(Optionable): @@ -24,7 +29,9 @@ class SpecOptions(Optionable): self._unknown_is_error = True with document: self.spec = Optionable - """ [string|list(string)=''] *Name/s of the source spec/s """ + """ [string|list(string)=''] *Name/s of the source spec/s. + The following names are uniform across distributors: '_desc', '_value', '_tolerance', '_footprint', + '_power', '_current', '_voltage', '_frequency', '_temp_coeff', '_manf' and '_size' """ self.field = '' """ *Name of the destination field """ self.policy = 'overwrite' @@ -34,9 +41,9 @@ class SpecOptions(Optionable): `new` copy only if the field doesn't exist. """ self.collision = 'warning' """ [warning,error,ignore] How to report a collision between the current value and the new value """ - self.compare = 'plain' - """ [plain,smart] How we compare the current value to determine a collision. - `plain` is a strict comparison. `smart` tries to extract any number and compare it """ + self.type = 'string' + """ [percent,voltage,power,current,value,string] How we compare the current value to determine a collision. + `value` is the component value i.e. resistance for R* """ self._field_example = 'RoHS' self._spec_example = 'rohs_status' @@ -67,7 +74,13 @@ class Spec_to_Field(BaseFilter): # noqa: F821 Currently this must be a `bom` output with KiCost enabled and a distributor that returns specs """ self.specs = SpecOptions """ [list(dict)|dict] *One or more specs to be copied """ + self.check_dist_coherence = True + """ Check that the data we got from different distributors is equivalent """ + self.check_dist_fields = Optionable + """ [string|list(string)=''] List of fields to include in the check. + For a full list of fields consult the `specs` option """ self._from = None + self._check_dist_fields_example = DEF_CHECK def config(self, parent): super().config(parent) @@ -77,21 +90,62 @@ class Spec_to_Field(BaseFilter): # noqa: F821 raise KiPlotConfigurationError("At least one spec must be provided ({})".format(str(self._tree))) if isinstance(self.specs, SpecOptions): self.specs = [self.specs] + if isinstance(self.check_dist_fields, type): + self.check_dist_fields = DEF_CHECK + else: + self.check_dist_fields = self.force_list(self.check_dist_fields) - def compare(self, cur_val, spec_val, how): + def _normalize(self, val, kind, comp): + val = val.strip() + if kind == 'string': + return val + if kind == 'percent': + # TODO: What about +20%, -10%? + res = re.match(r"(?:\+/-|±|\+-)?\s*(\d+)\s*%?", val) + if not res: + return val + return res.group(1)+'%' + if kind == 'value': + if not comp.ref_prefix or comp.ref_prefix[0] not in "RLC": + return val + res = comp_match(val, comp.ref_prefix, comp.ref, relax_severity=True, stronger=True) + if not res: + return val + return str(res) + # voltage,power,current + new_val = re.sub(r"\s*(Volts?|V|Watts?|W|Amperes?|Ampers|Amp|A)", '', val) + # Change things like 0.125 to 125 m + res = comp_match(new_val, ' ', comp.ref, relax_severity=True, stronger=True) + if res is None: + if ',' not in new_val: + return val + # Some distributor APIs can return multiple values separated by comma + res_many = [] + for v in new_val.split(','): + res = comp_match(v.strip(), ' ', comp.ref, relax_severity=True, stronger=True) + if res is not None: + res.unit = UNITS[kind] + res_many.append(res) + if not res_many: + return val + res = res_many[0] + reference = str(res) + if len(res_many) > 1: + for r in res_many[1:]: + if str(r) != reference: + logger.warning(W_FLDCOLLISION+'Inconsistencies in multiple values {}: `{}` ({} vs {})'. + format(comp.ref, val, r, reference)) + res.unit = UNITS[kind] + return str(res) + + def normalize(self, old_val, kind, comp): + val = self._normalize(old_val, kind, comp) + logger.debugl(3, "- Normalize {} -> {}".format(old_val, val)) + return val + + def compare(self, cur_val, spec_val): cur_val = cur_val.lower().strip() spec_val = spec_val.lower().strip() - if how == 'plain': - logger.debugl(3, f" - Compare {cur_val} == {spec_val}") - return cur_val == spec_val - # smart - cur_match = re.match(r'(.*?)(\d+)(.*?)', cur_val) - if cur_match: - spec_match = re.match(r'(.*?)(\d+)(.*?)', spec_val) - if spec_match: - logger.debugl(3, f" - Compare {int(cur_match.group(2))} == {int(spec_match.group(2))}") - return int(cur_match.group(2)) == int(spec_match.group(2)) - logger.debugl(3, f" - Compare {cur_val} == {spec_val}") return cur_val == spec_val def solve_from(self): @@ -103,20 +157,73 @@ class Spec_to_Field(BaseFilter): # noqa: F821 run_output(out) self._from = out + def update_extra_info(self, res, attr, ei, dattr, pattern="{}", units=""): + if dattr in ei: + # Don't overwrite collected data + return + value = res.get_extra(attr) + if not value: + # No result for this + return + if isinstance(value, float): + if int(value) == value: + value = int(value) + if units: + # Change things like 0.125 to 125 m + v, pow = get_prefix(value, '') + parsed = ParsedValue(v, pow, units) + value = str(parsed) + ei[dattr] = pattern.format(value) + + def check_coherent(self, c): + if not self.check_dist_coherence: + return + extra_info = {} + for d, dd in c.kicost_part.dd.items(): + ei = dd.extra_info + if not ei: + # We got nothing for this distributor + continue + if 'desc' in ei and 'value' not in ei: + # Very incomplete extra info, try to fill it + desc = ei['desc'] + logger.debugl(3, '- Parsing {}'.format(desc)) + res = comp_match(desc, c.ref_prefix, c.ref, relax_severity=True, stronger=True) + if res: + ei['value'] = str(res) + self.update_extra_info(res, 'tolerance', ei, 'tolerance', "{}%") + self.update_extra_info(res, 'voltage_rating', ei, 'voltage', units="V") + self.update_extra_info(res, 'size', ei, 'footprint') + self.update_extra_info(res, 'characteristic', ei, 'temp_coeff') + self.update_extra_info(res, 'power_rating', ei, 'power', units="W") + logger.debugl(3, '- New extra_info: {}'.format(ei)) + for n, v in ei.items(): + if '_'+n not in self.check_dist_fields: + continue + if n not in extra_info: + # First time we see it + extra_info[n] = (self.normalize(v, EI_TYPES[n], c), d) + else: + cur_val, cur_dist = extra_info[n] + v = self.normalize(v, EI_TYPES[n], c) + if not self.compare(cur_val, v): + desc = "`{}` vs `{}` collision, `{}` != `{}`".format(d, cur_dist, v, cur_val) + logger.error(desc) + def filter(self, comp): self.solve_from() - for d, dd in comp.kicost_part.dd.items(): - logger.error(f"{d} {dd.extra_info}") + self.check_coherent(comp) for s in self.specs: field = s.field.lower() spec_name = [] - spec_val = [] + spec_val = set() for sp in s.spec: name, val = get_spec(comp.kicost_part, sp) if name: spec_name.append(name) if val: - spec_val.append(val) + val = self.normalize(val, s.type, comp) + spec_val.add(val) spec_name = ','.join(spec_name) spec_val = ' '.join(spec_val) if not spec_name or not spec_val: @@ -125,10 +232,11 @@ class Spec_to_Field(BaseFilter): # noqa: F821 has_field = comp.is_field(field) cur_val = comp.get_field_value(field) if has_field else None if cur_val: + cur_val = self.normalize(cur_val, s.type, comp) if cur_val == spec_val: # Already there continue - if not self.compare(cur_val, spec_val, s.compare): + if not self.compare(cur_val, spec_val): # Collision desc = "{} field `{}` collision, has `{}`, found `{}`".format(comp.ref, s.field, cur_val, spec_val) if s.collision == 'warning':