[Filters][Spec_to_Field][Added] Dist coherence and dist desc
- We can now try to parse the description of the component - We can also try to compare the results from different distributors
This commit is contained in:
Salvador E. Tropea
parent
58ee992c0f
commit
b30ed4a6fc
|
|
@ -201,8 +201,11 @@ def initialize():
|
|||
parser = Lark(g, start='main') # , debug=DEBUG)
|
||||
|
||||
|
||||
def parse(text, with_extra=False):
|
||||
def parse(text, with_extra=False, stronger=False):
|
||||
initialize()
|
||||
if stronger:
|
||||
text = text.replace('+/-', ' +/-')
|
||||
text = text.replace(' - ', ' ')
|
||||
try:
|
||||
tree = parser.parse(text)
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ def value_from_grammar(r):
|
|||
return parsed
|
||||
|
||||
|
||||
def comp_match(component, ref_prefix, ref=None):
|
||||
def comp_match(component, ref_prefix, ref=None, relax_severity=False, stronger=False):
|
||||
"""
|
||||
Return a normalized value and units for a given component value string
|
||||
Also tries to separate extra data, i.e. tolerance, using a complex parser
|
||||
|
|
@ -199,18 +199,29 @@ def comp_match(component, ref_prefix, ref=None):
|
|||
# Ignore case
|
||||
match = re.compile(match_string(), flags=re.IGNORECASE)
|
||||
|
||||
log_func_warn = logger.debug if relax_severity else logger.warning
|
||||
where = ' in {}'.format(ref) if ref is not None else ''
|
||||
result = match.match(component)
|
||||
if not result:
|
||||
# This is used to parse things like "1/8 W", but we get "1/8" here
|
||||
result = re.match(r'(\d+)\/(\d+)', component)
|
||||
if result:
|
||||
val = int(result.group(1))/int(result.group(2))
|
||||
val, pow = get_prefix(val, '')
|
||||
parsed = ParsedValue(val, pow, get_unit('', ref_prefix))
|
||||
# Cache the result
|
||||
parser_cache[original+ref_prefix] = parsed
|
||||
return parsed
|
||||
if not result:
|
||||
# Failed with the regex, try with the parser
|
||||
result = parse(ref_prefix[0]+' '+with_commas, with_extra=True)
|
||||
result = parse(ref_prefix[0]+' '+with_commas, with_extra=True, stronger=stronger)
|
||||
if result:
|
||||
result = value_from_grammar(result)
|
||||
if result and result.get_extra('discarded'):
|
||||
discarded = " ".join(list(map(lambda x: '`'+x+'`', result.get_extra('discarded'))))
|
||||
logger.warning(W_BADVAL4 + "Malformed value: `{}` (discarded: {}{})".format(original, discarded, where))
|
||||
log_func_warn(W_BADVAL4+"Malformed value: `{}` (discarded: {}{})".format(original, discarded, where))
|
||||
if not result:
|
||||
logger.warning(W_BADVAL1 + "Malformed value: `{}` (no match{})".format(original, where))
|
||||
log_func_warn(W_BADVAL1+"Malformed value: `{}` (no match{})".format(original, where))
|
||||
return None
|
||||
# Cache the result
|
||||
parser_cache[original+ref_prefix] = result
|
||||
|
|
@ -218,7 +229,7 @@ def comp_match(component, ref_prefix, ref=None):
|
|||
|
||||
value, prefix, units, post = result.groups()
|
||||
if value == '.':
|
||||
logger.warning(W_BADVAL2 + "Malformed value: `{}` (reduced to decimal point{})".format(original, where))
|
||||
log_func_warn(W_BADVAL2+"Malformed value: `{}` (reduced to decimal point{})".format(original, where))
|
||||
return None
|
||||
if value == '':
|
||||
value = '0'
|
||||
|
|
@ -229,12 +240,11 @@ def comp_match(component, ref_prefix, ref=None):
|
|||
# We will also have a trailing number
|
||||
if post:
|
||||
if "." in value:
|
||||
logger.warning(W_BADVAL3 + "Malformed value: `{}` (unit split, but contains decimal point{})".
|
||||
format(original, where))
|
||||
log_func_warn(W_BADVAL3+"Malformed value: `{}` (unit split, but contains decimal point{})".format(original, where))
|
||||
return None
|
||||
value = float(value)
|
||||
postValue = float(post) / (10 ** len(post))
|
||||
val = value * 1.0 + postValue
|
||||
postValue = float(post)/(10**len(post))
|
||||
val = value*1.0+postValue
|
||||
else:
|
||||
val = float(value)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
# Project: KiBot (formerly KiPlot)
|
||||
# Description: Extracts information from the distributor spec and fills fields
|
||||
import re
|
||||
from .bom.units import comp_match, get_prefix, ParsedValue
|
||||
from .bom.xlsx_writer import get_spec
|
||||
from .error import KiPlotConfigurationError
|
||||
from .kiplot import look_for_output, run_output
|
||||
|
|
@ -15,6 +16,10 @@ from .macros import macros, document, filter_class # noqa: F401
|
|||
from . import log
|
||||
|
||||
logger = log.get_logger()
|
||||
UNITS = {'voltage': 'V', 'power': 'W', 'current': 'A'}
|
||||
EI_TYPES = {'value': 'value', 'tolerance': 'percent', 'footprint': 'string', 'power': 'power', 'current': 'current',
|
||||
'voltage': 'voltage', 'frequency': 'string', 'temp_coeff': 'string', 'manf': 'string', 'size': 'string'}
|
||||
DEF_CHECK = ['_value', '_tolerance', '_power', '_current', '_voltage', '_temp_coeff']
|
||||
|
||||
|
||||
class SpecOptions(Optionable):
|
||||
|
|
@ -24,7 +29,9 @@ class SpecOptions(Optionable):
|
|||
self._unknown_is_error = True
|
||||
with document:
|
||||
self.spec = Optionable
|
||||
""" [string|list(string)=''] *Name/s of the source spec/s """
|
||||
""" [string|list(string)=''] *Name/s of the source spec/s.
|
||||
The following names are uniform across distributors: '_desc', '_value', '_tolerance', '_footprint',
|
||||
'_power', '_current', '_voltage', '_frequency', '_temp_coeff', '_manf' and '_size' """
|
||||
self.field = ''
|
||||
""" *Name of the destination field """
|
||||
self.policy = 'overwrite'
|
||||
|
|
@ -34,9 +41,9 @@ class SpecOptions(Optionable):
|
|||
`new` copy only if the field doesn't exist. """
|
||||
self.collision = 'warning'
|
||||
""" [warning,error,ignore] How to report a collision between the current value and the new value """
|
||||
self.compare = 'plain'
|
||||
""" [plain,smart] How we compare the current value to determine a collision.
|
||||
`plain` is a strict comparison. `smart` tries to extract any number and compare it """
|
||||
self.type = 'string'
|
||||
""" [percent,voltage,power,current,value,string] How we compare the current value to determine a collision.
|
||||
`value` is the component value i.e. resistance for R* """
|
||||
self._field_example = 'RoHS'
|
||||
self._spec_example = 'rohs_status'
|
||||
|
||||
|
|
@ -67,7 +74,13 @@ class Spec_to_Field(BaseFilter): # noqa: F821
|
|||
Currently this must be a `bom` output with KiCost enabled and a distributor that returns specs """
|
||||
self.specs = SpecOptions
|
||||
""" [list(dict)|dict] *One or more specs to be copied """
|
||||
self.check_dist_coherence = True
|
||||
""" Check that the data we got from different distributors is equivalent """
|
||||
self.check_dist_fields = Optionable
|
||||
""" [string|list(string)=''] List of fields to include in the check.
|
||||
For a full list of fields consult the `specs` option """
|
||||
self._from = None
|
||||
self._check_dist_fields_example = DEF_CHECK
|
||||
|
||||
def config(self, parent):
|
||||
super().config(parent)
|
||||
|
|
@ -77,21 +90,62 @@ class Spec_to_Field(BaseFilter): # noqa: F821
|
|||
raise KiPlotConfigurationError("At least one spec must be provided ({})".format(str(self._tree)))
|
||||
if isinstance(self.specs, SpecOptions):
|
||||
self.specs = [self.specs]
|
||||
if isinstance(self.check_dist_fields, type):
|
||||
self.check_dist_fields = DEF_CHECK
|
||||
else:
|
||||
self.check_dist_fields = self.force_list(self.check_dist_fields)
|
||||
|
||||
def compare(self, cur_val, spec_val, how):
|
||||
def _normalize(self, val, kind, comp):
|
||||
val = val.strip()
|
||||
if kind == 'string':
|
||||
return val
|
||||
if kind == 'percent':
|
||||
# TODO: What about +20%, -10%?
|
||||
res = re.match(r"(?:\+/-|±|\+-)?\s*(\d+)\s*%?", val)
|
||||
if not res:
|
||||
return val
|
||||
return res.group(1)+'%'
|
||||
if kind == 'value':
|
||||
if not comp.ref_prefix or comp.ref_prefix[0] not in "RLC":
|
||||
return val
|
||||
res = comp_match(val, comp.ref_prefix, comp.ref, relax_severity=True, stronger=True)
|
||||
if not res:
|
||||
return val
|
||||
return str(res)
|
||||
# voltage,power,current
|
||||
new_val = re.sub(r"\s*(Volts?|V|Watts?|W|Amperes?|Ampers|Amp|A)", '', val)
|
||||
# Change things like 0.125 to 125 m
|
||||
res = comp_match(new_val, ' ', comp.ref, relax_severity=True, stronger=True)
|
||||
if res is None:
|
||||
if ',' not in new_val:
|
||||
return val
|
||||
# Some distributor APIs can return multiple values separated by comma
|
||||
res_many = []
|
||||
for v in new_val.split(','):
|
||||
res = comp_match(v.strip(), ' ', comp.ref, relax_severity=True, stronger=True)
|
||||
if res is not None:
|
||||
res.unit = UNITS[kind]
|
||||
res_many.append(res)
|
||||
if not res_many:
|
||||
return val
|
||||
res = res_many[0]
|
||||
reference = str(res)
|
||||
if len(res_many) > 1:
|
||||
for r in res_many[1:]:
|
||||
if str(r) != reference:
|
||||
logger.warning(W_FLDCOLLISION+'Inconsistencies in multiple values {}: `{}` ({} vs {})'.
|
||||
format(comp.ref, val, r, reference))
|
||||
res.unit = UNITS[kind]
|
||||
return str(res)
|
||||
|
||||
def normalize(self, old_val, kind, comp):
|
||||
val = self._normalize(old_val, kind, comp)
|
||||
logger.debugl(3, "- Normalize {} -> {}".format(old_val, val))
|
||||
return val
|
||||
|
||||
def compare(self, cur_val, spec_val):
|
||||
cur_val = cur_val.lower().strip()
|
||||
spec_val = spec_val.lower().strip()
|
||||
if how == 'plain':
|
||||
logger.debugl(3, f" - Compare {cur_val} == {spec_val}")
|
||||
return cur_val == spec_val
|
||||
# smart
|
||||
cur_match = re.match(r'(.*?)(\d+)(.*?)', cur_val)
|
||||
if cur_match:
|
||||
spec_match = re.match(r'(.*?)(\d+)(.*?)', spec_val)
|
||||
if spec_match:
|
||||
logger.debugl(3, f" - Compare {int(cur_match.group(2))} == {int(spec_match.group(2))}")
|
||||
return int(cur_match.group(2)) == int(spec_match.group(2))
|
||||
logger.debugl(3, f" - Compare {cur_val} == {spec_val}")
|
||||
return cur_val == spec_val
|
||||
|
||||
def solve_from(self):
|
||||
|
|
@ -103,20 +157,73 @@ class Spec_to_Field(BaseFilter): # noqa: F821
|
|||
run_output(out)
|
||||
self._from = out
|
||||
|
||||
def update_extra_info(self, res, attr, ei, dattr, pattern="{}", units=""):
|
||||
if dattr in ei:
|
||||
# Don't overwrite collected data
|
||||
return
|
||||
value = res.get_extra(attr)
|
||||
if not value:
|
||||
# No result for this
|
||||
return
|
||||
if isinstance(value, float):
|
||||
if int(value) == value:
|
||||
value = int(value)
|
||||
if units:
|
||||
# Change things like 0.125 to 125 m
|
||||
v, pow = get_prefix(value, '')
|
||||
parsed = ParsedValue(v, pow, units)
|
||||
value = str(parsed)
|
||||
ei[dattr] = pattern.format(value)
|
||||
|
||||
def check_coherent(self, c):
|
||||
if not self.check_dist_coherence:
|
||||
return
|
||||
extra_info = {}
|
||||
for d, dd in c.kicost_part.dd.items():
|
||||
ei = dd.extra_info
|
||||
if not ei:
|
||||
# We got nothing for this distributor
|
||||
continue
|
||||
if 'desc' in ei and 'value' not in ei:
|
||||
# Very incomplete extra info, try to fill it
|
||||
desc = ei['desc']
|
||||
logger.debugl(3, '- Parsing {}'.format(desc))
|
||||
res = comp_match(desc, c.ref_prefix, c.ref, relax_severity=True, stronger=True)
|
||||
if res:
|
||||
ei['value'] = str(res)
|
||||
self.update_extra_info(res, 'tolerance', ei, 'tolerance', "{}%")
|
||||
self.update_extra_info(res, 'voltage_rating', ei, 'voltage', units="V")
|
||||
self.update_extra_info(res, 'size', ei, 'footprint')
|
||||
self.update_extra_info(res, 'characteristic', ei, 'temp_coeff')
|
||||
self.update_extra_info(res, 'power_rating', ei, 'power', units="W")
|
||||
logger.debugl(3, '- New extra_info: {}'.format(ei))
|
||||
for n, v in ei.items():
|
||||
if '_'+n not in self.check_dist_fields:
|
||||
continue
|
||||
if n not in extra_info:
|
||||
# First time we see it
|
||||
extra_info[n] = (self.normalize(v, EI_TYPES[n], c), d)
|
||||
else:
|
||||
cur_val, cur_dist = extra_info[n]
|
||||
v = self.normalize(v, EI_TYPES[n], c)
|
||||
if not self.compare(cur_val, v):
|
||||
desc = "`{}` vs `{}` collision, `{}` != `{}`".format(d, cur_dist, v, cur_val)
|
||||
logger.error(desc)
|
||||
|
||||
def filter(self, comp):
|
||||
self.solve_from()
|
||||
for d, dd in comp.kicost_part.dd.items():
|
||||
logger.error(f"{d} {dd.extra_info}")
|
||||
self.check_coherent(comp)
|
||||
for s in self.specs:
|
||||
field = s.field.lower()
|
||||
spec_name = []
|
||||
spec_val = []
|
||||
spec_val = set()
|
||||
for sp in s.spec:
|
||||
name, val = get_spec(comp.kicost_part, sp)
|
||||
if name:
|
||||
spec_name.append(name)
|
||||
if val:
|
||||
spec_val.append(val)
|
||||
val = self.normalize(val, s.type, comp)
|
||||
spec_val.add(val)
|
||||
spec_name = ','.join(spec_name)
|
||||
spec_val = ' '.join(spec_val)
|
||||
if not spec_name or not spec_val:
|
||||
|
|
@ -125,10 +232,11 @@ class Spec_to_Field(BaseFilter): # noqa: F821
|
|||
has_field = comp.is_field(field)
|
||||
cur_val = comp.get_field_value(field) if has_field else None
|
||||
if cur_val:
|
||||
cur_val = self.normalize(cur_val, s.type, comp)
|
||||
if cur_val == spec_val:
|
||||
# Already there
|
||||
continue
|
||||
if not self.compare(cur_val, spec_val, s.compare):
|
||||
if not self.compare(cur_val, spec_val):
|
||||
# Collision
|
||||
desc = "{} field `{}` collision, has `{}`, found `{}`".format(comp.ref, s.field, cur_val, spec_val)
|
||||
if s.collision == 'warning':
|
||||
|
|
|
|||
Loading…
Reference in New Issue