templates
/
KiBot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added full KiBoM configuration from the YAML config.

This commit is contained in:
Salvador E. Tropea 2020-07-22 18:33:53 -03:00
parent 97e95ff7c5
commit 165d9aa15d
13 changed files with 616 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
kiplot/kiplot.py
View File

@ -66,6 +66,8 @@ def check_version(command, version):
logger.debug('Running: '+str(cmd))
result = run(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True)
z = re.match(command + r' (\d+\.\d+\.\d+)', result.stdout)
if not z:
z = re.search(r'Version: (\d+\.\d+\.\d+)', result.stdout)
if not z:
logger.error('Unable to determine ' + command + ' version:\n' +
result.stdout)

333
kiplot/out_kibom.py
View File

@ -1,15 +1,291 @@
import os
from glob import (glob)
from re import search
from tempfile import NamedTemporaryFile
from subprocess import (check_output, STDOUT, CalledProcessError)
from .misc import (CMD_KIBOM, URL_KIBOM, BOM_ERROR)
from .kiplot import (check_script)
from .gs import (GS)
from .optionable import BaseOptions
from .optionable import Optionable, BaseOptions
from .error import KiPlotConfigurationError
from kiplot.macros import macros, document, output_class # noqa: F401
from . import log
logger = log.get_logger(__name__)
CONFIG_FILENAME = 'config.kibom.ini'
class KiBoMRegex(Optionable):
""" Implements the pair column/regex """
def __init__(self):
super().__init__()
self._unkown_is_error = True
with document:
self.column = ''
""" Name of the column to apply the regular expression """
self.regex = ''
""" Regular expression to match """
self.field = None
""" {column} """
self.regexp = None
""" {regex} """ # pragma: no cover
def __str__(self):
return self.column+'\t'+self.regex
class KiBoMColumns(Optionable):
""" Information for the BoM columns """
def __init__(self):
super().__init__()
self._unkown_is_error = True
with document:
self.field = ''
""" Name of the field to use for this column """
self.name = ''
""" Name to display in the header. The field is used when empty """
self.join = Optionable
""" [list(string)|string] List of fields to join to this column """ # pragma: no cover
def config(self):
super().config()
if not self.field:
raise KiPlotConfigurationError("Missing or empty `field` in columns list ({})".format(str(self._tree)))
if isinstance(self.join, type):
self.join = None
elif isinstance(self.join, list):
self.join = '\t'.join(self.join)
class KiBoMConfig(Optionable):
""" Implements the .ini options """
def __init__(self):
super().__init__()
with document:
self.ignore_dnf = True
""" Exclude DNF (Do Not Fit) components """
self.html_generate_dnf = True
""" Generate a separated section for DNF (Do Not Fit) components (HTML only) """
self.use_alt = False
""" Print grouped references in the alternate compressed style eg: R1-R7,R18 """
self.number_rows = True
""" First column is the row number """
self.group_connectors = True
""" Connectors with the same footprints will be grouped together, independent of the name of the connector """
self.test_regex = True
""" Each component group will be tested against a number of regular-expressions (see ``). """
self.merge_blank_fields = True
""" Component groups with blank fields will be merged into the most compatible group, where possible """
self.fit_field = 'Config'
""" Field name used to determine if a particular part is to be fitted (also DNC and variants) """
self.datasheet_as_link = ''
""" Column with links to the datasheet (HTML only) """
self.hide_headers = False
""" Hide column headers """
self.hide_pcb_info = False
""" Hide project information """
self.digikey_link = Optionable
""" [string|list(string)] Column/s containing Digi-Key part numbers, will be linked to web page (HTML only) """
self.group_fields = Optionable
""" [list(string)] List of fields used for sorting individual components into groups.
Components which match (comparing *all* fields) will be grouped together.
Field names are case-insensitive.
If empty: ['Part', 'Part Lib', 'Value', 'Footprint', 'Footprint Lib'] is used """
self.component_aliases = Optionable
""" [list(list(string))] A series of values which are considered to be equivalent for the part name.
Each entry is a list of equivalen names. Example: ['c', 'c_small', 'cap' ]
will ensure the equivalent capacitor symbols can be grouped together.
If empty the following aliases are used:
- ['r', 'r_small', 'res', 'resistor']
- ['l', 'l_small', 'inductor']
- ['c', 'c_small', 'cap', 'capacitor']
- ['sw', 'switch']
- ['zener', 'zenersmall']
- ['d', 'diode', 'd_small'] """
self.include_only = KiBoMRegex
""" [list(dict)] A series of regular expressions used to select included parts.
If there are any regex defined here, only components that match against ANY of them will be included.
Column names are case-insensitive.
If empty all the components are included """
self.exclude_any = KiBoMRegex
""" [list(dict)] A series of regular expressions used to exclude parts.
If a component matches ANY of these, it will be excluded.
Column names are case-insensitive.
If empty the following list is used:
- column: References
..regex: '^TP[0-9]*'
- column: References
..regex: '^FID'
- column: Part
..regex: 'mount.*hole'
- column: Part
..regex: 'solder.*bridge'
- column: Part
..regex: 'test.*point'
- column: Footprint
..regex 'test.*point'
- column: Footprint
..regex: 'mount.*hole'
- column: Footprint
..regex: 'fiducial' """
self.columns = KiBoMColumns
""" [list(dict)|list(string)] List of columns to display.
Can be just the name of the field """ # pragma: no cover
@staticmethod
def _create_minimal_ini():
""" KiBoM config to get only the headers """
with NamedTemporaryFile(mode='w', delete=False) as f:
f.write('[BOM_OPTIONS]\n')
f.write('output_file_name = %O\n')
f.write('hide_pcb_info = 1\n')
f.write('\n[IGNORE_COLUMNS]\n')
f.write('\n[REGEX_EXCLUDE]\n')
f.write('Part\t.*\n')
f.close()
return f.name
@staticmethod
def _get_columns():
""" Create a list of valid columns """
check_script(CMD_KIBOM, URL_KIBOM, '1.8.0')
config = None
csv = None
columns = None
try:
xml = os.path.splitext(os.path.abspath(GS.sch_file))[0]+'.xml'
config = os.path.abspath(KiBoMConfig._create_minimal_ini())
with NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as f:
csv = f.name
cmd = [CMD_KIBOM, '--cfg', config, '-d', os.path.dirname(csv), '-s', ',', xml, csv]
logger.debug('Running: '+str(cmd))
cmd_output = check_output(cmd, stderr=STDOUT)
with open(csv, 'rt') as f:
columns = f.readline().rstrip().split(',')
except CalledProcessError as e:
logger.error('Failed to get the column names for `{}`, error {}'.format(xml, e.returncode))
if e.output:
logger.debug('Output from command: '+e.output.decode())
exit(BOM_ERROR)
finally:
if config:
os.remove(config)
if csv:
os.remove(csv)
logger.debug('Output from command:\n'+cmd_output.decode())
return columns
def config(self):
super().config()
# digikey_link
if isinstance(self.digikey_link, type):
self.digikey_link = None
elif isinstance(self.digikey_link, list):
self.digikey_link = '\t'.join(self.digikey_link)
# group_fields
if isinstance(self.group_fields, type):
self.group_fields = None
# component_aliases
if isinstance(self.component_aliases, type):
self.component_aliases = None
else:
self.component_aliases = ['\t'.join(a) for a in self.component_aliases]
# include_only
if isinstance(self.include_only, type):
self.include_only = None
else:
self.include_only = [str(r) for r in self.include_only]
# exclude_any
if isinstance(self.exclude_any, type):
self.exclude_any = None
else:
self.exclude_any = [str(r) for r in self.exclude_any]
# columns
if isinstance(self.columns, type):
self.columns = None
self.col_rename = None
self.join = None
self.ignore = None
else:
# This is tricky
# Lower case available columns
valid_columns = self._get_columns()
valid_columns_l = {c.lower(): c for c in valid_columns}
logger.debug("Valid columns: "+str(valid_columns))
# Create the different lists
columns = []
columns_l = {}
self.col_rename = []
self.join = []
for col in self.columns:
if isinstance(col, str):
# Just a string, add to the list of used
new_col = col
else:
# A complete entry
new_col = col.field
# A column rename
if col.name:
self.col_rename.append(col.field+'\t'+col.name)
# Attach other columns
if col.join:
self.join.append(col.field+'\t'+col.join)
# Check this is a valid column
if new_col.lower() not in valid_columns_l:
raise KiPlotConfigurationError('Invalid column name `{}`'.format(new_col))
columns.append(new_col)
columns_l[new_col.lower()] = new_col
# Create a list of the columns we don't want
self.ignore = [c for c in valid_columns_l.keys() if c not in columns_l]
# And this is the ordered list with the case style defined by the user
self.columns = columns
def write_bool(self, attr):
""" Write a .INI bool option """
self.f.write('{} = {}\n'.format(attr, '1' if getattr(self, attr) else '0'))
def write_str(self, attr):
""" Write a .INI string option """
val = getattr(self, attr)
if val:
self.f.write('{} = {}\n'.format(attr, val))
def write_vector(self, vector, section):
""" Write a .INI section filled with a vector of strings """
if vector:
self.f.write('\n[{}]\n'.format(section))
for v in vector:
self.f.write(v+'\n')
def save(self, filename):
""" Create an INI file for KiBoM """
logger.debug("Saving KiBoM config to `{}`".format(filename))
with open(filename, 'wt') as f:
self.f = f
f.write('[BOM_OPTIONS]\n')
self.write_bool('ignore_dnf')
self.write_bool('html_generate_dnf')
self.write_bool('use_alt')
self.write_bool('number_rows')
self.write_bool('group_connectors')
self.write_bool('test_regex')
self.write_bool('merge_blank_fields')
self.write_str('fit_field')
self.write_str('datasheet_as_link')
self.write_bool('hide_headers')
self.write_bool('hide_pcb_info')
self.write_str('digikey_link')
# Ask to keep the output name
f.write('output_file_name = %O\n')
self.write_vector(self.group_fields, 'GROUP_FIELDS')
self.write_vector(self.include_only, 'REGEX_INCLUDE')
self.write_vector(self.exclude_any, 'REGEX_EXCLUDE')
self.write_vector(self.columns, 'COLUMN_ORDER')
self.write_vector(self.ignore, 'IGNORE_COLUMNS')
self.write_vector(self.col_rename, 'COLUMN_RENAME')
self.write_vector(self.join, 'JOIN')
self.write_vector(self.component_aliases, 'COMPONENT_ALIASES')
class KiBoMOptions(BaseOptions):
def __init__(self):
@ -22,28 +298,51 @@ class KiBoMOptions(BaseOptions):
are output to the BoM. To specify multiple variants,
with a BOM file exported for each variant, separate
variants with the ';' (semicolon) character """
self.conf = 'bom.ini'
""" BoM configuration file, relative to PCB """
self.conf = KiBoMConfig
""" [string|dict] BoM configuration file, relative to PCB.
You can also define the configuration here, will be stored in `config.kibom.ini` """
self.separator = ','
""" CSV Separator """
self.output = '%f-%i.%x'
""" filename for the output (%i=bom)"""
self.format = 'HTML'
""" [HTML,CSV] format for the BoM """ # pragma: no cover
""" [HTML,CSV,XML,XLSX] format for the BoM """ # pragma: no cover
def config(self):
super().config()
if isinstance(self.conf, type):
self.conf = 'bom.ini'
elif isinstance(self.conf, str):
if not self.conf:
self.conf = 'bom.ini'
else:
# A configuration
conf = os.path.abspath(os.path.join(GS.out_dir, CONFIG_FILENAME))
self.conf.save(conf)
self.conf = conf
def run(self, output_dir, board):
check_script(CMD_KIBOM, URL_KIBOM)
check_script(CMD_KIBOM, URL_KIBOM, '1.8.0')
format = self.format.lower()
prj = os.path.splitext(os.path.abspath(GS.pcb_file))[0]
config = os.path.join(os.path.dirname(os.path.abspath(GS.pcb_file)), self.conf)
logger.debug('Doing BoM, format {} prj: {} config: {}'.format(format, prj, config))
prj = os.path.splitext(os.path.abspath(GS.sch_file))[0]
config = os.path.join(os.path.dirname(os.path.abspath(GS.sch_file)), self.conf)
if self.output:
force_output = True
output = self.expand_filename_sch(output_dir, self.output, 'bom', format)
else:
force_output = False
output = os.path.basename(prj)+'.'+format
logger.debug('Doing BoM, format {} prj: {} config: {} output: {}'.format(format, prj, config, output))
cmd = [CMD_KIBOM,
'-n', str(self.number),
'--cfg', config,
'-s', self.separator]
'-s', self.separator,
'-d', output_dir]
if GS.debug_enabled:
cmd.append('-v')
if self.variant:
cmd.extend(['-r', self.variant])
cmd.extend([prj+'.xml', os.path.join(output_dir, os.path.basename(prj))+'.'+format])
cmd.extend([prj+'.xml', output])
logger.debug('Running: '+str(cmd))
try:
cmd_output = check_output(cmd, stderr=STDOUT)
@ -52,10 +351,14 @@ class KiBoMOptions(BaseOptions):
if e.output:
logger.debug('Output from command: '+e.output.decode())
exit(BOM_ERROR)
prj = os.path.basename(prj)
for f in glob(os.path.join(output_dir, prj)+'*.tmp'):
# I'm not sure when these files are left, but they are annoying
os.remove(f) # pragma: no cover
if force_output:
# When we create the .ini we can control the name.
# But when the user does it we can trust the settings.
m = search(r'Saving BOM File: (.*)', cmd_output.decode())
if m and m.group(1) != output:
cur = m.group(1)
logger.debug('Renaming output file: {} -> {}'.format(cur, output))
os.rename(cur, output)
logger.debug('Output from command:\n'+cmd_output.decode())

1
kiplot/out_pdf_pcb_print.py
View File

@ -1,4 +1,3 @@
import os
from subprocess import (call)
from .pre_base import BasePreFlight
from .error import (KiPlotConfigurationError)

53
tests/test_plot/test_bom.py
View File

@ -32,9 +32,9 @@ def test_bom_ok():
ctx.run(no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'bom.sch')])
# Check all outputs are there
# Default format is PRJ_bom_REVISION
name = os.path.join(BOM_DIR, prj+'_bom_')
csv = name+'.csv'
html = name+'_(pp).html'
name = os.path.join(BOM_DIR, prj)
csv = name+'-bom.csv'
html = name+'_bom__(pp).html'
ctx.expect_out_file(csv)
ctx.expect_out_file(html)
ctx.search_in_file(csv, ['R,R1,100', 'R,R2,200', 'C,C1,1uF'])
@ -46,3 +46,50 @@ def test_bom_fail():
ctx = context.TestContext('BoM_fail', 'bom_no_xml', 'bom', BOM_DIR)
ctx.run(BOM_ERROR)
ctx.clean_up()
def test_bom_cfg_1():
prj = 'bom'
ctx = context.TestContext('BoMConfig1', prj, 'bom_cfg', BOM_DIR)
ctx.run(no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'bom.sch')])
name = os.path.join(BOM_DIR, prj)
csv = name+'-bom.csv'
ctx.expect_out_file(csv)
ctx.search_in_file(csv, ['R,R1,100 ~', 'R,R2,200 ~', 'C,C1,1uF ~'])
ctx.clean_up()
def test_bom_cfg_2():
prj = 'bom'
ctx = context.TestContext('BoMConfig2', prj, 'bom_cfg2', BOM_DIR)
ctx.run(no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'bom.sch')])
name = os.path.join(BOM_DIR, prj)
csv = name+'-bom.csv'
ctx.expect_out_file(csv)
ctx.search_in_file(csv, ['R,100,R1', 'R,200,R2'])
ctx.search_not_in_file(csv, ['C,1uF,C1'])
ctx.clean_up()
def test_bom_cfg_3():
""" Without any column """
prj = 'bom'
ctx = context.TestContext('BoMConfig3', prj, 'bom_cfg3', BOM_DIR)
ctx.run(no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'bom.sch')])
name = os.path.join(BOM_DIR, prj)
csv = name+'-bom.csv'
ctx.expect_out_file(csv)
ctx.search_in_file(csv, ['R,R1,100', 'R,R2,200', 'C,C1,1uF'])
ctx.clean_up()
def test_bom_cfg_4():
""" Without join """
prj = 'bom'
ctx = context.TestContext('BoMConfig4', prj, 'bom_cfg4', BOM_DIR)
ctx.run(no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'bom.sch')])
name = os.path.join(BOM_DIR, prj)
csv = name+'-bom.csv'
ctx.expect_out_file(csv)
ctx.search_in_file(csv, ['R,100,R1', 'R,200,R2', 'C,1uF,C1'])
ctx.clean_up()

28
tests/test_plot/test_yaml_errors.py
View File

@ -38,6 +38,9 @@ Tests various errors in the config file
- Unknown section
- HPGL wrong pen_number
- KiBoM wrong format
- Invalid column name
- Failed to get columns
- Column without field
- PcbDraw
- Wrong color
@ -53,7 +56,7 @@ sys.path.insert(0, os.path.dirname(prev_dir))
# Utils import
from utils import context
sys.path.insert(0, os.path.dirname(prev_dir))
from kiplot.misc import (EXIT_BAD_CONFIG, PLOT_ERROR)
from kiplot.misc import (EXIT_BAD_CONFIG, PLOT_ERROR, BOM_ERROR)
PRJ = 'fail-project'
@ -399,11 +402,32 @@ def test_error_hpgl_pen_num():
def test_error_bom_wrong_format():
ctx = context.TestContext('BoMWrongFormat', PRJ, 'error_bom_wrong_format', '')
ctx.run(EXIT_BAD_CONFIG)
ctx.run(EXIT_BAD_CONFIG, no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'bom.sch')])
assert ctx.search_err("Option .?format.? must be any of")
ctx.clean_up()
def test_error_bom_column():
ctx = context.TestContext('BoMColumn', PRJ, 'error_bom_column', '')
ctx.run(EXIT_BAD_CONFIG, no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'bom.sch')])
assert ctx.search_err("Invalid column name .?Impossible.?")
ctx.clean_up()
def test_error_bom_no_columns():
ctx = context.TestContext('BoMNoColumns', PRJ, 'error_bom_column', '')
ctx.run(BOM_ERROR, no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'bom_no_xml.sch')])
assert ctx.search_err("Failed to get the column names")
ctx.clean_up()
def test_error_bom_no_field():
ctx = context.TestContext('BoMNoField', PRJ, 'error_bom_no_field', '')
ctx.run(EXIT_BAD_CONFIG, no_board_file=True, extra=['-e', os.path.join(ctx.get_board_dir(), 'fail-erc.sch')])
assert ctx.search_err("Missing or empty .?field.?")
ctx.clean_up()
def test_error_wrong_boolean():
ctx = context.TestContext('WrongBoolean', PRJ, 'error_wrong_boolean', '')
ctx.run(EXIT_BAD_CONFIG)

2
tests/yaml_samples/bom.kiplot.yaml
View File

@ -10,6 +10,8 @@ outputs:
options:
format: HTML # HTML or CSV
variant: pp
output: '' # Keep KiBoM name
conf: ''
- name: 'bom_csv'
comment: "Bill of Materials in CSV format"

21
tests/yaml_samples/bom_cfg.kiplot.yaml Normal file
View File

@ -0,0 +1,21 @@
# Example KiPlot config file
kiplot:
version: 1
outputs:
- name: 'bom_csv'
comment: "Bill of Materials in CSV format"
type: kibom
dir: BoM
options:
format: CSV # HTML or CSV
conf:
ignore_dnf: true
columns:
- Part
- References
- field: Value
join:
- Footprint
- Datasheet

33
tests/yaml_samples/bom_cfg2.kiplot.yaml Normal file
View File

@ -0,0 +1,33 @@
# Example KiPlot config file
kiplot:
version: 1
outputs:
- name: 'bom_csv'
comment: "Bill of Materials in CSV format"
type: kibom
dir: BoM
options:
format: CSV # HTML or CSV
conf:
ignore_dnf: true
digikey_link:
- digikey#
- digikey_alt#
component_aliases:
- ['r', 'r_small', 'res', 'resistor']
- - 'l'
- 'l_small'
- 'inductor'
include_only:
- field: References
regex: R\d
exclude_any:
- field: References
regex: C\d
columns:
- Part
- field: Value
name: Valor
join: Footprint
- References

14
tests/yaml_samples/bom_cfg3.kiplot.yaml Normal file
View File

@ -0,0 +1,14 @@
# Example KiPlot config file
kiplot:
version: 1
outputs:
- name: 'bom_csv'
comment: "Bill of Materials in CSV format"
type: kibom
dir: BoM
options:
format: CSV # HTML or CSV
conf:
ignore_dnf: true

18
tests/yaml_samples/bom_cfg4.kiplot.yaml Normal file
View File

@ -0,0 +1,18 @@
# Example KiPlot config file
kiplot:
version: 1
outputs:
- name: 'bom_csv'
comment: "Bill of Materials in CSV format"
type: kibom
dir: BoM
options:
format: CSV # HTML or CSV
conf:
ignore_dnf: true
columns:
- Part
- field: Value
name: Valor
- References

33
tests/yaml_samples/error_bom_column.kiplot.yaml Normal file
View File

@ -0,0 +1,33 @@
# Example KiPlot config file
kiplot:
version: 1
outputs:
- name: 'bom_csv'
comment: "Bill of Materials in CSV format"
type: kibom
dir: BoM
options:
format: CSV # HTML or CSV
conf:
ignore_dnf: true
digikey_link:
- digikey#
- digikey_alt#
component_aliases:
- ['r', 'r_small', 'res', 'resistor']
- - 'l'
- 'l_small'
- 'inductor'
include_only:
- field: References
regex: R\d
exclude_any:
- field: References
regex: C\d
columns:
- Part
- field: Impossible
name: Valor
join: Footprint
- References

32
tests/yaml_samples/error_bom_no_field.kiplot.yaml Normal file
View File

@ -0,0 +1,32 @@
# Example KiPlot config file
kiplot:
version: 1
outputs:
- name: 'bom_csv'
comment: "Bill of Materials in CSV format"
type: kibom
dir: BoM
options:
format: CSV # HTML or CSV
conf:
ignore_dnf: true
digikey_link:
- digikey#
- digikey_alt#
component_aliases:
- ['r', 'r_small', 'res', 'resistor']
- - 'l'
- 'l_small'
- 'inductor'
include_only:
- field: References
regex: R\d
exclude_any:
- field: References
regex: C\d
columns:
- Part
- name: Valor
join: Footprint
- References

67
tests/yaml_samples/kibom_1.kiplot.yaml Normal file
View File

@ -0,0 +1,67 @@
# Example KiPlot config file
kiplot:
version: 1
outputs:
- name: 'bom_html'
comment: "Bill of Materials in HTML format"
type: kibom
dir: BoM
options:
format: CSV # HTML or CSV
variant: pp
conf:
ignore_dnf: False
html_generate_dnf: False
use_alt: True
number_rows: False
group_connectors: False
test_regex: False
merge_blank_fields: False
fit_field: 'Configure'
datasheet_as_link: 'manf#'
hide_headers: False
hide_pcb_info: True
digikey_link:
- digikey#
- digikey_alt#
group_fields:
- Part
- Part Lib
component_aliases:
- ['r', 'r_small', 'res', 'resistor']
- ['l', 'l_small', 'inductor']
- - c
- c_small
- cap
- capacitor
include_only:
- column: 'References'
regex: 'C.*'
exclude_any:
- column: References
regex: '^TP[0-9]*'
- column: References
regex: '^FID'
- column: Part
regex: 'mount.*hole'
- column: Part
regex: 'solder.*bridge'
- column: Part
regex: 'test.*point'
- column: Footprint
regex: 'test.*point'
- column: Footprint
regex: 'mount.*hole'
- column: Footprint
regex: 'fiducial'
columns:
- References
- field: Value
name: Valor
- Part
- field: Description
join:
- Footprint
- Footprint Lib