From ab3bd7f0b3e9951cf3814552cb6267db7f0b050e Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Mon, 4 Oct 2021 14:44:43 -0300 Subject: [PATCH] Added a mechanism to import filters and variants. - Also to restrict which outputs are imported. - Fixes #88 --- CHANGELOG.md | 1 + README.md | 43 +++++++ docs/README.in | 43 +++++++ kibot/config_reader.py | 109 ++++++++++++++++-- kibot/misc.py | 3 + kibot/registrable.py | 8 ++ tests/test_plot/test_position.py | 10 ++ .../simple_position_rot_4.kibot.yaml | 30 +++++ .../simple_position_rot_4f.kibot.yaml | 18 +++ 9 files changed, 256 insertions(+), 9 deletions(-) create mode 100644 tests/yaml_samples/simple_position_rot_4.kibot.yaml create mode 100644 tests/yaml_samples/simple_position_rot_4f.kibot.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d4ca0a0..ee8d0725 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - skip_top: top components aren't rotated. - skip_bottom: bottom components aren't rotated. - XLSX BoM: option to control the logo scale (#84) +- Import mechanism for filters and variants (#88) ### Changed - Internal BoM: now components with different Tolerance, Voltage, Current diff --git a/README.md b/README.md index 52089d38..217da845 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ * [Supported outputs](#supported-outputs) * [Consolidating BoMs](#consolidating-boms) * [Importing outputs from another file](#importing-outputs-from-another-file) + * [Importing filters and variants from another file](#importing-filters-and-variants-from-another-file) * [Usage](#usage) * [Installation](#installation) * [Usage for CI/CD](#usage-for-cicd) @@ -1757,6 +1758,48 @@ import: This will import all the outputs from the listed files. +#### Importing filters and variants from another file + +This is a more complex case of the previous [Importing outputs from another file](#importing-outputs-from-another-file). +In this case you must use the more general syntax: + +```yaml +import: + - file: FILE_CONTAINING_THE_YAML_DEFINITIONS + outputs: LIST_OF_OUTPUTS + filters: LIST_OF_FILTERS + variants: LIST_OF_VARIANTS +``` + +This syntax is flexible. If you don't define which `outputs`, `filters` and/or `variants` all will be imported. So you can just omit them, like this: + +```yaml +import: + - file: FILE_CONTAINING_THE_YAML_DEFINITIONS +``` + +The `LIST_OF_items` can be a YAML list or just one string. Here is an example: + +```yaml +import: + - file: FILE_CONTAINING_THE_YAML_DEFINITIONS + outputs: one_name + filters: ['name1', 'name2'] +``` + +This will import the `one_name` output and the `name1` and `name2` filters. As `variants` is omitted, all variants will be imported. +You can also use the `all` and `none` special names, like this: + +```yaml +import: + - file: FILE_CONTAINING_THE_YAML_DEFINITIONS + outputs: all + filters: all + variants: none +``` + +This will import all outputs and filters, but not variants. + ## Usage If you need a template for the configuration file try: diff --git a/docs/README.in b/docs/README.in index 706b693e..4256f4e7 100644 --- a/docs/README.in +++ b/docs/README.in @@ -35,6 +35,7 @@ * [Supported outputs](#supported-outputs) * [Consolidating BoMs](#consolidating-boms) * [Importing outputs from another file](#importing-outputs-from-another-file) + * [Importing filters and variants from another file](#importing-filters-and-variants-from-another-file) * [Usage](#usage) * [Installation](#installation) * [Usage for CI/CD](#usage-for-cicd) @@ -760,6 +761,48 @@ import: This will import all the outputs from the listed files. +#### Importing filters and variants from another file + +This is a more complex case of the previous [Importing outputs from another file](#importing-outputs-from-another-file). +In this case you must use the more general syntax: + +```yaml +import: + - file: FILE_CONTAINING_THE_YAML_DEFINITIONS + outputs: LIST_OF_OUTPUTS + filters: LIST_OF_FILTERS + variants: LIST_OF_VARIANTS +``` + +This syntax is flexible. If you don't define which `outputs`, `filters` and/or `variants` all will be imported. So you can just omit them, like this: + +```yaml +import: + - file: FILE_CONTAINING_THE_YAML_DEFINITIONS +``` + +The `LIST_OF_items` can be a YAML list or just one string. Here is an example: + +```yaml +import: + - file: FILE_CONTAINING_THE_YAML_DEFINITIONS + outputs: one_name + filters: ['name1', 'name2'] +``` + +This will import the `one_name` output and the `name1` and `name2` filters. As `variants` is omitted, all variants will be imported. +You can also use the `all` and `none` special names, like this: + +```yaml +import: + - file: FILE_CONTAINING_THE_YAML_DEFINITIONS + outputs: all + filters: all + variants: none +``` + +This will import all outputs and filters, but not variants. + ## Usage If you need a template for the configuration file try: diff --git a/kibot/config_reader.py b/kibot/config_reader.py index e9aeac01..507e3dbb 100644 --- a/kibot/config_reader.py +++ b/kibot/config_reader.py @@ -171,6 +171,30 @@ class CfgYamlReader(object): except KiPlotConfigurationError as e: config_error("In `global` section: "+str(e)) + @staticmethod + def _config_error_import(fname, error): + if fname is None: + fname = '*unnamed*' + config_error('{} in {} import'.format(error, fname)) + + @staticmethod + def _parse_import_items(kind, fname, value): + if isinstance(value, str): + if value == 'all': + return None + elif value == 'none': + return [] + return [value] + if isinstance(value, list): + values = [] + for v in value: + if isinstance(v, str): + values.append(v) + else: + CfgYamlReader._config_error_import(fname, '`{}` items must be strings ({})'.format(kind, str(v))) + return values + CfgYamlReader._config_error_import(fname, '`{}` must be a string or a list ({})'.format(kind, str(v))) + def _parse_import(self, imp, name): """ Get imports """ logger.debug("Parsing imports: {}".format(imp)) @@ -179,20 +203,87 @@ class CfgYamlReader(object): # Import the files dir = os.path.dirname(os.path.abspath(name)) outputs = [] - for fn in imp: - if not isinstance(fn, str): - config_error("`import` items must be strings ({})".format(str(fn))) + for entry in imp: + if isinstance(entry, str): + fn = entry + outs = None + fils = [] + vars = [] + elif isinstance(entry, dict): + fname = outs = fils = vars = None + for k, v in entry.items(): + if k == 'file': + if not isinstance(v, str): + config_error("`import.file` must be a string ({})".format(str(v))) + fn = v + elif k == 'outputs': + outs = _parse_import_items('outputs', fname, v) + elif k == 'filters': + fils = _parse_import_items('filters', fname, v) + elif k == 'variants': + vars = _parse_import_items('variants', fname, v) + else: + self._config_error_import(fname, "unknown import entry `{}`".format(str(v))) + else: + config_error("`import` items must be strings or dicts ({})".format(str(fn))) if not os.path.isabs(fn): fn = os.path.join(dir, fn) if not os.path.isfile(fn): config_error("missing import file `{}`".format(fn)) + fn_rel = os.path.relpath(fn) data = self.load_yaml(open(fn)) - if 'outputs' in data: - outs = self._parse_outputs(data['outputs']) - outputs.extend(outs) - logger.debug('Outputs loaded from `{}`: {}'.format(os.path.relpath(fn), list(map(lambda c: c.name, outs)))) - else: - logger.warning(W_NOOUTPUTS+"No outputs found in `{}`".format(fn)) + # Outputs + if (outs is None or len(outs) > 0) and 'outputs' in data: + i_outs = self._parse_outputs(data['outputs']) + if outs is not None: + sel_outs = [] + for o in i_outs: + if o.name in outs: + sel_outs.append(o) + outs.remove(o) + for o in outs: + logger.warning(W_UNKOUT+"can't import `{}` output from `{}` (missing)".format(o, fn_rel)) + else: + sel_outs = i_outs + if len(sel_outs) == 0: + logger.warning(W_NOOUTPUTS+"No outputs found in `{}`".format(fn_rel)) + else: + outputs.extend(sel_outs) + logger.debug('Outputs loaded from `{}`: {}'.format(fn_rel, list(map(lambda c: c.name, sel_outs)))) + # Filters + if fils is None or len(fils) > 0 and 'filters' in data: + i_fils = self._parse_filters(data['filters']) + if fils is not None: + sel_fils = {} + for f in fils: + if f in i_fils: + sel_fils[f] = i_fils[f] + else: + logger.warning(W_UNKOUT+"can't import `{}` filter from `{}` (missing)".format(f, fn_rel)) + else: + sel_fils = i_fils + if len(sel_fils) == 0: + logger.warning(W_NOFILTERS+"No filters found in `{}`".format(fn_rel)) + else: + RegOutput.add_filters(sel_fils) + logger.debug('Filters loaded from `{}`: {}'.format(fn_rel, sel_fils.keys())) + # Variants + if vars is None or len(vars) > 0 and 'variants' in data: + i_vars = self._parse_variants(data['variants']) + if vars is not None: + sel_vars = {} + for f in vars: + if f in i_vars: + sel_vars[f] = i_vars[f] + else: + logger.warning(W_UNKOUT+"can't import `{}` variant from `{}` (missing)".format(f, fn_rel)) + else: + sel_vars = i_vars + if len(sel_vars) == 0: + logger.warning(W_NOVARIANTS+"No variants found in `{}`".format(fn_rel)) + else: + RegOutput.add_variants(sel_vars) + logger.debug('Variants loaded from `{}`: {}'.format(fn_rel, sel_vars.keys())) return outputs def load_yaml(self, fstream): diff --git a/kibot/misc.py b/kibot/misc.py index aba822e2..f49b0104 100644 --- a/kibot/misc.py +++ b/kibot/misc.py @@ -198,6 +198,9 @@ W_UNKDIST = '(W063) ' W_UNKCUR = '(W064) ' W_NONETLIST = '(W065) ' W_NOKICOST = '(W066) ' +W_UNKOUT = '(W067) ' +W_NOFILTERS = '(W068) ' +W_NOVARIANTS = '(W069) ' class Rect(object): diff --git a/kibot/registrable.py b/kibot/registrable.py index d1f84368..a6670ea7 100644 --- a/kibot/registrable.py +++ b/kibot/registrable.py @@ -49,6 +49,10 @@ class RegOutput(Optionable, Registrable): def set_variants(variants): RegOutput._def_variants = variants + @staticmethod + def add_variants(variants): + RegOutput._def_variants.update(variants) + @staticmethod def is_variant(name): return name in RegOutput._def_variants @@ -61,6 +65,10 @@ class RegOutput(Optionable, Registrable): def set_filters(filters): RegOutput._def_filters = filters + @staticmethod + def add_filters(filters): + RegOutput._def_filters.update(filters) + @staticmethod def is_filter(name): return name in RegOutput._def_filters diff --git a/tests/test_plot/test_position.py b/tests/test_plot/test_position.py index d70f6175..c6ce3452 100644 --- a/tests/test_plot/test_position.py +++ b/tests/test_plot/test_position.py @@ -239,6 +239,16 @@ def test_position_rot_3(test_dir): ctx.clean_up() +def test_position_rot_4(test_dir): + prj = 'light_control' + ctx = context.TestContext(test_dir, 'test_position_rot_4', prj, 'simple_position_rot_4', POS_DIR) + ctx.run(extra_debug=True) + output = prj+'_cpl_jlc_aux.csv' + ctx.expect_out_file(output) + ctx.compare_txt(output) + ctx.clean_up() + + def test_rot_bottom(test_dir): ctx = context.TestContext(test_dir, 'test_rot_bottom', 'comp_bottom', 'simple_position_rot_bottom', POS_DIR) ctx.run() diff --git a/tests/yaml_samples/simple_position_rot_4.kibot.yaml b/tests/yaml_samples/simple_position_rot_4.kibot.yaml new file mode 100644 index 00000000..85b7b6de --- /dev/null +++ b/tests/yaml_samples/simple_position_rot_4.kibot.yaml @@ -0,0 +1,30 @@ +kibot: + version: 1 + +import: + - file: simple_position_rot_4f.kibot.yaml + +outputs: + - name: 'position' + comment: "Pick and place file, JLC style" + type: position + options: + variant: rotated + output: '%f_cpl_jlc_aux.%x' + format: CSV + units: millimeters + separate_files_for_front_and_back: false + only_smd: true + columns: + - id: Ref + name: Designator + - Val + - Package + - id: PosX + name: "Mid X" + - id: PosY + name: "Mid Y" + - id: Rot + name: Rotation + - id: Side + name: Layer diff --git a/tests/yaml_samples/simple_position_rot_4f.kibot.yaml b/tests/yaml_samples/simple_position_rot_4f.kibot.yaml new file mode 100644 index 00000000..24f5bd3a --- /dev/null +++ b/tests/yaml_samples/simple_position_rot_4f.kibot.yaml @@ -0,0 +1,18 @@ +kibot: + version: 1 + +filters: + - name: only_jlc_parts + comment: 'Only parts with JLC code' + type: generic + include_only: + - column: 'LCSC#' + regex: '^C\d+' + +variants: + - name: rotated + comment: 'Just a place holder for the rotation filter' + type: kibom + variant: rotated + pre_transform: _rot_footprint +