Added a mechanism to import filters and variants.

- Also to restrict which outputs are imported.
- Fixes #88

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - skip_top: top components aren't rotated.
   - skip_bottom: bottom components aren't rotated.
 - XLSX BoM: option to control the logo scale (#84)
+- Import mechanism for filters and variants (#88)
 
 ### Changed
 - Internal BoM: now components with different Tolerance, Voltage, Current

README.md

@@ -35,6 +35,7 @@
     * [Supported outputs](#supported-outputs)
     * [Consolidating BoMs](#consolidating-boms)
     * [Importing outputs from another file](#importing-outputs-from-another-file)
+    * [Importing filters and variants from another file](#importing-filters-and-variants-from-another-file)
 * [Usage](#usage)
 * [Installation](#installation)
 * [Usage for CI/CD](#usage-for-cicd)
@@ -1757,6 +1758,48 @@ import:
 
 This will import all the outputs from the listed files.
 
+#### Importing filters and variants from another file
+
+This is a more complex case of the previous [Importing outputs from another file](#importing-outputs-from-another-file).
+In this case you must use the more general syntax:
+
+```yaml
+import:
+  - file: FILE_CONTAINING_THE_YAML_DEFINITIONS
+    outputs: LIST_OF_OUTPUTS
+    filters: LIST_OF_FILTERS
+    variants: LIST_OF_VARIANTS
+```
+
+This syntax is flexible. If you don't define which `outputs`, `filters` and/or `variants` all will be imported. So you can just omit them, like this:
+
+```yaml
+import:
+  - file: FILE_CONTAINING_THE_YAML_DEFINITIONS
+```
+
+The `LIST_OF_items` can be a YAML list or just one string. Here is an example:
+
+```yaml
+import:
+  - file: FILE_CONTAINING_THE_YAML_DEFINITIONS
+    outputs: one_name
+    filters: ['name1', 'name2']
+```
+
+This will import the `one_name` output and the `name1` and `name2` filters. As `variants` is omitted, all variants will be imported.
+You can also use the `all` and `none` special names, like this:
+
+```yaml
+import:
+  - file: FILE_CONTAINING_THE_YAML_DEFINITIONS
+    outputs: all
+    filters: all
+    variants: none
+```
+
+This will import all outputs and filters, but not variants.
+
 ## Usage
 
 If you need a template for the configuration file try:

docs/README.in

@@ -35,6 +35,7 @@
     * [Supported outputs](#supported-outputs)
     * [Consolidating BoMs](#consolidating-boms)
     * [Importing outputs from another file](#importing-outputs-from-another-file)
+    * [Importing filters and variants from another file](#importing-filters-and-variants-from-another-file)
 * [Usage](#usage)
 * [Installation](#installation)
 * [Usage for CI/CD](#usage-for-cicd)
@@ -760,6 +761,48 @@ import:
 
 This will import all the outputs from the listed files.
 
+#### Importing filters and variants from another file
+
+This is a more complex case of the previous [Importing outputs from another file](#importing-outputs-from-another-file).
+In this case you must use the more general syntax:
+
+```yaml
+import:
+  - file: FILE_CONTAINING_THE_YAML_DEFINITIONS
+    outputs: LIST_OF_OUTPUTS
+    filters: LIST_OF_FILTERS
+    variants: LIST_OF_VARIANTS
+```
+
+This syntax is flexible. If you don't define which `outputs`, `filters` and/or `variants` all will be imported. So you can just omit them, like this:
+
+```yaml
+import:
+  - file: FILE_CONTAINING_THE_YAML_DEFINITIONS
+```
+
+The `LIST_OF_items` can be a YAML list or just one string. Here is an example:
+
+```yaml
+import:
+  - file: FILE_CONTAINING_THE_YAML_DEFINITIONS
+    outputs: one_name
+    filters: ['name1', 'name2']
+```
+
+This will import the `one_name` output and the `name1` and `name2` filters. As `variants` is omitted, all variants will be imported.
+You can also use the `all` and `none` special names, like this:
+
+```yaml
+import:
+  - file: FILE_CONTAINING_THE_YAML_DEFINITIONS
+    outputs: all
+    filters: all
+    variants: none
+```
+
+This will import all outputs and filters, but not variants.
+
 ## Usage
 
 If you need a template for the configuration file try:

kibot/config_reader.py

@@ -171,6 +171,30 @@ class CfgYamlReader(object):
         except KiPlotConfigurationError as e:
             config_error("In `global` section: "+str(e))
 
+    @staticmethod
+    def _config_error_import(fname, error):
+        if fname is None:
+            fname = '*unnamed*'
+        config_error('{} in {} import'.format(error, fname))
+
+    @staticmethod
+    def _parse_import_items(kind, fname, value):
+        if isinstance(value, str):
+            if value == 'all':
+                return None
+            elif value == 'none':
+                return []
+            return [value]
+        if isinstance(value, list):
+            values = []
+            for v in value:
+                if isinstance(v, str):
+                    values.append(v)
+                else:
+                    CfgYamlReader._config_error_import(fname, '`{}` items must be strings ({})'.format(kind, str(v)))
+            return values
+        CfgYamlReader._config_error_import(fname, '`{}` must be a string or a list ({})'.format(kind, str(v)))
+
     def _parse_import(self, imp, name):
         """ Get imports """
         logger.debug("Parsing imports: {}".format(imp))
@@ -179,20 +203,87 @@ class CfgYamlReader(object):
         # Import the files
         dir = os.path.dirname(os.path.abspath(name))
         outputs = []
-        for fn in imp:
+        for entry in imp:
-            if not isinstance(fn, str):
+            if isinstance(entry, str):
-                config_error("`import` items must be strings ({})".format(str(fn)))
+                fn = entry
+                outs = None
+                fils = []
+                vars = []
+            elif isinstance(entry, dict):
+                fname = outs = fils = vars = None
+                for k, v in entry.items():
+                    if k == 'file':
+                        if not isinstance(v, str):
+                            config_error("`import.file` must be a string ({})".format(str(v)))
+                        fn = v
+                    elif k == 'outputs':
+                        outs = _parse_import_items('outputs', fname, v)
+                    elif k == 'filters':
+                        fils = _parse_import_items('filters', fname, v)
+                    elif k == 'variants':
+                        vars = _parse_import_items('variants', fname, v)
+                    else:
+                        self._config_error_import(fname, "unknown import entry `{}`".format(str(v)))
+            else:
+                config_error("`import` items must be strings or dicts ({})".format(str(fn)))
             if not os.path.isabs(fn):
                 fn = os.path.join(dir, fn)
             if not os.path.isfile(fn):
                 config_error("missing import file `{}`".format(fn))
+            fn_rel = os.path.relpath(fn)
             data = self.load_yaml(open(fn))
-            if 'outputs' in data:
+            # Outputs
-                outs = self._parse_outputs(data['outputs'])
+            if (outs is None or len(outs) > 0) and 'outputs' in data:
-                outputs.extend(outs)
+                i_outs = self._parse_outputs(data['outputs'])
-                logger.debug('Outputs loaded from `{}`: {}'.format(os.path.relpath(fn), list(map(lambda c: c.name, outs))))
+                if outs is not None:
+                    sel_outs = []
+                    for o in i_outs:
+                        if o.name in outs:
+                            sel_outs.append(o)
+                            outs.remove(o)
+                    for o in outs:
+                        logger.warning(W_UNKOUT+"can't import `{}` output from `{}` (missing)".format(o, fn_rel))
                 else:
-                logger.warning(W_NOOUTPUTS+"No outputs found in `{}`".format(fn))
+                    sel_outs = i_outs
+                if len(sel_outs) == 0:
+                    logger.warning(W_NOOUTPUTS+"No outputs found in `{}`".format(fn_rel))
+                else:
+                    outputs.extend(sel_outs)
+                logger.debug('Outputs loaded from `{}`: {}'.format(fn_rel, list(map(lambda c: c.name, sel_outs))))
+            # Filters
+            if fils is None or len(fils) > 0 and 'filters' in data:
+                i_fils = self._parse_filters(data['filters'])
+                if fils is not None:
+                    sel_fils = {}
+                    for f in fils:
+                        if f in i_fils:
+                            sel_fils[f] = i_fils[f]
+                        else:
+                            logger.warning(W_UNKOUT+"can't import `{}` filter from `{}` (missing)".format(f, fn_rel))
+                else:
+                    sel_fils = i_fils
+                if len(sel_fils) == 0:
+                    logger.warning(W_NOFILTERS+"No filters found in `{}`".format(fn_rel))
+                else:
+                    RegOutput.add_filters(sel_fils)
+                    logger.debug('Filters loaded from `{}`: {}'.format(fn_rel, sel_fils.keys()))
+            # Variants
+            if vars is None or len(vars) > 0 and 'variants' in data:
+                i_vars = self._parse_variants(data['variants'])
+                if vars is not None:
+                    sel_vars = {}
+                    for f in vars:
+                        if f in i_vars:
+                            sel_vars[f] = i_vars[f]
+                        else:
+                            logger.warning(W_UNKOUT+"can't import `{}` variant from `{}` (missing)".format(f, fn_rel))
+                else:
+                    sel_vars = i_vars
+                if len(sel_vars) == 0:
+                    logger.warning(W_NOVARIANTS+"No variants found in `{}`".format(fn_rel))
+                else:
+                    RegOutput.add_variants(sel_vars)
+                    logger.debug('Variants loaded from `{}`: {}'.format(fn_rel, sel_vars.keys()))
         return outputs
 
     def load_yaml(self, fstream):

kibot/misc.py

@@ -198,6 +198,9 @@ W_UNKDIST = '(W063) '
 W_UNKCUR = '(W064) '
 W_NONETLIST = '(W065) '
 W_NOKICOST = '(W066) '
+W_UNKOUT = '(W067) '
+W_NOFILTERS = '(W068) '
+W_NOVARIANTS = '(W069) '
 
 
 class Rect(object):

kibot/registrable.py

@@ -49,6 +49,10 @@ class RegOutput(Optionable, Registrable):
     def set_variants(variants):
         RegOutput._def_variants = variants
 
+    @staticmethod
+    def add_variants(variants):
+        RegOutput._def_variants.update(variants)
+
     @staticmethod
     def is_variant(name):
         return name in RegOutput._def_variants
@@ -61,6 +65,10 @@ class RegOutput(Optionable, Registrable):
     def set_filters(filters):
         RegOutput._def_filters = filters
 
+    @staticmethod
+    def add_filters(filters):
+        RegOutput._def_filters.update(filters)
+
     @staticmethod
     def is_filter(name):
         return name in RegOutput._def_filters

tests/test_plot/test_position.py

@@ -239,6 +239,16 @@ def test_position_rot_3(test_dir):
     ctx.clean_up()
 
 
+def test_position_rot_4(test_dir):
+    prj = 'light_control'
+    ctx = context.TestContext(test_dir, 'test_position_rot_4', prj, 'simple_position_rot_4', POS_DIR)
+    ctx.run(extra_debug=True)
+    output = prj+'_cpl_jlc_aux.csv'
+    ctx.expect_out_file(output)
+    ctx.compare_txt(output)
+    ctx.clean_up()
+
+
 def test_rot_bottom(test_dir):
     ctx = context.TestContext(test_dir, 'test_rot_bottom', 'comp_bottom', 'simple_position_rot_bottom', POS_DIR)
     ctx.run()

tests/yaml_samples/simple_position_rot_4.kibot.yaml

@@ -0,0 +1,30 @@
+kibot:
+  version: 1
+
+import:
+  - file: simple_position_rot_4f.kibot.yaml
+
+outputs:
+  - name: 'position'
+    comment: "Pick and place file, JLC style"
+    type: position
+    options:
+      variant: rotated
+      output: '%f_cpl_jlc_aux.%x'
+      format: CSV
+      units: millimeters
+      separate_files_for_front_and_back: false
+      only_smd: true
+      columns:
+        - id: Ref
+          name: Designator
+        - Val
+        - Package
+        - id: PosX
+          name: "Mid X"
+        - id: PosY
+          name: "Mid Y"
+        - id: Rot
+          name: Rotation
+        - id: Side
+          name: Layer

tests/yaml_samples/simple_position_rot_4f.kibot.yaml

@@ -0,0 +1,18 @@
+kibot:
+  version: 1
+
+filters:
+  - name: only_jlc_parts
+    comment: 'Only parts with JLC code'
+    type: generic
+    include_only:
+      - column: 'LCSC#'
+        regex: '^C\d+'
+
+variants:
+  - name: rotated
+    comment: 'Just a place holder for the rotation filter'
+    type: kibom
+    variant: rotated
+    pre_transform: _rot_footprint
+