From 281ed3be7eb3df99513b24c62757c5ec49077b1c Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Wed, 24 May 2023 09:39:06 -0300 Subject: [PATCH] [Imports][Added] Allow to define @TAGS@ values during import - Also added defaults - BTW: disabled the YAML lint crap that insists in checking excluded files --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 2 + README.md | 75 +++++++++++++++ docs/README.in | 75 +++++++++++++++ kibot/config_reader.py | 96 +++++++++++++++---- tests/test_plot/test_misc.py | 13 +++ .../definitions_gerbers.kibot.yaml | 14 +++ .../definitions_level_1.kibot.yaml | 12 +++ tests/yaml_samples/definitions_top.kibot.yaml | 14 +++ 9 files changed, 284 insertions(+), 19 deletions(-) create mode 100644 tests/yaml_samples/definitions_gerbers.kibot.yaml create mode 100644 tests/yaml_samples/definitions_level_1.kibot.yaml create mode 100644 tests/yaml_samples/definitions_top.kibot.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 384f990c..ee636952 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,6 @@ files: .*/Makefile| .*\.sh| .*\.py| - .*\.yaml| .*\.md )$ exclude: @@ -14,6 +13,7 @@ exclude: submodules/.*| kibot/PyPDF2/.*| kibot/PcbDraw/.*| + tests/yaml_samples/definitions_*| tests/yaml_samples/simple_position_csv_pre.kibot.yaml )$ repos: diff --git a/CHANGELOG.md b/CHANGELOG.md index ffd4fc4a..6fbed851 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 policy implementation. See [KiCad issue 14360](https://gitlab.com/kicad/code/kicad/-/issues/14360). (#441) + - Default values for @TAGS@ + - Parametrizable imports - Command line: - `--list-variants` List all available variants (See #434) - `--only-names` to make `--list` list only output names diff --git a/README.md b/README.md index c9a830ca..3414cbdc 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,14 @@ * [Consolidating BoMs](#consolidating-boms) * [Importing outputs from another file](#importing-outputs-from-another-file) * [Importing other stuff from another file](#importing-other-stuff-from-another-file) + * [Parametrizable imports](#parametrizable-imports) * [Importing internal templates](#importing-internal-templates) * [Using other output as base for a new one](#using-other-output-as-base-for-a-new-one) * [Grouping outputs](#grouping-outputs) * [Doing YAML substitution or preprocessing](#doing-yaml-substitution-or-preprocessing) + * [Default definitions](#default-definitions) + * [Definitions during import](#definitions-during-import) + * [Recursive definitions expansion](#recursive-definitions-expansion) * [Usage](#usage) * [Usage for CI/CD](#usage-for-cicd) * [GitHub Actions](#usage-of-github-actions) @@ -5385,6 +5389,10 @@ import: is_external: true ``` +#### Parametrizable imports + +You can create imports that are parametrizable. For this you must use the mechanism explained in +the [Doing YAML substitution or preprocessing](#doing-yaml-substitution-or-preprocessing) section. #### Importing internal templates @@ -5574,6 +5582,73 @@ This is applied to all YAML files loaded, so this propagates to all the imported You can use `-E` as many times as you need. +#### Default definitions + +A configuration file using the `@VARIABLE@` tags won't be usable unless you provide proper +values for **all** de used variables. When using various tags this could be annoying. +KiBot supports defining default values for the tags. Here is an example: + +```yaml +kibot: + version: 1 + +outputs: + - name: 'gerbers_@ID@' + comment: "Gerbers with definitions" + type: gerber + output_id: _@ID@ + layers: @LAYERS@ +... +definitions: + ID: def_id + LAYERS: F.Cu +``` + +Note that from the YAML point this is two documents in the same file. The second document +is used to provide default values for the definitions. As defaults they have the lowest +precedence. + +#### Definitions during import + +When importing a configuration you can specify values for the `@VARIABLE@` tags. This +enables the creation of parametrizable imports. Using the example depicted in +[Default definitions](#default-definitions) saved to a file named *simple.kibot.yaml* +you can use: + +```yaml +kibot: + version: 1 + +import: + - file: simple.kibot.yaml + definitions: + ID: external_copper + LAYERS: "[F.Cu, B.Cu]" +``` + +This will import *simple.kibot.yaml* and use these particular values. Note that they +have more precedence than the definitions found in *simple.kibot.yaml*, but less +precedence than any value passed from the command line. + +#### Recursive definitions expansion + +When KiBot expands the `@VARIABLE@` tags it first applies all the replacements defined +in the command line, and then all the values collected from the `definitions`. After +doing a round of replacements KiBot tries to do another. This process is repeated until +nothing is replaced or we reach 20 iterations. So you can define a tag that contains +another tag. + +As an example, if the configuration shown in [Definitions during import](#definitions-during-import) +is stored in a file named *top.kibot.yaml* you could use: + +```shell +kibot -v -c top.kibot.yaml -E ID=@LAYERS@ +``` + +This will generate gerbers for the front/top and bottom layers using *[F.Cu, B.Cu]* as +output id. So you'll get *light_control-B_Cu_[F.Cu, B.Cu].gbr* and +*light_control-F_Cu_[F.Cu, B.Cu].gbr*. + ## Usage For a quick start just go to the project's dir and run: diff --git a/docs/README.in b/docs/README.in index bd9bff58..b02ce6b9 100644 --- a/docs/README.in +++ b/docs/README.in @@ -58,10 +58,14 @@ * [Consolidating BoMs](#consolidating-boms) * [Importing outputs from another file](#importing-outputs-from-another-file) * [Importing other stuff from another file](#importing-other-stuff-from-another-file) + * [Parametrizable imports](#parametrizable-imports) * [Importing internal templates](#importing-internal-templates) * [Using other output as base for a new one](#using-other-output-as-base-for-a-new-one) * [Grouping outputs](#grouping-outputs) * [Doing YAML substitution or preprocessing](#doing-yaml-substitution-or-preprocessing) + * [Default definitions](#default-definitions) + * [Definitions during import](#definitions-during-import) + * [Recursive definitions expansion](#recursive-definitions-expansion) * [Usage](#usage) * [Usage for CI/CD](#usage-for-cicd) * [GitHub Actions](#usage-of-github-actions) @@ -1276,6 +1280,10 @@ import: is_external: true ``` +#### Parametrizable imports + +You can create imports that are parametrizable. For this you must use the mechanism explained in +the [Doing YAML substitution or preprocessing](#doing-yaml-substitution-or-preprocessing) section. #### Importing internal templates @@ -1465,6 +1473,73 @@ This is applied to all YAML files loaded, so this propagates to all the imported You can use `-E` as many times as you need. +#### Default definitions + +A configuration file using the `@VARIABLE@` tags won't be usable unless you provide proper +values for **all** de used variables. When using various tags this could be annoying. +KiBot supports defining default values for the tags. Here is an example: + +```yaml +kibot: + version: 1 + +outputs: + - name: 'gerbers_@ID@' + comment: "Gerbers with definitions" + type: gerber + output_id: _@ID@ + layers: @LAYERS@ +... +definitions: + ID: def_id + LAYERS: F.Cu +``` + +Note that from the YAML point this is two documents in the same file. The second document +is used to provide default values for the definitions. As defaults they have the lowest +precedence. + +#### Definitions during import + +When importing a configuration you can specify values for the `@VARIABLE@` tags. This +enables the creation of parametrizable imports. Using the example depicted in +[Default definitions](#default-definitions) saved to a file named *simple.kibot.yaml* +you can use: + +```yaml +kibot: + version: 1 + +import: + - file: simple.kibot.yaml + definitions: + ID: external_copper + LAYERS: "[F.Cu, B.Cu]" +``` + +This will import *simple.kibot.yaml* and use these particular values. Note that they +have more precedence than the definitions found in *simple.kibot.yaml*, but less +precedence than any value passed from the command line. + +#### Recursive definitions expansion + +When KiBot expands the `@VARIABLE@` tags it first applies all the replacements defined +in the command line, and then all the values collected from the `definitions`. After +doing a round of replacements KiBot tries to do another. This process is repeated until +nothing is replaced or we reach 20 iterations. So you can define a tag that contains +another tag. + +As an example, if the configuration shown in [Definitions during import](#definitions-during-import) +is stored in a file named *top.kibot.yaml* you could use: + +```shell +kibot -v -c top.kibot.yaml -E ID=@LAYERS@ +``` + +This will generate gerbers for the front/top and bottom layers using *[F.Cu, B.Cu]* as +output id. So you'll get *light_control-B_Cu_[F.Cu, B.Cu].gbr* and +*light_control-F_Cu_[F.Cu, B.Cu].gbr*. + ## Usage For a quick start just go to the project's dir and run: diff --git a/kibot/config_reader.py b/kibot/config_reader.py index e725c82d..7d9f841a 100644 --- a/kibot/config_reader.py +++ b/kibot/config_reader.py @@ -9,13 +9,15 @@ Class to read KiBot config files """ +from copy import deepcopy import collections +from collections import OrderedDict import difflib import io -import os import json +import os +import re from sys import (exit, maxsize) -from collections import OrderedDict from .error import KiPlotConfigurationError, config_error from .misc import (NO_YAML_MODULE, EXIT_BAD_ARGS, EXAMPLE_CFG, WONT_OVERWRITE, W_NOOUTPUTS, W_UNKOUT, W_NOFILTERS, @@ -62,6 +64,15 @@ def update_dict(d, u): return d +def do_replace(k, v, content, replaced): + key = '@'+k+'@' + if key in content: + logger.debugl(2, '- Replacing {} -> {}'.format(key, v)) + content = content.replace(key, str(v)) + replaced = True + return content, replaced + + class CollectedImports(object): def __init__(self): super().__init__() @@ -478,7 +489,7 @@ class CfgYamlReader(object): raise KiPlotConfigurationError("Missing import file `{}`".format(fn)) return fn, is_internal - def _parse_import(self, imp, name, apply=True, depth=0): + def _parse_import(self, imp, name, collected_definitions, apply=True, depth=0): """ Get imports """ logger.debug("Parsing imports: {}".format(imp)) depth += 1 @@ -491,6 +502,7 @@ class CfgYamlReader(object): all_collected = CollectedImports() for entry in imp: explicit_fils = explicit_vars = explicit_globals = explicit_pres = explicit_groups = False + local_defs = {} if isinstance(entry, str): is_external = True fn = entry @@ -531,6 +543,10 @@ class CfgYamlReader(object): elif k == 'groups': groups = self._parse_import_items(k, fn, v) explicit_groups = True + elif k == 'definitions': + if not isinstance(v, dict): + CfgYamlReader._config_error_import(fn, 'definitions must be a dict') + local_defs = v else: self._config_error_import(fn, "Unknown import entry `{}`".format(str(v))) if fn is None: @@ -539,13 +555,19 @@ class CfgYamlReader(object): raise KiPlotConfigurationError("`import` items must be strings or dicts ({})".format(str(entry))) fn, is_internal = self.check_import_file_name(dir_name, fn, is_external) fn_rel = os.path.relpath(fn) - data = self.load_yaml(open(fn)) + # Create a new dict for definitions applying the new ones and nake it the last + cur_definitions = deepcopy(collected_definitions[-1]) + cur_definitions.update(local_defs) + collected_definitions.append(cur_definitions) + # Now load the YAML + data = self.load_yaml(open(fn), collected_definitions) if 'import' in data: # Do a recursive import - imported = self._parse_import(data['import'], fn, apply=False, depth=depth) + imported = self._parse_import(data['import'], fn, collected_definitions, apply=False, depth=depth) else: # Nothing to import, start fresh imported = CollectedImports() + collected_definitions.pop() # Parse and filter all stuff, add them to all_collected # Outputs all_collected.outputs.extend(self._parse_import_outputs(outs, explicit_outs, fn_rel, data, imported)) @@ -572,18 +594,55 @@ class CfgYamlReader(object): RegOutput.add_groups(all_collected.groups, fn_rel) return all_collected - def load_yaml(self, fstream): - if GS.cli_defines: - # Load the file to memory so we can preprocess it - content = fstream.read() + def load_yaml(self, fstream, collected_definitions): + # We support some sort of defaults for the -E definitions + # To implement it we use a separated "document" inside the same file + # Load the file to memory so we can preprocess it + content = fstream.read() + docs = re.split(r"^\.\.\.$", content, flags=re.M) + local_defs = None + if len(docs) > 1: + definitions = None + for doc in docs: + if re.search(r"^kibot:\s*$", doc, flags=re.M): + content = doc + elif re.search(r"^definitions:\s*$", doc, flags=re.M): + definitions = doc + if definitions: + logger.debug("Found local definitions") + try: + data = yaml.safe_load(io.StringIO(definitions)) + except yaml.YAMLError as e: + raise KiPlotConfigurationError("Error loading YAML "+str(e)) + local_defs = data.get('definitions') + if not local_defs: + raise KiPlotConfigurationError("Error loading default definitions from config") + if not isinstance(local_defs, dict): + raise KiPlotConfigurationError("Error default definitions must be a dict") + logger.debug("- Local definitions: "+str(local_defs)) + logger.debug("- Current definitions: "+str(collected_definitions[-1])) + local_defs.update(collected_definitions[-1]) + collected_definitions[-1] = local_defs + logger.debug("- Updated definitions: "+str(collected_definitions[-1])) + # Apply the definitions + if GS.cli_defines or collected_definitions[-1]: logger.debug('Applying preprocessor definitions') - # Replace all - for k, v in GS.cli_defines.items(): - key = '@'+k+'@' - logger.debugl(2, '- Replacing {} -> {}'.format(key, v)) - content = content.replace(key, v) - # Create an stream from the string - fstream = io.StringIO(content) + replaced = True + depth = 0 + while replaced and depth < 20: + replaced = False + depth += 1 + # Replace all + logger.debug("- Applying CLI definitions: "+str(GS.cli_defines)) + for k, v in GS.cli_defines.items(): + content, replaced = do_replace(k, v, content, replaced) + logger.debug("- Applying collected definitions: "+str(collected_definitions[-1])) + for k, v in collected_definitions[-1].items(): + content, replaced = do_replace(k, v, content, replaced) + if depth >= 20: + logger.error('Maximum depth of definition replacements reached, loop?') + # Create an stream from the string + fstream = io.StringIO(content) try: data = yaml.safe_load(fstream) except yaml.YAMLError as e: @@ -606,7 +665,8 @@ class CfgYamlReader(object): :param fstream: file stream of a config YAML file """ - data = self.load_yaml(fstream) + collected_definitions = [{}] + data = self.load_yaml(fstream, collected_definitions) # Analyze the version # Currently just checks for v1 v1 = data.get('kiplot', None) @@ -622,7 +682,7 @@ class CfgYamlReader(object): # Look for imports v1 = data.get('import', None) if v1: - self._parse_import(v1, fstream.name) + self._parse_import(v1, fstream.name, collected_definitions) # Look for globals # If no globals defined initialize them with default values self._parse_global(data.get('global', {})) diff --git a/tests/test_plot/test_misc.py b/tests/test_plot/test_misc.py index 589c524c..9fd6b428 100644 --- a/tests/test_plot/test_misc.py +++ b/tests/test_plot/test_misc.py @@ -1703,3 +1703,16 @@ def test_value_split_1(test_dir): ctx.run() ctx.expect_out_file_d(prj+context.KICAD_SCH_EXT) ctx.clean_up() + + +def test_definitions_1(test_dir): + prj = 'simple_2layer' + ctx = context.TestContext(test_dir, prj, 'definitions_top', 'gerberdir') + ctx.run() + for la in ['B_Cu', 'F_Cu']: + for copy in range(2): + ctx.expect_out_file(f'{prj}-{la}_copper_{copy+1}.gbr') + for la in ['B_Silkscreen', 'F_Silkscreen']: + for copy in range(2): + ctx.expect_out_file(f'{prj}-{la}_silk_{copy+1}.gbr') + ctx.clean_up() diff --git a/tests/yaml_samples/definitions_gerbers.kibot.yaml b/tests/yaml_samples/definitions_gerbers.kibot.yaml new file mode 100644 index 00000000..06eb4f19 --- /dev/null +++ b/tests/yaml_samples/definitions_gerbers.kibot.yaml @@ -0,0 +1,14 @@ +kibot: + version: 1 + +outputs: + - name: 'gerbers_@ID@' + comment: "Gerbers with definitions" + type: gerber + output_id: _@ID@ + layers: @LAYERS@ +... +--- +definitions: + ID: def_id + LAYERS: F.Cu diff --git a/tests/yaml_samples/definitions_level_1.kibot.yaml b/tests/yaml_samples/definitions_level_1.kibot.yaml new file mode 100644 index 00000000..13058a94 --- /dev/null +++ b/tests/yaml_samples/definitions_level_1.kibot.yaml @@ -0,0 +1,12 @@ +kibot: + version: 1 + +import: + # Copy 1 + - file: definitions_gerbers.kibot.yaml + definitions: + ID: @ID@_1 + # Copy 2 + - file: definitions_gerbers.kibot.yaml + definitions: + ID: @ID@_2 diff --git a/tests/yaml_samples/definitions_top.kibot.yaml b/tests/yaml_samples/definitions_top.kibot.yaml new file mode 100644 index 00000000..0abdbed5 --- /dev/null +++ b/tests/yaml_samples/definitions_top.kibot.yaml @@ -0,0 +1,14 @@ +kibot: + version: 1 + +import: + # Generate gerbers for the top and bottom copper + - file: definitions_level_1.kibot.yaml + definitions: + LAYERS: "[F.Cu, B.Cu]" + ID: copper + # Generate gerbers for the top and bottom silk screen + - file: definitions_level_1.kibot.yaml + definitions: + LAYERS: "[F.SilkS, B.SilkS]" + ID: silk