Makefile generation.

CHANGELOG.md

@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - [P-Ban](https://www.p-ban.com/)
   - [PCBWay](https://www.pcbway.com)
 - Support for ZIP/TAR/RAR generation.
+- Makefile generation.
 
 ### Changed
 - Now the default output name applies to the DRC and ERC report names.

README.md

@@ -913,6 +913,7 @@ Next time you need this list just use an alias, like this:
                          Extension .html will be added automatically.
                          Note that this name is used only when output is ''.
         - `netlist_file`: [string=''] Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name.
+                          Leave it blank for most uses, data will be extracted from the PCB.
         - `no_blacklist_virtual`: [boolean=false] Do not blacklist virtual components.
                                   IBoM option, avoid using in conjunction with KiBot variants/filters.
         - `no_redraw_on_drag`: [boolean=false] Do not redraw pcb on drag by default.
@@ -1435,7 +1436,7 @@ KiBot: KiCad automation tool for documents generation
 
 Usage:
   kibot [-b BOARD] [-e SCHEMA] [-c CONFIG] [-d OUT_DIR] [-s PRE]
-         [-q | -v...] [-i] [-g DEF]... [TARGET...]
+         [-q | -v...] [-i] [-m MKFILE] [-g DEF]... [TARGET...]
   kibot [-v...] [-c PLOT_CONFIG] --list
   kibot [-v...] [-b BOARD] [-d OUT_DIR] [-p | -P] --example
   kibot [-v...] --help-filters
@@ -1463,6 +1464,7 @@ Options:
   --help-preflights                List supported preflights and details
   -i, --invert-sel                 Generate the outputs not listed as targets
   -l, --list                       List available outputs (in the config file)
+  -m MKFILE, --makefile MKFILE     Generate a Makefile (no targets created)
   -p, --copy-options               Copy plot options from the PCB file
   -P, --copy-and-expand            As -p but expand the list of layers
   -q, --quiet                      Remove information logs

docs/samples/generic_plot.kibot.yaml

@@ -508,7 +508,8 @@ outputs:
       # Extension .html will be added automatically.
       # Note that this name is used only when output is ''
       name_format: 'ibom'
-      # [string=''] Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name
+      # [string=''] Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name.
+      # Leave it blank for most uses, data will be extracted from the PCB
       netlist_file: ''
       # [boolean=false] Do not blacklist virtual components.
       # IBoM option, avoid using in conjunction with KiBot variants/filters

kibot/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (c) 2020 Salvador E. Tropea
-# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial
+# Copyright (c) 2020-2021 Salvador E. Tropea
+# Copyright (c) 2020-2021 Instituto Nacional de Tecnología Industrial
 # Copyright (c) 2018 John Beard
 # License: GPL-3.0
 # Project: KiBot (formerly KiPlot)
@@ -9,7 +9,7 @@
 
 Usage:
   kibot [-b BOARD] [-e SCHEMA] [-c CONFIG] [-d OUT_DIR] [-s PRE]
-         [-q | -v...] [-i] [-g DEF]... [TARGET...]
+         [-q | -v...] [-i] [-m MKFILE] [-g DEF]... [TARGET...]
   kibot [-v...] [-c PLOT_CONFIG] --list
   kibot [-v...] [-b BOARD] [-d OUT_DIR] [-p | -P] --example
   kibot [-v...] --help-filters
@@ -37,6 +37,7 @@ Options:
   --help-preflights                List supported preflights and details
   -i, --invert-sel                 Generate the outputs not listed as targets
   -l, --list                       List available outputs (in the config file)
+  -m MKFILE, --makefile MKFILE     Generate a Makefile (no targets created)
   -p, --copy-options               Copy plot options from the PCB file
   -P, --copy-and-expand            As -p but expand the list of layers
   -q, --quiet                      Remove information logs
@@ -47,7 +48,7 @@ Options:
 
 """
 __author__ = 'Salvador E. Tropea, John Beard'
-__copyright__ = 'Copyright 2018-2020, Salvador E. Tropea/INTI/John Beard'
+__copyright__ = 'Copyright 2018-2021, Salvador E. Tropea/INTI/John Beard'
 __credits__ = ['Salvador E. Tropea', 'John Beard']
 __license__ = 'GPL v3+'
 __email__ = 'stropea@inti.gob.ar'
@@ -75,7 +76,7 @@ from .misc import NO_PCB_FILE, NO_SCH_FILE, EXIT_BAD_ARGS, W_VARSCH, W_VARCFG, W
 from .pre_base import (BasePreFlight)
 from .config_reader import (CfgYamlReader, print_outputs_help, print_output_help, print_preflights_help, create_example,
                             print_filters_help)
-from .kiplot import (generate_outputs, load_actions, config_output)
+from .kiplot import (generate_outputs, load_actions, config_output, generate_makefile)
 
 
 def list_pre_and_outs(logger, outputs):
@@ -302,8 +303,12 @@ def main():
     GS.set_sch(solve_schematic(args.schematic, args.board_file))
     # Determine the PCB file
     GS.set_pcb(solve_board_file(GS.sch_file, args.board_file))
-    # Do all the job (pre-flight + outputs)
-    generate_outputs(outputs, args.target, args.invert_sel, args.skip_pre)
+    if args.makefile:
+        # Only create a makefile
+        generate_makefile(args.makefile, plot_config, outputs)
+    else:
+        # Do all the job (pre-flight + outputs)
+        generate_outputs(outputs, args.target, args.invert_sel, args.skip_pre)
     # Print total warnings
     logger.log_totals()
 

kibot/kiplot.py

@@ -17,11 +17,12 @@ from subprocess import run, PIPE, call
 from glob import glob
 from distutils.version import StrictVersion
 from importlib.util import (spec_from_file_location, module_from_spec)
+from collections import OrderedDict
 
 from .gs import GS
 from .misc import (PLOT_ERROR, MISSING_TOOL, CMD_EESCHEMA_DO, URL_EESCHEMA_DO, CORRUPTED_PCB,
                    EXIT_BAD_ARGS, CORRUPTED_SCH, EXIT_BAD_CONFIG, WRONG_INSTALL, UI_SMD, UI_VIRTUAL, KICAD_VERSION_5_99,
-                   MOD_SMD, MOD_THROUGH_HOLE, MOD_VIRTUAL, W_PCBNOSCH, W_NONEEDSKIP)
+                   MOD_SMD, MOD_THROUGH_HOLE, MOD_VIRTUAL, W_PCBNOSCH, W_NONEEDSKIP, W_WRONGCHAR, name2make)
 from .error import PlotError, KiPlotConfigurationError, config_error, trace_dump
 from .pre_base import BasePreFlight
 from .kicad.v5_sch import Schematic, SchFileError
@@ -288,3 +289,75 @@ def generate_outputs(outputs, target, invert, skip_pre):
             run_output(out)
         else:
             logger.debug('Skipping `%s` output', str(out))
+
+
+def adapt_file_name(name):
+    name = os.path.relpath(name)
+    name = name.replace(' ', r'\ ')
+    if '$' in name:
+        logger.warning(W_WRONGCHAR+'Wrong character in file name `{}`'.format(name))
+    return name
+
+
+def generate_makefile(makefile, cfg_file, outputs):
+    logger.info('- Creating makefile `{}` from `{}`'.format(makefile, cfg_file))
+    with open(makefile, 'wt') as f:
+        f.write('#!/usr/bin/make\n')
+        f.write('# Automatically generated by KiBot from `{}`\n'.format(cfg_file))
+        f.write('KIBOT=kibot\n')
+        f.write('DEBUG=\n')
+        f.write('CONFIG={}\n'.format(cfg_file))
+        f.write('SCH={}\n'.format(os.path.relpath(GS.sch_file)))
+        f.write('PCB={}\n'.format(os.path.relpath(GS.pcb_file)))
+        f.write('DEST={}\n'.format(os.path.relpath(GS.out_dir)))
+        f.write('KIBOT_CMD=$(KIBOT) $(DEBUG) -c $(CONFIG) -e $(SCH) -b $(PCB) -d $(DEST)\n')
+        f.write('LOGFILE=kibot_error.log\n')
+        f.write('\n')
+        # Configure all outputs
+        GS.outputs = outputs
+        for out in outputs:
+            config_output(out)
+        # Get all targets and dependencies
+        targets = OrderedDict()
+        dependencies = OrderedDict()
+        comments = {}
+        ori_names = {}
+        is_pre = set()
+        # Preflights
+        pres = BasePreFlight.get_in_use_objs()
+        for pre in pres:
+            tg = pre.get_targets()
+            if not tg:
+                continue
+            name = pre._name
+            targets[name] = [adapt_file_name(fn) for fn in tg]
+            dependencies[name] = [adapt_file_name(fn) for fn in pre.get_dependencies()]
+            is_pre.add(name)
+        # Outputs
+        for out in outputs:
+            name = name2make(out.name)
+            ori_names[name] = out.name
+            targets[name] = [adapt_file_name(fn) for fn in out.get_targets(os.path.join(GS.out_dir, out.dir))]
+            dependencies[name] = [adapt_file_name(fn) for fn in out.get_dependencies()]
+            if out.comment:
+                comments[name] = out.comment
+        # all target
+        f.write('#\n# Default target\n#\n')
+        f.write('all: '+' '.join(targets.keys())+'\n\n')
+        # Generate the output targets
+        f.write('#\n# Available targets (outputs)\n#\n')
+        for name, target in targets.items():
+            f.write(name+': '+' '.join(target)+'\n\n')
+        # Generate the output dependencies
+        f.write('#\n# Rules and dependencies\n#\n')
+        for name, dep in dependencies.items():
+            if name in comments:
+                f.write('# '+comments[name]+'\n')
+            f.write(' '.join(targets[name])+': '+' '.join(dep)+'\n')
+            if name in is_pre:
+                skip = filter(lambda n: n != name, is_pre)
+                f.write('\t@$(KIBOT_CMD) -s {} -i 2>> $(LOGFILE)\n\n'.format(','.join(skip)))
+            else:
+                f.write('\t@$(KIBOT_CMD) -s all {} 2>> $(LOGFILE)\n\n'.format(ori_names[name]))
+        # Mark all outputs as PHONY
+        f.write('.PHONY: '+' '.join(targets.keys())+'\n')

kibot/misc.py

@@ -5,6 +5,8 @@
 # Project: KiBot (formerly KiPlot)
 """ Miscellaneous definitions """
 
+import re
+
 # Error levels
 INTERNAL_ERROR = 1    # Unhandled exceptions
 WRONG_ARGUMENTS = 2   # This is what argsparse uses
@@ -163,6 +165,7 @@ W_MISS3D = '(W047) '
 W_FAILDL = '(W048) '
 W_NOLAYER = '(W049) '
 W_EMPTYZIP = '(W050) '
+W_WRONGCHAR = '(W051) '
 
 
 class Rect(object):
@@ -185,3 +188,7 @@ class Rect(object):
             self.y1 = min(self.y1, wxRect.y)
             self.x2 = max(self.x2, wxRect.x+wxRect.width)
             self.y2 = max(self.y2, wxRect.y+wxRect.height)
+
+
+def name2make(name):
+    return re.sub(r'[ \$\.\\\/]', '_', name)

kibot/out_base.py

@@ -55,6 +55,14 @@ class BaseOutput(RegOutput):
             return []
         return self.options.get_targets(self, out_dir)
 
+    def get_dependencies(self):
+        """ Returns a list of files needed to create this output """
+        if self._sch_related:
+            if GS.sch:
+                return GS.sch.get_files()
+            return [GS.sch_file]
+        return [GS.pcb_file]
+
     def config(self):
         super().config()
         if getattr(self, 'options', None) and isinstance(self.options, type):

kibot/out_compress.py

@@ -108,17 +108,14 @@ class CompressOptions(BaseOptions):
             ext += '.'+sub_ext
         return ext
 
-    def get_targets(self, parent, out_dir):
-        return [self.expand_filename(out_dir, self.output, parent.name, self.solve_extension())]
-
-    def run(self, output_dir, parent):
-        # Output file name
-        output = self.expand_filename(output_dir, self.output, GS.current_output, self.solve_extension())
-        logger.debug('Collecting files')
+    def get_files(self, output, parent, no_out_expand=False):
         output_real = os.path.realpath(output)
-        # Collect the files
         files = OrderedDict()
         for f in self.files:
+            if f.from_output and no_out_expand:
+                # Just the name of the output
+                files[f.from_output] = 1
+                continue
             # Get the list of candidates
             files_list = None
             if f.from_output:
@@ -155,6 +152,22 @@ class CompressOptions(BaseOptions):
                 else:
                     dest = os.path.relpath(dest, GS.out_dir)
                 files[fname_real] = dest
+        return files
+
+    def get_targets(self, parent, out_dir):
+        return [self.expand_filename(out_dir, self.output, parent.name, self.solve_extension())]
+
+    def get_dependencies(self, parent):
+        output = self.get_targets(parent, GS.out_dir)[0]
+        files = self.get_files(output, parent, no_out_expand=True)
+        return files.keys()
+
+    def run(self, output_dir, parent):
+        # Output file name
+        output = self.get_targets(parent, output_dir)[0]
+        logger.debug('Collecting files')
+        # Collect the files
+        files = self.get_files(output, parent)
         logger.debug('Generating `{}` archive'.format(output))
         if self.format == 'ZIP':
             self.create_zip(output, files)
@@ -175,5 +188,8 @@ class Compress(BaseOutput):  # noqa: F821
             self.options = CompressOptions
             """ [dict] Options for the `compress` output """
 
+    def get_dependencies(self):
+        return self.options.get_dependencies(self)
+
     def run(self, output_dir):
         self.options.run(output_dir, self)

kibot/out_ibom.py

@@ -59,7 +59,8 @@ class IBoMOptions(VariantOptions):
             self.sort_order = 'C,R,L,D,U,Y,X,F,SW,A,~,HS,CNN,J,P,NT,MH'
             """ Default sort order for components. Must contain '~' once """
             self.netlist_file = ''
-            """ Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name """
+            """ Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name.
+                Leave it blank for most uses, data will be extracted from the PCB """
             self.extra_fields = ''
             """ Comma separated list of extra fields to pull from netlist or xml file """
             self.normalize_field_case = False
@@ -101,6 +102,12 @@ class IBoMOptions(VariantOptions):
         logger.error('Please use a name generated by KiBot or specify the name explicitly.')
         return []
 
+    def get_dependencies(self):
+        files = [GS.pcb_file]
+        if os.path.isfile(self.netlist_file):
+            files.append(self.netlist_file)
+        return files
+
     def run(self, output_dir):
         super().run(output_dir)
         check_script(CMD_IBOM, URL_IBOM)
@@ -156,3 +163,6 @@ class IBoM(BaseOutput):  # noqa: F821
         with document:
             self.options = IBoMOptions
             """ [dict] Options for the `ibom` output """
+
+    def get_dependencies(self):
+        return self.options.get_dependencies()

kibot/out_kibom.py

@@ -417,3 +417,9 @@ class KiBoM(BaseOutput):  # noqa: F821
             self.options = KiBoMOptions
             """ [dict] Options for the `kibom` output """
         self._sch_related = True
+
+    def get_dependencies(self):
+        files = super().get_dependencies()
+        if isinstance(self.options.conf, str):
+            files.append(self.options.conf)
+        return files

kibot/out_pcbdraw.py

@@ -303,3 +303,9 @@ class PcbDraw(BaseOutput):  # noqa: F821
         with document:
             self.options = PcbDrawOptions
             """ [dict] Options for the `pcbdraw` output """
+
+    def get_dependencies(self):
+        files = super().get_dependencies()
+        if isinstance(self.options.style, str):
+            files.append(self.options.style)
+        return files

kibot/out_step.py

@@ -88,7 +88,7 @@ class STEPOptions(VariantOptions):
             for model in models_l:
                 models.push_front(model)
 
-    def list_components(self):
+    def download_models(self):
         """ Check we have the 3D models.
             Inform missing models.
             Try to download the missing models """
@@ -141,6 +141,20 @@ class STEPOptions(VariantOptions):
                 models.push_front(model)
         return models_replaced
 
+    def list_models(self):
+        """ Get the list of 3D models """
+        # Load KiCad configuration so we can expand the 3D models path
+        KiConf.init(GS.pcb_file)
+        models = set()
+        # Look for all the footprints
+        for m in GS.board.GetModules():
+            # Look for all the 3D models for this footprint
+            for m3d in m.Models():
+                full_name = KiConf.expand_env(m3d.m_Filename)
+                if os.path.isfile(full_name):
+                    models.add(full_name)
+        return models.keys()
+
     def save_board(self, dir):
         """ Save the PCB to a temporal file """
         with NamedTemporaryFile(mode='w', suffix='.kicad_pcb', delete=False, dir=dir) as f:
@@ -151,7 +165,7 @@ class STEPOptions(VariantOptions):
 
     def filter_components(self, dir):
         if not self._comps:
-            if self.list_components():
+            if self.download_models():
                 # Some missing components found and we downloaded them
                 # Save the fixed board
                 ret = self.save_board(dir)
@@ -171,7 +185,7 @@ class STEPOptions(VariantOptions):
                 while not models.empty():
                     rem_m_models.append(models.pop())
                 rem_models.append(rem_m_models)
-        self.list_components()
+        self.download_models()
         fname = self.save_board(dir)
         self.undo_3d_models_rename()
         # Undo the removing
@@ -245,3 +259,8 @@ class STEP(BaseOutput):  # noqa: F821
         with document:
             self.options = STEPOptions
             """ [dict] Options for the `step` output """
+
+    def get_dependencies(self):
+        files = super().get_dependencies()
+        files.extend(self.options.list_models())
+        return files

kibot/pre_base.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (c) 2020 Salvador E. Tropea
-# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial
+# Copyright (c) 2020-2021 Salvador E. Tropea
+# Copyright (c) 2020-2021 Instituto Nacional de Tecnología Industrial
 # License: GPL-3.0
 # Project: KiBot (formerly KiPlot)
 from .gs import GS
@@ -88,3 +88,19 @@ class BasePreFlight(Registrable):
 
     def apply(self):
         pass
+
+    def get_dependencies(self):
+        """ Returns a list of files needed to run this preflight """
+        files = []
+        if self._sch_related:
+            if GS.sch:
+                files.extend(GS.sch.get_files())
+            else:
+                files.append(GS.sch_file)
+        if self._pcb_related:
+            files.append(GS.pcb_file)
+        return files
+
+    def get_targets(self):
+        """ Returns a list of targets generated by this preflight """
+        return []

kibot/pre_drc.py

@@ -26,11 +26,15 @@ class Run_DRC(BasePreFlight):  # noqa: F821
         self._enabled = value
         self._pcb_related = True
 
+    def get_targets(self):
+        """ Returns a list of targets generated by this preflight """
+        return [Optionable.expand_filename(None, GS.out_dir, GS.def_global_output, 'drc', 'txt')]
+
     def run(self):
         check_script(CMD_PCBNEW_RUN_DRC, URL_PCBNEW_RUN_DRC, '1.4.0')
         if GS.board is None:
             load_board()
-        output = Optionable.expand_filename(None, GS.out_dir, GS.def_global_output, 'drc', 'txt')
+        output = self.get_targets()[0]
         logger.debug('DRC report: '+output)
         cmd = [CMD_PCBNEW_RUN_DRC, 'run_drc', '-o', output]
         if GS.filter_file:

kibot/pre_erc.py

@@ -26,12 +26,16 @@ class Run_ERC(BasePreFlight):  # noqa: F821
         self._enabled = value
         self._sch_related = True
 
+    def get_targets(self):
+        """ Returns a list of targets generated by this preflight """
+        return [Optionable.expand_filename_sch(None, GS.out_dir, GS.def_global_output, 'erc', 'txt')]
+
     def run(self):
         check_eeschema_do()
         # The schematic is loaded only before executing an output related to it.
         # But here we need data from it.
         load_sch()
-        output = Optionable.expand_filename_sch(None, GS.out_dir, GS.def_global_output, 'erc', 'txt')
+        output = self.get_targets()[0]
         logger.debug('ERC report: '+output)
         cmd = [CMD_EESCHEMA_DO, 'run_erc', '-o', output]
         if GS.filter_file:

kibot/pre_update_xml.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (c) 2020 Salvador E. Tropea
-# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial
+# Copyright (c) 2020-2021 Salvador E. Tropea
+# Copyright (c) 2020-2021 Instituto Nacional de Tecnología Industrial
 # License: GPL-3.0
 # Project: KiBot (formerly KiPlot)
 from sys import (exit)
@@ -26,6 +26,10 @@ class Update_XML(BasePreFlight):  # noqa: F821
         self._enabled = value
         self._sch_related = True
 
+    def get_targets(self):
+        """ Returns a list of targets generated by this preflight """
+        return [GS.sch_no_ext+'.xml']
+
     def run(self):
         check_eeschema_do()
         cmd = [CMD_EESCHEMA_DO, 'bom_xml', GS.sch_file, GS.out_dir]