From 73cb98f1132c82ba76ab3a5592c4a7a381cf9eab Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Fri, 22 Jan 2021 17:22:18 -0300 Subject: [PATCH] Makefile generation. --- CHANGELOG.md | 1 + README.md | 4 +- docs/samples/generic_plot.kibot.yaml | 3 +- kibot/__main__.py | 19 ++++--- kibot/kiplot.py | 75 +++++++++++++++++++++++++++- kibot/misc.py | 7 +++ kibot/out_base.py | 8 +++ kibot/out_compress.py | 32 +++++++++--- kibot/out_ibom.py | 12 ++++- kibot/out_kibom.py | 6 +++ kibot/out_pcbdraw.py | 6 +++ kibot/out_step.py | 25 ++++++++-- kibot/pre_base.py | 20 +++++++- kibot/pre_drc.py | 6 ++- kibot/pre_erc.py | 6 ++- kibot/pre_update_xml.py | 8 ++- 16 files changed, 210 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 075df460..e2c01b2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [P-Ban](https://www.p-ban.com/) - [PCBWay](https://www.pcbway.com) - Support for ZIP/TAR/RAR generation. +- Makefile generation. ### Changed - Now the default output name applies to the DRC and ERC report names. diff --git a/README.md b/README.md index 47454297..4c34ffa4 100644 --- a/README.md +++ b/README.md @@ -913,6 +913,7 @@ Next time you need this list just use an alias, like this: Extension .html will be added automatically. Note that this name is used only when output is ''. - `netlist_file`: [string=''] Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name. + Leave it blank for most uses, data will be extracted from the PCB. - `no_blacklist_virtual`: [boolean=false] Do not blacklist virtual components. IBoM option, avoid using in conjunction with KiBot variants/filters. - `no_redraw_on_drag`: [boolean=false] Do not redraw pcb on drag by default. @@ -1435,7 +1436,7 @@ KiBot: KiCad automation tool for documents generation Usage: kibot [-b BOARD] [-e SCHEMA] [-c CONFIG] [-d OUT_DIR] [-s PRE] - [-q | -v...] [-i] [-g DEF]... [TARGET...] + [-q | -v...] [-i] [-m MKFILE] [-g DEF]... [TARGET...] kibot [-v...] [-c PLOT_CONFIG] --list kibot [-v...] [-b BOARD] [-d OUT_DIR] [-p | -P] --example kibot [-v...] --help-filters @@ -1463,6 +1464,7 @@ Options: --help-preflights List supported preflights and details -i, --invert-sel Generate the outputs not listed as targets -l, --list List available outputs (in the config file) + -m MKFILE, --makefile MKFILE Generate a Makefile (no targets created) -p, --copy-options Copy plot options from the PCB file -P, --copy-and-expand As -p but expand the list of layers -q, --quiet Remove information logs diff --git a/docs/samples/generic_plot.kibot.yaml b/docs/samples/generic_plot.kibot.yaml index f069dfe3..c9ddf049 100644 --- a/docs/samples/generic_plot.kibot.yaml +++ b/docs/samples/generic_plot.kibot.yaml @@ -508,7 +508,8 @@ outputs: # Extension .html will be added automatically. # Note that this name is used only when output is '' name_format: 'ibom' - # [string=''] Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name + # [string=''] Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name. + # Leave it blank for most uses, data will be extracted from the PCB netlist_file: '' # [boolean=false] Do not blacklist virtual components. # IBoM option, avoid using in conjunction with KiBot variants/filters diff --git a/kibot/__main__.py b/kibot/__main__.py index 271df8c8..64fd3f14 100644 --- a/kibot/__main__.py +++ b/kibot/__main__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 Salvador E. Tropea -# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial +# Copyright (c) 2020-2021 Salvador E. Tropea +# Copyright (c) 2020-2021 Instituto Nacional de Tecnología Industrial # Copyright (c) 2018 John Beard # License: GPL-3.0 # Project: KiBot (formerly KiPlot) @@ -9,7 +9,7 @@ Usage: kibot [-b BOARD] [-e SCHEMA] [-c CONFIG] [-d OUT_DIR] [-s PRE] - [-q | -v...] [-i] [-g DEF]... [TARGET...] + [-q | -v...] [-i] [-m MKFILE] [-g DEF]... [TARGET...] kibot [-v...] [-c PLOT_CONFIG] --list kibot [-v...] [-b BOARD] [-d OUT_DIR] [-p | -P] --example kibot [-v...] --help-filters @@ -37,6 +37,7 @@ Options: --help-preflights List supported preflights and details -i, --invert-sel Generate the outputs not listed as targets -l, --list List available outputs (in the config file) + -m MKFILE, --makefile MKFILE Generate a Makefile (no targets created) -p, --copy-options Copy plot options from the PCB file -P, --copy-and-expand As -p but expand the list of layers -q, --quiet Remove information logs @@ -47,7 +48,7 @@ Options: """ __author__ = 'Salvador E. Tropea, John Beard' -__copyright__ = 'Copyright 2018-2020, Salvador E. Tropea/INTI/John Beard' +__copyright__ = 'Copyright 2018-2021, Salvador E. Tropea/INTI/John Beard' __credits__ = ['Salvador E. Tropea', 'John Beard'] __license__ = 'GPL v3+' __email__ = 'stropea@inti.gob.ar' @@ -75,7 +76,7 @@ from .misc import NO_PCB_FILE, NO_SCH_FILE, EXIT_BAD_ARGS, W_VARSCH, W_VARCFG, W from .pre_base import (BasePreFlight) from .config_reader import (CfgYamlReader, print_outputs_help, print_output_help, print_preflights_help, create_example, print_filters_help) -from .kiplot import (generate_outputs, load_actions, config_output) +from .kiplot import (generate_outputs, load_actions, config_output, generate_makefile) def list_pre_and_outs(logger, outputs): @@ -302,8 +303,12 @@ def main(): GS.set_sch(solve_schematic(args.schematic, args.board_file)) # Determine the PCB file GS.set_pcb(solve_board_file(GS.sch_file, args.board_file)) - # Do all the job (pre-flight + outputs) - generate_outputs(outputs, args.target, args.invert_sel, args.skip_pre) + if args.makefile: + # Only create a makefile + generate_makefile(args.makefile, plot_config, outputs) + else: + # Do all the job (pre-flight + outputs) + generate_outputs(outputs, args.target, args.invert_sel, args.skip_pre) # Print total warnings logger.log_totals() diff --git a/kibot/kiplot.py b/kibot/kiplot.py index e6d654ac..8e959c80 100644 --- a/kibot/kiplot.py +++ b/kibot/kiplot.py @@ -17,11 +17,12 @@ from subprocess import run, PIPE, call from glob import glob from distutils.version import StrictVersion from importlib.util import (spec_from_file_location, module_from_spec) +from collections import OrderedDict from .gs import GS from .misc import (PLOT_ERROR, MISSING_TOOL, CMD_EESCHEMA_DO, URL_EESCHEMA_DO, CORRUPTED_PCB, EXIT_BAD_ARGS, CORRUPTED_SCH, EXIT_BAD_CONFIG, WRONG_INSTALL, UI_SMD, UI_VIRTUAL, KICAD_VERSION_5_99, - MOD_SMD, MOD_THROUGH_HOLE, MOD_VIRTUAL, W_PCBNOSCH, W_NONEEDSKIP) + MOD_SMD, MOD_THROUGH_HOLE, MOD_VIRTUAL, W_PCBNOSCH, W_NONEEDSKIP, W_WRONGCHAR, name2make) from .error import PlotError, KiPlotConfigurationError, config_error, trace_dump from .pre_base import BasePreFlight from .kicad.v5_sch import Schematic, SchFileError @@ -288,3 +289,75 @@ def generate_outputs(outputs, target, invert, skip_pre): run_output(out) else: logger.debug('Skipping `%s` output', str(out)) + + +def adapt_file_name(name): + name = os.path.relpath(name) + name = name.replace(' ', r'\ ') + if '$' in name: + logger.warning(W_WRONGCHAR+'Wrong character in file name `{}`'.format(name)) + return name + + +def generate_makefile(makefile, cfg_file, outputs): + logger.info('- Creating makefile `{}` from `{}`'.format(makefile, cfg_file)) + with open(makefile, 'wt') as f: + f.write('#!/usr/bin/make\n') + f.write('# Automatically generated by KiBot from `{}`\n'.format(cfg_file)) + f.write('KIBOT=kibot\n') + f.write('DEBUG=\n') + f.write('CONFIG={}\n'.format(cfg_file)) + f.write('SCH={}\n'.format(os.path.relpath(GS.sch_file))) + f.write('PCB={}\n'.format(os.path.relpath(GS.pcb_file))) + f.write('DEST={}\n'.format(os.path.relpath(GS.out_dir))) + f.write('KIBOT_CMD=$(KIBOT) $(DEBUG) -c $(CONFIG) -e $(SCH) -b $(PCB) -d $(DEST)\n') + f.write('LOGFILE=kibot_error.log\n') + f.write('\n') + # Configure all outputs + GS.outputs = outputs + for out in outputs: + config_output(out) + # Get all targets and dependencies + targets = OrderedDict() + dependencies = OrderedDict() + comments = {} + ori_names = {} + is_pre = set() + # Preflights + pres = BasePreFlight.get_in_use_objs() + for pre in pres: + tg = pre.get_targets() + if not tg: + continue + name = pre._name + targets[name] = [adapt_file_name(fn) for fn in tg] + dependencies[name] = [adapt_file_name(fn) for fn in pre.get_dependencies()] + is_pre.add(name) + # Outputs + for out in outputs: + name = name2make(out.name) + ori_names[name] = out.name + targets[name] = [adapt_file_name(fn) for fn in out.get_targets(os.path.join(GS.out_dir, out.dir))] + dependencies[name] = [adapt_file_name(fn) for fn in out.get_dependencies()] + if out.comment: + comments[name] = out.comment + # all target + f.write('#\n# Default target\n#\n') + f.write('all: '+' '.join(targets.keys())+'\n\n') + # Generate the output targets + f.write('#\n# Available targets (outputs)\n#\n') + for name, target in targets.items(): + f.write(name+': '+' '.join(target)+'\n\n') + # Generate the output dependencies + f.write('#\n# Rules and dependencies\n#\n') + for name, dep in dependencies.items(): + if name in comments: + f.write('# '+comments[name]+'\n') + f.write(' '.join(targets[name])+': '+' '.join(dep)+'\n') + if name in is_pre: + skip = filter(lambda n: n != name, is_pre) + f.write('\t@$(KIBOT_CMD) -s {} -i 2>> $(LOGFILE)\n\n'.format(','.join(skip))) + else: + f.write('\t@$(KIBOT_CMD) -s all {} 2>> $(LOGFILE)\n\n'.format(ori_names[name])) + # Mark all outputs as PHONY + f.write('.PHONY: '+' '.join(targets.keys())+'\n') diff --git a/kibot/misc.py b/kibot/misc.py index ce30bdcd..5c8f202c 100644 --- a/kibot/misc.py +++ b/kibot/misc.py @@ -5,6 +5,8 @@ # Project: KiBot (formerly KiPlot) """ Miscellaneous definitions """ +import re + # Error levels INTERNAL_ERROR = 1 # Unhandled exceptions WRONG_ARGUMENTS = 2 # This is what argsparse uses @@ -163,6 +165,7 @@ W_MISS3D = '(W047) ' W_FAILDL = '(W048) ' W_NOLAYER = '(W049) ' W_EMPTYZIP = '(W050) ' +W_WRONGCHAR = '(W051) ' class Rect(object): @@ -185,3 +188,7 @@ class Rect(object): self.y1 = min(self.y1, wxRect.y) self.x2 = max(self.x2, wxRect.x+wxRect.width) self.y2 = max(self.y2, wxRect.y+wxRect.height) + + +def name2make(name): + return re.sub(r'[ \$\.\\\/]', '_', name) diff --git a/kibot/out_base.py b/kibot/out_base.py index 76189939..639131e2 100644 --- a/kibot/out_base.py +++ b/kibot/out_base.py @@ -55,6 +55,14 @@ class BaseOutput(RegOutput): return [] return self.options.get_targets(self, out_dir) + def get_dependencies(self): + """ Returns a list of files needed to create this output """ + if self._sch_related: + if GS.sch: + return GS.sch.get_files() + return [GS.sch_file] + return [GS.pcb_file] + def config(self): super().config() if getattr(self, 'options', None) and isinstance(self.options, type): diff --git a/kibot/out_compress.py b/kibot/out_compress.py index a41602a6..853c7bb8 100644 --- a/kibot/out_compress.py +++ b/kibot/out_compress.py @@ -108,17 +108,14 @@ class CompressOptions(BaseOptions): ext += '.'+sub_ext return ext - def get_targets(self, parent, out_dir): - return [self.expand_filename(out_dir, self.output, parent.name, self.solve_extension())] - - def run(self, output_dir, parent): - # Output file name - output = self.expand_filename(output_dir, self.output, GS.current_output, self.solve_extension()) - logger.debug('Collecting files') + def get_files(self, output, parent, no_out_expand=False): output_real = os.path.realpath(output) - # Collect the files files = OrderedDict() for f in self.files: + if f.from_output and no_out_expand: + # Just the name of the output + files[f.from_output] = 1 + continue # Get the list of candidates files_list = None if f.from_output: @@ -155,6 +152,22 @@ class CompressOptions(BaseOptions): else: dest = os.path.relpath(dest, GS.out_dir) files[fname_real] = dest + return files + + def get_targets(self, parent, out_dir): + return [self.expand_filename(out_dir, self.output, parent.name, self.solve_extension())] + + def get_dependencies(self, parent): + output = self.get_targets(parent, GS.out_dir)[0] + files = self.get_files(output, parent, no_out_expand=True) + return files.keys() + + def run(self, output_dir, parent): + # Output file name + output = self.get_targets(parent, output_dir)[0] + logger.debug('Collecting files') + # Collect the files + files = self.get_files(output, parent) logger.debug('Generating `{}` archive'.format(output)) if self.format == 'ZIP': self.create_zip(output, files) @@ -175,5 +188,8 @@ class Compress(BaseOutput): # noqa: F821 self.options = CompressOptions """ [dict] Options for the `compress` output """ + def get_dependencies(self): + return self.options.get_dependencies(self) + def run(self, output_dir): self.options.run(output_dir, self) diff --git a/kibot/out_ibom.py b/kibot/out_ibom.py index a490bde1..77826ed0 100644 --- a/kibot/out_ibom.py +++ b/kibot/out_ibom.py @@ -59,7 +59,8 @@ class IBoMOptions(VariantOptions): self.sort_order = 'C,R,L,D,U,Y,X,F,SW,A,~,HS,CNN,J,P,NT,MH' """ Default sort order for components. Must contain '~' once """ self.netlist_file = '' - """ Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name """ + """ Path to netlist or xml file. You can use '%F.xml' to avoid specifying the project name. + Leave it blank for most uses, data will be extracted from the PCB """ self.extra_fields = '' """ Comma separated list of extra fields to pull from netlist or xml file """ self.normalize_field_case = False @@ -101,6 +102,12 @@ class IBoMOptions(VariantOptions): logger.error('Please use a name generated by KiBot or specify the name explicitly.') return [] + def get_dependencies(self): + files = [GS.pcb_file] + if os.path.isfile(self.netlist_file): + files.append(self.netlist_file) + return files + def run(self, output_dir): super().run(output_dir) check_script(CMD_IBOM, URL_IBOM) @@ -156,3 +163,6 @@ class IBoM(BaseOutput): # noqa: F821 with document: self.options = IBoMOptions """ [dict] Options for the `ibom` output """ + + def get_dependencies(self): + return self.options.get_dependencies() diff --git a/kibot/out_kibom.py b/kibot/out_kibom.py index a54d3221..f1643036 100644 --- a/kibot/out_kibom.py +++ b/kibot/out_kibom.py @@ -417,3 +417,9 @@ class KiBoM(BaseOutput): # noqa: F821 self.options = KiBoMOptions """ [dict] Options for the `kibom` output """ self._sch_related = True + + def get_dependencies(self): + files = super().get_dependencies() + if isinstance(self.options.conf, str): + files.append(self.options.conf) + return files diff --git a/kibot/out_pcbdraw.py b/kibot/out_pcbdraw.py index 5358ff4f..ebb4db6e 100644 --- a/kibot/out_pcbdraw.py +++ b/kibot/out_pcbdraw.py @@ -303,3 +303,9 @@ class PcbDraw(BaseOutput): # noqa: F821 with document: self.options = PcbDrawOptions """ [dict] Options for the `pcbdraw` output """ + + def get_dependencies(self): + files = super().get_dependencies() + if isinstance(self.options.style, str): + files.append(self.options.style) + return files diff --git a/kibot/out_step.py b/kibot/out_step.py index 59dc8dc5..14ab8d00 100644 --- a/kibot/out_step.py +++ b/kibot/out_step.py @@ -88,7 +88,7 @@ class STEPOptions(VariantOptions): for model in models_l: models.push_front(model) - def list_components(self): + def download_models(self): """ Check we have the 3D models. Inform missing models. Try to download the missing models """ @@ -141,6 +141,20 @@ class STEPOptions(VariantOptions): models.push_front(model) return models_replaced + def list_models(self): + """ Get the list of 3D models """ + # Load KiCad configuration so we can expand the 3D models path + KiConf.init(GS.pcb_file) + models = set() + # Look for all the footprints + for m in GS.board.GetModules(): + # Look for all the 3D models for this footprint + for m3d in m.Models(): + full_name = KiConf.expand_env(m3d.m_Filename) + if os.path.isfile(full_name): + models.add(full_name) + return models.keys() + def save_board(self, dir): """ Save the PCB to a temporal file """ with NamedTemporaryFile(mode='w', suffix='.kicad_pcb', delete=False, dir=dir) as f: @@ -151,7 +165,7 @@ class STEPOptions(VariantOptions): def filter_components(self, dir): if not self._comps: - if self.list_components(): + if self.download_models(): # Some missing components found and we downloaded them # Save the fixed board ret = self.save_board(dir) @@ -171,7 +185,7 @@ class STEPOptions(VariantOptions): while not models.empty(): rem_m_models.append(models.pop()) rem_models.append(rem_m_models) - self.list_components() + self.download_models() fname = self.save_board(dir) self.undo_3d_models_rename() # Undo the removing @@ -245,3 +259,8 @@ class STEP(BaseOutput): # noqa: F821 with document: self.options = STEPOptions """ [dict] Options for the `step` output """ + + def get_dependencies(self): + files = super().get_dependencies() + files.extend(self.options.list_models()) + return files diff --git a/kibot/pre_base.py b/kibot/pre_base.py index ae4f3372..5aef30dc 100644 --- a/kibot/pre_base.py +++ b/kibot/pre_base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 Salvador E. Tropea -# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial +# Copyright (c) 2020-2021 Salvador E. Tropea +# Copyright (c) 2020-2021 Instituto Nacional de Tecnología Industrial # License: GPL-3.0 # Project: KiBot (formerly KiPlot) from .gs import GS @@ -88,3 +88,19 @@ class BasePreFlight(Registrable): def apply(self): pass + + def get_dependencies(self): + """ Returns a list of files needed to run this preflight """ + files = [] + if self._sch_related: + if GS.sch: + files.extend(GS.sch.get_files()) + else: + files.append(GS.sch_file) + if self._pcb_related: + files.append(GS.pcb_file) + return files + + def get_targets(self): + """ Returns a list of targets generated by this preflight """ + return [] diff --git a/kibot/pre_drc.py b/kibot/pre_drc.py index 49aff6fa..350e7c2a 100644 --- a/kibot/pre_drc.py +++ b/kibot/pre_drc.py @@ -26,11 +26,15 @@ class Run_DRC(BasePreFlight): # noqa: F821 self._enabled = value self._pcb_related = True + def get_targets(self): + """ Returns a list of targets generated by this preflight """ + return [Optionable.expand_filename(None, GS.out_dir, GS.def_global_output, 'drc', 'txt')] + def run(self): check_script(CMD_PCBNEW_RUN_DRC, URL_PCBNEW_RUN_DRC, '1.4.0') if GS.board is None: load_board() - output = Optionable.expand_filename(None, GS.out_dir, GS.def_global_output, 'drc', 'txt') + output = self.get_targets()[0] logger.debug('DRC report: '+output) cmd = [CMD_PCBNEW_RUN_DRC, 'run_drc', '-o', output] if GS.filter_file: diff --git a/kibot/pre_erc.py b/kibot/pre_erc.py index a95f7cdf..eb48173f 100644 --- a/kibot/pre_erc.py +++ b/kibot/pre_erc.py @@ -26,12 +26,16 @@ class Run_ERC(BasePreFlight): # noqa: F821 self._enabled = value self._sch_related = True + def get_targets(self): + """ Returns a list of targets generated by this preflight """ + return [Optionable.expand_filename_sch(None, GS.out_dir, GS.def_global_output, 'erc', 'txt')] + def run(self): check_eeschema_do() # The schematic is loaded only before executing an output related to it. # But here we need data from it. load_sch() - output = Optionable.expand_filename_sch(None, GS.out_dir, GS.def_global_output, 'erc', 'txt') + output = self.get_targets()[0] logger.debug('ERC report: '+output) cmd = [CMD_EESCHEMA_DO, 'run_erc', '-o', output] if GS.filter_file: diff --git a/kibot/pre_update_xml.py b/kibot/pre_update_xml.py index 44112bfe..febbd093 100644 --- a/kibot/pre_update_xml.py +++ b/kibot/pre_update_xml.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 Salvador E. Tropea -# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial +# Copyright (c) 2020-2021 Salvador E. Tropea +# Copyright (c) 2020-2021 Instituto Nacional de Tecnología Industrial # License: GPL-3.0 # Project: KiBot (formerly KiPlot) from sys import (exit) @@ -26,6 +26,10 @@ class Update_XML(BasePreFlight): # noqa: F821 self._enabled = value self._sch_related = True + def get_targets(self): + """ Returns a list of targets generated by this preflight """ + return [GS.sch_no_ext+'.xml'] + def run(self): check_eeschema_do() cmd = [CMD_EESCHEMA_DO, 'bom_xml', GS.sch_file, GS.out_dir]