diff --git a/CHANGELOG.md b/CHANGELOG.md index 4945b09d..3ccbb960 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [JLCPCB](https://jlcpcb.com/) - [P-Ban](https://www.p-ban.com/) - [PCBWay](https://www.pcbway.com) +- Support for ZIP/TAR/RAR generation. ### Changed - Now the default output name applies to the DRC and ERC report names. diff --git a/README.md b/README.md index 279e27de..258085f0 100644 --- a/README.md +++ b/README.md @@ -646,6 +646,26 @@ Next time you need this list just use an alias, like this: - `style`: [string='modern-blue'] Head style: modern-blue, modern-green, modern-red and classic.. - `title`: [string='KiBot Bill of Materials'] BoM title. +* Archiver (files compressor) + * Type: `compress` + * Description: Generates a compressed file containing output files. + This is used to generate groups of files in compressed file format. + * Valid keys: + - `comment`: [string=''] A comment for documentation purposes. + - `dir`: [string='.'] Output directory for the generated files. + - `name`: [string=''] Used to identify this particular output definition. + - `options`: [dict] Options for the `compress` output. + * Valid keys: + - `compression`: [string='auto'] [auto,stored,deflated,bzip2,lzma] Compression algorithm. Use auto to let KiBot select a suitable one. + - `files`: [list(dict)] Which files will be included. + * Valid keys: + - `dest`: [string=''] Destination directory inside the archive, empty means the same of the file. + - `filter`: [string='.*'] A regular expression that source files must match. + - `source`: [string='*'] File names to add, wildcards allowed. Use ** for recursive match. + Note this pattern is applied to the output dir specified with -d comman line option. + - `format`: [string='ZIP'] [ZIP,TAR,RAR] Output file format. + - `output`: [string='%f-%i%v.%x'] Name for the generated archive (%i=name of the output %x=according to format). Affected by global options. + * DXF (Drawing Exchange Format) * Type: `dxf` * Description: Exports the PCB to 2D mechanical EDA tools (like AutoCAD). @@ -1450,7 +1470,8 @@ Options: - For ERC, DRC, BoM XML update and PCB/SCH print install [KiCad Automation Scripts](https://github.com/INTI-CMNB/kicad-automation-scripts/) - BoM files (HTML/CSV/TSV/TXT/XML/XLSX) can be generated using the internal BoM generator or using [KiBoM](https://github.com/INTI-CMNB/KiBoM). - For interactive BoM install [InteractiveHtmlBom](https://github.com/INTI-CMNB/InteractiveHtmlBom) -- For SVG/PNG/JPG beauty PCB render [PcbDraw](https://github.com/INTI-CMNB/PcbDraw) +- For SVG/PNG/JPG beauty PCB render [PcbDraw](https://github.com/INTI-CMNB/PcbDraw). Also install the convert (from imagemagick) and rsvg-convert (from librsvg2-bin) tools. +- To create RAR files install the rar tool. ### Installation on Ubuntu/Debian diff --git a/debian/control b/debian/control index 49cdb53e..171e6589 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: kibot Architecture: all Multi-Arch: foreign Depends: ${misc:Depends}, ${python3:Depends}, python3-yaml, kicad (>= 5.1.0), python3-wxgtk4.0 -Recommends: kibom.inti-cmnb (>= 1.8.0), interactivehtmlbom.inti-cmnb, pcbdraw, imagemagick, librsvg2-bin, python3-xlsxwriter +Recommends: kibom.inti-cmnb (>= 1.8.0), interactivehtmlbom.inti-cmnb, pcbdraw, imagemagick, librsvg2-bin, python3-xlsxwriter, rar Description: KiCad Bot KiBot is a program which helps you to automate the generation of KiCad output documents easily, repeatable, and most of all, scriptably. diff --git a/docs/README.in b/docs/README.in index a2fd0dec..adac2e41 100644 --- a/docs/README.in +++ b/docs/README.in @@ -568,7 +568,8 @@ kibot --list - For ERC, DRC, BoM XML update and PCB/SCH print install [KiCad Automation Scripts](https://github.com/INTI-CMNB/kicad-automation-scripts/) - BoM files (HTML/CSV/TSV/TXT/XML/XLSX) can be generated using the internal BoM generator or using [KiBoM](https://github.com/INTI-CMNB/KiBoM). - For interactive BoM install [InteractiveHtmlBom](https://github.com/INTI-CMNB/InteractiveHtmlBom) -- For SVG/PNG/JPG beauty PCB render [PcbDraw](https://github.com/INTI-CMNB/PcbDraw) +- For SVG/PNG/JPG beauty PCB render [PcbDraw](https://github.com/INTI-CMNB/PcbDraw). Also install the convert (from imagemagick) and rsvg-convert (from librsvg2-bin) tools. +- To create RAR files install the rar tool. ### Installation on Ubuntu/Debian diff --git a/docs/samples/Elecrow.kibot.yaml b/docs/samples/Elecrow.kibot.yaml index 48ec8bdb..4c98ca9d 100644 --- a/docs/samples/Elecrow.kibot.yaml +++ b/docs/samples/Elecrow.kibot.yaml @@ -46,3 +46,12 @@ outputs: pth_id: '' npth_id: '-NPTH' output: "%f%i.TXT" + + - name: Elecrow + comment: ZIP file for Elecrow + type: compress + dir: Elecrow + options: + files: + - source: Elecrow/* + dest: / diff --git a/docs/samples/Elecrow_stencil.kibot.yaml b/docs/samples/Elecrow_stencil.kibot.yaml index 33d22f58..03587a79 100644 --- a/docs/samples/Elecrow_stencil.kibot.yaml +++ b/docs/samples/Elecrow_stencil.kibot.yaml @@ -48,3 +48,12 @@ outputs: pth_id: '' npth_id: '-NPTH' output: "%f%i.TXT" + + - name: Elecrow + comment: ZIP file for Elecrow + type: compress + dir: Elecrow + options: + files: + - source: Elecrow/* + dest: / diff --git a/docs/samples/FusionPCB.kibot.yaml b/docs/samples/FusionPCB.kibot.yaml index c19e9206..d9449a11 100644 --- a/docs/samples/FusionPCB.kibot.yaml +++ b/docs/samples/FusionPCB.kibot.yaml @@ -46,3 +46,12 @@ outputs: pth_and_npth_single_file: true use_aux_axis_as_origin: true output: "%f.TXT" + + - name: FusionPCB + comment: ZIP file for FusionPCB + type: compress + dir: FusionPCB + options: + files: + - source: FusionPCB/* + dest: / diff --git a/docs/samples/JLCPCB.kibot.yaml b/docs/samples/JLCPCB.kibot.yaml index ef4dd337..f1f0ce6d 100644 --- a/docs/samples/JLCPCB.kibot.yaml +++ b/docs/samples/JLCPCB.kibot.yaml @@ -52,3 +52,12 @@ outputs: npth_id: '-NPTH' metric_units: false output: "%f%i.%x" + + - name: JLCPCB + comment: ZIP file for JLCPCB + type: compress + dir: JLCPCB + options: + files: + - source: JLCPCB/* + dest: / diff --git a/docs/samples/JLCPCB_stencil.kibot.yaml b/docs/samples/JLCPCB_stencil.kibot.yaml index 8f959d8c..cbba9aea 100644 --- a/docs/samples/JLCPCB_stencil.kibot.yaml +++ b/docs/samples/JLCPCB_stencil.kibot.yaml @@ -45,3 +45,12 @@ outputs: npth_id: '-NPTH' metric_units: false output: "%f%i.%x" + + - name: JLCPCB + comment: ZIP file for JLCPCB + type: compress + dir: JLCPCB + options: + files: + - source: JLCPCB/* + dest: / diff --git a/docs/samples/P-Ban.kibot.yaml b/docs/samples/P-Ban.kibot.yaml index 3003bce2..5f055496 100644 --- a/docs/samples/P-Ban.kibot.yaml +++ b/docs/samples/P-Ban.kibot.yaml @@ -59,3 +59,12 @@ outputs: left_digits: 3 right_digits: 3 output: "%f.drl" + + - name: P-Ban + comment: ZIP file for P-Ban + type: compress + dir: P-Ban + options: + files: + - source: P-Ban/* + dest: / diff --git a/docs/samples/PCBWay.kibot.yaml b/docs/samples/PCBWay.kibot.yaml index c8a2dbb3..12f915d9 100644 --- a/docs/samples/PCBWay.kibot.yaml +++ b/docs/samples/PCBWay.kibot.yaml @@ -51,3 +51,13 @@ outputs: pth_id: '' npth_id: '-NPTH' output: "%f%i.drl" + + - name: PCBWay + comment: ZIP file for PCBWay + type: compress + dir: PCBWay + options: + format: ZIP + files: + - source: PCBWay/* + dest: / diff --git a/docs/samples/generic_plot.kibot.yaml b/docs/samples/generic_plot.kibot.yaml index e09e1fcf..5dca2204 100644 --- a/docs/samples/generic_plot.kibot.yaml +++ b/docs/samples/generic_plot.kibot.yaml @@ -161,6 +161,29 @@ outputs: # [string='KiBot Bill of Materials'] BoM title title: 'KiBot Bill of Materials' + # Archiver (files compressor): + # This is used to generate groups of files in compressed file format. + - name: 'compress_example' + comment: 'Generates a compressed file containing output files.' + type: 'compress' + dir: 'Example/compress_dir' + options: + # [string='auto'] [auto,stored,deflated,bzip2,lzma] Compression algorithm. Use auto to let KiBot select a suitable one + compression: 'auto' + # [list(dict)] Which files will be included + files: + # [string=''] Destination directory inside the archive, empty means the same of the file + - dest: '' + # [string='.*'] A regular expression that source files must match + filter: '.*' + # [string='*'] File names to add, wildcards allowed. Use ** for recursive match. + # Note this pattern is applied to the output dir specified with -d comman line option + source: '*' + # [string='ZIP'] [ZIP,TAR,RAR] Output file format + format: 'ZIP' + # [string='%f-%i%v.%x'] Name for the generated archive (%i=name of the output %x=according to format). Affected by global options + output: '%f-%i%v.%x' + # DXF (Drawing Exchange Format): # This output is what you get from the File/Plot menu in pcbnew. - name: 'dxf_example' diff --git a/kibot/gs.py b/kibot/gs.py index 3929a3b7..a110f897 100644 --- a/kibot/gs.py +++ b/kibot/gs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 Salvador E. Tropea -# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial +# Copyright (c) 2020-2021 Salvador E. Tropea +# Copyright (c) 2020-2021 Instituto Nacional de Tecnología Industrial # License: GPL-3.0 # Project: KiBot (formerly KiPlot) import os @@ -57,6 +57,8 @@ class GS(object): pcb_comp = None # Current variant/s variant = None + # Name for the output we are generating + current_output = None # Global defaults # This is used as default value for classes supporting "output" option def_global_output = '%f-%i%v.%x' diff --git a/kibot/kiplot.py b/kibot/kiplot.py index f694ddf8..164b66ee 100644 --- a/kibot/kiplot.py +++ b/kibot/kiplot.py @@ -266,6 +266,7 @@ def generate_outputs(outputs, target, invert, skip_pre): load_sch() config_output(out) logger.info('- '+str(out)) + GS.current_output = out.name try: out.run(get_output_dir(out.dir), board) except PlotError as e: diff --git a/kibot/misc.py b/kibot/misc.py index a801b328..ce30bdcd 100644 --- a/kibot/misc.py +++ b/kibot/misc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 Salvador E. Tropea -# Copyright (c) 2020 Instituto Nacional de Tecnología Industrial +# Copyright (c) 2020-2021 Salvador E. Tropea +# Copyright (c) 2020-2021 Instituto Nacional de Tecnología Industrial # License: GPL-3.0 # Project: KiBot (formerly KiPlot) """ Miscellaneous definitions """ @@ -162,6 +162,7 @@ W_MISFLDNAME = '(W046) ' W_MISS3D = '(W047) ' W_FAILDL = '(W048) ' W_NOLAYER = '(W049) ' +W_EMPTYZIP = '(W050) ' class Rect(object): diff --git a/kibot/out_compress.py b/kibot/out_compress.py new file mode 100644 index 00000000..0d4ccb86 --- /dev/null +++ b/kibot/out_compress.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2021 Salvador E. Tropea +# Copyright (c) 2021 Instituto Nacional de Tecnología Industrial +# License: GPL-3.0 +# Project: KiBot (formerly KiPlot) +import re +import os +import glob +from sys import exit +from subprocess import check_output, STDOUT, CalledProcessError +from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED, ZIP_BZIP2, ZIP_LZMA +from tarfile import open as tar_open +from collections import OrderedDict +from .gs import GS +from .misc import MISSING_TOOL, WRONG_INSTALL, W_EMPTYZIP +from .optionable import Optionable, BaseOptions +from .macros import macros, document, output_class # noqa: F401 +from . import log + +logger = log.get_logger(__name__) + + +class FilesList(Optionable): + def __init__(self): + super().__init__() + with document: + self.source = '*' + """ File names to add, wildcards allowed. Use ** for recursive match. + Note this pattern is applied to the output dir specified with -d comman line option """ + self.filter = '.*' + """ A regular expression that source files must match """ + self.dest = '' + """ Destination directory inside the archive, empty means the same of the file """ + + +class CompressOptions(BaseOptions): + ZIP_ALGORITHMS = {'auto': ZIP_DEFLATED, + 'stored': ZIP_STORED, + 'deflated': ZIP_DEFLATED, + 'bzip2': ZIP_BZIP2, + 'lzma': ZIP_LZMA} + TAR_MODE = {'auto': 'bz2', + 'stored': '', + 'deflated': 'gz', + 'bzip2': 'bz2', + 'lzma': 'xz'} + + def __init__(self): + with document: + self.output = GS.def_global_output + """ Name for the generated archive (%i=name of the output %x=according to format) """ + self.format = 'ZIP' + """ [ZIP,TAR,RAR] Output file format """ + self.compression = 'auto' + """ [auto,stored,deflated,bzip2,lzma] Compression algorithm. Use auto to let KiBot select a suitable one """ + self.files = FilesList + """ [list(dict)] Which files will be included """ + super().__init__() + + def config(self): + super().config() + if isinstance(self.files, type): + self.files = [] + logger.warning(W_EMPTYZIP+' No files provided, creating an empty archive') + + def create_zip(self, output, files): + with ZipFile(output, 'w', compression=self.ZIP_ALGORITHMS[self.compression], compresslevel=9) as zip: + for fname, dest in files.items(): + logger.debug('Adding '+fname+' as '+dest) + zip.write(fname, dest) + + def create_tar(self, output, files): + with tar_open(output, 'w:'+self.TAR_MODE[self.compression]) as tar: + for fname, dest in files.items(): + logger.debug('Adding '+fname+' as '+dest) + tar.add(fname, dest) + + def create_rar(self, output, files): + if os.path.isfile(output): + os.remove(output) + for fname, dest in files.items(): + logger.debug('Adding '+fname+' as '+dest) + cmd = ['rar', 'a', '-m5', '-ep', '-ap'+os.path.dirname(dest), output, fname] + try: + check_output(cmd, stderr=STDOUT) + except FileNotFoundError: + logger.error('Missing `rar` command, install it') + exit(MISSING_TOOL) + except CalledProcessError as e: # pragma: no cover + logger.error('Failed to invoke rar command, error {}'.format(e.returncode)) + if e.output: + logger.debug('Output from command: '+e.output.decode()) + exit(WRONG_INSTALL) + + def solve_extension(self): + if self.format == 'ZIP': + return 'zip' + if self.format == 'RAR': + return 'rar' + # TAR + ext = 'tar' + sub_ext = self.TAR_MODE[self.compression] + if sub_ext: + ext += '.'+sub_ext + return ext + + def run(self, output_dir, board): + # Output file name + logger.debug('output_dir '+output_dir) + logger.debug('GS.out_dir '+GS.out_dir) + output = self.expand_filename(output_dir, self.output, GS.current_output, self.solve_extension()) + logger.debug('Collecting files') + output_real = os.path.realpath(output) + # Collect the files + files = OrderedDict() + for f in self.files: + for fname in filter(re.compile(f.filter).match, glob.iglob(os.path.join(GS.out_dir, f.source), recursive=True)): + fname_real = os.path.realpath(fname) + # Avoid including the output + if fname_real == output_real: + continue + # Compute the destination directory inside the archive + dest = fname + if f.dest: + dest = os.path.join(f.dest, os.path.basename(fname)) + else: + dest = os.path.relpath(dest, GS.out_dir) + files[fname_real] = dest + logger.debug('Generating `{}` archive'.format(output)) + if self.format == 'ZIP': + self.create_zip(output, files) + elif self.format == 'TAR': + self.create_tar(output, files) + elif self.format == 'RAR': + self.create_rar(output, files) + + +@output_class +class Compress(BaseOutput): # noqa: F821 + """ Archiver (files compressor) + Generates a compressed file containing output files. + This is used to generate groups of files in compressed file format. """ + def __init__(self): + super().__init__() + with document: + self.options = CompressOptions + """ [dict] Options for the `compress` output """