diff --git a/README.md b/README.md index 1e02ce30..3770a990 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,10 @@ Notes: - Mandatory for `kicost` - Optional to find components costs and specs for `bom` +[**PcbDraw**](https://github.com/INTI-CMNB/pcbdraw) v0.9.0 (tool) +- Mandatory for `pcbdraw` +- Optional to create realistic solder masks for `pcb_print` + [**Interactive HTML BoM**](https://github.com/INTI-CMNB/InteractiveHtmlBom) v2.4.1.4 (tool) - Mandatory for `ibom` @@ -127,16 +131,13 @@ Notes: [**LXML**](https://pypi.org/project/LXML/) (python module) [Debian](https://packages.debian.org/bullseye/python3-lxml) - Mandatory for `pcb_print` -[**PcbDraw**](https://github.com/INTI-CMNB/pcbdraw) v0.9.0 (tool) -- Mandatory for `pcbdraw` - [**QRCodeGen**](https://pypi.org/project/QRCodeGen/) (python module) (PyPi dependency) [Debian](https://packages.debian.org/bullseye/python3-qrcodegen) - Mandatory for `qr_lib` [**Colorama**](https://pypi.org/project/Colorama/) (python module) (PyPi dependency) [Debian](https://packages.debian.org/bullseye/python3-colorama) - Optional to get color messages in a portable way for general use -[**RSVG tools**](https://cran.r-project.org/web/packages/rsvg/index.html) (tool) [Debian](https://packages.debian.org/bullseye/librsvg2-bin) +[**RSVG tools**](https://gitlab.gnome.org/GNOME/librsvg) (tool) [Debian](https://packages.debian.org/bullseye/librsvg2-bin) - Optional to: - Create outputs preview for `navigate_results` - Create PNG icons for `navigate_results` diff --git a/kibot/dep_downloader.py b/kibot/dep_downloader.py new file mode 100644 index 00000000..b236434f --- /dev/null +++ b/kibot/dep_downloader.py @@ -0,0 +1,440 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2022 Salvador E. Tropea +# Copyright (c) 2022 Instituto Nacional de Tecnología Industrial +# License: GPL-3.0 +# Project: KiBot (formerly KiPlot) +import os +import re +import subprocess +import requests +import platform +import io +import tarfile +import stat +import json +import fnmatch +from sys import exit +from shutil import which, rmtree, move +from .kiplot import search_as_plugin +from .misc import MISSING_TOOL, TRY_INSTALL_CHECK, W_DOWNTOOL, W_MISSTOOL, USER_AGENT +from . import log + +logger = log.get_logger() +ver_re = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?(?:[\.-](\d+))?') +home_bin = os.environ.get('HOME') or os.environ.get('username') +if home_bin is not None: + home_bin = os.path.join(home_bin, '.local', 'share', 'kibot', 'bin') +EXEC_PERM = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH +last_stderr = None +binary_tools_cache = {} + + +def download(url): + logger.debug('- Trying to download '+url) + r = requests.get(url, allow_redirects=True, headers={'User-Agent': USER_AGENT}, timeout=20) + if r.status_code != 200: + logger.debug('- Failed to download `{}`'.format(url)) + return None + return r.content + + +def write_executable(command, content): + dest_bin = os.path.join(home_bin, command) + os.makedirs(home_bin, exist_ok=True) + with open(dest_bin, 'wb') as f: + f.write(content) + os.chmod(dest_bin, EXEC_PERM) + return dest_bin + + +def try_download_tar_ball(dep, url, name, name_in_tar=None): + if name_in_tar is None: + name_in_tar = name + content = download(url) + if content is None: + return None + # Try to extract the binary + dest_file = None + try: + with tarfile.open(fileobj=io.BytesIO(content), mode='r') as tar: + for entry in tar: + if entry.type != tarfile.REGTYPE or not fnmatch.fnmatch(entry.name, name_in_tar): + continue + dest_file = write_executable(name, tar.extractfile(entry).read()) + except Exception as e: + logger.debug('- Failed to extract {}'.format(e)) + return None + # Is this usable? + cmd = check_tool_binary_version(dest_file, dep) + if cmd is None: + return None + # logger.warning(W_DOWNTOOL+'Using downloaded `{}` tool, please visit {} for details'.format(name, dep.url)) + return cmd + + +def git_downloader(dep): + # Currently only for Linux x86_64/x86_32 + # arm, arm64, mips64el and mipsel are also there, just not implemented + system = platform.system() + plat = platform.platform() + if system != 'Linux' or 'x86_' not in plat: + logger.debug('- No binary for this system') + return None + # Try to download it + arch = 'amd64' if 'x86_64' in plat else 'i386' + url = 'https://github.com/EXALAB/git-static/raw/master/output/'+arch+'/bin/git' + content = download(url) + if content is None: + return None + dest_bin = write_executable(dep.command+'.real', content.replace(b'/root/output', b'/tmp/kibogit')) + # Now create the wrapper + git_real = dest_bin + dest_bin = dest_bin[:-5] + logger.error(f'{dest_bin} -> {git_real}') + if os.path.isfile(dest_bin): + os.remove(dest_bin) + with open(dest_bin, 'wt') as f: + f.write('#!/bin/sh\n') + f.write('rm /tmp/kibogit\n') + f.write('ln -s {} /tmp/kibogit\n'.format(home_bin[:-3])) + f.write('{} "$@"\n'.format(git_real)) + os.chmod(dest_bin, EXEC_PERM) + return check_tool_binary_version(dest_bin, dep) + + +def convert_downloader(dep): + # Currently only for Linux x86_64 + system = platform.system() + plat = platform.platform() + if system != 'Linux' or 'x86_64' not in plat: + logger.debug('- No binary for this system') + return None + # Get the download page + content = download(dep.url_down) + if content is None: + return None + # Look for the URL + res = re.search(r'href\s*=\s*"([^"]+)">magick<', content.decode()) + if not res: + logger.debug('- No `magick` download') + return None + url = res.group(1) + # Get the binary + content = download(url) + if content is None: + return None + # Can we run the AppImage? + dest_bin = write_executable(dep.command, content) + cmd = check_tool_binary_version(dest_bin, dep) + if cmd is not None: + logger.warning(W_DOWNTOOL+'Using downloaded `{}` tool, please visit {} for details'.format(dep.name, dep.url)) + return cmd + # Was because we don't have FUSE support + if 'libfuse.so' not in last_stderr and 'FUSE' not in last_stderr: + logger.debug('- Unknown fail reason: `{}`'.format(last_stderr)) + return None + # Uncompress it + unc_dir = os.path.join(home_bin, 'squashfs-root') + if os.path.isdir(unc_dir): + rmtree(unc_dir) + cmd = [dest_bin, '--appimage-extract'] + try: + res_run = subprocess.run(cmd, check=True, capture_output=True, cwd=home_bin) + except Exception as e: + logger.debug('- Failed to execute `{}` ({})'.format(cmd[0], e)) + return None + if not os.path.isdir(unc_dir): + logger.debug('- Failed to uncompress `{}` ({})'.format(cmd[0], res_run.stderr.decode())) + return None + # Now copy the important stuff + # Binaries + src_dir, _, bins = next(os.walk(os.path.join(unc_dir, 'usr', 'bin'))) + if not len(bins): + logger.debug('- No binaries found after extracting {}'.format(dest_bin)) + return None + for f in bins: + dst_file = os.path.join(home_bin, f) + if os.path.isfile(dst_file): + os.remove(dst_file) + move(os.path.join(src_dir, f), dst_file) + # Libs (to ~/.local/share/kibot/lib/ImageMagick/lib/ or similar) + src_dir = os.path.join(unc_dir, 'usr', 'lib') + if not os.path.isdir(src_dir): + logger.debug('- No libraries found after extracting {}'.format(dest_bin)) + return None + dst_dir = os.path.join(home_bin, '..', 'lib', 'ImageMagick') + if os.path.isdir(dst_dir): + rmtree(dst_dir) + os.makedirs(dst_dir, exist_ok=True) + move(src_dir, dst_dir) + lib_dir = os.path.join(dst_dir, 'lib') + # Config (to ~/.local/share/kibot/etc/ImageMagick-7/ or similar) + src_dir, dirs, _ = next(os.walk(os.path.join(unc_dir, 'usr', 'etc'))) + if len(dirs) != 1: + logger.debug('- More than one config dir found {}'.format(dirs)) + return None + src_dir = os.path.join(src_dir, dirs[0]) + dst_dir = os.path.join(home_bin, '..', 'etc') + os.makedirs(dst_dir, exist_ok=True) + dst_dir_name = os.path.join(dst_dir, dirs[0]) + if os.path.isdir(dst_dir_name): + rmtree(dst_dir_name) + move(src_dir, dst_dir) + # Now create the wrapper + os.remove(dest_bin) + magick_bin = dest_bin[:-len(dep.command)]+'magick' + with open(dest_bin, 'wt') as f: + f.write('#!/bin/sh\n') + # Include the downloaded libs + f.write('export LD_LIBRARY_PATH="{}:$LD_LIBRARY_PATH"\n'.format(lib_dir)) + # Also look for gs in our download dir + f.write('export PATH="$PATH:{}"\n'.format(home_bin)) + # Get the config from the downloaded config + f.write('export MAGICK_CONFIGURE_PATH="{}"\n'.format(dst_dir_name)) + # Use the `convert` tool + f.write('{} convert "$@"\n'.format(magick_bin)) + os.chmod(dest_bin, EXEC_PERM) + # Is this usable? + return check_tool_binary_version(dest_bin, dep) + + +def gs_downloader(dep): + # Currently only for Linux x86 + system = platform.system() + plat = platform.platform() + if system != 'Linux' or 'x86_' not in plat: + logger.debug('- No binary for this system') + return None + # Get the download page + url = 'https://api.github.com/repos/ArtifexSoftware/ghostpdl-downloads/releases/latest' + r = requests.get(url, allow_redirects=True) + if r.status_code != 200: + logger.debug('- Failed to download `{}`'.format(dep.url_down)) + return None + # Look for the valid tarball + arch = 'x86_64' if 'x86_64' in plat else 'x86' + url = None + pattern = 'ghostscript*linux-'+arch+'*' + try: + data = json.loads(r.content) + for a in data['assets']: + if fnmatch.fnmatch(a['name'], pattern): + url = a['browser_download_url'] + except Exception as e: + logger.debug('- Failed to find a download ({})'.format(e)) + if url is None: + logger.debug('- No suitable binary') + return None + # Try to download it + res = try_download_tar_ball(dep, url, 'ghostscript', 'ghostscript-*/gs*') + if res is not None: + short_gs = res[:-11]+'gs' + long_gs = res + if not os.path.isfile(short_gs): + os.symlink(long_gs, short_gs) + return res + + +def rsvg_downloader(dep): + # Currently only for Linux x86_64 + system = platform.system() + plat = platform.platform() + if system != 'Linux' or 'x86_64' not in plat: + logger.debug('- No binary for this system') + return None + # Get the download page + url = 'https://api.github.com/repos/set-soft/rsvg-convert-aws-lambda-binary/releases/latest' + r = requests.get(url, allow_redirects=True) + if r.status_code != 200: + logger.debug('- Failed to download `{}`'.format(dep.url_down)) + return None + # Look for the valid tarball + url = None + try: + data = json.loads(r.content) + for a in data['assets']: + if 'linux-x86_64' in a['name']: + url = a['browser_download_url'] + except Exception as e: + logger.debug('- Failed to find a download ({})'.format(e)) + if url is None: + logger.debug('- No suitable binary') + return None + # Try to download it + return try_download_tar_ball(dep, url, 'rsvg-convert') + + +def rar_downloader(dep): + # Get the download page + r = requests.get(dep.url_down, allow_redirects=True) + if r.status_code != 200: + logger.debug('- Failed to download `{}`'.format(dep.url_down)) + return None + # Try to figure out the right package + system = platform.system() + OSs = {'Linux': 'rarlinux', 'Darwin': 'rarmacos'} + if system not in OSs: + return None + name = OSs[system] + plat = platform.platform() + if 'arm64' in plat: + name += '-arm' + elif 'x86_64' in plat: + name += '-x64' + else: + name += '-x32' + res = re.search('href="([^"]+{}[^"]+)"'.format(name), r.content.decode()) + if not res: + return None + # Try to download it + return try_download_tar_ball(dep, dep.url+res.group(1), 'rar', name_in_tar='rar/rar') + + +def do_int(v): + return int(v) if v is not None else 0 + + +def run_command(cmd, only_first_line=True, pre_ver_text=None, no_err_2=False): + global last_stderr + try: + res_run = subprocess.run(cmd, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + if e.returncode != 2 or not no_err_2: + logger.debug('- Failed to run %s, error %d' % (cmd[0], e.returncode)) + last_stderr = e.stderr.decode() + if e.output: + logger.debug('- Output from command: '+e.output.decode()) + return None + except Exception as e: + logger.debug('- Failed to run {}, error {}'.format(cmd[0], e)) + return None + last_stderr = res_run.stderr.decode() + res = res_run.stdout.decode().strip() + if only_first_line: + res = res.split('\n')[0] + pre_vers = (cmd[0]+' version ', cmd[0]+' ', pre_ver_text) + for pre_ver in pre_vers: + if pre_ver and res.startswith(pre_ver): + res = res[len(pre_ver):] + res = ver_re.search(res) + if res: + return tuple(map(do_int, res.groups())) + return None + + +def check_tool_binary_version(full_name, dep): + logger.debugl(2, '- Checking version for `{}`'.format(full_name)) + if dep.no_cmd_line_version: + # No way to know the version, assume we can use it + logger.debugl(2, "- This tool doesn't have a version option") + return full_name + # Do we need a particular version? + needs = (0, 0, 0) + for r in dep.roles: + if r.version and r.version > needs: + needs = r.version + if needs == (0, 0, 0): + # Any version is Ok + logger.debugl(2, '- No particular version needed') + else: + logger.debugl(2, '- Needed version {}'.format(needs)) + # Check the version + if full_name in binary_tools_cache: + version = binary_tools_cache[full_name] + logger.debugl(2, '- Cached version {}'.format(version)) + else: + cmd = [full_name, dep.help_option] + if dep.is_kicad_plugin: + cmd.insert(0, 'python3') + version = run_command(cmd, no_err_2=dep.no_cmd_line_version_old) + binary_tools_cache[full_name] = version + logger.debugl(2, '- Found version {}'.format(version)) + return full_name if version is not None and version >= needs else None + + +def check_tool_binary_system(dep): + logger.debugl(2, '- Looking for tool `{}` at system level'.format(dep.command)) + if dep.is_kicad_plugin: + full_name = search_as_plugin(dep.command, dep.plugin_dirs) + else: + full_name = which(dep.command) + if full_name is None: + return None + return check_tool_binary_version(full_name, dep) + + +def using_downloaded(dep): + logger.warning(W_DOWNTOOL+'Using downloaded `{}` tool, please visit {} for details'.format(dep.command, dep.url)) + + +def check_tool_binary_local(dep): + logger.debugl(2, '- Looking for tool `{}` at user level'.format(dep.command)) + home = os.environ.get('HOME') or os.environ.get('username') + if home is None: + return None + full_name = os.path.join(home_bin, dep.command) + if not os.path.isfile(full_name) or not os.access(full_name, os.X_OK): + return None + cmd = check_tool_binary_version(full_name, dep) + if cmd is not None: + using_downloaded(dep) + return cmd + + +def try_download_tool_binary(dep): + if dep.downloader is None or home_bin is None: + return None + logger.info('- Trying to download {} ({})'.format(dep.name, dep.url_down)) + res = None + # res = dep.downloader(dep) + # return res + try: + res = dep.downloader(dep) + if res: + using_downloaded(dep) + except Exception as e: + logger.error('- Failed to download {}: {}'.format(dep.name, e)) + return res + + +def check_tool_binary(dep): + logger.debugl(2, '- Checking binary tool {}'.format(dep.name)) + cmd = check_tool_binary_system(dep) + if cmd is not None: + return cmd + cmd = check_tool_binary_local(dep) + if cmd is not None: + return cmd + return try_download_tool_binary(dep) + + +def check_tool_python(dep): + return None + + +def do_log_err(msg, fatal): + if fatal: + logger.error(msg) + else: + logger.warning(W_MISSTOOL+msg) + + +def check_tool(dep, fatal=False): + logger.debug('Starting tool check for {}'.format(dep.name)) + if dep.is_python: + cmd = check_tool_python(dep) + else: + cmd = check_tool_binary(dep) + logger.debug('- Returning `{}`'.format(cmd)) + if cmd is None: + do_log_err('Missing `{}` command ({}), install it'.format(dep.command, dep.name), fatal) + if dep.url: + do_log_err('Home page: '+dep.url, fatal) + if dep.url_down: + do_log_err('Download page: '+dep.url_down, fatal) + if dep.deb_package: + do_log_err('Debian package: '+dep.deb_package, fatal) + do_log_err(TRY_INSTALL_CHECK, fatal) + if fatal: + exit(MISSING_TOOL) + return cmd diff --git a/kibot/misc.py b/kibot/misc.py index 2bdb0a63..ce1ac5e9 100644 --- a/kibot/misc.py +++ b/kibot/misc.py @@ -243,6 +243,7 @@ W_PDMASKFAIL = '(W089) ' W_MISSTOOL = '(W090) ' W_NOTYET = '(W091) ' W_NOMATCH = '(W092) ' +W_DOWNTOOL = '(W093) ' # Somehow arbitrary, the colors are real, but can be different PCB_MAT_COLORS = {'fr1': "937042", 'fr2': "949d70", 'fr3': "adacb4", 'fr4': "332B16", 'fr5': "6cc290"} PCB_FINISH_COLORS = {'hal': "8b898c", 'hasl': "8b898c", 'imag': "8b898c", 'enig': "cfb96e", 'enepig': "cfb96e", @@ -260,6 +261,8 @@ SOLDER_COLORS = {'green': ("#285e3a", "#208b47"), SILK_COLORS = {'black': "0b1013", 'white': "d5dce4"} # KiCad 6 uses IUs for SVGs, but KiCad 5 uses a very different scale based on inches KICAD5_SVG_SCALE = 116930/297002200 +# Some browser name to pretend +USER_AGENT = 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1' class Rect(object): @@ -316,7 +319,7 @@ class ToolDependency(object): """ Class used to define tools needed for an output """ def __init__(self, output, name, url=None, url_down=None, is_python=False, deb=None, in_debian=True, extra_deb=None, roles=None, plugin_dirs=None, command=None, pypi_name=None, module_name=None, no_cmd_line_version=False, - help_option=None, no_cmd_line_version_old=False): + help_option=None, no_cmd_line_version_old=False, downloader=None): # The associated output self.output = output # Name of the tool @@ -342,6 +345,7 @@ class ToolDependency(object): # URLs self.url = url self.url_down = url_down + self.downloader = downloader # Can be installed as a KiCad plug-in? self.is_kicad_plugin = plugin_dirs is not None self.plugin_dirs = plugin_dirs @@ -366,6 +370,28 @@ def kiauto_dependency(output, version=None): in_debian=False, pypi_name='kiauto', command='pcbnew_do', roles=role) -def git_dependency(output): - return ToolDependency(output, 'Git', 'https://git-scm.com/', +def git_dependency(output, downloader): + return ToolDependency(output, 'Git', 'https://git-scm.com/', downloader=downloader, roles=ToolDependencyRole(desc='Find commit hash and/or date')) + + +def rsvg_dependency(output, downloader, roles=None): + return ToolDependency(output, 'RSVG tools', 'https://gitlab.gnome.org/GNOME/librsvg', deb='librsvg2-bin', + command='rsvg-convert', downloader=downloader, roles=roles) + + +def gs_dependency(output, downloader, roles=None): + return ToolDependency(output, 'Ghostscript', 'https://www.ghostscript.com/', + url_down='https://github.com/ArtifexSoftware/ghostpdl-downloads/releases', + downloader=downloader, roles=roles) + + +def convert_dependency(output, downloader, roles=None): + return ToolDependency(output, 'ImageMagick', 'https://imagemagick.org/', command='convert', + url_down='https://imagemagick.org/script/download.php', + downloader=downloader, roles=roles) + + +def pcbdraw_dependency(output, downloader, roles=None): + return ToolDependency(output, 'PcbDraw', URL_PCBDRAW, url_down=URL_PCBDRAW+'/releases', in_debian=False, + downloader=downloader, roles=roles) diff --git a/kibot/out_compress.py b/kibot/out_compress.py index 4ab439ff..5c52d018 100644 --- a/kibot/out_compress.py +++ b/kibot/out_compress.py @@ -14,17 +14,17 @@ from tarfile import open as tar_open from collections import OrderedDict from .gs import GS from .kiplot import config_output, get_output_dir, run_output -from .misc import (MISSING_TOOL, WRONG_INSTALL, W_EMPTYZIP, WRONG_ARGUMENTS, INTERNAL_ERROR, ToolDependency, - ToolDependencyRole, TRY_INSTALL_CHECK) +from .misc import (WRONG_INSTALL, W_EMPTYZIP, WRONG_ARGUMENTS, INTERNAL_ERROR, ToolDependency, ToolDependencyRole) from .optionable import Optionable, BaseOptions from .registrable import RegOutput, RegDependency from .macros import macros, document, output_class # noqa: F401 +from .dep_downloader import rar_downloader, check_tool from . import log logger = log.get_logger() -RegDependency.register(ToolDependency('compress', 'RAR', 'https://www.rarlab.com/', - url_down='https://www.rarlab.com/download.htm', help_option='-?', - roles=ToolDependencyRole(desc='Compress in RAR format'))) +rar_dep = ToolDependency('compress', 'RAR', 'https://www.rarlab.com/', url_down='https://www.rarlab.com/download.htm', + help_option='-?', downloader=rar_downloader, roles=ToolDependencyRole(desc='Compress in RAR format')) +RegDependency.register(rar_dep) class FilesList(Optionable): @@ -101,15 +101,15 @@ class CompressOptions(BaseOptions): def create_rar(self, output, files): if os.path.isfile(output): os.remove(output) + command = check_tool(rar_dep, fatal=True) + if command is None: + return for fname, dest in files.items(): - logger.debug('Adding '+fname+' as '+dest) - cmd = ['rar', 'a', '-m5', '-ep', '-ap'+os.path.dirname(dest), output, fname] + logger.debugl(2, 'Adding '+fname+' as '+dest) + cmd = [command, 'a', '-m5', '-ep', '-ap'+os.path.dirname(dest), output, fname] + logger.debugl(2, '- Running {}'.format(cmd)) try: check_output(cmd, stderr=STDOUT) - except FileNotFoundError: - logger.error('Missing `rar` command, install it') - logger.error(TRY_INSTALL_CHECK) - exit(MISSING_TOOL) except CalledProcessError as e: logger.error('Failed to invoke rar command, error {}'.format(e.returncode)) if e.output: @@ -138,11 +138,13 @@ class CompressOptions(BaseOptions): # Get the list of candidates files_list = None if f.from_output: + logger.debugl(2, '- From output `{}`'.format(f.from_output)) out = RegOutput.get_output(f.from_output) if out is not None: config_output(out) out_dir = get_output_dir(out.dir, out, dry=True) files_list = out.get_targets(out_dir) + logger.debugl(2, '- List of files: {}'.format(files_list)) if out_dir not in dirs_list: dirs_list.append(out_dir) else: @@ -163,6 +165,9 @@ class CompressOptions(BaseOptions): out_dir = out_dir_cwd if f.from_cwd else out_dir_default source = f.expand_filename_both(f.source, make_safe=False) files_list = glob.iglob(os.path.join(out_dir, source), recursive=True) + if GS.debug_level > 1: + files_list = list(files_list) + logger.debug('- Pattern {} list of files: {}'.format(source, files_list)) # Filter and adapt them for fname in filter(re.compile(f.filter).match, files_list): fname_real = os.path.realpath(fname) diff --git a/kibot/out_download_datasheets.py b/kibot/out_download_datasheets.py index 94f206b6..afd74c98 100644 --- a/kibot/out_download_datasheets.py +++ b/kibot/out_download_datasheets.py @@ -9,12 +9,11 @@ import requests from .out_base import VariantOptions from .fil_base import DummyFilter from .error import KiPlotConfigurationError -from .misc import W_UNKFLD, W_ALRDOWN, W_FAILDL +from .misc import W_UNKFLD, W_ALRDOWN, W_FAILDL, USER_AGENT from .gs import GS from .macros import macros, document, output_class # noqa: F401 from . import log logger = log.get_logger() -USER_AGENT = 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1' def is_url(ds): diff --git a/kibot/out_navigate_results.py b/kibot/out_navigate_results.py index 3c5d1f07..8b19901c 100644 --- a/kibot/out_navigate_results.py +++ b/kibot/out_navigate_results.py @@ -8,31 +8,27 @@ import os import subprocess import pprint -from shutil import copy2, which +from shutil import copy2 from math import ceil from struct import unpack from tempfile import NamedTemporaryFile from .gs import GS from .optionable import BaseOptions from .kiplot import config_output, get_output_dir -from .misc import W_NOTYET, W_MISSTOOL, TRY_INSTALL_CHECK, ToolDependencyRole, ToolDependency +from .misc import (W_NOTYET, W_MISSTOOL, ToolDependencyRole, rsvg_dependency, gs_dependency, convert_dependency) from .registrable import RegOutput, RegDependency +from .dep_downloader import check_tool, rsvg_downloader, gs_downloader, convert_downloader from .macros import macros, document, output_class # noqa: F401 from . import log, __version__ -SVGCONV = 'rsvg-convert' -CONVERT = 'convert' -PS2IMG = 'ghostscript' -RegDependency.register(ToolDependency('navigate_results', 'RSVG tools', - 'https://cran.r-project.org/web/packages/rsvg/index.html', deb='librsvg2-bin', - command=SVGCONV, - roles=[ToolDependencyRole(desc='Create outputs preview'), - ToolDependencyRole(desc='Create PNG icons')])) -RegDependency.register(ToolDependency('navigate_results', 'Ghostscript', 'https://www.ghostscript.com/', - url_down='https://github.com/ArtifexSoftware/ghostpdl-downloads/releases', - roles=ToolDependencyRole(desc='Create outputs preview'))) -RegDependency.register(ToolDependency('navigate_results', 'ImageMagick', 'https://imagemagick.org/', command='convert', - roles=ToolDependencyRole(desc='Create outputs preview'))) +rsvg_dep = rsvg_dependency('navigate_results', rsvg_downloader, roles=[ToolDependencyRole(desc='Create outputs preview'), + ToolDependencyRole(desc='Create PNG icons')]) +gs_dep = gs_dependency('navigate_results', gs_downloader, roles=ToolDependencyRole(desc='Create outputs preview')) +convert_dep = convert_dependency('navigate_results', convert_downloader, + roles=ToolDependencyRole(desc='Create outputs preview')) +RegDependency.register(rsvg_dep) +RegDependency.register(gs_dep) +RegDependency.register(convert_dep) logger = log.get_logger() CAT_IMAGE = {'PCB': 'pcbnew', 'Schematic': 'eeschema', @@ -146,11 +142,6 @@ def _run_command(cmd): return True -def svg_to_png(svg_file, png_file, width): - cmd = [SVGCONV, '-w', str(width), '-f', 'png', '-o', png_file, svg_file] - return _run_command(cmd) - - def get_png_size(file): with open(file, 'rb') as f: s = f.read() @@ -182,6 +173,10 @@ class Navigate_ResultsOptions(BaseOptions): node = node[c] node[out.name] = out + def svg_to_png(self, svg_file, png_file, width): + cmd = [self.rsvg_command, '-w', str(width), '-f', 'png', '-o', png_file, svg_file] + return _run_command(cmd) + def copy(self, img, width): """ Copy an SVG icon to the images/ dir. Tries to convert it to PNG. """ @@ -192,7 +187,7 @@ class Navigate_ResultsOptions(BaseOptions): src = os.path.join(self.img_src_dir, 'images', img+'.svg') dst = os.path.join(self.out_dir, 'images', img_w) id = img_w - if self.svg2png_avail and svg_to_png(src, dst+'.png', width): + if self.rsvg_command is not None and self.svg_to_png(src, dst+'.png', width): img_w += '.png' else: copy2(src, dst+'.svg') @@ -202,24 +197,21 @@ class Navigate_ResultsOptions(BaseOptions): return name def can_be_converted(self, ext): - if ext in IMAGEABLES_SVG and not self.svg2png_avail: - logger.warning(W_MISSTOOL+"Missing SVG to PNG converter: "+SVGCONV) - logger.warning(W_MISSTOOL+TRY_INSTALL_CHECK) + if ext in IMAGEABLES_SVG and self.rsvg_command is None: + logger.warning(W_MISSTOOL+"Missing SVG to PNG converter") return False if ext in IMAGEABLES_GS and not self.ps2img_avail: - logger.warning(W_MISSTOOL+"Missing PS/PDF to PNG converter: "+PS2IMG) - logger.warning(W_MISSTOOL+TRY_INSTALL_CHECK) + logger.warning(W_MISSTOOL+"Missing PS/PDF to PNG converter") return False - if ext in IMAGEABLES_SIMPLE and not self.convert_avail: - logger.warning(W_MISSTOOL+"Missing Imagemagick converter: "+CONVERT) - logger.warning(W_MISSTOOL+TRY_INSTALL_CHECK) + if ext in IMAGEABLES_SIMPLE and self.convert_command is None: + logger.warning(W_MISSTOOL+"Missing {} converter".format(convert_dep.name)) return False return ext in IMAGEABLES_SVG or ext in IMAGEABLES_GS or ext in IMAGEABLES_SIMPLE def get_image_for_cat(self, cat): img = None # Check if we have an output that can represent this category - if cat in CAT_REP and self.convert_avail: + if cat in CAT_REP and self.convert_command is not None: outs_rep = CAT_REP[cat] rep_file = None # Look in all outputs @@ -251,6 +243,8 @@ class Navigate_ResultsOptions(BaseOptions): if not os.path.isfile(file): logger.warning(W_NOTYET+"{} not yet generated, using an icon".format(os.path.relpath(file))) return False, None, None + if self.convert_command is None: + return False, None, None # Create a unique name using the output name and the generated file name bfname = os.path.splitext(os.path.basename(file))[0] fname = os.path.join(self.out_dir, 'images', out_name+'_'+bfname+'.png') @@ -263,10 +257,10 @@ class Navigate_ResultsOptions(BaseOptions): with NamedTemporaryFile(mode='w', suffix='.png', delete=False) as f: tmp_name = f.name logger.debug('Temporal convert: {} -> {}'.format(file, tmp_name)) - if not svg_to_png(file, tmp_name, BIG_ICON): + if not self.svg_to_png(file, tmp_name, BIG_ICON): return False, None, None file = tmp_name - cmd = [CONVERT, file, + cmd = [self.convert_command, file, # Size for the big icons (width) '-resize', str(BIG_ICON)+'x'] if not no_icon: @@ -448,9 +442,9 @@ class Navigate_ResultsOptions(BaseOptions): logger.debug('Collected outputs:\n'+pprint.pformat(o_tree)) with open(os.path.join(self.out_dir, 'styles.css'), 'wt') as f: f.write(STYLE) - self.svg2png_avail = which(SVGCONV) is not None - self.convert_avail = which(CONVERT) is not None - self.ps2img_avail = which(PS2IMG) is not None + self.rsvg_command = check_tool(rsvg_dep) + self.convert_command = check_tool(convert_dep) + self.ps2img_avail = check_tool(gs_dep) # Create the pages self.home = name self.back_img = self.copy('back', MID_ICON) diff --git a/kibot/out_pcb_print.py b/kibot/out_pcb_print.py index 8532b855..fe3754a2 100644 --- a/kibot/out_pcb_print.py +++ b/kibot/out_pcb_print.py @@ -23,10 +23,12 @@ from .kicad.config import KiConf from .kicad.v5_sch import SchError from .kicad.pcb import PCB from .misc import (CMD_PCBNEW_PRINT_LAYERS, URL_PCBNEW_PRINT_LAYERS, PDF_PCB_PRINT, MISSING_TOOL, W_PDMASKFAIL, - KICAD5_SVG_SCALE, W_MISSTOOL, ToolDependency, ToolDependencyRole, TRY_INSTALL_CHECK) + KICAD5_SVG_SCALE, W_MISSTOOL, ToolDependency, ToolDependencyRole, TRY_INSTALL_CHECK, rsvg_dependency, + gs_dependency, convert_dependency, pcbdraw_dependency) from .kiplot import check_script, exec_with_retry, add_extra_options from .registrable import RegDependency from .create_pdf import create_pdf_from_pages +from .dep_downloader import check_tool, rsvg_downloader, gs_downloader, convert_downloader from .macros import macros, document, output_class # noqa: F401 from .drill_marks import DRILL_MARKS_MAP, add_drill_marks from .layer import Layer, get_priority @@ -34,8 +36,6 @@ from . import __version__ from . import log logger = log.get_logger() -SVG2PDF = 'rsvg-convert' -PDF2PS = 'pdf2ps' VIATYPE_THROUGH = 3 VIATYPE_BLIND_BURIED = 2 VIATYPE_MICROVIA = 1 @@ -43,15 +43,15 @@ POLY_FILL_STYLE = ("fill:{0}; fill-opacity:1.0; stroke:{0}; stroke-width:1; stro "stroke-linejoin:round;fill-rule:evenodd;") DRAWING_LAYERS = ['Dwgs.User', 'Cmts.User', 'Eco1.User', 'Eco2.User'] EXTRA_LAYERS = ['F.Fab', 'B.Fab', 'F.CrtYd', 'B.CrtYd'] -RegDependency.register(ToolDependency('pcb_print', 'RSVG tools', - 'https://cran.r-project.org/web/packages/rsvg/index.html', deb='librsvg2-bin', - command=SVG2PDF, - roles=ToolDependencyRole(desc='Create PDF, PNG, EPS and PS formats'))) -RegDependency.register(ToolDependency('pcb_print', 'Ghostscript', 'https://www.ghostscript.com/', - url_down='https://github.com/ArtifexSoftware/ghostpdl-downloads/releases', - roles=ToolDependencyRole(desc='Create PS files'))) -RegDependency.register(ToolDependency('pcb_print', 'ImageMagick', 'https://imagemagick.org/', command='convert', - roles=ToolDependencyRole(desc='Create monochrome prints'))) +rsvg_dep = rsvg_dependency('pcb_print', rsvg_downloader, roles=ToolDependencyRole(desc='Create PDF, PNG, EPS and PS formats')) +gs_dep = gs_dependency('pcb_print', gs_downloader, roles=ToolDependencyRole(desc='Create PS files')) +convert_dep = convert_dependency('pcb_print', convert_downloader, roles=ToolDependencyRole(desc='Create monochrome prints')) +pcbdraw_dep = pcbdraw_dependency('pcb_print', None, roles=ToolDependencyRole(desc='Create realistic solder masks', + version=(0, 9, 0))) +RegDependency.register(rsvg_dep) +RegDependency.register(gs_dep) +RegDependency.register(convert_dep) +RegDependency.register(pcbdraw_dep) RegDependency.register(ToolDependency('pcb_print', 'LXML', is_python=True)) @@ -114,39 +114,6 @@ def get_size(svg): return float(view_box[2]), float(view_box[3]) -def svg_to_pdf(input_folder, svg_file, pdf_file): - # Note: rsvg-convert uses 90 dpi but KiCad (and the docs I found) says SVG pt is 72 dpi - cmd = [SVG2PDF, '-d', '72', '-p', '72', '-f', 'pdf', '-o', os.path.join(input_folder, pdf_file), - os.path.join(input_folder, svg_file)] - _run_command(cmd) - - -def svg_to_png(input_folder, svg_file, png_file, width): - cmd = [SVG2PDF, '-w', str(width), '-f', 'png', '-o', os.path.join(input_folder, png_file), - os.path.join(input_folder, svg_file)] - _run_command(cmd) - - -def svg_to_eps(input_folder, svg_file, eps_file): - cmd = [SVG2PDF, '-d', '72', '-p', '72', '-f', 'eps', '-o', os.path.join(input_folder, eps_file), - os.path.join(input_folder, svg_file)] - _run_command(cmd) - - -def pdf_to_ps(ps_file, output): - cmd = [PDF2PS, ps_file, output] - _run_command(cmd) - - -def create_pdf_from_svg_pages(input_folder, input_files, output_fn): - svg_files = [] - for svg_file in input_files: - pdf_file = svg_file.replace('.svg', '.pdf') - svg_to_pdf(input_folder, svg_file, pdf_file) - svg_files.append(os.path.join(input_folder, pdf_file)) - create_pdf_from_pages(svg_files, output_fn) - - class LayerOptions(Layer): """ Data for a layer """ def __init__(self): @@ -627,16 +594,13 @@ class PCB_PrintOptions(VariantOptions): not self.last_worksheet.has_images): return if monochrome: - if which('convert') is None: - logger.error('`convert` not installed. install `imagemagick` or equivalent') - logger.error(TRY_INSTALL_CHECK) - exit(MISSING_TOOL) + convert_command = check_tool(convert_dep, fatal=True) for img in self.last_worksheet.images: with NamedTemporaryFile(mode='wb', suffix='.png', delete=False) as f: f.write(img.data) fname = f.name dest = fname.replace('.png', '_gray.png') - _run_command(['convert', fname, '-set', 'colorspace', 'Gray', '-separate', '-average', dest]) + _run_command([convert_command, fname, '-set', 'colorspace', 'Gray', '-separate', '-average', dest]) with open(dest, 'rb') as f: img.data = f.read() os.remove(fname) @@ -855,16 +819,40 @@ class PCB_PrintOptions(VariantOptions): logger.debug('- Autoscale: {}'.format(scale)) return scale + def svg_to_pdf(self, input_folder, svg_file, pdf_file): + # Note: rsvg-convert uses 90 dpi but KiCad (and the docs I found) says SVG pt is 72 dpi + cmd = [self.rsvg_command, '-d', '72', '-p', '72', '-f', 'pdf', '-o', os.path.join(input_folder, pdf_file), + os.path.join(input_folder, svg_file)] + _run_command(cmd) + + def svg_to_png(self, input_folder, svg_file, png_file, width): + cmd = [self.rsvg_command, '-w', str(width), '-f', 'png', '-o', os.path.join(input_folder, png_file), + os.path.join(input_folder, svg_file)] + _run_command(cmd) + + def svg_to_eps(self, input_folder, svg_file, eps_file): + cmd = [self.rsvg_command, '-d', '72', '-p', '72', '-f', 'eps', '-o', os.path.join(input_folder, eps_file), + os.path.join(input_folder, svg_file)] + _run_command(cmd) + + def pdf_to_ps(self, ps_file, output): + cmd = [self.gs_command, '-q', '-dNOPAUSE', '-dBATCH', '-P-', '-dSAFER', '-sDEVICE=ps2write', '-sOutputFile='+output, + '-c', 'save', 'pop', '-f', ps_file] + _run_command(cmd) + + def create_pdf_from_svg_pages(self, input_folder, input_files, output_fn): + svg_files = [] + for svg_file in input_files: + pdf_file = svg_file.replace('.svg', '.pdf') + self.svg_to_pdf(input_folder, svg_file, pdf_file) + svg_files.append(os.path.join(input_folder, pdf_file)) + create_pdf_from_pages(svg_files, output_fn) + def generate_output(self, output): - if self.format != 'SVG' and which(SVG2PDF) is None: - logger.error('`{}` not installed. Install `librsvg2-bin` or equivalent'.format(SVG2PDF)) - logger.error(TRY_INSTALL_CHECK) - exit(MISSING_TOOL) - if self.format == 'PS' and which(PDF2PS) is None: - logger.error('`{}` not installed. '.format(PDF2PS)) - logger.error('Install `librsvg2-bin` or equivalent') - logger.error(TRY_INSTALL_CHECK) - exit(MISSING_TOOL) + if self.format != 'SVG': + self.rsvg_command = check_tool(rsvg_dep, fatal=True) + if self.format == 'PS': + self.gs_command = check_tool(gs_dep, fatal=True) output_dir = os.path.dirname(output) if self.keep_temporal_files: temp_dir_base = output_dir @@ -959,20 +947,20 @@ class PCB_PrintOptions(VariantOptions): id = self._expand_id+('_page_'+page_str) out_file = self.expand_filename(output_dir, self.output, id, self._expand_ext) if self.format == 'PNG': - svg_to_png(temp_dir, assembly_file, out_file, self.png_width) + self.svg_to_png(temp_dir, assembly_file, out_file, self.png_width) else: - svg_to_eps(temp_dir, assembly_file, out_file) + self.svg_to_eps(temp_dir, assembly_file, out_file) pages.append(os.path.join(page_str, assembly_file)) self.restore_title() # Join all pages in one file if self.format in ['PDF', 'PS']: logger.debug('- Creating output file {}'.format(output)) if self.format == 'PDF': - create_pdf_from_svg_pages(temp_dir_base, pages, output) + self.create_pdf_from_svg_pages(temp_dir_base, pages, output) else: ps_file = os.path.join(temp_dir, GS.pcb_basename+'.ps') - create_pdf_from_svg_pages(temp_dir_base, pages, ps_file) - pdf_to_ps(ps_file, output) + self.create_pdf_from_svg_pages(temp_dir_base, pages, ps_file) + self.pdf_to_ps(ps_file, output) # Remove the temporal files if not self.keep_temporal_files: rmtree(temp_dir_base) @@ -1011,12 +999,12 @@ class PCB_Print(BaseOutput): # noqa: F821 if not realistic_solder_mask: logger.warning(W_MISSTOOL+'Missing PcbDraw tool, disabling `realistic_solder_mask`') # Check we can convert SVGs - if which(SVG2PDF) is None: - logger.warning(W_MISSTOOL+'Missing {} tool, disabling most printed formats'.format(SVG2PDF)) + if check_tool(rsvg_dep) is None: + logger.warning(W_MISSTOOL+'Disabling most printed formats') disabled |= {'PDF', 'PNG', 'EPS', 'PS'} # Check we can convert to PS - if which(PDF2PS) is None: - logger.warning(W_MISSTOOL+'Missing {} tool, disabling postscript printed format'.format(PDF2PS)) + if check_tool(gs_dep) is None: + logger.warning(W_MISSTOOL+'Disabling postscript printed format') disabled.add('PS') # Generate one output for each format for fmt in ['PDF', 'SVG', 'PNG', 'EPS', 'PS']: diff --git a/kibot/out_pcbdraw.py b/kibot/out_pcbdraw.py index 6f8006ee..185955fd 100644 --- a/kibot/out_pcbdraw.py +++ b/kibot/out_pcbdraw.py @@ -7,29 +7,27 @@ import os from tempfile import NamedTemporaryFile # Here we import the whole module to make monkeypatch work import subprocess -import shutil from .misc import (PCBDRAW, PCBDRAW_ERR, URL_PCBDRAW, W_AMBLIST, W_UNRETOOL, W_USESVG2, W_USEIMAGICK, PCB_MAT_COLORS, - PCB_FINISH_COLORS, SOLDER_COLORS, SILK_COLORS, ToolDependency, ToolDependencyRole, TRY_INSTALL_CHECK) + PCB_FINISH_COLORS, SOLDER_COLORS, SILK_COLORS, ToolDependencyRole, rsvg_dependency, convert_dependency, + pcbdraw_dependency) from .kiplot import check_script from .registrable import RegDependency from .gs import GS from .optionable import Optionable from .out_base import VariantOptions +from .dep_downloader import check_tool, rsvg_downloader, convert_downloader from .macros import macros, document, output_class # noqa: F401 from . import log logger = log.get_logger() -SVG2PNG = 'rsvg-convert' -CONVERT = 'convert' # 0.9.0 implements KiCad 6 support MIN_VERSION = '0.9.0' -RegDependency.register(ToolDependency('pcbdraw', 'RSVG tools', 'https://cran.r-project.org/web/packages/rsvg/index.html', - deb='librsvg2-bin', command=SVG2PNG, - roles=ToolDependencyRole(desc='Create PNG and JPG images'))) -RegDependency.register(ToolDependency('pcbdraw', 'ImageMagick', 'https://imagemagick.org/', command='convert', - roles=ToolDependencyRole(desc='Create JPG images'))) -RegDependency.register(ToolDependency('pcbdraw', 'PcbDraw', URL_PCBDRAW, url_down=URL_PCBDRAW+'/releases', in_debian=False, - roles=ToolDependencyRole(version=(0, 9, 0)))) +rsvg_dep = rsvg_dependency('pcbdraw', rsvg_downloader, roles=ToolDependencyRole(desc='Create PNG and JPG images')) +convert_dep = convert_dependency('pcbdraw', convert_downloader, roles=ToolDependencyRole(desc='Create JPG images')) +pcbdraw_dep = pcbdraw_dependency('pcbdraw', None, roles=ToolDependencyRole(version=(0, 9, 0))) +RegDependency.register(rsvg_dep) +RegDependency.register(convert_dep) +RegDependency.register(pcbdraw_dep) class PcbDrawStyle(Optionable): @@ -248,19 +246,21 @@ class PcbDrawOptions(VariantOptions): cmd.append(output) else: # PNG and JPG outputs are unreliable - if shutil.which(SVG2PNG) is None: - logger.warning(W_UNRETOOL + '`{}` not installed, using unreliable PNG/JPG conversion'.format(SVG2PNG)) - logger.warning(W_USESVG2 + 'If you experiment problems install `librsvg2-bin` or equivalent') - logger.warning(W_USESVG2 + TRY_INSTALL_CHECK) - cmd.append(output) - elif shutil.which(CONVERT) is None: - logger.warning(W_UNRETOOL + '`{}` not installed, using unreliable PNG/JPG conversion'.format(CONVERT)) - logger.warning(W_USEIMAGICK + 'If you experiment problems install `imagemagick` or equivalent') - logger.warning(W_USEIMAGICK + TRY_INSTALL_CHECK) + self.rsvg_command = check_tool(rsvg_dep) + if self.rsvg_command is None: + logger.warning(W_UNRETOOL + '`{}` not installed, using unreliable PNG/JPG conversion'.format(rsvg_dep.name)) + logger.warning(W_USESVG2 + 'If you experiment problems install it') cmd.append(output) else: - svg = _get_tmp_name('.svg') - cmd.append(svg) + self.convert_command = check_tool(convert_dep) + if self.convert_command is None: + logger.warning(W_UNRETOOL + '`{}` not installed, using unreliable PNG/JPG conversion'. + format(convert_dep.name)) + logger.warning(W_USEIMAGICK + 'If you experiment problems install it') + cmd.append(output) + else: + svg = _get_tmp_name('.svg') + cmd.append(svg) return svg def get_targets(self, out_dir): @@ -318,8 +318,8 @@ class PcbDrawOptions(VariantOptions): if svg is not None: # Manually convert the SVG to PNG png = _get_tmp_name('.png') - _run_command([SVG2PNG, '-d', str(self.dpi), '-p', str(self.dpi), svg, '-o', png], svg) - cmd = [CONVERT, '-trim', png] + _run_command([self.rsvg_command, '-d', str(self.dpi), '-p', str(self.dpi), svg, '-o', png], svg) + cmd = [self.convert_command, '-trim', png] if self.format == 'jpg': cmd += ['-quality', '85%'] cmd.append(name) diff --git a/kibot/pre_any_replace.py b/kibot/pre_any_replace.py index bf3aad06..174e65cb 100644 --- a/kibot/pre_any_replace.py +++ b/kibot/pre_any_replace.py @@ -12,10 +12,12 @@ from .misc import FAILED_EXECUTE, W_EMPTREP, W_BADCHARS from .optionable import Optionable from .pre_base import BasePreFlight from .gs import GS +from .dep_downloader import check_tool from .macros import macros, document, pre_class # noqa: F401 from . import log logger = log.get_logger() +re_git = re.compile(r'([^a-zA-Z_]|^)(git) ') class TagReplaceBase(Optionable): @@ -86,7 +88,7 @@ class Base_Replace(BasePreFlight): # noqa: F821 "\n before: 'Git hash: <'" "\n after: '>'".format(cls._context, cls._context)) - def replace(self, file): + def replace(self, file, git_dep): logger.debug('Applying replacements to `{}`'.format(file)) with open(file, 'rt') as f: content = f.read() @@ -95,7 +97,12 @@ class Base_Replace(BasePreFlight): # noqa: F821 for r in o.replace_tags: text = r.text if not text: - cmd = ['/bin/bash', '-c', r.command] + command = r.command + if re_git.search(command): + git_command = check_tool(git_dep, fatal=True) + command = re_git.sub(r'\1'+git_command+' ', command) + cmd = ['/bin/bash', '-c', command] + logger.debugl(2, 'Running: {}'.format(cmd)) result = run(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True) if result.returncode: logger.error('Failed to execute:\n{}\nreturn code {}'.format(r.command, result.returncode)) diff --git a/kibot/pre_pcb_replace.py b/kibot/pre_pcb_replace.py index 6a9375bc..1bc4a4ad 100644 --- a/kibot/pre_pcb_replace.py +++ b/kibot/pre_pcb_replace.py @@ -7,11 +7,13 @@ from .gs import GS from .pre_any_replace import TagReplaceBase, Base_ReplaceOptions, Base_Replace from .registrable import RegDependency from .misc import git_dependency +from .dep_downloader import git_downloader from .macros import macros, document, pre_class # noqa: F401 from . import log logger = log.get_logger() -RegDependency.register(git_dependency('pcb_replace')) +git_dep = git_dependency('pcb_replace', git_downloader) +RegDependency.register(git_dep) class TagReplacePCB(TagReplaceBase): @@ -56,6 +58,6 @@ class PCB_Replace(Base_Replace): # noqa: F821 t.after = '")' t._relax_check = True o.replace_tags.append(t) - self.replace(GS.pcb_file) + self.replace(GS.pcb_file, git_dep) # Force the schematic reload GS.board = None diff --git a/kibot/pre_sch_replace.py b/kibot/pre_sch_replace.py index ee765aa5..e37254df 100644 --- a/kibot/pre_sch_replace.py +++ b/kibot/pre_sch_replace.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2021 Salvador E. Tropea -# Copyright (c) 2021 Instituto Nacional de Tecnología Industrial +# Copyright (c) 2021-2022 Salvador E. Tropea +# Copyright (c) 2021-2022 Instituto Nacional de Tecnología Industrial # License: GPL-3.0 # Project: KiBot (formerly KiPlot) import os @@ -9,11 +9,13 @@ from .kiplot import load_sch from .pre_any_replace import TagReplaceBase, Base_ReplaceOptions, Base_Replace from .registrable import RegDependency from .misc import git_dependency +from .dep_downloader import git_downloader from .macros import macros, document, pre_class # noqa: F401 from . import log logger = log.get_logger() -RegDependency.register(git_dependency('sch_replace')) +git_dep = git_dependency('sch_replace', git_downloader) +RegDependency.register(git_dep) class TagReplaceSCH(TagReplaceBase): @@ -68,6 +70,6 @@ class SCH_Replace(Base_Replace): # noqa: F821 load_sch() os.environ['KIBOT_TOP_SCH_NAME'] = GS.sch_file for file in GS.sch.get_files(): - self.replace(file) + self.replace(file, git_dep) # Force the schematic reload GS.sch = None diff --git a/kibot/pre_set_text_variables.py b/kibot/pre_set_text_variables.py index 26b4cf22..2c3c232e 100644 --- a/kibot/pre_set_text_variables.py +++ b/kibot/pre_set_text_variables.py @@ -6,6 +6,7 @@ import os import sys import json +import re from subprocess import run, PIPE from .error import KiPlotConfigurationError from .misc import FAILED_EXECUTE, W_EMPTREP, git_dependency @@ -13,11 +14,14 @@ from .optionable import Optionable from .pre_base import BasePreFlight from .gs import GS from .registrable import RegDependency +from .dep_downloader import git_downloader, check_tool from .macros import macros, document, pre_class # noqa: F401 from . import log logger = log.get_logger() -RegDependency.register(git_dependency('set_text_variables')) +git_dep = git_dependency('set_text_variables', git_downloader) +RegDependency.register(git_dep) +re_git = re.compile(r'([^a-zA-Z_]|^)(git) ') class KiCadVariable(Optionable): @@ -112,7 +116,11 @@ class Set_Text_Variables(BasePreFlight): # noqa: F821 for r in o: text = r.text if not text: - cmd = ['/bin/bash', '-c', r.command] + command = r.command + if re_git.search(command): + git_command = check_tool(git_dep, fatal=True) + command = re_git.sub(r'\1'+git_command+' ', command) + cmd = ['/bin/bash', '-c', command] result = run(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True) if result.returncode: logger.error('Failed to execute:\n{}\nreturn code {}'.format(r.command, result.returncode)) diff --git a/src/kibot-check b/src/kibot-check index da89f827..b6ff77b8 100755 --- a/src/kibot-check +++ b/src/kibot-check @@ -20,6 +20,7 @@ deps = '{\ "Colorama": {\ "command": "colorama",\ "deb_package": "python3-colorama",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 100,\ @@ -47,6 +48,7 @@ deps = '{\ "Distutils": {\ "command": "distutils",\ "deb_package": "python3-distutils",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 1000000,\ @@ -74,6 +76,7 @@ deps = '{\ "Ghostscript": {\ "command": "ghostscript",\ "deb_package": "ghostscript",\ + "downloader": {},\ "extra_deb": null,\ "help_option": "--version",\ "importance": 2,\ @@ -106,6 +109,7 @@ deps = '{\ "Git": {\ "command": "git",\ "deb_package": "git",\ + "downloader": {},\ "extra_deb": null,\ "help_option": "--version",\ "importance": 3,\ @@ -144,6 +148,7 @@ deps = '{\ "ImageMagick": {\ "command": "convert",\ "deb_package": "imagemagick",\ + "downloader": {},\ "extra_deb": null,\ "help_option": "--version",\ "importance": 3,\ @@ -177,11 +182,12 @@ deps = '{\ }\ ],\ "url": "https://imagemagick.org/",\ - "url_down": null\ + "url_down": "https://imagemagick.org/script/download.php"\ },\ "Interactive HTML BoM": {\ "command": "generate_interactive_bom.py",\ "deb_package": "interactive html bom",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 10000,\ @@ -217,6 +223,7 @@ deps = '{\ "KiBoM": {\ "command": "KiBOM_CLI.py",\ "deb_package": "kibom",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 10000,\ @@ -247,6 +254,7 @@ deps = '{\ "KiCad Automation tools": {\ "command": "pcbnew_do",\ "deb_package": "kicad automation tools",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 110000,\ @@ -357,6 +365,7 @@ deps = '{\ "KiCost": {\ "command": "kicost",\ "deb_package": "kicost",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 10001,\ @@ -397,6 +406,7 @@ deps = '{\ "LXML": {\ "command": "lxml",\ "deb_package": "python3-lxml",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 10000,\ @@ -424,6 +434,7 @@ deps = '{\ "Pandoc": {\ "command": "pandoc",\ "deb_package": "pandoc",\ + "downloader": null,\ "extra_deb": [\ "texlive-latex-base",\ "texlive-latex-recommended"\ @@ -453,19 +464,30 @@ deps = '{\ "PcbDraw": {\ "command": "pcbdraw",\ "deb_package": "pcbdraw",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ - "importance": 10000,\ + "importance": 10001,\ "in_debian": false,\ "is_kicad_plugin": false,\ "is_python": false,\ "name": "PcbDraw",\ "no_cmd_line_version": false,\ "no_cmd_line_version_old": false,\ - "output": "pcbdraw",\ + "output": "pcb_print",\ "plugin_dirs": null,\ "pypi_name": "PcbDraw",\ "roles": [\ + {\ + "desc": "Create realistic solder masks",\ + "mandatory": false,\ + "output": "pcb_print",\ + "version": [\ + 0,\ + 9,\ + 0\ + ]\ + },\ {\ "desc": null,\ "mandatory": true,\ @@ -483,6 +505,7 @@ deps = '{\ "PyYAML": {\ "command": "pyyaml",\ "deb_package": "python3-yaml",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 1000000,\ @@ -510,6 +533,7 @@ deps = '{\ "QRCodeGen": {\ "command": "qrcodegen",\ "deb_package": "python3-qrcodegen",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 10000,\ @@ -537,6 +561,7 @@ deps = '{\ "RAR": {\ "command": "rar",\ "deb_package": "rar",\ + "downloader": {},\ "extra_deb": null,\ "help_option": "-?",\ "importance": 1,\ @@ -563,6 +588,7 @@ deps = '{\ "RSVG tools": {\ "command": "rsvg-convert",\ "deb_package": "librsvg2-bin",\ + "downloader": {},\ "extra_deb": null,\ "help_option": "--version",\ "importance": 4,\ @@ -601,12 +627,13 @@ deps = '{\ "version": null\ }\ ],\ - "url": "https://cran.r-project.org/web/packages/rsvg/index.html",\ + "url": "https://gitlab.gnome.org/GNOME/librsvg",\ "url_down": null\ },\ "Requests": {\ "command": "requests",\ "deb_package": "python3-requests",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 1000000,\ @@ -634,6 +661,7 @@ deps = '{\ "XLSXWriter": {\ "command": "xlsxwriter",\ "deb_package": "python3-xlsxwriter",\ + "downloader": null,\ "extra_deb": null,\ "help_option": "--version",\ "importance": 1,\ diff --git a/tests/test_plot/test_misc_2.py b/tests/test_plot/test_misc_2.py index 0f48970b..e967246e 100644 --- a/tests/test_plot/test_misc_2.py +++ b/tests/test_plot/test_misc_2.py @@ -11,8 +11,7 @@ from kibot.out_base import BaseOutput from kibot.gs import GS from kibot.kiplot import load_actions, _import, load_board, search_as_plugin, generate_makefile from kibot.registrable import RegOutput, RegFilter -from kibot.misc import (MISSING_TOOL, WRONG_INSTALL, BOM_ERROR, DRC_ERROR, ERC_ERROR, PDF_PCB_PRINT, CMD_PCBNEW_PRINT_LAYERS, - KICAD2STEP_ERR) +from kibot.misc import (WRONG_INSTALL, BOM_ERROR, DRC_ERROR, ERC_ERROR, PDF_PCB_PRINT, CMD_PCBNEW_PRINT_LAYERS, KICAD2STEP_ERR) from kibot.bom.columnlist import ColumnList from kibot.bom.units import get_prefix from kibot.__main__ import detect_kicad @@ -80,21 +79,22 @@ def run_compress(ctx, test_import_fail=False): return pytest_wrapped_e -def test_no_rar(test_dir, caplog, monkeypatch): - global mocked_check_output_FNF - mocked_check_output_FNF = True - # Create a silly context to get the output path - ctx = context.TestContext(test_dir, 'test_v5', 'empty_zip', '') - # The file we pretend to compress - ctx.create_dummy_out_file('Test.txt') - # We will patch subprocess.check_output to make rar fail - with monkeypatch.context() as m: - patch_functions(m) - pytest_wrapped_e = run_compress(ctx) - # Check we exited because rar isn't installed - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == MISSING_TOOL - assert "Missing `rar` command" in caplog.text +# No longer possible, we trust in check_tool, it won't return an unexistent file name, so we don't catch FileNoFound +# def test_no_rar(test_dir, caplog, monkeypatch): +# global mocked_check_output_FNF +# mocked_check_output_FNF = True +# # Create a silly context to get the output path +# ctx = context.TestContext(test_dir, 'test_v5', 'empty_zip', '') +# # The file we pretend to compress +# ctx.create_dummy_out_file('Test.txt') +# # We will patch subprocess.check_output to make rar fail +# with monkeypatch.context() as m: +# patch_functions(m) +# pytest_wrapped_e = run_compress(ctx) +# # Check we exited because rar isn't installed +# assert pytest_wrapped_e.type == SystemExit +# assert pytest_wrapped_e.value.code == MISSING_TOOL +# assert "Missing `rar` command" in caplog.text def test_rar_fail(test_dir, caplog, monkeypatch): diff --git a/tests/test_plot/test_pcbdraw.py b/tests/test_plot/test_pcbdraw.py index 93d2fe77..f1b6dea8 100644 --- a/tests/test_plot/test_pcbdraw.py +++ b/tests/test_plot/test_pcbdraw.py @@ -5,10 +5,14 @@ For debug information use: pytest-3 --log-cli-level debug """ import coverage +import logging from shutil import which +from os import access +from importlib import reload from . import context from kibot.mcpyrate import activate # noqa: F401 from kibot.out_pcbdraw import PcbDrawOptions +import kibot.log OUT_DIR = 'PcbDraw' cov = coverage.Coverage() @@ -33,17 +37,29 @@ def test_pcbdraw_simple(test_dir): def no_rsvg_convert(name): + logging.debug('no_rsvg_convert called') if name == 'rsvg-convert': + logging.debug('no_rsvg_convert returns None') return None return which(name) def no_convert(name): + logging.debug('no_convert called') if name == 'convert': + logging.debug('no_convert returns None') return None return which(name) +def no_convert_access(name, attrs): + logging.debug('no_convert_access') + if name.endswith('/convert'): + logging.debug('no_convert_access returns False') + return False + return access(name, attrs) + + def no_run(cmd, stderr): return b"" @@ -53,6 +69,10 @@ def test_pcbdraw_miss_rsvg(caplog, monkeypatch): with monkeypatch.context() as m: m.setattr("shutil.which", no_rsvg_convert) m.setattr("subprocess.check_output", no_run) + # Reload the module so we get the above patches + reload(kibot.dep_downloader) + old_lev = kibot.log.debug_level + kibot.log.debug_level = 2 o = PcbDrawOptions() o.style = '' o.remap = None @@ -63,6 +83,7 @@ def test_pcbdraw_miss_rsvg(caplog, monkeypatch): o.run('') cov.stop() cov.save() + kibot.log.debug_level = old_lev assert 'using unreliable PNG/JPG' in caplog.text, caplog.text assert 'librsvg2-bin' in caplog.text, caplog.text @@ -72,6 +93,9 @@ def test_pcbdraw_miss_convert(caplog, monkeypatch): with monkeypatch.context() as m: m.setattr("shutil.which", no_convert) m.setattr("subprocess.check_output", no_run) + m.setattr("os.access", no_convert_access) + # Reload the module so we get the above patches + reload(kibot.dep_downloader) o = PcbDrawOptions() o.style = '' o.remap = None