diff --git a/README.md b/README.md index e521aa7b..88f472ae 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ Notes: [**RAR**](https://www.rarlab.com/) [![Tool](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/llave-inglesa-22x22.png)](https://www.rarlab.com/) [![Debian](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/debian-openlogo-22x22.png)](https://packages.debian.org/bullseye/rar) ![Auto-download](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/auto_download-22x22.png) - Optional to compress in RAR format for `compress` -[**XLSXWriter**](https://pypi.org/project/XLSXWriter/) [![Python module](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/Python-logo-notext-22x22.png)](https://pypi.org/project/XLSXWriter/) [![PyPi dependency](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/PyPI_logo_simplified-22x22.png)](https://pypi.org/project/XLSXWriter/) [![Debian](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/debian-openlogo-22x22.png)](https://packages.debian.org/bullseye/python3-xlsxwriter) +[**XLSXWriter**](https://pypi.org/project/XLSXWriter/) [![Python module](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/Python-logo-notext-22x22.png)](https://pypi.org/project/XLSXWriter/) [![PyPi dependency](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/PyPI_logo_simplified-22x22.png)](https://pypi.org/project/XLSXWriter/) [![Debian](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/debian-openlogo-22x22.png)](https://packages.debian.org/bullseye/python3-xlsxwriter) ![Auto-download](https://raw.githubusercontent.com/INTI-CMNB/KiBot/master/docs/images/auto_download-22x22.png) - Optional to create XLSX files for `bom` diff --git a/kibot/bom/bom_writer.py b/kibot/bom/bom_writer.py index 05ad3194..9e47d940 100644 --- a/kibot/bom/bom_writer.py +++ b/kibot/bom/bom_writer.py @@ -17,7 +17,6 @@ This is just a hub that calls the real BoM writer: from .csv_writer import write_csv from .html_writer import write_html from .xml_writer import write_xml -from .xlsx_writer import write_xlsx from .. import log logger = log.get_logger() @@ -43,6 +42,8 @@ def write_bom(filename, ext, groups, headings, cfg): elif ext in ["xml"]: result = write_xml(filename, groups, headings, head_names, cfg) elif ext in ["xlsx"]: + # We delay the module load to give out_bom the chance to install XLSXWriter dependencies + from .xlsx_writer import write_xlsx result = write_xlsx(filename, groups, headings, head_names, cfg) if result: diff --git a/kibot/bom/xlsx_writer.py b/kibot/bom/xlsx_writer.py index 194e00a0..b5396607 100644 --- a/kibot/bom/xlsx_writer.py +++ b/kibot/bom/xlsx_writer.py @@ -19,10 +19,13 @@ from base64 import b64decode from .columnlist import ColumnList from .kibot_logo import KIBOT_LOGO from .. import log -from ..misc import W_NOKICOST, W_UNKDIST, KICOST_ERROR, W_BADFIELD, TRY_INSTALL_CHECK +from ..misc import W_NOKICOST, W_UNKDIST, KICOST_ERROR, W_BADFIELD from ..error import trace_dump from ..gs import GS from .. import __version__ +# Init the logger first +logger = log.get_logger() +# XLSX Writer support try: from xlsxwriter import Workbook XLSX_SUPPORT = True @@ -31,8 +34,6 @@ except ModuleNotFoundError: class Workbook(): pass -# Init the logger first -logger = log.get_logger() # KiCost support try: # Give priority to submodules @@ -46,13 +47,7 @@ try: init_all_loggers, create_worksheet, Spreadsheet, get_distributors_list, get_dist_name_from_label, set_distributors_progress, is_valid_api) KICOST_SUPPORT = True -except ModuleNotFoundError: - KICOST_SUPPORT = False - ProgressConsole = object except ImportError: - logger.error("Installed KiCost is older than the version we support.") - logger.error("Try installing the last release or the current GIT code.") - logger.error(TRY_INSTALL_CHECK) KICOST_SUPPORT = False ProgressConsole = object @@ -712,8 +707,6 @@ def write_xlsx(filename, groups, col_fields, head_names, cfg): cfg = BoMOptions object with all the configuration """ if not XLSX_SUPPORT: - logger.error('Python xlsxwriter module not installed (Debian: python3-xlsxwriter)') - logger.error(TRY_INSTALL_CHECK) return False link_datasheet = -1 diff --git a/kibot/dep_downloader.py b/kibot/dep_downloader.py index fb04c3c4..5d0e701b 100644 --- a/kibot/dep_downloader.py +++ b/kibot/dep_downloader.py @@ -3,6 +3,7 @@ # Copyright (c) 2022 Instituto Nacional de Tecnología Industrial # License: GPL-3.0 # Project: KiBot (formerly KiPlot) +import importlib import os import re import subprocess @@ -17,7 +18,7 @@ import site from sys import exit, stdout from shutil import which, rmtree, move from math import ceil -from .misc import MISSING_TOOL, TRY_INSTALL_CHECK, W_DOWNTOOL, W_MISSTOOL, USER_AGENT +from .misc import MISSING_TOOL, TRY_INSTALL_CHECK, W_DOWNTOOL, W_MISSTOOL, USER_AGENT, version_str2tuple from .gs import GS from . import log @@ -147,18 +148,7 @@ def untar(data): return os.path.abspath(dir_name) -def pytool_downloader(dep, system, plat): - # Check if we have a github repo as download page - logger.debug('- Download URL: '+str(dep.url_down)) - if not dep.url_down: - return None - res = re.match(r'^https://github.com/([^/]+)/([^/]+)/', dep.url_down) - if res is None: - return None - user = res.group(1) - prj = res.group(2) - logger.debugl(2, '- GitHub repo: {}/{}'.format(user, prj)) - url = 'https://api.github.com/repos/{}/{}/releases/latest'.format(user, prj) +def check_pip(): # Check if we have pip and wheel pip_command = which('pip3') if pip_command is not None: @@ -177,14 +167,45 @@ def pytool_downloader(dep, system, plat): logger.debugl(2, '- Wheel v{}'.format(wheel.__version__)) except ImportError: wheel_ok = False - if not wheel_ok: - cmd = [pip_command, 'install', '--no-warn-script-location', '-U', 'wheel'] - logger.debug('- Trying to install wheel: `{}`'.format(cmd)) - try: - res_run = subprocess.run(cmd, check=True, capture_output=True) - except Exception as e: - logger.debug('- Failed to install wheel ({})'.format(e)) - return None + if not wheel_ok and not pip_install(pip_command, name='wheel'): + return None + return pip_command + + +def pip_install(pip_command, dest=None, name='.'): + cmd = [pip_command, 'install', '-U', '--no-warn-script-location', name] + logger.debug('- Running: {}'.format(cmd)) + try: + res_run = subprocess.run(cmd, check=True, capture_output=True, cwd=dest) + logger.debugl(3, '- Output from pip:\n'+res_run.stdout.decode()) + except Exception as e: + logger.debug('- Failed to install `{}` using pip ({})'.format(name, e)) + out = res_run.stderr.decode() + if out: + logger.debug('- StdErr: '+out) + out = res_run.stdout.decode() + if out: + logger.debug('- StdOut: '+out) + return False + return True + + +def pytool_downloader(dep, system, plat): + # Check if we have a github repo as download page + logger.debug('- Download URL: '+str(dep.url_down)) + if not dep.url_down: + return None + res = re.match(r'^https://github.com/([^/]+)/([^/]+)/', dep.url_down) + if res is None: + return None + user = res.group(1) + prj = res.group(2) + logger.debugl(2, '- GitHub repo: {}/{}'.format(user, prj)) + url = 'https://api.github.com/repos/{}/{}/releases/latest'.format(user, prj) + # Check if we have pip and wheel + pip_command = check_pip() + if pip_command is None: + return None # Look for the last release data = download(url, progress=False) if data is None: @@ -203,25 +224,25 @@ def pytool_downloader(dep, system, plat): return None logger.debugl(2, '- Uncompressed tarball to: '+dest) # Try to pip install it - cmd = [pip_command, 'install', '-U', '--no-warn-script-location', '.'] - logger.debug('- Running: {}'.format(cmd)) - try: - res_run = subprocess.run(cmd, check=True, capture_output=True, cwd=dest) - logger.debugl(3, '- Output from pip:\n'+res_run.stdout.decode()) - except Exception as e: - logger.debug('- Failed to install using pip ({})'.format(e)) - out = res_run.stderr.decode() - if out: - logger.debug('- StdErr: '+out) - out = res_run.stdout.decode() - if out: - logger.debug('- StdOut: '+out) + if not pip_install(pip_command, dest=dest): return None rmtree(dest) # Check it was successful return check_tool_binary_version(os.path.join(site.USER_BASE, 'bin', dep.command), dep, no_cache=True) +def python_downloader(dep): + logger.info('- Trying to install {} (from PyPi)'.format(dep.name)) + # Check if we have pip and wheel + pip_command = check_pip() + if pip_command is None: + return False + # Try to pip install it + if not pip_install(pip_command, name=dep.pypi_name.lower()): + return False + return True + + def git_downloader(dep, system, plat): # Currently only for Linux x86_64/x86_32 # arm, arm64, mips64el and mipsel are also there, just not implemented @@ -582,7 +603,49 @@ def check_tool_binary(dep): return try_download_tool_binary(dep) -def check_tool_python(dep): +def check_tool_python_version(mod, dep): + logger.debugl(2, '- Checking version for `{}`'.format(dep.name)) + global version_check_fail + version_check_fail = False + # Do we need a particular version? + needs = (0, 0, 0) + for r in dep.roles: + if r.version and r.version > needs: + needs = r.version + if needs == (0, 0, 0): + # Any version is Ok + logger.debugl(2, '- No particular version needed') + else: + logger.debugl(2, '- Needed version {}'.format(needs)) + # Check the version + if hasattr(mod, '__version__'): + version = version_str2tuple(mod.__version__) + else: + version = 'Ok' + logger.debugl(2, '- Found version {}'.format(version)) + version_check_fail = version != 'Ok' and version < needs + return None if version_check_fail else mod + + +def check_tool_python(dep, reload): + # Try to load the module + try: + mod = importlib.import_module(dep.module_name) + return check_tool_python_version(mod, dep) + except ModuleNotFoundError: + pass + # Not installed, try to download it + if not python_downloader(dep): + return None + # Check we can use it + try: + mod = importlib.import_module(dep.module_name) + res = check_tool_python_version(mod, dep) + if res is not None and reload is not None: + res = importlib.reload(reload) + return res + except ModuleNotFoundError: + pass return None @@ -618,18 +681,20 @@ def show_roles(roles, fatal): do_log_err('- {}{}'.format(o.desc, get_version(o)), fatal) -def check_tool(dep, fatal=False): +def check_tool(dep, fatal=False, reload=None): logger.debug('Starting tool check for {}'.format(dep.name)) if dep.is_python: - cmd = check_tool_python(dep) + cmd = check_tool_python(dep, reload) + type = 'python module' else: cmd = check_tool_binary(dep) + type = 'command' logger.debug('- Returning `{}`'.format(cmd)) if cmd is None: if version_check_fail: - do_log_err('Upgrade `{}` command ({})'.format(dep.command, dep.name), fatal) + do_log_err('Upgrade `{}` {} ({})'.format(dep.command, type, dep.name), fatal) else: - do_log_err('Missing `{}` command ({}), install it'.format(dep.command, dep.name), fatal) + do_log_err('Missing `{}` {} ({}), install it'.format(dep.command, type, dep.name), fatal) if dep.url: do_log_err('Home page: '+dep.url, fatal) if dep.url_down: diff --git a/kibot/kiplot.py b/kibot/kiplot.py index 849aaa80..8304452b 100644 --- a/kibot/kiplot.py +++ b/kibot/kiplot.py @@ -21,7 +21,7 @@ from collections import OrderedDict from .gs import GS from .registrable import RegOutput -from .misc import (PLOT_ERROR, CORRUPTED_PCB, EXIT_BAD_ARGS, CORRUPTED_SCH, +from .misc import (PLOT_ERROR, CORRUPTED_PCB, EXIT_BAD_ARGS, CORRUPTED_SCH, version_str2tuple, EXIT_BAD_CONFIG, WRONG_INSTALL, UI_SMD, UI_VIRTUAL, TRY_INSTALL_CHECK, MOD_SMD, MOD_THROUGH_HOLE, MOD_VIRTUAL, W_PCBNOSCH, W_NONEEDSKIP, W_WRONGCHAR, name2make, W_TIMEOUT, W_KIAUTO, W_VARSCH, NO_SCH_FILE, NO_PCB_FILE, W_VARPCB, NO_YAML_MODULE, WRONG_ARGUMENTS) @@ -737,7 +737,7 @@ def discover_files(dest_dir): def yaml_dump(f, tree): - if tuple(map(int, yaml.__version__.split('.'))) < (3, 14): + if version_str2tuple(yaml.__version__) < (3, 14): f.write(yaml.dump(tree)) else: # sort_keys was introduced after 3.13 diff --git a/kibot/misc.py b/kibot/misc.py index f86f9269..d13fe265 100644 --- a/kibot/misc.py +++ b/kibot/misc.py @@ -304,6 +304,10 @@ def hide_stderr(): os.dup2(newstderr, 2) +def version_str2tuple(ver): + return tuple(map(int, ver.split('.'))) + + class ToolDependencyRole(object): """ Class used to define the role of a tool """ def __init__(self, desc=None, version=None, output=None): diff --git a/kibot/out_bom.py b/kibot/out_bom.py index 684bf159..db5d8303 100644 --- a/kibot/out_bom.py +++ b/kibot/out_bom.py @@ -18,11 +18,10 @@ from .error import KiPlotConfigurationError from .kiplot import get_board_comps_data, load_any_sch from .bom.columnlist import ColumnList, BoMError from .bom.bom import do_bom -from .bom.xlsx_writer import KICOST_SUPPORT from .var_kibom import KiBoM from .fil_base import (BaseFilter, apply_exclude_filter, apply_fitted_filter, apply_fixed_filter, reset_filters, KICOST_NAME_TRANSLATIONS) -from .dep_downloader import pytool_downloader +from .dep_downloader import check_tool, pytool_downloader, python_downloader from .macros import macros, document, output_class # noqa: F401 from . import log # To debug the `with document` we can use: @@ -41,8 +40,9 @@ DEFAULT_ALIASES = [['r', 'r_small', 'res', 'resistor'], kicost_dep = kicost_dependency('bom', pytool_downloader, roles=ToolDependencyRole(desc='Find components costs and specs', version=(1, 1, 8))) RegDependency.register(kicost_dep) -RegDependency.register(ToolDependency('bom', 'XLSXWriter', is_python=True, - roles=ToolDependencyRole(desc='Create XLSX files'))) +xlsx_dep = ToolDependency('bom', 'XLSXWriter', is_python=True, roles=ToolDependencyRole(desc='Create XLSX files'), + downloader=python_downloader) +RegDependency.register(xlsx_dep) class BoMJoinField(Optionable): @@ -691,6 +691,10 @@ class BoMOptions(BaseOptions): def run(self, output): format = self.format.lower() + if format == 'xlsx': + if self.xlsx.kicost: + check_tool(kicost_dep, fatal=True) + check_tool(xlsx_dep, fatal=True) # Add some info needed for the output to the config object. # So all the configuration is contained in one object. self.source = GS.sch_basename @@ -856,17 +860,22 @@ class BoM(BaseOutput): # noqa: F821 if join_fields: logger.debug(' - Fields to join with Value: {}'.format(join_fields)) # Create a generic version - for fmt in ['HTML', 'CSV', 'TXT', 'TSV', 'XML', 'XLSX']: + SIMP_FMT = ['HTML', 'CSV', 'TXT', 'TSV', 'XML'] + XYRS_FMT = ['HTML'] + if check_tool(xlsx_dep) is not None: + SIMP_FMT.append('XLSX') + XYRS_FMT.append('XLSX') + for fmt in SIMP_FMT: outs.append(BoM.create_bom(fmt, 'Generic', group_fields, join_fields, fld_names)) if GS.board: # Create an example showing the positional fields cols = ColumnList.COLUMNS_DEFAULT + ColumnList.COLUMNS_EXTRA - for fmt in ['HTML', 'XLSX']: + for fmt in XYRS_FMT: gb = BoM.create_bom(fmt, 'Positional', group_fields, None, fld_names, cols) gb['options'][fmt.lower()] = {'style': 'modern-red'} outs.append(gb) # Create a costs version - if KICOST_SUPPORT: # and dists? + if check_tool(kicost_dep) is not None: # and dists? logger.debug(' - KiCost distributors {}'.format(dists)) grp = group_fields if group_fields: diff --git a/src/kibot-check b/src/kibot-check index 5539bfc2..fb70b818 100755 --- a/src/kibot-check +++ b/src/kibot-check @@ -673,7 +673,7 @@ deps = '{\ "XLSXWriter": {\ "command": "xlsxwriter",\ "deb_package": "python3-xlsxwriter",\ - "downloader": null,\ + "downloader": {},\ "extra_deb": null,\ "help_option": "--version",\ "importance": 1,\