KiBot/tests/test_plot/test_dep_downloader.py

"""
Tests for Dependencies Downloader

For debug information use:
pytest-3 --log-cli-level debug
"""
import os
import coverage
import yaml
import logging
import importlib
import pytest
import shutil
import subprocess
import sys
from . import context
from kibot.mcpyrate import activate  # noqa: F401
import kibot.dep_downloader as downloader
import kibot.out_compress as compress
import kibot.out_kibom as kibom
from kibot.misc import MISSING_TOOL
import kibot.log as log

cov = coverage.Coverage()
bin_dir = os.path.join('.local', 'share', 'kibot', 'bin')
bin_dir_py = os.path.join('.local', 'bin')
DEP_PYTHON_MODULE_FOOBAR = """
  - name: FooBar
    python_module: true
    role: mandatory
"""


def try_dependency(ctx, caplog, monkeypatch, docstring, name_dep, downloader_name, b_dir, use_wrapper=False):
    with monkeypatch.context() as m:
        # Force the downloader to use the output dir instead of HOME
        home = os.path.abspath(ctx.output_dir)
        m.setenv("HOME", home)
        m.setattr("site.USER_BASE", os.path.join(home, '.local'))
        # Refresh the module with actual dependencies
        mod = importlib.reload(downloader)
        mod.register_deps('test', yaml.safe_load(docstring))
        # Get the dependency
        dep = mod.used_deps['test:'+name_dep]
        # Download it
        cov.load()
        cov.start()
        downloader_func = getattr(mod, downloader_name+'_downloader')
        if use_wrapper:
            dep.downloader = downloader_func
            res, version = mod.try_download_tool_binary(dep)
            if res:
                res, version = mod.check_tool_binary_local(dep)
        else:
            res, version = downloader_func(dep, 'Linux', 'x86_64')
        cov.stop()
        cov.save()
        # We should get the following name:
        logging.debug('Result: {} Version: {}'.format(res, version))
        with open(ctx.get_out_path('caplog.txt'), 'wt') as f:
            f.write(caplog.text)
        full_name = os.path.join(home, b_dir, dep.command)
        assert res == full_name or res == full_name.replace('/bin', '/local/bin')
        # We executed the file


def try_dependency_module(ctx, caplog, monkeypatch, docstring, name_dep, downloader_name):
    # Note: every attempt to install in a chosen dir failed, even when the module was there and in the sys.path the
    # importlib call miserably failed.
    caplog.set_level(logging.DEBUG)
    with monkeypatch.context():
        # Refresh the module with actual dependencies
        mod = importlib.reload(downloader)
        mod.register_deps('test', yaml.safe_load(docstring))
        # Get the dependency
        dep = mod.used_deps['test:'+name_dep]
        logging.info(f"downloader_name: {downloader_name}")
        # Download it
        cov.load()
        cov.start()
        # Python module
        downloader_func = getattr(mod, downloader_name)
        res, version = downloader_func(dep)
        cov.stop()
        cov.save()
        # We should get the following name:
        logging.debug('Result: {} Version: {}'.format(res, version))
        assert res is not None
        assert res.__file__ is not None


@pytest.mark.indep
def test_dep_rar(test_dir, caplog, monkeypatch):
    """ Check the rar_downloader """
    # Create a context to get an output directory
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    try_dependency(ctx, caplog, monkeypatch, compress.__doc__, 'rar', 'rar', bin_dir, use_wrapper=True)


@pytest.mark.slow
@pytest.mark.indep
def test_dep_pytool(test_dir, caplog, monkeypatch):
    """ Check the pytool_downloader """
    # Create a context to get an output directory
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    log.debug_level = 10
    try_dependency(ctx, caplog, monkeypatch, kibom.__doc__, 'kibom', 'pytool', bin_dir_py)


@pytest.mark.slow
@pytest.mark.indep
def test_dep_rsvg(test_dir, caplog, monkeypatch):
    """ Check the rsvg_downloader """
    # Create a context to get an output directory
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    log.debug_level = 10
    dep = '  - from: RSVG\n    role: mandatory\n'
    try_dependency(ctx, caplog, monkeypatch, downloader.__doc__+dep, 'rsvg', 'rsvg', bin_dir)


@pytest.mark.indep
def test_dep_git(test_dir, caplog, monkeypatch):
    """ Check the git_downloader """
    # Create a context to get an output directory
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    log.debug_level = 10
    dep = '  - from: Git\n    role: mandatory\n'
    try_dependency(ctx, caplog, monkeypatch, downloader.__doc__+dep, 'git', 'git', bin_dir)


@pytest.mark.slow
@pytest.mark.indep
def test_dep_gs(test_dir, caplog, monkeypatch):
    """ Check the git_downloader """
    # Create a context to get an output directory
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    log.debug_level = 10
    dep = '  - from: Ghostscript\n    role: mandatory\n'
    try_dependency(ctx, caplog, monkeypatch, downloader.__doc__+dep, 'ghostscript', 'gs', bin_dir)
    os.remove(os.path.join(ctx.output_dir, bin_dir, 'gs'))


# @pytest.mark.xfail(True, reason="URL down", run=True, raises=AssertionError)
# https://imagemagick.org/archive/binaries/magick
@pytest.mark.slow
@pytest.mark.indep
def test_dep_convert(test_dir, caplog, monkeypatch):
    """ Check the convert_downloader """
    # Create a context to get an output directory
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    log.debug_level = 10
    dep = '  - from: ImageMagick\n    role: mandatory\n'
    try_dependency(ctx, caplog, monkeypatch, downloader.__doc__+dep, 'imagemagick', 'convert', bin_dir)


@pytest.mark.indep
def test_dep_python(test_dir, caplog, monkeypatch):
    """ Check the python_downloader """
    # Create a context to get an output directory
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    log.debug_level = 10
    # Ensure we don't have engineering-notation
    try:
        import engineering_notation
        logging.debug('Test module is already installed, using pip to uninstall ...')
        cmd = ['pip', 'uninstall', '-y', 'engineering-notation']
        res = subprocess.run(cmd, capture_output=True)
        if res.returncode == 1 and b'externally-managed-environment' in res.stderr:
            cmd.insert(-1, '--break-system-packages')
            res = subprocess.run(cmd, capture_output=True)
        # Why pip does this???!!!
        dir_name = os.path.dirname(engineering_notation.__file__)
        if os.path.isdir(dir_name):
            logging.debug('Silly pip left things that will allow importing a non-existent module, removing it')
            shutil.rmtree(dir_name)
        logging.debug('Removing engineering_notation from memory')
        del sys.modules["engineering_notation"]
    except Exception as e:
        logging.error(e)
    dep = 'Dependencies:\n  - name: engineering_notation\n    role: mandatory\n    python_module: true\n'
    try_dependency_module(ctx, caplog, monkeypatch, dep, 'engineering_notation', 'check_tool_python')


def try_function(ctx, caplog, monkeypatch, fun_to_test, dep='', dep2=None, disable_download=True):
    log.debug_level = 10
    # Refresh the module with actual dependencies
    mod = importlib.reload(downloader)
    mod.register_deps('test', yaml.safe_load(downloader.__doc__+dep))
    if dep2 is not None:
        mod.register_deps('test2', yaml.safe_load(dep2))
    mod.disable_auto_download = disable_download
    cov.load()
    cov.start()
    res = fun_to_test(mod)
    cov.stop()
    cov.save()
    with open(ctx.get_out_path('caplog.txt'), 'wt') as f:
        f.write(caplog.text)
    return res


def do_test_check_tool_dep_get_ver_fatal(mod):
    with pytest.raises(SystemExit) as pytest_wrapped_e:
        mod.check_tool_dep_get_ver('test', 'foobar', fatal=True)
    return pytest_wrapped_e


@pytest.mark.indep
def test_check_tool_dep_get_ver_1(test_dir, caplog, monkeypatch):
    """ Check for missing stuff in check_tool_dep_get_ver
        Also checks show_roles, get_version and do_log_error """
    # Create a context to get an output directory
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    dep = """
  - name: FooBar
    version: 1.3.0.4
    extra_deb: ['foobar-extra-debian', 'deb2']
    arch: foobar-arch (AUR)
    extra_arch: ['foobar-extra-arch', 'aur2']
    command: foobar
    role: Do this and this
"""
    pytest_wrapped_e = try_function(ctx, caplog, monkeypatch, do_test_check_tool_dep_get_ver_fatal, dep=dep)
    # Check the messages
    assert "Missing `foobar` command (FooBar), install it" in caplog.text
    assert "AUR package: foobar-arch (AUR)" in caplog.text
    assert "Recommended extra Arch packages: foobar-extra-arch aur2" in caplog.text
    assert "Recommended extra Debian packages: foobar-extra-debian deb2" in caplog.text
    assert "Used to do this and this (v1.3.0.4)" in caplog.text
    assert "This is not an official package" in caplog.text
    assert pytest_wrapped_e.type == SystemExit
    assert pytest_wrapped_e.value.code == MISSING_TOOL


def do_check_tool_python(mod):
    mod.python_downloader = lambda x: True
    return mod.check_tool_python(mod.used_deps['test:foobar'])


@pytest.mark.indep
def test_check_tool_python_1(test_dir, caplog, monkeypatch):
    """ Download disabled case """
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    try_function(ctx, caplog, monkeypatch, do_check_tool_python, dep=DEP_PYTHON_MODULE_FOOBAR)


@pytest.mark.indep
def test_check_tool_python_2(test_dir, caplog, monkeypatch):
    """ Download enabled, but fails """
    ctx = context.TestContext(test_dir, 'bom', 'bom')
    try_function(ctx, caplog, monkeypatch, do_check_tool_python, dep=DEP_PYTHON_MODULE_FOOBAR, disable_download=False)