diff --git a/.coveragerc b/.coveragerc index 02df6f45..1760fef3 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,6 +8,7 @@ omit = */kibot/docopt.py */kibot/mcpyrate/* */kibot/PyPDF2/* */kibot/svgutils/* +parallel = true [report] exclude_lines = diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index ca004166..2f24ccf8 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -49,8 +49,14 @@ jobs: python3-coverage erase # Create the caches with macros python3-coverage run -a src/kibot --help-outputs > /dev/null - pytest-3 --durations=0 --test_dir output + # Individual run for specific tests # pytest-3 --log-cli-level debug -k "test_ibom_parse_fail" --test_dir output + # Run the 90% faster tests (under 3 s) + # Do it in parallel + pytest-3 -v --durations=0 -m "not slow" -n 2 --test_dir output + # Run the slowest at the end and exit on the first error + pytest-3 -v --durations=0 -m slow -x --test_dir output + python3-coverage combine python3-coverage report python3-coverage html -d output/htmlcov - name: Store results diff --git a/Makefile b/Makefile index 4e307775..e75d807b 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ lint: doc test_tmp: lint $(PY_COV) erase $(PYTEST) + $(PY_COV) combine $(PY_COV) report test: lint @@ -46,7 +47,9 @@ test: lint rm -f example.kibot.yaml rm -f tests/.local $(PY_COV) erase - $(PYTEST) --test_dir output + $(PYTEST) -m "not slow" -n 2 --test_dir output + $(PYTEST) -m "slow" --test_dir output + $(PY_COV) combine $(PY_COV) report $(PY_COV) html x-www-browser htmlcov/index.html @@ -58,6 +61,7 @@ test1: rm -f tests/.local $(PY_COV) erase $(PYTEST) --log-cli-level debug -k "test_bom_ok" --test_dir output + $(PY_COV) combine $(PY_COV) report $(PY_COV) html #x-www-browser htmlcov/index.html @@ -77,7 +81,8 @@ test_docker_local_1: /bin/bash -c "flake8 . --count --statistics ; python3-coverage run -a src/kibot --help-outputs > /dev/null; pytest-3 --log-cli-level debug -k 'test_print_pcb_svg_simple_2' --test_dir output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ tests/.config/kiplot/plugins/__pycache__/ tests/test_plot/fake_pcbnew/__pycache__/ tests/.config/kibot/plugins/__pycache__/ .coverage htmlcov/" #$(PY_COV) report #x-www-browser htmlcov/index.html - rm .coverage + # The coverage used in the image is incompatible + $(PY_COV) erase test_docker_local_1_ki6: rm -rf output @@ -86,7 +91,7 @@ test_docker_local_1_ki6: # Run in the same directory to make the __pycache__ valid # Also change the owner of the files to the current user (we run as root like in GitHub) docker run --rm -v $(CWD):$(CWD) --workdir="$(CWD)" setsoft/kicad_auto_test:ki6 \ - /bin/bash -c "flake8 . --count --statistics ; python3-coverage run -a src/kibot --help-outputs > /dev/null; pytest-3 --log-cli-level debug -k 'test_kicad_conf_user' --test_dir output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ tests/.config/kiplot/plugins/__pycache__/ tests/test_plot/fake_pcbnew/__pycache__/ tests/.config/kibot/plugins/__pycache__/ .coverage htmlcov/" + /bin/bash -c "flake8 . --count --statistics ; python3-coverage run -a src/kibot --help-outputs > /dev/null; pytest-3 --log-cli-level debug -k 'test_dep_pytool' --test_dir output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ tests/.config/kiplot/plugins/__pycache__/ tests/test_plot/fake_pcbnew/__pycache__/ tests/.config/kibot/plugins/__pycache__/ .coverage htmlcov/" #$(PY_COV) report #x-www-browser htmlcov/index.html @@ -98,8 +103,9 @@ test_docker_local: # Also change the owner of the files to the current user (we run as root like in GitHub) docker run --rm -v $(CWD):$(CWD) --workdir="$(CWD)" setsoft/kicad_auto_test:latest \ /bin/bash -c "flake8 . --count --statistics ; python3-coverage run -a src/kibot --help-outputs ; pytest-3 --test_dir output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ .coverage htmlcov/" - $(PY_COV) report - x-www-browser htmlcov/index.html + # $(PY_COV) report + # x-www-browser htmlcov/index.html + $(PY_COV) erase test_docker_local_ki6: rm -rf output @@ -109,6 +115,7 @@ test_docker_local_ki6: # Also change the owner of the files to the current user (we run as root like in GitHub) docker run --rm -v $(CWD):$(CWD) --workdir="$(CWD)" setsoft/kicad_auto_test:ki6 \ /bin/bash -c "flake8 . --count --statistics ; python3-coverage run -a src/kibot --help-outputs ; pytest-3 --test_dir output ; $(PY_COV) html; chown -R $(USER_ID):$(GROUP_ID) output/ tests/board_samples/ .coverage htmlcov/" + $(PY_COV) combine $(PY_COV) report x-www-browser htmlcov/index.html @@ -124,7 +131,7 @@ docker_shell: single_test: rm -rf pp - -$(PY_COV) run -a src/kibot --help-list-outputs > /dev/null + -$(PY_COV) run src/kibot --help-list-outputs > /dev/null -$(PYTEST) --log-cli-level debug -k "$(SINGLE_TEST)" --test_dir pp @echo "********************" Output @cat pp/*/output.txt diff --git a/pytest.ini b/pytest.ini index 1708edf4..6423a6c4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,5 @@ [pytest] python_files = tests/test_plot/test_*.py +markers = + webtest: mark a test as a webtest. + slow: mark test as slow. diff --git a/tests/test_plot/test_dep_downloader.py b/tests/test_plot/test_dep_downloader.py index 90486027..0c777a48 100644 --- a/tests/test_plot/test_dep_downloader.py +++ b/tests/test_plot/test_dep_downloader.py @@ -9,6 +9,7 @@ import coverage import yaml import logging import importlib +import pytest import shutil import subprocess import sys @@ -120,6 +121,7 @@ def test_dep_gs(test_dir, caplog, monkeypatch): os.remove(os.path.join(ctx.output_dir, bin_dir, 'gs')) +@pytest.mark.slow def test_dep_convert(test_dir, caplog, monkeypatch): """ Check the convert_downloader """ # Create a context to get an output directory diff --git a/tests/test_plot/test_ibom.py b/tests/test_plot/test_ibom.py index eff74d9b..ce8984af 100644 --- a/tests/test_plot/test_ibom.py +++ b/tests/test_plot/test_ibom.py @@ -9,10 +9,11 @@ For debug information use: pytest-3 --log-cli-level debug """ -import os -import re import json import logging +import os +import pytest +import re from . import context from utils.lzstring import LZString from kibot.misc import (BOM_ERROR) @@ -87,6 +88,7 @@ def test_ibom_all_ops(test_dir): ctx.clean_up() +@pytest.mark.slow def test_ibom_variant_1(test_dir): prj = 'kibom-variante' ctx = context.TestContext(test_dir, prj, 'ibom_variant_1', BOM_DIR) diff --git a/tests/test_plot/test_misc.py b/tests/test_plot/test_misc.py index 9ee6021c..8d6239a5 100644 --- a/tests/test_plot/test_misc.py +++ b/tests/test_plot/test_misc.py @@ -31,6 +31,7 @@ import os import re import shutil import logging +import pytest import subprocess import json from . import context @@ -742,6 +743,7 @@ def check_makefile(ctx, mkfile, prj, dbg, txt): ctx.search_err(r'Wrong character in file name `(.*)/test_v5-top\$.svg') +@pytest.mark.slow def test_makefile_1(test_dir): prj = 'test_v5' ctx = context.TestContext(test_dir, prj, 'makefile_1') @@ -752,6 +754,7 @@ def test_makefile_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_makefile_2(test_dir): prj = 'test_v5' ctx = context.TestContext(test_dir, prj, 'makefile_1') @@ -898,6 +901,7 @@ def test_cli_order(test_dir): ctx.clean_up() +@pytest.mark.slow def test_qr_lib_1(test_dir): prj = 'qr_test/qr_test' ctx = context.TestContext(test_dir, prj, 'qr_lib_1', POS_DIR) @@ -959,6 +963,7 @@ def test_report_simple_1(test_dir): ctx.clean_up(keep_project=True) +@pytest.mark.slow def test_report_simple_2(test_dir): prj = 'light_control' ctx = context.TestContext(test_dir, prj, 'report_simple_2', POS_DIR) @@ -1117,6 +1122,7 @@ def test_annotate_pcb_tbrl_small_grid(test_dir): {'C1': '1u', 'C2': '2u', 'R1': '2', 'R2': '1', 'C3': '3u', 'C4': '4u', 'R3': '3', 'R4': '4'}) +@pytest.mark.slow def test_gencad_1(test_dir): prj = 'gencad' ctx = context.TestContext(test_dir, prj, 'gencad_1') @@ -1133,6 +1139,7 @@ def test_gencad_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_quick_start_1(test_dir): """ Very naive test to see if it doesn't crash """ prj = 'light_control' @@ -1172,6 +1179,7 @@ def test_quick_start_1(test_dir): ctx.expect_out_file(os.path.join('Browse', 'light_control-navigate.html')) +@pytest.mark.slow def test_netlist_classic_1(test_dir): prj = 'light_control' dir_o = 'Export' @@ -1180,6 +1188,7 @@ def test_netlist_classic_1(test_dir): ctx.expect_out_file(os.path.join(dir_o, prj+'-netlist.net')) +@pytest.mark.slow def test_netlist_ipc_1(test_dir): prj = 'light_control' dir_o = 'Export' diff --git a/tests/test_plot/test_preflight.py b/tests/test_plot/test_preflight.py index 0f5c15ed..caae208a 100644 --- a/tests/test_plot/test_preflight.py +++ b/tests/test_plot/test_preflight.py @@ -13,15 +13,17 @@ For debug information use: pytest-3 --log-cli-level debug """ -import os -import logging -import re import json +import logging +import pytest +import os +import re from subprocess import run, PIPE from . import context from kibot.misc import DRC_ERROR, ERC_ERROR, BOM_ERROR, CORRUPTED_PCB, CORRUPTED_SCH, EXIT_BAD_CONFIG +@pytest.mark.slow def test_erc_1(test_dir): prj = 'bom' ctx = context.TestContext(test_dir, prj, 'erc', '') @@ -31,6 +33,7 @@ def test_erc_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_erc_fail_1(test_dir): """ Using an SCH with ERC errors """ prj = 'fail-erc' @@ -49,6 +52,7 @@ def test_erc_fail_2(test_dir): ctx.clean_up() +@pytest.mark.slow def test_erc_warning_1(test_dir): """ Using an SCH with ERC warnings """ prj = 'warning-project' @@ -60,6 +64,7 @@ def test_erc_warning_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_erc_warning_2(test_dir): """ Using an SCH with ERC warnings as errors """ prj = 'warning-project' @@ -130,6 +135,7 @@ def test_drc_time_out(test_dir): ctx.clean_up() +@pytest.mark.slow def test_update_xml_1(test_dir): prj = 'bom' ctx = context.TestContext(test_dir, prj, 'update_xml', '') @@ -149,6 +155,7 @@ def test_update_xml_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_update_xml_fail(test_dir): """ Using a dummy SCH """ prj = '3Rs' diff --git a/tests/test_plot/test_print_pcb.py b/tests/test_plot/test_print_pcb.py index 2abb5c05..6ee54cad 100644 --- a/tests/test_plot/test_print_pcb.py +++ b/tests/test_plot/test_print_pcb.py @@ -9,6 +9,7 @@ pytest-3 --log-cli-level debug """ import logging +import pytest from . import context PDF_DIR = 'Layers' PDF_FILE = 'bom-F_Cu+F_SilkS.pdf' @@ -16,6 +17,7 @@ PDF_FILE_B = 'PCB_Bot.pdf' PDF_FILE_C = 'PCB_Bot_def.pdf' +@pytest.mark.slow def test_print_pcb_simple(test_dir): prj = 'bom' ctx = context.TestContext(test_dir, prj, 'print_pcb', PDF_DIR) @@ -25,6 +27,7 @@ def test_print_pcb_simple(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_pcb_svg_simple_1(test_dir): prj = 'bom' ctx = context.TestContext(test_dir, prj, 'print_pcb_svg') @@ -36,6 +39,7 @@ def test_print_pcb_svg_simple_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_pcb_svg_simple_2(test_dir): """ Check the portrait version is OK """ prj = 'bom_portrait' @@ -48,6 +52,7 @@ def test_print_pcb_svg_simple_2(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_pcb_refill_1(test_dir): prj = 'zone-refill' ctx = context.TestContext(test_dir, prj, 'print_pcb_zone-refill') @@ -57,6 +62,7 @@ def test_print_pcb_refill_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_pcb_refill_2(test_dir): """ Using KiCad 6 colors """ if context.ki5(): @@ -69,6 +75,7 @@ def test_print_pcb_refill_2(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_variant_1(test_dir): prj = 'kibom-variant_3_txt' ctx = context.TestContext(test_dir, prj, 'print_pcb_variant_1') @@ -81,6 +88,7 @@ def test_print_variant_1(test_dir): ctx.clean_up(keep_project=True) +@pytest.mark.slow def test_print_pcb_options(test_dir): prj = 'bom' ctx = context.TestContext(test_dir, prj, 'print_pcb_options', PDF_DIR) @@ -91,6 +99,7 @@ def test_print_pcb_options(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_wrong_paste(test_dir): prj = 'wrong_paste' ctx = context.TestContext(test_dir, prj, 'wrong_paste', PDF_DIR) @@ -101,6 +110,7 @@ def test_print_wrong_paste(test_dir): ctx.clean_up() +@pytest.mark.slow def test_pcb_print_simple_1(test_dir): prj = 'light_control' ctx = context.TestContext(test_dir, prj, 'pcb_print_2') diff --git a/tests/test_plot/test_print_sch.py b/tests/test_plot/test_print_sch.py index 371502fa..c8a98476 100644 --- a/tests/test_plot/test_print_sch.py +++ b/tests/test_plot/test_print_sch.py @@ -12,6 +12,7 @@ pytest-3 --log-cli-level debug import os import logging import coverage +import pytest from . import context from kibot.misc import (PDF_SCH_PRINT, SVG_SCH_PRINT) from kibot.kicad.v5_sch import Schematic, SchFileError, DrawPoligon, Pin @@ -25,6 +26,7 @@ NI_DIR = 'no_inductor' cov = coverage.Coverage() +@pytest.mark.slow def test_print_sch_ok(test_dir): prj = 'bom_no_xml' # bom has meta data, here we test no meta-data ctx = context.TestContext(test_dir, prj, 'print_sch') @@ -34,6 +36,7 @@ def test_print_sch_ok(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_sch_fail(test_dir): prj = 'print_err' ctx = context.TestContextSCH(test_dir, prj, 'print_sch') @@ -42,6 +45,7 @@ def test_print_sch_fail(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_sch_svg_ok(test_dir): prj = 'bom_no_xml' # bom has meta data, here we test no meta-data ctx = context.TestContext(test_dir, prj, 'print_sch_svg') @@ -51,6 +55,7 @@ def test_print_sch_svg_ok(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_sch_svg_fail(test_dir): prj = 'print_err' ctx = context.TestContext(test_dir, prj, 'print_sch_svg') @@ -97,6 +102,7 @@ def test_sch_variant_ni_2(test_dir): check_l1(ctx) +@pytest.mark.slow def test_print_sch_variant_ni_1(test_dir): """ Using a variant """ prj = 'test_v5_wks' # Is the most complete, contains every KiCad object I know @@ -109,6 +115,7 @@ def test_print_sch_variant_ni_1(test_dir): ctx.clean_up(keep_project=True) +@pytest.mark.slow def test_print_sch_svg_variant_ni_1(test_dir): """ SVG using a variant """ prj = 'test_v5' # Is the most complete, contains every KiCad object I know @@ -121,6 +128,7 @@ def test_print_sch_svg_variant_ni_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_print_sch_variant_ni_2(test_dir): """ Using a filter """ prj = 'test_v5' # Is the most complete, contains every KiCad object I know diff --git a/tests/test_plot/test_step.py b/tests/test_plot/test_step.py index 92a1aa24..537a22d8 100644 --- a/tests/test_plot/test_step.py +++ b/tests/test_plot/test_step.py @@ -9,6 +9,7 @@ pytest-3 --log-cli-level debug """ import os +import pytest from glob import glob from . import context @@ -17,6 +18,7 @@ STEP_DIR = '3D' # STEP_FILE = 'bom.step' +@pytest.mark.slow def test_step_1(test_dir): prj = 'bom' ctx = context.TestContext(test_dir, prj, 'step_simple', STEP_DIR) @@ -31,6 +33,7 @@ def test_step_1(test_dir): ctx.clean_up() +@pytest.mark.slow def test_step_2(test_dir): prj = 'bom_fake_models' yaml = 'step_simple_2' @@ -44,6 +47,7 @@ def test_step_2(test_dir): ctx.clean_up(keep_project=True) +@pytest.mark.slow def test_step_3(test_dir): prj = 'bom' ctx = context.TestContext(test_dir, prj, 'step_simple_3', STEP_DIR) @@ -53,6 +57,7 @@ def test_step_3(test_dir): ctx.clean_up() +@pytest.mark.slow def test_step_gl_env(test_dir): prj = 'bom' ctx = context.TestContext(test_dir, prj, 'step_gl_env', STEP_DIR) @@ -67,6 +72,7 @@ def test_step_gl_env(test_dir): ctx.clean_up() +@pytest.mark.slow def test_step_variant_1(test_dir): prj = 'kibom-variant_3' ctx = context.TestContext(test_dir, prj, 'step_variant_1') @@ -78,6 +84,7 @@ def test_step_variant_1(test_dir): ctx.clean_up(keep_project=True) +@pytest.mark.slow def test_render_3d_variant_1(test_dir): # Text variables to ensure they are rendered. # Traces diff --git a/tests/utils/context.py b/tests/utils/context.py index d76cc3a6..cd60a117 100644 --- a/tests/utils/context.py +++ b/tests/utils/context.py @@ -276,7 +276,7 @@ class TestContext(object): f.write('Dummy file\n') def do_run(self, cmd, ret_val=None, use_a_tty=False, chdir_out=False): - cmd_base = [COVERAGE_SCRIPT, 'run', '-a'] + cmd_base = [COVERAGE_SCRIPT, 'run'] if chdir_out: cwd = os.getcwd() cmd_base.append('--rcfile='+os.path.join(cwd, '.coveragerc'))