From 6fa164c75f1433b0cb227ae0293cde34a5d5a142 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Thu, 28 Sep 2023 23:32:32 +0100 Subject: [PATCH] Test new config options Adds tests for the new config options across ini, CLI and kwarg where relevant. Asserts that they work and have the expected order of precedence. Still need to add dedicated tests for all the other options. --- setup.cfg | 1 + tests/test_baseline_path.py | 48 ++++++++++++++++++++++++ tests/test_default_backend.py | 35 ++++++++++++++++++ tests/test_default_style.py | 37 +++++++++++++++++++ tests/test_default_tolerance.py | 58 +++++++++++++++++++++++++++++ tests/test_generate_summary.py | 63 ++++++++++++++++++++++++++++++++ tests/test_hash_library.py | 52 ++++++++++++++++++++++++++ tests/test_use_full_test_name.py | 52 ++++++++++++++++++++++++++ 8 files changed, 346 insertions(+) create mode 100644 tests/test_baseline_path.py create mode 100644 tests/test_default_backend.py create mode 100644 tests/test_default_style.py create mode 100644 tests/test_default_tolerance.py create mode 100644 tests/test_generate_summary.py create mode 100644 tests/test_hash_library.py create mode 100644 tests/test_use_full_test_name.py diff --git a/setup.cfg b/setup.cfg index 7f9a3e3..f8026f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ filterwarnings = error ignore:distutils Version classes are deprecated ignore:the imp module is deprecated in favour of importlib + ignore:The NumPy module was reloaded [flake8] max-line-length = 100 diff --git a/tests/test_baseline_path.py b/tests/test_baseline_path.py new file mode 100644 index 0000000..1517ac1 --- /dev/null +++ b/tests/test_baseline_path.py @@ -0,0 +1,48 @@ +import shutil +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected_baseline_path, success_expected", + [ + ("dir1", None, None, "dir1", True), + ("dir1", "dir2", None, "dir2", True), + ("dir1", "dir2", "dir3", "dir3", True), + ("dir1", "dir2", "dir3", "dir2", False), + (None, None, "dir3", "dir3", True), + ], +) +def test_config(pytester, ini, cli, kwarg, expected_baseline_path, success_expected): + (pytester.path / expected_baseline_path).mkdir() + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + pytester.path / expected_baseline_path / "test_mpl.png", + ) + ini = f"mpl-baseline-path = {pytester.path / ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + {ini} + """ + ) + kwarg = f"baseline_dir='{pytester.path / kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-baseline-path={pytester.path / cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_default_backend.py b/tests/test_default_backend.py new file mode 100644 index 0000000..b84624a --- /dev/null +++ b/tests/test_default_backend.py @@ -0,0 +1,35 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("backend1", None, None, "backend1"), + ("backend1", "backend2", None, "backend2"), + ("backend1", "backend2", "backend3", "backend3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = f"mpl-default-backend = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + kwarg = f"backend='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-backend={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines([f"*ModuleNotFoundError: No module named 'matplotlib.backends.backend_{expected}'*"]) diff --git a/tests/test_default_style.py b/tests/test_default_style.py new file mode 100644 index 0000000..1b0c1e6 --- /dev/null +++ b/tests/test_default_style.py @@ -0,0 +1,37 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("sty1", None, None, "sty1"), + ("sty1", "sty2", None, "sty2"), + ("sty1", "sty2", "sty3", "sty3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = "mpl-default-style = " + ini if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-baseline-path = {pytester.path} + {ini} + """ + ) + kwarg = f"style='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + + cli = "--mpl-default-style=" + cli if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines([f"*OSError: '{expected}' is not a valid package style*"]) diff --git a/tests/test_default_tolerance.py b/tests/test_default_tolerance.py new file mode 100644 index 0000000..c12832c --- /dev/null +++ b/tests/test_default_tolerance.py @@ -0,0 +1,58 @@ +from pathlib import Path + +import pytest +from PIL import Image, ImageDraw + +TEST_NAME = "test_base_style" + + +@pytest.fixture(scope="module") +def baseline_image(tmpdir_factory): + path = Path(__file__).parent / "baseline" / "2.0.x" / f"{TEST_NAME}.png" + image = Image.open(path) + draw = ImageDraw.Draw(image) + draw.rectangle(((0, 0), (100, 100)), fill="red") + output = Path(tmpdir_factory.mktemp("data").join(f"{TEST_NAME}.png")) + print(output) + image.save(output) + return output + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + (40, None, None, True), + (30, 40, None, True), + (30, 30, 40, True), + (30, 40, 30, False), + (40, 30, 30, False), + ], +) +def test_config(pytester, baseline_image, ini, cli, kwarg, success_expected): + ini = f"mpl-default-tolerance = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {baseline_image.parent} + {ini} + """ + ) + kwarg = f"tolerance={kwarg}" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def {TEST_NAME}(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-tolerance={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_generate_summary.py b/tests/test_generate_summary.py new file mode 100644 index 0000000..3632207 --- /dev/null +++ b/tests/test_generate_summary.py @@ -0,0 +1,63 @@ +import json + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, expected", + [ + ("json", None, {"json"}), + ("json", "html", {"html"}), + ("basic-html", "json", {"json"}), + (None, "json,basic-html,html", {"json", "basic-html", "html"}), + ], +) +def test_config(pytester, ini, cli, expected): + ini = f"mpl-generate-summary = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-results-path = {pytester.path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-generate-summary={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + + json_summary = pytester.path / "results.json" + if "json" in expected: + with open(json_summary) as fp: + results = json.load(fp) + assert "test_config.test_mpl" in results + else: + assert not json_summary.exists() + + html_summary = pytester.path / "fig_comparison.html" + if "html" in expected: + with open(html_summary) as fp: + raw = fp.read() + assert "bootstrap" in raw + assert "test_config.test_mpl" in raw + else: + assert not html_summary.exists() + + basic_html_summary = pytester.path / "fig_comparison_basic.html" + if "basic-html" in expected: + with open(basic_html_summary) as fp: + raw = fp.read() + assert "bootstrap" not in raw + assert "test_config.test_mpl" in raw + else: + assert not basic_html_summary.exists() diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py new file mode 100644 index 0000000..29a01c6 --- /dev/null +++ b/tests/test_hash_library.py @@ -0,0 +1,52 @@ +import json + +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + ("bad", None, None, False), + ("good", None, None, True), + ("bad", "good", None, True), + ("bad", "bad", "good", False), # Note: CLI overrides kwarg + ("bad", "good", "bad", True), + ], +) +def test_config(pytester, ini, cli, kwarg, success_expected): + hash_libraries = { + "good": (pytester.path / "good_hash_library.json", + "b1e03274b2df4130e0894afd6c0faa76805e1851eec32f38e86d2423de0d9186"), + "bad": (pytester.path / "bad_hash_library.json", "bad-value"), + } + for library_path, hash_val in hash_libraries.values(): + with open(library_path, "w") as fp: + json.dump({f"test_config.test_mpl": hash_val}, fp) + + ini = f"mpl-hash-library = {hash_libraries[ini][0]}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + + kwarg = f"hash_library='{hash_libraries[kwarg][0]}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 3, 2]) + return fig + """ + ) + + cli = f"--mpl-hash-library={hash_libraries[cli][0]}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_use_full_test_name.py b/tests/test_use_full_test_name.py new file mode 100644 index 0000000..f78bd48 --- /dev/null +++ b/tests/test_use_full_test_name.py @@ -0,0 +1,52 @@ +import shutil +from pathlib import Path + +import pytest + +FULL_TEST_NAME = "test_config.TestClass.test_mpl" +SHORT_TEST_NAME = "test_mpl" + + +@pytest.mark.parametrize( + "ini, cli, expected_baseline_name, success_expected", + [ + (None, None, SHORT_TEST_NAME, True), + (False, None, SHORT_TEST_NAME, True), + (True, None, FULL_TEST_NAME, True), + (False, True, FULL_TEST_NAME, True), + (None, True, FULL_TEST_NAME, True), + (True, True, "bad_name", False), + ], +) +def test_config(pytester, ini, cli, expected_baseline_name, success_expected): + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + pytester.path / f"{expected_baseline_name}.png", + ) + ini = f"mpl-use-full-test-name = {ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {pytester.path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + class TestClass: + @pytest.mark.mpl_image_compare + def test_mpl(self): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = "--mpl-use-full-test-name" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1)