Merge pull request #108 from Cadair/always_compare

Implement always compare mode for hash library

README.rst

@@ -84,19 +84,19 @@ The hash library can be generated with
 ``--mpl-generate-hash-library=path_to_file.json``. The hash library to be used
 can either be specified via the ``--mpl-hash-library=`` command line argument,
 or via the ``hash_library=`` keyword argument to the
-``@pytest.mark.mpl_image_comapre`` decorator.
+``@pytest.mark.mpl_image_compare`` decorator.
 
 
 Hybrid Mode: Hashes and Images
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 It is possible to configure both hashes and baseline images. In this scenario
-the hashes will be compared first, and the baseline images used if the hash
+the hashes will be compared first, with the baseline images only used if the hash
 comparison fails.
 
 This is especially useful if the baseline images are external to the repository
-with the tests in, and can be accessed remotely. In this situation if the hashes
-match the baseline images wont be retrieved, saving time and bandwidth. Also it
+containing the tests, and are accessed via HTTP. In this situation, if the hashes
+match, the baseline images won't be retrieved, saving time and bandwidth. Also, it
 allows the tests to be modified and the hashes updated to reflect the changes
 without having to modify the external images.
 
@@ -111,16 +111,16 @@ against, the tests can be run with::
 
 and the tests will pass if the images are the same. If you omit the
 ``--mpl`` option, the tests will run but will only check that the code
-runs without checking the output images.
+runs, without checking the output images.
 
 
 Generating a Failure Summary
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-By specifying the ``--mpl-generate-summary=html`` CLI argument a HTML summary
+By specifying the ``--mpl-generate-summary=html`` CLI argument, a HTML summary
 page will be generated showing the baseline, diff and result image for each
 failing test. If no baseline images are configured, just the result images will
-be displayed.
+be displayed. (See also, the **Results always** section below.)
 
 Options
 -------
@@ -182,6 +182,24 @@ are run. In addition, if both this option and the ``baseline_dir``
 option in the ``mpl_image_compare`` decorator are used, the one in the
 decorator takes precedence.
 
+Results always
+^^^^^^^^^^^^^^
+
+By default, result images are only generated for tests that fail.
+Passing ``--mpl-results-always`` to pytest will force result images
+to be generated for all tests, even for tests that pass.
+If a baseline image exists, a diff image will also be generated.
+All of these images will be shown in the summary (if requested).
+
+This option is useful for always *comparing* the result images again
+the baseline images, while only *assessing* the tests against the
+hash library.
+If you only update your baseline images after merging a PR, this
+option means that the generated summary will always show how the
+PR affects the baseline images, with the success status of each
+test (based on the hash library) also shown in the generated
+summary.
+
 Base style
 ^^^^^^^^^^
 

pytest_mpl/plugin.py

@@ -155,6 +155,12 @@ def pytest_addoption(parser):
     results_path_help = "directory for test results, relative to location where py.test is run"
     group.addoption('--mpl-results-path', help=results_path_help, action='store')
     parser.addini('mpl-results-path', help=results_path_help)
+
+    results_always_help = "Always generate result images, not just for failed tests."
+    group.addoption('--mpl-results-always', action='store_true',
+                    help=results_always_help)
+    parser.addini('mpl-results-always', help=results_always_help)
+
     parser.addini('mpl-use-full-test-name', help="use fully qualified test name as the filename.",
                   type='bool')
 
@@ -175,6 +181,8 @@ def pytest_configure(config):
         results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path")
         hash_library = config.getoption("--mpl-hash-library")
         generate_summary = config.getoption("--mpl-generate-summary")
+        results_always = (config.getoption("--mpl-results-always") or
+                          config.getini("mpl-results-always"))
 
         if config.getoption("--mpl-baseline-relative"):
             baseline_relative_dir = config.getoption("--mpl-baseline-path")
@@ -205,7 +213,8 @@ def pytest_configure(config):
                                                       results_dir=results_dir,
                                                       hash_library=hash_library,
                                                       generate_hash_library=generate_hash_lib,
-                                                      generate_summary=generate_summary))
+                                                      generate_summary=generate_summary,
+                                                      results_always=results_always))
 
     else:
 
@@ -262,7 +271,8 @@ def __init__(self,
                  results_dir=None,
                  hash_library=None,
                  generate_hash_library=None,
-                 generate_summary=None
+                 generate_summary=None,
+                 results_always=False
                  ):
         self.config = config
         self.baseline_dir = baseline_dir
@@ -274,6 +284,7 @@ def __init__(self,
         if generate_summary and generate_summary.lower() not in ("html",):
             raise ValueError(f"The mpl summary type '{generate_summary}' is not supported.")
         self.generate_summary = generate_summary
+        self.results_always = results_always
 
         # Generate the containing dir for all test results
         if not self.results_dir:
@@ -282,6 +293,7 @@ def __init__(self,
 
         # We need global state to store all the hashes generated over the run
         self._generated_hash_library = {}
+        self._test_results = {}
 
     def get_compare(self, item):
         """
@@ -389,7 +401,6 @@ def generate_baseline_image(self, item, fig):
                     **savefig_kwargs)
 
         close_mpl_figure(fig)
-        pytest.skip("Skipping test, since generating image")
 
     def generate_image_hash(self, item, fig):
         """
@@ -455,6 +466,10 @@ def load_hash_library(self, library_path):
             return json.load(fp)
 
     def compare_image_to_hash_library(self, item, fig, result_dir):
+        new_test = False
+        hash_comparison_pass = False
+        baseline_image_path = None
+
         compare = self.get_compare(item)
         savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})
 
@@ -470,41 +485,59 @@ def compare_image_to_hash_library(self, item, fig, result_dir):
         test_hash = self.generate_image_hash(item, fig)
 
         if hash_name not in hash_library:
-            return (f"Hash for test '{hash_name}' not found in {hash_library_filename}. "
-                    f"Generated hash is {test_hash}.")
+            new_test = True
+            error_message = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. "
+                             f"Generated hash is {test_hash}.")
 
-        if test_hash == hash_library[hash_name]:
-            return
+        # Save the figure for later summary (will be removed later if not needed)
+        test_image = (result_dir / "result.png").absolute()
+        fig.savefig(str(test_image), **savefig_kwargs)
 
-        error_message = (f"Hash {test_hash} doesn't match hash "
-                         f"{hash_library[hash_name]} in library "
-                         f"{hash_library_filename} for test {hash_name}.")
+        if not new_test:
+            if test_hash == hash_library[hash_name]:
+                hash_comparison_pass = True
+            else:
+                error_message = (f"Hash {test_hash} doesn't match hash "
+                                 f"{hash_library[hash_name]} in library "
+                                 f"{hash_library_filename} for test {hash_name}.")
 
         # If the compare has only been specified with hash and not baseline
         # dir, don't attempt to find a baseline image at the default path.
-        if not self.baseline_directory_specified(item):
-            # Save the figure for later summary
-            test_image = (result_dir / "result.png").absolute()
-            fig.savefig(str(test_image), **savefig_kwargs)
+        if not hash_comparison_pass and not self.baseline_directory_specified(item) or new_test:
             return error_message
 
-        try:
-            baseline_image_path = self.obtain_baseline_image(item, result_dir)
-            baseline_image = baseline_image_path
-            baseline_image = None if not baseline_image.exists() else baseline_image
-        except Exception:
-            baseline_image = None
+        # If this is not a new test try and get the baseline image.
+        if not new_test:
+            baseline_error = None
+            # Ignore Errors here as it's possible the reference image dosen't exist yet.
+            try:
+                baseline_image_path = self.obtain_baseline_image(item, result_dir)
+                baseline_image = baseline_image_path
+                if baseline_image and not baseline_image.exists():
+                    baseline_image = None
+                # Get the baseline and generate a diff image, always so that
+                # --mpl-results-always can be respected.
+                baseline_comparison = self.compare_image_to_baseline(item, fig, result_dir)
+            except Exception as e:
+                baseline_image = None
+                baseline_error = e
+
+        # If the hash comparison passes then return
+        if hash_comparison_pass:
+            return
 
         if baseline_image is None:
             error_message += f"\nUnable to find baseline image for {item}."
+            if baseline_error:
+                error_message += f"\n{baseline_error}"
             return error_message
 
         # Override the tolerance (if not explicitly set) to 0 as the hashes are not forgiving
         tolerance = compare.kwargs.get('tolerance', None)
         if not tolerance:
             compare.kwargs['tolerance'] = 0
 
-        comparison_error = (self.compare_image_to_baseline(item, fig, result_dir) or
+        comparison_error = (baseline_comparison or
                             "\nHowever, the comparison to the baseline image succeeded.")
 
         return f"{error_message}\n{comparison_error}"
@@ -548,14 +581,17 @@ def item_function_wrapper(*args, **kwargs):
                 if remove_text:
                     remove_ticks_and_titles(fig)
 
+                test_name = self.generate_test_name(item)
+
                 # What we do now depends on whether we are generating the
                 # reference images or simply running the test.
                 if self.generate_dir is not None:
                     self.generate_baseline_image(item, fig)
+                    if self.generate_hash_library is None:
+                        pytest.skip("Skipping test, since generating image.")
 
                 if self.generate_hash_library is not None:
-                    hash_name = self.generate_test_name(item)
-                    self._generated_hash_library[hash_name] = self.generate_image_hash(item, fig)
+                    self._generated_hash_library[test_name] = self.generate_image_hash(item, fig)
 
                 # Only test figures if not generating images
                 if self.generate_dir is None:
@@ -571,8 +607,11 @@ def item_function_wrapper(*args, **kwargs):
 
                     close_mpl_figure(fig)
 
+                    self._test_results[str(pathify(test_name))] = msg or True
+
                     if msg is None:
-                        shutil.rmtree(result_dir)
+                        if not self.results_always:
+                            shutil.rmtree(result_dir)
                     else:
                         pytest.fail(msg, pytrace=False)
 
@@ -592,8 +631,10 @@ def generate_summary_html(self, dir_list):
             f.write(HTML_INTRO)
 
             for directory in dir_list:
+                test_name = directory.parts[-1]
+                test_result = 'passed' if self._test_results[test_name] is True else 'failed'
                 f.write('<tr>'
-                        f'<td>{directory.parts[-1]}\n'
+                        f'<td>{test_name} ({test_result})\n'
                         f'<td><img src="{directory / "baseline.png"}"></td>\n'
                         f'<td><img src="{directory / "result-failed-diff.png"}"></td>\n'
                         f'<td><img src="{directory / "result.png"}"></td>\n'

tests/baseline/hashes/mpl20_ft2104.json → tests/baseline/hashes/mpl20_ft261.json

@@ -8,5 +8,9 @@
   "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057",
   "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7",
   "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93",
-  "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498"
-}
+  "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498",
+  "test_pytest_mpl.test_hash_succeeds": "480062c2239ed9d70e361d1a5b578dc2aa756971161ac6e7287b492ae6118c59",
+  "test.test_modified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b",
+  "test.test_new": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b",
+  "test.test_unmodified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b"
+}

tests/baseline/hashes/mpl21_ft2104.json → tests/baseline/hashes/mpl21_ft261.json

@@ -8,5 +8,9 @@
   "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057",
   "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7",
   "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93",
-  "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498"
-}
+  "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498",
+  "test_pytest_mpl.test_hash_succeeds": "17b65dd0247b0dfd8c1b4b079352414ae0fe03c0a3e79d63c8b8670d84d4098f",
+  "test.test_modified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336",
+  "test.test_new": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336",
+  "test.test_unmodified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336"
+}

tests/baseline/hashes/mpl22_ft2104.json → tests/baseline/hashes/mpl22_ft261.json

@@ -8,5 +8,9 @@
   "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057",
   "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7",
   "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93",
-  "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498"
-}
+  "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498",
+  "test_pytest_mpl.test_hash_succeeds": "e80557c8784fb920fb79b03b26dc072649a98811f00a8c212df8761e4351acde",
+  "test.test_modified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7",
+  "test.test_new": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7",
+  "test.test_unmodified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7"
+}

tests/baseline/hashes/mpl30_ft2104.json → tests/baseline/hashes/mpl30_ft261.json

@@ -8,5 +8,9 @@
   "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057",
   "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7",
   "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93",
-  "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498"
-}
+  "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498",
+  "test_pytest_mpl.test_hash_succeeds": "4e1157a93733cdb327f1741afdb0525f4d0e3f12e60b54f72c93db9f9c9ae27f",
+  "test.test_modified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898",
+  "test.test_new": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898",
+  "test.test_unmodified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898"
+}

tests/baseline/hashes/mpl31_ft261.json

@@ -9,5 +9,8 @@
   "test_pytest_mpl.test_parametrized[5]": "be7dc9de64a5d6fd458c1f930d4aa56cf8196ddb0e8b5b07ab79a1f0ea9eb820",
   "test_pytest_mpl.test_parametrized[50]": "a8ae2427337803dc864784d88c4428a6af5a3e47d2bfc84c98b68b25fde75704",
   "test_pytest_mpl.test_parametrized[500]": "590ef42388378173e293bd37e95ff22d8e753d53327d1fb5d6bdf2bac4f84d01",
-  "test_pytest_mpl.test_hash_succeeds": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f"
-}
+  "test_pytest_mpl.test_hash_succeeds": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f",
+  "test.test_modified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2",
+  "test.test_new": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2",
+  "test.test_unmodified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2"
+}

tests/baseline/hashes/mpl32_ft261.json

@@ -9,5 +9,8 @@
   "test_pytest_mpl.test_parametrized[5]": "9b2b5b1df784c8f9a5fc624840138fe7b4dbdd42cf592fe5733c9c825e5dda91",
   "test_pytest_mpl.test_parametrized[50]": "fcf0566ef5514674e2b4bf1e9b4c7f52451c6f98abdc75dc876f43c97a23bc32",
   "test_pytest_mpl.test_parametrized[500]": "38dccccfc980b44359bc1b325bef48471bc084db37ed622af00a553792a8b093",
-  "test_pytest_mpl.test_hash_succeeds": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04"
-}
+  "test_pytest_mpl.test_hash_succeeds": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04",
+  "test.test_modified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0",
+  "test.test_new": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0",
+  "test.test_unmodified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0"
+}

tests/baseline/hashes/mpl33_ft261.json

@@ -9,5 +9,8 @@
   "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7",
   "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93",
   "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498",
-  "test_pytest_mpl.test_hash_succeeds": "55ad74a44c99606f1df1e79f67a59a4986bddc2b48ea2b2d7ea8365db91dc7ca"
+  "test_pytest_mpl.test_hash_succeeds": "55ad74a44c99606f1df1e79f67a59a4986bddc2b48ea2b2d7ea8365db91dc7ca",
+  "test.test_modified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181",
+  "test.test_new": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181",
+  "test.test_unmodified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181"
 }