test(backport): Use coverage over pytest-cov (#2495)

* Backport parts of PR #2054

.github/workflows/ci.yml

@@ -60,14 +60,19 @@ jobs:
     - name: List installed Python packages
       run: python -m pip list
 
-    - name: Test with pytest
+    - name: Test with pytest and coverage
       run: |
-        pytest --ignore tests/benchmarks/ --ignore tests/contrib --ignore tests/test_notebooks.py
+        coverage run --module pytest --ignore tests/contrib --ignore tests/benchmarks --ignore tests/test_notebooks.py
 
     - name: Launch a tmate session if tests fail
       if: failure() && github.event_name == 'workflow_dispatch'
       uses: mxschmitt/action-tmate@v3
 
+    - name: Coverage report for core project
+      run: |
+        coverage report
+        coverage xml
+
       # Report coverage for oldest and newest Python tested to deal with version differences
     - name: Report core project coverage with Codecov
       if: >-
@@ -82,7 +87,12 @@ jobs:
 
     - name: Test Contrib module with pytest
       run: |
-        pytest tests/contrib --mpl --mpl-baseline-path tests/contrib/baseline
+        coverage run --append --module pytest tests/contrib --mpl --mpl-baseline-path tests/contrib/baseline
+
+    - name: Coverage report with contrib
+      run: |
+        coverage report
+        coverage xml
 
     - name: Report contrib coverage with Codecov
       if: github.event_name != 'schedule' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'

pyproject.toml

@@ -98,10 +98,9 @@ all = ["pyhf[backends,xmlio,contrib,shellcomplete]"]
 
 # Developer extras
 test = [
-    "pyhf[all]",
     "scikit-hep-testdata>=0.4.11",
     "pytest>=6.0",
-    "pytest-cov>=2.5.1",
+    "coverage[toml]>=6.0.0",
     "pytest-mock",
     "requests-mock>=1.9.0",
     "pytest-benchmark[histogram]",
@@ -131,7 +130,7 @@ docs = [
     "ipython!=8.7.0",  # c.f. https://github.com/scikit-hep/pyhf/pull/2068
 ]
 develop = [
-    "pyhf[test,docs]",
+    "pyhf[all,test,docs]",
     "tbump>=6.7.0",
     "pre-commit",
     "nox",
@@ -181,14 +180,9 @@ minversion = "6.0"
 xfail_strict = true
 addopts = [
     "-ra",
-    "--cov=pyhf",
-    "--cov-branch",
     "--showlocals",
     "--strict-markers",
     "--strict-config",
-    "--cov-report=term-missing",
-    "--cov-report=xml",
-    "--cov-report=html",
     "--doctest-modules",
     "--doctest-glob='*.rst'",
 ]
@@ -232,7 +226,22 @@ filterwarnings = [
     'ignore:`np.bool8` is a deprecated alias for `np.bool_`:DeprecationWarning',  # numpy via tensorflow
     "ignore:module 'sre_constants' is deprecated:DeprecationWarning",  # tensorflow v2.12.0+ for Python 3.11+
     "ignore:ml_dtypes.float8_e4m3b11 is deprecated.",  #FIXME: Can remove when jaxlib>=0.4.12
+    "ignore:jsonschema.RefResolver is deprecated as of v4.18.0, in favor of the:DeprecationWarning",  # Issue #2139
     "ignore:Skipping device Apple Paravirtual device that does not support Metal 2.0:UserWarning",  # Can't fix given hardware/virtualized device
+    'ignore:Type google._upb._message.[A-Z]+ uses PyType_Spec with a metaclass that has custom:DeprecationWarning',  # protobuf via tensorflow
+]
+
+[tool.coverage.run]
+source = ["pyhf"]
+branch = true
+omit = ["*/pyhf/typing.py"]
+
+[tool.coverage.report]
+precision = 1
+sort = "cover"
+show_missing = true
+exclude_also = [
+    "if TYPE_CHECKING:"
 ]
 
 [tool.mypy]