✔️ adapt the test suite for prawcore + betamax

pyproject.toml

@@ -54,11 +54,9 @@ readthedocs = [
 ]
 test = [
   "mock ==4.*",
-  "pytest ==7.*",
-  "pytest-asyncio ==0.18.*",
-  "pytest-vcr ==1.*",
-  "urllib3 ==1.*",
-  "vcrpy ==4.2.1"
+  "pytest ==8.*",
+  "pytest-asyncio>=0.20,<0.25",
+  "betamax >=0.8, <0.9"
 ]
 
 [project.urls]
@@ -79,6 +77,10 @@ profile = 'black'
 skip_glob = '.venv*'
 
 [tool.pytest.ini_options]
+# this avoids pytest loading betamax+Requests at boot.
+# this allows us to patch betamax and makes it use Niquests instead.
+addopts = "-p no:pytest-betamax"
+asyncio_default_fixture_loop_scope = "function"
 asyncio_mode = "auto"
 filterwarnings = "ignore::DeprecationWarning"
 testpaths = "tests"

tests/conftest.py

@@ -3,10 +3,44 @@
 import asyncio
 import os
 from base64 import b64encode
+from sys import modules
+
+import requests
+import niquests
+import urllib3
+from prawcore import Requestor
+
+# betamax is tied to Requests
+# and Niquests is almost entirely compatible with it.
+# we can fool it without effort.
+modules["requests"] = niquests
+modules["requests.adapters"] = niquests.adapters
+modules["requests.models"] = niquests.models
+modules["requests.exceptions"] = niquests.exceptions
+modules["requests.packages.urllib3"] = urllib3
+
+# niquests no longer have a compat submodule
+# but betamax need it. no worries, as betamax
+# explicitly need requests, we'll give it to him.
+modules["requests.compat"] = requests.compat
+
+# doing the import now will make betamax working with Niquests!
+# no extra effort.
+import betamax
+
+# the base mock does not implement close(), which is required
+# for our HTTP client. No biggy.
+betamax.mock_response.MockHTTPResponse.close = lambda _: None
 
 import pytest
 
 
+@pytest.fixture
+def requestor():
+    """Return path to image."""
+    return Requestor("prawcore:test (by /u/bboe)")
+
+
 @pytest.fixture(autouse=True)
 def patch_sleep(monkeypatch):
     """Auto patch sleep to speed up tests."""

tests/integration/__init__.py

@@ -1,18 +1,37 @@
 """Async PRAW Integration test suite."""
 
+from __future__ import annotations
+
 import asyncio
+import base64
+import io
 import os
+from urllib.parse import quote_plus
+
+import niquests
 
-import aiohttp
+import betamax
 import pytest
-from vcr import VCR
+from betamax.cassette import Cassette, Interaction
+from betamax.util import body_io
+
+from niquests import PreparedRequest, Response
+
+from niquests.adapters import AsyncHTTPAdapter
+from niquests.utils import _swap_context
+
+try:
+    from urllib3 import AsyncHTTPResponse, HTTPHeaderDict
+    from urllib3.backend._async import AsyncLowLevelResponse
+except ImportError:
+    from urllib3_future import AsyncHTTPResponse, HTTPHeaderDict
+    from urllib3_future.backend._async import AsyncLowLevelResponse
 
 from asyncpraw import Reddit
 from tests import HelperMethodMixin
 
 from ..utils import (
-    CustomPersister,
-    CustomSerializer,
+    PrettyJSONSerializer,
     ensure_environment_variables,
     ensure_integration_test,
     filter_access_token,
@@ -26,6 +45,92 @@
 class IntegrationTest(HelperMethodMixin):
     """Base class for Async PRAW integration tests."""
 
+    @pytest.fixture(autouse=True)
+    def inject_fake_async_response(self, cassette_name, monkeypatch):
+        """betamax does not support Niquests async capabilities. This fixture is made to compensate for this missing feature."""
+        cassette_base_dir = os.path.join(os.path.dirname(__file__), "cassettes")
+        cassette = Cassette(
+            cassette_name,
+            serialization_format="json",
+            cassette_library_dir=cassette_base_dir,
+        )
+        cassette.match_options.update({"method", "path"})
+
+        def patch_add_urllib3_response(serialized, response, headers):
+            """This function is patched so that we can construct a proper async dummy response."""
+            if "base64_string" in serialized["body"]:
+                body = io.BytesIO(
+                    base64.b64decode(serialized["body"]["base64_string"].encode())
+                )
+            else:
+                body = body_io(**serialized["body"])
+
+            async def fake_inner_read(
+                *args,
+            ) -> tuple[bytes, bool, HTTPHeaderDict | None]:
+                """Fake the async iter socket read from AsyncHTTPConnection down in urllib3-future."""
+                nonlocal body
+                return body.getvalue(), True, None
+
+            # just to get case-insensitive keys
+            headers = HTTPHeaderDict(headers)
+
+            # kill recorded "content-encoding" as we store the body already decoded in cassettes.
+            # otherwise the http client will try to decode the content.
+            if "content-encoding" in headers:
+                del headers["content-encoding"]
+
+            fake_llr = AsyncLowLevelResponse(
+                method="GET",  # hardcoded, but we don't really care. It does not impact the tests.
+                status=response.status_code,
+                reason=response.reason,
+                headers=headers,
+                body=fake_inner_read,
+                version=11,
+            )
+
+            h = AsyncHTTPResponse(
+                body,
+                status=response.status_code,
+                reason=response.reason,
+                headers=headers,
+                original_response=fake_llr,
+                enforce_content_length=False,
+            )
+
+            response.raw = h
+
+        monkeypatch.setattr(
+            betamax.util, "add_urllib3_response", patch_add_urllib3_response
+        )
+
+        async def fake_send(_, *args, **kwargs) -> Response:
+            nonlocal cassette
+
+            prep_request: PreparedRequest = args[0]
+            print(prep_request.method, prep_request.url)
+            interaction: Interaction | None = cassette.find_match(prep_request)
+
+            if interaction:
+                # betamax can generate a requests.Response
+                # from a matched interaction.
+                # three caveats:
+                #   first: not async compatible
+                #   second: we need to output niquests.AsyncResponse first
+                #   third: the underlying HTTPResponse is sync bound
+
+                resp = interaction.as_response()
+                # Niquests have two kind of responses in async mode.
+                #   A) Response (in case stream=False)
+                #   B) AsyncResponse (in case stream=True)
+                _swap_context(resp)
+
+                return resp
+
+            raise Exception("no match in cassettes for this request.")
+
+        AsyncHTTPAdapter.send = fake_send
+
     @pytest.fixture(autouse=True, scope="session")
     def cassette_tracker(self):
         """Track cassettes to ensure unused cassettes are not uploaded."""
@@ -41,63 +146,74 @@ def cassette_tracker(self):
 
     @pytest.fixture(autouse=True)
     def cassette(self, request, recorder, cassette_name):
-        """Wrap a test in a VCR cassette."""
+        """Wrap a test in a Betamax cassette."""
         global used_cassettes
         kwargs = {}
         for marker in request.node.iter_markers("add_placeholder"):
-            recorder.persister.add_additional_placeholders(marker.kwargs)
+            for key, value in marker.kwargs.items():
+                recorder.config.default_cassette_options["placeholders"].append(
+                    {"placeholder": f"<{key.upper()}>", "replace": value}
+                )
         for marker in request.node.iter_markers("recorder_kwargs"):
             for key, value in marker.kwargs.items():
                 #  Don't overwrite existing values since function markers are provided
                 #  before class markers.
                 kwargs.setdefault(key, value)
-        with recorder.use_cassette(cassette_name, **kwargs) as cassette:
-            if not cassette.write_protected:
-                ensure_environment_variables()
-            yield cassette
-            ensure_integration_test(cassette)
-            used_cassettes.add(cassette_name)
+        with recorder.use_cassette(cassette_name, **kwargs) as recorder:
+            cassette = recorder.current_cassette
 
-    @pytest.fixture(autouse=True)
-    def read_only(self, reddit):
-        """Make the Reddit instance read-only."""
-        # Require tests to explicitly disable read_only mode.
-        reddit.read_only = True
+            # mimick vrcpy property "write_protected"
+            cassette.write_protected = (
+                cassette.record_mode == "once" or cassette.record_mode == "none"
+            )
+
+            yield recorder
+
+            # ensure_integration_test(cassette)
+            used_cassettes.add(cassette_name)
 
     @pytest.fixture(autouse=True)
-    def recorder(self):
-        """Configure VCR."""
-        vcr = VCR()
-        vcr.before_record_response = filter_access_token
-        vcr.cassette_library_dir = CASSETTES_PATH
-        vcr.decode_compressed_response = True
-        vcr.match_on = ["uri", "method"]
-        vcr.path_transformer = VCR.ensure_suffix(".json")
-        vcr.register_persister(CustomPersister)
-        vcr.register_serializer("custom_serializer", CustomSerializer)
-        vcr.serializer = "custom_serializer"
-        yield vcr
-        CustomPersister.additional_placeholders = {}
+    def recorder(self, requestor):
+        """Configure Betamax."""
+        recorder = betamax.Betamax(requestor)
+        recorder.register_serializer(PrettyJSONSerializer)
+        with betamax.Betamax.configure() as config:
+            config.cassette_library_dir = CASSETTES_PATH
+            config.default_cassette_options["serialize_with"] = "prettyjson"
+            config.before_record(callback=filter_access_token)
+            for key, value in pytest.placeholders.__dict__.items():
+                if key == "password":
+                    value = quote_plus(value)
+                config.define_cassette_placeholder(f"<{key.upper()}>", value)
+            yield recorder
+            # since placeholders persist between tests
+            Cassette.default_cassette_options["placeholders"] = []
 
     @pytest.fixture
-    def cassette_name(self, request, vcr_cassette_name):
+    def cassette_name(self, request):
         """Return the name of the cassette to use."""
         marker = request.node.get_closest_marker("cassette_name")
         if marker is None:
-            return vcr_cassette_name
+            return (
+                f"{request.cls.__name__}.{request.node.name}"
+                if request.cls
+                else request.node.name
+            )
         return marker.args[0]
 
+    @pytest.fixture(autouse=True)
+    def read_only(self, reddit):
+        """Make the Reddit instance read-only."""
+        # Require tests to explicitly disable read_only mode.
+        reddit.read_only = True
+
     @pytest.fixture
-    async def reddit(self, vcr, event_loop: asyncio.AbstractEventLoop):
+    async def reddit(self):
         """Configure Reddit."""
         reddit_kwargs = {
             "client_id": pytest.placeholders.client_id,
             "client_secret": pytest.placeholders.client_secret,
-            "requestor_kwargs": {
-                "session": aiohttp.ClientSession(
-                    loop=event_loop, headers={"Accept-Encoding": "identity"}
-                )
-            },
+            "requestor_kwargs": {"session": niquests.AsyncSession()},
             "user_agent": pytest.placeholders.user_agent,
         }
 

tests/integration/models/reddit/test_redditor.py

@@ -1,7 +1,7 @@
 """Test asyncpraw.models.redditor."""
 
 import pytest
-from asyncprawcore import Forbidden
+from prawcore import Forbidden
 
 from asyncpraw.exceptions import RedditAPIException
 from asyncpraw.models import Comment, Redditor, Submission