HTTPRequest -> http.Request, add request.authority

Author: mhils

examples/addons/duplicate-modify-replay.py

@@ -4,7 +4,7 @@
 
 def request(flow):
     # Avoid an infinite loop by not replaying already replayed requests
-    if flow.request.is_replay:
+    if flow.is_replay == "request":
         return
     flow = flow.copy()
     # Only interactive tools have a view. If we have one, add a duplicate entry

mitmproxy/addons/clientplayback.py

@@ -17,6 +17,7 @@
 from mitmproxy.coretypes import basethread
 from mitmproxy.net import server_spec, tls
 from mitmproxy.net.http import http1
+from mitmproxy.net.http.url import hostport
 from mitmproxy.utils import human
 
 
@@ -46,7 +47,7 @@ def replay(self, f):  # pragma: no cover
         f.live = True
         r = f.request
         bsl = human.parse_size(self.options.body_size_limit)
-        first_line_format_backup = r.first_line_format
+        authority_backup = r.authority
         server = None
         try:
             f.response = None
@@ -79,9 +80,9 @@ def replay(self, f):  # pragma: no cover
                             sni=f.server_conn.sni,
                             **tls.client_arguments_from_options(self.options)
                         )
-                        r.first_line_format = "relative"
+                        r.authority = b""
                     else:
-                        r.first_line_format = "absolute"
+                        r.authority = hostport(r.scheme, r.host, r.port)
                 else:
                     server_address = (r.host, r.port)
                     server = connections.ServerConnection(server_address)
@@ -91,7 +92,7 @@ def replay(self, f):  # pragma: no cover
                             sni=f.server_conn.sni,
                             **tls.client_arguments_from_options(self.options)
                         )
-                    r.first_line_format = "relative"
+                    r.authority = ""
 
                 server.wfile.write(http1.assemble_request(r))
                 server.wfile.flush()
@@ -101,9 +102,7 @@ def replay(self, f):  # pragma: no cover
                     f.server_conn.close()
                 f.server_conn = server
 
-                f.response = http.HTTPResponse.wrap(
-                    http1.read_response(server.rfile, r, body_size_limit=bsl)
-                )
+                f.response = http1.read_response(server.rfile, r, body_size_limit=bsl)
             response_reply = self.channel.ask("response", f)
             if response_reply == exceptions.Kill:
                 raise exceptions.Kill()
@@ -115,7 +114,7 @@ def replay(self, f):  # pragma: no cover
         except Exception as e:
             self.channel.tell("log", log.LogEntry(repr(e), "error"))
         finally:
-            r.first_line_format = first_line_format_backup
+            r.authority = authority_backup
             f.live = False
             if server and server.connected():
                 server.finish()
@@ -200,7 +199,7 @@ def start_replay(self, flows: typing.Sequence[flow.Flow]) -> None:
             lst.append(hf)
             # Prepare the flow for replay
             hf.backup()
-            hf.request.is_replay = True
+            hf.is_replay = "request"
             hf.response = None
             hf.error = None
             # https://github.com/mitmproxy/mitmproxy/issues/2197

mitmproxy/addons/dumper.py

@@ -127,7 +127,7 @@ def _echo_request_line(self, flow):
                     human.format_address(flow.client_conn.address)
                 )
             )
-        elif flow.request.is_replay:
+        elif flow.is_replay == "request":
             client = click.style("[replay]", fg="yellow", bold=True)
         else:
             client = ""
@@ -166,7 +166,7 @@ def _echo_request_line(self, flow):
         self.echo(line)
 
     def _echo_response_line(self, flow):
-        if flow.response.is_replay:
+        if flow.is_replay == "response":
             replay = click.style("[replay] ", fg="yellow", bold=True)
         else:
             replay = ""

mitmproxy/addons/intercept.py

@@ -37,7 +37,7 @@ def process_flow(self, f):
         if self.filt:
             should_intercept = all([
                 self.filt(f),
-                not f.request.is_replay,
+                not f.is_replay == "request",
             ])
             if should_intercept and ctx.options.intercept_active:
                 f.intercept()

mitmproxy/addons/mapremote.py

@@ -47,4 +47,4 @@ def request(self, flow: http.HTTPFlow) -> None:
                 # this is a bit messy: setting .url also updates the host header,
                 # so we really only do that if the replacement affected the URL.
                 if url != new_url:
-                    flow.request.url = new_url
+                    flow.request.url = new_url  # type: ignore

mitmproxy/addons/serverplayback.py

@@ -202,10 +202,10 @@ def request(self, f: http.HTTPFlow) -> None:
             if rflow:
                 assert rflow.response
                 response = rflow.response.copy()
-                response.is_replay = True
                 if ctx.options.server_replay_refresh:
                     response.refresh()
                 f.response = response
+                f.is_replay = "response"
             elif ctx.options.server_replay_kill_extra:
                 ctx.log.warn(
                     "server_playback: killed non-replay request {}".format(

mitmproxy/coretypes/serializable.py

@@ -1,5 +1,8 @@
 import abc
 import uuid
+from typing import Type, TypeVar
+
+T = TypeVar('T', bound='Serializable')
 
 
 class Serializable(metaclass=abc.ABCMeta):
@@ -9,7 +12,7 @@ class Serializable(metaclass=abc.ABCMeta):
 
     @classmethod
     @abc.abstractmethod
-    def from_state(cls, state):
+    def from_state(cls: Type[T], state) -> T:
         """
         Create a new object from the given state.
         """
@@ -29,7 +32,7 @@ def set_state(self, state):
         """
         raise NotImplementedError()
 
-    def copy(self):
+    def copy(self: T) -> T:
         state = self.get_state()
         if isinstance(state, dict) and "id" in state:
             state["id"] = str(uuid.uuid4())

mitmproxy/flow.py

@@ -77,6 +77,7 @@ def __init__(
         self._backup: typing.Optional[Flow] = None
         self.reply: typing.Optional[controller.Reply] = None
         self.marked: bool = False
+        self.is_replay: typing.Optional[str] = None
         self.metadata: typing.Dict[str, typing.Any] = dict()
 
     _stateobject_attributes = dict(
@@ -86,6 +87,7 @@ def __init__(
         server_conn=connections.ServerConnection,
         type=str,
         intercepted=bool,
+        is_replay=str,
         marked=bool,
         metadata=typing.Dict[str, typing.Any],
     )

mitmproxy/http.py

@@ -1,142 +1,13 @@
 import html
-from typing import Optional
-
+import time
+from typing import Optional, Tuple
 from mitmproxy import connections
 from mitmproxy import flow
 from mitmproxy import version
 from mitmproxy.net import http
 
-
-class HTTPRequest(http.Request):
-    """
-    A mitmproxy HTTP request.
-    """
-
-    # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Request` and
-    # may be removed in the future.
-
-    def __init__(
-            self,
-            first_line_format,
-            method,
-            scheme,
-            host,
-            port,
-            path,
-            http_version,
-            headers,
-            content,
-            trailers=None,
-            timestamp_start=None,
-            timestamp_end=None,
-            is_replay=False,
-    ):
-        http.Request.__init__(
-            self,
-            first_line_format,
-            method,
-            scheme,
-            host,
-            port,
-            path,
-            http_version,
-            headers,
-            content,
-            trailers,
-            timestamp_start,
-            timestamp_end,
-        )
-        # Is this request replayed?
-        self.is_replay = is_replay
-        self.stream = None
-
-    def get_state(self):
-        state = super().get_state()
-        state["is_replay"] = self.is_replay
-        return state
-
-    def set_state(self, state):
-        state = state.copy()
-        self.is_replay = state.pop("is_replay")
-        super().set_state(state)
-
-    @classmethod
-    def wrap(self, request):
-        """
-        Wraps an existing :py:class:`mitmproxy.net.http.Request`.
-        """
-        req = HTTPRequest(
-            first_line_format=request.data.first_line_format,
-            method=request.data.method,
-            scheme=request.data.scheme,
-            host=request.data.host,
-            port=request.data.port,
-            path=request.data.path,
-            http_version=request.data.http_version,
-            headers=request.data.headers,
-            content=request.data.content,
-            trailers=request.data.trailers,
-            timestamp_start=request.data.timestamp_start,
-            timestamp_end=request.data.timestamp_end,
-        )
-        return req
-
-    def __hash__(self):
-        return id(self)
-
-
-class HTTPResponse(http.Response):
-    """
-    A mitmproxy HTTP response.
-    """
-
-    # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Response` and
-    # may be removed in the future.
-
-    def __init__(
-            self,
-            http_version,
-            status_code,
-            reason,
-            headers,
-            content,
-            trailers=None,
-            timestamp_start=None,
-            timestamp_end=None,
-            is_replay=False
-    ):
-        http.Response.__init__(
-            self,
-            http_version,
-            status_code,
-            reason,
-            headers,
-            content,
-            trailers,
-            timestamp_start=timestamp_start,
-            timestamp_end=timestamp_end,
-        )
-
-        # Is this request replayed?
-        self.is_replay = is_replay
-        self.stream = None
-
-    @classmethod
-    def wrap(self, response):
-        """
-        Wraps an existing :py:class:`mitmproxy.net.http.Response`.
-        """
-        resp = HTTPResponse(
-            http_version=response.data.http_version,
-            status_code=response.data.status_code,
-            reason=response.data.reason,
-            headers=response.data.headers,
-            content=response.data.content,
-            trailers=response.data.trailers,
-            timestamp_start=response.data.timestamp_start,
-            timestamp_end=response.data.timestamp_end,
-        )
-        return resp
+HTTPRequest = http.Request
+HTTPResponse = http.Response
 
 
 class HTTPFlow(flow.Flow):
@@ -197,8 +68,7 @@ def make_error_response(
         message: str = "",
         headers: Optional[http.Headers] = None,
 ) -> HTTPResponse:
-    reason = http.status_codes.RESPONSES.get(status_code, "Unknown")
-    body = """
+    body: bytes = """
         <html>
             <head>
                 <title>{status_code} {reason}</title>
@@ -210,7 +80,7 @@ def make_error_response(
         </html>
     """.strip().format(
         status_code=status_code,
-        reason=reason,
+        reason=http.status_codes.RESPONSES.get(status_code, "Unknown"),
         message=html.escape(message),
     ).encode("utf8", "replace")
 
@@ -222,19 +92,23 @@ def make_error_response(
             Content_Type="text/html"
         )
 
-    return HTTPResponse(
-        b"HTTP/1.1",
-        status_code,
-        reason,
-        headers,
-        body,
-    )
+    return HTTPResponse.make(status_code, body, headers)
 
 
-def make_connect_request(address):
+def make_connect_request(address: Tuple[str, int]) -> HTTPRequest:
     return HTTPRequest(
-        "authority", b"CONNECT", None, address[0], address[1], None, b"HTTP/1.1",
-        http.Headers(), b""
+        host=address[0],
+        port=address[1],
+        method=b"CONNECT",
+        scheme=b"",
+        authority=f"{address[0]}:{address[1]}".encode(),
+        path=b"",
+        http_version=b"HTTP/1.1",
+        headers=http.Headers(),
+        content=b"",
+        trailers=None,
+        timestamp_start=time.time(),
+        timestamp_end=time.time(),
     )
 
 
@@ -247,9 +121,11 @@ def make_connect_response(http_version):
         b"Connection established",
         http.Headers(),
         b"",
+        None,
+        time.time(),
+        time.time(),
     )
 
 
-expect_continue_response = HTTPResponse(
-    b"HTTP/1.1", 100, b"Continue", http.Headers(), b""
-)
+def make_expect_continue_response():
+    return HTTPResponse.make(100)