feat: initial implementation of plug_caisson usage (#1902)

* feat: initial implementation of plug_caisson usage

* chore: add failing tests

* chore: remove dbg statement

* chore: bump plug_caisson version

* chore: remove :length option

* chore: version bump

VERSION

@@ -1 +1 @@
-1.5.13
+1.5.14

lib/logflare_web/endpoint.ex

@@ -33,10 +33,8 @@ defmodule LogflareWeb.Endpoint do
   plug(Plug.Logger, log: :debug)
 
   plug(Plug.Parsers,
-    parsers: [:urlencoded, :multipart, :json],
-    pass: ["*/*"],
-    json_decoder: Jason,
-    body_reader: {LogflareWeb.Plugs.CompressedBodyReader, :read_body, []}
+    parsers: [:urlencoded, :multipart],
+    pass: ["*/*"]
   )
 
   plug(Plug.MethodOverride)

lib/logflare_web/router.ex

@@ -56,7 +56,7 @@ defmodule LogflareWeb.Router do
     plug(Plug.Parsers,
       parsers: [:json, :bert, :syslog, :ndjson],
       json_decoder: Jason,
-      body_reader: {LogflareWeb.Plugs.CompressedBodyReader, :read_body, []}
+      body_reader: {PlugCaisson, :read_body, []}
     )
 
     plug(:accepts, ["json", "bert"])

mix.exs

@@ -71,6 +71,7 @@ defmodule Logflare.Mixfile do
       {:bandit, ">= 0.7.7"},
       {:plug_crypto, "~> 1.2.2"},
       {:cors_plug, "~> 2.0"},
+      {:plug_caisson, "~> 0.1"},
 
       # Oauth
       {:ueberauth_google, "~> 0.8"},

mix.lock

@@ -98,6 +98,7 @@
   "phoenix_template": {:hex, :phoenix_template, "1.0.3", "32de561eefcefa951aead30a1f94f1b5f0379bc9e340bb5c667f65f1edfa4326", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "16f4b6588a4152f3cc057b9d0c0ba7e82ee23afa65543da535313ad8d25d8e2c"},
   "phoenix_view": {:hex, :phoenix_view, "2.0.3", "4d32c4817fce933693741deeb99ef1392619f942633dde834a5163124813aad3", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "cd34049af41be2c627df99cd4eaa71fc52a328c0c3d8e7d4aa28f880c30e7f64"},
   "plug": {:hex, :plug, "1.15.2", "94cf1fa375526f30ff8770837cb804798e0045fd97185f0bb9e5fcd858c792a3", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "02731fa0c2dcb03d8d21a1d941bdbbe99c2946c0db098eee31008e04c6283615"},
+  "plug_caisson": {:hex, :plug_caisson, "0.1.2", "bc37ede724aa1f5758a321f707792442c852800227219a078672ea5c9226da02", [:mix], [{:brotli, "~> 0.3.2", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "36d879d6de67d5d0727f776a4a32d4a3e463ef8bab570c80a88090de9b4deb11"},
   "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},
   "poison": {:hex, :poison, "5.0.0", "d2b54589ab4157bbb82ec2050757779bfed724463a544b6e20d79855a9e43b24", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "11dc6117c501b80c62a7594f941d043982a1bd05a1184280c0d9166eb4d8d3fc"},
   "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"},

test/logflare_web/controllers/log_controller_test.exs

@@ -97,6 +97,19 @@ defmodule LogflareWeb.LogControllerTest do
       :timer.sleep(2000)
     end
 
+    test ":create ingestion with gzip", %{conn: conn, source: source} do
+      batch = for _i <- 1..1500, do: @valid
+      payload = :zlib.gzip(Jason.encode!(%{"batch" => batch}))
+      conn =
+        conn
+        |> Plug.Conn.put_req_header("content-encoding", "gzip" )
+        |> Plug.Conn.put_req_header("content-type", "application/json" )
+        |> post(~p"/logs?#{[source: source.token]}", payload)
+
+      assert json_response(conn, 200) == %{"message" => "Logged!"}
+      # wait for all logs to be ingested before removing all stubs
+      :timer.sleep(2000)
+    end
     test ":create ingestion batch with `batch` key", %{conn: conn, source: source} do
       conn =
         conn