diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index a81ac99..a922c3b 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -34,8 +34,10 @@ jobs: otp: '25' - elixir: '1.15' otp: '26' + - elixir: '1.16' + otp: '26' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: erlef/setup-beam@v1 with: otp-version: ${{matrix.otp}} @@ -48,13 +50,13 @@ jobs: runs-on: ubuntu-latest name: Static analysis steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set mix.lock file hash run: | mix_hash="${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}" echo "mix_hash=$mix_hash" >> $GITHUB_ENV - name: Cache PLT files - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | _build/dev/*.plt @@ -66,6 +68,6 @@ jobs: uses: erlef/setup-beam@v1 with: otp-version: '26' - elixir-version: '1.15' + elixir-version: '1.16' - run: mix do deps.get, compile - run: mix check diff --git a/.tool-versions b/.tool-versions index 38d92fa..15e28fa 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 26.0.2 -elixir 1.15.4-otp-26 +erlang 26.2.1 +elixir 1.16.0-otp-26 diff --git a/CHANGELOG.md b/CHANGELOG.md index fd77cbe..7f1d1dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.9.0] - 2024-04-24 + +### Added + +- Added sending of pools size metrics via `telemetry`. [Working with metrics guide](https://hexdocs.pm/poolex/pool-metrics.html) + ## [0.8.0] - 2023-08-30 ### Changed @@ -233,7 +239,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Supported main interface `Poolex.run/3` with `:timeout` option. -[unreleased]: https://github.com/general-CbIC/poolex/compare/v0.8.0...HEAD +[unreleased]: https://github.com/general-CbIC/poolex/compare/v0.9.0...HEAD +[0.9.0]: https://github.com/general-CbIC/poolex/compare/v0.8.0...v0.9.0 [0.8.0]: https://github.com/general-CbIC/poolex/compare/v0.7.6...v0.8.0 [0.7.6]: https://github.com/general-CbIC/poolex/compare/v0.7.5...v0.7.6 [0.7.5]: https://github.com/general-CbIC/poolex/compare/v0.7.4...v0.7.5 diff --git a/README.md b/README.md index 26a6991..4ffd6b1 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,17 @@ Poolex is a library for managing pools of workers. Inspired by [poolboy](https://github.com/devinus/poolboy). +## Table of Contents + +- [Poolex](#poolex) + - [Table of Contents](#table-of-contents) + - [Features](#features) + - [Requirements](#requirements) + - [Installation](#installation) + - [Usage](#usage) + - [Guides](#guides) + - [Contributions](#contributions) + ## Features With `poolex` you can: @@ -31,24 +42,6 @@ With `poolex` you can: | Erlang/OTP | >= 22 | | Elixir | >= 1.7 | -## Table of Contents - -- [Installation](#installation) -- [Getting Started](https://hexdocs.pm/poolex/getting-started.html) - - [Starting pool of workers](https://hexdocs.pm/poolex/getting-started.html#starting-pool-of-workers) - - [Poolex configuration options](https://hexdocs.pm/poolex/getting-started.html#starting-pool-of-workers) - - [Working with the pool](https://hexdocs.pm/poolex/getting-started.html#working-with-the-pool) -- [Migration from `:poolboy`](https://hexdocs.pm/poolex/migration-from-poolboy.html) -- [Example of use](https://hexdocs.pm/poolex/example-of-use.html) - - [Defining the worker](https://hexdocs.pm/poolex/example-of-use.html#defining-the-worker) - - [Configuring Poolex](https://hexdocs.pm/poolex/example-of-use.html#configuring-poolex) - - [Using Poolex](https://hexdocs.pm/poolex/example-of-use.html#using-poolex) -- [Workers and callers implementations](https://hexdocs.pm/poolex/workers-and-callers-implementations.html) - - [Callers](https://hexdocs.pm/poolex/workers-and-callers-implementations.html#callers) - - [Workers](https://hexdocs.pm/poolex/workers-and-callers-implementations.html#workers) - - [Writing custom implementations](https://hexdocs.pm/poolex/workers-and-callers-implementations.html#writing-custom-implementations) -- [Contributions](#contributions) - ## Installation Add `:poolex` to your list of dependencies in `mix.exs`: @@ -56,7 +49,7 @@ Add `:poolex` to your list of dependencies in `mix.exs`: ```elixir def deps do [ - {:poolex, "~> 0.7.0"} + {:poolex, "~> 0.9.0"} ] end ``` @@ -85,6 +78,25 @@ iex> Poolex.run(:worker_pool, &(is_pid?(&1)), checkout_timeout: 1_000) A detailed description of the available configuration or examples of use can be found in [documentation](https://hexdocs.pm/poolex/getting-started.html). +## Guides + +- [Getting Started](https://hexdocs.pm/poolex/getting-started.html) + - [Starting pool of workers](https://hexdocs.pm/poolex/getting-started.html#starting-pool-of-workers) + - [Poolex configuration options](https://hexdocs.pm/poolex/getting-started.html#starting-pool-of-workers) + - [Working with the pool](https://hexdocs.pm/poolex/getting-started.html#working-with-the-pool) +- [Migration from `:poolboy`](https://hexdocs.pm/poolex/migration-from-poolboy.html) +- [Example of use](https://hexdocs.pm/poolex/example-of-use.html) + - [Defining the worker](https://hexdocs.pm/poolex/example-of-use.html#defining-the-worker) + - [Configuring Poolex](https://hexdocs.pm/poolex/example-of-use.html#configuring-poolex) + - [Using Poolex](https://hexdocs.pm/poolex/example-of-use.html#using-poolex) +- [Working with metrics](https://hexdocs.pm/poolex/pool-metrics.html) + - [Pool size metrics](https://hexdocs.pm/poolex/pool-metrics.html#pool-size-metrics) + - [Integration with PromEx](https://hexdocs.pm/poolex/pool-metrics.html#integration-with-promex) +- [Workers and callers implementations](https://hexdocs.pm/poolex/workers-and-callers-implementations.html) + - [Callers](https://hexdocs.pm/poolex/workers-and-callers-implementations.html#callers) + - [Workers](https://hexdocs.pm/poolex/workers-and-callers-implementations.html#workers) + - [Writing custom implementations](https://hexdocs.pm/poolex/workers-and-callers-implementations.html#writing-custom-implementations) + ## Contributions If you feel something can be improved or have any questions about specific behaviors or pieces of implementation, please feel free to file an issue. Proposed changes should be taken to issues before any PRs to save time on code that might not be merged upstream. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..afe52fa --- /dev/null +++ b/TODO.md @@ -0,0 +1,21 @@ +# Ideas for implementing + +## Pool Metrics + +I want to make a simple way to analyze running pools to set their optimal configuration. For example, we launched a `pool` in production with a maximum overflow of 0 (we do not want to create more processes than the designated number) and a pool size 200. + +Using metrics, we see that typically, our application uses 10-20 processes, and there are spikes when up to 180 workers are exploited. If our processes are heavyweight and, for example, open persistent connections to storage, then by analyzing metrics, we can significantly save resources. In this case, we can set the pool size to 20 and `max_overflow` to 180. This way, we will have one overall pool size limit of 200, and we will avoid uncontrolled waste of all resources, but at the same time, we will only keep up to 20 processes in memory at times when this is not required. + +### Metrics to be implemented + +- [x] Pool size + - [x] Idle workers count + - [x] Busy workers count + - [x] Is max_overflow used? +- [ ] Usage time + - [ ] How long are workers busy? + - [ ] How long the application waits of workers from pool? + +## Implementations metrics + +To be described... diff --git a/docs/guides/getting-started.cheatmd b/docs/guides/getting-started.cheatmd index 28e24d5..5a44e27 100644 --- a/docs/guides/getting-started.cheatmd +++ b/docs/guides/getting-started.cheatmd @@ -27,16 +27,17 @@ The second argument should contain a set of options for starting the pool. ## Poolex configuration options | Option | Description | Example | Default value | -|------------------------|------------------------------------------------------|-----------------------|-----------------------------------| -| `pool_id` | Identifier by which you will access the pool | `:my_pool` | **option is required** | -| `worker_module` | Name of module that implements our worker | `MyApp.Worker` | **option is required** | -| `workers_count` | How many workers should be running in the pool | `5` | **option is required** | -| `max_overflow` | How many workers can be created over the limit | `2` | `0` | -| `worker_args` | List of arguments passed to the start function | `[:gg, "wp"]` | `[]` | -| `worker_start_fun` | Name of the function that starts the worker | `:run` | `:start_link` | -| `busy_workers_impl` | Module that describes how to work with busy workers | `SomeBusyWorkersImpl` | `Poolex.Workers.Impl.List` | -| `idle_workers_impl` | Module that describes how to work with idle workers | `SomeIdleWorkersImpl` | `Poolex.Workers.Impl.List` | -| `waiting_callers_impl` | Module that describes how to work with callers queue | `WaitingCallersImpl` | `Poolex.Callers.Impl.ErlangQueue` | +|------------------------|------------------------------------------------------|-----------------------|-------------------------| +| `pool_id` | Identifier by which you will access the pool | `:my_pool` | **option is required** | +| `worker_module` | Name of module that implements our worker | `MyApp.Worker` | **option is required** | +| `workers_count` | How many workers should be running in the pool | `5` | **option is required** | +| `max_overflow` | How many workers can be created over the limit | `2` | `0` | +| `worker_args` | List of arguments passed to the start function | `[:gg, "wp"]` | `[]` | +| `worker_start_fun` | Name of the function that starts the worker | `:run` | `:start_link` | +| `busy_workers_impl` | Module that describes how to work with busy workers | `SomeBusyWorkersImpl` | `Poolex.Workers.Impl.List` | +| `idle_workers_impl` | Module that describes how to work with idle workers | `SomeIdleWorkersImpl` | `Poolex.Workers.Impl.List` | +| `waiting_callers_impl` | Module that describes how to work with callers queue | `WaitingCallersImpl` | `Poolex.Callers.Impl.ErlangQueue` | +| `pool_size_metrics` | Whether to dispatch pool size metrics | `true` | `false` | ## Working with the pool diff --git a/docs/guides/migration-from-poolboy.cheatmd b/docs/guides/migration-from-poolboy.cheatmd index 964dd65..901f689 100644 --- a/docs/guides/migration-from-poolboy.cheatmd +++ b/docs/guides/migration-from-poolboy.cheatmd @@ -12,7 +12,7 @@ If you are using `:poolboy` and want to use `Poolex` instead, then you need to f defp deps do [ - {:poolboy, "~> 1.5.0"} -+ {:poolex, "~> 0.7.0"} ++ {:poolex, "~> 0.9.0"} ] end ``` diff --git a/docs/guides/pool-metrics.cheatmd b/docs/guides/pool-metrics.cheatmd new file mode 100644 index 0000000..ec4db58 --- /dev/null +++ b/docs/guides/pool-metrics.cheatmd @@ -0,0 +1,30 @@ +# Working with metrics + +You can quickly analyze and optimize your pool's production settings with the metrics presented by the library. + +## Pool size metrics + +The Poolex library presents **an idle/busy worker count metric**. These metrics help estimate a pool load and the number of workers used. + +Also, there is **an overflow metric**. It shows how long pools are forced to use additional workers. + +You can handle them by using `:telemetry.attach/4`: + +```elixir +:telemetry.attach( + "my-lovely-pool-size-metrics", + [:poolex, :metrics, :pool_size], + &MyApp.handle_event/4, + nil +) +``` + +For example, your application can write metrics to the console: [PoolexExample.MetricsHandler](https://github.com/general-CbIC/poolex/blob/develop/examples/poolex_example/lib/poolex_example/metrics_handler.ex). + +More about using `telemetry` [here](https://hexdocs.pm/telemetry/readme.html). + +## Integration with PromEx + +There is a plugin that works with the [PromEx](https://github.com/akoutmos/prom_ex) library: [Poolex.PromEx](https://hex.pm/packages/poolex_prom_ex). + +About installation of this plugin you can read [here](https://hexdocs.pm/poolex_prom_ex/readme.html#installation). diff --git a/examples/poolex_example/lib/poolex_example/application.ex b/examples/poolex_example/lib/poolex_example/application.ex index da62249..273a6c4 100644 --- a/examples/poolex_example/lib/poolex_example/application.ex +++ b/examples/poolex_example/lib/poolex_example/application.ex @@ -9,9 +9,17 @@ defmodule PoolexExample.Application do pool_id: :worker_pool, worker_module: PoolexExample.Worker, workers_count: 5, - max_overflow: 2} + max_overflow: 2, + pool_size_metrics: true} ] + :telemetry.attach( + "poolex_metrics", + [:poolex, :metrics, :pool_size], + &PoolexExample.MetricsHandler.handle_event/4, + nil + ) + Supervisor.start_link(children, strategy: :one_for_one) end end diff --git a/examples/poolex_example/lib/poolex_example/metrics_handler.ex b/examples/poolex_example/lib/poolex_example/metrics_handler.ex new file mode 100644 index 0000000..011c442 --- /dev/null +++ b/examples/poolex_example/lib/poolex_example/metrics_handler.ex @@ -0,0 +1,10 @@ +defmodule PoolexExample.MetricsHandler do + def handle_event([:poolex, :metrics, :pool_size], measurements, metadata, _config) do + IO.puts(""" + [Pool: #{metadata.pool_id}]: + - Idle workers: #{measurements.idle_workers_count} + - Busy workers: #{measurements.busy_workers_count} + - Overflowed: #{measurements.overflowed} + """) + end +end diff --git a/examples/poolex_example/mix.exs b/examples/poolex_example/mix.exs index 47b8184..de68ae8 100644 --- a/examples/poolex_example/mix.exs +++ b/examples/poolex_example/mix.exs @@ -21,8 +21,9 @@ defmodule PoolexExample.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ + {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:poolex, path: "../.."}, - {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false} + {:telemetry, "~> 1.0"} ] end end diff --git a/examples/poolex_example/mix.lock b/examples/poolex_example/mix.lock index 853661a..4a5eae8 100644 --- a/examples/poolex_example/mix.lock +++ b/examples/poolex_example/mix.lock @@ -1,5 +1,7 @@ %{ - "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"}, + "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "poolex": {:hex, :poolex, "0.4.0", "cc3c5eb921a2ea5886953a2ea3beb0e36148d7d91dd9b628df74c3abb5ea7981", [:mix], [], "hexpm", "0441bf433f8fca4127b68c5985354cb772bba049057bca842f6e73df4f631c17"}, + "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, + "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, } diff --git a/lib/poolex.ex b/lib/poolex.ex index ccffa6c..f326ebd 100644 --- a/lib/poolex.ex +++ b/lib/poolex.ex @@ -30,6 +30,7 @@ defmodule Poolex do alias Poolex.Private.BusyWorkers alias Poolex.Private.DebugInfo alias Poolex.Private.IdleWorkers + alias Poolex.Private.Metrics alias Poolex.Private.Monitoring alias Poolex.Private.State alias Poolex.Private.WaitingCallers @@ -47,6 +48,7 @@ defmodule Poolex do | `busy_workers_impl` | Module that describes how to work with busy workers | `SomeBusyWorkersImpl` | `Poolex.Workers.Impl.List` | | `idle_workers_impl` | Module that describes how to work with idle workers | `SomeIdleWorkersImpl` | `Poolex.Workers.Impl.List` | | `waiting_callers_impl` | Module that describes how to work with callers queue | `WaitingCallersImpl` | `Poolex.Callers.Impl.ErlangQueue` | + | `pool_size_metrics` | Whether to dispatch pool size metrics | `true` | `false` | """ @typedoc """ @@ -66,6 +68,7 @@ defmodule Poolex do | {:busy_workers_impl, module()} | {:idle_workers_impl, module()} | {:waiting_callers_impl, module()} + | {:pool_size_metrics, boolean()} @typedoc """ Process id of `worker`. @@ -290,7 +293,14 @@ defmodule Poolex do |> BusyWorkers.init(busy_workers_impl) |> WaitingCallers.init(waiting_callers_impl) - {:ok, state} + {:ok, state, {:continue, opts}} + end + + @impl GenServer + def handle_continue(opts, state) do + Metrics.start_poller(opts) + + {:noreply, state} end @spec start_workers(non_neg_integer(), State.t(), Monitoring.monitor_id()) :: [pid] diff --git a/lib/poolex/private/metrics.ex b/lib/poolex/private/metrics.ex new file mode 100644 index 0000000..ecaf656 --- /dev/null +++ b/lib/poolex/private/metrics.ex @@ -0,0 +1,59 @@ +defmodule Poolex.Private.Metrics do + @moduledoc """ + Functions for dispatching metrics. + """ + + @doc """ + Dispatches metrics with current count of idle workers. + """ + @spec dispatch_pool_size_metrics(Poolex.pool_id()) :: :ok + def dispatch_pool_size_metrics(pool_id) do + debug_info = Poolex.get_debug_info(pool_id) + + :telemetry.execute( + [:poolex, :metrics, :pool_size], + %{ + idle_workers_count: debug_info.idle_workers_count, + busy_workers_count: debug_info.busy_workers_count, + overflowed: convert_overflow_to_number(debug_info.overflow > 0) + }, + %{pool_id: pool_id} + ) + end + + @spec convert_overflow_to_number(boolean()) :: integer() + defp convert_overflow_to_number(true), do: 1 + defp convert_overflow_to_number(false), do: 0 + + @doc """ + Starts a telemetry poller for dispatching metrics. + """ + @spec start_poller(list(Poolex.poolex_option())) :: GenServer.on_start() + def start_poller(opts) do + pool_id = Keyword.fetch!(opts, :pool_id) + measurements = collect_measurements(opts) + + if measurements == [] do + :ok + else + :telemetry_poller.start_link( + measurements: measurements, + period: :timer.seconds(1), + name: :"#{pool_id}_metrics_poller" + ) + end + end + + @spec collect_measurements(list(Poolex.poolex_option())) :: list() + defp collect_measurements(opts) do + pool_id = Keyword.fetch!(opts, :pool_id) + + if Keyword.get(opts, :pool_size_metrics, false) do + [ + {Poolex.Private.Metrics, :dispatch_pool_size_metrics, [pool_id]} + ] + else + [] + end + end +end diff --git a/mix.exs b/mix.exs index f558a12..8226069 100644 --- a/mix.exs +++ b/mix.exs @@ -15,7 +15,7 @@ defmodule Poolex.MixProject do package: package(), source_url: "https://github.com/general-CbIC/poolex", start_permanent: Mix.env() == :prod, - version: "0.8.0" + version: "0.9.0" ] end @@ -30,10 +30,13 @@ defmodule Poolex.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ + {:telemetry, "~> 1.0"}, + {:telemetry_poller, "~> 1.0"}, + # Development dependencies {:credo, ">= 0.0.0", only: [:dev], runtime: false}, {:dialyxir, ">= 0.0.0", only: [:dev], runtime: false}, {:doctor, ">= 0.0.0", only: [:dev], runtime: false}, - {:ex_check, "~> 0.15.0", only: [:dev], runtime: false}, + {:ex_check, "~> 0.16.0", only: [:dev], runtime: false}, {:ex_doc, ">= 0.0.0", only: [:dev], runtime: false}, {:makeup_diff, ">= 0.0.0", only: [:dev], runtime: false} ] @@ -54,12 +57,13 @@ defmodule Poolex.MixProject do [ main: "readme", extras: [ - "docs/CONTRIBUTING.md", - "docs/guides/workers-and-callers-implementations.md", + "README.md", + "docs/guides/getting-started.cheatmd", "docs/guides/migration-from-poolboy.cheatmd", "docs/guides/example-of-use.cheatmd", - "docs/guides/getting-started.cheatmd", - "README.md" + "docs/guides/pool-metrics.cheatmd", + "docs/guides/workers-and-callers-implementations.md", + "docs/CONTRIBUTING.md" ], groups_for_extras: [ Guides: Path.wildcard("docs/guides/*md") diff --git a/mix.lock b/mix.lock index 507aa32..76773dd 100644 --- a/mix.lock +++ b/mix.lock @@ -1,18 +1,20 @@ %{ - "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, - "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"}, - "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, - "dialyxir": {:hex, :dialyxir, "1.4.0", "6b698401c16de79e8596b73dca63762255e70e4bbe26423530e173917220d5fc", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "c7ecaa1da27debae488ab09d9827ec58a0161c7821972b6d2cb26c1614648849"}, + "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, + "credo": {:hex, :credo, "1.7.5", "643213503b1c766ec0496d828c90c424471ea54da77c8a168c725686377b9545", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f799e9b5cd1891577d8c773d245668aa74a2fcd15eb277f51a0131690ebfb3fd"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "doctor": {:hex, :doctor, "0.21.0", "20ef89355c67778e206225fe74913e96141c4d001cb04efdeba1a2a9704f1ab5", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "a227831daa79784eb24cdeedfa403c46a4cb7d0eab0e31232ec654314447e4e0"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "ex_check": {:hex, :ex_check, "0.15.0", "074b94c02de11c37bba1ca82ae5cc4926e6ccee862e57a485b6ba60fca2d8dc1", [:mix], [], "hexpm", "33848031a0c7e4209c3b4369ce154019788b5219956220c35ca5474299fb6a0e"}, - "ex_doc": {:hex, :ex_doc, "0.30.6", "5f8b54854b240a2b55c9734c4b1d0dd7bdd41f71a095d42a70445c03cf05a281", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bd48f2ddacf4e482c727f9293d9498e0881597eae6ddc3d9562bd7923375109f"}, - "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, - "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, - "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, + "ex_check": {:hex, :ex_check, "0.16.0", "07615bef493c5b8d12d5119de3914274277299c6483989e52b0f6b8358a26b5f", [:mix], [], "hexpm", "4d809b72a18d405514dda4809257d8e665ae7cf37a7aee3be6b74a34dec310f5"}, + "ex_doc": {:hex, :ex_doc, "0.32.1", "21e40f939515373bcdc9cffe65f3b3543f05015ac6c3d01d991874129d173420", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5142c9db521f106d61ff33250f779807ed2a88620e472ac95dc7d59c380113da"}, + "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, + "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, "makeup_diff": {:hex, :makeup_diff, "0.1.0", "5be352b6aa6f07fa6a236e3efd7ba689a03f28fb5d35b7a0fa0a1e4a64f6d8bb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "186bad5bb433a8afeb16b01423950e440072284a4103034ca899180343b9b4ac"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, - "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, + "makeup_erlang": {:hex, :makeup_erlang, "0.1.5", "e0ff5a7c708dda34311f7522a8758e23bfcd7d8d8068dc312b5eb41c6fd76eba", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "94d2e986428585a21516d7d7149781480013c56e30c6a233534bedf38867a59a"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, + "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, } diff --git a/test/poolex_metrics_test.exs b/test/poolex_metrics_test.exs new file mode 100644 index 0000000..fde45a6 --- /dev/null +++ b/test/poolex_metrics_test.exs @@ -0,0 +1,46 @@ +defmodule PoolexMetricsTest do + use Poolex.MetricsCase + + import PoolHelpers + + alias Poolex.Private.Metrics + + @tag telemetry_events: [[:poolex, :metrics, :pool_size]] + test "pool size metrics" do + pool_id = + start_pool( + worker_module: SomeWorker, + workers_count: 5, + pool_size_metrics: true, + max_overflow: 5 + ) + + assert_telemetry_event( + [:poolex, :metrics, :pool_size], + %{idle_workers_count: 5, busy_workers_count: 0, overflowed: 0}, + %{pool_id: ^pool_id} + ) + + launch_long_task(pool_id) + + Metrics.dispatch_pool_size_metrics(pool_id) + + assert_telemetry_event( + [:poolex, :metrics, :pool_size], + %{idle_workers_count: 4, busy_workers_count: 1, overflowed: 0}, + %{pool_id: ^pool_id} + ) + + Enum.each(1..5, fn _ -> + launch_long_task(pool_id) + end) + + Metrics.dispatch_pool_size_metrics(pool_id) + + assert_telemetry_event( + [:poolex, :metrics, :pool_size], + %{idle_workers_count: 0, busy_workers_count: 6, overflowed: 1}, + %{pool_id: ^pool_id} + ) + end +end diff --git a/test/poolex_test.exs b/test/poolex_test.exs index dd7fab2..ee467bd 100644 --- a/test/poolex_test.exs +++ b/test/poolex_test.exs @@ -1,5 +1,7 @@ defmodule PoolexTest do use ExUnit.Case, async: false + import PoolHelpers + doctest Poolex describe "debug info" do @@ -580,35 +582,4 @@ defmodule PoolexTest do assert elem(message_3, 3) == pool_pid end end - - defp start_pool(options) do - pool_name = - 1..10 - |> Enum.map(fn _ -> Enum.random(?a..?z) end) - |> to_string() - |> String.to_atom() - - options = Keyword.put(options, :pool_id, pool_name) - {:ok, _pid} = start_supervised({Poolex, options}) - - pool_name - end - - defp launch_long_task(pool_id, delay \\ :timer.seconds(4)) do - launch_long_tasks(pool_id, 1, delay) - end - - defp launch_long_tasks(pool_id, count, delay \\ :timer.seconds(4)) do - for _i <- 1..count do - spawn(fn -> - Poolex.run( - pool_id, - fn pid -> GenServer.call(pid, {:do_some_work_with_delay, delay}) end, - checkout_timeout: 100 - ) - end) - end - - :timer.sleep(10) - end end diff --git a/test/support/metrics_case.ex b/test/support/metrics_case.ex new file mode 100644 index 0000000..b590a4a --- /dev/null +++ b/test/support/metrics_case.ex @@ -0,0 +1,107 @@ +defmodule Poolex.MetricsCase do + @moduledoc """ + This module defines the setup for tests requiring metrics tests. + + Available tags: + - `telemetry_events`: list the Telemetry events to listen + - `metrics`: Specify the list of Telemetry.Metrics to used (format: [Module, :function, [args]]) + + Available assertions: + - `assert_telemetry_event(name, measurements, metadata \\ %{})` + - `assert_metric(name, measurement, metadata \\ %{})` + + ## Example: + + @tag telemetry_events: [[:user, :subscription, :email_confirmation]], + metrics: [MayApp.Metrics, :metrics, []] + test "my test" do + ... + assert_telemetry_event([:user, :subscription, :email_confirmation], %{count: 1}, %{result: :error}) + assert_metric([:user, :subscription, :email_confirmation, :count], 1, %{success: false}) + end + + """ + use ExUnit.CaseTemplate + + using do + quote do + import Poolex.MetricsCase + end + end + + setup tags do + if telemetry_events = tags[:telemetry_events] do + metrics = get_metrics_from_tag(tags) + + self = self() + + groups = Enum.group_by(metrics, & &1.event_name) + + :telemetry.attach_many( + tags[:test], + telemetry_events, + fn name, measurements, metadata, _config -> + send(self, {:telemetry_event, name, measurements, metadata}) + + # Send related metrics + if Enum.count(metrics) > 0 do + Enum.each(Map.get(groups, name, []), fn metric -> + send( + self, + {:metric, metric.name, Map.get(measurements, metric.measurement), + extract_tags(metric, metadata)} + ) + end) + end + end, + nil + ) + end + + :ok + end + + defp extract_tags(metric, metadata) do + tag_values = metric.tag_values.(metadata) + Map.take(tag_values, metric.tags) + end + + defp get_metrics_from_tag(%{metrics: [m, f, args]}) do + apply(m, f, args) + end + + defp get_metrics_from_tag(_) do + [] + end + + @doc """ + Assert that given event has been sent. + """ + defmacro assert_telemetry_event(name, measurements, metadata \\ %{}), + do: do_assert_telemetry_event(name, measurements, metadata) + + defp do_assert_telemetry_event(name, measurements, %{}) do + do_assert_telemetry_event(name, measurements, Macro.escape(%{})) + end + + defp do_assert_telemetry_event(name, measurements, metadata) do + do_assert_receive(:telemetry_event, name, measurements, metadata) + end + + defmacro assert_metric(name, measurement, metadata \\ %{}), + do: do_assert_metric(name, measurement, metadata) + + defp do_assert_metric(name, measurement, %{}) do + do_assert_metric(name, measurement, Macro.escape(%{})) + end + + defp do_assert_metric(name, measurement, metadata) do + do_assert_receive(:metric, name, measurement, metadata) + end + + defp do_assert_receive(msg_type, name, measurement, metadata) do + quote do + assert_receive {unquote(msg_type), unquote(name), unquote(measurement), unquote(metadata)} + end + end +end diff --git a/test/support/pool_helpers.ex b/test/support/pool_helpers.ex new file mode 100644 index 0000000..8fcd5ce --- /dev/null +++ b/test/support/pool_helpers.ex @@ -0,0 +1,39 @@ +defmodule PoolHelpers do + @moduledoc """ + Module with helpers functions for launching pools and long tasks. + """ + + @spec start_pool(list(Poolex.poolex_option())) :: Poolex.pool_id() + def start_pool(options) do + pool_name = + 1..10 + |> Enum.map(fn _ -> Enum.random(?a..?z) end) + |> to_string() + |> String.to_atom() + + options = Keyword.put(options, :pool_id, pool_name) + {:ok, _pid} = ExUnit.Callbacks.start_supervised({Poolex, options}) + + pool_name + end + + @spec launch_long_task(Poolex.pool_id(), timeout()) :: :ok + def launch_long_task(pool_id, delay \\ :timer.seconds(4)) do + launch_long_tasks(pool_id, 1, delay) + end + + @spec launch_long_tasks(Poolex.pool_id(), non_neg_integer(), timeout()) :: :ok + def launch_long_tasks(pool_id, count, delay \\ :timer.seconds(4)) do + for _i <- 1..count do + spawn(fn -> + Poolex.run( + pool_id, + fn pid -> GenServer.call(pid, {:do_some_work_with_delay, delay}) end, + checkout_timeout: 100 + ) + end) + end + + :timer.sleep(10) + end +end