Skip to content

Commit

Permalink
add max pool to boto3 client
Browse files Browse the repository at this point in the history
  • Loading branch information
vincentclaes committed May 5, 2022
1 parent 372d9db commit bb9a9f2
Showing 1 changed file with 11 additions and 3 deletions.
14 changes: 11 additions & 3 deletions stepview/data.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import concurrent.futures
import boto3
import botocore.client
import pendulum

from rich.table import Table
Expand Down Expand Up @@ -59,6 +60,7 @@ class Time:


NOW = pendulum.now()
MAX_POOL_CONNECTIONS = 100

PERIODS_MAPPING = {
Time.MINUTE: Periods(NOW.subtract(minutes=1), NOW, "microseconds"),
Expand Down Expand Up @@ -140,9 +142,15 @@ def run_for_state_machine(


def run_for_profile(profile_name: str, period: str) -> Table:
sfn_client = boto3.Session(profile_name=profile_name).client("stepfunctions")
sfn_client = boto3.Session(
profile_name=profile_name
).client(
"stepfunctions",
config=botocore.client.Config(max_pool_connections=MAX_POOL_CONNECTIONS)
)
cloudwatch_resource = boto3.Session(profile_name=profile_name).resource(
"cloudwatch"
"cloudwatch",
config=botocore.client.Config(max_pool_connections=MAX_POOL_CONNECTIONS)
)
state_machines = sfn_client.list_state_machines().get("stateMachines")
if state_machines:
Expand All @@ -156,7 +164,7 @@ def _run_for_state_machine(state_machine):
)

with concurrent.futures.ThreadPoolExecutor(
min(len(state_machines), 100)
min(len(state_machines), MAX_POOL_CONNECTIONS)
) as thread:
state_machine_generator = thread.map(_run_for_state_machine, state_machines)
return state_machine_generator
Expand Down

0 comments on commit bb9a9f2

Please sign in to comment.