Significantly reduced number of requests

src/dipdup/indexes/evm_subsquid_events/index.py

@@ -152,14 +152,15 @@ async def _synchronize(self, sync_level: int) -> None:
                 typename = handler.contract.module_name
                 topics.add(self.topics[typename][handler.name])
 
-            # NOTE: Requesting logs by batches of NODE_BATCH_SIZE.
+            # NOTE: Requesting logs by batches of datasource_config.http.batch_size or NODE_BATCH_SIZE.
+            batch_size = self.random_node._http_config.batch_size
             batch_first_level = first_level
             while batch_first_level <= sync_level:
                 # NOTE: We need block timestamps for each level, so fetch them separately and match with logs.
                 timestamps: dict[int, int] = {}
                 tasks: deque[asyncio.Task[Any]] = deque()
 
-                batch_last_level = min(batch_first_level + NODE_BATCH_SIZE, sync_level)
+                batch_last_level = min(batch_first_level + batch_size, sync_level)
                 level_logs_task = asyncio.create_task(
                     self.random_node.get_logs(
                         {
@@ -168,13 +169,14 @@ async def _synchronize(self, sync_level: int) -> None:
                         }
                     )
                 )
-                tasks.append(level_logs_task)
 
                 async def _fetch_timestamp(level: int, timestamps: dict[int, int]) -> None:
                     block = await self.random_node.get_block_by_level(level)
                     timestamps[level] = int(block['timestamp'], 16)
 
-                for level in range(batch_first_level, batch_last_level + 1):
+                level = batch_last_level
+                level_logs = await level_logs_task
+                for level in [int(log['blockNumber'], 16) for log in level_logs]:
                     tasks.append(
                         asyncio.create_task(
                             _fetch_timestamp(level, timestamps),
@@ -184,7 +186,6 @@ async def _fetch_timestamp(level: int, timestamps: dict[int, int]) -> None:
 
                 await asyncio.gather(*tasks)
 
-                level_logs = await level_logs_task
                 parsed_level_logs = tuple(
                     EvmNodeLogData.from_json(
                         log,