diff --git a/pdm.lock b/pdm.lock index 6ee1c7cf6..1ee9e943b 100644 --- a/pdm.lock +++ b/pdm.lock @@ -1486,15 +1486,15 @@ files = [ [[package]] name = "sentry-sdk" -version = "1.34.0" +version = "1.35.0" summary = "Python client for Sentry (https://sentry.io)" dependencies = [ "certifi", "urllib3>=1.26.11; python_version >= \"3.6\"", ] files = [ - {file = "sentry-sdk-1.34.0.tar.gz", hash = "sha256:e5d0d2b25931d88fa10986da59d941ac6037f742ab6ff2fce4143a27981d60c3"}, - {file = "sentry_sdk-1.34.0-py2.py3-none-any.whl", hash = "sha256:76dd087f38062ac6c1e30ed6feb533ee0037ff9e709974802db7b5dbf2e5db21"}, + {file = "sentry-sdk-1.35.0.tar.gz", hash = "sha256:04e392db9a0d59bd49a51b9e3a92410ac5867556820465057c2ef89a38e953e9"}, + {file = "sentry_sdk-1.35.0-py2.py3-none-any.whl", hash = "sha256:a7865952701e46d38b41315c16c075367675c48d049b90a4cc2e41991ebc7efa"}, ] [[package]] diff --git a/requirements.dev.txt b/requirements.dev.txt index b2a97cf34..b3a4e784d 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -95,7 +95,7 @@ rlp==3.0.0 ruamel-yaml==0.18.5 ruamel-yaml-clib==0.2.7 ruff==0.1.5 -sentry-sdk==1.34.0 +sentry-sdk==1.35.0 setuptools==68.2.2 six==1.16.0 sniffio==1.3.0 diff --git a/requirements.txt b/requirements.txt index 0df59921e..beed8f11c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,7 +77,7 @@ rfc3339-validator==0.1.4 rlp==3.0.0 ruamel-yaml==0.18.5 ruamel-yaml-clib==0.2.7 -sentry-sdk==1.34.0 +sentry-sdk==1.35.0 setuptools==68.2.2 six==1.16.0 sniffio==1.3.0 diff --git a/src/dipdup/codegen/__init__.py b/src/dipdup/codegen/__init__.py index 38d305cfa..6be8087ad 100644 --- a/src/dipdup/codegen/__init__.py +++ b/src/dipdup/codegen/__init__.py @@ -72,8 +72,6 @@ async def init( await self.generate_system_hooks() await self.generate_handlers() - # self._package.verify() - @abstractmethod async def generate_abi(self) -> None: ... diff --git a/src/dipdup/datasources/evm_subsquid.py b/src/dipdup/datasources/evm_subsquid.py index e51ed26c1..f34043b3a 100644 --- a/src/dipdup/datasources/evm_subsquid.py +++ b/src/dipdup/datasources/evm_subsquid.py @@ -26,8 +26,6 @@ 'address': True, 'data': True, 'topics': True, - # 'blockNumber': True, - # 'blockHash': True, } @@ -68,7 +66,7 @@ async def iter_event_logs( ) -> AsyncIterator[tuple[SubsquidEventData, ...]]: current_level = first_level - # TODO: smarter query optimizator + # TODO: Smarter query optimizator topics_by_address = defaultdict(list) for address, topic in topics: topics_by_address[address].append(topic) diff --git a/src/dipdup/datasources/tezos_tzkt.py b/src/dipdup/datasources/tezos_tzkt.py index 72bbebcf3..702925633 100644 --- a/src/dipdup/datasources/tezos_tzkt.py +++ b/src/dipdup/datasources/tezos_tzkt.py @@ -1057,7 +1057,7 @@ def _get_request_params( offset: int | None = None, limit: int | None = None, select: tuple[str, ...] | None = None, - values: bool = False, # return only list of chosen values instead of dict + values: bool = False, cursor: bool = False, sort: str | None = None, **kwargs: Any, @@ -1074,8 +1074,8 @@ def _get_request_params( params['offset.cr'] = offset else: params['offset'] = offset + # NOTE: If `values` is set request will return list of lists instead of list of dicts. if select: - # filter fields params['select.values' if values else 'select'] = ','.join(select) if sort: if sort.startswith('-'): diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index acff12dbd..a7710a2e5 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -542,7 +542,7 @@ async def run(self) -> None: advanced = self._config.advanced tasks: set[Task[None]] = set() - # verify before start so obvious mistakes can be seen instantly + # NOTE: Verify package before indexing to ensure that all modules are importable self._ctx.package.verify() async with AsyncExitStack() as stack: diff --git a/src/dipdup/indexes/evm_subsquid_events/index.py b/src/dipdup/indexes/evm_subsquid_events/index.py index abd35c6a6..2bf001900 100644 --- a/src/dipdup/indexes/evm_subsquid_events/index.py +++ b/src/dipdup/indexes/evm_subsquid_events/index.py @@ -160,34 +160,35 @@ async def _synchronize(self, sync_level: int) -> None: topics.add(self.topics[typename][handler.name]) # Requesting blocks info by batch - windows = ((i, min(i + NODE_BATCH_SIZE, sync_level)) for i in range(first_level, sync_level + 1, NODE_BATCH_SIZE + 1)) - for start_level, end_level in windows: - # NOTE: Get random one every time - # NOTE: Data for blocks start_level and end_level will be included + batch_first_level = first_level + while batch_first_level <= sync_level: + batch_last_level = min(batch_first_level + NODE_BATCH_SIZE, sync_level) level_logs = await self.random_node.get_logs( { - 'fromBlock': hex(start_level), - 'toBlock': hex(end_level), + 'fromBlock': hex(batch_first_level), + 'toBlock': hex(batch_last_level), } ) - # get timestamps for levels - timestamps = {} - for level in range(start_level, end_level + 1): + # NOTE: We need block timestamps for each level, so fetch them separately and match with logs. + timestamps: dict[int, int] = {} + for level in range(batch_first_level, batch_last_level + 1): block = await self.random_node.get_block_by_level(level) - try: - timestamps[hex(level)] = int(block['timestamp'], 16) - except TypeError as e: - raise FrameworkException(f'Block {level} not found') from e - - # match timestamps with logs - parsed_level_logs = tuple(EvmNodeLogData.from_json(log, timestamps[log['blockNumber']]) - for log in level_logs) - await self._process_level_events(parsed_level_logs, self.topics, sync_level) + timestamps[level] = int(block['timestamp'], 16) + + parsed_level_logs = tuple( + EvmNodeLogData.from_json( + log, + timestamps[int(log['blockNumber'], 16)], + ) + for log in level_logs + ) await self._process_level_events(parsed_level_logs, sync_level) if self._config.expose_metrics: Metrics.set_sqd_processor_last_block(level) + + batch_first_level = batch_last_level + 1 else: sync_level = min(sync_level, subsquid_sync_level) fetcher = self._create_fetcher(first_level, sync_level) diff --git a/src/dipdup/models/tezos_tzkt.py b/src/dipdup/models/tezos_tzkt.py index b63a9679a..37836a6c1 100644 --- a/src/dipdup/models/tezos_tzkt.py +++ b/src/dipdup/models/tezos_tzkt.py @@ -562,14 +562,11 @@ class TzktTokenBalanceData(HasLevel): transfers_count: int first_level: int first_time: datetime - # level is not defined in tzkt balances data, so it is - # Level of the block where the token balance was last changed. + # NOTE: Level of the block where the token balance has been changed for the last time. last_level: int last_time: datetime - # owner account account_address: str | None = None account_alias: str | None = None - # token object tzkt_token_id: int | None = None contract_address: str | None = None contract_alias: str | None = None diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 33b40d1b6..40f3c1b49 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -21,6 +21,7 @@ _logger = logging.getLogger(__name__) +# NOTE: All templates are stored in src/dipdup/projects TEMPLATES: dict[str, tuple[str, ...]] = { 'evm': ( 'demo_evm_events', @@ -147,8 +148,6 @@ def answers_from_terminal() -> Answers: options.append(_answers['template']) comments.append(_answers['description']) - # list of options can contain folder name of template or folder name of template with description - # all project templates are in src/dipdup/projects _, template = prompt_anyof( 'Choose a project template:', options=tuple(options), @@ -183,7 +182,6 @@ def answers_from_terminal() -> Answers: value=answers['description'], ) - # define author and license for new indexer answers['license'] = survey.routines.input( 'Enter project license (DipDup itself is MIT-licensed.): ', value=answers['license'], @@ -266,7 +264,7 @@ def render_base( include: set[str] | None = None, ) -> None: """Render base from template""" - # NOTE: Common base + # NOTE: Render common base _render_templates( answers=answers, path=Path('base'), @@ -274,9 +272,9 @@ def render_base( include=include, exists=True, ) - + # NOTE: Don't forget to update replay.yaml with new values _render( - answers, + answers=answers, template_path=Path(__file__).parent / 'templates' / 'replay.yaml.j2', output_path=Path('configs') / 'replay.yaml', force=force, diff --git a/src/dipdup/prometheus.py b/src/dipdup/prometheus.py index 87121eed6..82bae320b 100644 --- a/src/dipdup/prometheus.py +++ b/src/dipdup/prometheus.py @@ -61,10 +61,7 @@ 'Number of http errors', ['url', 'status'], ) -_http_errors_in_row = Histogram( - 'dipdup_http_errors_in_row', - """The number of consecutive failed requests""" -) +_http_errors_in_row = Histogram('dipdup_http_errors_in_row', """The number of consecutive failed requests""") _callback_duration = Histogram( 'dipdup_callback_duration_seconds', 'Duration of callback execution', @@ -80,8 +77,7 @@ 'Current chain height as reported by the archive', ) _sqd_processor_archive_http_errors_in_row = Histogram( - 'sqd_processor_archive_http_errors_in_row', - """The number of consecutive failed Archive requests""" + 'sqd_processor_archive_http_errors_in_row', """The number of consecutive failed Archive requests""" ) @@ -154,7 +150,7 @@ def set_levels_to_realtime(cls, index: str, levels: int) -> None: @classmethod def set_sqd_processor_last_block(cls, last_block: int) -> None: _sqd_processor_last_block.set(last_block) - + @classmethod def set_sqd_processor_chain_height(cls, chain_height: int) -> None: _sqd_processor_chain_height.set(chain_height)