From 920d02effcd1800c061b72544c7cc8083beed864 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskii Date: Fri, 10 Jan 2025 20:48:20 -0300 Subject: [PATCH] Fix parsing EVM tuples (#1182) --- CHANGELOG.md | 7 +++- docs/16.thanks.md | 8 ++++ docs/7.references/2.config.md | 20 +--------- docs/9.release-notes/_8.0_changelog.md | 5 +++ docs/config.rst | 2 - src/dipdup/abi/evm.py | 38 +++++++++++++++---- src/dipdup/indexes/evm_events/matcher.py | 9 ++++- .../indexes/evm_transactions/matcher.py | 4 +- src/dipdup/utils.py | 15 +++++++- 9 files changed, 74 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b16bdb13..5c6cb54b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,11 @@ Releases prior to 7.0 has been removed from this file to declutter search result ### Fixed -- subsquid: Fixed float type for `timestamp` field on event / transaction deserialization. -- subsquid: Fixed empty field base conversion on event deserialization. +- evm: Fixed sending JSONRPC requests via web3.py provider. +- evm: Fixed parsing tuple types in ABI. +- evm.subsquid: Fixed type of `timestamp` field of event/transaction models. +- evm.subsquid: Fixed empty field base conversion on event deserialization. +- starknet: Fixed parsing contract addresses starting with `0x0`. ## [8.1.3] - 2024-12-20 diff --git a/docs/16.thanks.md b/docs/16.thanks.md index 72293f571..ac7f5d040 100644 --- a/docs/16.thanks.md +++ b/docs/16.thanks.md @@ -31,7 +31,9 @@ We are grateful to all the people who helped us with the project. - [Florian PAUTOT](https://github.com/0x666c6f) - [gdsoumya](https://github.com/gdsoumya) - [Göran Sandström](https://github.com/veqtor) +- [hoka](https://github.com/hokaxbt) - [Igor Sereda](https://github.com/igorsereda) +- [Ilia Batii](https://github.com/baitcode) - [Javier Graciá Carpio](https://github.com/jagracar) - [JoE11-y](https://github.com/JoE11-y) - [Karan Dua](https://github.com/Karantezsure) @@ -49,4 +51,10 @@ We are grateful to all the people who helped us with the project. - [Soham Das](https://github.com/tosoham) - [tomsib2001](https://github.com/tomsib2001) +Also, these people helped heavily with [pysignalr](https://github.com/baking-bad/), a library we have developed to use in DipDup: + +- [Caio Barbieri](https://github.com/caiolombello) +- [MichaelMKKelly](https://github.com/MichaelMKKelly) +- [Ola Lidholm](https://github.com/olalid) + If we forgot to mention you, or you want to update your record, please, open an issue or pull request. diff --git a/docs/7.references/2.config.md b/docs/7.references/2.config.md index 0d7ba7876..b26fd9361 100644 --- a/docs/7.references/2.config.md +++ b/docs/7.references/2.config.md @@ -140,24 +140,6 @@ description: "Config file reference"
-## dipdup.config.DatasourceConfig - -class dipdup.config.DatasourceConfig(*args) -

Base class for datasource configs

-
-
Parameters:
-
    -

  • kind – Defined by child class

    • -

  • url – URL of the API

    • -

  • http – HTTP connection tunables

    • - -
-
-
-
- -
- ## dipdup.config.evm.EvmContractConfig class dipdup.config.evm.EvmContractConfig(*args) @@ -439,7 +421,7 @@ description: "Config file reference"
-## id0 +## dipdup.config.DatasourceConfig class dipdup.config.DatasourceConfig(*args)

Base class for datasource configs

diff --git a/docs/9.release-notes/_8.0_changelog.md b/docs/9.release-notes/_8.0_changelog.md index f20cb17da..b14b83961 100644 --- a/docs/9.release-notes/_8.0_changelog.md +++ b/docs/9.release-notes/_8.0_changelog.md @@ -36,8 +36,12 @@ - database: Fixed concurrency issue when using `get_or_create` method. - evm.events: Fixed matching logs when filtering by topic0. - evm.events: Improve fetching event batches from node. +- evm.subsquid: Fixed empty field base conversion on event deserialization. +- evm.subsquid: Fixed type of `timestamp` field of event/transaction models. - evm.subsquid: Fixed typo in `iter_events` method name. - evm: Fixed crash when contract ABI contains overloaded methods. +- evm: Fixed parsing tuple types in ABI. +- evm: Fixed sending JSONRPC requests via web3.py provider. - install: Fixed reinstalling package when `--force` flag is used. - models: Fixed `CachedModel` preloading. - models: Fixed setting default value for `Meta.maxsize`. @@ -45,6 +49,7 @@ - performance: Add index name to fetcher and realtime queues. - performance: Fixed estimation indexing speed in levels per second. - starknet.events: Fixed filtering events by key. +- starknet: Fixed parsing contract addresses starting with `0x0`. - subsquid: Fixed missing entry in `dipdup_head` internal table. - tezos.big_maps: Fixed logging status message in `skip_history` mode. - tezos.big_maps: Respect order of handlers in `skip_history` mode. diff --git a/docs/config.rst b/docs/config.rst index 552bde5ef..269b2d28b 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -2,13 +2,11 @@ .. autoclass:: dipdup.config.DipDupConfig -.. autoclass:: dipdup.config.AbiDatasourceConfig .. autoclass:: dipdup.config.abi_etherscan.AbiEtherscanDatasourceConfig .. autoclass:: dipdup.config.AdvancedConfig .. autoclass:: dipdup.config.ApiConfig .. autoclass:: dipdup.config.coinbase.CoinbaseDatasourceConfig .. autoclass:: dipdup.config.ContractConfig -.. autoclass:: dipdup.config.DatasourceConfig .. autoclass:: dipdup.config.evm.EvmContractConfig .. autoclass:: dipdup.config.evm_node.EvmNodeDatasourceConfig .. autoclass:: dipdup.config.evm_events.EvmEventsHandlerConfig diff --git a/src/dipdup/abi/evm.py b/src/dipdup/abi/evm.py index b4e4e3448..21f66fc2e 100644 --- a/src/dipdup/abi/evm.py +++ b/src/dipdup/abi/evm.py @@ -29,8 +29,6 @@ 'bytes': 'string', 'bool': 'boolean', 'string': 'string', - # TODO: arrays and tuples - # https://docs.soliditylang.org/en/develop/abi-spec.html#types 'tuple': 'object', } @@ -52,12 +50,25 @@ def _convert_name(name: str) -> str: def jsonschema_from_abi(abi: dict[str, Any]) -> dict[str, Any]: + properties, required = {}, [] + for item in abi['inputs']: + name = _convert_name(item['name']) + if item['type'] == 'tuple': + properties[name] = { + 'type': 'object', + 'properties': { + _convert_name(i['name']): {'type': _convert_type(i['type'])} for i in item['components'] + }, + } + else: + properties[name] = {'type': _convert_type(item['type'])} + required.append(name) + return { '$schema': 'http://json-schema.org/draft/2019-09/schema#', 'type': 'object', - 'properties': {_convert_name(i['name']): {'type': _convert_type(i['type'])} for i in abi['inputs']}, - 'required': [_convert_name(i['name']) for i in abi['inputs']], - 'additionalProperties': False, + 'properties': properties, + 'required': required, } @@ -108,12 +119,17 @@ def _convert_abi(abi_path: Path) -> EvmAbi: ) ) elif abi_item['type'] == 'event': - inputs = tuple((i['type'], i['indexed']) for i in abi_item['inputs']) + inputs = [] + for item in abi_item['inputs']: + if (type_ := item['type']) == 'tuple': + type_ = '(' + ','.join(c['type'] for c in item['components']) + ')' + inputs.append((type_, item['indexed'])) + events.append( EvmEventAbi( name=abi_item['name'], topic0=topic0_from_abi(abi_item), - inputs=inputs, + inputs=tuple(inputs), topic_count=len([i for i in inputs if i[1]]), ) ) @@ -178,7 +194,13 @@ def topic0_from_abi(event: dict[str, Any]) -> str: if event.get('type') != 'event': raise FrameworkException(f'`{event["name"]}` is not an event') - signature = f'{event["name"]}({",".join([i["type"] for i in event["inputs"]])})' + types = [] + from eth_utils.abi import collapse_if_tuple + + for input in event['inputs']: + types.append(collapse_if_tuple(input)) + + signature = f'{event["name"]}({",".join(types)})' return '0x' + eth_utils.crypto.keccak(text=signature).hex() diff --git a/src/dipdup/indexes/evm_events/matcher.py b/src/dipdup/indexes/evm_events/matcher.py index afb916c83..804682f8e 100644 --- a/src/dipdup/indexes/evm_events/matcher.py +++ b/src/dipdup/indexes/evm_events/matcher.py @@ -40,7 +40,13 @@ def decode_event_data( non_indexed_bytes = decode_hex(data) if non_indexed_bytes: - non_indexed_values = iter(decode_abi(tuple(n for n, i in inputs if not i), non_indexed_bytes)) + non_indexed_values = iter( + decode_abi( + types=tuple(n for n, i in inputs if not i), + data=non_indexed_bytes, + strict=False, + ) + ) else: # NOTE: Node truncates trailing zeros in event data non_indexed_values = cycle((0,)) @@ -81,6 +87,7 @@ def prepare_event_handler_args( type_=type_, data=data, plain=True, + nested=True, ) return EvmEvent( data=matched_event, diff --git a/src/dipdup/indexes/evm_transactions/matcher.py b/src/dipdup/indexes/evm_transactions/matcher.py index b414f6edb..ddfa953a7 100644 --- a/src/dipdup/indexes/evm_transactions/matcher.py +++ b/src/dipdup/indexes/evm_transactions/matcher.py @@ -49,8 +49,10 @@ def prepare_transaction_handler_args( name=handler_config.method, signature=handler_config.signature, )['inputs'] + from eth_utils.abi import collapse_if_tuple + data = decode_abi( - types=tuple(input['type'] for input in inputs), + types=tuple(collapse_if_tuple(input) for input in inputs), data=decode_hex(matched_transaction.input[10:]), strict=False, ) diff --git a/src/dipdup/utils.py b/src/dipdup/utils.py index 7cf9acfe7..a0f418728 100644 --- a/src/dipdup/utils.py +++ b/src/dipdup/utils.py @@ -200,13 +200,26 @@ def parse_object( type_: type[ObjectT], data: Mapping[str, Any] | Sequence[Any] | None, plain: bool = False, + nested: bool = False, ) -> ObjectT: try: if plain is False or data is None: return type_.model_validate(data) model_keys = tuple(field.alias or key for key, field in type_.model_fields.items()) - return type_(**dict(zip(model_keys, data, strict=True))) + model_dict = dict(zip(model_keys, data, strict=True)) + + if nested: + for k, v in model_dict.items(): + if not isinstance(v, list | tuple): + continue + + # NOTE: Might be `from_` or other reserved keyword + field_k = '{k}_ ' if k not in type_.model_fields else k + nested_type = type_.model_fields[field_k].annotation + model_dict[k] = parse_object(nested_type, v, plain=True) # type: ignore[arg-type] + + return type_(**model_dict) except ValidationError as e: raise InvalidDataError(f'Failed to parse: {e.errors()}', type_, data) from e except ValueError as e: