Merge branch 'next' into aux/8.2.0rc1

CHANGELOG.md

@@ -23,6 +23,16 @@ Releases prior to 7.0 has been removed from this file to declutter search result
 
 - evm.etherscan: Datasource has been renamed from `abi.etherscan` to `evm.etherscan` for consistency.
 
+## [8.1.4] - 2025-01-12
+
+### Fixed
+
+- evm: Fixed sending JSONRPC requests via web3.py provider.
+- evm: Fixed parsing tuple types in ABI.
+- evm.subsquid: Fixed type of `timestamp` field of event/transaction models.
+- evm.subsquid: Fixed empty field base conversion on event deserialization.
+- starknet: Fixed parsing contract addresses starting with `0x0`.
+
 ## [8.1.3] - 2024-12-20
 
 ### Fixed
@@ -578,7 +588,8 @@ Releases prior to 7.0 has been removed from this file to declutter search result
 
 <!-- Versions -->
 [Unreleased]: https://github.com/dipdup-io/dipdup/compare/8.2.0rc1...HEAD
-[8.2.0rc1]: https://github.com/dipdup-io/dipdup/compare/8.1.3...8.2.0rc1
+[8.2.0rc1]: https://github.com/dipdup-io/dipdup/compare/8.1.4...8.2.0rc1
+[8.1.4]: https://github.com/dipdup-io/dipdup/compare/8.1.3...8.1.4
 [8.1.3]: https://github.com/dipdup-io/dipdup/compare/8.1.2...8.1.3
 [8.1.2]: https://github.com/dipdup-io/dipdup/compare/8.1.1...8.1.2
 [8.1.1]: https://github.com/dipdup-io/dipdup/compare/8.1.0...8.1.1

docs/16.thanks.md

@@ -31,7 +31,9 @@ We are grateful to all the people who helped us with the project.
 - [Florian PAUTOT](https://github.com/0x666c6f)
 - [gdsoumya](https://github.com/gdsoumya)
 - [Göran Sandström](https://github.com/veqtor)
+- [hoka](https://github.com/hokaxbt)
 - [Igor Sereda](https://github.com/igorsereda)
+- [Ilia Batii](https://github.com/baitcode)
 - [Javier Graciá Carpio](https://github.com/jagracar)
 - [JoE11-y](https://github.com/JoE11-y)
 - [Karan Dua](https://github.com/Karantezsure)
@@ -49,4 +51,10 @@ We are grateful to all the people who helped us with the project.
 - [Soham Das](https://github.com/tosoham)
 - [tomsib2001](https://github.com/tomsib2001)
 
+Also, these people helped heavily with [pysignalr](https://github.com/baking-bad/), a library we have developed to use in DipDup:
+
+- [Caio Barbieri](https://github.com/caiolombello)
+- [MichaelMKKelly](https://github.com/MichaelMKKelly)
+- [Ola Lidholm](https://github.com/olalid)
+
 If we forgot to mention you, or you want to update your record, please, open an issue or pull request.

docs/9.release-notes/_8.1_changelog.md

@@ -20,8 +20,13 @@
 - config: Fixed starknet index validation error.
 - context: Allow to add Starknet contracts in runtime.
 - database: Ignore non-existent immutable table on schema wipe.
+- evm.subsquid: Fixed empty field base conversion on event deserialization.
+- evm.subsquid: Fixed type of `timestamp` field of event/transaction models.
+- evm: Fixed parsing tuple types in ABI.
+- evm: Fixed sending JSONRPC requests via web3.py provider.
 - metrics: Fixed indexed objects counter.
 - starknet.events: Fixed event ID calculation.
 - starknet: Added support for struct and array types, as well as u256 and ByteArray handlers.
 - starknet: Fixed event payload parsing (account for keys field).
 - starknet: Fixed missing class property in node datasource.
+- starknet: Fixed parsing contract addresses starting with `0x0`.

docs/config.rst

@@ -2,6 +2,8 @@
 
 .. autoclass:: dipdup.config.DipDupConfig
 
+.. autoclass:: dipdup.config.abi_etherscan.AbiEtherscanDatasourceConfig
+.. autoclass:: dipdup.config.ContractConfig
 .. autoclass:: dipdup.config.AdvancedConfig
 .. autoclass:: dipdup.config.ApiConfig
 .. autoclass:: dipdup.config.coinbase.CoinbaseDatasourceConfig

requirements.txt

@@ -8,7 +8,7 @@ aiosignal==1.3.2
 aiosqlite==0.20.0
 aiosubstrate==0.1.0
 annotated-types==0.7.0
-anyio==4.7.0
+anyio==4.8.0
 appdirs==1.4.4
 apscheduler==3.11.0
 argcomplete==3.5.2
@@ -67,7 +67,7 @@ prometheus-client==0.21.1
 propcache==0.2.1
 pycryptodome==3.21.0
 pydantic-core==2.27.2
-pydantic[email]==2.10.4; python_version ~= "3.11"
+pydantic[email]==2.10.5; python_version ~= "3.11"
 pyhumps==3.8.0
 pypika-tortoise==0.2.2
 pysignalr==1.1.0

src/dipdup/abi/evm.py

@@ -29,8 +29,6 @@
     'bytes': 'string',
     'bool': 'boolean',
     'string': 'string',
-    # TODO: arrays and tuples
-    # https://docs.soliditylang.org/en/develop/abi-spec.html#types
     'tuple': 'object',
 }
 
@@ -52,12 +50,25 @@ def _convert_name(name: str) -> str:
 
 
 def jsonschema_from_abi(abi: dict[str, Any]) -> dict[str, Any]:
+    properties, required = {}, []
+    for item in abi['inputs']:
+        name = _convert_name(item['name'])
+        if item['type'] == 'tuple':
+            properties[name] = {
+                'type': 'object',
+                'properties': {
+                    _convert_name(i['name']): {'type': _convert_type(i['type'])} for i in item['components']
+                },
+            }
+        else:
+            properties[name] = {'type': _convert_type(item['type'])}
+        required.append(name)
+
     return {
         '$schema': 'http://json-schema.org/draft/2019-09/schema#',
         'type': 'object',
-        'properties': {_convert_name(i['name']): {'type': _convert_type(i['type'])} for i in abi['inputs']},
-        'required': [_convert_name(i['name']) for i in abi['inputs']],
-        'additionalProperties': False,
+        'properties': properties,
+        'required': required,
     }
 
 
@@ -108,12 +119,17 @@ def _convert_abi(abi_path: Path) -> EvmAbi:
                 )
             )
         elif abi_item['type'] == 'event':
-            inputs = tuple((i['type'], i['indexed']) for i in abi_item['inputs'])
+            inputs = []
+            for item in abi_item['inputs']:
+                if (type_ := item['type']) == 'tuple':
+                    type_ = '(' + ','.join(c['type'] for c in item['components']) + ')'
+                inputs.append((type_, item['indexed']))
+
             events.append(
                 EvmEventAbi(
                     name=abi_item['name'],
                     topic0=topic0_from_abi(abi_item),
-                    inputs=inputs,
+                    inputs=tuple(inputs),
                     topic_count=len([i for i in inputs if i[1]]),
                 )
             )
@@ -178,7 +194,13 @@ def topic0_from_abi(event: dict[str, Any]) -> str:
     if event.get('type') != 'event':
         raise FrameworkException(f'`{event["name"]}` is not an event')
 
-    signature = f'{event["name"]}({",".join([i["type"] for i in event["inputs"]])})'
+    types = []
+    from eth_utils.abi import collapse_if_tuple
+
+    for input in event['inputs']:
+        types.append(collapse_if_tuple(input))
+
+    signature = f'{event["name"]}({",".join(types)})'
     return '0x' + eth_utils.crypto.keccak(text=signature).hex()
 
 

src/dipdup/config/starknet.py

@@ -22,6 +22,9 @@
 
 _HEX_ADDRESS_REGEXP = re.compile(r'(0x)?[0-9a-f]{1,64}', re.IGNORECASE | re.ASCII)
 
+# Spec: https://github.com/starkware-libs/starknet-specs/blob/master/api/starknet_api_openrpc.json
+_TRUNCATED_STARKNET_ADDRESS_REGEXP = re.compile(r'^0x(0|[a-fA-F1-9]{1}[a-fA-F0-9]{0,62})$', re.ASCII)
+
 
 def _validate_starknet_address(v: str) -> str:
     """
@@ -32,15 +35,21 @@ def _validate_starknet_address(v: str) -> str:
         return v
 
     if _HEX_ADDRESS_REGEXP.fullmatch(v) is None:
-        raise ValueError(f'{v} is not a valid Starknet contract address')
-
+        raise ValueError(
+            f'{v} is not a valid contract address (check if it is a hex string in the form 0x[64 hex chars])'
+        )
+
+    # Following code is similar to:
+    #   https://github.com/software-mansion/starknet.py/blob/a8d73538d409d9ef7c756921e43d10925f2838bc/starknet_py/net/client_utils.py#L60
+    #   starknet_py.net.client_utils._to_rpc_felt method
+    #
     # Convert hex to decimal and check if it's less than 2**251
     numeric_value = int(v, 16)
-    if not (numeric_value < 2**251):
+    truncated_value = hex(numeric_value)
+    if not _TRUNCATED_STARKNET_ADDRESS_REGEXP.fullmatch(truncated_value):
         raise ValueError(f'{v} is not a valid Starknet contract address')
 
-    # TODO: Probably needs to be to normalized as in EVM case
-    return v
+    return truncated_value
 
 
 type StarknetAddress = Annotated[Hex, AfterValidator(_validate_starknet_address)]

src/dipdup/datasources/__init__.py

@@ -193,7 +193,7 @@ async def _jsonrpc_request(
         raw: bool = False,
         ws: bool = False,
     ) -> Any:
-        request_id = uuid4().hex
+        request_id = str(int(uuid4()))
         request = {
             'jsonrpc': '2.0',
             'id': request_id,

src/dipdup/indexes/evm_events/matcher.py

@@ -40,7 +40,13 @@ def decode_event_data(
 
     non_indexed_bytes = decode_hex(data)
     if non_indexed_bytes:
-        non_indexed_values = iter(decode_abi(tuple(n for n, i in inputs if not i), non_indexed_bytes))
+        non_indexed_values = iter(
+            decode_abi(
+                types=tuple(n for n, i in inputs if not i),
+                data=non_indexed_bytes,
+                strict=False,
+            )
+        )
     else:
         # NOTE: Node truncates trailing zeros in event data
         non_indexed_values = cycle((0,))
@@ -81,6 +87,7 @@ def prepare_event_handler_args(
         type_=type_,
         data=data,
         plain=True,
+        nested=True,
     )
     return EvmEvent(
         data=matched_event,

src/dipdup/indexes/evm_transactions/matcher.py

@@ -49,8 +49,10 @@ def prepare_transaction_handler_args(
         name=handler_config.method,
         signature=handler_config.signature,
     )['inputs']
+    from eth_utils.abi import collapse_if_tuple
+
     data = decode_abi(
-        types=tuple(input['type'] for input in inputs),
+        types=tuple(collapse_if_tuple(input) for input in inputs),
         data=decode_hex(matched_transaction.input[10:]),
         strict=False,
     )

src/dipdup/utils.py

@@ -200,13 +200,26 @@ def parse_object(
     type_: type[ObjectT],
     data: Mapping[str, Any] | Sequence[Any] | None,
     plain: bool = False,
+    nested: bool = False,
 ) -> ObjectT:
     try:
         if plain is False or data is None:
             return type_.model_validate(data)
 
         model_keys = tuple(field.alias or key for key, field in type_.model_fields.items())
-        return type_(**dict(zip(model_keys, data, strict=True)))
+        model_dict = dict(zip(model_keys, data, strict=True))
+
+        if nested:
+            for k, v in model_dict.items():
+                if not isinstance(v, list | tuple):
+                    continue
+
+                # NOTE: Might be `from_` or other reserved keyword
+                field_k = '{k}_ ' if k not in type_.model_fields else k
+                nested_type = type_.model_fields[field_k].annotation
+                model_dict[k] = parse_object(nested_type, v, plain=True)  # type: ignore[arg-type]
+
+        return type_(**model_dict)
     except ValidationError as e:
         raise InvalidDataError(f'Failed to parse: {e.errors()}', type_, data) from e
     except ValueError as e: