Merge branch 'next' into dev/substrate-events

CHANGELOG.md

@@ -15,17 +15,15 @@ Releases prior to 7.0 has been removed from this file to declutter search result
 - substrate.subscan: Added `substrate.subscan` datasource to fetch ABIs from Subscan.
 - substrate.subsquid: Added `substrate.subsquid` datasource to fetch historical data from Squid Network.
 
+## [Unreleased]
+
 ### Fixed
 
 - cli: Don't wrap exceptions with `CallbackError` to avoid shadowing the original exception.
 - cli: Fixed `--template` option being ignored when `--quiet` flag is set.
 - config: Fixed setting default loglevels when `logging` is a dict.
-
-## [Unreleased]
-
-### Fixed
-
 - config: Fixed parsing config files after updating to pydantic 2.10.3.
+- metrics: Fixed indexed objects counter.
 
 ## [8.1.2] - 2024-12-10
 

docs/7.references/2.config.md

@@ -376,6 +376,24 @@ description: "Config file reference"
 
 <dl class="py class">
 
+## dipdup.config.IndexConfig
+
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.</span></span><span class="sig-name descname"><span class="pre">IndexConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span></dt>
+<dd><p>Index config</p>
+<dl class="field-list simple">
+<dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>kind</strong> – Defined by child class</p></li>
+<li><p><strong>datasources</strong> – Aliases of index datasources in <cite>datasources</cite> section</p></li>
+
+<li><p><strong>kwargs</strong> (<em>Any</em>)</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py class">
+
 ## dipdup.config.DatasourceConfig
 
 <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.</span></span><span class="sig-name descname"><span class="pre">DatasourceConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span></dt>
@@ -622,13 +640,13 @@ description: "Config file reference"
 
 ## dipdup.config.tezos.TezosIndexConfig
 
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.tezos.</span></span><span class="sig-name descname"><span class="pre">TezosIndexConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span></em><span class="sig-paren">)</span></dt>
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.tezos.</span></span><span class="sig-name descname"><span class="pre">TezosIndexConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">kind</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">datasources</span></span></em><span class="sig-paren">)</span></dt>
 <dd><p>TzKT index config</p>
 <dl class="field-list simple">
 <dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
-<li><p><strong>kind</strong> – starts with ‘tezos’</p></li>
-<li><p><strong>datasources</strong> – <cite>tezos</cite> datasources to use</p></li>
+<li><p><strong>kind</strong> (<em>str</em>) – starts with ‘tezos’</p></li>
+<li><p><strong>datasources</strong> (<em>tuple</em><em>[</em><em>str</em><em> | </em><a class="reference internal" href="#dipdupconfigtezos_tzkttezostzktdatasourceconfig" title="dipdup.config.tezos_tzkt.TezosTzktDatasourceConfig" target="_self"><em>TezosTzktDatasourceConfig</em></a><em>, </em><em>...</em><em>]</em>) – <cite>tezos</cite> datasources to use</p></li>
 
 </ul>
 </dd>
@@ -1082,13 +1100,13 @@ description: "Config file reference"
 
 ## dipdup.config.starknet_events.StarknetEventsIndexConfig
 
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.starknet_events.</span></span><span class="sig-name descname"><span class="pre">StarknetEventsIndexConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span></em><span class="sig-paren">)</span></dt>
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.starknet_events.</span></span><span class="sig-name descname"><span class="pre">StarknetEventsIndexConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">kind</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">datasources</span></span></em><span class="sig-paren">)</span></dt>
 <dd><p>Starknet events index config</p>
 <dl class="field-list simple">
 <dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
-<li><p><strong>kind</strong> – Always ‘starknet.events’</p></li>
-<li><p><strong>datasources</strong> – Aliases of index datasources in <cite>datasources</cite> section</p></li>
+<li><p><strong>kind</strong> (<em>Literal</em><em>[</em><em>'starknet.events'</em><em>]</em>) – Always ‘starknet.events’</p></li>
+<li><p><strong>datasources</strong> (<em>tuple</em><em>[</em><em>str</em><em> | </em><a class="reference internal" href="#dipdupconfigstarknet_subsquidstarknetsubsquiddatasourceconfig" title="dipdup.config.starknet_subsquid.StarknetSubsquidDatasourceConfig" target="_self"><em>StarknetSubsquidDatasourceConfig</em></a><em> | </em><a class="reference internal" href="#dipdupconfigstarknet_nodestarknetnodedatasourceconfig" title="dipdup.config.starknet_node.StarknetNodeDatasourceConfig" target="_self"><em>StarknetNodeDatasourceConfig</em></a><em>, </em><em>...</em><em>]</em>) – Aliases of index datasources in <cite>datasources</cite> section</p></li>
 <li><p><strong>handlers</strong> – Event handlers</p></li>
 <li><p><strong>first_level</strong> – Level to start indexing from</p></li>
 <li><p><strong>last_level</strong> – Level to stop indexing at</p></li>
@@ -1174,13 +1192,13 @@ description: "Config file reference"
 
 ## dipdup.config.substrate_events.SubstrateEventsIndexConfig
 
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.substrate_events.</span></span><span class="sig-name descname"><span class="pre">SubstrateEventsIndexConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span></em><span class="sig-paren">)</span></dt>
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.substrate_events.</span></span><span class="sig-name descname"><span class="pre">SubstrateEventsIndexConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">kind</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">datasources</span></span></em><span class="sig-paren">)</span></dt>
 <dd><p>Subsquid datasource config</p>
 <dl class="field-list simple">
 <dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
-<li><p><strong>kind</strong> – Always ‘substrate.events’</p></li>
-<li><p><strong>datasources</strong> – <cite>substrate</cite> datasources to use</p></li>
+<li><p><strong>kind</strong> (<em>Literal</em><em>[</em><em>'substrate.events'</em><em>]</em>) – Always ‘substrate.events’</p></li>
+<li><p><strong>datasources</strong> (<em>tuple</em><em>[</em><em>str</em><em> | </em><a class="reference internal" href="#dipdupconfigsubstrate_subsquidsubstratesubsquiddatasourceconfig" title="dipdup.config.substrate_subsquid.SubstrateSubsquidDatasourceConfig" target="_self"><em>SubstrateSubsquidDatasourceConfig</em></a><em> | </em><a class="reference internal" href="#dipdupconfigsubstrate_subscansubstratesubscandatasourceconfig" title="dipdup.config.substrate_subscan.SubstrateSubscanDatasourceConfig" target="_self"><em>SubstrateSubscanDatasourceConfig</em></a><em> | </em><em>SubstrateNodeDatasourceConfig</em><em>, </em><em>...</em><em>]</em>) – <cite>substrate</cite> datasources to use</p></li>
 <li><p><strong>handlers</strong> – Event handlers</p></li>
 <li><p><strong>first_level</strong> – Level to start indexing from</p></li>
 <li><p><strong>last_level</strong> – Level to stop indexing and disable this index</p></li>

docs/9.release-notes/_8.0_changelog.md

@@ -47,6 +47,7 @@
 - evm.subsquid: Fixed typo in `iter_events` method name.
 - evm: Fixed crash when contract ABI contains overloaded methods.
 - install: Fixed reinstalling package when `--force` flag is used.
+- metrics: Fixed indexed objects counter.
 - models: Fixed `CachedModel` preloading.
 - models: Fixed setting default value for `Meta.maxsize`.
 - package: Create package in-place if cwd equals package name.

docs/config.rst

@@ -18,6 +18,7 @@
 .. autoclass:: dipdup.config.HookConfig
 .. autoclass:: dipdup.config.HttpConfig
 .. autoclass:: dipdup.config.http.HttpDatasourceConfig
+.. autoclass:: dipdup.config.IndexConfig
 .. autoclass:: dipdup.config.DatasourceConfig
 .. autoclass:: dipdup.config.IndexTemplateConfig
 .. autoclass:: dipdup.config.ipfs.IpfsDatasourceConfig

src/dipdup/codegen/tezos.py

@@ -84,7 +84,7 @@ def preprocess_storage_jsonschema(schema: dict[str, Any]) -> dict[str, Any]:
 class TezosCodeGenerator(CodeGenerator):
     """Generates package based on config, invoked from `init` CLI command"""
 
-    kind = 'substrate'
+    kind = 'tezos'
 
     def __init__(
         self,

src/dipdup/utils.py

@@ -228,6 +228,10 @@ def json_dumps(obj: Any | str, option: int | None = orjson.OPT_INDENT_2) -> byte
     )
 
 
+def json_dumps_plain(obj: Any | str) -> str:
+    return json_dumps(obj, None).decode()
+
+
 class Watchdog:
     def __init__(self, timeout: int) -> None:
         self._watchdog = asyncio.Event()