storage/consensus: Update indexes to better serve the related accounts queries

.changelog/890.feature.2.md

@@ -0,0 +1,9 @@
+storage/runtime: Support method filter for native transfers
+
+Two special values for `method` filter are added:
+
+- `method=native_transfers`: Returns "likely to be native" transactions
+  - All accounts.Transfer transactions and EVM Calls with empty data
+
+- `method=evm.Call_no_native`: Returns EVM Calls which are not "likely to be
+native" transfers

.changelog/890.feature.md

@@ -0,0 +1 @@
+storage/consensus: Update indexes to better serve the related accounts queries

api/spec/v1.yaml

@@ -939,7 +939,14 @@ paths:
           name: method
           schema:
             type: string
-          description: A filter on the runtime transaction method.
+          description: |
+            A filter on the runtime transaction method.
+
+            In addition to the existing method names, the following special values are supported:
+              - 'native_transfers': Returns transactions "likely to be native transfers".
+                - These include accounts.Transfer transactions and evm.Calls with an empty 'body' field.
+
+              - 'evm.Call_no_native': Returns EVM calls that are "not likely to be native transfers".
           example: 'accounts.Transfer'
       responses:
         '200':

cmd/api/api.go

@@ -31,7 +31,7 @@ const (
 	// The path portion with which all v1 API endpoints start.
 	v1BaseURL = "/v1"
 
-	defaultRequestHandleTimeout = 10 * time.Second
+	defaultRequestHandleTimeout = 20 * time.Second
 )
 
 var (

storage/client/queries/queries.go

@@ -537,7 +537,22 @@ const (
 			($2::bigint IS NULL OR txs.round = $2::bigint) AND
 			($3::text IS NULL OR txs.tx_hash = $3::text OR txs.tx_eth_hash = $3::text) AND
 			($4::text IS NULL OR rel.account_address = $4::text) AND
-			($5::text IS NULL or txs.method = $5::text) AND
+			(
+				CASE
+					-- No filtering on method.
+					WHEN $5::text IS NULL THEN
+						TRUE
+					-- Special case to return are 'likely to be native transfers'.
+					WHEN $5::text = 'native_transfers' THEN
+						(txs.method = 'accounts.Transfer' OR (txs.method = 'evm.Call' AND (txs.body ->> 'data') = ''))
+					-- Special case to return all evm.Calls that are likely not native transfers.
+					WHEN $5::text = 'evm.Call_no_native' THEN
+						(txs.method = 'evm.Call' AND (txs.body ->> 'data') != '')
+					-- Regular case.
+					ELSE
+						(txs.method = $5::text)
+				END
+			) AND
 			($6::timestamptz IS NULL OR txs.timestamp >= $6::timestamptz) AND
 			($7::timestamptz IS NULL OR txs.timestamp < $7::timestamptz)
 		GROUP BY

storage/migrations/00_consensus.up.sql

@@ -81,7 +81,9 @@ CREATE TABLE chain.transactions
 CREATE INDEX ix_transactions_sender_block ON chain.transactions (sender, block);
 CREATE INDEX ix_transactions_tx_hash ON chain.transactions (tx_hash);
 --`method` is a possible external API parameter; `block` lets us efficiently retrieve the most recent N txs with a given method.
-CREATE INDEX ix_transactions_method_height ON chain.transactions (method, block);
+CREATE INDEX ix_transactions_method_height ON chain.transactions (method, block); -- Removed in 12_related_transactions_method_idx.up.sql.
+-- Added in 12_related_transactions_method_idx.up.sql.
+CREATE INDEX ix_transactions_method_block_tx_index ON chain.transactions (method, block DESC, tx_index);
 
 CREATE TABLE chain.events
 (
@@ -346,7 +348,9 @@ CREATE TABLE chain.accounts_related_transactions
   FOREIGN KEY (tx_block, tx_index) REFERENCES chain.transactions(block, tx_index) DEFERRABLE INITIALLY DEFERRED
 );
 CREATE INDEX ix_accounts_related_transactions_block ON chain.accounts_related_transactions (tx_block);
-CREATE INDEX ix_accounts_related_transactions_address_block ON chain.accounts_related_transactions(account_address, tx_block);
+CREATE INDEX ix_accounts_related_transactions_address_block ON chain.accounts_related_transactions(account_address, tx_block); -- Removed in 12_related_transactions_method_idx.up.sql.
+-- Added in 12_related_transactions_method_idx.up.sql.
+-- CREATE INDEX ix_accounts_related_transactions_address_block_desc_tx_index ON chain.accounts_related_transactions (account_address, tx_block DESC, tx_index);
 
 -- Tracks the current (consensus) height of the node.
 CREATE TABLE chain.latest_node_heights

storage/migrations/01_runtimes.up.sql

@@ -103,6 +103,13 @@ CREATE INDEX ix_runtime_transactions_to ON chain.runtime_transactions(runtime, "
 CREATE INDEX ix_runtime_transactions_to_abi_parsed_at ON chain.runtime_transactions (runtime, "to", abi_parsed_at);
 -- CREATE INDEX ix_runtime_transactions_method_round ON chain.runtime_transactions (runtime, method, round, tx_index); -- Added in 08_runtime_transactions_method_idx.up.sql
 
+-- Added in 12_related_transactions_method_idx.up.sql.
+-- Indexes for efficient query of 'likely native transfers':
+-- EVM Calls, where the body is an empty data field (likely native transfers)
+-- CREATE INDEX ix_runtime_transactions_evm_call_empty_data ON chain.runtime_transactions (runtime, round, tx_index) WHERE method = 'evm.Call' AND (body ->> 'data') = '';
+-- EVM Calls, where the body is non-empty data field (likely not native transfers).
+-- CREATE INDEX ix_runtime_transactions_evm_call_non_empty_data ON chain.runtime_transactions (runtime, round, tx_index) WHERE method = 'evm.Call' AND (body ->> 'data') != '';
+
 CREATE TABLE chain.runtime_transaction_signers
 (
   runtime        runtime NOT NULL,

storage/migrations/12_related_transactions_method_idx.up.sql

@@ -0,0 +1,15 @@
+BEGIN;
+
+CREATE INDEX IF NOT EXISTS ix_transactions_method_block_tx_index ON chain.transactions (method, block DESC, tx_index);
+DROP INDEX IF EXISTS chain.ix_transactions_method_height;
+
+CREATE INDEX IF NOT EXISTS ix_accounts_related_transactions_address_block_desc_tx_index ON chain.accounts_related_transactions (account_address, tx_block DESC, tx_index);
+DROP INDEX IF EXISTS chain.ix_accounts_related_transactions_address_block;
+
+-- Indexes for efficient query of 'likely native transfers':
+-- EVM Calls, where the body is an empty data field (likely native transfers)
+CREATE INDEX IF NOT EXISTS ix_runtime_transactions_evm_call_empty_data ON chain.runtime_transactions (runtime, round, tx_index) WHERE method = 'evm.Call' AND (body ->> 'data') = '';
+-- EVM Calls, where the body is non-empty data field (likely not native transfers).
+CREATE INDEX IF NOT EXISTS ix_runtime_transactions_evm_call_non_empty_data ON chain.runtime_transactions (runtime, round, tx_index) WHERE method = 'evm.Call' AND (body ->> 'data') != '';
+
+COMMIT;

tests/e2e_regression/common_test_cases.sh

@@ -64,6 +64,9 @@ commonTestCases=(
   'emerald_blocks                     /v1/emerald/blocks'
   'emerald_txs                        /v1/emerald/transactions'
   'emerald_txs_by_method              /v1/emerald/transactions?method=consensus.Withdraw'
+  'emerald_txs_native_transfers       /v1/emerald/transactions?method=native_transfers'
+  'emerald_txs_evm_call_no_native     /v1/emerald/transactions?method=evm.Call_no_native'
+  'emerald_txs_evm_call               /v1/emerald/transactions?method=evm.Call'
   'emerald_events                     /v1/emerald/events'
   'emerald_events_by_type             /v1/emerald/events?type=accounts.transfer'
   'emerald_tokens                     /v1/emerald/evm_tokens'

tests/e2e_regression/damask/expected/emerald_txs_evm_call.body

@@ -0,0 +1,5 @@
+{
+  "is_total_count_clipped": false,
+  "total_count": 0,
+  "transactions": []
+}

tests/e2e_regression/damask/expected/emerald_txs_evm_call.headers

@@ -0,0 +1,6 @@
+HTTP/1.1 200 OK
+Content-Type: application/json
+Vary: Origin
+Date: UNINTERESTING
+Content-Length: UNINTERESTING
+

tests/e2e_regression/damask/expected/emerald_txs_evm_call_no_native.body

@@ -0,0 +1,5 @@
+{
+  "is_total_count_clipped": false,
+  "total_count": 0,
+  "transactions": []
+}

tests/e2e_regression/damask/expected/emerald_txs_evm_call_no_native.headers

@@ -0,0 +1,6 @@
+HTTP/1.1 200 OK
+Content-Type: application/json
+Vary: Origin
+Date: UNINTERESTING
+Content-Length: UNINTERESTING
+

tests/e2e_regression/damask/expected/emerald_txs_native_transfers.body

@@ -0,0 +1,5 @@
+{
+  "is_total_count_clipped": false,
+  "total_count": 0,
+  "transactions": []
+}

tests/e2e_regression/damask/expected/emerald_txs_native_transfers.headers

@@ -0,0 +1,6 @@
+HTTP/1.1 200 OK
+Content-Type: application/json
+Vary: Origin
+Date: UNINTERESTING
+Content-Length: UNINTERESTING
+