From 3f5ed179d94a6d3991aacba462d67f02510a0686 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 8 Jan 2024 17:48:59 -0500 Subject: [PATCH 01/18] initial commit --- aggregator.example..json | 4 ++ auth_settings.example.json | 12 ++++ projects.example.json | 15 ++++ settings.example.json | 144 +++++++++++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 aggregator.example..json create mode 100644 auth_settings.example.json create mode 100644 projects.example.json create mode 100644 settings.example.json diff --git a/aggregator.example..json b/aggregator.example..json new file mode 100644 index 0000000..6757624 --- /dev/null +++ b/aggregator.example..json @@ -0,0 +1,4 @@ +{ + "config": [ + ] +} \ No newline at end of file diff --git a/auth_settings.example.json b/auth_settings.example.json new file mode 100644 index 0000000..b6d0e8f --- /dev/null +++ b/auth_settings.example.json @@ -0,0 +1,12 @@ +{ + "redis": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null + }, + "bind": { + "host": "0.0.0.0", + "port": 8555 + } +} diff --git a/projects.example.json b/projects.example.json new file mode 100644 index 0000000..8de3741 --- /dev/null +++ b/projects.example.json @@ -0,0 +1,15 @@ +{ + "config": [ + { + "project_type": "poolContract_total_supply", + "projects":[ + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + ], + "processor":{ + "module": "snapshotter.modules.computes.pool_total_supply", + "class_name": "AssetTotalSupplyProcessor" + }, + "preload_tasks":[] + } + ] +} \ No newline at end of file diff --git a/settings.example.json b/settings.example.json new file mode 100644 index 0000000..2157213 --- /dev/null +++ b/settings.example.json @@ -0,0 +1,144 @@ +{ + "namespace": "aavev3", + "core_api": { + "host": "0.0.0.0", + "port": 8002, + "auth": { + "enabled": false, + "header_key": "X-API-KEY" + }, + "public_rate_limit": "20000/day;300/minute;10/second" + }, + "instance_id": "account-address", + "rpc": { + "full_nodes": [{ + "url": "https://rpc-url", + "rate_limit": "100000000/day;20000/minute;1000/second" + }], + "archive_nodes": [], + "force_archive_blocks": 100, + "retry": 5, + "request_time_out": 5, + "skip_epoch_threshold_blocks": 30, + "polling_interval": 10, + "connection_limits":{ + "max_connections": 100, + "max_keepalive_connections": 50, + "keepalive_expiry": 300 + } + }, + "rlimit": { + "file_descriptors": 2048 + }, + "rabbitmq": { + "user": "guest", + "password": "guest", + "host": "rabbitmq", + "port": 5672, + "setup": { + "core": { + "exchange": "powerloom-backend" + }, + "callbacks": { + "exchange": "powerloom-backend-callbacks" + }, + "commit_payload": { + "exchange": "powerloom-backend-commit-payload" + }, + "event_detector": { + "exchange": "powerloom-backend-event-detector" + }, + "delegated_worker": { + "exchange": "powerloom-backend-delegated-worker" + } + } + }, + "reporting": { + "slack_url": "https://slack-reporting-url", + "service_url": "https://powerloom-reporting-url" + }, + "redis": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null, + "ssl": false, + "cluster_mode": false + }, + "redis_reader": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null + }, + "logs": { + "trace_enabled": false, + "write_to_files": true + }, + "projects_config_path": "config/projects.json", + "aggregator_config_path": "config/aggregator.json", + "preloader_config_path": "config/preloader.json", + "pair_contract_abi": "snapshotter/static/abis/UniswapV2Pair.json", + "protocol_state": { + "abi": "snapshotter/static/abis/ProtocolContract.json", + "address": "0x9038EcB07A057A3E1023F1a3a68d404DF73018B4" + }, + "callback_worker_config": { + "num_delegate_workers": 0, + "num_snapshot_workers": 8, + "num_aggregation_workers": 12 + }, + "ipfs": { + "url": "ipfs-writer-url", + "url_auth": { + "apiKey": "ipfs-writer-key", + "apiSecret": "ipfs-writer-secret" + }, + "reader_url": "ipfs-reader-url", + "reader_url_auth": { + "apiKey": "ipfs-reader-key", + "apiSecret": "ipfs-reader-secret" + }, + "write_rate_limit": { + "req_per_sec": 10, + "burst": 10 + }, + "timeout": 60, + "local_cache_path": "/home/ubuntu/local_cache/", + "connection_limits":{ + "max_connections": 100, + "max_keepalive_connections": 50, + "keepalive_expiry": 300 + }, + "remote_pinning": { + "enabled": false, + "service_name": "", + "service_endpoint": "", + "service_token": "", + "background_pinning": false + } + }, + "web3storage":{ + "upload_snapshots": false, + "upload_aggregates": true, + "url": "https://api.web3.storage", + "upload_url_suffix": "/upload", + "api_token": "web3-storage-token", + "timeout": 30, + "max_idle_conns": 5, + "idle_conn_timeout": 300 + }, + "anchor_chain_rpc": { + "full_nodes": [{ + "url": "https://prost-rpc-url", + "rate_limit": "100000000/day;20000/minute;1000/second" + }], + "retry": 5, + "request_time_out": 5, + "connection_limits":{ + "max_connections": 100, + "max_keepalive_connections": 50, + "keepalive_expiry": 300 + } + } +} \ No newline at end of file From c3d743f726c987c125fad718447b42fffef4b3d5 Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 9 Jan 2024 01:07:54 -0500 Subject: [PATCH 02/18] add preloader.json --- preloader.json | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 preloader.json diff --git a/preloader.json b/preloader.json new file mode 100644 index 0000000..24cd4b7 --- /dev/null +++ b/preloader.json @@ -0,0 +1,27 @@ +{ + "preloaders": [ + { + "task_type": "block_transactions", + "module": "snapshotter.utils.preloaders.tx_receipts.preloader", + "class_name": "TxPreloadWorker" + }, + { + "task_type": "block_details", + "module": "snapshotter.utils.preloaders.block_details.preloader", + "class_name": "BlockDetailsPreloader" + }, + { + "task_type": "eth_price", + "module": "snapshotter.utils.preloaders.eth_price.preloader", + "class_name": "EthPricePreloader" + } + ], + "delegate_tasks": [ + { + "task_type": "txreceipt", + "module": "snapshotter.utils.preloaders.tx_receipts.delegated_worker.tx_receipts", + "class_name": "TxReceiptProcessor" + } + ], + "timeout": 60 +} From 6eee52eb80fb9c7ccdf82c4ec6ddf4f519361469 Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 9 Jan 2024 21:05:16 -0500 Subject: [PATCH 03/18] add: top asset aggregator for dashboard --- aggregator.example..json | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/aggregator.example..json b/aggregator.example..json index 6757624..8808827 100644 --- a/aggregator.example..json +++ b/aggregator.example..json @@ -1,4 +1,17 @@ { "config": [ + { + "project_type": "aggregate_top_assets", + "aggregate_on": "MultiProject", + "projects_to_wait_for": [ + "poolContract_total_supply:0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48:aavev3", + "poolContract_total_supply:0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2:aavev3", + "poolContract_total_supply:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3" + ], + "processor": { + "module": "snapshotter.modules.computes.aggregate.multi_aave_top_assets", + "class_name": "AggreagateTopAssetsProcessor" + } + } ] -} \ No newline at end of file +} From 2fe28338e8e352c7f888fb362dc013cb419099c1 Mon Sep 17 00:00:00 2001 From: Seth Date: Thu, 11 Jan 2024 00:08:44 -0500 Subject: [PATCH 04/18] fix: update projects for aggregate, fix naming --- aggregator.example..json => aggregator.example.json | 0 projects.example.json | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) rename aggregator.example..json => aggregator.example.json (100%) diff --git a/aggregator.example..json b/aggregator.example.json similarity index 100% rename from aggregator.example..json rename to aggregator.example.json diff --git a/projects.example.json b/projects.example.json index 8de3741..181ce16 100644 --- a/projects.example.json +++ b/projects.example.json @@ -3,7 +3,9 @@ { "project_type": "poolContract_total_supply", "projects":[ - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599" ], "processor":{ "module": "snapshotter.modules.computes.pool_total_supply", From 87753e9dd52e3b8b14f3ff918de92e4588d8093a Mon Sep 17 00:00:00 2001 From: Seth Date: Fri, 12 Jan 2024 22:02:11 -0500 Subject: [PATCH 05/18] add: 24h stats aggregate and bulk pricing preload --- aggregator.example.json | 15 +++++++++++++++ preloader.json | 11 +++-------- projects.example.json | 4 +++- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/aggregator.example.json b/aggregator.example.json index 8808827..ce08bee 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -12,6 +12,21 @@ "module": "snapshotter.modules.computes.aggregate.multi_aave_top_assets", "class_name": "AggreagateTopAssetsProcessor" } + }, + { + "project_type": "aggregate_24h_stats", + "aggregate_on": "MultiProject", + "projects_to_wait_for": [ + "poolContract_total_supply:0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48:aavev3", + "poolContract_total_supply:0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2:aavev3", + "poolContract_total_supply:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3" + ], + "processor": { + "module": "snapshotter.modules.computes.aggregate.multi_aave_stats", + "class_name": "AggreagateMarketStatsProcessor" + } } ] } + + diff --git a/preloader.json b/preloader.json index 24cd4b7..cfc176c 100644 --- a/preloader.json +++ b/preloader.json @@ -1,19 +1,14 @@ { "preloaders": [ - { - "task_type": "block_transactions", - "module": "snapshotter.utils.preloaders.tx_receipts.preloader", - "class_name": "TxPreloadWorker" - }, { "task_type": "block_details", "module": "snapshotter.utils.preloaders.block_details.preloader", "class_name": "BlockDetailsPreloader" }, { - "task_type": "eth_price", - "module": "snapshotter.utils.preloaders.eth_price.preloader", - "class_name": "EthPricePreloader" + "task_type": "bulk_price", + "module": "snapshotter.modules.computes.utils.preloaders.pricing.preloader", + "class_name": "BulkPricePreloader" } ], "delegate_tasks": [ diff --git a/projects.example.json b/projects.example.json index 181ce16..8016da5 100644 --- a/projects.example.json +++ b/projects.example.json @@ -11,7 +11,9 @@ "module": "snapshotter.modules.computes.pool_total_supply", "class_name": "AssetTotalSupplyProcessor" }, - "preload_tasks":[] + "preload_tasks":[ + "bulk_price" + ] } ] } \ No newline at end of file From b286c4190cd6ed274637eb5d3d6f63d8d32090c3 Mon Sep 17 00:00:00 2001 From: Seth Date: Sat, 13 Jan 2024 01:03:01 -0500 Subject: [PATCH 06/18] add: increase tokens to snapshot in example files --- aggregator.example.json | 24 ++++++++++++++++++++++-- projects.example.json | 12 +++++++++++- settings.example.json | 2 +- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/aggregator.example.json b/aggregator.example.json index ce08bee..6304d5d 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -6,7 +6,17 @@ "projects_to_wait_for": [ "poolContract_total_supply:0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48:aavev3", "poolContract_total_supply:0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2:aavev3", - "poolContract_total_supply:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3" + "poolContract_total_supply:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3", + "poolContract_total_supply:0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0:aavev3", + "poolContract_total_supply:0x6b175474e89094c44da98b954eedeac495271d0f:aavev3", + "poolContract_total_supply:0x514910771af9ca656af840dff83e8264ecf986ca:aavev3", + "poolContract_total_supply:0xae78736cd615f374d3085123a210448e74fc6393:aavev3", + "poolContract_total_supply:0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9:aavev3", + "poolContract_total_supply:0xbe9895146f7af43049ca1c1ae358b0541ea49704:aavev3", + "poolContract_total_supply:0xd533a949740bb3306d119cc777fa900ba034cd52:aavev3", + "poolContract_total_supply:0xba100000625a3754423978a60c9317c58a424e3d:aavev3", + "poolContract_total_supply:0x1f9840a85d5af5bf1d1762f925bdaddc4201f984:aavev3", + "poolContract_total_supply:0x111111111117dc0aa78b770fa6a738034120c302:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_top_assets", @@ -19,7 +29,17 @@ "projects_to_wait_for": [ "poolContract_total_supply:0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48:aavev3", "poolContract_total_supply:0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2:aavev3", - "poolContract_total_supply:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3" + "poolContract_total_supply:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3", + "poolContract_total_supply:0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0:aavev3", + "poolContract_total_supply:0x6b175474e89094c44da98b954eedeac495271d0f:aavev3", + "poolContract_total_supply:0x514910771af9ca656af840dff83e8264ecf986ca:aavev3", + "poolContract_total_supply:0xae78736cd615f374d3085123a210448e74fc6393:aavev3", + "poolContract_total_supply:0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9:aavev3", + "poolContract_total_supply:0xbe9895146f7af43049ca1c1ae358b0541ea49704:aavev3", + "poolContract_total_supply:0xd533a949740bb3306d119cc777fa900ba034cd52:aavev3", + "poolContract_total_supply:0xba100000625a3754423978a60c9317c58a424e3d:aavev3", + "poolContract_total_supply:0x1f9840a85d5af5bf1d1762f925bdaddc4201f984:aavev3", + "poolContract_total_supply:0x111111111117dc0aa78b770fa6a738034120c302:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_stats", diff --git a/projects.example.json b/projects.example.json index 8016da5..9ed34a1 100644 --- a/projects.example.json +++ b/projects.example.json @@ -5,7 +5,17 @@ "projects":[ "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", - "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599" + "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", + "0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0", + "0x6b175474e89094c44da98b954eedeac495271d0f", + "0x514910771af9ca656af840dff83e8264ecf986ca", + "0xae78736cd615f374d3085123a210448e74fc6393", + "0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9", + "0xbe9895146f7af43049ca1c1ae358b0541ea49704", + "0xd533a949740bb3306d119cc777fa900ba034cd52", + "0xba100000625a3754423978a60c9317c58a424e3d", + "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", + "0x111111111117dc0aa78b770fa6a738034120c302" ], "processor":{ "module": "snapshotter.modules.computes.pool_total_supply", diff --git a/settings.example.json b/settings.example.json index 2157213..7ed36f2 100644 --- a/settings.example.json +++ b/settings.example.json @@ -81,7 +81,7 @@ "pair_contract_abi": "snapshotter/static/abis/UniswapV2Pair.json", "protocol_state": { "abi": "snapshotter/static/abis/ProtocolContract.json", - "address": "0x9038EcB07A057A3E1023F1a3a68d404DF73018B4" + "address": "0xa45F08e931471158f80d154FC5FC154F5575bB94" }, "callback_worker_config": { "num_delegate_workers": 0, From e7963323e8fa6d8d1fdf7d5256f62b14101eea89 Mon Sep 17 00:00:00 2001 From: Seth Date: Thu, 18 Jan 2024 08:16:22 -0500 Subject: [PATCH 07/18] add: add all assets to example files --- aggregator.example.json | 36 +++++++++++++++++++++++++++++++----- preloader.json | 5 +++++ projects.example.json | 19 +++++++++++++++++-- settings.example.json | 2 +- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/aggregator.example.json b/aggregator.example.json index 6304d5d..ee539a9 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -16,7 +16,21 @@ "poolContract_total_supply:0xd533a949740bb3306d119cc777fa900ba034cd52:aavev3", "poolContract_total_supply:0xba100000625a3754423978a60c9317c58a424e3d:aavev3", "poolContract_total_supply:0x1f9840a85d5af5bf1d1762f925bdaddc4201f984:aavev3", - "poolContract_total_supply:0x111111111117dc0aa78b770fa6a738034120c302:aavev3" + "poolContract_total_supply:0x111111111117dc0aa78b770fa6a738034120c302:aavev3", + "poolContract_total_supply:0xdac17f958d2ee523a2206206994597c13d831ec7:aavev3", + "poolContract_total_supply:0x5f98805a4e8be255a32880fdec7f6728c6568ba0:aavev3", + "poolContract_total_supply:0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f:aavev3", + "poolContract_total_supply:0x5a98fcbea516cf06857215779fd812ca3bef1b32:aavev3", + "poolContract_total_supply:0xc18360217d8f7ab5e7c516566761ea12ce7f9d72:aavev3", + "poolContract_total_supply:0x853d955acef822db058eb8505911ed77f175b99e:aavev3", + "poolContract_total_supply:0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f:aavev3", + "poolContract_total_supply:0xd33526068d116ce69f19a9ee46f0bd304f21a51f:aavev3", + "poolContract_total_supply:0x83f20f44975d03b1b09e64809b757c47f942beea:aavev3", + "poolContract_total_supply:0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6:aavev3", + "poolContract_total_supply:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", + "poolContract_total_supply:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", + "poolContract_total_supply:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", + "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_top_assets", @@ -39,7 +53,21 @@ "poolContract_total_supply:0xd533a949740bb3306d119cc777fa900ba034cd52:aavev3", "poolContract_total_supply:0xba100000625a3754423978a60c9317c58a424e3d:aavev3", "poolContract_total_supply:0x1f9840a85d5af5bf1d1762f925bdaddc4201f984:aavev3", - "poolContract_total_supply:0x111111111117dc0aa78b770fa6a738034120c302:aavev3" + "poolContract_total_supply:0x111111111117dc0aa78b770fa6a738034120c302:aavev3", + "poolContract_total_supply:0xdac17f958d2ee523a2206206994597c13d831ec7:aavev3", + "poolContract_total_supply:0x5f98805a4e8be255a32880fdec7f6728c6568ba0:aavev3", + "poolContract_total_supply:0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f:aavev3", + "poolContract_total_supply:0x5a98fcbea516cf06857215779fd812ca3bef1b32:aavev3", + "poolContract_total_supply:0xc18360217d8f7ab5e7c516566761ea12ce7f9d72:aavev3", + "poolContract_total_supply:0x853d955acef822db058eb8505911ed77f175b99e:aavev3", + "poolContract_total_supply:0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f:aavev3", + "poolContract_total_supply:0xd33526068d116ce69f19a9ee46f0bd304f21a51f:aavev3", + "poolContract_total_supply:0x83f20f44975d03b1b09e64809b757c47f942beea:aavev3", + "poolContract_total_supply:0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6:aavev3", + "poolContract_total_supply:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", + "poolContract_total_supply:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", + "poolContract_total_supply:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", + "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_stats", @@ -47,6 +75,4 @@ } } ] -} - - +} \ No newline at end of file diff --git a/preloader.json b/preloader.json index cfc176c..260d08d 100644 --- a/preloader.json +++ b/preloader.json @@ -9,6 +9,11 @@ "task_type": "bulk_price", "module": "snapshotter.modules.computes.utils.preloaders.pricing.preloader", "class_name": "BulkPricePreloader" + }, + { + "task_type": "bulk_asset", + "module": "snapshotter.modules.computes.utils.preloaders.asset_data.preloader", + "class_name": "BulkAssetDataPreloader" } ], "delegate_tasks": [ diff --git a/projects.example.json b/projects.example.json index 9ed34a1..abbb77e 100644 --- a/projects.example.json +++ b/projects.example.json @@ -15,14 +15,29 @@ "0xd533a949740bb3306d119cc777fa900ba034cd52", "0xba100000625a3754423978a60c9317c58a424e3d", "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", - "0x111111111117dc0aa78b770fa6a738034120c302" + "0x111111111117dc0aa78b770fa6a738034120c302", + "0xdac17f958d2ee523a2206206994597c13d831ec7", + "0x5f98805a4e8be255a32880fdec7f6728c6568ba0", + "0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f", + "0x5a98fcbea516cf06857215779fd812ca3bef1b32", + "0xc18360217d8f7ab5e7c516566761ea12ce7f9d72", + "0x853d955acef822db058eb8505911ed77f175b99e", + "0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f", + "0xd33526068d116ce69f19a9ee46f0bd304f21a51f", + "0x83f20f44975d03b1b09e64809b757c47f942beea", + "0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6", + "0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202", + "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", + "0xf939e0a03fb07f59a73314e73794be0e57ac1b4e", + "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2" ], "processor":{ "module": "snapshotter.modules.computes.pool_total_supply", "class_name": "AssetTotalSupplyProcessor" }, "preload_tasks":[ - "bulk_price" + "bulk_asset", + "block_details" ] } ] diff --git a/settings.example.json b/settings.example.json index 7ed36f2..e0989d1 100644 --- a/settings.example.json +++ b/settings.example.json @@ -81,7 +81,7 @@ "pair_contract_abi": "snapshotter/static/abis/UniswapV2Pair.json", "protocol_state": { "abi": "snapshotter/static/abis/ProtocolContract.json", - "address": "0xa45F08e931471158f80d154FC5FC154F5575bB94" + "address": "0xB155EF3cBF0b09e62C6Ae69393e507fab57549C2" }, "callback_worker_config": { "num_delegate_workers": 0, From 4f8ca83f7f99adc9cd379562faec3839cd5d9127 Mon Sep 17 00:00:00 2001 From: Seth Date: Sat, 27 Jan 2024 19:56:41 -0500 Subject: [PATCH 08/18] add: aggregate for 6hr single project apr --- aggregator.example.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aggregator.example.json b/aggregator.example.json index ee539a9..8778f52 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -1,5 +1,16 @@ { "config": [ + { + "project_type": "aggregate_poolContract_6h_apr_avg", + "aggregate_on": "SingleProject", + "filters": { + "projectId": "poolContract_total_supply" + }, + "processor": { + "module": "snapshotter.modules.computes.aggregate.single_aave_apr", + "class_name": "AggreagateSingleAprProcessor" + } + }, { "project_type": "aggregate_top_assets", "aggregate_on": "MultiProject", From cd475c1312396c435b102f225b9ca28af4eb84a5 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 29 Jan 2024 19:51:22 -0500 Subject: [PATCH 09/18] chore: rename 6h aggregate module --- aggregator.example.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aggregator.example.json b/aggregator.example.json index 8778f52..301129f 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -1,13 +1,13 @@ { "config": [ { - "project_type": "aggregate_poolContract_6h_apr_avg", + "project_type": "aggregate_poolContract_6h_apr", "aggregate_on": "SingleProject", "filters": { "projectId": "poolContract_total_supply" }, "processor": { - "module": "snapshotter.modules.computes.aggregate.single_aave_apr", + "module": "snapshotter.modules.computes.aggregate.single_aave_apr_6h", "class_name": "AggreagateSingleAprProcessor" } }, From b3946597603ceadd55cf300b585aa9bfa383333b Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 29 Jan 2024 19:54:31 -0500 Subject: [PATCH 10/18] add: 24h apr aggregate config --- aggregator.example.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aggregator.example.json b/aggregator.example.json index 301129f..a8b9a2f 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -11,6 +11,17 @@ "class_name": "AggreagateSingleAprProcessor" } }, + { + "project_type": "aggregate_poolContract_24h_apr", + "aggregate_on": "SingleProject", + "filters": { + "projectId": "aggregate_poolContract_6h_apr" + }, + "processor": { + "module": "snapshotter.modules.computes.aggregate.single_aave_apr_24h", + "class_name": "AggreagateSingleAprProcessor" + } + }, { "project_type": "aggregate_top_assets", "aggregate_on": "MultiProject", From 45d9998d51261abced413c9344aef41ec6d7b620 Mon Sep 17 00:00:00 2001 From: Seth Date: Wed, 7 Feb 2024 16:19:38 -0500 Subject: [PATCH 11/18] add: supply volume projects and aggregates --- aggregator.example.json | 11 +++++++++++ preloader.json | 5 +++++ projects.example.json | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/aggregator.example.json b/aggregator.example.json index a8b9a2f..9a6d9cd 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -22,6 +22,17 @@ "class_name": "AggreagateSingleAprProcessor" } }, + { + "project_type": "aggregate_poolContract_volume_24h", + "aggregate_on": "SingleProject", + "filters": { + "projectId": "poolContract_supply_volume" + }, + "processor": { + "module": "snapshotter.modules.computes.aggregate.single_aave_volume_24h", + "class_name": "AggregateSupplyVolumeProcessor" + } + }, { "project_type": "aggregate_top_assets", "aggregate_on": "MultiProject", diff --git a/preloader.json b/preloader.json index 260d08d..a0bb82d 100644 --- a/preloader.json +++ b/preloader.json @@ -14,6 +14,11 @@ "task_type": "bulk_asset", "module": "snapshotter.modules.computes.utils.preloaders.asset_data.preloader", "class_name": "BulkAssetDataPreloader" + }, + { + "task_type": "bulk_event", + "module": "snapshotter.modules.computes.utils.preloaders.volume_events.preloader", + "class_name": "AaveBulkVolumeEventsPreloader" } ], "delegate_tasks": [ diff --git a/projects.example.json b/projects.example.json index abbb77e..1654a89 100644 --- a/projects.example.json +++ b/projects.example.json @@ -39,6 +39,47 @@ "bulk_asset", "block_details" ] + }, + { + "project_type": "poolContract_supply_volume", + "projects":[ + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", + "0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0", + "0x6b175474e89094c44da98b954eedeac495271d0f", + "0x514910771af9ca656af840dff83e8264ecf986ca", + "0xae78736cd615f374d3085123a210448e74fc6393", + "0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9", + "0xbe9895146f7af43049ca1c1ae358b0541ea49704", + "0xd533a949740bb3306d119cc777fa900ba034cd52", + "0xba100000625a3754423978a60c9317c58a424e3d", + "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", + "0x111111111117dc0aa78b770fa6a738034120c302", + "0xdac17f958d2ee523a2206206994597c13d831ec7", + "0x5f98805a4e8be255a32880fdec7f6728c6568ba0", + "0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f", + "0x5a98fcbea516cf06857215779fd812ca3bef1b32", + "0xc18360217d8f7ab5e7c516566761ea12ce7f9d72", + "0x853d955acef822db058eb8505911ed77f175b99e", + "0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f", + "0xd33526068d116ce69f19a9ee46f0bd304f21a51f", + "0x83f20f44975d03b1b09e64809b757c47f942beea", + "0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6", + "0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202", + "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", + "0xf939e0a03fb07f59a73314e73794be0e57ac1b4e", + "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2" + ], + "processor":{ + "module": "snapshotter.modules.computes.pool_supply_volume", + "class_name": "AssetSupplyVolumeProcessor" + }, + "preload_tasks":[ + "bulk_asset", + "block_details", + "bulk_event" + ] } ] } \ No newline at end of file From ba14f3f25ff37f300c1b1e785643ce76c99d035a Mon Sep 17 00:00:00 2001 From: Seth Date: Wed, 7 Feb 2024 21:31:11 -0500 Subject: [PATCH 12/18] add: multi aggregate for 24h asset volume --- aggregator.example.json | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/aggregator.example.json b/aggregator.example.json index 9a6d9cd..dde2c1a 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -106,6 +106,43 @@ "module": "snapshotter.modules.computes.aggregate.multi_aave_stats", "class_name": "AggreagateMarketStatsProcessor" } + }, + { + "project_type": "aggregate_top_volume", + "aggregate_on": "MultiProject", + "projects_to_wait_for": [ + "aggregate_poolContract_volume_24h:0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48:aavev3", + "aggregate_poolContract_volume_24h:0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2:aavev3", + "aggregate_poolContract_volume_24h:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3", + "aggregate_poolContract_volume_24h:0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0:aavev3", + "aggregate_poolContract_volume_24h:0x6b175474e89094c44da98b954eedeac495271d0f:aavev3", + "aggregate_poolContract_volume_24h:0x514910771af9ca656af840dff83e8264ecf986ca:aavev3", + "aggregate_poolContract_volume_24h:0xae78736cd615f374d3085123a210448e74fc6393:aavev3", + "aggregate_poolContract_volume_24h:0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9:aavev3", + "aggregate_poolContract_volume_24h:0xbe9895146f7af43049ca1c1ae358b0541ea49704:aavev3", + "aggregate_poolContract_volume_24h:0xd533a949740bb3306d119cc777fa900ba034cd52:aavev3", + "aggregate_poolContract_volume_24h:0xba100000625a3754423978a60c9317c58a424e3d:aavev3", + "aggregate_poolContract_volume_24h:0x1f9840a85d5af5bf1d1762f925bdaddc4201f984:aavev3", + "aggregate_poolContract_volume_24h:0x111111111117dc0aa78b770fa6a738034120c302:aavev3", + "aggregate_poolContract_volume_24h:0xdac17f958d2ee523a2206206994597c13d831ec7:aavev3", + "aggregate_poolContract_volume_24h:0x5f98805a4e8be255a32880fdec7f6728c6568ba0:aavev3", + "aggregate_poolContract_volume_24h:0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f:aavev3", + "aggregate_poolContract_volume_24h:0x5a98fcbea516cf06857215779fd812ca3bef1b32:aavev3", + "aggregate_poolContract_volume_24h:0xc18360217d8f7ab5e7c516566761ea12ce7f9d72:aavev3", + "aggregate_poolContract_volume_24h:0x853d955acef822db058eb8505911ed77f175b99e:aavev3", + "aggregate_poolContract_volume_24h:0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f:aavev3", + "aggregate_poolContract_volume_24h:0xd33526068d116ce69f19a9ee46f0bd304f21a51f:aavev3", + "aggregate_poolContract_volume_24h:0x83f20f44975d03b1b09e64809b757c47f942beea:aavev3", + "aggregate_poolContract_volume_24h:0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6:aavev3", + "aggregate_poolContract_volume_24h:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", + "aggregate_poolContract_volume_24h:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", + "aggregate_poolContract_volume_24h:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", + "aggregate_poolContract_volume_24h:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3" + ], + "processor": { + "module": "snapshotter.modules.computes.aggregate.multi_aave_top_volume", + "class_name": "AggreagateTopVolumeProcessor" + } } ] } \ No newline at end of file From ed2ad00eaa87680b4a0c04a772fc694bbded49fd Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 12 Feb 2024 17:08:12 -0500 Subject: [PATCH 13/18] chore: update projects with new asset --- projects.example.json | 6 ++++-- settings.example.json | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/projects.example.json b/projects.example.json index 1654a89..6082a9a 100644 --- a/projects.example.json +++ b/projects.example.json @@ -29,7 +29,8 @@ "0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202", "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", "0xf939e0a03fb07f59a73314e73794be0e57ac1b4e", - "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2" + "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", + "0x6c3ea9036406852006290770BEdFcAbA0e23A0e8" ], "processor":{ "module": "snapshotter.modules.computes.pool_total_supply", @@ -69,7 +70,8 @@ "0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202", "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", "0xf939e0a03fb07f59a73314e73794be0e57ac1b4e", - "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2" + "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", + "0x6c3ea9036406852006290770BEdFcAbA0e23A0e8" ], "processor":{ "module": "snapshotter.modules.computes.pool_supply_volume", diff --git a/settings.example.json b/settings.example.json index e0989d1..4f78f84 100644 --- a/settings.example.json +++ b/settings.example.json @@ -81,7 +81,7 @@ "pair_contract_abi": "snapshotter/static/abis/UniswapV2Pair.json", "protocol_state": { "abi": "snapshotter/static/abis/ProtocolContract.json", - "address": "0xB155EF3cBF0b09e62C6Ae69393e507fab57549C2" + "address": "0xd532ad02DDB0BEF52B20967D607c0816b3E64450" }, "callback_worker_config": { "num_delegate_workers": 0, From c140249f17e5b7b4d739513810ee4856f7dca01d Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 12 Feb 2024 17:46:40 -0500 Subject: [PATCH 14/18] chore: update aggregates for new projects --- aggregator.example.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/aggregator.example.json b/aggregator.example.json index dde2c1a..b4e795b 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -63,7 +63,8 @@ "poolContract_total_supply:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", "poolContract_total_supply:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", "poolContract_total_supply:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", - "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3" + "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", + "poolContract_total_supply:0x6c3ea9036406852006290770BEdFcAbA0e23A0e8:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_top_assets", @@ -100,7 +101,8 @@ "poolContract_total_supply:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", "poolContract_total_supply:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", "poolContract_total_supply:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", - "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3" + "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", + "poolContract_total_supply:0x6c3ea9036406852006290770BEdFcAbA0e23A0e8:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_stats", @@ -137,7 +139,8 @@ "aggregate_poolContract_volume_24h:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", "aggregate_poolContract_volume_24h:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", "aggregate_poolContract_volume_24h:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", - "aggregate_poolContract_volume_24h:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3" + "aggregate_poolContract_volume_24h:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", + "aggregate_poolContract_volume_24h:0x6c3ea9036406852006290770BEdFcAbA0e23A0e8:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_top_volume", From 50a81fe48d0e8404220ef43bde5288d396e62a67 Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 20 Feb 2024 11:23:15 -0500 Subject: [PATCH 15/18] add: new token to projects/aggregates --- aggregator.example.json | 6 +++--- projects.example.json | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aggregator.example.json b/aggregator.example.json index b4e795b..eeb9647 100644 --- a/aggregator.example.json +++ b/aggregator.example.json @@ -64,7 +64,7 @@ "poolContract_total_supply:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", "poolContract_total_supply:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", - "poolContract_total_supply:0x6c3ea9036406852006290770BEdFcAbA0e23A0e8:aavev3" + "poolContract_total_supply:0x6c3ea9036406852006290770bedfcaba0e23a0e8:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_top_assets", @@ -102,7 +102,7 @@ "poolContract_total_supply:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", "poolContract_total_supply:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", - "poolContract_total_supply:0x6c3ea9036406852006290770BEdFcAbA0e23A0e8:aavev3" + "poolContract_total_supply:0x6c3ea9036406852006290770bedfcaba0e23a0e8:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_stats", @@ -140,7 +140,7 @@ "aggregate_poolContract_volume_24h:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", "aggregate_poolContract_volume_24h:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", "aggregate_poolContract_volume_24h:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", - "aggregate_poolContract_volume_24h:0x6c3ea9036406852006290770BEdFcAbA0e23A0e8:aavev3" + "aggregate_poolContract_volume_24h:0x6c3ea9036406852006290770bedfcaba0e23a0e8:aavev3" ], "processor": { "module": "snapshotter.modules.computes.aggregate.multi_aave_top_volume", diff --git a/projects.example.json b/projects.example.json index 6082a9a..810d661 100644 --- a/projects.example.json +++ b/projects.example.json @@ -30,7 +30,7 @@ "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", "0xf939e0a03fb07f59a73314e73794be0e57ac1b4e", "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", - "0x6c3ea9036406852006290770BEdFcAbA0e23A0e8" + "0x6c3ea9036406852006290770bedfcaba0e23a0e8" ], "processor":{ "module": "snapshotter.modules.computes.pool_total_supply", @@ -71,7 +71,7 @@ "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", "0xf939e0a03fb07f59a73314e73794be0e57ac1b4e", "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", - "0x6c3ea9036406852006290770BEdFcAbA0e23A0e8" + "0x6c3ea9036406852006290770bedfcaba0e23a0e8" ], "processor":{ "module": "snapshotter.modules.computes.pool_supply_volume", From 59962e5bf4723f08bb6cb7fb9a84fbf6d25871c1 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 11 Mar 2024 14:58:10 -0400 Subject: [PATCH 16/18] fix: settings.example.json protocol state contract entry for autofill --- settings.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.example.json b/settings.example.json index 4f78f84..e6a8089 100644 --- a/settings.example.json +++ b/settings.example.json @@ -81,7 +81,7 @@ "pair_contract_abi": "snapshotter/static/abis/UniswapV2Pair.json", "protocol_state": { "abi": "snapshotter/static/abis/ProtocolContract.json", - "address": "0xd532ad02DDB0BEF52B20967D607c0816b3E64450" + "address": "protocol-state-contract" }, "callback_worker_config": { "num_delegate_workers": 0, From 582045837fd66b8da1d528410d5984bbe304bcfa Mon Sep 17 00:00:00 2001 From: Seth Date: Wed, 13 Mar 2024 16:26:59 -0400 Subject: [PATCH 17/18] update: rpc polling interval for faster 1h chain --- settings.example.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings.example.json b/settings.example.json index e6a8089..fe3487c 100644 --- a/settings.example.json +++ b/settings.example.json @@ -1,5 +1,5 @@ { - "namespace": "aavev3", + "namespace": "relevant-namespace", "core_api": { "host": "0.0.0.0", "port": 8002, @@ -20,7 +20,7 @@ "retry": 5, "request_time_out": 5, "skip_epoch_threshold_blocks": 30, - "polling_interval": 10, + "polling_interval": 8, "connection_limits":{ "max_connections": 100, "max_keepalive_connections": 50, From ed913d0ad3c97f3f5d901bb225b411ae2cf1bfea Mon Sep 17 00:00:00 2001 From: Seth Date: Wed, 27 Mar 2024 22:43:51 -0400 Subject: [PATCH 18/18] chore: update polling interval --- settings.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.example.json b/settings.example.json index fe3487c..c2971ec 100644 --- a/settings.example.json +++ b/settings.example.json @@ -20,7 +20,7 @@ "retry": 5, "request_time_out": 5, "skip_epoch_threshold_blocks": 30, - "polling_interval": 8, + "polling_interval": 3, "connection_limits":{ "max_connections": 100, "max_keepalive_connections": 50,