diff --git a/aggregator.example.json b/aggregator.example.json new file mode 100644 index 0000000..eeb9647 --- /dev/null +++ b/aggregator.example.json @@ -0,0 +1,151 @@ +{ + "config": [ + { + "project_type": "aggregate_poolContract_6h_apr", + "aggregate_on": "SingleProject", + "filters": { + "projectId": "poolContract_total_supply" + }, + "processor": { + "module": "snapshotter.modules.computes.aggregate.single_aave_apr_6h", + "class_name": "AggreagateSingleAprProcessor" + } + }, + { + "project_type": "aggregate_poolContract_24h_apr", + "aggregate_on": "SingleProject", + "filters": { + "projectId": "aggregate_poolContract_6h_apr" + }, + "processor": { + "module": "snapshotter.modules.computes.aggregate.single_aave_apr_24h", + "class_name": "AggreagateSingleAprProcessor" + } + }, + { + "project_type": "aggregate_poolContract_volume_24h", + "aggregate_on": "SingleProject", + "filters": { + "projectId": "poolContract_supply_volume" + }, + "processor": { + "module": "snapshotter.modules.computes.aggregate.single_aave_volume_24h", + "class_name": "AggregateSupplyVolumeProcessor" + } + }, + { + "project_type": "aggregate_top_assets", + "aggregate_on": "MultiProject", + "projects_to_wait_for": [ + "poolContract_total_supply:0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48:aavev3", + "poolContract_total_supply:0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2:aavev3", + "poolContract_total_supply:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3", + "poolContract_total_supply:0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0:aavev3", + "poolContract_total_supply:0x6b175474e89094c44da98b954eedeac495271d0f:aavev3", + "poolContract_total_supply:0x514910771af9ca656af840dff83e8264ecf986ca:aavev3", + "poolContract_total_supply:0xae78736cd615f374d3085123a210448e74fc6393:aavev3", + "poolContract_total_supply:0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9:aavev3", + "poolContract_total_supply:0xbe9895146f7af43049ca1c1ae358b0541ea49704:aavev3", + "poolContract_total_supply:0xd533a949740bb3306d119cc777fa900ba034cd52:aavev3", + "poolContract_total_supply:0xba100000625a3754423978a60c9317c58a424e3d:aavev3", + "poolContract_total_supply:0x1f9840a85d5af5bf1d1762f925bdaddc4201f984:aavev3", + "poolContract_total_supply:0x111111111117dc0aa78b770fa6a738034120c302:aavev3", + "poolContract_total_supply:0xdac17f958d2ee523a2206206994597c13d831ec7:aavev3", + "poolContract_total_supply:0x5f98805a4e8be255a32880fdec7f6728c6568ba0:aavev3", + "poolContract_total_supply:0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f:aavev3", + "poolContract_total_supply:0x5a98fcbea516cf06857215779fd812ca3bef1b32:aavev3", + "poolContract_total_supply:0xc18360217d8f7ab5e7c516566761ea12ce7f9d72:aavev3", + "poolContract_total_supply:0x853d955acef822db058eb8505911ed77f175b99e:aavev3", + "poolContract_total_supply:0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f:aavev3", + "poolContract_total_supply:0xd33526068d116ce69f19a9ee46f0bd304f21a51f:aavev3", + "poolContract_total_supply:0x83f20f44975d03b1b09e64809b757c47f942beea:aavev3", + "poolContract_total_supply:0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6:aavev3", + "poolContract_total_supply:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", + "poolContract_total_supply:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", + "poolContract_total_supply:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", + "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", + "poolContract_total_supply:0x6c3ea9036406852006290770bedfcaba0e23a0e8:aavev3" + ], + "processor": { + "module": "snapshotter.modules.computes.aggregate.multi_aave_top_assets", + "class_name": "AggreagateTopAssetsProcessor" + } + }, + { + "project_type": "aggregate_24h_stats", + "aggregate_on": "MultiProject", + "projects_to_wait_for": [ + "poolContract_total_supply:0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48:aavev3", + "poolContract_total_supply:0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2:aavev3", + "poolContract_total_supply:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3", + "poolContract_total_supply:0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0:aavev3", + "poolContract_total_supply:0x6b175474e89094c44da98b954eedeac495271d0f:aavev3", + "poolContract_total_supply:0x514910771af9ca656af840dff83e8264ecf986ca:aavev3", + "poolContract_total_supply:0xae78736cd615f374d3085123a210448e74fc6393:aavev3", + "poolContract_total_supply:0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9:aavev3", + "poolContract_total_supply:0xbe9895146f7af43049ca1c1ae358b0541ea49704:aavev3", + "poolContract_total_supply:0xd533a949740bb3306d119cc777fa900ba034cd52:aavev3", + "poolContract_total_supply:0xba100000625a3754423978a60c9317c58a424e3d:aavev3", + "poolContract_total_supply:0x1f9840a85d5af5bf1d1762f925bdaddc4201f984:aavev3", + "poolContract_total_supply:0x111111111117dc0aa78b770fa6a738034120c302:aavev3", + "poolContract_total_supply:0xdac17f958d2ee523a2206206994597c13d831ec7:aavev3", + "poolContract_total_supply:0x5f98805a4e8be255a32880fdec7f6728c6568ba0:aavev3", + "poolContract_total_supply:0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f:aavev3", + "poolContract_total_supply:0x5a98fcbea516cf06857215779fd812ca3bef1b32:aavev3", + "poolContract_total_supply:0xc18360217d8f7ab5e7c516566761ea12ce7f9d72:aavev3", + "poolContract_total_supply:0x853d955acef822db058eb8505911ed77f175b99e:aavev3", + "poolContract_total_supply:0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f:aavev3", + "poolContract_total_supply:0xd33526068d116ce69f19a9ee46f0bd304f21a51f:aavev3", + "poolContract_total_supply:0x83f20f44975d03b1b09e64809b757c47f942beea:aavev3", + "poolContract_total_supply:0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6:aavev3", + "poolContract_total_supply:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", + "poolContract_total_supply:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", + "poolContract_total_supply:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", + "poolContract_total_supply:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", + "poolContract_total_supply:0x6c3ea9036406852006290770bedfcaba0e23a0e8:aavev3" + ], + "processor": { + "module": "snapshotter.modules.computes.aggregate.multi_aave_stats", + "class_name": "AggreagateMarketStatsProcessor" + } + }, + { + "project_type": "aggregate_top_volume", + "aggregate_on": "MultiProject", + "projects_to_wait_for": [ + "aggregate_poolContract_volume_24h:0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48:aavev3", + "aggregate_poolContract_volume_24h:0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2:aavev3", + "aggregate_poolContract_volume_24h:0x2260fac5e5542a773aa44fbcfedf7c193bc2c599:aavev3", + "aggregate_poolContract_volume_24h:0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0:aavev3", + "aggregate_poolContract_volume_24h:0x6b175474e89094c44da98b954eedeac495271d0f:aavev3", + "aggregate_poolContract_volume_24h:0x514910771af9ca656af840dff83e8264ecf986ca:aavev3", + "aggregate_poolContract_volume_24h:0xae78736cd615f374d3085123a210448e74fc6393:aavev3", + "aggregate_poolContract_volume_24h:0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9:aavev3", + "aggregate_poolContract_volume_24h:0xbe9895146f7af43049ca1c1ae358b0541ea49704:aavev3", + "aggregate_poolContract_volume_24h:0xd533a949740bb3306d119cc777fa900ba034cd52:aavev3", + "aggregate_poolContract_volume_24h:0xba100000625a3754423978a60c9317c58a424e3d:aavev3", + "aggregate_poolContract_volume_24h:0x1f9840a85d5af5bf1d1762f925bdaddc4201f984:aavev3", + "aggregate_poolContract_volume_24h:0x111111111117dc0aa78b770fa6a738034120c302:aavev3", + "aggregate_poolContract_volume_24h:0xdac17f958d2ee523a2206206994597c13d831ec7:aavev3", + "aggregate_poolContract_volume_24h:0x5f98805a4e8be255a32880fdec7f6728c6568ba0:aavev3", + "aggregate_poolContract_volume_24h:0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f:aavev3", + "aggregate_poolContract_volume_24h:0x5a98fcbea516cf06857215779fd812ca3bef1b32:aavev3", + "aggregate_poolContract_volume_24h:0xc18360217d8f7ab5e7c516566761ea12ce7f9d72:aavev3", + "aggregate_poolContract_volume_24h:0x853d955acef822db058eb8505911ed77f175b99e:aavev3", + "aggregate_poolContract_volume_24h:0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f:aavev3", + "aggregate_poolContract_volume_24h:0xd33526068d116ce69f19a9ee46f0bd304f21a51f:aavev3", + "aggregate_poolContract_volume_24h:0x83f20f44975d03b1b09e64809b757c47f942beea:aavev3", + "aggregate_poolContract_volume_24h:0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6:aavev3", + "aggregate_poolContract_volume_24h:0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202:aavev3", + "aggregate_poolContract_volume_24h:0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0:aavev3", + "aggregate_poolContract_volume_24h:0xf939e0a03fb07f59a73314e73794be0e57ac1b4e:aavev3", + "aggregate_poolContract_volume_24h:0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2:aavev3", + "aggregate_poolContract_volume_24h:0x6c3ea9036406852006290770bedfcaba0e23a0e8:aavev3" + ], + "processor": { + "module": "snapshotter.modules.computes.aggregate.multi_aave_top_volume", + "class_name": "AggreagateTopVolumeProcessor" + } + } + ] +} \ No newline at end of file diff --git a/auth_settings.example.json b/auth_settings.example.json new file mode 100644 index 0000000..b6d0e8f --- /dev/null +++ b/auth_settings.example.json @@ -0,0 +1,12 @@ +{ + "redis": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null + }, + "bind": { + "host": "0.0.0.0", + "port": 8555 + } +} diff --git a/preloader.json b/preloader.json new file mode 100644 index 0000000..a0bb82d --- /dev/null +++ b/preloader.json @@ -0,0 +1,32 @@ +{ + "preloaders": [ + { + "task_type": "block_details", + "module": "snapshotter.utils.preloaders.block_details.preloader", + "class_name": "BlockDetailsPreloader" + }, + { + "task_type": "bulk_price", + "module": "snapshotter.modules.computes.utils.preloaders.pricing.preloader", + "class_name": "BulkPricePreloader" + }, + { + "task_type": "bulk_asset", + "module": "snapshotter.modules.computes.utils.preloaders.asset_data.preloader", + "class_name": "BulkAssetDataPreloader" + }, + { + "task_type": "bulk_event", + "module": "snapshotter.modules.computes.utils.preloaders.volume_events.preloader", + "class_name": "AaveBulkVolumeEventsPreloader" + } + ], + "delegate_tasks": [ + { + "task_type": "txreceipt", + "module": "snapshotter.utils.preloaders.tx_receipts.delegated_worker.tx_receipts", + "class_name": "TxReceiptProcessor" + } + ], + "timeout": 60 +} diff --git a/projects.example.json b/projects.example.json new file mode 100644 index 0000000..810d661 --- /dev/null +++ b/projects.example.json @@ -0,0 +1,87 @@ +{ + "config": [ + { + "project_type": "poolContract_total_supply", + "projects":[ + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", + "0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0", + "0x6b175474e89094c44da98b954eedeac495271d0f", + "0x514910771af9ca656af840dff83e8264ecf986ca", + "0xae78736cd615f374d3085123a210448e74fc6393", + "0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9", + "0xbe9895146f7af43049ca1c1ae358b0541ea49704", + "0xd533a949740bb3306d119cc777fa900ba034cd52", + "0xba100000625a3754423978a60c9317c58a424e3d", + "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", + "0x111111111117dc0aa78b770fa6a738034120c302", + "0xdac17f958d2ee523a2206206994597c13d831ec7", + "0x5f98805a4e8be255a32880fdec7f6728c6568ba0", + "0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f", + "0x5a98fcbea516cf06857215779fd812ca3bef1b32", + "0xc18360217d8f7ab5e7c516566761ea12ce7f9d72", + "0x853d955acef822db058eb8505911ed77f175b99e", + "0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f", + "0xd33526068d116ce69f19a9ee46f0bd304f21a51f", + "0x83f20f44975d03b1b09e64809b757c47f942beea", + "0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6", + "0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202", + "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", + "0xf939e0a03fb07f59a73314e73794be0e57ac1b4e", + "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", + "0x6c3ea9036406852006290770bedfcaba0e23a0e8" + ], + "processor":{ + "module": "snapshotter.modules.computes.pool_total_supply", + "class_name": "AssetTotalSupplyProcessor" + }, + "preload_tasks":[ + "bulk_asset", + "block_details" + ] + }, + { + "project_type": "poolContract_supply_volume", + "projects":[ + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", + "0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0", + "0x6b175474e89094c44da98b954eedeac495271d0f", + "0x514910771af9ca656af840dff83e8264ecf986ca", + "0xae78736cd615f374d3085123a210448e74fc6393", + "0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9", + "0xbe9895146f7af43049ca1c1ae358b0541ea49704", + "0xd533a949740bb3306d119cc777fa900ba034cd52", + "0xba100000625a3754423978a60c9317c58a424e3d", + "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", + "0x111111111117dc0aa78b770fa6a738034120c302", + "0xdac17f958d2ee523a2206206994597c13d831ec7", + "0x5f98805a4e8be255a32880fdec7f6728c6568ba0", + "0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f", + "0x5a98fcbea516cf06857215779fd812ca3bef1b32", + "0xc18360217d8f7ab5e7c516566761ea12ce7f9d72", + "0x853d955acef822db058eb8505911ed77f175b99e", + "0x40d16fc0246ad3160ccc09b8d0d3a2cd28ae6c2f", + "0xd33526068d116ce69f19a9ee46f0bd304f21a51f", + "0x83f20f44975d03b1b09e64809b757c47f942beea", + "0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6", + "0xdefa4e8a7bcba345f687a2f1456f5edd9ce97202", + "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", + "0xf939e0a03fb07f59a73314e73794be0e57ac1b4e", + "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", + "0x6c3ea9036406852006290770bedfcaba0e23a0e8" + ], + "processor":{ + "module": "snapshotter.modules.computes.pool_supply_volume", + "class_name": "AssetSupplyVolumeProcessor" + }, + "preload_tasks":[ + "bulk_asset", + "block_details", + "bulk_event" + ] + } + ] +} \ No newline at end of file diff --git a/settings.example.json b/settings.example.json new file mode 100644 index 0000000..c2971ec --- /dev/null +++ b/settings.example.json @@ -0,0 +1,144 @@ +{ + "namespace": "relevant-namespace", + "core_api": { + "host": "0.0.0.0", + "port": 8002, + "auth": { + "enabled": false, + "header_key": "X-API-KEY" + }, + "public_rate_limit": "20000/day;300/minute;10/second" + }, + "instance_id": "account-address", + "rpc": { + "full_nodes": [{ + "url": "https://rpc-url", + "rate_limit": "100000000/day;20000/minute;1000/second" + }], + "archive_nodes": [], + "force_archive_blocks": 100, + "retry": 5, + "request_time_out": 5, + "skip_epoch_threshold_blocks": 30, + "polling_interval": 3, + "connection_limits":{ + "max_connections": 100, + "max_keepalive_connections": 50, + "keepalive_expiry": 300 + } + }, + "rlimit": { + "file_descriptors": 2048 + }, + "rabbitmq": { + "user": "guest", + "password": "guest", + "host": "rabbitmq", + "port": 5672, + "setup": { + "core": { + "exchange": "powerloom-backend" + }, + "callbacks": { + "exchange": "powerloom-backend-callbacks" + }, + "commit_payload": { + "exchange": "powerloom-backend-commit-payload" + }, + "event_detector": { + "exchange": "powerloom-backend-event-detector" + }, + "delegated_worker": { + "exchange": "powerloom-backend-delegated-worker" + } + } + }, + "reporting": { + "slack_url": "https://slack-reporting-url", + "service_url": "https://powerloom-reporting-url" + }, + "redis": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null, + "ssl": false, + "cluster_mode": false + }, + "redis_reader": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null + }, + "logs": { + "trace_enabled": false, + "write_to_files": true + }, + "projects_config_path": "config/projects.json", + "aggregator_config_path": "config/aggregator.json", + "preloader_config_path": "config/preloader.json", + "pair_contract_abi": "snapshotter/static/abis/UniswapV2Pair.json", + "protocol_state": { + "abi": "snapshotter/static/abis/ProtocolContract.json", + "address": "protocol-state-contract" + }, + "callback_worker_config": { + "num_delegate_workers": 0, + "num_snapshot_workers": 8, + "num_aggregation_workers": 12 + }, + "ipfs": { + "url": "ipfs-writer-url", + "url_auth": { + "apiKey": "ipfs-writer-key", + "apiSecret": "ipfs-writer-secret" + }, + "reader_url": "ipfs-reader-url", + "reader_url_auth": { + "apiKey": "ipfs-reader-key", + "apiSecret": "ipfs-reader-secret" + }, + "write_rate_limit": { + "req_per_sec": 10, + "burst": 10 + }, + "timeout": 60, + "local_cache_path": "/home/ubuntu/local_cache/", + "connection_limits":{ + "max_connections": 100, + "max_keepalive_connections": 50, + "keepalive_expiry": 300 + }, + "remote_pinning": { + "enabled": false, + "service_name": "", + "service_endpoint": "", + "service_token": "", + "background_pinning": false + } + }, + "web3storage":{ + "upload_snapshots": false, + "upload_aggregates": true, + "url": "https://api.web3.storage", + "upload_url_suffix": "/upload", + "api_token": "web3-storage-token", + "timeout": 30, + "max_idle_conns": 5, + "idle_conn_timeout": 300 + }, + "anchor_chain_rpc": { + "full_nodes": [{ + "url": "https://prost-rpc-url", + "rate_limit": "100000000/day;20000/minute;1000/second" + }], + "retry": 5, + "request_time_out": 5, + "connection_limits":{ + "max_connections": 100, + "max_keepalive_connections": 50, + "keepalive_expiry": 300 + } + } +} \ No newline at end of file