2024-06-10-12-11-39

.tool-versions

@@ -0,0 +1 @@
+python 3.11.1

Makefile

@@ -10,7 +10,9 @@ perms:
 up: perms docker-spin-up warehouse-migration
 
 down:
-	docker compose down
+	docker compose down --volumes --rmi all
+
+restart: down up
 
 sh:
 	docker exec -ti webserver bash

containers/airflow/Dockerfile

@@ -1,3 +1,3 @@
-FROM apache/airflow:2.2.0
+FROM apache/airflow:2.9.2
 COPY requirements.txt /
 RUN pip install --no-cache-dir -r /requirements.txt

containers/airflow/requirements.txt

@@ -1,9 +1,10 @@
-black==22.8.0
-flake8==5.0.4
-mypy==0.971
-isort==5.10.1
-moto[all]==4.0.6
-pytest==7.0.1
-pytest-mock==3.6.1
-apache-airflow-client==2.3.0
-yoyo-migrations==8.0.0
+black==24.4.2
+flake8==7.0.0
+mypy==1.10.0
+isort==5.13.2
+moto[all]==5.0.9
+pytest==8.2.2
+pytest-mock==3.14.0
+apache-airflow-client==2.9.0
+yoyo-migrations==8.2.0
+duckdb==1.0.0

dags/coincap_el.py

@@ -0,0 +1,33 @@
+from airflow import DAG
+from datetime import datetime, timedelta
+from airflow.decorators import task
+import requests
+import os
+import csv
+
+
+with DAG(
+    'coincap_el',
+    description='A simple DAG to fetch data from CoinCap Exchanges API and write to a file',
+    schedule_interval=timedelta(days=1),
+    start_date=datetime(2023, 1, 1),
+    catchup=False,
+) as dag:
+
+    @task
+    def fetch_coincap_exchanges():
+        url = "https://api.coincap.io/v2/exchanges"
+        response = requests.get(url)
+        data = response.json()
+
+        exchanges = data['data']
+        if exchanges:
+            keys = exchanges[0].keys()
+            with open(f'{os.getenv("AIRFLOW_HOME")}/dags/files/coincap_exchanges.csv', 'w') as f:
+                dict_writer = csv.DictWriter(f, fieldnames=keys)
+                dict_writer.writeheader()
+                dict_writer.writerows(exchanges)
+
+    fetch_coincap_exchanges()
+
+

dags/files/coincap_exchanges.csv

@@ -0,0 +1,74 @@
+exchangeId,name,rank,percentTotalVolume,volumeUsd,tradingPairs,socket,exchangeUrl,updated
+binance,Binance,1,37.700716079681428695000000000000000000,4279352519.4607505424725000,743,True,https://www.binance.com/,1718035730868
+gdax,Coinbase Pro,2,7.066287232275180771000000000000000000,802083812.0623211646861849,215,True,https://pro.coinbase.com/,1718035749926
+probit,ProBit,3,4.978532687393347699000000000000000000,565105881.6491967855804652,336,False,https://www.probit.com/en-us/,1718035738259
+gate,Gate,4,4.439365712702816601000000000000000000,503905835.8284370979492146,1273,False,https://gate.io/,1718035750443
+bibox,Bibox,5,4.324756757423087369000000000000000000,490896742.8315695938840736,155,False,https://www.bibox.com,1718035747847
+whitebit,WhiteBIT,6,4.294077825079210249000000000000000000,487414422.5981114227110546,87,False,https://whitebit.com,1718035739705
+kraken,Kraken,7,3.884815681605614980000000000000000000,440959682.0278777702894739,322,False,https://kraken.com,1718035749681
+digifinex,DigiFinex,8,3.690633682368124733000000000000000000,418918370.5070409714415671,128,False,https://www.digifinex.com/,1718035750594
+uniswap-v3,Uniswap (V3),9,3.674344176148787290000000000000000000,417069372.7497261642520274,268,False,https://uniswap.org/,1718035750675
+bitmax,AscendEX (Bitmax),10,3.573091537626089247000000000000000000,405576335.5127576437478852,175,False,https://www.bitmax.io/,1718035697005
+huobi,Huobi,11,2.472915202150425480000000000000000000,280696946.9324901085151890,128,True,https://www.hbg.com/,1718035742635
+crypto,Crypto.com Exchange,12,2.286956548750056045000000000000000000,259589055.2345581000390929,116,False,https://api.crypto.com/,1718035750675
+yobit,YoBit,13,2.196360384876938390000000000000000000,249305618.6731806941609672,61,False,https://yobit.net,1718035750501
+lbank,LBank,14,2.141883094461997430000000000000000000,243121982.0149842930462251,95,False,https://www.lbank.info,1718035750175
+kucoin,Kucoin,15,2.065134057375702748000000000000000000,234410312.3340354367297397,564,False,https://www.kucoin.io/,1718035715372
+bigone,BigONE,16,1.956450653844174187000000000000000000,222073819.9516831534438782,61,False,https://big.one/,1718035746109
+tidex,Tidex,17,1.412344036838044219000000000000000000,160313082.6378543087327694,21,False,https://tidex.com/,1718035749511
+currency-com,Currency.com,18,1.376293603365589039000000000000000000,156221051.2562236696485308,70,False,https://currency.com/,1718035750229
+bitstamp,Bitstamp,19,1.205107896633454925000000000000000000,136790014.8840886007295066,95,True,https://www.bitstamp.net/,1718035749702
+bitfinex,Bitfinex,20,1.046291683840976876000000000000000000,118763021.4734518098809364,185,True,https://bitfinex.com,1718035718179
+hitbtc,HitBTC,21,0.557914104206524764000000000000000000,63328004.7634324391745742,692,True,https://hitbtc.com/,1718035739297
+exmo,Exmo,22,0.396976568149045427000000000000000000,45060223.0866129155143308,92,False,https://www.exmo.com/,1718035721062
+btcturk,BTCTurk,23,0.365036657467140337000000000000000000,41434771.0671053632443591,77,False,https://www.btcturk.com/,1718035714574
+dex-trade,Dex-Trade,24,0.361176069845430296000000000000000000,40996561.4763207562759327,87,False,https://dex-trade.com/,1718035735729
+coincheck,coincheck,25,0.337804293363720878000000000000000000,38343665.6968376772892188,1,False,coincheck,1718035743508
+bitflyer,Bitflyer,26,0.309605894596886247000000000000000000,35142907.1607777338843884,4,False,https://bitflyer.com/,1718035745555
+gemini,Gemini,27,0.228502530968990769000000000000000000,25936984.3145316618949062,61,True,https://gemini.com/,1718035749428
+alterdice,Alterdice,28,0.220220799760043066000000000000000000,24996937.2544280079871049,2,False,https://alterdice.com/,1718035750170
+independentreserve,Independent Reserve,29,0.216909308400738050000000000000000000,24621054.7682263248998744,96,False,https://www.independentreserve.com/,1718035750323
+coinex,CoinEx,30,0.188705345731625987000000000000000000,21419664.6818483810984377,487,False,https://www.coinex.com/,1718035750663
+uniswap,Uniswap (V2),31,0.186984277649414630000000000000000000,21224308.7894507827464407,361,False,https://www.uniswap.io/,1718035750673
+bitkub,Bitkub,32,0.165048485450390612000000000000000000,18734409.4619456453413402,43,False,https://www.bitkub.com/,1718035727762
+bitso,Bitso,33,0.124859606572165958000000000000000000,14172629.2634384220146082,27,True,https://www.bitso.com/,1718035720887
+mercatox,Mercatox,34,0.123766237524129274000000000000000000,14048522.5599867638139030,313,False,https://mercatox.com/,1718035738662
+bitbank,Bitbank,35,0.090254243358922879000000000000000000,10244625.6695422952497585,4,False,https://www.bibank.com,1718035748796
+itbit,Itbit,36,0.078951284392393212000000000000000000,8961643.5153426148000000,5,False,https://www.itbit.com/,1718035744057
+indodax,Indodax,37,0.072780406431232643000000000000000000,8261196.2852538106219551,155,False,https://indodax.com/,1718035749278
+luno,Luno,38,0.042958340932188365000000000000000000,4876137.7399696214561359,24,True,https://www.luno.com/,1718035748945
+max-exchange,Max,39,0.039976286754560999000000000000000000,4537649.1809929994603397,22,False,https://max.maicoin.com/,1718035740234
+sushi,SushiSwap,40,0.032866335355183398000000000000000000,3730609.1139061060200294,269,False,https://sushi.com/,1718035727968
+binanceus,Binance.US,41,0.025543018869530807000000000000000000,2899350.2914622570251910,43,False,https://www.binance.us/,1718035731100
+btcmarket,BTCMarket,42,0.020661881836890669000000000000000000,2345299.6465271793806253,10,False,https://www.btcmarkets.net/,1718035744290
+wazirx,WazirX,43,0.0089798493454963937313000000000000000000,1019289.4172038941451539,285,False,https://wazirx.com/,1718035731627
+paribu,Paribu,44,0.0072765089806766497544000000000000000000,825945.7717865321123092,9,False,https://www.paribu.com/,1718035737563
+btcbox,BTCBOX,45,0.0040325934268046054452000000000000000000,457733.7153088624004922,1,False,https://www.btcbox.co.jp,1718035737283
+coinmate,Coinmate,46,0.0039047986591942410640000000000000000000,443227.9202573447725673,3,True,https://coinmate.io/,1718035740253
+gopax,GOPAX,47,0.0009731141950521474601000000000000000000,110456.7529571076469329,31,False,https://www.gopax.co.kr/,1718035740475
+cex,Cex,48,0.0009708938075076185186000000000000000000,110204.7200510806936759,71,False,https://www.cex.io/,1718035710325
+btctradeua,BTC Trade UA,49,0.0000017295270721021649000000000000000000,196.3160598284984982,14,False,https://btc-trade.com.ua/,1718035699098
+okcoin,OkCoin,65,,,0,,https://www.okcoin.com,1681374633809
+mxc-com,MXC.COM,64,,,0,,https://www.mxc.com/,0
+kuna,Kuna,63,,,0,,https://kuna.io/,1677701260207
+kickex,KickEX,62,,,0,,https://kickex.com,1717176256937
+hotbit,Hotbit,61,,,0,,https://www.hotbit.io/,1684920757709
+exmarkets,ExMarkets,60,,,0,,https://exmarkets.com/,1716390699724
+cointiger,CoinTiger,59,,,0,,https://www.cointiger.com/,1707114784242
+coinbene,Coinbene,58,,,0,,https://www.coinbene.com,0
+blockchain-com,Blockchain.com,57,,,0,,https://blockchain.com/,0
+bkex,Bkex,56,,,0,,https://www.bkex.com/,0
+bittrex,Bittrex,55,,,0,,https://bittrex.com/,0
+bitpanda-pro,Bitpanda Pro,53,,,0,,https://www.bitpanda.com/en/pro,1704192020599
+bithumbglobal,Bithumb Global,52,,,0,,https://www.bithumb.pro/,1692326335935
+bitbay,BitBay,51,,,0,,https://bitbay.net,0
+zb,ZB,50,,,0,,https://www.zb.com/,1662264169320
+bitrue,Bitrue,54,,,0,,https://www.bitrue.com/,1718035750455
+qryptos,Liquid,69,,,124,False,https://www.qryptos.com/,1718035701712
+quoine,Quoine,70,,,19,False,https://quoine.com/,1718035708882
+thorchain,THORChain,72,,,0,,https://thorchain.org/,1668435396236
+zaif,Zaif,73,,,0,,https://zaif.jp/,1666250109990
+therocktrading,The Rock Trading,71,,,0,,https://www.therocktrading.com,1676660097265
+poloniex,Poloniex,68,,,0,,https://poloniex.com/,1692177923613
+osmosis,Osmosis,67,,,0,,https://app.osmosis.zone/,0
+okex,Okex,66,,,0,,https://www.okex.com/,0

docker-compose.yml

@@ -71,7 +71,7 @@ x-airflow-common:
 services:
   postgres:
     container_name: postgres
-    image: postgres:13
+    image: postgres:16
     environment:
       POSTGRES_USER: airflow
       POSTGRES_PASSWORD: airflow
@@ -135,7 +135,7 @@ services:
       - "3000:3000"
 
   warehouse:
-    image: postgres:13
+    image: postgres:16
     container_name: warehouse
     environment:
       POSTGRES_USER: ${POSTGRES_USER}