Momentum_bt_1.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Jupyter Extensions - convenience as an extension\n",
    "For Jupyter to bring more added value I recommend you to install extensions\n",
    "pip Commands:\n",
    "\n",
    "pip install jupyter_nbextensions_configurator jupyter_contrib_nbextensions\n",
    "jupyter contrib nbextension install --user\n",
    "jupyter nbextensions_configurator enable --user\n",
    "\n",
    "My Favorites are:\n",
    "* Collapsible headings\n",
    "* Code folding\n",
    "* Notify\n",
    "* Hinterland (can be nice, but I think I didn't come to use it correctly)\n",
    "              \n",
    "Thanks to https://becominghuman.ai/enhance-your-jupyter-experience-with-these-notebook-widgets-a2717921f678"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "code_folding": [],
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:90% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# stretch the code to 90% of the browser width\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:90% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Potential error messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#Indexing errors, i.e. ValueError: Length of values (3519) does not match length of index (3529)\n",
    "    #There are actually 3 data input:\n",
    "    #1. The folder with all the stocks, 2. the benchmark (I tend to use QQQ), 3. the bond (I used TLT)\n",
    "    #These errors are in my experience due to one or several of the input csvs do not cover the time span inserted in the \"fromdate\" / \"todate\" fields in globalparams, \n",
    "    #i.e. the todate goes to 2021 althogh the csvs only cover until mid 2020\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import glob\n",
    "import webbrowser\n",
    "import os.path\n",
    "import os\n",
    "import pandas as pd\n",
    "import backtrader as bt\n",
    "import backtrader.analyzers as btanal\n",
    "import pyfolio as pf\n",
    "import numpy as np\n",
    "import sys\n",
    "from scipy.stats import linregress\n",
    "import quantstats as qs\n",
    "import seaborn as sns\n",
    "import optunity.metrics\n",
    "import math\n",
    "import plotly.graph_objects as go\n",
    "import multiprocess #can pickle when multiprocessing can't\n",
    "\n",
    "from matplotlib.colors import LinearSegmentedColormap\n",
    "from matplotlib.patches import Rectangle\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import warnings \n",
    "warnings.simplefilter(action='ignore', category=Warning)\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "#Imports for walkforward analysis\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "from sklearn.utils import indexable\n",
    "from sklearn.utils.validation import _num_samples\n",
    "import backtrader.indicators as btind\n",
    "import datetime as dt\n",
    "import pandas as pd\n",
    "import pandas_datareader as web\n",
    "from pandas import Series, DataFrame\n",
    "import random\n",
    "import optunity.metrics\n",
    "import backtrader.analyzers as btanal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "globalparams = dict(path='/home/mmd/Data/NASDAQ2021/',      # location of your csv files, downloaded from Yahoo.\n",
    "                    fromdate=datetime.datetime(2020, 3, 5),\n",
    "                    todate=datetime.datetime(2021, 3, 13),\n",
    "                    strategy=\"St\",                          # HSt = Buy and Hold, St = Standard Strategy for non-survivorship free bias momentum\n",
    "                    update_csv=False,                        # If True, the csv files will be updated with the timeframe given in fromdate/todate\n",
    "                    \n",
    "                    # Momentum Strategy specifics(= St)        \n",
    "                    mperiod=200,           # lookback period for momentum - default 90 periods\n",
    "                    selcperc=2,            # number of stocks to select from the universe\n",
    "                    time_interval=\"week\", # if it is not \"week\" then it is \"month\"\n",
    "                    time_between=1,        # time between time interval, e.g. 1 means weekly, 2 means every 2 weeks etc. same for months\n",
    "                    \n",
    "                    roc = True,        # Whether the rate of change should be taken into consideration\n",
    "                    rocperiod=200,     # rate of change to make sure stock is increasing\n",
    "                    \n",
    "                    rebal=True,            # apply proper rebalancing when stocks are sold than have a share above the equal weights\n",
    "\n",
    "                    #General optimization parameters\n",
    "                    params_combination= \"23\",               # 1: # of stocks picked from universe, 2: momentum period, 3: roc period\n",
    "                    optimization_target=\"SQN\",          # Relevant for heatmap & optunity. \n",
    "                                                            # Potential Inputs: returns, Sharpe, Sortino, SQN (needs more than 30 trades!), VWR (variablility weighted returns)\n",
    "                    heatmap_yaxisr=np.arange(1, 7, 3),      # y axis range of parameters for heatmap\n",
    "                    heatmap_xaxisr=np.arange(100, 210, 50), # x axis (mperiod) range of parameters for heatmap\n",
    "                    heatmap_zaxisr=np.arange(100, 210, 50), # z axis (ROC) range of parameters for heatmap                    \n",
    "                    \n",
    "                    #Heatmap section\n",
    "                    heatmap=False,  # Does not work with Hold and Buy, Multiprocessing only here implemented (works in normal and walkforward runs)                  \n",
    "                    \n",
    "                    #optimization with optunity (various algorithms e.g. Partical Swarm Optimization)\n",
    "                    optunity=False, # Either optunity or heatmap, Multiprocessing not implemented\n",
    "                    num_eval=1,    # how many evaluations optunity will try // also for Walk Forward Analysis                    \n",
    "                    \n",
    "                    #Walk-forward-analysis, either optunity or heatmap must be switched on\n",
    "                    walk=False,\n",
    "                    fixed_length=False, # by setting it False the training data will will grow over time, otherwise it will keep the size given under train_splits\n",
    "                    n_splits=3,         # how many chunks the data should have\n",
    "                    \n",
    "                    cash=1000000,               # amount of initial captal      \n",
    "                    commission=0.005,           # broker fee/ask/bid spread\n",
    "                    reserve=0.05,               # % of cash that won't be spent due to otherwise canceled orders\n",
    "                    printlog=True,\n",
    "                    ranking_timeanalysis=True, # excel file with rankings of all stocks over time, only works in \"normal\" run (no heatmap/Walk-forward/optunity), \n",
    "                                                # if it doesn't work, shorten the time period in fromdate\n",
    "                    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading data from Yahoo\n",
    "Enter symbol, from , to, path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "if globalparams[\"update_csv\"]:\n",
    "    import yahoofinance as yf\n",
    "    from csv import reader\n",
    "    # open file in read mode\n",
    "    from1 = globalparams[\"fromdate\"].strftime('%Y-%m-%d')\n",
    "    to = globalparams[\"todate\"].strftime('%Y-%m-%d')\n",
    "    #Download all \n",
    "    with open(r'/home/mmd/Data/NASDAQ2021.csv', 'r') as read_obj:\n",
    "        # pass the file object to reader() to get the reader object\n",
    "        csv_reader = reader(read_obj)\n",
    "        # Iterate over each row in the csv using reader object\n",
    "        for row in csv_reader:\n",
    "            # row variable is a list that represents a row in csv\n",
    "            string = ''\n",
    "            path = r'/home/mmd/Data/NASDAQ2021/'\n",
    "            hist = yf.HistoricalPrices(string.join(row),  from1, to, frequency='1d')\n",
    "            hist.to_csv(path + string.join(row) + '.csv')\n",
    "    #Download Benchmark\n",
    "    row = \"QQQ\"\n",
    "    string = ''\n",
    "    path = '/home/mmd/Data/'\n",
    "    hist = yf.HistoricalPrices(string.join(row),  from1, to, frequency='1d')\n",
    "    hist.to_csv(path + string.join(row) + '.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function section\n",
    "Momentum (with and without volatility correction) [thanks to Teddy Koker https://teddykoker.com/2019/05/momentum-strategy-from-stocks-on-the-move-in-python/ ]  and heatmap for analysis of variable combinations and [thanks to samwindham1 https://github.com/samwindham1/algo-trader/blob/master/backtest/util/analyzers.py ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     24,
     151
    ]
   },
   "outputs": [],
   "source": [
    "#Momentum + ROC\n",
    "class MomentumEnhanced(bt.Indicator):\n",
    "    lines = ('trend',)\n",
    "    params = (('period', 200), ('rperiod', 200))\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.addminperiod(self.params.period)\n",
    "        if globalparams[\"roc\"]:\n",
    "            self.roc = bt.ind.ROC(self.data, period=self.p.rperiod)\n",
    "                \n",
    "    def next(self):              \n",
    "        #Momentum\n",
    "        returns = np.log(self.data.get(size=self.p.period))\n",
    "        x = np.arange(len(returns))\n",
    "        slope, _, rvalue, _, _ = linregress(x, returns)\n",
    "        momentum_annualized = (1 + slope) ** 252\n",
    "        \n",
    "        #ROC\n",
    "        rocround = math.ceil(self.roc[0]) if globalparams[\"roc\"] else 1\n",
    "        roc_final = min(1, max(0, rocround)) if globalparams[\"roc\"] else 1\n",
    "\n",
    "        #final line formula\n",
    "        self.lines.trend[0] = momentum_annualized * roc_final\n",
    "\n",
    "# Great Heatmap to analyse different combinations of parameters\n",
    "def my_heatmap1(data):\n",
    "    data = np.array(data)\n",
    "    xs = np.unique(data[:, 1].astype(int))\n",
    "    ys = np.unique(data[:, 0].astype(int))\n",
    "    vals = data[:, 2].reshape(len(ys), len(xs))\n",
    "    \n",
    "    min_val_ndx = np.unravel_index(np.argmin(vals, axis=None), vals.shape)\n",
    "    max_val_ndx = np.unravel_index(np.argmax(vals, axis=None), vals.shape)\n",
    "    \n",
    "    cmap = LinearSegmentedColormap.from_list('', ['red', 'orange', 'yellow', 'chartreuse', 'limegreen'])\n",
    "    ax = sns.heatmap(vals, xticklabels=xs, yticklabels=ys, cmap='viridis', annot=True, fmt='.2f')\n",
    "\n",
    "    ax.add_patch(Rectangle(min_val_ndx[::-1], 1, 1, fill=False, edgecolor='blue', lw=3, clip_on=False))\n",
    "    ax.add_patch(Rectangle(max_val_ndx[::-1], 1, 1, fill=False, edgecolor='blue', lw=3, clip_on=False))\n",
    "    print(\"Average of all iterations\", [sum(i)/len(i) for i in zip(*data)][2])\n",
    "    #plt.tight_layout()\n",
    "    \n",
    "    plt.show()\n",
    "\n",
    "def plotly_4d(data):\n",
    "    df1 = pd.DataFrame.from_records(data, columns=['selcperc', 'mperiod', 'rocperiod', 'value'])\n",
    "    df = df1[~(df1['selcperc'].isin([2]))]\n",
    "    df= df.pivot_table(index=['mperiod'], columns=['rocperiod'], values='value').reset_index()\n",
    "    del df['mperiod']\n",
    "    df.drop(df.index[0])\n",
    "    print(df)\n",
    "    fig = go.Figure(data=[go.Surface(z=df.values)])\n",
    "    fig.update_traces(contours_z=dict(show=True, usecolormap=True,\n",
    "                                      highlightcolor=\"limegreen\", project_z=True))\n",
    "    fig.update_layout(title_text=\"Analysis of Strategy\", autosize=False, width=700, height=700,\n",
    "                  margin=dict(l=65, r=50, b=65, t=90))\n",
    "    fig.show() \n",
    "  \n",
    "    \n",
    "class Sortino(bt.Analyzer):\n",
    "    params = (('timeframe', bt.TimeFrame.Days),('compression', 1),('riskfreerate', 0.00),('factor', None),('convertrate', True),('annualize', True),)\n",
    "\n",
    "    RATEFACTORS = {\n",
    "        bt.TimeFrame.Days: 252,\n",
    "        bt.TimeFrame.Weeks: 52,\n",
    "        bt.TimeFrame.Months: 12,\n",
    "        bt.TimeFrame.Years: 1,\n",
    "    }\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Sortino, self).__init__()\n",
    "        self.ret = btanal.TimeReturn(\n",
    "            timeframe=self.p.timeframe,\n",
    "            compression=self.p.compression)\n",
    "        self.ratio = 0.0\n",
    "\n",
    "    def stop(self):\n",
    "        returns = list(self.ret.get_analysis().values())\n",
    "\n",
    "        rate = self.p.riskfreerate\n",
    "\n",
    "        factor = None\n",
    "        if self.p.timeframe in self.RATEFACTORS:\n",
    "            # Get the conversion factor from the default table\n",
    "            factor = self.RATEFACTORS[self.p.timeframe]\n",
    "\n",
    "        if factor is not None:\n",
    "            # A factor was found\n",
    "            if self.p.convertrate:\n",
    "                # Standard: downgrade annual returns to timeframe factor\n",
    "                rate = pow(1.0 + rate, 1.0 / factor) - 1.0\n",
    "            else:\n",
    "                # Else upgrade returns to yearly returns\n",
    "                returns = [pow(1.0 + x, factor) - 1.0 for x in returns]\n",
    "\n",
    "        if len(returns):\n",
    "            # Sortino Ratio = (R - T) / TDD\n",
    "            #   R = Avg Returns\n",
    "            #   T = Target (risk-free rate)\n",
    "            #   TDD = Downside Risk\n",
    "            ret_free_avg = np.mean(returns) - rate\n",
    "            tdd = math.sqrt(np.mean([min(0, r - rate)**2 for r in returns]))\n",
    "\n",
    "            try:\n",
    "                ratio = ret_free_avg / tdd\n",
    "\n",
    "                if factor is not None and \\\n",
    "                        self.p.convertrate and self.p.annualize:\n",
    "\n",
    "                    ratio = math.sqrt(factor) * ratio\n",
    "            except (ValueError, TypeError, ZeroDivisionError):\n",
    "                ratio = None\n",
    "\n",
    "        else:\n",
    "            # no returns\n",
    "            ratio = None\n",
    "\n",
    "        self.ratio = ratio\n",
    "\n",
    "    def get_analysis(self):\n",
    "        return dict(sortino=self.ratio)\n",
    "\n",
    "# Picking optimization target and subsequent analyser functions for optimization\n",
    "def opta(i):\n",
    "    switcher={\n",
    "            \"VWR\":['cerebro.addanalyzer(btanal.VWR, _name=\"VWR\")','results1[0].analyzers.VWR.get_analysis()[\"vwr\"]'],\n",
    "            \"Sortino\":['cerebro.addanalyzer(Sortino, _name=\"sortino\")','results1[0].analyzers.sortino.get_analysis()[\"sortino\"]'],\n",
    "            \"returns\":['1==1','cerebro.broker.getvalue()'], #TODO must be tested separately\n",
    "            \"Sharpe\":['cerebro.addanalyzer(btanal.SharpeRatio, _name=\"sharpe\")','results1[0].analyzers.sharpe.get_analysis()[\"sharperatio\"]'],\n",
    "            \"SQN\":['cerebro.addanalyzer(btanal.SQN, _name=\"sqn\")','results1[0].analyzers.sqn.get_analysis()[\"sqn\"]'],\n",
    "         }\n",
    "    return switcher.get(i,\"Optimization Target not available\")\n",
    "\n",
    "def params_comb(i): \n",
    "    switcher={\n",
    "            \"12\":[['globalparams[\"heatmap_yaxisr\"]','globalparams[\"heatmap_xaxisr\"]','[globalparams[\"rocperiod\"],globalparams[\"rocperiod\"]*1.0001]'],['results_list.append([i, j, k, PnL])','2','1','2']], #p1, p2, p3 in optunity maximize\n",
    "            \"13\":[['globalparams[\"heatmap_yaxisr\"]','[globalparams[\"mperiod\"],globalparams[\"mperiod\"]*1.0001]','globalparams[\"heatmap_zaxisr\"]'],['results_list.append([i, k, j, PnL])','1','2','1']],\n",
    "            \"23\":[['[globalparams[\"selcperc\"],globalparams[\"selcperc\"]*1.0001]','globalparams[\"heatmap_xaxisr\"]','globalparams[\"heatmap_zaxisr\"]'],['results_list.append([j, k, i, PnL])','0','0','1']],\n",
    "            \"123\":[['globalparams[\"heatmap_yaxisr\"]','globalparams[\"heatmap_xaxisr\"]','globalparams[\"heatmap_zaxisr\"]'],['results_list.append([i, j, k, PnL])','0','1','2']],\n",
    "         }\n",
    "    return switcher.get(i,\"Optimization Target not available\")\n",
    "\n",
    "def params_comb_optunity(i): \n",
    "    switcher={\n",
    "            \"12\":[['[globalparams[\"heatmap_yaxisr\"][0],globalparams[\"heatmap_yaxisr\"][-1]]','[globalparams[\"heatmap_xaxisr\"][0],globalparams[\"heatmap_xaxisr\"][-1]]','[globalparams[\"rocperiod\"],globalparams[\"rocperiod\"]*1.0001]'],['results_list.append([i, j, k, PnL])','0','1','2']], #p1, p2, p3 in optunity maximize\n",
    "            \"13\":[['[globalparams[\"heatmap_yaxisr\"][0],globalparams[\"heatmap_yaxisr\"][-1]]','[globalparams[\"mperiod\"],globalparams[\"mperiod\"]*1.0001]','[globalparams[\"heatmap_zaxisr\"][0],globalparams[\"heatmap_zaxisr\"][-1]]'],['results_list.append([i, k, j, PnL])','0','2','1']],\n",
    "            \"23\":[['[globalparams[\"selcperc\"],globalparams[\"selcperc\"]*1.0001]','[globalparams[\"heatmap_xaxisr\"][0],globalparams[\"heatmap_xaxisr\"][-1]]','[globalparams[\"heatmap_zaxisr\"][0],globalparams[\"heatmap_zaxisr\"][-1]]'],['results_list.append([j, k, i, PnL])','2','0','1']],\n",
    "            \"123\":[['[globalparams[\"heatmap_yaxisr\"][0],globalparams[\"heatmap_yaxisr\"][-1]]','[globalparams[\"heatmap_xaxisr\"][0],globalparams[\"heatmap_xaxisr\"][-1]]','[globalparams[\"heatmap_zaxisr\"][0],globalparams[\"heatmap_zaxisr\"][-1]]'],['results_list.append([i, j, k, PnL])','0','1','2']],\n",
    "         }\n",
    "    return switcher.get(i,\"Optimization Target not available\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Momentum Strategy\n",
    "Originally it is a combination of the momentum strategy example given on the official website:\n",
    "https://www.backtrader.com/blog/2019-05-20-momentum-strategy/momentum-strategy/\n",
    "and  https://teddykoker.com/2019/05/momentum-strategy-from-stocks-on-the-move-in-python/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     93
    ],
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "class St(bt.Strategy):\n",
    "    params = dict(\n",
    "        selcperc=globalparams[\"selcperc\"],  # percentage of stocks to select from the universe\n",
    "        mperiod=globalparams[\"mperiod\"],  # lookback period for momentum - default 90 periods\n",
    "        rocperiod=globalparams[\"rocperiod\"],   # rate of change to make sure stock is increasing\n",
    "        momentumplus=MomentumEnhanced, # choose which momentum function is used\n",
    "        reserve=globalparams[\"reserve\"],  # 5% reserve capital\n",
    "        monthdays=[1],\n",
    "        monthcarry=True,\n",
    "        weekdays=[5],\n",
    "        weekcarry=True,\n",
    "        printlog=globalparams[\"printlog\"],\n",
    "        WFA=False,          #Switch for the last \"putting the fragments together run\" of the Walk Forward Analysis\n",
    "        start_dates=None,  # Starting days for trading periods (a list)\n",
    "        end_dates=None,\n",
    "        \n",
    "    )\n",
    "\n",
    "    def __init__(self):\n",
    "        self.bench = self.data0\n",
    "        #self.bond = self.data1\n",
    "        self.stocks = self.datas[1:]\n",
    "        # calculate 1st the amount of stocks that will be selected\n",
    "\n",
    "        # counter for time in between\n",
    "        self.timer_i = 0\n",
    "        self.posdata =[]\n",
    "        \n",
    "        # This is the set up of the timer that makes the strategy being executed at the given time\n",
    "        if globalparams[\"time_interval\"] == \"week\":\n",
    "            self.add_timer(\n",
    "                when=bt.timer.SESSION_START,\n",
    "                weekdays=self.p.weekdays,\n",
    "                weekcarry=self.p.weekcarry,\n",
    "            )  \n",
    "        else:\n",
    "            self.add_timer(\n",
    "                when=bt.timer.SESSION_START,\n",
    "                monthdays=self.p.monthdays,\n",
    "                monthcarry=self.p.monthcarry,\n",
    "            )\n",
    "\n",
    "        # preparation for Walk Forward Analysis\n",
    "        if self.p.WFA == True:\n",
    "            self.mperiod = dict()\n",
    "            self.vperiod = dict()\n",
    "            self.ms = dict()\n",
    "            self.selnum = dict()\n",
    "\n",
    "            self.date_combos = [c for c in zip(self.p.start_dates, self.p.end_dates)]\n",
    "\n",
    "            # Additional indexing, allowing for differing start/end dates\n",
    "            for sd, ed, f, s, t in zip(self.p.start_dates, self.p.end_dates, self.p.mperiod, self.p.rocperiod, self.p.selcperc):\n",
    "                self.selnum[(sd, ed)] = int(t) # of stocks bought\n",
    "                \n",
    "                # returns, volatilities and momentums\n",
    "                self.ms[(sd, ed)] = [self.p.momentumplus(d, period=int(f), rperiod=int(s)) for d in self.stocks] #toDo how to implement it with two parameters\n",
    "        \n",
    "        #Non walk forward analysis\n",
    "        else:\n",
    "            self.selnum = int(self.p.selcperc)\n",
    "            # allocation perc per stock\n",
    "            ms = [self.p.momentumplus(d,period=int(self.p.mperiod), rperiod=int(self.p.rocperiod)) for d in self.stocks]\n",
    "\n",
    "            # simple rank formula: (momentum * net payout) / volatility\n",
    "            # the highest ranked: low vol, large momentum, large payout\n",
    "            self.ranks = {d: m for d, m in zip(self.stocks, ms)}\n",
    "            \n",
    "                \n",
    "    def log(self, arg):\n",
    "        if self.p.printlog:\n",
    "            print('{} {}'.format(self.datetime.date(), arg))\n",
    "        \n",
    "    # This section is for logging of orders in greater detail to figure out whether the strategy is actually having no problem with orders\n",
    "    def notify_order(self, order):\n",
    "        if order.status in [order.Accepted]:\n",
    "            # Buy/Sell order submitted/accepted to/by broker - Nothing to do\n",
    "            return\n",
    "        if order.status in [order.Submitted]:\n",
    "            if order.isbuy():\n",
    "            \n",
    "                dt, dn = self.datetime.date(), order.data._name\n",
    "                self.log('Buy {} {} {} Price {:.2f} Value {:.2f} Size {} Cash {:.2f}'.format(\n",
    "                        order.getstatusname(), dt, dn, order.created.price, order.created.size * order.created.price , order.created.size, self.broker.getcash()))\n",
    "            if order.issell():\n",
    "                dt, dn = self.datetime.date(), order.data._name\n",
    "                self.log('Sell {} {} {} Price {:.2f} Value {:.2f} Size {}'.format(\n",
    "                        order.getstatusname(), dt, dn, order.created.price, order.created.size * order.created.price, order.created.size))\n",
    "\n",
    "            # Buy/Sell order submitted/accepted to/by broker - Nothing to do\n",
    "            return\n",
    "\n",
    "        # Check if an order has been completed\n",
    "        # Attention: broker could reject order if not enough cash\n",
    "        if order.status in [order.Completed]:\n",
    "            if order.isbuy():\n",
    "                dt, dn = self.datetime.date(), order.data._name\n",
    "                self.log('Buy {} {} Price {:.2f} Value {:.2f} Size {}'.format(\n",
    "                    dt, dn, order.executed.price, order.executed.value, order.executed.size))\n",
    "                \n",
    "            if order.issell():# Sell\n",
    "                dt, dn = self.datetime.date(), order.data._name\n",
    "                self.log('Sell {} {} Price {:.2f} Value {:.2f} Size {}'.format(\n",
    "                    dt, dn, order.executed.price, order.executed.price*order.executed.size, order.executed.size))\n",
    "\n",
    "        elif order.status in [order.Canceled, order.Margin, order.Rejected]:\n",
    "            self.log('Order Canceled/Margin/Rejected')\n",
    "           \n",
    "    \n",
    "    # This is the function using the timer to execute the rebalance \n",
    "    def notify_timer(self, timer, when, *args, **kwargs):\n",
    "        #print('strategy notify_timer with tid {}, when {} _getminperstatus {}'.\n",
    "        #      format(timer.p.tid, when, int(self._getminperstatus())))\n",
    "        if self._getminperstatus() < 0:\n",
    "            self.timer_i += 1\n",
    "            if self.timer_i == globalparams[\"time_between\"]:\n",
    "                self.timer_i = 0\n",
    "                self.rebalance()\n",
    "  \n",
    "    def next(self):\n",
    "        pass # must be filled with a pass\n",
    "\n",
    "    \n",
    "    # Actual order giving by a ranking takes place here\n",
    "    def rebalance(self):\n",
    "        \n",
    "        # Walkforward Analysis Section (no triple momentum possible)\n",
    "        if self.p.WFA == True:\n",
    "            # Determine which set of moving averages to use\n",
    "            curdate = self.datetime.date(0)\n",
    "            dtidx = None  # Will be index\n",
    "            # Determine which period (if any) we are in\n",
    "            for sd, ed in self.date_combos:\n",
    "                if sd <= curdate and curdate <= ed:\n",
    "                    dtidx = (sd, ed)\n",
    "                    self.ranks = {d: m for d, m in zip(self.stocks, self.ms[dtidx])}\n",
    "                    \n",
    "                    # sort data and current rank\n",
    "                    ranks = sorted(\n",
    "                        self.ranks.items(),  # get the (d, rank), pair\n",
    "                        key=lambda x: x[1][0],  # use rank (elem 1) and current time \"0\"\n",
    "                        reverse=True,  # highest ranked 1st ... please\n",
    "                    )\n",
    "\n",
    "                    # put top ranked in dict with data as key to test for presence\n",
    "                    rtop = dict(ranks[:self.selnum[dtidx]])\n",
    "\n",
    "                    # For logging purposes of stocks leaving the portfolio\n",
    "                    rbot = dict(ranks[self.selnum[dtidx]:])\n",
    "\n",
    "                    # prepare quick lookup list of stocks currently holding a position\n",
    "                    self.posdata = [d for d, pos in self.getpositions().items() if pos]\n",
    "                    \n",
    "            if dtidx is None:  # Not in any window\n",
    "                pass # Don't engage in trades\n",
    "            else:\n",
    "                self.momentum_core(self.posdata, rtop, rbot, dtidx)             \n",
    "\n",
    "        # Non walk forward section\n",
    "        else:\n",
    "            # sort data and current rank\n",
    "            ranks = sorted(\n",
    "                self.ranks.items(),  # get the (d, rank), pair\n",
    "                key=lambda x: x[1][0],  # use rank (elem 1) and current time \"0\"\n",
    "                reverse=True,  # highest ranked 1st ... please\n",
    "            )\n",
    "            \n",
    "            # put top ranked in dict with data as key to test for presence\n",
    "            rtop = dict(ranks[:self.selnum])\n",
    "\n",
    "            # For logging purposes of stocks leaving the portfolio\n",
    "            rbot = dict(ranks[self.selnum:])\n",
    "            \n",
    "            # prepare quick lookup list of stocks currently holding a position\n",
    "            self.posdata = [d for d, pos in self.getpositions().items() if pos]\n",
    "                \n",
    "            self.momentum_core(self.posdata, rtop, rbot, 0)\n",
    "            \n",
    "    def momentum_core(self, posdata, rtop, rbot, dtidx):\n",
    "        # remove those no longer top ranked\n",
    "        # do this first to issue sell orders and free cash\n",
    "        sell = False\n",
    "        for d in (d for d in posdata if d not in rtop):\n",
    "            self.log('Leave {} - Rank {:.2f}'.format(d._name, rbot[d][0]))\n",
    "            self.order_target_percent(d, target=0.0)\n",
    "            if self.p.WFA == True:\n",
    "                if (0.95 / self.selnum[dtidx]) < self.broker.getposition(d).size * self.broker.getposition(d).adjbase / self.broker.get_value():\n",
    "                    sell = globalparams[\"rebal\"]\n",
    "            else:\n",
    "                if (0.95 / self.selnum) < self.broker.getposition(d).size * self.broker.getposition(d).adjbase / self.broker.get_value():\n",
    "                    sell = globalparams[\"rebal\"]\n",
    "\n",
    "        sum_stocks = 0\n",
    "        # rebalance those already top ranked and still there\n",
    "        for d in (d for d in posdata if d in rtop):\n",
    "            if self.p.WFA == True:\n",
    "                if sell and ((0.95 / self.selnum[dtidx]) < self.broker.getposition(d).size * self.broker.getposition(d).adjbase / self.broker.get_value()):\n",
    "                    self.log('Rebal {} - Rank {:.2f}'.format(d._name, rtop[d][0]))\n",
    "                    self.order_target_percent(d, target=0.95 * rtop[d][0]/ (0.95 / self.selnum[dtidx]))\n",
    "            else:\n",
    "                if sell and ((0.95 / self.selnum) < self.broker.getposition(d).size * self.broker.getposition(d).adjbase / self.broker.get_value()):\n",
    "                    self.log('Rebal {} - Rank {:.2f}'.format(d._name, rtop[d][0]))\n",
    "                    self.order_target_percent(d, target=0.95 * rtop[d][0]/(0.95 / self.selnum))\n",
    "                    \n",
    "            #Rebalances are just pricy operations which are needed for the buying part ... since it was based on equal weights...\n",
    "            del rtop[d]  # remove it, to simplify next iteration\n",
    "            sum_stocks += self.broker.getposition(d).size * self.broker.getposition(d).adjbase # Calulate the value of the stocks owned\n",
    "\n",
    "\n",
    "        if len(rtop):\n",
    "            sizer_perc = (0.95 - sum_stocks / self.broker.get_value()) / len(rtop) # Have a equal sizer for the cash (you want to invest in)\n",
    "            sum_stocks = 0\n",
    "\n",
    "        # issue a target order for the newly top ranked stocks\n",
    "        # do this last, as this will generate buy orders consuming cash\n",
    "        for d in rtop:\n",
    "            self.log('Enter {} - Rank {:.2f}'.format(d._name, rtop[d][0]))\n",
    "            self.order_target_percent(d, target=sizer_perc)\n",
    "           \n",
    "    def stop(self):\n",
    "        if globalparams[\"ranking_timeanalysis\"]:\n",
    "            t = 1\n",
    "            for k,v in self.ranks.items():\n",
    "                if t == 1:\n",
    "                    time = [k.num2date(x) for x in (k.datetime).get(ago=0, size=len(k))]\n",
    "                    myrankings = pd.DataFrame(columns=[k._name], index=time)\n",
    "                    t +=1\n",
    "                else:\n",
    "                    myrankings[k._name] = 1\n",
    "                list = []\n",
    "                for i in v.get(size=len(v)):\n",
    "                    list.append(i)\n",
    "                myrankings[k._name] = list \n",
    "            myrankings.to_excel(\"output.xlsx\")\n",
    "        pnl = round(self.broker.getvalue() - globalparams[\"cash\"],2)\n",
    "        self.log('Final PnL: {}'.format(\n",
    "            pnl))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Walk forward analysis\n",
    "Thanks to https://ntguardian.wordpress.com/2017/06/19/walk-forward-analysis-demonstration-backtrader/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     18,
     91
    ]
   },
   "outputs": [],
   "source": [
    "def walk_forward_analysis(datafeeds):\n",
    "    # display dataframes more orderly\n",
    "    pd.set_option('display.max_rows', 10)\n",
    "    pd.set_option('display.max_columns', 10)\n",
    "    pd.set_option('display.width', 1000)\n",
    "\n",
    "    globalparameters = dict(strategy=globalparams['strategy'],            # if a different strategy is used\n",
    "                        n_splits=globalparams[\"n_splits\"],                # how many chunks the data should have\n",
    "                        fixed_length=globalparams[\"fixed_length\"],         # by setting it False the training data will will grow over time, otherwise it will keep the size given under train_splits\n",
    "                        train_splits=1,             # how many splits should be used to train the model (be aware of these two variables may not work with your strategy, i.e. SMA100 but two training splits are just 110 days or less than 100 days)\n",
    "                        test_splits=1,              # how many splits should the data be tested on?\n",
    "                        cash=globalparams[\"cash\"],\n",
    "                        commission=globalparams[\"commission\"],\n",
    "                        num_evals=globalparams[\"num_eval\"],               # how often should the optimizer try to optimize\n",
    "                        )\n",
    "\n",
    "\n",
    "    class TimeSeriesSplitImproved(TimeSeriesSplit):\n",
    "\n",
    "        def split(self, X, y=None, groups=None, fixed_length=False,\n",
    "                  train_splits=1, test_splits=1):\n",
    "\n",
    "            X, y, groups = indexable(X, y, groups)\n",
    "            n_samples = _num_samples(X)\n",
    "            n_splits = self.n_splits\n",
    "            n_folds = n_splits + 1\n",
    "            train_splits, test_splits = int(train_splits), int(test_splits)\n",
    "            if n_folds > n_samples:\n",
    "                raise ValueError(\n",
    "                    (\"Cannot have number of folds ={0} greater\"\n",
    "                     \" than the number of samples: {1}.\").format(n_folds,\n",
    "                                                                 n_samples))\n",
    "            if ((n_folds - train_splits - test_splits) == 0 and test_splits > 0):\n",
    "                raise ValueError(\n",
    "                    (\"Both train_splits and test_splits must be positive\"\n",
    "                     \" integers.\"))\n",
    "            indices = np.arange(n_samples)\n",
    "            split_size = (n_samples // n_folds)\n",
    "            test_size = split_size * test_splits\n",
    "            train_size = split_size * train_splits\n",
    "            test_starts = range(train_size + n_samples % n_folds,\n",
    "                                n_samples - (test_size - split_size),\n",
    "                                split_size)\n",
    "\n",
    "            if fixed_length:\n",
    "                for i, test_start in zip(range(len(test_starts)),\n",
    "                                         test_starts):\n",
    "                    rem = 0\n",
    "                    if i == 0:\n",
    "                        rem = n_samples % n_folds\n",
    "                    yield (indices[(test_start - train_size - rem):test_start],\n",
    "                           indices[test_start:test_start + test_size])\n",
    "            else:\n",
    "                for test_start in test_starts:\n",
    "                    yield (indices[:test_start],\n",
    "                           indices[test_start:test_start + test_size])\n",
    "\n",
    "    class AcctStats(bt.Analyzer):\n",
    "        \"\"\"A simple analyzer that gets the gain in the value of the account; should be self-explanatory\"\"\"\n",
    "\n",
    "        def __init__(self):\n",
    "            self.start_val = self.strategy.broker.get_value()\n",
    "            self.end_val = None\n",
    "\n",
    "        def stop(self):\n",
    "            self.end_val = self.strategy.broker.get_value()\n",
    "\n",
    "        def get_analysis(self):\n",
    "            return {\"start\": self.start_val, \"end\": self.end_val,\n",
    "                    \"growth\": self.end_val - self.start_val, \"return\": self.end_val / self.start_val}\n",
    "\n",
    "    for df in datafeeds.values():\n",
    "        df[\"OpenInterest\"] = 0  # PandasData reader expects an OpenInterest column;\n",
    "\n",
    "    tscv = TimeSeriesSplitImproved(globalparameters['n_splits'])\n",
    "    split = tscv.split(datafeeds[list(datafeeds.keys())[2]], fixed_length=globalparameters['fixed_length'], train_splits=globalparameters['train_splits'], test_splits=globalparameters['test_splits'])\n",
    "    walk_forward_results = list()\n",
    "    \n",
    "    heatmap_data = []\n",
    "    # Be prepared: this will take a while\n",
    "    i = 0\n",
    "    for train, test in split:\n",
    "        i += 1\n",
    "        # TRAINING\n",
    "        # Optimize with optunity\n",
    "        def runstrat(p1=globalparams[\"selcperc\"], p2=globalparams[\"mperiod\"], p3=globalparams[\"rocperiod\"], WFA=False):\n",
    "            cerebro = bt.Cerebro(maxcpus=0,stdstats=False, runonce=False)\n",
    "            cerebro.addstrategy(eval(globalparameters['strategy']), selcperc=p1, mperiod=p2, rocperiod=p3, printlog=False) #change the parameters that should be depicted in heatmap\n",
    "            cerebro.broker.setcash(globalparams[\"cash\"])\n",
    "            cerebro.broker.setcommission(commission=globalparams[\"commission\"])\n",
    "            eval(opta(globalparams[\"optimization_target\"])[0])\n",
    "            for s, df in datafeeds.items():\n",
    "                data = bt.feeds.PandasData(dataname=df.iloc[train], name=s)  # Add a subset of data\n",
    "                cerebro.adddata(data)     \n",
    "            cerebro.broker.set_coc(True)\n",
    "            results1 = cerebro.run()\n",
    "            print(round(p1,0), round(p2,0), round(p3,0), round(eval(opta(globalparams[\"optimization_target\"])[1]),2), round(cerebro.broker.getvalue(),0))\n",
    "            if globalparams[\"heatmap\"]: \n",
    "                return [round(p1,0), round(p2,0), round(p3,0), round(eval(opta(globalparams[\"optimization_target\"])[1]),2), round(cerebro.broker.getvalue()/globalparams[\"cash\"],2)]\n",
    "            if globalparams[\"optunity\"]:\n",
    "                return round(eval(opta(globalparams[\"optimization_target\"])[1]),2)\n",
    "\n",
    "        #Either use Optunity for quick search of optimum\n",
    "        if globalparams[\"optunity\"]:\n",
    "            opt = optunity.maximize(runstrat, num_evals=globalparams[\"num_eval\"], pmap=optunity.pmap, p1=eval(params_comb_optunity(globalparams[\"params_combination\"])[0][0]), \n",
    "                                    p2=eval(params_comb_optunity(globalparams[\"params_combination\"])[0][1]), \n",
    "                                    p3=eval(params_comb_optunity(globalparams[\"params_combination\"])[0][2]))\n",
    "            optimal_pars, details, _ = opt\n",
    "            optimal_pars['var1'] = int(optimal_pars['p1'])\n",
    "            optimal_pars['var2'] = int(optimal_pars['p2'])\n",
    "            optimal_pars['var3'] = int(optimal_pars['p3'])\n",
    "        \n",
    "        # or heatmap showing how variables i = (y axis/ p1) and j = (x axis /p2) impact strategy outcome\n",
    "        if globalparams[\"heatmap\"]:     \n",
    "            pool = multiprocess.Pool(processes=7, maxtasksperchild=1)  \n",
    "            noduplicates = []\n",
    "            runt = 1\n",
    "            for i in eval(params_comb(globalparams[\"params_combination\"])[0][0]):\n",
    "                i = int(i)\n",
    "                for j in eval(params_comb(globalparams[\"params_combination\"])[0][1]):\n",
    "                    j = int(j)\n",
    "                    for k in eval(params_comb(globalparams[\"params_combination\"])[0][2]):  \n",
    "                        k = int(k)\n",
    "                        if runt == 1:\n",
    "                            noduplicates.append([i, j, k])\n",
    "                        else:\n",
    "                            if [i, j, k] in noduplicates:\n",
    "                                continue\n",
    "                            else:\n",
    "                                noduplicates.append([i, j, k])                            \n",
    "                        runt += 1\n",
    "                        \n",
    "            def Convert(lst):\n",
    "                lst2 = [[\"p1\",\"p2\",\"p3\"] for x in range(len(lst))]\n",
    "                res_dct = []\n",
    "                for x in range(len(lst)):\n",
    "                    res_dct.append(dict(zip(lst2[x],lst[x])))\n",
    "                return res_dct\n",
    "            \n",
    "            Convert(noduplicates)\n",
    "\n",
    "            results_list = pool.starmap_async(runstrat, noduplicates,chunksize=1)\n",
    "            pool.close()            \n",
    "            pool.join()\n",
    "            \n",
    "            #sort list since multiprocessing messes up the order\n",
    "            results_list = sorted(results_list.get(), key = lambda x:(x[0], x[1], x[2]))\n",
    "            \n",
    "            if globalparams[\"heatmap\"]:\n",
    "                optimal_pars = {}\n",
    "                #print(results_list)\n",
    "                #best return from the heatmap loop\n",
    "                #print(max(results_list, key=lambda x: x[3]), max(results_list, key=lambda x: x[3])[0], datafeeds[list(datafeeds.keys())[2]].iloc[test[0]].name)\n",
    "                optimal_pars['var1'] = max(results_list, key=lambda x: x[3])[0]\n",
    "                optimal_pars['var2'] = max(results_list, key=lambda x: x[3])[1]\n",
    "                optimal_pars['var3'] = max(results_list, key=lambda x: x[3])[2]\n",
    "                print(\"Optimal vars 1/2/3 \",optimal_pars['var1'], optimal_pars['var2'], optimal_pars['var3'])\n",
    "            \n",
    "            if len(globalparams[\"params_combination\"]) == 2:\n",
    "                for i in range(len(results_list)):\n",
    "                    results_list[i].pop(eval(params_comb(globalparams[\"params_combination\"])[1][1]))\n",
    "                my_heatmap1(results_list)\n",
    "                for i in range(len(results_list)):\n",
    "                    results_list[i].pop(2)\n",
    "                my_heatmap1(results_list)\n",
    "            else:\n",
    "                plotly_4d(results_list)\n",
    "            \n",
    "        \n",
    "        results_list = []\n",
    "        \n",
    "        # TESTING\n",
    "        tester = bt.Cerebro(stdstats=False, maxcpus=None, runonce=False)\n",
    "        tester.broker.set_cash(globalparameters['cash'])\n",
    "        tester.broker.set_coc(True)\n",
    "        tester.broker.setcommission(globalparameters['commission'])\n",
    "        tester.addanalyzer(AcctStats)\n",
    "        tester.addanalyzer(bt.analyzers.SharpeRatio, riskfreerate=0.0)\n",
    "\n",
    "        tester.addstrategy(eval(globalparameters['strategy']), selcperc=optimal_pars['var1'], mperiod=optimal_pars['var2'], rocperiod=optimal_pars['var3'], \n",
    "                           WFA=False, printlog=False)  # Test with optimal combination toDO like above int vs float\n",
    "        for s, df in datafeeds.items():\n",
    "            data = bt.feeds.PandasData(dataname=df.iloc[test], name=s)  # Add a subset of data\n",
    "            tester.adddata(data)\n",
    "\n",
    "        res = tester.run()\n",
    "        res_dict = res[0].analyzers.acctstats.get_analysis()\n",
    "        res_dict[\"var1\"] = optimal_pars['var1']\n",
    "        res_dict[\"var2\"] = optimal_pars['var2']\n",
    "        res_dict[\"var3\"] = optimal_pars['var3']\n",
    "        res_dict[\"sharpe\"] = res[0].analyzers.sharperatio.get_analysis()['sharperatio']\n",
    "        res_dict[\"start_date\"] = datafeeds[list(datafeeds.keys())[2]].iloc[test[0]].name\n",
    "        res_dict[\"end_date\"] = datafeeds[list(datafeeds.keys())[2]].iloc[test[-1]].name\n",
    "        walk_forward_results.append(res_dict)\n",
    "    \n",
    "    \n",
    "    wfdf = DataFrame(walk_forward_results)\n",
    "    print(wfdf.loc[:, wfdf.columns != 'start'])\n",
    "\n",
    "    cerebro_wf = bt.Cerebro(stdstats=False, maxcpus=None, runonce=False)\n",
    "\n",
    "    for s, df in datafeeds.items():\n",
    "        data = bt.feeds.PandasData(dataname=df, name=s, fromdate=wfdf.iloc[0, wfdf.columns.get_loc('start_date')], todate=wfdf.iloc[-1, wfdf.columns.get_loc('end_date')])\n",
    "        cerebro_wf.adddata(data)  # Give the data to cerebro\n",
    "\n",
    "    cerebro_wf.broker.setcash(globalparameters['cash'])\n",
    "    cerebro_wf.broker.setcommission(globalparameters['commission'])\n",
    "    cerebro_wf.broker.set_coc(True)\n",
    "\n",
    "    cerebro_wf.addstrategy(eval(globalparams['strategy']),\n",
    "                           # Give the results of the above optimization to SMACWalkForward (NOT OPTIONAL)\n",
    "                           selcperc=[f for f in wfdf.var1],\n",
    "                           mperiod=[s for s in wfdf.var2],\n",
    "                           rocperiod=[t for t in wfdf.var3],\n",
    "                           start_dates=[sd.date() for sd in wfdf.start_date],\n",
    "                           end_dates=[ed.date() for ed in wfdf.end_date],\n",
    "                           WFA=True,\n",
    "                           )\n",
    "\n",
    "    cerebro_wf.addobservermulti(bt.observers.BuySell)  # Plots up/down arrows\n",
    "    cerebro_wf.addanalyzer(bt.analyzers.SharpeRatio, riskfreerate=0.0)\n",
    "    cerebro_wf.addanalyzer(btanal.PyFolio)\n",
    "\n",
    "    results = cerebro_wf.run()\n",
    "    print(f\"Sharpe: {results[0].analyzers.sharperatio.get_analysis()['sharperatio']:.3f}\")\n",
    "    \n",
    "    # Quantstats thanks to https://algotrading101.com/learn/backtrader-for-backtesting/\n",
    "    returns, positions, transactions, gross_lev = results[0].analyzers.pyfolio.get_pf_items()\n",
    "    returns.index = returns.index.tz_convert(None)\n",
    "    qs.reports.html(returns, output='stats.html', title='Walkforward '+ globalparams[\"strategy\"])\n",
    "    webbrowser.open('stats.html')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exection section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def run(args=None):\n",
    "    #optreturn=False otherwise the heatmap doesn't work\n",
    "    cerebro = bt.Cerebro(maxcpus=1, preload=False, optdatas=False, optreturn=False, stdstats=True) \n",
    "    \n",
    "    #plotsize\n",
    "    plt.rcParams['figure.figsize'] = [7, 7]\n",
    "    plt.rcParams['figure.dpi'] = 200\n",
    "        \n",
    "    # <<<Data loading section>>>\n",
    "    # Parse from/to-date\n",
    "    fromdate = globalparams[\"fromdate\"]\n",
    "    todate = globalparams[\"todate\"]\n",
    "    datafeeds = {}\n",
    "    \n",
    "    # Add SPY/QQQ as \"Benchmark\"\n",
    "    df0 = pd.read_csv('/home/mmd/Data/QQQ.csv', index_col=0, parse_dates=True)\n",
    "    benchdata = bt.feeds.PandasData(dataname=df0,name=\"QQQ\",fromdate=fromdate, todate=todate,  plot=False)\n",
    "    cerebro.adddata(benchdata)\n",
    "    df0 = df0.loc[fromdate : todate]\n",
    "    datafeeds.update({\"QQQ\": df0})\n",
    "    \n",
    "    # add all the data files available in the directory datadir\n",
    "    for fname in glob.glob(os.path.join(globalparams[\"path\"], '*')):\n",
    "        df = pd.read_csv(fname, index_col=0, parse_dates=True)\n",
    "        # Add dataframes to a dictionary for walkforward analysis\n",
    "        if len(df)>210:\n",
    "            cerebro.adddata(bt.feeds.PandasData(dataname=df,name=os.path.basename(fname).replace(\".csv\", \"\"),fromdate=fromdate, todate=todate, plot=False))\n",
    "        df = df.loc[fromdate : todate]\n",
    "        datafeeds.update({os.path.basename(fname).replace(\".csv\", \"\"): df})\n",
    "            \n",
    "    results_list = []\n",
    "    # <<<Cerebro loading section>>>\n",
    "    if globalparams[\"walk\"]:\n",
    "        walk_forward_analysis(datafeeds)\n",
    "    else:\n",
    "        ready_list = []\n",
    "        def dummy_func(index):\n",
    "            global ready_list\n",
    "            ready_list.append(index)\n",
    "            \n",
    "        # Optimization by Optunity/Heatmap (must be stated outside of other classes/functions)\n",
    "        def runstrat(p1=globalparams[\"selcperc\"], p2=globalparams[\"mperiod\"], p3=globalparams[\"rocperiod\"]):\n",
    "            cerebro = bt.Cerebro(maxcpus=0,stdstats=True)\n",
    "            cerebro.addstrategy(St, selcperc=p1, mperiod=p2, rocperiod=p3, printlog=False) #change the parameters that should be depicted in heatmap\n",
    "            cerebro.broker.setcash(globalparams[\"cash\"])\n",
    "            cerebro.broker.setcommission(commission=globalparams[\"commission\"])\n",
    "            eval(opta(globalparams[\"optimization_target\"])[0])\n",
    "            cerebro.adddata(benchdata) \n",
    "            for fname in glob.glob(os.path.join(globalparams[\"path\"], '*')):\n",
    "                df = pd.read_csv(fname, index_col=0, parse_dates=True)\n",
    "                if len(df)>200:\n",
    "                    cerebro.adddata(bt.feeds.PandasData(dataname=df,name=os.path.basename(fname).replace(\".csv\", \"\"),fromdate=fromdate, todate=todate, plot=False))        \n",
    "\n",
    "            cerebro.broker.set_coc(True)\n",
    "            results1 = cerebro.run()\n",
    "            print(round(p1,0), round(p2,0), round(p3,0), round(eval(opta(globalparams[\"optimization_target\"])[1]),2), round(cerebro.broker.getvalue()/globalparams[\"cash\"],2))\n",
    "\n",
    "            #the return is as far as I can see just legacy since the line above will be used\n",
    "            if globalparams[\"heatmap\"]: \n",
    "                return [round(p1,0), round(p2,0), round(p3,0), round(eval(opta(globalparams[\"optimization_target\"])[1]),1), round(cerebro.broker.getvalue()/globalparams[\"cash\"],1)]\n",
    "            if globalparams[\"optunity\"]:\n",
    "                return round(eval(opta(globalparams[\"optimization_target\"])[1]),2)\n",
    "            \n",
    "        #Either use Optunity for quick search of optimum\n",
    "        if globalparams[\"optunity\"]:\n",
    "            opt = optunity.maximize(runstrat, num_evals=globalparams[\"num_eval\"], pmap=optunity.pmap, p1=eval(params_comb_optunity(globalparams[\"params_combination\"])[0][0]), \n",
    "                                    p2=eval(params_comb_optunity(globalparams[\"params_combination\"])[0][1]), \n",
    "                                    p3=eval(params_comb_optunity(globalparams[\"params_combination\"])[0][2]))\n",
    "            optimal_pars, details, _ = opt\n",
    "            print('Optimal Parameters:')\n",
    "            print('selcperc = %.4f' % optimal_pars['p1'])\n",
    "            print('mperiod = %.4f' % optimal_pars['p2'])\n",
    "            print('rocperiod = %.4f' % optimal_pars['p3'])\n",
    "\n",
    "        # or heatmap showing how variables i = (y axis/ p1) and j = (x axis /p2) impact strategy outcome\n",
    "        if globalparams[\"heatmap\"]:     \n",
    "            pool = multiprocess.Pool(processes=7, maxtasksperchild=1)  \n",
    "            noduplicates = []\n",
    "            runt = 1\n",
    "            for i in eval(params_comb(globalparams[\"params_combination\"])[0][0]):\n",
    "                i = int(i)\n",
    "                for j in eval(params_comb(globalparams[\"params_combination\"])[0][1]):\n",
    "                    j = int(j)\n",
    "                    for k in eval(params_comb(globalparams[\"params_combination\"])[0][2]):  \n",
    "                        k = int(k)\n",
    "                        if runt == 1:\n",
    "                            noduplicates.append([i, j, k])\n",
    "                        else:\n",
    "                            if [i, j, k] in noduplicates:\n",
    "                                continue\n",
    "                            else:\n",
    "                                noduplicates.append([i, j, k])                            \n",
    "                        runt += 1\n",
    "                        \n",
    "            def Convert(lst):\n",
    "                lst2 = [[\"p1\",\"p2\",\"p3\"] for x in range(len(lst))]\n",
    "                res_dct = []\n",
    "                for x in range(len(lst)):\n",
    "                    res_dct.append(dict(zip(lst2[x],lst[x])))\n",
    "                return res_dct\n",
    "            \n",
    "            Convert(noduplicates)\n",
    "\n",
    "            results_list = pool.starmap_async(runstrat, noduplicates,chunksize=1)\n",
    "            pool.close()            \n",
    "            pool.join()\n",
    "            \n",
    "            #sort list since multiprocessing messes up the order\n",
    "            results_list = sorted(results_list.get(), key = lambda x:(x[0], x[1], x[2]))\n",
    "            \n",
    "            if len(globalparams[\"params_combination\"]) == 2:\n",
    "                for i in range(len(results_list)):\n",
    "                    results_list[i].pop(eval(params_comb(globalparams[\"params_combination\"])[1][1]))\n",
    "                my_heatmap1(results_list)\n",
    "                for i in range(len(results_list)):\n",
    "                    results_list[i].pop(2)\n",
    "                my_heatmap1(results_list)\n",
    "            else:\n",
    "                plotly_4d(results_list)\n",
    "            return\n",
    "\n",
    "        # add strategy\n",
    "        if globalparams[\"optunity\"]: #Pick other than Buy and Hold Strategy for optunity optimziation\n",
    "            cerebro.addstrategy(eval(gmatplotliblobalparams[\"strategy\"]), selcperc=optimal_pars['p1'], mperiod=int(optimal_pars['p2']), rocperiod=int(optimal_pars['p3']))\n",
    "        else:    \n",
    "            cerebro.addstrategy(eval(globalparams[\"strategy\"]))\n",
    "\n",
    "        # set the cash, cheat on close and commission\n",
    "        cerebro.broker.setcash(globalparams[\"cash\"])\n",
    "        cerebro.broker.set_coc(True)\n",
    "        cerebro.broker.setcommission(commission=globalparams[\"commission\"])\n",
    "\n",
    "        # Adding Analysers\n",
    "        cerebro.addanalyzer(btanal.PyFolio)                # Needed to use PyFolio & Quanstat\n",
    "        cerebro.addanalyzer(Sortino, _name=\"sortino\") \n",
    "        cerebro.addobserver(bt.observers.Benchmark, data=benchdata, timeframe=bt.TimeFrame.NoTimeFrame)\n",
    "\n",
    "        results = cerebro.run(maxcpus=0)#maxcpu=1 otherwise pickling multiprocessing errors\n",
    "\n",
    "        # <<<Performance analysing section section>>>\n",
    "        cerebro.plot()\n",
    "\n",
    "        # Basic performance evaluation ... final value ... minus starting cash\n",
    "        pnl = cerebro.broker.get_value() - globalparams[\"cash\"]\n",
    "        print(\"Sortino Ratio : \", results[0].analyzers.sortino.get_analysis()[\"sortino\"])\n",
    "        print('Profit ... or Loss: {:.2f}'.format(pnl))\n",
    "\n",
    "        # Quantstats thanks to https://algotrading101.com/learn/backtrader-for-backtesting/\n",
    "        # Does not work with optstrategy\n",
    "        returns, positions, transactions, gross_lev = results[0].analyzers.pyfolio.get_pf_items()\n",
    "        returns.index = returns.index.tz_convert(None)\n",
    "        qs.reports.html(returns, output='stats.html', title='Momentum '+ globalparams[\"strategy\"] + \" \" + str(int(globalparams[\"selcperc\"])) + \" # of stocks picked\")\n",
    "        webbrowser.open('stats.html')\n",
    "\n",
    "# <<<Execute starting section>>>    \n",
    "if __name__ == '__main__':\n",
    "    run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "notify_time": "5"
 },
 "nbformat": 4,
 "nbformat_minor": 4
}