ML_Project_DataPrep.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "pip install ta talipp arch"
      ],
      "metadata": {
        "id": "6eqkF_UI7FSa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4a398b5f-ae42-421b-b526-2528673aa627"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting ta\n",
            "  Downloading ta-0.11.0.tar.gz (25 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting talipp\n",
            "  Downloading talipp-1.9.1-py3-none-any.whl (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.0/43.0 kB\u001b[0m \u001b[31m804.4 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting arch\n",
            "  Downloading arch-6.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (981 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.7/981.7 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from ta) (1.23.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from ta) (1.5.3)\n",
            "Requirement already satisfied: scipy>=1.5 in /usr/local/lib/python3.10/dist-packages (from arch) (1.11.4)\n",
            "Requirement already satisfied: statsmodels>=0.12 in /usr/local/lib/python3.10/dist-packages (from arch) (0.14.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->ta) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->ta) (2023.3.post1)\n",
            "Requirement already satisfied: patsy>=0.5.2 in /usr/local/lib/python3.10/dist-packages (from statsmodels>=0.12->arch) (0.5.3)\n",
            "Requirement already satisfied: packaging>=21.3 in /usr/local/lib/python3.10/dist-packages (from statsmodels>=0.12->arch) (23.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from patsy>=0.5.2->statsmodels>=0.12->arch) (1.16.0)\n",
            "Building wheels for collected packages: ta\n",
            "  Building wheel for ta (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ta: filename=ta-0.11.0-py3-none-any.whl size=29413 sha256=925d3191f29c531e7a42d5047035468f0a2b3948209b1a914ddbb66a78be3a11\n",
            "  Stored in directory: /root/.cache/pip/wheels/5f/67/4f/8a9f252836e053e532c6587a3230bc72a4deb16b03a829610b\n",
            "Successfully built ta\n",
            "Installing collected packages: talipp, ta, arch\n",
            "Successfully installed arch-6.2.0 ta-0.11.0 talipp-1.9.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "from sklearn import model_selection\n",
        "from sklearn import linear_model\n",
        "from sklearn import metrics\n",
        "from datetime import datetime, timedelta\n",
        "from sklearn.metrics import r2_score\n",
        "import gzip\n",
        "from sklearn.linear_model import ElasticNetCV\n",
        "from sklearn.model_selection import TimeSeriesSplit\n",
        "from sklearn.model_selection import PredefinedSplit\n",
        "import xgboost as xgb\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import datetime\n",
        "\n",
        "from sklearn import model_selection\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "\n",
        "from talipp.indicators import *\n",
        "from talipp.ohlcv import OHLCVFactory\n",
        "import ta\n",
        "import arch\n",
        "from statsmodels.tsa.arima.model import ARIMA"
      ],
      "metadata": {
        "id": "lScrpvcyg7S-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "metadata": {
        "id": "-xqpNrtQg-C5"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "rAZ3_0Q2jpg6",
        "outputId": "8b2457c0-b9f4-4da7-931c-69ab5eb0ffe7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd"
      ],
      "metadata": {
        "id": "WgPeujqFjuNm",
        "outputId": "610be730-f27b-4733-8aae-33e387fe864f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/root\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd .."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uEps0nUHvvm3",
        "outputId": "26977886-2068-4133-82be-6d3248f730bf"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd content/drive/MyDrive"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3YItjKUBvw1I",
        "outputId": "a4c9fea1-7a39-4273-bb2b-0f417aa7cab4"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_file = \"Historic_Data.parquet.gzip\"\n",
        "data = pd.read_parquet(data_file)\n",
        "\n",
        "vishesh_file = \"Vishesh_Data.parquet.gzip\"\n",
        "vishesh = pd.read_parquet(vishesh_file)\n",
        "\n",
        "wrds_file = \"WRDS_Financial_Data.parquet.gzip\"\n",
        "wrds = pd.read_parquet(wrds_file)"
      ],
      "metadata": {
        "id": "5RB1RrqrpMNw"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.replace(-np.inf, np.nan, inplace=True)\n",
        "data = data.drop(columns=['TICKER','COMNAM','TSYMBOL','NUMTRD','CUSIP'])\n",
        "data['date'] = pd.to_datetime(data['date'])\n",
        "data = data.fillna(data.groupby(data['date'].dt.year).transform('median'))\n",
        "\n",
        "ticker = vishesh.set_index('PERMNO')['TICKER'].to_dict()\n",
        "data = data.reset_index()\n",
        "data['TICKER'] = data['PERMNO'].map(ticker)\n",
        "data = data.drop(columns=['index'])\n",
        "\n",
        "wrds = wrds.drop(columns=['qdate','public_date','TICKER','cusip','staff_sale'])\n",
        "wrds['adate'] = pd.to_datetime(wrds['adate'])\n",
        "wrds['divyield'] = wrds['divyield'].str.rstrip('%').astype('float') / 100.0\n",
        "wrds = wrds.fillna(wrds.median())\n",
        "wrds['year'] = wrds['adate'].dt.year\n",
        "wrds = wrds.drop_duplicates(subset=['permno', 'adate'], keep='first')\n",
        "wrds = wrds.drop(columns=['adate'])"
      ],
      "metadata": {
        "id": "m2gIDYTeWhIK"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data['PERMNO'] = data['PERMNO'].astype(float)\n",
        "data['date'] = pd.to_datetime(data['date'])\n",
        "duplicate_rows = data[data.duplicated(subset=['PERMNO', 'date'], keep=False)]\n",
        "data = data.drop_duplicates(subset=['PERMNO', 'date'], keep='first')\n",
        "data = data.dropna()\n",
        "data.info(verbose=True, show_counts=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gOO_6zJLtmr9",
        "outputId": "60c737cf-0415-41f6-dcba-2767ced38f0f"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 3213291 entries, 0 to 5200378\n",
            "Data columns (total 13 columns):\n",
            " #   Column   Non-Null Count    Dtype         \n",
            "---  ------   --------------    -----         \n",
            " 0   PERMNO   3213291 non-null  float64       \n",
            " 1   date     3213291 non-null  datetime64[ns]\n",
            " 2   BIDLO    3213291 non-null  float64       \n",
            " 3   ASKHI    3213291 non-null  float64       \n",
            " 4   PRC      3213291 non-null  float64       \n",
            " 5   VOL      3213291 non-null  float64       \n",
            " 6   RET      3213291 non-null  float64       \n",
            " 7   BID      3213291 non-null  float64       \n",
            " 8   ASK      3213291 non-null  float64       \n",
            " 9   SHROUT   3213291 non-null  float64       \n",
            " 10  OPENPRC  3213291 non-null  float64       \n",
            " 11  RETX     3213291 non-null  float64       \n",
            " 12  TICKER   3213291 non-null  object        \n",
            "dtypes: datetime64[ns](1), float64(11), object(1)\n",
            "memory usage: 343.2+ MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "merged_data = pd.merge(data, wrds, how='left', left_on=['PERMNO', data['date'].dt.year], right_on=['permno', 'year'])\n",
        "merged_data = merged_data.drop(columns=['permno', 'year'])"
      ],
      "metadata": {
        "id": "S8c7pDOh61uS"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "merged_data = merged_data.drop_duplicates(subset=['PERMNO', 'date'], keep='first')\n",
        "merged_data = merged_data.set_index(['PERMNO', 'date'], drop=True, verify_integrity=True)\n",
        "data = merged_data\n",
        "data.info(verbose=True, show_counts=True)"
      ],
      "metadata": {
        "id": "nn2dIw60jZX0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data"
      ],
      "metadata": {
        "id": "jswUvkGhlfd_",
        "outputId": "554b88e6-c8e5-4e59-ab7a-38f43c4050a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 648
        }
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                        BIDLO      ASKHI        PRC        VOL       RET  \\\n",
              "PERMNO  date                                                               \n",
              "10057.0 1992-01-02    7.00000    7.25000    7.25000    23400.0  0.000000   \n",
              "        1992-01-03    7.12500    7.25000    7.25000     9100.0  0.000000   \n",
              "        1992-01-06    7.12500    7.37500    7.37500    15300.0  0.017241   \n",
              "        1992-01-07    7.37500    7.62500    7.62500    21400.0  0.033898   \n",
              "        1992-01-08    7.62500    8.75000    8.50000   165500.0  0.114754   \n",
              "...                       ...        ...        ...        ...       ...   \n",
              "93096.0 2022-12-23  244.32001  248.78000  247.55000  1452446.0  0.008227   \n",
              "        2022-12-27  244.64999  248.24500  247.17000  1643906.0 -0.001535   \n",
              "        2022-12-28  244.77000  249.27499  245.00999  1196778.0 -0.008739   \n",
              "        2022-12-29  244.99300  249.25500  248.00999   960106.0  0.012244   \n",
              "        2022-12-30  244.80000  248.63499  246.25000  1145845.0 -0.004879   \n",
              "\n",
              "                          BID        ASK    SHROUT    OPENPRC      RETX  ...  \\\n",
              "PERMNO  date                                                             ...   \n",
              "10057.0 1992-01-02   23.50000   24.00000    6291.0   28.75000  0.000000  ...   \n",
              "        1992-01-03   23.50000   24.00000    6291.0   28.75000  0.000000  ...   \n",
              "        1992-01-06   23.50000   24.00000    6291.0   28.75000  0.017241  ...   \n",
              "        1992-01-07   23.50000   24.00000    6291.0   28.75000  0.033898  ...   \n",
              "        1992-01-08   23.50000   24.00000    6291.0   28.75000  0.114754  ...   \n",
              "...                       ...        ...       ...        ...       ...  ...   \n",
              "93096.0 2022-12-23  247.73000  247.74001  225572.0  244.32001  0.008227  ...   \n",
              "        2022-12-27  247.23000  247.24001  225572.0  247.42999 -0.001535  ...   \n",
              "        2022-12-28  245.02000  245.10001  225572.0  247.46001 -0.008739  ...   \n",
              "        2022-12-29  248.05000  248.06000  225572.0  245.64000  0.012244  ...   \n",
              "        2022-12-30  246.25999  246.31000  223575.0  247.33000 -0.007096  ...   \n",
              "\n",
              "                   pay_turn  sale_invcap  sale_equity  sale_nwc  rd_sale  \\\n",
              "PERMNO  date                                                               \n",
              "10057.0 1992-01-02   15.988        2.614        2.752     4.683    0.015   \n",
              "        1992-01-03   15.988        2.614        2.752     4.683    0.015   \n",
              "        1992-01-06   15.988        2.614        2.752     4.683    0.015   \n",
              "        1992-01-07   15.988        2.614        2.752     4.683    0.015   \n",
              "        1992-01-08   15.988        2.614        2.752     4.683    0.015   \n",
              "...                     ...          ...          ...       ...      ...   \n",
              "93096.0 2022-12-23    6.292        1.771        5.465   105.460    0.000   \n",
              "        2022-12-27    6.292        1.771        5.465   105.460    0.000   \n",
              "        2022-12-28    6.292        1.771        5.465   105.460    0.000   \n",
              "        2022-12-29    6.292        1.771        5.465   105.460    0.000   \n",
              "        2022-12-30    6.292        1.771        5.465   105.460    0.000   \n",
              "\n",
              "                    adv_sale  accrual    ptb  PEG_trailing  divyield  \n",
              "PERMNO  date                                                          \n",
              "10057.0 1992-01-02     0.000   -0.142  0.959         0.785    0.0432  \n",
              "        1992-01-03     0.000   -0.142  0.959         0.785    0.0432  \n",
              "        1992-01-06     0.000   -0.142  0.959         0.785    0.0432  \n",
              "        1992-01-07     0.000   -0.142  0.959         0.785    0.0432  \n",
              "        1992-01-08     0.000   -0.142  0.959         0.785    0.0432  \n",
              "...                      ...      ...    ...           ...       ...  \n",
              "93096.0 2022-12-23     0.003   -0.018  7.189         0.981    0.0237  \n",
              "        2022-12-27     0.003   -0.018  7.189         0.981    0.0237  \n",
              "        2022-12-28     0.003   -0.018  7.189         0.981    0.0237  \n",
              "        2022-12-29     0.003   -0.018  7.189         0.981    0.0237  \n",
              "        2022-12-30     0.003   -0.018  7.189         0.981    0.0237  \n",
              "\n",
              "[2551424 rows x 79 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e5d78e63-1587-47cd-a5d8-abf4a9873279\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>BIDLO</th>\n",
              "      <th>ASKHI</th>\n",
              "      <th>PRC</th>\n",
              "      <th>VOL</th>\n",
              "      <th>RET</th>\n",
              "      <th>BID</th>\n",
              "      <th>ASK</th>\n",
              "      <th>SHROUT</th>\n",
              "      <th>OPENPRC</th>\n",
              "      <th>RETX</th>\n",
              "      <th>...</th>\n",
              "      <th>pay_turn</th>\n",
              "      <th>sale_invcap</th>\n",
              "      <th>sale_equity</th>\n",
              "      <th>sale_nwc</th>\n",
              "      <th>rd_sale</th>\n",
              "      <th>adv_sale</th>\n",
              "      <th>accrual</th>\n",
              "      <th>ptb</th>\n",
              "      <th>PEG_trailing</th>\n",
              "      <th>divyield</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PERMNO</th>\n",
              "      <th>date</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">10057.0</th>\n",
              "      <th>1992-01-02</th>\n",
              "      <td>7.00000</td>\n",
              "      <td>7.25000</td>\n",
              "      <td>7.25000</td>\n",
              "      <td>23400.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>23.50000</td>\n",
              "      <td>24.00000</td>\n",
              "      <td>6291.0</td>\n",
              "      <td>28.75000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>15.988</td>\n",
              "      <td>2.614</td>\n",
              "      <td>2.752</td>\n",
              "      <td>4.683</td>\n",
              "      <td>0.015</td>\n",
              "      <td>0.000</td>\n",
              "      <td>-0.142</td>\n",
              "      <td>0.959</td>\n",
              "      <td>0.785</td>\n",
              "      <td>0.0432</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1992-01-03</th>\n",
              "      <td>7.12500</td>\n",
              "      <td>7.25000</td>\n",
              "      <td>7.25000</td>\n",
              "      <td>9100.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>23.50000</td>\n",
              "      <td>24.00000</td>\n",
              "      <td>6291.0</td>\n",
              "      <td>28.75000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>15.988</td>\n",
              "      <td>2.614</td>\n",
              "      <td>2.752</td>\n",
              "      <td>4.683</td>\n",
              "      <td>0.015</td>\n",
              "      <td>0.000</td>\n",
              "      <td>-0.142</td>\n",
              "      <td>0.959</td>\n",
              "      <td>0.785</td>\n",
              "      <td>0.0432</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1992-01-06</th>\n",
              "      <td>7.12500</td>\n",
              "      <td>7.37500</td>\n",
              "      <td>7.37500</td>\n",
              "      <td>15300.0</td>\n",
              "      <td>0.017241</td>\n",
              "      <td>23.50000</td>\n",
              "      <td>24.00000</td>\n",
              "      <td>6291.0</td>\n",
              "      <td>28.75000</td>\n",
              "      <td>0.017241</td>\n",
              "      <td>...</td>\n",
              "      <td>15.988</td>\n",
              "      <td>2.614</td>\n",
              "      <td>2.752</td>\n",
              "      <td>4.683</td>\n",
              "      <td>0.015</td>\n",
              "      <td>0.000</td>\n",
              "      <td>-0.142</td>\n",
              "      <td>0.959</td>\n",
              "      <td>0.785</td>\n",
              "      <td>0.0432</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1992-01-07</th>\n",
              "      <td>7.37500</td>\n",
              "      <td>7.62500</td>\n",
              "      <td>7.62500</td>\n",
              "      <td>21400.0</td>\n",
              "      <td>0.033898</td>\n",
              "      <td>23.50000</td>\n",
              "      <td>24.00000</td>\n",
              "      <td>6291.0</td>\n",
              "      <td>28.75000</td>\n",
              "      <td>0.033898</td>\n",
              "      <td>...</td>\n",
              "      <td>15.988</td>\n",
              "      <td>2.614</td>\n",
              "      <td>2.752</td>\n",
              "      <td>4.683</td>\n",
              "      <td>0.015</td>\n",
              "      <td>0.000</td>\n",
              "      <td>-0.142</td>\n",
              "      <td>0.959</td>\n",
              "      <td>0.785</td>\n",
              "      <td>0.0432</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1992-01-08</th>\n",
              "      <td>7.62500</td>\n",
              "      <td>8.75000</td>\n",
              "      <td>8.50000</td>\n",
              "      <td>165500.0</td>\n",
              "      <td>0.114754</td>\n",
              "      <td>23.50000</td>\n",
              "      <td>24.00000</td>\n",
              "      <td>6291.0</td>\n",
              "      <td>28.75000</td>\n",
              "      <td>0.114754</td>\n",
              "      <td>...</td>\n",
              "      <td>15.988</td>\n",
              "      <td>2.614</td>\n",
              "      <td>2.752</td>\n",
              "      <td>4.683</td>\n",
              "      <td>0.015</td>\n",
              "      <td>0.000</td>\n",
              "      <td>-0.142</td>\n",
              "      <td>0.959</td>\n",
              "      <td>0.785</td>\n",
              "      <td>0.0432</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">93096.0</th>\n",
              "      <th>2022-12-23</th>\n",
              "      <td>244.32001</td>\n",
              "      <td>248.78000</td>\n",
              "      <td>247.55000</td>\n",
              "      <td>1452446.0</td>\n",
              "      <td>0.008227</td>\n",
              "      <td>247.73000</td>\n",
              "      <td>247.74001</td>\n",
              "      <td>225572.0</td>\n",
              "      <td>244.32001</td>\n",
              "      <td>0.008227</td>\n",
              "      <td>...</td>\n",
              "      <td>6.292</td>\n",
              "      <td>1.771</td>\n",
              "      <td>5.465</td>\n",
              "      <td>105.460</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.003</td>\n",
              "      <td>-0.018</td>\n",
              "      <td>7.189</td>\n",
              "      <td>0.981</td>\n",
              "      <td>0.0237</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2022-12-27</th>\n",
              "      <td>244.64999</td>\n",
              "      <td>248.24500</td>\n",
              "      <td>247.17000</td>\n",
              "      <td>1643906.0</td>\n",
              "      <td>-0.001535</td>\n",
              "      <td>247.23000</td>\n",
              "      <td>247.24001</td>\n",
              "      <td>225572.0</td>\n",
              "      <td>247.42999</td>\n",
              "      <td>-0.001535</td>\n",
              "      <td>...</td>\n",
              "      <td>6.292</td>\n",
              "      <td>1.771</td>\n",
              "      <td>5.465</td>\n",
              "      <td>105.460</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.003</td>\n",
              "      <td>-0.018</td>\n",
              "      <td>7.189</td>\n",
              "      <td>0.981</td>\n",
              "      <td>0.0237</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2022-12-28</th>\n",
              "      <td>244.77000</td>\n",
              "      <td>249.27499</td>\n",
              "      <td>245.00999</td>\n",
              "      <td>1196778.0</td>\n",
              "      <td>-0.008739</td>\n",
              "      <td>245.02000</td>\n",
              "      <td>245.10001</td>\n",
              "      <td>225572.0</td>\n",
              "      <td>247.46001</td>\n",
              "      <td>-0.008739</td>\n",
              "      <td>...</td>\n",
              "      <td>6.292</td>\n",
              "      <td>1.771</td>\n",
              "      <td>5.465</td>\n",
              "      <td>105.460</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.003</td>\n",
              "      <td>-0.018</td>\n",
              "      <td>7.189</td>\n",
              "      <td>0.981</td>\n",
              "      <td>0.0237</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2022-12-29</th>\n",
              "      <td>244.99300</td>\n",
              "      <td>249.25500</td>\n",
              "      <td>248.00999</td>\n",
              "      <td>960106.0</td>\n",
              "      <td>0.012244</td>\n",
              "      <td>248.05000</td>\n",
              "      <td>248.06000</td>\n",
              "      <td>225572.0</td>\n",
              "      <td>245.64000</td>\n",
              "      <td>0.012244</td>\n",
              "      <td>...</td>\n",
              "      <td>6.292</td>\n",
              "      <td>1.771</td>\n",
              "      <td>5.465</td>\n",
              "      <td>105.460</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.003</td>\n",
              "      <td>-0.018</td>\n",
              "      <td>7.189</td>\n",
              "      <td>0.981</td>\n",
              "      <td>0.0237</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2022-12-30</th>\n",
              "      <td>244.80000</td>\n",
              "      <td>248.63499</td>\n",
              "      <td>246.25000</td>\n",
              "      <td>1145845.0</td>\n",
              "      <td>-0.004879</td>\n",
              "      <td>246.25999</td>\n",
              "      <td>246.31000</td>\n",
              "      <td>223575.0</td>\n",
              "      <td>247.33000</td>\n",
              "      <td>-0.007096</td>\n",
              "      <td>...</td>\n",
              "      <td>6.292</td>\n",
              "      <td>1.771</td>\n",
              "      <td>5.465</td>\n",
              "      <td>105.460</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.003</td>\n",
              "      <td>-0.018</td>\n",
              "      <td>7.189</td>\n",
              "      <td>0.981</td>\n",
              "      <td>0.0237</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2551424 rows × 79 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e5d78e63-1587-47cd-a5d8-abf4a9873279')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e5d78e63-1587-47cd-a5d8-abf4a9873279 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e5d78e63-1587-47cd-a5d8-abf4a9873279');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-8f9a4c0b-7398-4641-aed9-21c4119f065d\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-8f9a4c0b-7398-4641-aed9-21c4119f065d')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-8f9a4c0b-7398-4641-aed9-21c4119f065d button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_14768203-bde2-4932-bbd8-5d10994e6621\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('data')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_14768203-bde2-4932-bbd8-5d10994e6621 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('data');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def add_technical_indicators(df):\n",
        "    grouped = df.groupby('PERMNO')\n",
        "\n",
        "    for permno, group in grouped:\n",
        "        # RSI (Relative Strength Index)\n",
        "        df.loc[permno, 'rsi_14'] = ta.momentum.RSIIndicator(group['PRC'], window=14).rsi()\n",
        "\n",
        "        # SMA (Simple Moving Average)\n",
        "        sma_indicator = ta.trend.SMAIndicator(group['PRC'], window=50)\n",
        "        df.loc[permno, 'sma_50'] = sma_indicator.sma_indicator()\n",
        "        # Convert 'sma_50' to float\n",
        "        df['sma_50'] = pd.to_numeric(df['sma_50'], errors='coerce').astype(float)\n",
        "\n",
        "        # EMA (Exponential Moving Average)\n",
        "        ema_indicator = ta.trend.EMAIndicator(group['PRC'], window=20)\n",
        "        df.loc[permno, 'ema_20'] = ema_indicator.ema_indicator()\n",
        "        # Convert 'ema_20' to float\n",
        "        df['ema_20'] = pd.to_numeric(df['ema_20'], errors='coerce').astype(float)\n",
        "\n",
        "        # MACD (Moving Average Convergence Divergence)\n",
        "        macd_indicator = ta.trend.MACD(group['PRC'], window_slow=26, window_fast=12, window_sign=9)\n",
        "        df.loc[permno, 'macd'] = macd_indicator.macd()\n",
        "        df.loc[permno, 'macd_diff'] = macd_indicator.macd_diff()\n",
        "        df.loc[permno, 'macd_signal'] = macd_indicator.macd_signal()\n",
        "        # Convert 'macd', 'macd_diff', and 'macd_signal' to float\n",
        "        df['macd'] = pd.to_numeric(df['macd'], errors='coerce').astype(float)\n",
        "        df['macd_diff'] = pd.to_numeric(df['macd_diff'], errors='coerce').astype(float)\n",
        "        df['macd_signal'] = pd.to_numeric(df['macd_signal'], errors='coerce').astype(float)\n",
        "\n",
        "        # ADX (Average Directional Index)\n",
        "        adx_indicator = ta.trend.ADXIndicator(group['ASKHI'], group['BIDLO'], group['PRC'], window=14)\n",
        "        df.loc[permno, 'adx'] = adx_indicator.adx()\n",
        "        df.loc[permno, 'adx_pos'] = adx_indicator.adx_pos()\n",
        "        df.loc[permno, 'adx_neg'] = adx_indicator.adx_neg()\n",
        "        # Convert 'adx', 'adx_pos', and 'adx_neg' to float\n",
        "        df['adx'] = pd.to_numeric(df['adx'], errors='coerce').astype(float)\n",
        "        df['adx_pos'] = pd.to_numeric(df['adx_pos'], errors='coerce').astype(float)\n",
        "        df['adx_neg'] = pd.to_numeric(df['adx_neg'], errors='coerce').astype(float)\n",
        "\n",
        "        # Stochastic RSI\n",
        "        stochrsi_indicator = ta.momentum.StochRSIIndicator(group['PRC'], window=14, smooth1=3, smooth2=3)\n",
        "        df.loc[permno, 'stochrsi'] = stochrsi_indicator.stochrsi()\n",
        "        df.loc[permno, 'stochrsi_k'] = stochrsi_indicator.stochrsi_k()\n",
        "        df.loc[permno, 'stochrsi_d'] = stochrsi_indicator.stochrsi_d()\n",
        "        # Convert 'stochrsi_k' and 'stochrsi_d' to float\n",
        "        df['stochrsi'] = pd.to_numeric(df['stochrsi'], errors = 'coerce').astype(float)\n",
        "        df['stochrsi_k'] = pd.to_numeric(df['stochrsi_k'], errors='coerce').astype(float)\n",
        "        df['stochrsi_d'] = pd.to_numeric(df['stochrsi_d'], errors='coerce').astype(float)\n",
        "\n",
        "        # Stochastic Oscillator\n",
        "        stoch_indicator = ta.momentum.StochasticOscillator(group['ASKHI'], group['BIDLO'], group['PRC'], window=14, smooth_window=3)\n",
        "        df.loc[permno, 'stoch'] = stoch_indicator.stoch()\n",
        "        df.loc[permno, 'stoch_signal'] = stoch_indicator.stoch_signal()\n",
        "        # Convert 'stoch' and 'stoch_signal' to float\n",
        "        df['stoch'] = pd.to_numeric(df['stoch'], errors='coerce').astype(float)\n",
        "        df['stoch_signal'] = pd.to_numeric(df['stoch_signal'], errors='coerce').astype(float)\n",
        "\n",
        "        # ROC (Rate of Change)\n",
        "        roc_indicator = ta.momentum.ROCIndicator(group['PRC'], window=12)\n",
        "        df.loc[permno, 'roc'] = roc_indicator.roc()\n",
        "        # Convert 'roc' to float\n",
        "        df['roc'] = pd.to_numeric(df['roc'], errors='coerce').astype(float)\n",
        "\n",
        "        # True Strength Index (TSI)\n",
        "        tsi_indicator = ta.momentum.tsi(group['PRC'], window_slow=25, window_fast=13)\n",
        "        df.loc[permno, 'tsi'] = tsi_indicator\n",
        "        # Convert 'tsi' to float\n",
        "        df['tsi'] = pd.to_numeric(df['tsi'], errors='coerce').astype(float)\n",
        "\n",
        "        # Ease of Movement (EoM)\n",
        "        eom_indicator = ta.volume.EaseOfMovementIndicator(group['ASKHI'], group['BIDLO'], group['VOL'], window=14)\n",
        "        df.loc[permno, 'eom'] = eom_indicator.ease_of_movement()\n",
        "        df.loc[permno, 'eom_sma'] = eom_indicator.sma_ease_of_movement()\n",
        "        # Convert 'eom' and 'eom_sma' to float\n",
        "        df['eom'] = pd.to_numeric(df['eom'], errors='coerce').astype(float)\n",
        "        df['eom_sma'] = pd.to_numeric(df['eom_sma'], errors='coerce').astype(float)\n",
        "\n",
        "        # Chaikin Money Flow (CMF)\n",
        "        cmf_indicator = ta.volume.ChaikinMoneyFlowIndicator(group['ASKHI'], group['BIDLO'], group['PRC'], group['VOL'], window=20)\n",
        "        df.loc[permno, 'cmf'] = cmf_indicator.chaikin_money_flow()\n",
        "        df['cmf'] = pd.to_numeric(df['cmf'], errors='coerce').astype(float)\n",
        "\n",
        "        # Accumulation/Distribution Index (ADI)\n",
        "        adi_indicator = ta.volume.AccDistIndexIndicator(group['ASKHI'], group['BIDLO'], group['PRC'], group['VOL'])\n",
        "        df.loc[permno, 'adi'] = adi_indicator.acc_dist_index()\n",
        "        df['adi'] = pd.to_numeric(df['adi'], errors='coerce').astype(float)\n",
        "\n",
        "        # Force Index (FI)\n",
        "        fi_indicator = ta.volume.ForceIndexIndicator(group['PRC'], group['VOL'], window=13)\n",
        "        df.loc[permno, 'fi'] = fi_indicator.force_index()\n",
        "        df['fi'] = pd.to_numeric(df['fi'], errors='coerce').astype(float)\n",
        "\n",
        "        # Money Flow Index (MFI)\n",
        "        mfi_indicator = ta.volume.MFIIndicator(group['ASKHI'], group['BIDLO'], group['PRC'], group['VOL'], window=14)\n",
        "        df.loc[permno, 'mfi'] = mfi_indicator.money_flow_index()\n",
        "        df['mfi'] = pd.to_numeric(df['mfi'], errors='coerce').astype(float)\n",
        "\n",
        "        # On-Balance Volume (OBV)\n",
        "        obv_indicator = ta.volume.OnBalanceVolumeIndicator(group['PRC'], group['VOL'])\n",
        "        df.loc[permno, 'obv'] = obv_indicator.on_balance_volume()\n",
        "        df['obv'] = pd.to_numeric(df['obv'], errors='coerce').astype(float)\n",
        "\n",
        "        # Average True Range (ATR)\n",
        "        atr_indicator = ta.volatility.AverageTrueRange(group['ASKHI'], group['BIDLO'], group['PRC'], window=14)\n",
        "        df.loc[permno, 'atr'] = atr_indicator.average_true_range()\n",
        "        df['atr'] = pd.to_numeric(df['atr'], errors='coerce').astype(float)\n",
        "\n",
        "        # Bollinger Bands\n",
        "        bollinger_indicator = ta.volatility.BollingerBands(group['PRC'], window=20, window_dev=2)\n",
        "        df.loc[permno, 'bollinger_hband'] = bollinger_indicator.bollinger_hband()\n",
        "        df.loc[permno, 'bollinger_hband_indicator'] = bollinger_indicator.bollinger_hband_indicator()\n",
        "        df.loc[permno, 'bollinger_lband'] = bollinger_indicator.bollinger_lband()\n",
        "        df.loc[permno, 'bollinger_lband_indicator'] = bollinger_indicator.bollinger_lband_indicator()\n",
        "        df.loc[permno, 'bollinger_mavg'] = bollinger_indicator.bollinger_mavg()\n",
        "        df.loc[permno, 'bollinger_pband'] = bollinger_indicator.bollinger_pband()\n",
        "        df.loc[permno, 'bollinger_wband'] = bollinger_indicator.bollinger_wband()\n",
        "\n",
        "        df['bollinger_hband'] = pd.to_numeric(df['bollinger_hband'], errors='coerce').astype(float)\n",
        "        df['bollinger_hband_indicator'] = pd.to_numeric(df['bollinger_hband_indicator'], errors='coerce').astype(float)\n",
        "        df['bollinger_lband'] = pd.to_numeric(df['bollinger_lband'], errors='coerce').astype(float)\n",
        "        df['bollinger_lband_indicator'] = pd.to_numeric(df['bollinger_lband_indicator'], errors='coerce').astype(float)\n",
        "        df['bollinger_mavg'] = pd.to_numeric(df['bollinger_mavg'], errors='coerce').astype(float)\n",
        "        df['bollinger_pband'] = pd.to_numeric(df['bollinger_pband'], errors='coerce').astype(float)\n",
        "        df['bollinger_wband'] = pd.to_numeric(df['bollinger_wband'], errors='coerce').astype(float)\n",
        "\n",
        "    return df"
      ],
      "metadata": {
        "id": "wZAD30QX3szP"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def add_fft(df):\n",
        "    grouped = df.groupby('PERMNO')\n",
        "\n",
        "    for permno, group in grouped:\n",
        "        df.loc[permno, 'fft'] = np.fft.fft(group['PRC'])\n",
        "\n",
        "    return df"
      ],
      "metadata": {
        "id": "RQpbtjGqJRs5"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def fit_garch_model(prices, p, q):\n",
        "    model = arch.arch_model(prices, vol='Garch', p=p, q=q)\n",
        "    result = model.fit(disp='off')\n",
        "    return result.conditional_volatility\n",
        "\n",
        "def addGARCH_of_differentfactors(df):\n",
        "    # create a series of p and q\n",
        "    p_values = range(1, 3)\n",
        "    q_values = range(1, 3)\n",
        "    grouped = df.groupby('PERMNO')\n",
        "\n",
        "    for permno, group in grouped:\n",
        "      for p in p_values:\n",
        "        for q in q_values:\n",
        "          df.loc[permno, f'GARCH_{p}_{q}'] = fit_garch_model(group['PRC'], p, q)\n",
        "\n",
        "    return df"
      ],
      "metadata": {
        "id": "zQb0gOKJKH1D"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def fit_arma_model(series, ar_order, ma_order):\n",
        "    arma_model = ARIMA(series, order=(ar_order, 0, ma_order))\n",
        "    arma_results = arma_model.fit()\n",
        "    return arma_results.fittedvalues\n",
        "\n",
        "def add_ARMA(df):\n",
        "    ar_values = range(1, 3)\n",
        "    ma_values = range(1, 3)\n",
        "    grouped = df.groupby('PERMNO')\n",
        "\n",
        "    for _, group in grouped:\n",
        "        for ar in ar_values:\n",
        "            for ma in ma_values:\n",
        "                arma_col_name = f'ARMA_{ar}_{ma}'\n",
        "                df.loc[group.index, arma_col_name] = fit_arma_model(group['PRC'], ar, ma)\n",
        "\n",
        "    return df"
      ],
      "metadata": {
        "id": "wkSVojB6D1ws"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = add_technical_indicators(data)\n",
        "data = data.dropna()\n",
        "data = add_fft(data)\n",
        "data = addGARCH_of_differentfactors(data)"
      ],
      "metadata": {
        "id": "5b6o8kZ6Jpcz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9d610471-4105-4c70-f42e-13ef7301eb73"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/arch/univariate/base.py:765: ConvergenceWarning: The optimizer returned code 8. The message is:\n",
            "Positive directional derivative for linesearch\n",
            "See scipy.optimize.fmin_slsqp for code meaning.\n",
            "\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data['fft'] = pd.to_numeric(data['fft'], errors='coerce').astype(float)\n",
        "data.info(verbose=True, show_counts=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w1DzxoqHDG4C",
        "outputId": "4a257782-57c7-45fc-b11d-0950486352b0"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "MultiIndex: 2492541 entries, (10057.0, Timestamp('1992-03-12 00:00:00')) to (93096.0, Timestamp('2022-12-30 00:00:00'))\n",
            "Data columns (total 115 columns):\n",
            " #    Column                     Non-Null Count    Dtype  \n",
            "---   ------                     --------------    -----  \n",
            " 0    BIDLO                      2492541 non-null  float64\n",
            " 1    ASKHI                      2492541 non-null  float64\n",
            " 2    PRC                        2492541 non-null  float64\n",
            " 3    VOL                        2492541 non-null  float64\n",
            " 4    RET                        2492541 non-null  float64\n",
            " 5    BID                        2492541 non-null  float64\n",
            " 6    ASK                        2492541 non-null  float64\n",
            " 7    SHROUT                     2492541 non-null  float64\n",
            " 8    OPENPRC                    2492541 non-null  float64\n",
            " 9    RETX                       2492541 non-null  float64\n",
            " 10   TICKER                     2492541 non-null  object \n",
            " 11   CAPEI                      2492541 non-null  float64\n",
            " 12   bm                         2492541 non-null  float64\n",
            " 13   evm                        2492541 non-null  float64\n",
            " 14   pe_op_basic                2492541 non-null  float64\n",
            " 15   pe_op_dil                  2492541 non-null  float64\n",
            " 16   pe_exi                     2492541 non-null  float64\n",
            " 17   pe_inc                     2492541 non-null  float64\n",
            " 18   ps                         2492541 non-null  float64\n",
            " 19   pcf                        2492541 non-null  float64\n",
            " 20   dpr                        2492541 non-null  float64\n",
            " 21   npm                        2492541 non-null  float64\n",
            " 22   opmbd                      2492541 non-null  float64\n",
            " 23   opmad                      2492541 non-null  float64\n",
            " 24   gpm                        2492541 non-null  float64\n",
            " 25   ptpm                       2492541 non-null  float64\n",
            " 26   cfm                        2492541 non-null  float64\n",
            " 27   roa                        2492541 non-null  float64\n",
            " 28   roe                        2492541 non-null  float64\n",
            " 29   roce                       2492541 non-null  float64\n",
            " 30   efftax                     2492541 non-null  float64\n",
            " 31   aftret_eq                  2492541 non-null  float64\n",
            " 32   aftret_invcapx             2492541 non-null  float64\n",
            " 33   aftret_equity              2492541 non-null  float64\n",
            " 34   pretret_noa                2492541 non-null  float64\n",
            " 35   pretret_earnat             2492541 non-null  float64\n",
            " 36   GProf                      2492541 non-null  float64\n",
            " 37   equity_invcap              2492541 non-null  float64\n",
            " 38   debt_invcap                2492541 non-null  float64\n",
            " 39   totdebt_invcap             2492541 non-null  float64\n",
            " 40   capital_ratio              2492541 non-null  float64\n",
            " 41   int_debt                   2492541 non-null  float64\n",
            " 42   int_totdebt                2492541 non-null  float64\n",
            " 43   cash_lt                    2492541 non-null  float64\n",
            " 44   invt_act                   2492541 non-null  float64\n",
            " 45   rect_act                   2492541 non-null  float64\n",
            " 46   debt_at                    2492541 non-null  float64\n",
            " 47   debt_ebitda                2492541 non-null  float64\n",
            " 48   short_debt                 2492541 non-null  float64\n",
            " 49   curr_debt                  2492541 non-null  float64\n",
            " 50   lt_debt                    2492541 non-null  float64\n",
            " 51   profit_lct                 2492541 non-null  float64\n",
            " 52   ocf_lct                    2492541 non-null  float64\n",
            " 53   cash_debt                  2492541 non-null  float64\n",
            " 54   fcf_ocf                    2492541 non-null  float64\n",
            " 55   lt_ppent                   2492541 non-null  float64\n",
            " 56   dltt_be                    2492541 non-null  float64\n",
            " 57   debt_assets                2492541 non-null  float64\n",
            " 58   debt_capital               2492541 non-null  float64\n",
            " 59   de_ratio                   2492541 non-null  float64\n",
            " 60   intcov                     2492541 non-null  float64\n",
            " 61   intcov_ratio               2492541 non-null  float64\n",
            " 62   cash_ratio                 2492541 non-null  float64\n",
            " 63   quick_ratio                2492541 non-null  float64\n",
            " 64   curr_ratio                 2492541 non-null  float64\n",
            " 65   cash_conversion            2492541 non-null  float64\n",
            " 66   inv_turn                   2492541 non-null  float64\n",
            " 67   at_turn                    2492541 non-null  float64\n",
            " 68   rect_turn                  2492541 non-null  float64\n",
            " 69   pay_turn                   2492541 non-null  float64\n",
            " 70   sale_invcap                2492541 non-null  float64\n",
            " 71   sale_equity                2492541 non-null  float64\n",
            " 72   sale_nwc                   2492541 non-null  float64\n",
            " 73   rd_sale                    2492541 non-null  float64\n",
            " 74   adv_sale                   2492541 non-null  float64\n",
            " 75   accrual                    2492541 non-null  float64\n",
            " 76   ptb                        2492541 non-null  float64\n",
            " 77   PEG_trailing               2492541 non-null  float64\n",
            " 78   divyield                   2492541 non-null  float64\n",
            " 79   rsi_14                     2492541 non-null  float64\n",
            " 80   sma_50                     2492541 non-null  float64\n",
            " 81   ema_20                     2492541 non-null  float64\n",
            " 82   macd                       2492541 non-null  float64\n",
            " 83   macd_diff                  2492541 non-null  float64\n",
            " 84   macd_signal                2492541 non-null  float64\n",
            " 85   adx                        2492541 non-null  float64\n",
            " 86   adx_pos                    2492541 non-null  float64\n",
            " 87   adx_neg                    2492541 non-null  float64\n",
            " 88   stochrsi                   2492541 non-null  float64\n",
            " 89   stochrsi_k                 2492541 non-null  float64\n",
            " 90   stochrsi_d                 2492541 non-null  float64\n",
            " 91   stoch                      2492541 non-null  float64\n",
            " 92   stoch_signal               2492541 non-null  float64\n",
            " 93   roc                        2492541 non-null  float64\n",
            " 94   tsi                        2492541 non-null  float64\n",
            " 95   eom                        2492541 non-null  float64\n",
            " 96   eom_sma                    2492541 non-null  float64\n",
            " 97   cmf                        2492541 non-null  float64\n",
            " 98   adi                        2492541 non-null  float64\n",
            " 99   fi                         2492541 non-null  float64\n",
            " 100  mfi                        2492541 non-null  float64\n",
            " 101  obv                        2492541 non-null  float64\n",
            " 102  atr                        2492541 non-null  float64\n",
            " 103  bollinger_hband            2492541 non-null  float64\n",
            " 104  bollinger_hband_indicator  2492541 non-null  float64\n",
            " 105  bollinger_lband            2492541 non-null  float64\n",
            " 106  bollinger_lband_indicator  2492541 non-null  float64\n",
            " 107  bollinger_mavg             2492541 non-null  float64\n",
            " 108  bollinger_pband            2492541 non-null  float64\n",
            " 109  bollinger_wband            2492541 non-null  float64\n",
            " 110  fft                        2492541 non-null  float64\n",
            " 111  GARCH_1_1                  2492541 non-null  float64\n",
            " 112  GARCH_1_2                  2492541 non-null  float64\n",
            " 113  GARCH_2_1                  2492541 non-null  float64\n",
            " 114  GARCH_2_2                  2492541 non-null  float64\n",
            "dtypes: float64(114), object(1)\n",
            "memory usage: 2.2+ GB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# save data to a paraquet.gzip file\n",
        "data.to_parquet('ML_Project_data.parquet.gzip', compression='gzip')"
      ],
      "metadata": {
        "id": "6zuUqVy8oYn4"
      },
      "execution_count": 45,
      "outputs": []
    }
  ]
}