Neural Network and Logistic Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "RnL8xke-upzm"
   },
   "outputs": [],
   "source": [
    "#import important library\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import io #Importing input module\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import linear_model  # Importing linear model\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as mpatches\n",
    "\n",
    "\n",
    "#import model library\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 480,
     "resources": {
      "http://localhost:8080/nbextensions/google.colab/files.js": {
       "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
       "headers": [
        [
         "content-type",
         "application/javascript"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      }
     }
    },
    "id": "jQHyqhAnzevs",
    "outputId": "0791201d-4fb6-47b0-80e9-6d4081131be1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SpMax_L</th>\n",
       "      <th>J_Dz(e)</th>\n",
       "      <th>nHM</th>\n",
       "      <th>F01[N-N]</th>\n",
       "      <th>F04[C-N]</th>\n",
       "      <th>NssssC</th>\n",
       "      <th>nCb-</th>\n",
       "      <th>C%</th>\n",
       "      <th>nCp</th>\n",
       "      <th>nO</th>\n",
       "      <th>...</th>\n",
       "      <th>C-026</th>\n",
       "      <th>F02[C-N]</th>\n",
       "      <th>nHDon</th>\n",
       "      <th>SpMax_B(m)</th>\n",
       "      <th>Psi_i_A</th>\n",
       "      <th>nN</th>\n",
       "      <th>SM6_B(m)</th>\n",
       "      <th>nArCOOR</th>\n",
       "      <th>nX</th>\n",
       "      <th>experimental_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.919</td>\n",
       "      <td>2.6909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>31.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.949</td>\n",
       "      <td>1.591</td>\n",
       "      <td>0</td>\n",
       "      <td>7.253</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.170</td>\n",
       "      <td>2.1144</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30.8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.315</td>\n",
       "      <td>1.967</td>\n",
       "      <td>0</td>\n",
       "      <td>7.257</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.932</td>\n",
       "      <td>3.2512</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26.7</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.076</td>\n",
       "      <td>2.417</td>\n",
       "      <td>0</td>\n",
       "      <td>7.601</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.000</td>\n",
       "      <td>2.7098</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.046</td>\n",
       "      <td>5.000</td>\n",
       "      <td>0</td>\n",
       "      <td>6.690</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.236</td>\n",
       "      <td>3.3944</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>29.4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.351</td>\n",
       "      <td>2.405</td>\n",
       "      <td>0</td>\n",
       "      <td>8.003</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>RB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050</th>\n",
       "      <td>5.431</td>\n",
       "      <td>2.8955</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>32.1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3.573</td>\n",
       "      <td>2.242</td>\n",
       "      <td>1</td>\n",
       "      <td>8.088</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NRB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1051</th>\n",
       "      <td>5.287</td>\n",
       "      <td>3.3732</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35.3</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3.787</td>\n",
       "      <td>3.083</td>\n",
       "      <td>3</td>\n",
       "      <td>9.278</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NRB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1052</th>\n",
       "      <td>4.869</td>\n",
       "      <td>1.7670</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>44.4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>3.848</td>\n",
       "      <td>2.576</td>\n",
       "      <td>5</td>\n",
       "      <td>9.537</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NRB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053</th>\n",
       "      <td>5.158</td>\n",
       "      <td>1.6914</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>56.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>5.808</td>\n",
       "      <td>2.055</td>\n",
       "      <td>8</td>\n",
       "      <td>11.055</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NRB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1054</th>\n",
       "      <td>5.076</td>\n",
       "      <td>2.6588</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>54.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.009</td>\n",
       "      <td>2.206</td>\n",
       "      <td>0</td>\n",
       "      <td>9.130</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NRB</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1055 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      SpMax_L  J_Dz(e)  nHM  F01[N-N]  F04[C-N]  NssssC  nCb-    C%  nCp  nO  \\\n",
       "0       3.919   2.6909    0         0         0       0     0  31.4    2   0   \n",
       "1       4.170   2.1144    0         0         0       0     0  30.8    1   1   \n",
       "2       3.932   3.2512    0         0         0       0     0  26.7    2   4   \n",
       "3       3.000   2.7098    0         0         0       0     0  20.0    0   2   \n",
       "4       4.236   3.3944    0         0         0       0     0  29.4    2   4   \n",
       "...       ...      ...  ...       ...       ...     ...   ...   ...  ...  ..   \n",
       "1050    5.431   2.8955    0         0         0       2     0  32.1    4   1   \n",
       "1051    5.287   3.3732    0         0         9       0     0  35.3    0   9   \n",
       "1052    4.869   1.7670    0         1         9       0     5  44.4    0   4   \n",
       "1053    5.158   1.6914    2         0        36       0     9  56.1    0   0   \n",
       "1054    5.076   2.6588    2         0         0       0     4  54.5    0   0   \n",
       "\n",
       "      ...  C-026  F02[C-N]  nHDon  SpMax_B(m)  Psi_i_A  nN  SM6_B(m)  nArCOOR  \\\n",
       "0     ...      0         0      0       2.949    1.591   0     7.253        0   \n",
       "1     ...      0         0      0       3.315    1.967   0     7.257        0   \n",
       "2     ...      0         0      1       3.076    2.417   0     7.601        0   \n",
       "3     ...      0         0      1       3.046    5.000   0     6.690        0   \n",
       "4     ...      0         0      0       3.351    2.405   0     8.003        0   \n",
       "...   ...    ...       ...    ...         ...      ...  ..       ...      ...   \n",
       "1050  ...      0         6      1       3.573    2.242   1     8.088        0   \n",
       "1051  ...      0         3      0       3.787    3.083   3     9.278        0   \n",
       "1052  ...      4        13      0       3.848    2.576   5     9.537        1   \n",
       "1053  ...      1        16      0       5.808    2.055   8    11.055        0   \n",
       "1054  ...      2         0      0       4.009    2.206   0     9.130        0   \n",
       "\n",
       "      nX  experimental_class  \n",
       "0      0                  RB  \n",
       "1      0                  RB  \n",
       "2      0                  RB  \n",
       "3      0                  RB  \n",
       "4      0                  RB  \n",
       "...   ..                 ...  \n",
       "1050   0                 NRB  \n",
       "1051   0                 NRB  \n",
       "1052   0                 NRB  \n",
       "1053   1                 NRB  \n",
       "1054   2                 NRB  \n",
       "\n",
       "[1055 rows x 42 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import dataset\n",
    "\n",
    "from google.colab import files\n",
    "uploaded = files.upload()\n",
    "\n",
    "import io\n",
    "data = pd.read_csv(io.BytesIO(uploaded['biodeg.csv']), sep=';')\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "03dityiwzopv"
   },
   "source": [
    "**Data processing**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6perK3uZzuD_",
    "outputId": "03f69488-9b2b-4373-fa52-01d88b27b358"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target variables  :  ['NRB' 'RB']\n"
     ]
    }
   ],
   "source": [
    "#Get binary classification of the response \n",
    "response = np.unique(data['experimental_class'])\n",
    "print('Target variables  : ', response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "ZSwjQtuNzxpP"
   },
   "outputs": [],
   "source": [
    "#convert the binary classification clasess into numerical values \n",
    "#so that it can be plotted \n",
    "#'RB' is equal to value 1 (true)\n",
    "#'NRB' is equal to value 0 (false)\n",
    "\n",
    "def y_to_numeric(y):\n",
    "  y_test_list = []\n",
    "  for i, value in enumerate(y):\n",
    "    if (value == response[1]):\n",
    "      y_test_list.append(1)\n",
    "    else:\n",
    "      y_test_list.append(0)\n",
    "\n",
    "  return np.array(y_test_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "OyqdkD6Czzdn"
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#Extract features and response into respective y and X variable\n",
    "y = data['experimental_class'].values\n",
    "y = y_to_numeric(y)\n",
    "X = data[data.keys()[:-1]]\n",
    "\n",
    "#Normalize data\n",
    "scaler = MinMaxScaler()\n",
    "X = scaler.fit_transform(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IPdC0gFX0Dof"
   },
   "source": [
    "**Features Selection**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gCNeozfb0Bpv",
    "outputId": "209903d3-1083-448e-ce5a-8d250a2b3350"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features selection using ANOVA method:\n",
      "--------------------------------------\n",
      "       Features       P Value       Score\n",
      "0       SpMax_L  5.741990e-41  195.999690\n",
      "26      SpMax_A  1.205494e-39  188.835426\n",
      "21  SpPosA_B(p)  5.126240e-36  169.390245\n",
      "38     SM6_B(m)  6.093413e-35  163.690168\n",
      "12    HyWi_B(m)  1.238373e-30  141.126544\n",
      "14        SM6_L  1.462180e-30  140.752426\n",
      "6          nCb-  1.769975e-29  135.151782\n",
      "32        C-026  2.619938e-26  118.916644\n",
      "2           nHM  3.022542e-23  103.463027\n",
      "35   SpMax_B(m)  7.808043e-22   96.411675\n",
      "13          LOC  8.342953e-20   86.365580\n",
      "33     F02[C-N]  6.274198e-19   82.057578\n",
      "37           nN  5.480339e-18   77.450978\n",
      "24    B03[C-Cl]  9.297153e-17   71.466604\n",
      "10     F03[C-N]  1.453584e-15   65.691318\n",
      "4      F04[C-N]  1.167523e-14   61.339528\n",
      "40           nX  1.920032e-12   50.773587\n",
      "7            C%  3.884058e-11   44.611199\n",
      "9            nO  6.867642e-09   34.129262\n",
      "30        TI2_L  1.393576e-08   32.709270\n",
      "5        NssssC  2.538854e-08   31.508325\n",
      "19       nArNO2  5.325140e-07   25.457085\n",
      "39      nArCOOR  1.072241e-06   24.076257\n",
      "17           Mi  1.814320e-05   18.545029\n",
      "22         nCIR  1.469638e-04   14.516704\n",
      "36      Psi_i_A  1.841175e-04   14.086633\n",
      "23    B01[C-Br]  1.924918e-04   14.001863\n",
      "11        SdssC  2.532684e-04   13.479685\n",
      "31         nCrt  5.245271e-04   12.101096\n",
      "3      F01[N-N]  7.795384e-04   11.355478\n",
      "20        nCRX3  1.751214e-03    9.843964\n",
      "28    B04[C-Br]  2.526082e-03    9.165594\n",
      "25        N-073  2.834218e-03    8.953312\n",
      "16           Me  2.926507e-03    8.894285\n",
      "18         nN-N  5.204057e-02    3.783036\n",
      "8           nCp  6.833389e-02    3.329425\n",
      "29          SdO  8.162462e-02    3.038067\n",
      "34        nHDon  3.741843e-01    0.790393\n",
      "27     Psi_i_1d  4.168589e-01    0.659673\n",
      "15     F03[C-O]  9.255871e-01    0.008727\n",
      "1       J_Dz(e)  9.508476e-01    0.003802\n"
     ]
    }
   ],
   "source": [
    "# Import the feature selection library\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_classif\n",
    "\n",
    "#apply SelectKBest class to extract best features from the dataset\n",
    "#ANOVA is used as the feature selection method\n",
    "#the lower p values acquired, the stronger the relationship with the response\n",
    "#the higher score acquired, the stronger the relationship with the response\n",
    "bestfeatures = SelectKBest(score_func=f_classif)\n",
    "fit = bestfeatures.fit(X,y)\n",
    "\n",
    "dfpvalues = pd.DataFrame(fit.pvalues_)\n",
    "dfscores = pd.DataFrame(fit.scores_)\n",
    "dfcolumns = pd.DataFrame(data.keys()[:-1])\n",
    "\n",
    "featureScores = pd.concat([dfcolumns,dfpvalues,dfscores],axis=1)\n",
    "featureScores.columns = ['Features','P Value', 'Score']  #naming the dataframe columns\n",
    "\n",
    "pvalueTop = featureScores.nsmallest(featureScores.shape[0],'P Value')\n",
    "\n",
    "#print the p value for each features in ascending order\n",
    "#the first on the list has the strongest correlation with the target response\n",
    "print (\"Features selection using ANOVA method:\")\n",
    "print (\"--------------------------------------\")\n",
    "print(pvalueTop) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "dgoGeHNK0Rwv"
   },
   "outputs": [],
   "source": [
    "# A feature is deemed significant if the p value acquired is less than 0.05\n",
    "# The bottom 7 features consists of p value that is higher than 0.5\n",
    "# So they are considered to have weak relationship with the target response\n",
    "weak_features = 7\n",
    "\n",
    "# Only select features with p value that is lower than 0.05\n",
    "# To ensure only significant features used for model training\n",
    "# Store best features in array for later us\n",
    "featuresBest = pvalueTop['Features'][:-weak_features or None] #Assign"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "g5Z9XW0u0UtP",
    "outputId": "008d1719-c51a-4bd1-fc27-345e09325dbd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(844, 34)\n",
      "(211, 34)\n"
     ]
    }
   ],
   "source": [
    "# Splitting the dataset into train and test set \n",
    "X = data[featuresBest].values\n",
    "seed_num = 10\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=seed_num)\n",
    "\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9RFxW2yBsISP"
   },
   "source": [
    "**Logistic Regression**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "iqm1sVCwyxC2"
   },
   "outputs": [],
   "source": [
    "# Importing libraries for logistic regression\n",
    "import numpy as np\n",
    "from sklearn import linear_model, decomposition\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "8hsJusC4yo3B"
   },
   "outputs": [],
   "source": [
    "# Creating objects required for pipelining\n",
    "std_slc = StandardScaler()\n",
    "pca = decomposition.PCA()\n",
    "logistic_Reg = linear_model.LogisticRegression(max_iter=2000, solver='liblinear')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "7FDv9M50ywxw"
   },
   "outputs": [],
   "source": [
    "# Assigning all three objects to pipe\n",
    "pipe = Pipeline(steps=[('std_slc', std_slc),\n",
    "                        ('pca', pca),\n",
    "                        ('logistic_Reg', logistic_Reg)])\n",
    "C = np.logspace(-4, 4, 50)\n",
    "penalty = ['l1', 'l2']\n",
    "parameters = dict(logistic_Reg__C=C,\n",
    "                  logistic_Reg__penalty=penalty)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NwBsUKvly0rj",
    "outputId": "26d2fcb7-6e94-45fa-889d-83963e1720fa"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(estimator=Pipeline(steps=[(&#x27;std_slc&#x27;, StandardScaler()),\n",
       "                                       (&#x27;pca&#x27;, PCA()),\n",
       "                                       (&#x27;logistic_Reg&#x27;,\n",
       "                                        LogisticRegression(max_iter=2000,\n",
       "                                                           solver=&#x27;liblinear&#x27;))]),\n",
       "             param_grid={&#x27;logistic_Reg__C&#x27;: array([1.00000000e-04, 1.45634848e-04, 2.12095089e-04, 3.08884360e-04,\n",
       "       4.49843267e-04, 6.55128557e-04, 9.54095476e-04, 1.38949549e-03,\n",
       "       2.02358965e-03, 2.94705170e-03, 4.2919342...\n",
       "       3.72759372e+00, 5.42867544e+00, 7.90604321e+00, 1.15139540e+01,\n",
       "       1.67683294e+01, 2.44205309e+01, 3.55648031e+01, 5.17947468e+01,\n",
       "       7.54312006e+01, 1.09854114e+02, 1.59985872e+02, 2.32995181e+02,\n",
       "       3.39322177e+02, 4.94171336e+02, 7.19685673e+02, 1.04811313e+03,\n",
       "       1.52641797e+03, 2.22299648e+03, 3.23745754e+03, 4.71486636e+03,\n",
       "       6.86648845e+03, 1.00000000e+04]),\n",
       "                         &#x27;logistic_Reg__penalty&#x27;: [&#x27;l1&#x27;, &#x27;l2&#x27;]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(estimator=Pipeline(steps=[(&#x27;std_slc&#x27;, StandardScaler()),\n",
       "                                       (&#x27;pca&#x27;, PCA()),\n",
       "                                       (&#x27;logistic_Reg&#x27;,\n",
       "                                        LogisticRegression(max_iter=2000,\n",
       "                                                           solver=&#x27;liblinear&#x27;))]),\n",
       "             param_grid={&#x27;logistic_Reg__C&#x27;: array([1.00000000e-04, 1.45634848e-04, 2.12095089e-04, 3.08884360e-04,\n",
       "       4.49843267e-04, 6.55128557e-04, 9.54095476e-04, 1.38949549e-03,\n",
       "       2.02358965e-03, 2.94705170e-03, 4.2919342...\n",
       "       3.72759372e+00, 5.42867544e+00, 7.90604321e+00, 1.15139540e+01,\n",
       "       1.67683294e+01, 2.44205309e+01, 3.55648031e+01, 5.17947468e+01,\n",
       "       7.54312006e+01, 1.09854114e+02, 1.59985872e+02, 2.32995181e+02,\n",
       "       3.39322177e+02, 4.94171336e+02, 7.19685673e+02, 1.04811313e+03,\n",
       "       1.52641797e+03, 2.22299648e+03, 3.23745754e+03, 4.71486636e+03,\n",
       "       6.86648845e+03, 1.00000000e+04]),\n",
       "                         &#x27;logistic_Reg__penalty&#x27;: [&#x27;l1&#x27;, &#x27;l2&#x27;]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;std_slc&#x27;, StandardScaler()), (&#x27;pca&#x27;, PCA()),\n",
       "                (&#x27;logistic_Reg&#x27;,\n",
       "                 LogisticRegression(max_iter=2000, solver=&#x27;liblinear&#x27;))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">PCA</label><div class=\"sk-toggleable__content\"><pre>PCA()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(max_iter=2000, solver=&#x27;liblinear&#x27;)</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(estimator=Pipeline(steps=[('std_slc', StandardScaler()),\n",
       "                                       ('pca', PCA()),\n",
       "                                       ('logistic_Reg',\n",
       "                                        LogisticRegression(max_iter=2000,\n",
       "                                                           solver='liblinear'))]),\n",
       "             param_grid={'logistic_Reg__C': array([1.00000000e-04, 1.45634848e-04, 2.12095089e-04, 3.08884360e-04,\n",
       "       4.49843267e-04, 6.55128557e-04, 9.54095476e-04, 1.38949549e-03,\n",
       "       2.02358965e-03, 2.94705170e-03, 4.2919342...\n",
       "       3.72759372e+00, 5.42867544e+00, 7.90604321e+00, 1.15139540e+01,\n",
       "       1.67683294e+01, 2.44205309e+01, 3.55648031e+01, 5.17947468e+01,\n",
       "       7.54312006e+01, 1.09854114e+02, 1.59985872e+02, 2.32995181e+02,\n",
       "       3.39322177e+02, 4.94171336e+02, 7.19685673e+02, 1.04811313e+03,\n",
       "       1.52641797e+03, 2.22299648e+03, 3.23745754e+03, 4.71486636e+03,\n",
       "       6.86648845e+03, 1.00000000e+04]),\n",
       "                         'logistic_Reg__penalty': ['l1', 'l2']})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Displaying flow of GridSearchCV\n",
    "clf_GS = GridSearchCV(pipe, parameters)\n",
    "clf_GS.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EyJoJ1zvy6aD",
    "outputId": "f5030e7d-240d-4286-ad9a-b43c1a4b9dcc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Penalty: l1\n",
      "Best C: 2.559547922699533\n",
      "\n",
      "LogisticRegression(C=2.559547922699533, max_iter=2000, penalty='l1',\n",
      "                   solver='liblinear')\n"
     ]
    }
   ],
   "source": [
    "# Displaying best penalty, best C and other parameters of logistic regression\n",
    "Best_Penalty = clf_GS.best_estimator_.get_params()['logistic_Reg__penalty']\n",
    "Best_C = clf_GS.best_estimator_.get_params()['logistic_Reg__C']\n",
    "print('Best Penalty:', Best_Penalty )\n",
    "print('Best C:', Best_C )\n",
    "print(); print(clf_GS.best_estimator_.get_params()['logistic_Reg'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "ZfspiXFEzCe4"
   },
   "outputs": [],
   "source": [
    "# Assigning linear_model.LogisticRegression to model\n",
    "model = linear_model.LogisticRegression(penalty=Best_Penalty, C=Best_C, max_iter=2000, solver='liblinear' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tQV0GI6BzDU-",
    "outputId": "49966437-90b5-4cf7-b92d-765d84c142fa"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression(C=2.559547922699533, max_iter=2000, penalty=&#x27;l1&#x27;,\n",
       "                   solver=&#x27;liblinear&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" checked><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(C=2.559547922699533, max_iter=2000, penalty=&#x27;l1&#x27;,\n",
       "                   solver=&#x27;liblinear&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LogisticRegression(C=2.559547922699533, max_iter=2000, penalty='l1',\n",
       "                   solver='liblinear')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Displaying the model of logistic regression\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "iE1oiN2WzF9J",
    "outputId": "75190733-be87-41a6-817f-42a37c6d35ae"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:\n",
      "0.8767772511848341\n",
      " \n",
      "Confusion Matrix:\n",
      "[[135  12]\n",
      " [ 14  50]]\n",
      " \n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.92      0.91       147\n",
      "           1       0.81      0.78      0.79        64\n",
      "\n",
      "    accuracy                           0.88       211\n",
      "   macro avg       0.86      0.85      0.85       211\n",
      "weighted avg       0.88      0.88      0.88       211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Accuracy, confusion matrix and classification report of logistic regression\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "y_pred = model.predict(X_test)\n",
    "print(\"Accuracy:\")\n",
    "print(accuracy_score(y_test, y_pred))\n",
    "print(\" \")\n",
    "print(\"Confusion Matrix:\")\n",
    "print(confusion_matrix(y_test, y_pred))\n",
    "print(\" \")\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sWisiqlK0ee7"
   },
   "source": [
    "**Neural Network**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "r-k3OxWq5dbv",
    "outputId": "ccd28149-a78b-4c3d-8e0a-a792014b01b6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: scikeras in c:\\users\\derek\\anaconda3\\lib\\site-packages (0.10.0)\n",
      "Requirement already satisfied: packaging>=0.21 in c:\\users\\derek\\anaconda3\\lib\\site-packages (from scikeras) (21.0)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in c:\\users\\derek\\anaconda3\\lib\\site-packages (from scikeras) (1.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\derek\\anaconda3\\lib\\site-packages (from packaging>=0.21->scikeras) (3.0.4)\n",
      "Requirement already satisfied: scipy>=1.3.2 in c:\\users\\derek\\anaconda3\\lib\\site-packages (from scikit-learn>=1.0.0->scikeras) (1.7.1)\n",
      "Requirement already satisfied: numpy>=1.17.3 in c:\\users\\derek\\anaconda3\\lib\\site-packages (from scikit-learn>=1.0.0->scikeras) (1.20.3)\n",
      "Requirement already satisfied: joblib>=1.1.1 in c:\\users\\derek\\anaconda3\\lib\\site-packages (from scikit-learn>=1.0.0->scikeras) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\derek\\anaconda3\\lib\\site-packages (from scikit-learn>=1.0.0->scikeras) (2.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install scikeras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "_TzlSNE50kqW"
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m        Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_38336/2645323697.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# import library for NN model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mloadtxt\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow'"
     ]
    }
   ],
   "source": [
    "# import library for NN model\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from numpy import loadtxt\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import Dropout\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from scikeras.wrappers import KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "BMzAlTpu0rZh"
   },
   "outputs": [],
   "source": [
    "# set default values\n",
    "input_size = featuresBest.shape[0]\n",
    "def_optimizer = 'adam'\n",
    "def_init = 'uniform'\n",
    "\n",
    "# Create a function that creates the model (required for KerasClassifier) \n",
    "# while accepting the hyperparameters we want to tune \n",
    "# we also pass some default values such as optimizer='adam'\n",
    "def create_model (optimizer=def_optimizer, init_mode=def_init):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(12, input_shape=(input_size,), kernel_initializer=init_mode, activation='relu'))\n",
    "    model.add(Dense(8, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "     # compile model\n",
    "    model.compile(loss='binary_crossentropy', \n",
    "                  optimizer=optimizer, \n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fpmQ54_5sHf3"
   },
   "source": [
    "**Hyperparameters Optimization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8YFvcBNR-lT8"
   },
   "outputs": [],
   "source": [
    "# Neural network model is influenced by a few hyperparameters\n",
    "# that would affect the the overall performance of the model\n",
    "# Optimization by this part of coding will focus on the following hyperparameters:\n",
    "# batch sizes, epochs, optimizer algorithm and initialization mode\n",
    "\n",
    "# Optimization is done using grid search method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rcEhovy2XcNr",
    "outputId": "6a595141-9263-4316-fa2a-2fc3251d8095"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# Optimize batch size and epoch\n",
    "print (\"The optimization process has been executed.\")\n",
    "print (\"Please wait for a while ...\")\n",
    "# create model\n",
    "model = KerasClassifier(model=create_model, verbose=0)\n",
    "# define the grid search parameters\n",
    "batch_size = [32, 64, 128]\n",
    "epochs = [20, 50, 100, 200]\n",
    "param_grid = dict(batch_size=batch_size, epochs=epochs)\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)\n",
    "grid_result = grid.fit(X_train, y_train)\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "\n",
    "best_batch, best_epoch = grid_result.best_params_['batch_size'], grid_result.best_params_['epochs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_poCQAw19cZk",
    "outputId": "24e31eb0-cc35-4b8c-eb4e-a55a8f27593a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The optimization process has been executed.\n",
      "Please wait for a while ...\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'KerasClassifier' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                  Traceback (most recent call last)",
      "\u001b[1;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'KerasClassifier' is not defined"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Optimize optimisation algorithm\n",
    "print (\"The optimization process has been executed.\")\n",
    "print (\"Please wait for a while ...\")\n",
    "# create model\n",
    "model = KerasClassifier(model=create_model, epochs=best_epoch, batch_size=best_batch, verbose=0)\n",
    "# define the grid search parameters\n",
    "optimizer = ['sgd', 'rmsprop', 'adam', 'adamax']\n",
    "param_grid = dict(model__optimizer=optimizer)\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)\n",
    "grid_result = grid.fit(X_train, y_train)\n",
    "# summarize results\n",
    "print(\"\\nBest: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "\n",
    "best_optimizer = grid_result.best_params_['model__optimizer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9q2_fltTcflB",
    "outputId": "ca43ad5b-5a5a-4a58-ec03-4cf1aa4391dd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The optimization process has been executed.\n",
      "Please wait for a while ...\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'KerasClassifier' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                  Traceback (most recent call last)",
      "\u001b[1;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'KerasClassifier' is not defined"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Optimize init mode\n",
    "print (\"The optimization process has been executed.\")\n",
    "print (\"Please wait for a while ...\")\n",
    "# create model\n",
    "model = KerasClassifier(model=create_model, epochs=best_epoch, batch_size=best_batch, optimizer=best_optimizer, verbose=0)\n",
    "# define the grid search parameters\n",
    "init_mode = ['uniform', 'lecun_uniform', 'glorot_uniform', 'he_uniform']\n",
    "param_grid = dict(model__init_mode=init_mode)\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)\n",
    "grid_result = grid.fit(X_train, y_train)\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "\n",
    "best_init = grid_result.best_params_['model__init_mode']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "k2PUtYsHjl3c",
    "outputId": "e233ce12-20bd-4340-fd6e-aff87314c512"
   },
   "outputs": [],
   "source": [
    "# Display\n",
    "print (\"Optimized parameters:\")\n",
    "print (\"---------------------\")\n",
    "print (\"Init mode: \", best_init)\n",
    "print (\"Optimizer: \", best_optimizer)\n",
    "print (\"Batch sizes: \", best_batch)\n",
    "print (\"Epochs: \", best_epoch)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7_fLUxQtsPd0"
   },
   "source": [
    "**Model Training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WVrsVxQ57p2M"
   },
   "outputs": [],
   "source": [
    "# Function to further optimize the value of epoch \n",
    "# Best epoch is chosen so that it minimize the validation loss\n",
    "stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yi_n4ZA57tHE",
    "outputId": "928de537-ca2a-4cf0-c0ba-9ef3f03632b9"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'best_optimizer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                  Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_38336/1486598900.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Train model using the optimal hyperparameters generated\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcreate_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbest_optimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minit_mode\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbest_init\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mmodel_history\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbest_epoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbest_batch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstop_early\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.25\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0moptimize_epoch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_history\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'val_loss'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'best_optimizer' is not defined"
     ]
    }
   ],
   "source": [
    "# Train model using the optimal hyperparameters generated\n",
    "model = create_model(optimizer=best_optimizer, init_mode=best_init)\n",
    "model_history = model.fit(X_train, y_train, epochs=best_epoch, batch_size=best_batch, callbacks=[stop_early], validation_split=0.25)\n",
    "\n",
    "optimize_epoch = len(model_history.history['val_loss'])\n",
    "print('Optimize epoch:', optimize_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 830
    },
    "id": "b2tiEAlpYyj_",
    "outputId": "e6443d7e-f7f8-4839-d0de-dc050b391043"
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'LogisticRegression' object has no attribute 'evaluate'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m             Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_53420/4007488940.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Display history of training, validation accuracy and loss for each epoch\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m# Best epoch is chosen so that it minimize the validation loss\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mscore\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Test loss:'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Test accuracy:'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'LogisticRegression' object has no attribute 'evaluate'"
     ]
    }
   ],
   "source": [
    "# Display history of training, validation accuracy and loss for each epoch\n",
    "# Best epoch is chosen so that it minimize the validation loss\n",
    "score = model.evaluate(X_train, y_train, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])\n",
    "\n",
    "fig, ax = plt.subplots(1, 1, figsize=(10,6))\n",
    "ax.plot(np.sqrt(model_history.history['accuracy']), 'r', label='train_acc')\n",
    "ax.plot(np.sqrt(model_history.history['val_accuracy']), 'b' ,label='validation')\n",
    "ax.set_xlabel(r'Epoch', fontsize=20)\n",
    "ax.set_ylabel(r'Accuracy', fontsize=20)\n",
    "ax.legend()\n",
    "ax.tick_params(labelsize=20)\n",
    "\n",
    "fig, ax = plt.subplots(1, 1, figsize=(10,6))\n",
    "ax.plot(np.sqrt(model_history.history['loss']), 'r', label='train')\n",
    "ax.plot(np.sqrt(model_history.history['val_loss']), 'b' ,label='validation')\n",
    "ax.set_xlabel(r'Epoch', fontsize=20)\n",
    "ax.set_ylabel(r'Loss', fontsize=20)\n",
    "ax.legend()\n",
    "ax.tick_params(labelsize=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "46F0XSBz747N",
    "outputId": "eb0f7992-5c27-4a0d-f49d-86570c112ca8"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'optimize_epoch' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                  Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_53420/777887175.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# retrain model based on the best epoch\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mmodel_history\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moptimize_epoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbest_batch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstop_early\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.25\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'optimize_epoch' is not defined"
     ]
    }
   ],
   "source": [
    "# retrain model based on the best epoch \n",
    "model_history = model.fit(X_train, y_train, epochs=optimize_epoch, batch_size=best_batch, callbacks=[stop_early], validation_split=0.25)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zkgclSMJ8RlU",
    "outputId": "7cae9e28-2f73-4ec2-e70e-83b1012e1369"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27/27 [==============================] - 0s 1ms/step - loss: 0.3567 - accuracy: 0.8258\n",
      "Accuracy: 82.58\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the keras model for training dataset\n",
    "_, accuracy = model.evaluate(X_train, y_train)\n",
    "accuracy = round((accuracy*100),2)\n",
    "print('Accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "n0ofG0f78WJs",
    "outputId": "2f879133-47b3-4d10-8e05-072f4e13e559"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7/7 [==============================] - 0s 2ms/step - loss: 0.2976 - accuracy: 0.8768\n",
      "Accuracy: 87.68\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the keras model for test dataset\n",
    "_, accuracy = model.evaluate(X_test, y_test)\n",
    "accuracy = round((accuracy*100),2)\n",
    "print('Accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dLtaQoSAr5Kb"
   },
   "outputs": [],
   "source": [
    "# Dictionary to translate binary into original categorical class\n",
    "convert_to_binary  = {'1': 'RB', '0': 'NRB'}\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SgOVXoBG8Ydt",
    "outputId": "682e8d64-d069-4b49-eee9-4c703a7f1e64"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value prediction:\n",
      "------------------\n",
      "Record [1] => NRB (expected  RB )\n",
      "Record [2] => NRB (expected  NRB )\n",
      "Record [3] => NRB (expected  NRB )\n",
      "Record [4] => NRB (expected  NRB )\n",
      "Record [5] => NRB (expected  NRB )\n",
      "Record [6] => NRB (expected  NRB )\n",
      "Record [7] => NRB (expected  NRB )\n",
      "Record [8] => NRB (expected  NRB )\n",
      "Record [9] => NRB (expected  NRB )\n",
      "Record [10] => RB (expected  RB )\n",
      "Record [11] => RB (expected  RB )\n",
      "Record [12] => NRB (expected  NRB )\n",
      "Record [13] => RB (expected  NRB )\n",
      "Record [14] => RB (expected  RB )\n",
      "Record [15] => RB (expected  RB )\n",
      "Record [16] => NRB (expected  NRB )\n",
      "Record [17] => RB (expected  RB )\n",
      "Record [18] => NRB (expected  NRB )\n",
      "Record [19] => NRB (expected  RB )\n",
      "Record [20] => NRB (expected  NRB )\n",
      "Record [21] => RB (expected  NRB )\n",
      "Record [22] => RB (expected  NRB )\n",
      "Record [23] => NRB (expected  NRB )\n",
      "Record [24] => NRB (expected  NRB )\n",
      "Record [25] => RB (expected  NRB )\n",
      "Record [26] => RB (expected  RB )\n",
      "Record [27] => RB (expected  RB )\n",
      "Record [28] => RB (expected  RB )\n",
      "Record [29] => NRB (expected  NRB )\n",
      "Record [30] => NRB (expected  NRB )\n",
      "Record [31] => NRB (expected  NRB )\n",
      "Record [32] => NRB (expected  NRB )\n",
      "Record [33] => NRB (expected  NRB )\n",
      "Record [34] => NRB (expected  NRB )\n",
      "Record [35] => RB (expected  RB )\n",
      "Record [36] => NRB (expected  NRB )\n",
      "Record [37] => NRB (expected  NRB )\n",
      "Record [38] => NRB (expected  NRB )\n",
      "Record [39] => NRB (expected  RB )\n",
      "Record [40] => NRB (expected  NRB )\n",
      "Record [41] => RB (expected  NRB )\n",
      "Record [42] => NRB (expected  NRB )\n",
      "Record [43] => NRB (expected  NRB )\n",
      "Record [44] => NRB (expected  NRB )\n",
      "Record [45] => RB (expected  RB )\n",
      "Record [46] => RB (expected  NRB )\n",
      "Record [47] => RB (expected  NRB )\n",
      "Record [48] => RB (expected  RB )\n",
      "Record [49] => NRB (expected  NRB )\n",
      "Record [50] => RB (expected  RB )\n",
      "Record [51] => NRB (expected  NRB )\n",
      "Record [52] => NRB (expected  RB )\n",
      "Record [53] => RB (expected  RB )\n",
      "Record [54] => RB (expected  RB )\n",
      "Record [55] => NRB (expected  NRB )\n",
      "Record [56] => NRB (expected  NRB )\n",
      "Record [57] => RB (expected  NRB )\n",
      "Record [58] => NRB (expected  NRB )\n",
      "Record [59] => RB (expected  NRB )\n",
      "Record [60] => RB (expected  RB )\n",
      "Record [61] => NRB (expected  NRB )\n",
      "Record [62] => RB (expected  RB )\n",
      "Record [63] => NRB (expected  NRB )\n",
      "Record [64] => NRB (expected  NRB )\n",
      "Record [65] => NRB (expected  NRB )\n",
      "Record [66] => NRB (expected  NRB )\n",
      "Record [67] => NRB (expected  NRB )\n",
      "Record [68] => NRB (expected  NRB )\n",
      "Record [69] => NRB (expected  NRB )\n",
      "Record [70] => NRB (expected  NRB )\n",
      "Record [71] => NRB (expected  NRB )\n",
      "Record [72] => NRB (expected  NRB )\n",
      "Record [73] => NRB (expected  NRB )\n",
      "Record [74] => NRB (expected  NRB )\n",
      "Record [75] => RB (expected  RB )\n",
      "Record [76] => NRB (expected  NRB )\n",
      "Record [77] => NRB (expected  NRB )\n",
      "Record [78] => RB (expected  NRB )\n",
      "Record [79] => RB (expected  NRB )\n",
      "Record [80] => RB (expected  NRB )\n",
      "Record [81] => NRB (expected  NRB )\n",
      "Record [82] => RB (expected  RB )\n",
      "Record [83] => NRB (expected  NRB )\n",
      "Record [84] => NRB (expected  NRB )\n",
      "Record [85] => NRB (expected  NRB )\n",
      "Record [86] => RB (expected  RB )\n",
      "Record [87] => NRB (expected  NRB )\n",
      "Record [88] => NRB (expected  NRB )\n",
      "Record [89] => RB (expected  RB )\n",
      "Record [90] => NRB (expected  NRB )\n",
      "Record [91] => NRB (expected  NRB )\n",
      "Record [92] => RB (expected  RB )\n",
      "Record [93] => NRB (expected  NRB )\n",
      "Record [94] => NRB (expected  NRB )\n",
      "Record [95] => NRB (expected  NRB )\n",
      "Record [96] => NRB (expected  NRB )\n",
      "Record [97] => NRB (expected  NRB )\n",
      "Record [98] => RB (expected  RB )\n",
      "Record [99] => RB (expected  RB )\n",
      "Record [100] => RB (expected  RB )\n",
      "Record [101] => RB (expected  NRB )\n",
      "Record [102] => NRB (expected  NRB )\n",
      "Record [103] => NRB (expected  NRB )\n",
      "Record [104] => NRB (expected  NRB )\n",
      "Record [105] => RB (expected  RB )\n",
      "Record [106] => NRB (expected  NRB )\n",
      "Record [107] => NRB (expected  NRB )\n",
      "Record [108] => RB (expected  RB )\n",
      "Record [109] => RB (expected  RB )\n",
      "Record [110] => NRB (expected  NRB )\n",
      "Record [111] => NRB (expected  NRB )\n",
      "Record [112] => RB (expected  RB )\n",
      "Record [113] => NRB (expected  NRB )\n",
      "Record [114] => NRB (expected  NRB )\n",
      "Record [115] => NRB (expected  RB )\n",
      "Record [116] => NRB (expected  NRB )\n",
      "Record [117] => NRB (expected  NRB )\n",
      "Record [118] => NRB (expected  NRB )\n",
      "Record [119] => NRB (expected  NRB )\n",
      "Record [120] => NRB (expected  NRB )\n",
      "Record [121] => NRB (expected  NRB )\n",
      "Record [122] => RB (expected  RB )\n",
      "Record [123] => NRB (expected  NRB )\n",
      "Record [124] => NRB (expected  NRB )\n",
      "Record [125] => NRB (expected  NRB )\n",
      "Record [126] => NRB (expected  NRB )\n",
      "Record [127] => RB (expected  NRB )\n",
      "Record [128] => NRB (expected  NRB )\n",
      "Record [129] => RB (expected  RB )\n",
      "Record [130] => NRB (expected  NRB )\n",
      "Record [131] => RB (expected  RB )\n",
      "Record [132] => NRB (expected  NRB )\n",
      "Record [133] => RB (expected  RB )\n",
      "Record [134] => NRB (expected  NRB )\n",
      "Record [135] => RB (expected  RB )\n",
      "Record [136] => NRB (expected  RB )\n",
      "Record [137] => NRB (expected  NRB )\n",
      "Record [138] => NRB (expected  NRB )\n",
      "Record [139] => RB (expected  RB )\n",
      "Record [140] => RB (expected  RB )\n",
      "Record [141] => NRB (expected  NRB )\n",
      "Record [142] => NRB (expected  NRB )\n",
      "Record [143] => RB (expected  RB )\n",
      "Record [144] => RB (expected  RB )\n",
      "Record [145] => NRB (expected  NRB )\n",
      "Record [146] => NRB (expected  NRB )\n",
      "Record [147] => NRB (expected  NRB )\n",
      "Record [148] => NRB (expected  NRB )\n",
      "Record [149] => RB (expected  RB )\n",
      "Record [150] => NRB (expected  NRB )\n",
      "Record [151] => NRB (expected  NRB )\n",
      "Record [152] => NRB (expected  NRB )\n",
      "Record [153] => NRB (expected  NRB )\n",
      "Record [154] => NRB (expected  NRB )\n",
      "Record [155] => NRB (expected  NRB )\n",
      "Record [156] => RB (expected  RB )\n",
      "Record [157] => RB (expected  RB )\n",
      "Record [158] => NRB (expected  NRB )\n",
      "Record [159] => NRB (expected  NRB )\n",
      "Record [160] => RB (expected  NRB )\n",
      "Record [161] => NRB (expected  RB )\n",
      "Record [162] => RB (expected  RB )\n",
      "Record [163] => NRB (expected  NRB )\n",
      "Record [164] => NRB (expected  NRB )\n",
      "Record [165] => NRB (expected  NRB )\n",
      "Record [166] => RB (expected  RB )\n",
      "Record [167] => NRB (expected  NRB )\n",
      "Record [168] => RB (expected  RB )\n",
      "Record [169] => RB (expected  RB )\n",
      "Record [170] => RB (expected  RB )\n",
      "Record [171] => NRB (expected  NRB )\n",
      "Record [172] => NRB (expected  NRB )\n",
      "Record [173] => NRB (expected  NRB )\n",
      "Record [174] => RB (expected  RB )\n",
      "Record [175] => NRB (expected  NRB )\n",
      "Record [176] => NRB (expected  NRB )\n",
      "Record [177] => RB (expected  RB )\n",
      "Record [178] => NRB (expected  RB )\n",
      "Record [179] => NRB (expected  NRB )\n",
      "Record [180] => RB (expected  RB )\n",
      "Record [181] => NRB (expected  NRB )\n",
      "Record [182] => NRB (expected  NRB )\n",
      "Record [183] => NRB (expected  NRB )\n",
      "Record [184] => NRB (expected  NRB )\n",
      "Record [185] => NRB (expected  NRB )\n",
      "Record [186] => NRB (expected  NRB )\n",
      "Record [187] => NRB (expected  NRB )\n",
      "Record [188] => NRB (expected  RB )\n",
      "Record [189] => NRB (expected  NRB )\n",
      "Record [190] => NRB (expected  NRB )\n",
      "Record [191] => NRB (expected  NRB )\n",
      "Record [192] => RB (expected  RB )\n",
      "Record [193] => NRB (expected  RB )\n",
      "Record [194] => NRB (expected  NRB )\n",
      "Record [195] => NRB (expected  NRB )\n",
      "Record [196] => NRB (expected  NRB )\n",
      "Record [197] => RB (expected  RB )\n",
      "Record [198] => NRB (expected  NRB )\n",
      "Record [199] => NRB (expected  NRB )\n",
      "Record [200] => NRB (expected  RB )\n",
      "Record [201] => NRB (expected  NRB )\n",
      "Record [202] => NRB (expected  NRB )\n",
      "Record [203] => NRB (expected  NRB )\n",
      "Record [204] => NRB (expected  NRB )\n",
      "Record [205] => RB (expected  RB )\n",
      "Record [206] => NRB (expected  NRB )\n",
      "Record [207] => RB (expected  RB )\n",
      "Record [208] => RB (expected  RB )\n",
      "Record [209] => NRB (expected  NRB )\n",
      "Record [210] => NRB (expected  NRB )\n",
      "Record [211] => NRB (expected  NRB )\n"
     ]
    }
   ],
   "source": [
    "# make class predictions with the model\n",
    "# display predictions in original categorical class\n",
    "y_pred = (model.predict(X_test) > 0.5).astype(int)\n",
    "\n",
    "print (\"Value prediction:\")\n",
    "print(\"------------------\")\n",
    "for i in range(X_test.shape[0]):\n",
    "\tprint('Record',[i+1],'=>', convert_to_binary[str(y_pred[i][0])], '(expected ', convert_to_binary[str(y_test[i])],')')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "la-03pRP8cMk",
    "outputId": "87346e3c-1d93-43df-c90c-6b35cea0bb9e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: \n",
      "---------\n",
      "0.8767772511848341\n",
      "\n",
      "\n",
      "Confusion Matric:\n",
      "-------------------\n",
      "[[132  15]\n",
      " [ 11  53]]\n",
      "\n",
      "\n",
      "Classification Report: \n",
      "----------------------\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.90      0.91       147\n",
      "           1       0.78      0.83      0.80        64\n",
      "\n",
      "    accuracy                           0.88       211\n",
      "   macro avg       0.85      0.86      0.86       211\n",
      "weighted avg       0.88      0.88      0.88       211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print(\"Analysis Neural Network:\")\n",
    "# print(\"------------------------------------------------\")\n",
    "\n",
    "#Neural Network Analysis\n",
    "print(\"Accuracy: \") \n",
    "print(\"---------\")\n",
    "print(accuracy_score(y_test, y_pred))\n",
    "\n",
    "print(\"\\n\\nConfusion Matric:\")\n",
    "print(\"-------------------\")\n",
    "print(confusion_matrix(y_test, y_pred))\n",
    "\n",
    "print(\"\\n\\nClassification Report: \")\n",
    "print(\"----------------------\")\n",
    "print(classification_report(y_test, y_pred))\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Neural Network and Logistic Regression complete.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}