Random Forest(Company_Data).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3cfe96fd",
   "metadata": {},
   "source": [
    "# Random Forest(Company_Data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e61db1cf",
   "metadata": {},
   "source": [
    "A cloth manufacturing company is interested to know about the segment or attributes causes high sale. \n",
    "Approach - A Random Forest can be built with target variable Sales (we will first convert it in categorical variable) & all other variable will be independent in the analysis.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "810b1a33",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:35.822710Z",
     "start_time": "2021-08-29T21:40:35.809719Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "#inline visualization\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b4eddb4d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:36.558228Z",
     "start_time": "2021-08-29T21:40:35.826708Z"
    }
   },
   "outputs": [],
   "source": [
    "import category_encoders as ce\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "34d67f2f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:36.711522Z",
     "start_time": "2021-08-29T21:40:36.562186Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sales</th>\n",
       "      <th>CompPrice</th>\n",
       "      <th>Income</th>\n",
       "      <th>Advertising</th>\n",
       "      <th>Population</th>\n",
       "      <th>Price</th>\n",
       "      <th>ShelveLoc</th>\n",
       "      <th>Age</th>\n",
       "      <th>Education</th>\n",
       "      <th>Urban</th>\n",
       "      <th>US</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.50</td>\n",
       "      <td>138</td>\n",
       "      <td>73</td>\n",
       "      <td>11</td>\n",
       "      <td>276</td>\n",
       "      <td>120</td>\n",
       "      <td>Bad</td>\n",
       "      <td>42</td>\n",
       "      <td>17</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11.22</td>\n",
       "      <td>111</td>\n",
       "      <td>48</td>\n",
       "      <td>16</td>\n",
       "      <td>260</td>\n",
       "      <td>83</td>\n",
       "      <td>Good</td>\n",
       "      <td>65</td>\n",
       "      <td>10</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.06</td>\n",
       "      <td>113</td>\n",
       "      <td>35</td>\n",
       "      <td>10</td>\n",
       "      <td>269</td>\n",
       "      <td>80</td>\n",
       "      <td>Medium</td>\n",
       "      <td>59</td>\n",
       "      <td>12</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.40</td>\n",
       "      <td>117</td>\n",
       "      <td>100</td>\n",
       "      <td>4</td>\n",
       "      <td>466</td>\n",
       "      <td>97</td>\n",
       "      <td>Medium</td>\n",
       "      <td>55</td>\n",
       "      <td>14</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.15</td>\n",
       "      <td>141</td>\n",
       "      <td>64</td>\n",
       "      <td>3</td>\n",
       "      <td>340</td>\n",
       "      <td>128</td>\n",
       "      <td>Bad</td>\n",
       "      <td>38</td>\n",
       "      <td>13</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sales  CompPrice  Income  Advertising  Population  Price ShelveLoc  Age  \\\n",
       "0   9.50        138      73           11         276    120       Bad   42   \n",
       "1  11.22        111      48           16         260     83      Good   65   \n",
       "2  10.06        113      35           10         269     80    Medium   59   \n",
       "3   7.40        117     100            4         466     97    Medium   55   \n",
       "4   4.15        141      64            3         340    128       Bad   38   \n",
       "\n",
       "   Education Urban   US  \n",
       "0         17   Yes  Yes  \n",
       "1         10   Yes  Yes  \n",
       "2         12   Yes  Yes  \n",
       "3         14   Yes  Yes  \n",
       "4         13   Yes   No  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('Company_Data.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "25376976",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:36.804968Z",
     "start_time": "2021-08-29T21:40:36.715521Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 400 entries, 0 to 399\n",
      "Data columns (total 11 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   Sales        400 non-null    float64\n",
      " 1   CompPrice    400 non-null    int64  \n",
      " 2   Income       400 non-null    int64  \n",
      " 3   Advertising  400 non-null    int64  \n",
      " 4   Population   400 non-null    int64  \n",
      " 5   Price        400 non-null    int64  \n",
      " 6   ShelveLoc    400 non-null    object \n",
      " 7   Age          400 non-null    int64  \n",
      " 8   Education    400 non-null    int64  \n",
      " 9   Urban        400 non-null    object \n",
      " 10  US           400 non-null    object \n",
      "dtypes: float64(1), int64(7), object(3)\n",
      "memory usage: 34.5+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e021b309",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:36.928005Z",
     "start_time": "2021-08-29T21:40:36.807968Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sales</th>\n",
       "      <th>CompPrice</th>\n",
       "      <th>Income</th>\n",
       "      <th>Advertising</th>\n",
       "      <th>Population</th>\n",
       "      <th>Price</th>\n",
       "      <th>Age</th>\n",
       "      <th>Education</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>400.000000</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>400.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.496325</td>\n",
       "      <td>124.975000</td>\n",
       "      <td>68.657500</td>\n",
       "      <td>6.635000</td>\n",
       "      <td>264.840000</td>\n",
       "      <td>115.795000</td>\n",
       "      <td>53.322500</td>\n",
       "      <td>13.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.824115</td>\n",
       "      <td>15.334512</td>\n",
       "      <td>27.986037</td>\n",
       "      <td>6.650364</td>\n",
       "      <td>147.376436</td>\n",
       "      <td>23.676664</td>\n",
       "      <td>16.200297</td>\n",
       "      <td>2.620528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>77.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.390000</td>\n",
       "      <td>115.000000</td>\n",
       "      <td>42.750000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>139.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>39.750000</td>\n",
       "      <td>12.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.490000</td>\n",
       "      <td>125.000000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>272.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>54.500000</td>\n",
       "      <td>14.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>9.320000</td>\n",
       "      <td>135.000000</td>\n",
       "      <td>91.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>398.500000</td>\n",
       "      <td>131.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>16.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>16.270000</td>\n",
       "      <td>175.000000</td>\n",
       "      <td>120.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>509.000000</td>\n",
       "      <td>191.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>18.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Sales   CompPrice      Income  Advertising  Population  \\\n",
       "count  400.000000  400.000000  400.000000   400.000000  400.000000   \n",
       "mean     7.496325  124.975000   68.657500     6.635000  264.840000   \n",
       "std      2.824115   15.334512   27.986037     6.650364  147.376436   \n",
       "min      0.000000   77.000000   21.000000     0.000000   10.000000   \n",
       "25%      5.390000  115.000000   42.750000     0.000000  139.000000   \n",
       "50%      7.490000  125.000000   69.000000     5.000000  272.000000   \n",
       "75%      9.320000  135.000000   91.000000    12.000000  398.500000   \n",
       "max     16.270000  175.000000  120.000000    29.000000  509.000000   \n",
       "\n",
       "            Price         Age   Education  \n",
       "count  400.000000  400.000000  400.000000  \n",
       "mean   115.795000   53.322500   13.900000  \n",
       "std     23.676664   16.200297    2.620528  \n",
       "min     24.000000   25.000000   10.000000  \n",
       "25%    100.000000   39.750000   12.000000  \n",
       "50%    117.000000   54.500000   14.000000  \n",
       "75%    131.000000   66.000000   16.000000  \n",
       "max    191.000000   80.000000   18.000000  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d87a7318",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:37.004963Z",
     "start_time": "2021-08-29T21:40:36.930984Z"
    }
   },
   "outputs": [],
   "source": [
    "## Performing EDA\n",
    "\n",
    "encoder = ce.OrdinalEncoder(cols=[\"ShelveLoc\", \"Urban\", \"US\"])\n",
    "sales = encoder.fit_transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "371432ec",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:37.083700Z",
     "start_time": "2021-08-29T21:40:37.008940Z"
    }
   },
   "outputs": [],
   "source": [
    "sale_val = []\n",
    "for value in df['Sales']:\n",
    "    if value <= 7.49:\n",
    "        sale_val.append(\"low\")\n",
    "    else:\n",
    "        sale_val.append(\"high\")\n",
    "        \n",
    "sales[\"sale_val\"]= sale_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "340b2a04",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:37.222677Z",
     "start_time": "2021-08-29T21:40:37.091703Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sales</th>\n",
       "      <th>CompPrice</th>\n",
       "      <th>Income</th>\n",
       "      <th>Advertising</th>\n",
       "      <th>Population</th>\n",
       "      <th>Price</th>\n",
       "      <th>ShelveLoc</th>\n",
       "      <th>Age</th>\n",
       "      <th>Education</th>\n",
       "      <th>Urban</th>\n",
       "      <th>US</th>\n",
       "      <th>sale_val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.50</td>\n",
       "      <td>138</td>\n",
       "      <td>73</td>\n",
       "      <td>11</td>\n",
       "      <td>276</td>\n",
       "      <td>120</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11.22</td>\n",
       "      <td>111</td>\n",
       "      <td>48</td>\n",
       "      <td>16</td>\n",
       "      <td>260</td>\n",
       "      <td>83</td>\n",
       "      <td>2</td>\n",
       "      <td>65</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.06</td>\n",
       "      <td>113</td>\n",
       "      <td>35</td>\n",
       "      <td>10</td>\n",
       "      <td>269</td>\n",
       "      <td>80</td>\n",
       "      <td>3</td>\n",
       "      <td>59</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.40</td>\n",
       "      <td>117</td>\n",
       "      <td>100</td>\n",
       "      <td>4</td>\n",
       "      <td>466</td>\n",
       "      <td>97</td>\n",
       "      <td>3</td>\n",
       "      <td>55</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.15</td>\n",
       "      <td>141</td>\n",
       "      <td>64</td>\n",
       "      <td>3</td>\n",
       "      <td>340</td>\n",
       "      <td>128</td>\n",
       "      <td>1</td>\n",
       "      <td>38</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sales  CompPrice  Income  Advertising  Population  Price  ShelveLoc  Age  \\\n",
       "0   9.50        138      73           11         276    120          1   42   \n",
       "1  11.22        111      48           16         260     83          2   65   \n",
       "2  10.06        113      35           10         269     80          3   59   \n",
       "3   7.40        117     100            4         466     97          3   55   \n",
       "4   4.15        141      64            3         340    128          1   38   \n",
       "\n",
       "   Education  Urban  US sale_val  \n",
       "0         17      1   1     high  \n",
       "1         10      1   1     high  \n",
       "2         12      1   1     high  \n",
       "3         14      1   1      low  \n",
       "4         13      1   2      low  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sales.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0da71a8f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:37.515142Z",
     "start_time": "2021-08-29T21:40:37.227656Z"
    }
   },
   "outputs": [],
   "source": [
    "## Train and Split the data \n",
    "x = sales.drop(['sale_val', 'Sales'],axis=1)\n",
    "y = sales['sale_val']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f9004e53",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:38.016917Z",
     "start_time": "2021-08-29T21:40:37.524142Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CompPrice</th>\n",
       "      <th>Income</th>\n",
       "      <th>Advertising</th>\n",
       "      <th>Population</th>\n",
       "      <th>Price</th>\n",
       "      <th>ShelveLoc</th>\n",
       "      <th>Age</th>\n",
       "      <th>Education</th>\n",
       "      <th>Urban</th>\n",
       "      <th>US</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>138</td>\n",
       "      <td>73</td>\n",
       "      <td>11</td>\n",
       "      <td>276</td>\n",
       "      <td>120</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>111</td>\n",
       "      <td>48</td>\n",
       "      <td>16</td>\n",
       "      <td>260</td>\n",
       "      <td>83</td>\n",
       "      <td>2</td>\n",
       "      <td>65</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>113</td>\n",
       "      <td>35</td>\n",
       "      <td>10</td>\n",
       "      <td>269</td>\n",
       "      <td>80</td>\n",
       "      <td>3</td>\n",
       "      <td>59</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>117</td>\n",
       "      <td>100</td>\n",
       "      <td>4</td>\n",
       "      <td>466</td>\n",
       "      <td>97</td>\n",
       "      <td>3</td>\n",
       "      <td>55</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>141</td>\n",
       "      <td>64</td>\n",
       "      <td>3</td>\n",
       "      <td>340</td>\n",
       "      <td>128</td>\n",
       "      <td>1</td>\n",
       "      <td>38</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>138</td>\n",
       "      <td>108</td>\n",
       "      <td>17</td>\n",
       "      <td>203</td>\n",
       "      <td>128</td>\n",
       "      <td>2</td>\n",
       "      <td>33</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>139</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>37</td>\n",
       "      <td>120</td>\n",
       "      <td>3</td>\n",
       "      <td>55</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>162</td>\n",
       "      <td>26</td>\n",
       "      <td>12</td>\n",
       "      <td>368</td>\n",
       "      <td>159</td>\n",
       "      <td>3</td>\n",
       "      <td>40</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>100</td>\n",
       "      <td>79</td>\n",
       "      <td>7</td>\n",
       "      <td>284</td>\n",
       "      <td>95</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>134</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>120</td>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     CompPrice  Income  Advertising  Population  Price  ShelveLoc  Age  \\\n",
       "0          138      73           11         276    120          1   42   \n",
       "1          111      48           16         260     83          2   65   \n",
       "2          113      35           10         269     80          3   59   \n",
       "3          117     100            4         466     97          3   55   \n",
       "4          141      64            3         340    128          1   38   \n",
       "..         ...     ...          ...         ...    ...        ...  ...   \n",
       "395        138     108           17         203    128          2   33   \n",
       "396        139      23            3          37    120          3   55   \n",
       "397        162      26           12         368    159          3   40   \n",
       "398        100      79            7         284     95          1   50   \n",
       "399        134      37            0          27    120          2   49   \n",
       "\n",
       "     Education  Urban  US  \n",
       "0           17      1   1  \n",
       "1           10      1   1  \n",
       "2           12      1   1  \n",
       "3           14      1   1  \n",
       "4           13      1   2  \n",
       "..         ...    ...  ..  \n",
       "395         14      1   1  \n",
       "396         11      2   1  \n",
       "397         18      1   1  \n",
       "398         12      1   1  \n",
       "399         16      1   1  \n",
       "\n",
       "[400 rows x 10 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "cc37aeff",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:38.244789Z",
     "start_time": "2021-08-29T21:40:38.022914Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      high\n",
       "1      high\n",
       "2      high\n",
       "3       low\n",
       "4       low\n",
       "       ... \n",
       "395    high\n",
       "396     low\n",
       "397     low\n",
       "398     low\n",
       "399    high\n",
       "Name: sale_val, Length: 400, dtype: object"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "eff044e6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:47.898162Z",
     "start_time": "2021-08-29T21:40:38.248785Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8124406457739792\n"
     ]
    }
   ],
   "source": [
    "## Random forest Classification\n",
    "num_trees = 200\n",
    "max_features = 4\n",
    "kfold = KFold(n_splits=15,  shuffle=True)\n",
    "model = RandomForestClassifier(n_estimators=num_trees, max_features=max_features)\n",
    "results_rfc = cross_val_score(model, x, y, cv=kfold)\n",
    "print(results_rfc.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "f2da7628",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:47.914127Z",
     "start_time": "2021-08-29T21:40:47.902169Z"
    }
   },
   "outputs": [],
   "source": [
    "## Various Ensemble Techniques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d35ec217",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:52.662083Z",
     "start_time": "2021-08-29T21:40:47.916071Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7899999999999998\n"
     ]
    }
   ],
   "source": [
    "## Bagging \n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "seed = 7\n",
    "kfold = KFold(n_splits=10, random_state=seed, shuffle=True)\n",
    "cart = DecisionTreeClassifier()\n",
    "num_trees = 100\n",
    "model = BaggingClassifier(base_estimator=cart, n_estimators=num_trees, random_state=seed)\n",
    "results_bag = cross_val_score(model, x, y, cv=kfold)\n",
    "print(results_bag.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4dd97f5e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:53.118648Z",
     "start_time": "2021-08-29T21:40:52.664831Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7699999999999999\n"
     ]
    }
   ],
   "source": [
    "## Boosting \n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "num_trees = 10\n",
    "seed=7\n",
    "kfold = KFold(n_splits=10, random_state=seed, shuffle=True)\n",
    "model = AdaBoostClassifier(n_estimators=num_trees, random_state=seed)\n",
    "results_boost = cross_val_score(model, x, y, cv=kfold)\n",
    "print(results_boost.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a31ae34f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:53.134646Z",
     "start_time": "2021-08-29T21:40:53.123641Z"
    }
   },
   "outputs": [],
   "source": [
    "## Stacking \n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import VotingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5e269e1a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-29T21:40:56.297740Z",
     "start_time": "2021-08-29T21:40:53.139629Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7625\n"
     ]
    }
   ],
   "source": [
    "# Creating the sub models\n",
    "estimators = []\n",
    "model1 = LogisticRegression(max_iter=500)\n",
    "estimators.append(('logistic', model1))\n",
    "model2 = DecisionTreeClassifier()\n",
    "estimators.append(('cart', model2))\n",
    "model3 = SVC()\n",
    "estimators.append(('svm', model3))\n",
    "\n",
    "# Creating the ensemble model\n",
    "ensemble = VotingClassifier(estimators)\n",
    "results_stack = cross_val_score(ensemble, x, y, cv=kfold)\n",
    "print(results_stack.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cafdaf0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}