no-output-engineering-graduate-salary-prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:44.102998Z",
     "iopub.status.busy": "2021-07-15T08:39:44.102283Z",
     "iopub.status.idle": "2021-07-15T08:39:45.595600Z",
     "shell.execute_reply": "2021-07-15T08:39:45.595082Z",
     "shell.execute_reply.started": "2021-07-15T08:18:17.779Z"
    },
    "papermill": {
     "duration": 1.540293,
     "end_time": "2021-07-15T08:39:45.595747",
     "exception": false,
     "start_time": "2021-07-15T08:39:44.055454",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pandas import DataFrame, read_csv, get_dummies\n",
    "from scipy.stats import zscore\n",
    "from statsmodels.stats.outliers_influence import variance_inflation_factor\n",
    "from matplotlib.pyplot import figure, subplot2grid\n",
    "from seaborn import set_theme,scatterplot,displot,barplot,countplot,heatmap\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import r2_score,mean_squared_error\n",
    "from numpy import where,abs,median,nan,sqrt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:45.679832Z",
     "iopub.status.busy": "2021-07-15T08:39:45.677495Z",
     "iopub.status.idle": "2021-07-15T08:39:45.681931Z",
     "shell.execute_reply": "2021-07-15T08:39:45.682417Z",
     "shell.execute_reply.started": "2021-07-15T08:18:18.259519Z"
    },
    "papermill": {
     "duration": 0.048346,
     "end_time": "2021-07-15T08:39:45.682584",
     "exception": false,
     "start_time": "2021-07-15T08:39:45.634238",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "set_theme(context=\"notebook\",style='darkgrid', palette='inferno')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:45.762820Z",
     "iopub.status.busy": "2021-07-15T08:39:45.762155Z",
     "iopub.status.idle": "2021-07-15T08:39:45.803732Z",
     "shell.execute_reply": "2021-07-15T08:39:45.803106Z",
     "shell.execute_reply.started": "2021-07-15T08:18:18.265506Z"
    },
    "papermill": {
     "duration": 0.082942,
     "end_time": "2021-07-15T08:39:45.803889",
     "exception": false,
     "start_time": "2021-07-15T08:39:45.720947",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = read_csv(\"Engineering_graduate_salary.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:45.884009Z",
     "iopub.status.busy": "2021-07-15T08:39:45.883376Z",
     "iopub.status.idle": "2021-07-15T08:39:45.911443Z",
     "shell.execute_reply": "2021-07-15T08:39:45.912452Z",
     "shell.execute_reply.started": "2021-07-15T08:18:18.301477Z"
    },
    "papermill": {
     "duration": 0.070404,
     "end_time": "2021-07-15T08:39:45.912699",
     "exception": false,
     "start_time": "2021-07-15T08:39:45.842295",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:45.994788Z",
     "iopub.status.busy": "2021-07-15T08:39:45.994143Z",
     "iopub.status.idle": "2021-07-15T08:39:46.030707Z",
     "shell.execute_reply": "2021-07-15T08:39:46.030072Z",
     "shell.execute_reply.started": "2021-07-15T08:18:18.319973Z"
    },
    "papermill": {
     "duration": 0.078259,
     "end_time": "2021-07-15T08:39:46.030850",
     "exception": false,
     "start_time": "2021-07-15T08:39:45.952591",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.039576,
     "end_time": "2021-07-15T08:39:46.109805",
     "exception": false,
     "start_time": "2021-07-15T08:39:46.070229",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "-1 in the data represents that student has not attempted that section\n",
    "\n",
    "For sake of convenience I will replace -1 with 0\n",
    "\n",
    "So if a student has not attempted the section it will add nothing to the Salary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:46.195445Z",
     "iopub.status.busy": "2021-07-15T08:39:46.194803Z",
     "iopub.status.idle": "2021-07-15T08:39:46.199192Z",
     "shell.execute_reply": "2021-07-15T08:39:46.199622Z",
     "shell.execute_reply.started": "2021-07-15T08:18:18.359916Z"
    },
    "papermill": {
     "duration": 0.049889,
     "end_time": "2021-07-15T08:39:46.199817",
     "exception": false,
     "start_time": "2021-07-15T08:39:46.149928",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.replace(to_replace=-1, value=0,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.03935,
     "end_time": "2021-07-15T08:39:46.279073",
     "exception": false,
     "start_time": "2021-07-15T08:39:46.239723",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Now let's plot correlation matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:46.365895Z",
     "iopub.status.busy": "2021-07-15T08:39:46.365290Z",
     "iopub.status.idle": "2021-07-15T08:39:50.610441Z",
     "shell.execute_reply": "2021-07-15T08:39:50.610922Z",
     "shell.execute_reply.started": "2021-07-15T08:18:18.368103Z"
    },
    "papermill": {
     "duration": 4.291585,
     "end_time": "2021-07-15T08:39:50.611087",
     "exception": false,
     "start_time": "2021-07-15T08:39:46.319502",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,16))\n",
    "plot = heatmap(df.corr(), annot=True)\n",
    "plot.set_title(\"Correlation Matrix\", fontsize=18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.052026,
     "end_time": "2021-07-15T08:39:50.715212",
     "exception": false,
     "start_time": "2021-07-15T08:39:50.663186",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "I will drop ID and DOB columns as they are not very relevant to the Salary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:50.822502Z",
     "iopub.status.busy": "2021-07-15T08:39:50.821820Z",
     "iopub.status.idle": "2021-07-15T08:39:50.827088Z",
     "shell.execute_reply": "2021-07-15T08:39:50.827537Z",
     "shell.execute_reply.started": "2021-07-15T08:18:22.754942Z"
    },
    "papermill": {
     "duration": 0.060558,
     "end_time": "2021-07-15T08:39:50.827708",
     "exception": false,
     "start_time": "2021-07-15T08:39:50.767150",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(columns=[\"ID\", \"DOB\"], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.052211,
     "end_time": "2021-07-15T08:39:50.932857",
     "exception": false,
     "start_time": "2021-07-15T08:39:50.880646",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "I will map Gender to integers so that it can be used in the calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:51.040385Z",
     "iopub.status.busy": "2021-07-15T08:39:51.039724Z",
     "iopub.status.idle": "2021-07-15T08:39:51.047934Z",
     "shell.execute_reply": "2021-07-15T08:39:51.047206Z",
     "shell.execute_reply.started": "2021-07-15T08:18:22.761937Z"
    },
    "papermill": {
     "duration": 0.063369,
     "end_time": "2021-07-15T08:39:51.048074",
     "exception": false,
     "start_time": "2021-07-15T08:39:50.984705",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"Gender\"] = df[\"Gender\"].replace({'m': 0, 'f': 1})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.052084,
     "end_time": "2021-07-15T08:39:51.151559",
     "exception": false,
     "start_time": "2021-07-15T08:39:51.099475",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "I will calculate [variance inflation factor](https://en.wikipedia.org/wiki/Variance_inflation_factor) which will help us identity if the there is [multicollinearity](https://en.wikipedia.org/wiki/Multicollinearity) in the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:51.258822Z",
     "iopub.status.busy": "2021-07-15T08:39:51.257858Z",
     "iopub.status.idle": "2021-07-15T08:39:51.543827Z",
     "shell.execute_reply": "2021-07-15T08:39:51.544322Z",
     "shell.execute_reply.started": "2021-07-15T08:18:22.778535Z"
    },
    "papermill": {
     "duration": 0.341105,
     "end_time": "2021-07-15T08:39:51.544588",
     "exception": false,
     "start_time": "2021-07-15T08:39:51.203483",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = df[['Gender', '10percentage', '12graduation', '12percentage', 'CollegeID', 'CollegeTier', 'collegeGPA', 'CollegeCityID', 'CollegeCityTier', 'GraduationYear', 'English', 'Logical', 'Quant', 'Domain', 'ComputerProgramming', 'ElectronicsAndSemicon', 'ComputerScience', 'MechanicalEngg', 'ElectricalEngg', 'TelecomEngg', 'CivilEngg', 'conscientiousness', 'agreeableness', 'extraversion', 'nueroticism', 'openess_to_experience', 'Salary']]\n",
    "\n",
    "vif_data = DataFrame()\n",
    "vif_data[\"Column\"] = X.columns\n",
    "\n",
    "vif_data[\"VIF\"] = [variance_inflation_factor(X.values, i) for i in range(len(X.columns))]\n",
    "\n",
    "vif_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:51.704621Z",
     "iopub.status.busy": "2021-07-15T08:39:51.703665Z",
     "iopub.status.idle": "2021-07-15T08:39:52.509577Z",
     "shell.execute_reply": "2021-07-15T08:39:52.510011Z",
     "shell.execute_reply.started": "2021-07-15T08:18:23.022499Z"
    },
    "papermill": {
     "duration": 0.865586,
     "end_time": "2021-07-15T08:39:52.510185",
     "exception": false,
     "start_time": "2021-07-15T08:39:51.644599",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = displot(df[\"Salary\"], kde=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.052725,
     "end_time": "2021-07-15T08:39:52.616204",
     "exception": false,
     "start_time": "2021-07-15T08:39:52.563479",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Our data looked very skewed, this can be because of Outliers (outliers are the points in the dataset which show high deviation from most of the data)\n",
    "\n",
    "To make it a more normalised distribution we can remove Outliers\n",
    "\n",
    "We can remove outliers using various algorithms and technique\n",
    "\n",
    "Here I am going to use Z-Score (Z-Score is the how much times the standard deviation a point is deviated from the mean value.)\n",
    "\n",
    "We will replace the values that are very largely deviated with the median value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:52.727906Z",
     "iopub.status.busy": "2021-07-15T08:39:52.725137Z",
     "iopub.status.idle": "2021-07-15T08:39:52.733468Z",
     "shell.execute_reply": "2021-07-15T08:39:52.732988Z",
     "shell.execute_reply.started": "2021-07-15T08:18:23.815872Z"
    },
    "papermill": {
     "duration": 0.064171,
     "end_time": "2021-07-15T08:39:52.733607",
     "exception": false,
     "start_time": "2021-07-15T08:39:52.669436",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"Salary\"] = df[\"Salary\"].replace([df[\"Salary\"][(abs(zscore(df[\"Salary\"])) > 3)]], median(df[\"Salary\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:52.881555Z",
     "iopub.status.busy": "2021-07-15T08:39:52.880432Z",
     "iopub.status.idle": "2021-07-15T08:39:53.277544Z",
     "shell.execute_reply": "2021-07-15T08:39:53.276998Z",
     "shell.execute_reply.started": "2021-07-15T08:18:23.825408Z"
    },
    "papermill": {
     "duration": 0.490232,
     "end_time": "2021-07-15T08:39:53.277681",
     "exception": false,
     "start_time": "2021-07-15T08:39:52.787449",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = displot(data=df[\"Salary\"], kde=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.05392,
     "end_time": "2021-07-15T08:39:53.386622",
     "exception": false,
     "start_time": "2021-07-15T08:39:53.332702",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "This looks more like a normal distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:53.530330Z",
     "iopub.status.busy": "2021-07-15T08:39:53.529407Z",
     "iopub.status.idle": "2021-07-15T08:39:53.771420Z",
     "shell.execute_reply": "2021-07-15T08:39:53.770609Z",
     "shell.execute_reply.started": "2021-07-15T08:18:24.270843Z"
    },
    "papermill": {
     "duration": 0.316692,
     "end_time": "2021-07-15T08:39:53.771593",
     "exception": false,
     "start_time": "2021-07-15T08:39:53.454901",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = scatterplot(x=df.Salary, y=df.Gender)\n",
    "plot.set_title(\"Scatter Plot of Gender with Salary\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:53.893343Z",
     "iopub.status.busy": "2021-07-15T08:39:53.892677Z",
     "iopub.status.idle": "2021-07-15T08:39:54.153585Z",
     "shell.execute_reply": "2021-07-15T08:39:54.153096Z",
     "shell.execute_reply.started": "2021-07-15T08:18:24.503143Z"
    },
    "papermill": {
     "duration": 0.322715,
     "end_time": "2021-07-15T08:39:54.153728",
     "exception": false,
     "start_time": "2021-07-15T08:39:53.831013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = scatterplot(x=df['10percentage'], y=df['12percentage'])\n",
    "plot.set_title(\"Scatter Plot of 10percentage with 12percentage\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:54.279524Z",
     "iopub.status.busy": "2021-07-15T08:39:54.278220Z",
     "iopub.status.idle": "2021-07-15T08:39:54.282833Z",
     "shell.execute_reply": "2021-07-15T08:39:54.283370Z",
     "shell.execute_reply.started": "2021-07-15T08:18:24.761277Z"
    },
    "papermill": {
     "duration": 0.070691,
     "end_time": "2021-07-15T08:39:54.283542",
     "exception": false,
     "start_time": "2021-07-15T08:39:54.212851",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"10percentage\"].corr(df[\"12percentage\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.057727,
     "end_time": "2021-07-15T08:39:54.400579",
     "exception": false,
     "start_time": "2021-07-15T08:39:54.342852",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "There is high correlation between 10 & 12 percentages\n",
    "\n",
    "Also there is high multicollinearity as seen in the VIF values\n",
    "\n",
    "So instead of using both, I will use only 12 percentage as 12 percentage is somewhat more important as compared to 10 percentage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:54.520440Z",
     "iopub.status.busy": "2021-07-15T08:39:54.519757Z",
     "iopub.status.idle": "2021-07-15T08:39:54.525773Z",
     "shell.execute_reply": "2021-07-15T08:39:54.526287Z",
     "shell.execute_reply.started": "2021-07-15T08:18:24.768635Z"
    },
    "papermill": {
     "duration": 0.068111,
     "end_time": "2021-07-15T08:39:54.526467",
     "exception": false,
     "start_time": "2021-07-15T08:39:54.458356",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(columns=[\"10percentage\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:54.646734Z",
     "iopub.status.busy": "2021-07-15T08:39:54.646093Z",
     "iopub.status.idle": "2021-07-15T08:39:54.898795Z",
     "shell.execute_reply": "2021-07-15T08:39:54.899235Z",
     "shell.execute_reply.started": "2021-07-15T08:18:24.780974Z"
    },
    "papermill": {
     "duration": 0.314189,
     "end_time": "2021-07-15T08:39:54.899508",
     "exception": false,
     "start_time": "2021-07-15T08:39:54.585319",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = scatterplot(x=df['CollegeID'], y=df['CollegeCityID'])\n",
    "plot.set_title(\"Scatter Plot of CollegeID with CollegeCityID\", fontsize=14)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.060161,
     "end_time": "2021-07-15T08:39:55.019934",
     "exception": false,
     "start_time": "2021-07-15T08:39:54.959773",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Looks like CollegeID and CollegeCityID is same for every value\n",
    "\n",
    "Also it had infinite VIF \n",
    "\n",
    "So it is neccessary to drop one of them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:55.145888Z",
     "iopub.status.busy": "2021-07-15T08:39:55.141713Z",
     "iopub.status.idle": "2021-07-15T08:39:55.151146Z",
     "shell.execute_reply": "2021-07-15T08:39:55.150694Z",
     "shell.execute_reply.started": "2021-07-15T08:18:25.073724Z"
    },
    "papermill": {
     "duration": 0.071952,
     "end_time": "2021-07-15T08:39:55.151295",
     "exception": false,
     "start_time": "2021-07-15T08:39:55.079343",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(columns=[\"CollegeID\"], inplace=True)\n",
    "df[\"CollegeCityID\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:55.280523Z",
     "iopub.status.busy": "2021-07-15T08:39:55.279883Z",
     "iopub.status.idle": "2021-07-15T08:39:55.540127Z",
     "shell.execute_reply": "2021-07-15T08:39:55.540589Z",
     "shell.execute_reply.started": "2021-07-15T08:18:25.084284Z"
    },
    "papermill": {
     "duration": 0.329414,
     "end_time": "2021-07-15T08:39:55.540771",
     "exception": false,
     "start_time": "2021-07-15T08:39:55.211357",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = scatterplot(x=df['12graduation'], y=df['GraduationYear'])\n",
    "plot.set_title(\"Scatter Plot of 12graduation with GraduationYear\", fontsize=14)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.062205,
     "end_time": "2021-07-15T08:39:55.664457",
     "exception": false,
     "start_time": "2021-07-15T08:39:55.602252",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "There is a point with 0 graduation year\n",
    "\n",
    "This can be a error in dataset\n",
    "Let's remove it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:55.827476Z",
     "iopub.status.busy": "2021-07-15T08:39:55.826823Z",
     "iopub.status.idle": "2021-07-15T08:39:55.832506Z",
     "shell.execute_reply": "2021-07-15T08:39:55.832940Z",
     "shell.execute_reply.started": "2021-07-15T08:18:25.521456Z"
    },
    "papermill": {
     "duration": 0.071378,
     "end_time": "2021-07-15T08:39:55.833133",
     "exception": false,
     "start_time": "2021-07-15T08:39:55.761755",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df[df[\"GraduationYear\"]> 1750]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:55.959912Z",
     "iopub.status.busy": "2021-07-15T08:39:55.959234Z",
     "iopub.status.idle": "2021-07-15T08:39:55.967681Z",
     "shell.execute_reply": "2021-07-15T08:39:55.967139Z",
     "shell.execute_reply.started": "2021-07-15T08:18:25.529397Z"
    },
    "papermill": {
     "duration": 0.073102,
     "end_time": "2021-07-15T08:39:55.967832",
     "exception": false,
     "start_time": "2021-07-15T08:39:55.894730",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"GraduationYear\"].corr(df[\"12graduation\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.06259,
     "end_time": "2021-07-15T08:39:56.094494",
     "exception": false,
     "start_time": "2021-07-15T08:39:56.031904",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "There is high correlation between 10 & 12 percentages\n",
    "\n",
    "Also there is high multicollinearity as seen in the VIF values\n",
    "\n",
    "So again instead of using both, I will use only GraduationYear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:56.224099Z",
     "iopub.status.busy": "2021-07-15T08:39:56.223411Z",
     "iopub.status.idle": "2021-07-15T08:39:56.226869Z",
     "shell.execute_reply": "2021-07-15T08:39:56.226263Z",
     "shell.execute_reply.started": "2021-07-15T08:18:25.546696Z"
    },
    "papermill": {
     "duration": 0.071031,
     "end_time": "2021-07-15T08:39:56.227016",
     "exception": false,
     "start_time": "2021-07-15T08:39:56.155985",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(columns=[\"12graduation\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:56.354413Z",
     "iopub.status.busy": "2021-07-15T08:39:56.353785Z",
     "iopub.status.idle": "2021-07-15T08:39:56.998143Z",
     "shell.execute_reply": "2021-07-15T08:39:56.998688Z",
     "shell.execute_reply.started": "2021-07-15T08:18:25.556799Z"
    },
    "papermill": {
     "duration": 0.709884,
     "end_time": "2021-07-15T08:39:56.998872",
     "exception": false,
     "start_time": "2021-07-15T08:39:56.288988",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,8))\n",
    "plot = scatterplot(x=df['12percentage'], y=df['collegeGPA'], hue=df.CollegeTier)\n",
    "plot.set_title(\"Scatter Plot of 12percentage with collegeGPA\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:57.143544Z",
     "iopub.status.busy": "2021-07-15T08:39:57.142856Z",
     "iopub.status.idle": "2021-07-15T08:39:57.375359Z",
     "shell.execute_reply": "2021-07-15T08:39:57.374720Z",
     "shell.execute_reply.started": "2021-07-15T08:18:26.093799Z"
    },
    "papermill": {
     "duration": 0.307486,
     "end_time": "2021-07-15T08:39:57.375500",
     "exception": false,
     "start_time": "2021-07-15T08:39:57.068014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = scatterplot(x=df[\"collegeGPA\"],y=df[\"Salary\"])\n",
    "plot.set_title(\"Scatter Plot of collegeGPA with Salary\", fontsize=14)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.071037,
     "end_time": "2021-07-15T08:39:57.517097",
     "exception": false,
     "start_time": "2021-07-15T08:39:57.446060",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "I will manually remove the outleirs as they are easily identifiable in the plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:57.666179Z",
     "iopub.status.busy": "2021-07-15T08:39:57.665553Z",
     "iopub.status.idle": "2021-07-15T08:39:57.670488Z",
     "shell.execute_reply": "2021-07-15T08:39:57.669983Z",
     "shell.execute_reply.started": "2021-07-15T08:18:26.327216Z"
    },
    "papermill": {
     "duration": 0.080631,
     "end_time": "2021-07-15T08:39:57.670630",
     "exception": false,
     "start_time": "2021-07-15T08:39:57.589999",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df[df[\"collegeGPA\"] > 40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:57.837169Z",
     "iopub.status.busy": "2021-07-15T08:39:57.836504Z",
     "iopub.status.idle": "2021-07-15T08:39:58.544484Z",
     "shell.execute_reply": "2021-07-15T08:39:58.544966Z",
     "shell.execute_reply.started": "2021-07-15T08:18:26.333244Z"
    },
    "papermill": {
     "duration": 0.802391,
     "end_time": "2021-07-15T08:39:58.545144",
     "exception": false,
     "start_time": "2021-07-15T08:39:57.742753",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(8,24))\n",
    "subplot2grid((3,1),(0,0))\n",
    "scatterplot(x=df['English'], y=df['Salary']).set_title(\"English\", fontsize=14)\n",
    "subplot2grid((3,1),(1,0))\n",
    "scatterplot(x=df['Logical'], y=df['Salary']).set_title(\"Logical\", fontsize=14)\n",
    "subplot2grid((3,1),(2,0))\n",
    "scatterplot(x=df['Quant'], y=df['Salary']).set_title(\"Quant\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:58.704072Z",
     "iopub.status.busy": "2021-07-15T08:39:58.703039Z",
     "iopub.status.idle": "2021-07-15T08:39:58.940216Z",
     "shell.execute_reply": "2021-07-15T08:39:58.939711Z",
     "shell.execute_reply.started": "2021-07-15T08:18:27.086713Z"
    },
    "papermill": {
     "duration": 0.317655,
     "end_time": "2021-07-15T08:39:58.940393",
     "exception": false,
     "start_time": "2021-07-15T08:39:58.622738",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = scatterplot(x=df['Domain'], y=df['Salary'])\n",
    "plot.set_title(\"Domain\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:39:59.134607Z",
     "iopub.status.busy": "2021-07-15T08:39:59.133935Z",
     "iopub.status.idle": "2021-07-15T08:40:00.752513Z",
     "shell.execute_reply": "2021-07-15T08:40:00.752992Z",
     "shell.execute_reply.started": "2021-07-15T08:18:27.325736Z"
    },
    "papermill": {
     "duration": 1.73236,
     "end_time": "2021-07-15T08:40:00.753162",
     "exception": false,
     "start_time": "2021-07-15T08:39:59.020802",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,32))\n",
    "subplot2grid((4,2),(0,0))\n",
    "scatterplot(x=df['ComputerProgramming'], y=df['Salary']).set_title(\"ComputerProgramming\", fontsize=14)\n",
    "subplot2grid((4,2),(0,1))\n",
    "scatterplot(x=df['ElectronicsAndSemicon'], y=df['Salary']).set_title(\"ElectronicsAndSemicon\", fontsize=14)\n",
    "subplot2grid((4,2),(1,0))\n",
    "scatterplot(x=df['ComputerScience'], y=df['Salary']).set_title(\"ComputerScience\", fontsize=14)\n",
    "subplot2grid((4,2),(1,1))\n",
    "scatterplot(x=df['MechanicalEngg'], y=df['Salary']).set_title(\"MechanicalEngg\", fontsize=14)\n",
    "subplot2grid((4,2),(2,0))\n",
    "scatterplot(x=df['ElectricalEngg'], y=df['Salary']).set_title(\"ElectricalEngg\", fontsize=14)\n",
    "subplot2grid((4,2),(2,1))\n",
    "scatterplot(x=df['TelecomEngg'], y=df['Salary']).set_title(\"TelecomEngg\", fontsize=14)\n",
    "subplot2grid((4,2),(3,0))\n",
    "scatterplot(x=df['CivilEngg'], y=df['Salary']).set_title(\"CivilEngg\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:00.937913Z",
     "iopub.status.busy": "2021-07-15T08:40:00.937242Z",
     "iopub.status.idle": "2021-07-15T08:40:18.717819Z",
     "shell.execute_reply": "2021-07-15T08:40:18.718300Z",
     "shell.execute_reply.started": "2021-07-15T08:18:28.994817Z"
    },
    "papermill": {
     "duration": 17.876575,
     "end_time": "2021-07-15T08:40:18.718471",
     "exception": false,
     "start_time": "2021-07-15T08:40:00.841896",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,16))\n",
    "plot= scatterplot(x=df[\"10board\"],y=df[\"12board\"])\n",
    "plot.set_title(\"Scatter Plot of 10board with 12board\", fontsize=14)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.101686,
     "end_time": "2021-07-15T08:40:18.921859",
     "exception": false,
     "start_time": "2021-07-15T08:40:18.820173",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Though 10board and 12board are not very same but still I will remove 10board so that there is no risk of multicollinearity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:19.135446Z",
     "iopub.status.busy": "2021-07-15T08:40:19.134691Z",
     "iopub.status.idle": "2021-07-15T08:40:19.137883Z",
     "shell.execute_reply": "2021-07-15T08:40:19.137371Z",
     "shell.execute_reply.started": "2021-07-15T08:18:46.781821Z"
    },
    "papermill": {
     "duration": 0.113373,
     "end_time": "2021-07-15T08:40:19.138031",
     "exception": false,
     "start_time": "2021-07-15T08:40:19.024658",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(columns=[\"10board\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:19.348570Z",
     "iopub.status.busy": "2021-07-15T08:40:19.347710Z",
     "iopub.status.idle": "2021-07-15T08:40:34.676846Z",
     "shell.execute_reply": "2021-07-15T08:40:34.677328Z",
     "shell.execute_reply.started": "2021-07-15T08:18:46.789564Z"
    },
    "papermill": {
     "duration": 15.438489,
     "end_time": "2021-07-15T08:40:34.677494",
     "exception": false,
     "start_time": "2021-07-15T08:40:19.239005",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,8))\n",
    "plot = countplot(x=df[\"12board\"])\n",
    "plot.set_xticklabels(plot.get_xticklabels(),rotation=\"vertical\")\n",
    "plot.set_title(\"Count Plot of 12board\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:34.909610Z",
     "iopub.status.busy": "2021-07-15T08:40:34.908905Z",
     "iopub.status.idle": "2021-07-15T08:40:34.911920Z",
     "shell.execute_reply": "2021-07-15T08:40:34.911393Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.272813Z"
    },
    "papermill": {
     "duration": 0.120572,
     "end_time": "2021-07-15T08:40:34.912055",
     "exception": false,
     "start_time": "2021-07-15T08:40:34.791483",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "board = df[\"12board\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.110524,
     "end_time": "2021-07-15T08:40:35.133466",
     "exception": false,
     "start_time": "2021-07-15T08:40:35.022942",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "There are many boards which are very rarely choosen by students\n",
    "\n",
    "So I will take major boards and categorize rest of them as other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:35.362566Z",
     "iopub.status.busy": "2021-07-15T08:40:35.361576Z",
     "iopub.status.idle": "2021-07-15T08:40:35.364973Z",
     "shell.execute_reply": "2021-07-15T08:40:35.364490Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.279936Z"
    },
    "papermill": {
     "duration": 0.119885,
     "end_time": "2021-07-15T08:40:35.365108",
     "exception": false,
     "start_time": "2021-07-15T08:40:35.245223",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "rare_board = board[board <= 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:35.600118Z",
     "iopub.status.busy": "2021-07-15T08:40:35.599489Z",
     "iopub.status.idle": "2021-07-15T08:40:35.605598Z",
     "shell.execute_reply": "2021-07-15T08:40:35.606020Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.296422Z"
    },
    "papermill": {
     "duration": 0.131232,
     "end_time": "2021-07-15T08:40:35.606192",
     "exception": false,
     "start_time": "2021-07-15T08:40:35.474960",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def remove_rare_board(value):\n",
    "    if value in rare_board:\n",
    "        return 'other'\n",
    "    else:\n",
    "        return value\n",
    "df[\"12board\"] = df[\"12board\"].apply(remove_rare_board)\n",
    "df[\"12board\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.109912,
     "end_time": "2021-07-15T08:40:35.826194",
     "exception": false,
     "start_time": "2021-07-15T08:40:35.716282",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Looks like there is another error as there is no '0' board\n",
    "I will replace this '0' board with 'cbse' as it is the most common one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:36.054452Z",
     "iopub.status.busy": "2021-07-15T08:40:36.053427Z",
     "iopub.status.idle": "2021-07-15T08:40:36.056116Z",
     "shell.execute_reply": "2021-07-15T08:40:36.056602Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.316251Z"
    },
    "papermill": {
     "duration": 0.119621,
     "end_time": "2021-07-15T08:40:36.056777",
     "exception": false,
     "start_time": "2021-07-15T08:40:35.937156",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"12board\"].replace(to_replace='0',value='cbse', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.109756,
     "end_time": "2021-07-15T08:40:36.276451",
     "exception": false,
     "start_time": "2021-07-15T08:40:36.166695",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "I will convert categorical data to numeric data using get_dummies, so we can use these in our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:36.507819Z",
     "iopub.status.busy": "2021-07-15T08:40:36.506826Z",
     "iopub.status.idle": "2021-07-15T08:40:36.514446Z",
     "shell.execute_reply": "2021-07-15T08:40:36.513909Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.325779Z"
    },
    "papermill": {
     "duration": 0.126585,
     "end_time": "2021-07-15T08:40:36.514604",
     "exception": false,
     "start_time": "2021-07-15T08:40:36.388019",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = get_dummies(df, columns=[\"12board\"], prefix=\"board_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:36.748514Z",
     "iopub.status.busy": "2021-07-15T08:40:36.745068Z",
     "iopub.status.idle": "2021-07-15T08:40:37.022845Z",
     "shell.execute_reply": "2021-07-15T08:40:37.022352Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.343779Z"
    },
    "papermill": {
     "duration": 0.396507,
     "end_time": "2021-07-15T08:40:37.022989",
     "exception": false,
     "start_time": "2021-07-15T08:40:36.626482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,8))\n",
    "plot = barplot(x=df[\"Degree\"],y=df[\"Salary\"])\n",
    "plot.set_xticklabels(plot.get_xticklabels(),rotation=\"vertical\")\n",
    "plot.set_title(\"Bar Plot of Degree with Salary\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:37.253524Z",
     "iopub.status.busy": "2021-07-15T08:40:37.252887Z",
     "iopub.status.idle": "2021-07-15T08:40:37.257023Z",
     "shell.execute_reply": "2021-07-15T08:40:37.256446Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.657644Z"
    },
    "papermill": {
     "duration": 0.122644,
     "end_time": "2021-07-15T08:40:37.257157",
     "exception": false,
     "start_time": "2021-07-15T08:40:37.134513",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"Degree\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.112874,
     "end_time": "2021-07-15T08:40:37.482085",
     "exception": false,
     "start_time": "2021-07-15T08:40:37.369211",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Looks like majority of students belong to B.Tech./B.E. but still degree is very important factor in determining Salary\n",
    "\n",
    "Again I will convert categorical data to numeric data using get_dummies, so we can use these in our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:37.713605Z",
     "iopub.status.busy": "2021-07-15T08:40:37.712930Z",
     "iopub.status.idle": "2021-07-15T08:40:37.721509Z",
     "shell.execute_reply": "2021-07-15T08:40:37.721963Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.668242Z"
    },
    "papermill": {
     "duration": 0.12759,
     "end_time": "2021-07-15T08:40:37.722162",
     "exception": false,
     "start_time": "2021-07-15T08:40:37.594572",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = get_dummies(df, columns=[\"Degree\"], prefix=\"degree_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:37.952009Z",
     "iopub.status.busy": "2021-07-15T08:40:37.950790Z",
     "iopub.status.idle": "2021-07-15T08:40:38.937461Z",
     "shell.execute_reply": "2021-07-15T08:40:38.936956Z",
     "shell.execute_reply.started": "2021-07-15T08:19:02.686022Z"
    },
    "papermill": {
     "duration": 1.103719,
     "end_time": "2021-07-15T08:40:38.937608",
     "exception": false,
     "start_time": "2021-07-15T08:40:37.833889",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,8))\n",
    "plot = countplot(x=df[\"Specialization\"])\n",
    "plot.set_xticklabels(plot.get_xticklabels(),rotation=\"vertical\")\n",
    "plot.set_title(\"Count Plot of Specialization\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:39.178173Z",
     "iopub.status.busy": "2021-07-15T08:40:39.177579Z",
     "iopub.status.idle": "2021-07-15T08:40:39.179467Z",
     "shell.execute_reply": "2021-07-15T08:40:39.179863Z",
     "shell.execute_reply.started": "2021-07-15T08:19:03.761622Z"
    },
    "papermill": {
     "duration": 0.124408,
     "end_time": "2021-07-15T08:40:39.180030",
     "exception": false,
     "start_time": "2021-07-15T08:40:39.055622",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "specializations = df[\"Specialization\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.116591,
     "end_time": "2021-07-15T08:40:39.410826",
     "exception": false,
     "start_time": "2021-07-15T08:40:39.294235",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "I will categorize rarely chosen specializations into other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:39.643891Z",
     "iopub.status.busy": "2021-07-15T08:40:39.642988Z",
     "iopub.status.idle": "2021-07-15T08:40:39.647704Z",
     "shell.execute_reply": "2021-07-15T08:40:39.648194Z",
     "shell.execute_reply.started": "2021-07-15T08:19:03.77026Z"
    },
    "papermill": {
     "duration": 0.122028,
     "end_time": "2021-07-15T08:40:39.648375",
     "exception": false,
     "start_time": "2021-07-15T08:40:39.526347",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "rare_specialization = specializations[specializations <= 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:39.882724Z",
     "iopub.status.busy": "2021-07-15T08:40:39.881621Z",
     "iopub.status.idle": "2021-07-15T08:40:39.895492Z",
     "shell.execute_reply": "2021-07-15T08:40:39.894854Z",
     "shell.execute_reply.started": "2021-07-15T08:19:03.782896Z"
    },
    "papermill": {
     "duration": 0.132294,
     "end_time": "2021-07-15T08:40:39.895632",
     "exception": false,
     "start_time": "2021-07-15T08:40:39.763338",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def remove_rare_specializations(value):\n",
    "    if value in rare_specialization:\n",
    "        return 'other'\n",
    "    else:\n",
    "        return value\n",
    "df[\"Specialization\"] = df[\"Specialization\"].apply(remove_rare_specializations)\n",
    "df[\"Specialization\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.114274,
     "end_time": "2021-07-15T08:40:40.124655",
     "exception": false,
     "start_time": "2021-07-15T08:40:40.010381",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Now let's convert it to numeric so we can use it in our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:40.361719Z",
     "iopub.status.busy": "2021-07-15T08:40:40.360728Z",
     "iopub.status.idle": "2021-07-15T08:40:40.370377Z",
     "shell.execute_reply": "2021-07-15T08:40:40.370835Z",
     "shell.execute_reply.started": "2021-07-15T08:19:03.801771Z"
    },
    "papermill": {
     "duration": 0.130564,
     "end_time": "2021-07-15T08:40:40.371019",
     "exception": false,
     "start_time": "2021-07-15T08:40:40.240455",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = get_dummies(df, columns=[\"Specialization\"], prefix=\"specialization_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:40.630569Z",
     "iopub.status.busy": "2021-07-15T08:40:40.629641Z",
     "iopub.status.idle": "2021-07-15T08:40:41.728545Z",
     "shell.execute_reply": "2021-07-15T08:40:41.729016Z",
     "shell.execute_reply.started": "2021-07-15T08:19:03.818474Z"
    },
    "papermill": {
     "duration": 1.223995,
     "end_time": "2021-07-15T08:40:41.729181",
     "exception": false,
     "start_time": "2021-07-15T08:40:40.505186",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,24))\n",
    "subplot2grid((4,2),(0,0))\n",
    "scatterplot(x=df['conscientiousness'], y=df['Salary']).set_title(\"conscientiousness\", fontsize=14)\n",
    "subplot2grid((4,2),(0,1))\n",
    "scatterplot(x=df['agreeableness'], y=df['Salary']).set_title(\"agreeableness\", fontsize=14)\n",
    "subplot2grid((4,2),(1,0))\n",
    "scatterplot(x=df['extraversion'], y=df['Salary']).set_title(\"extraversion\", fontsize=14)\n",
    "subplot2grid((4,2),(1,1))\n",
    "scatterplot(x=df['nueroticism'], y=df['Salary']).set_title(\"nueroticism\", fontsize=14)\n",
    "subplot2grid((4,2),(2,0))\n",
    "scatterplot(x=df['openess_to_experience'], y=df['Salary']).set_title(\"openess_to_experience\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:41.987978Z",
     "iopub.status.busy": "2021-07-15T08:40:41.987228Z",
     "iopub.status.idle": "2021-07-15T08:40:42.217707Z",
     "shell.execute_reply": "2021-07-15T08:40:42.218203Z",
     "shell.execute_reply.started": "2021-07-15T08:19:04.973106Z"
    },
    "papermill": {
     "duration": 0.363789,
     "end_time": "2021-07-15T08:40:42.218402",
     "exception": false,
     "start_time": "2021-07-15T08:40:41.854613",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot = scatterplot(x=df[\"CollegeTier\"], y=df[\"CollegeCityTier\"])\n",
    "plot.set_title(\"Scatter Plot of CollegeTier with CollegeCityTier\", fontsize=14)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.126033,
     "end_time": "2021-07-15T08:40:42.470812",
     "exception": false,
     "start_time": "2021-07-15T08:40:42.344779",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Let's remove CollegeCityTier as it is not as relevant as CollegeTier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:42.724724Z",
     "iopub.status.busy": "2021-07-15T08:40:42.723767Z",
     "iopub.status.idle": "2021-07-15T08:40:42.730680Z",
     "shell.execute_reply": "2021-07-15T08:40:42.731222Z",
     "shell.execute_reply.started": "2021-07-15T08:19:05.225136Z"
    },
    "papermill": {
     "duration": 0.135476,
     "end_time": "2021-07-15T08:40:42.731417",
     "exception": false,
     "start_time": "2021-07-15T08:40:42.595941",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(columns=[\"CollegeCityTier\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:42.994638Z",
     "iopub.status.busy": "2021-07-15T08:40:42.993682Z",
     "iopub.status.idle": "2021-07-15T08:40:43.434975Z",
     "shell.execute_reply": "2021-07-15T08:40:43.434457Z",
     "shell.execute_reply.started": "2021-07-15T08:19:05.232318Z"
    },
    "papermill": {
     "duration": 0.573276,
     "end_time": "2021-07-15T08:40:43.435107",
     "exception": false,
     "start_time": "2021-07-15T08:40:42.861831",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "figure(figsize=(16,8))\n",
    "plot = countplot(x=df[\"CollegeState\"])\n",
    "plot.set_xticklabels(plot.get_xticklabels(),rotation=\"vertical\")\n",
    "plot.set_title(\"Count Plot of CollegeState\", fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:43.692710Z",
     "iopub.status.busy": "2021-07-15T08:40:43.692008Z",
     "iopub.status.idle": "2021-07-15T08:40:43.699141Z",
     "shell.execute_reply": "2021-07-15T08:40:43.699702Z",
     "shell.execute_reply.started": "2021-07-15T08:19:05.707147Z"
    },
    "papermill": {
     "duration": 0.137752,
     "end_time": "2021-07-15T08:40:43.699894",
     "exception": false,
     "start_time": "2021-07-15T08:40:43.562142",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"CollegeState\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:43.963320Z",
     "iopub.status.busy": "2021-07-15T08:40:43.959262Z",
     "iopub.status.idle": "2021-07-15T08:40:43.970395Z",
     "shell.execute_reply": "2021-07-15T08:40:43.969857Z",
     "shell.execute_reply.started": "2021-07-15T08:19:05.71791Z"
    },
    "papermill": {
     "duration": 0.142607,
     "end_time": "2021-07-15T08:40:43.970558",
     "exception": false,
     "start_time": "2021-07-15T08:40:43.827951",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = get_dummies(df, columns=[\"CollegeState\"], prefix=\"state_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:44.231503Z",
     "iopub.status.busy": "2021-07-15T08:40:44.230805Z",
     "iopub.status.idle": "2021-07-15T08:40:44.237980Z",
     "shell.execute_reply": "2021-07-15T08:40:44.237392Z",
     "shell.execute_reply.started": "2021-07-15T08:19:05.734293Z"
    },
    "papermill": {
     "duration": 0.138215,
     "end_time": "2021-07-15T08:40:44.238112",
     "exception": false,
     "start_time": "2021-07-15T08:40:44.099897",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.129838,
     "end_time": "2021-07-15T08:40:44.495979",
     "exception": false,
     "start_time": "2021-07-15T08:40:44.366141",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "I will use Ridge Regression as there is some multicollinearity\n",
    "\n",
    "Ridge provides advantage in this case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-15T08:40:44.756029Z",
     "iopub.status.busy": "2021-07-15T08:40:44.755366Z",
     "iopub.status.idle": "2021-07-15T08:40:44.818229Z",
     "shell.execute_reply": "2021-07-15T08:40:44.819486Z",
     "shell.execute_reply.started": "2021-07-15T08:19:05.744051Z"
    },
    "papermill": {
     "duration": 0.195933,
     "end_time": "2021-07-15T08:40:44.819771",
     "exception": false,
     "start_time": "2021-07-15T08:40:44.623838",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = df.copy()\n",
    "X = data.drop(columns=[\"Salary\"])\n",
    "y = data[[\"Salary\"]]\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y, test_size=0.4,random_state=49, shuffle=True)\n",
    "\n",
    "model = Ridge(alpha=0.5)\n",
    "model.fit(X_train,y_train)\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "print(sqrt(mean_squared_error(y_test,y_pred)))\n",
    "print(r2_score(y_test,y_pred))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "1ae46ccb3f3b5e600c6269f17f42033cb93fa3afca6206e5230d3d0f143258cc"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit ('engineering-graduate-salary-analysis-ev1blrql': pipenv)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 70.034163,
   "end_time": "2021-07-15T08:40:46.415132",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-07-15T08:39:36.380969",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}