Feature_selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  6.    148.     72.    ...  33.6     0.627  50.   ]\n",
      " [  1.     85.     66.    ...  26.6     0.351  31.   ]\n",
      " [  8.    183.     64.    ...  23.3     0.672  32.   ]\n",
      " ...\n",
      " [  5.    121.     72.    ...  26.2     0.245  30.   ]\n",
      " [  1.    126.     60.    ...  30.1     0.349  47.   ]\n",
      " [  1.     93.     70.    ...  30.4     0.315  23.   ]]\n",
      "[1. 0. 1. 0. 1. 0. 1. 0. 1. 1. 0. 1. 0. 1. 1. 1. 1. 1. 0. 1. 0. 0. 1. 1.\n",
      " 1. 1. 1. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 1. 1. 1. 0. 0. 0. 1. 0. 1. 0. 0.\n",
      " 1. 0. 0. 0. 0. 1. 0. 0. 1. 0. 0. 0. 0. 1. 0. 0. 1. 0. 1. 0. 0. 0. 1. 0.\n",
      " 1. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 1. 0. 0. 0. 1. 0. 0. 0. 0. 1. 0. 0.\n",
      " 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 1. 0. 0. 1. 1. 1. 0. 0. 0.\n",
      " 1. 0. 0. 0. 1. 1. 0. 0. 1. 1. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 1. 1. 0. 0. 0. 1. 0. 0. 0. 0. 1. 1. 0. 0.\n",
      " 0. 0. 1. 1. 0. 0. 0. 1. 0. 1. 0. 1. 0. 0. 0. 0. 0. 1. 1. 1. 1. 1. 0. 0.\n",
      " 1. 1. 0. 1. 0. 1. 1. 1. 0. 0. 0. 0. 0. 0. 1. 1. 0. 1. 0. 0. 0. 1. 1. 1.\n",
      " 1. 0. 1. 1. 1. 1. 0. 0. 0. 0. 0. 1. 0. 0. 1. 1. 0. 0. 0. 1. 1. 1. 1. 0.\n",
      " 0. 0. 1. 1. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 0. 1. 0. 1. 0. 0.\n",
      " 1. 0. 1. 0. 0. 1. 1. 0. 0. 0. 0. 0. 1. 0. 0. 0. 1. 0. 0. 1. 1. 0. 0. 1.\n",
      " 0. 0. 0. 1. 1. 1. 0. 0. 1. 0. 1. 0. 1. 1. 0. 1. 0. 0. 1. 0. 1. 1. 0. 0.\n",
      " 1. 0. 1. 0. 0. 1. 0. 1. 0. 1. 1. 1. 0. 0. 1. 0. 1. 0. 0. 0. 1. 0. 0. 0.\n",
      " 0. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 1. 1. 1. 0. 1.\n",
      " 1. 0. 0. 1. 0. 0. 1. 0. 0. 1. 1. 0. 0. 0. 0. 1. 0. 0. 1. 0. 0. 0. 0. 0.\n",
      " 0. 0. 1. 1. 1. 0. 0. 1. 0. 0. 1. 0. 0. 1. 0. 1. 1. 0. 1. 0. 1. 0. 1. 0.\n",
      " 1. 1. 0. 0. 0. 0. 1. 1. 0. 1. 0. 1. 0. 0. 0. 0. 1. 1. 0. 1. 0. 1. 0. 0.\n",
      " 0. 0. 0. 1. 0. 0. 0. 0. 1. 0. 0. 1. 1. 1. 0. 0. 1. 0. 0. 1. 0. 0. 0. 1.\n",
      " 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0.\n",
      " 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 1. 0. 0. 0. 1. 0.\n",
      " 0. 0. 1. 0. 0. 0. 1. 0. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 1. 1. 1. 1. 0. 0. 1. 1. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.\n",
      " 0. 1. 0. 1. 1. 0. 0. 0. 1. 0. 1. 0. 1. 0. 1. 0. 1. 0. 0. 1. 0. 0. 1. 0.\n",
      " 0. 0. 0. 1. 1. 0. 1. 0. 0. 0. 0. 1. 1. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 1. 0. 0. 1. 0. 0. 0. 1. 0. 0. 0. 1. 1.\n",
      " 1. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 1. 0. 1. 1. 1. 1. 0. 1. 1. 0. 0. 0. 0.\n",
      " 0. 0. 0. 1. 1. 0. 1. 0. 0. 1. 0. 1. 0. 0. 0. 0. 0. 1. 0. 1. 0. 1. 0. 1.\n",
      " 1. 0. 0. 0. 0. 1. 1. 0. 0. 0. 1. 0. 1. 1. 0. 0. 1. 0. 0. 1. 1. 0. 0. 1.\n",
      " 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 1. 1. 1. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1.\n",
      " 0. 0. 1. 0. 1. 1. 1. 0. 0. 1. 1. 1. 0. 1. 0. 1. 0. 1. 0. 0. 0. 0. 1. 0.]\n",
      "[ 111.52  1411.887   17.605   53.108 2175.565  127.669    5.393  181.304]\n",
      "[[148.    0.   33.6  50. ]\n",
      " [ 85.    0.   26.6  31. ]\n",
      " [183.    0.   23.3  32. ]\n",
      " [ 89.   94.   28.1  21. ]\n",
      " [137.  168.   43.1  33. ]]\n"
     ]
    }
   ],
   "source": [
    "# Feature Extraction with Univariate Statistical Tests (Chi-squared for classification)\n",
    "import pandas\n",
    "import numpy\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "# load data\n",
    "#url = \"https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv\"\n",
    "names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']\n",
    "dataframe = pandas.read_csv(\"C:\\\\Users\\\\kasi\\\\Desktop\\\\datascience_project\\\\pima-indians-diabetes.data.csv\", names=names)\n",
    "array = dataframe.values\n",
    "X = array[:,0:8]\n",
    "print(X)\n",
    "\n",
    "Y = array[:,8]\n",
    "print(Y)\n",
    "# feature extraction\n",
    "test = SelectKBest(score_func=chi2, k=4)\n",
    "fit = test.fit(X, Y)\n",
    "# summarize scores\n",
    "numpy.set_printoptions(precision=3)\n",
    "print(fit.scores_)\n",
    "features = fit.transform(X)\n",
    "# summarize selected features\n",
    "print(features[0:5,:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can see the scores for each attribute and the 4 attributes chosen (those with the highest scores): plas, test, mass and age."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Recursive Feature Elimination\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Features: %d 3\n",
      "Selected Features: %s [ True False False False False  True  True False]\n",
      "Feature Ranking: %s [1 2 3 5 6 1 1 4]\n"
     ]
    }
   ],
   "source": [
    "# Feature Extraction with RFE\n",
    "from pandas import read_csv\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "# load data\n",
    "#url = \"https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv\"\n",
    "names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']\n",
    "#dataframe = read_csv(url, names=names)\n",
    "dataframe = pandas.read_csv(\"C:\\\\Users\\\\kasi\\\\Desktop\\\\datascience_project\\\\pima-indians-diabetes.data.csv\", names=names)\n",
    "array = dataframe.values\n",
    "X = array[:,0:8]\n",
    "Y = array[:,8]\n",
    "# feature extraction\n",
    "model = LogisticRegression()\n",
    "rfe = RFE(model, 3)\n",
    "fit = rfe.fit(X, Y)\n",
    "print(\"Num Features: %d\",fit.n_features_)\n",
    "print(\"Selected Features: %s\", fit.support_)\n",
    "print(\"Feature Ranking: %s\",fit.ranking_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Principal Component Analysis\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explained Variance: %s [0.889 0.062 0.026]\n",
      "[[-2.022e-03  9.781e-02  1.609e-02  6.076e-02  9.931e-01  1.401e-02\n",
      "   5.372e-04 -3.565e-03]\n",
      " [-2.265e-02 -9.722e-01 -1.419e-01  5.786e-02  9.463e-02 -4.697e-02\n",
      "  -8.168e-04 -1.402e-01]\n",
      " [-2.246e-02  1.434e-01 -9.225e-01 -3.070e-01  2.098e-02 -1.324e-01\n",
      "  -6.400e-04 -1.255e-01]]\n"
     ]
    }
   ],
   "source": [
    "# Feature Extraction with PCA\n",
    "import numpy\n",
    "from pandas import read_csv\n",
    "from sklearn.decomposition import PCA\n",
    "# load data\n",
    "#url = \"https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv\"\n",
    "names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']\n",
    "dataframe = pandas.read_csv(\"C:\\\\Users\\\\kasi\\\\Desktop\\\\datascience_project\\\\pima-indians-diabetes.data.csv\", names=names)\n",
    "array = dataframe.values\n",
    "X = array[:,0:8]\n",
    "Y = array[:,8]\n",
    "# feature extraction\n",
    "pca = PCA(n_components=3)\n",
    "fit = pca.fit(X)\n",
    "# summarize components\n",
    "print(\"Explained Variance: %s\",fit.explained_variance_ratio_)\n",
    "print(fit.components_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4. Feature Importance\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.12  0.237 0.095 0.077 0.064 0.153 0.122 0.131]\n"
     ]
    }
   ],
   "source": [
    "# Feature Importance with Extra Trees Classifier\n",
    "from pandas import read_csv\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "# load data\n",
    "#url = \"https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv\"\n",
    "names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']\n",
    "#dataframe = read_csv(url, names=names)\n",
    "dataframe = pandas.read_csv(\"C:\\\\Users\\\\kasi\\\\Desktop\\\\datascience_project\\\\pima-indians-diabetes.data.csv\", names=names)\n",
    "array = dataframe.values\n",
    "X = array[:,0:8]\n",
    "Y = array[:,8]\n",
    "# feature extraction\n",
    "model = ExtraTreesClassifier()\n",
    "model.fit(X, Y)\n",
    "print(model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}