PCA+SVM_result0.746.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:95% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# split 20% for validation set\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.svm import SVC\n",
    "from time import time\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "\n",
    "from IPython.core.display import display, HTML \n",
    "display(HTML(\"<style>.container { width:95% !important; }</style>\"))  # Use all space available in the browser\n",
    "\n",
    "pd.options.mode.chained_assignment = None                             # removes unnecessary error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training = pd.read_csv(\"train.csv\")\n",
    "test = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "''' Remove useless Features: '''\n",
    "\n",
    "result = training.loc[:,'Survived']                                   # needed later for the training\n",
    "testIds = test.loc[:,'PassengerId']                                   # needed later for the submission\n",
    "training = training.drop(['Survived','Ticket','PassengerId','Name','Fare','Age'], axis=1)\n",
    "test = test.drop(['Ticket','PassengerId','Name','Fare','Age'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "''' transform Features into numbers if necessary (training): '''\n",
    "\n",
    "# Sex:\n",
    "training.loc[:,'Sex'][np.where(training['Sex'] == 'male')[0]] = 1           # male female into 1:male 0:female\n",
    "training.loc[:,'Sex'][np.where(training['Sex'] == 'female')[0]] = 0         # male female into 1:male 0:female\n",
    "\n",
    "# Cabins:\n",
    "givenCabins = np.where(training['Cabin'] == training['Cabin'])[0]                                    # find indexes where the cabin is given\n",
    "training.loc[:,'Cabin'][givenCabins] = [cab[:1] for cab in training.loc[:,'Cabin'][givenCabins]]     # The Cabin Number seems Irrelevant to me so I removed it\n",
    "training.loc[:,'Cabin'][givenCabins] = [ord(cab)-64 for cab in training.loc[:,'Cabin'][givenCabins]] # replaced the deck letter with a Number\n",
    "training.loc[:,'Cabin'][np.where(training['Cabin'] != training['Cabin'])[0]] = -1                    # replace NaN Values with -1\n",
    "\n",
    "# Embarked:\n",
    "givenEmbarked = np.where(training['Embarked'] == training['Embarked'])[0]   # find indexes where Embarked is given\n",
    "embarkedColumn = training.loc[:,'Embarked'][givenEmbarked]\n",
    "embarkedColumn = [ord(char)-64 for char in embarkedColumn]\n",
    "embarkedColumn = [1 if num==3 else num for num in embarkedColumn]           # Embarked letter num 3  -> class 1\n",
    "embarkedColumn = [2 if num==17 else num for num in embarkedColumn]          # Embarked letter num 17 -> class 2\n",
    "embarkedColumn = [3 if num==19 else num for num in embarkedColumn]          # Embarked letter num 19 -> class 3\n",
    "embarkedClass = [\"C\",\"Q\",\"S\"] # C = Class 1, Q = Class 2, S = Class 3\n",
    "training.loc[:,'Embarked'][givenEmbarked] = embarkedColumn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "''' transform Features into numbers if necessary (test): '''\n",
    "\n",
    "# Sex:\n",
    "test.loc[:,'Sex'][np.where(test['Sex'] == 'male')[0]] = 1           # male female into 1:male 0:female\n",
    "test.loc[:,'Sex'][np.where(test['Sex'] == 'female')[0]] = 0         # male female into 1:male 0:female\n",
    "\n",
    "# Cabins:\n",
    "givenCabins = np.where(test['Cabin'] == test['Cabin'])[0]                                    # find indexes where the cabin is given\n",
    "test.loc[:,'Cabin'][givenCabins] = [cab[:1] for cab in test.loc[:,'Cabin'][givenCabins]]     # The Cabin Number seems Irrelevant to me so I removed it\n",
    "test.loc[:,'Cabin'][givenCabins] = [ord(cab)-64 for cab in test.loc[:,'Cabin'][givenCabins]] # replaced the deck letter with a Number\n",
    "test.loc[:,'Cabin'][np.where(test['Cabin'] != test['Cabin'])[0]] = -1                    # replace NaN Values with -1\n",
    "\n",
    "# Embarked:\n",
    "givenEmbarked = np.where(test['Embarked'] == test['Embarked'])[0]   # find indexes where Embarked is given\n",
    "embarkedColumn = test.loc[:,'Embarked'][givenEmbarked]\n",
    "embarkedColumn = [ord(char)-64 for char in embarkedColumn]\n",
    "embarkedColumn = [1 if num==3 else num for num in embarkedColumn]           # Embarked letter num 3  -> class 1\n",
    "embarkedColumn = [2 if num==17 else num for num in embarkedColumn]          # Embarked letter num 17 -> class 2\n",
    "embarkedColumn = [3 if num==19 else num for num in embarkedColumn]          # Embarked letter num 19 -> class 3\n",
    "embarkedClass = [\"C\",\"Q\",\"S\"] # C = Class 1, Q = Class 2, S = Class 3\n",
    "test.loc[:,'Embarked'][givenEmbarked] = embarkedColumn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lets look at the first 10 data points\n",
      "   Pclass Sex  SibSp  Parch Cabin Embarked\n",
      "0       3   1      1      0    -1        3\n",
      "1       1   0      1      0     3        1\n",
      "2       3   0      0      0    -1        3\n",
      "3       1   0      1      0     3        3\n",
      "4       3   1      0      0    -1        3\n",
      "5       3   1      0      0    -1        2\n",
      "6       1   1      0      0     5        3\n",
      "7       3   1      3      1    -1        3\n",
      "8       3   0      0      2    -1        3\n",
      "9       2   0      1      0    -1        1\n"
     ]
    }
   ],
   "source": [
    "print(\"Lets look at the first 10 data points\")\n",
    "print(training[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed 2 rows from training_clean because at least one feature was None\n"
     ]
    }
   ],
   "source": [
    "''' Remove rows that still contain NaN: '''\n",
    "\n",
    "nullRows = np.where(pd.isnull(training).values)[0]\n",
    "result = result.drop(result.index[nullRows])\n",
    "nullRowsTest = np.where(pd.isnull(test).values)[0]\n",
    "testIds = testIds.drop(testIds.index[nullRowsTest])\n",
    "\n",
    "shapeBefore = training.shape[0]\n",
    "training = training.dropna(axis=0, how='any') # remove empty rows\n",
    "test = test.dropna(axis=0, how='any') # remove empty rows\n",
    "\n",
    "print(\"Removed\", shapeBefore - training.shape[0], \"rows from training_clean because at least one feature was None\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(889, 6)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "''' Get most meaningful features or combinations of features through PCA: '''\n",
    "n_components = training.shape[1]\n",
    "pca = PCA(n_components=n_components, whiten=True, svd_solver=\"randomized\").fit(training)\n",
    "eigenvalues = pca.explained_variance_\n",
    "\n",
    "X_train_pca = pca.transform(training)\n",
    "X_test_pca = pca.transform(test)\n",
    "print(X_train_pca.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 54.405s\n"
     ]
    }
   ],
   "source": [
    "'''Fitting the classifier to the training set'''\n",
    "\n",
    "t0 = time()\n",
    "param_grid = {'C': [1e3, 5e3, 1e4, 5e4, 1e5],\n",
    "              'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1], }\n",
    "clf = GridSearchCV(SVC(kernel='rbf', class_weight='balanced'), param_grid)\n",
    "clf = clf.fit(X_train_pca, result)\n",
    "print(\"done in %0.3fs\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    PassengerId  Survived\n",
      "1           893         1\n",
      "2           894         0\n",
      "3           895         0\n",
      "4           896         0\n",
      "5           897         0\n",
      "6           898         1\n",
      "7           899         0\n",
      "8           900         1\n",
      "9           901         0\n",
      "10          902         0\n",
      "11          903         0\n",
      "12          904         1\n",
      "13          905         0\n",
      "14          906         1\n"
     ]
    }
   ],
   "source": [
    "'''Predicting alive or dead on the test set'''\n",
    "y_pred = clf.predict(X_test_pca)\n",
    "submission_array = np.column_stack((testIds,y_pred))\n",
    "submission = pd.DataFrame(submission_array, columns=['PassengerId','Survived'])  # 1st row as the column names\n",
    "print(submission[1:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('submission1.csv', index=False) # result: 0.746 !"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}