destination_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook creates a prediction for rows where the origin destination distance is exactly the same as at least one of the training rows.\n",
    "\n",
    "This takes advantage of a mistake made by Expedia: They gave a precise distance between the user's city and the exact hotel that they chose in the end. In the training data set, there are some items that repeat the `orig_destination_distance`. For about 95% of them there is 1-to-1 correspondance between that distance and the `hotel_cluster`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from math import ceil\n",
    "with open('datapath.txt') as f:\n",
    "    datapath=f.readlines()[0].rstrip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "totaltrainrows=37670293\n",
    "ndestinations=62106\n",
    "totaltestrows=2528243"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "testdata=pd.read_csv(datapath+'test.csv', nrows=None, usecols=[0,6,7],dtype={0:np.uint32,6:np.uint16,7:np.float64},index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "testdata.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6648023943900963"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testdata.shape[0]/float(totaltestrows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>orig_destination_distance</th>\n",
       "      <th>user_location_city</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5539.0567</td>\n",
       "      <td>37449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5873.2923</td>\n",
       "      <td>37449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3975.9776</td>\n",
       "      <td>17440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1508.5975</td>\n",
       "      <td>34156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>66.7913</td>\n",
       "      <td>36345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>359.8521</td>\n",
       "      <td>48189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>237.3465</td>\n",
       "      <td>48189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>216.5785</td>\n",
       "      <td>24811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2337.6754</td>\n",
       "      <td>48189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2539.7995</td>\n",
       "      <td>48189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2660.9654</td>\n",
       "      <td>48189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2452.2781</td>\n",
       "      <td>24103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>7449.5906</td>\n",
       "      <td>36086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>25.8282</td>\n",
       "      <td>5938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>534.3984</td>\n",
       "      <td>39058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>911.1921</td>\n",
       "      <td>20170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>3974.8078</td>\n",
       "      <td>41495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1383.5711</td>\n",
       "      <td>28950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>382.0833</td>\n",
       "      <td>1271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>28.2090</td>\n",
       "      <td>8978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>191.2790</td>\n",
       "      <td>8657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>279.5189</td>\n",
       "      <td>8657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>53.1477</td>\n",
       "      <td>30455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>4708.7364</td>\n",
       "      <td>50661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>7755.2049</td>\n",
       "      <td>14703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>7342.3791</td>\n",
       "      <td>14703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>98.4142</td>\n",
       "      <td>55926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1514.4627</td>\n",
       "      <td>1803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>219.3531</td>\n",
       "      <td>21728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1035.2867</td>\n",
       "      <td>24562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528207</th>\n",
       "      <td>720.4032</td>\n",
       "      <td>38374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528208</th>\n",
       "      <td>919.8944</td>\n",
       "      <td>38374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528209</th>\n",
       "      <td>955.8293</td>\n",
       "      <td>46963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528211</th>\n",
       "      <td>666.1290</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528212</th>\n",
       "      <td>61.4925</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528213</th>\n",
       "      <td>482.1640</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528214</th>\n",
       "      <td>561.1356</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528215</th>\n",
       "      <td>524.9776</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528216</th>\n",
       "      <td>98.9595</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528217</th>\n",
       "      <td>134.8539</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528218</th>\n",
       "      <td>7759.9406</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528219</th>\n",
       "      <td>244.3949</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528220</th>\n",
       "      <td>780.8954</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528221</th>\n",
       "      <td>654.8646</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528222</th>\n",
       "      <td>1814.9161</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528223</th>\n",
       "      <td>1824.0325</td>\n",
       "      <td>42396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528224</th>\n",
       "      <td>184.1005</td>\n",
       "      <td>17552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528225</th>\n",
       "      <td>5187.0656</td>\n",
       "      <td>2086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528226</th>\n",
       "      <td>448.0108</td>\n",
       "      <td>40604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528227</th>\n",
       "      <td>274.9252</td>\n",
       "      <td>40604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528231</th>\n",
       "      <td>221.9894</td>\n",
       "      <td>35390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528233</th>\n",
       "      <td>25.8312</td>\n",
       "      <td>13982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528234</th>\n",
       "      <td>2300.3771</td>\n",
       "      <td>40193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528235</th>\n",
       "      <td>1766.4050</td>\n",
       "      <td>19630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528236</th>\n",
       "      <td>296.8726</td>\n",
       "      <td>47766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528237</th>\n",
       "      <td>2369.5146</td>\n",
       "      <td>18037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528238</th>\n",
       "      <td>2466.1247</td>\n",
       "      <td>18037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528239</th>\n",
       "      <td>4701.6346</td>\n",
       "      <td>48951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528240</th>\n",
       "      <td>1077.2278</td>\n",
       "      <td>17078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2528241</th>\n",
       "      <td>266.7678</td>\n",
       "      <td>28561</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1680782 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         orig_destination_distance  user_location_city\n",
       "id                                                    \n",
       "0                        5539.0567               37449\n",
       "1                        5873.2923               37449\n",
       "2                        3975.9776               17440\n",
       "3                        1508.5975               34156\n",
       "4                          66.7913               36345\n",
       "5                         359.8521               48189\n",
       "6                         237.3465               48189\n",
       "7                         216.5785               24811\n",
       "8                        2337.6754               48189\n",
       "9                        2539.7995               48189\n",
       "10                       2660.9654               48189\n",
       "25                       2452.2781               24103\n",
       "26                       7449.5906               36086\n",
       "27                         25.8282                5938\n",
       "28                        534.3984               39058\n",
       "29                        911.1921               20170\n",
       "30                       3974.8078               41495\n",
       "32                       1383.5711               28950\n",
       "33                        382.0833                1271\n",
       "34                         28.2090                8978\n",
       "35                        191.2790                8657\n",
       "36                        279.5189                8657\n",
       "37                         53.1477               30455\n",
       "38                       4708.7364               50661\n",
       "40                       7755.2049               14703\n",
       "41                       7342.3791               14703\n",
       "42                         98.4142               55926\n",
       "43                       1514.4627                1803\n",
       "44                        219.3531               21728\n",
       "45                       1035.2867               24562\n",
       "...                            ...                 ...\n",
       "2528207                   720.4032               38374\n",
       "2528208                   919.8944               38374\n",
       "2528209                   955.8293               46963\n",
       "2528211                   666.1290               42396\n",
       "2528212                    61.4925               42396\n",
       "2528213                   482.1640               42396\n",
       "2528214                   561.1356               42396\n",
       "2528215                   524.9776               42396\n",
       "2528216                    98.9595               42396\n",
       "2528217                   134.8539               42396\n",
       "2528218                  7759.9406               42396\n",
       "2528219                   244.3949               42396\n",
       "2528220                   780.8954               42396\n",
       "2528221                   654.8646               42396\n",
       "2528222                  1814.9161               42396\n",
       "2528223                  1824.0325               42396\n",
       "2528224                   184.1005               17552\n",
       "2528225                  5187.0656                2086\n",
       "2528226                   448.0108               40604\n",
       "2528227                   274.9252               40604\n",
       "2528231                   221.9894               35390\n",
       "2528233                    25.8312               13982\n",
       "2528234                  2300.3771               40193\n",
       "2528235                  1766.4050               19630\n",
       "2528236                   296.8726               47766\n",
       "2528237                  2369.5146               18037\n",
       "2528238                  2466.1247               18037\n",
       "2528239                  4701.6346               48951\n",
       "2528240                  1077.2278               17078\n",
       "2528241                   266.7678               28561\n",
       "\n",
       "[1680782 rows x 2 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testdata[['orig_destination_distance','user_location_city']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code takes advantage of the fact that sets are refered to by reference in python. We can pass this set to another data frame, but all it is passing is something that says \"look in this spot in memory for this object\". This means that `[set()]*testdata.shape[0]` will not work for our purposes, as it creates a list, where each element is the exact same object in memory. Thus, if we change one element in the list, we'll change every one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "testdata['hotel_cluster']=[set() for i in xrange(testdata.shape[0])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This creates an iterator over the training data, using just the columns we want. Each element of the iterator is a `DataFrame`. We can than iterate over it, extract and use the info we want from each chunk, after which it is discarded from memory and the next chunk is loaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nchunks=8\n",
    "chunksize=ceil(float(totaltrainrows)/nchunks)\n",
    "trainit=pd.read_csv(datapath+'train.csv',iterator=True,chunksize=chunksize,usecols=[5,6,23],dtype={5:np.uint16,6:np.float64,23:np.uint8})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def add_cluster(x):\n",
    "    x['hotel_cluster_tst'].add(x['hotel_cluster_tr'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because the sets are objects passed by reference, we can modify them in the data frame created in the for loop, and they will also get modified in `testdata`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "CPU times: user 19min 8s, sys: 15.7 s, total: 19min 24s\n",
      "Wall time: 19min 20s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "nchunk=0\n",
    "for chunk in trainit:\n",
    "    chunk=pd.merge(chunk,testdata.reset_index(),on=['orig_destination_distance','user_location_city'],how='inner',suffixes=['_tr','_tst'])\n",
    "    chunk.apply(add_cluster,axis=1)\n",
    "    print(nchunk)\n",
    "    nchunk+=1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Garbage collection doesn't work well in an interactive session, so I can save some memory like so:\n",
    "chunk=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "settostring=lambda x: ' '.join(str(i) for i in x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When we convert the sets to booleans, empty sets are `False`, all others are `True`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "prediction=testdata[testdata['hotel_cluster'].astype(bool)]['hotel_cluster'].map(settostring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "prediction.to_csv(datapath+'dist_predict.csv',index=True,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5073299214294299"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction.shape[0]/float(testdata.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are, however, some data points where this method doesn't work perfectly. This is probably because hotels can change their cluster seasonally, or as aspects of the hotel change. I haven't yet figured out what to do with these."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "checklen=lambda x: len(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bads=testdata[testdata['hotel_cluster'].apply(len)>5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22, 3)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bads.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are only 8 actual problems to deal with though:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "orig_destination_distance\n",
       "55.7481      10\n",
       "61.5983       4\n",
       "69.1495       2\n",
       "141.5581      1\n",
       "195.8534      2\n",
       "1698.9480     1\n",
       "2375.0267     1\n",
       "3378.0289     1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bads.groupby('orig_destination_distance').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_location_city</th>\n",
       "      <th>orig_destination_distance</th>\n",
       "      <th>hotel_cluster</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>40071</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47815</th>\n",
       "      <td>27655</td>\n",
       "      <td>2375.0267</td>\n",
       "      <td>{3, 83, 20, 85, 57, 30, 53}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170559</th>\n",
       "      <td>28561</td>\n",
       "      <td>61.5983</td>\n",
       "      <td>{73, 25, 89, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244716</th>\n",
       "      <td>2086</td>\n",
       "      <td>195.8534</td>\n",
       "      <td>{3, 20, 53, 57, 60, 93, 30, 85}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278646</th>\n",
       "      <td>28561</td>\n",
       "      <td>61.5983</td>\n",
       "      <td>{73, 25, 89, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451279</th>\n",
       "      <td>32561</td>\n",
       "      <td>141.5581</td>\n",
       "      <td>{40, 41, 44, 83, 86, 90, 60, 31}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>646412</th>\n",
       "      <td>35390</td>\n",
       "      <td>3378.0289</td>\n",
       "      <td>{64, 99, 36, 9, 46, 80}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>690018</th>\n",
       "      <td>28561</td>\n",
       "      <td>61.5983</td>\n",
       "      <td>{73, 25, 89, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>835472</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>944406</th>\n",
       "      <td>51733</td>\n",
       "      <td>1698.9480</td>\n",
       "      <td>{98, 80, 21, 25, 59, 95}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1007332</th>\n",
       "      <td>24562</td>\n",
       "      <td>69.1495</td>\n",
       "      <td>{98, 68, 70, 25, 28, 95}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1032008</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1291974</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1301232</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1357583</th>\n",
       "      <td>2086</td>\n",
       "      <td>195.8534</td>\n",
       "      <td>{3, 20, 53, 57, 60, 93, 30, 85}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1497455</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1567900</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1950882</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2087261</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2186077</th>\n",
       "      <td>2037</td>\n",
       "      <td>55.7481</td>\n",
       "      <td>{73, 89, 25, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2450016</th>\n",
       "      <td>28561</td>\n",
       "      <td>61.5983</td>\n",
       "      <td>{73, 25, 89, 90, 59, 30}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2487747</th>\n",
       "      <td>24562</td>\n",
       "      <td>69.1495</td>\n",
       "      <td>{98, 68, 70, 25, 28, 95}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_location_city  orig_destination_distance  \\\n",
       "id                                                       \n",
       "40071                  2037                    55.7481   \n",
       "47815                 27655                  2375.0267   \n",
       "170559                28561                    61.5983   \n",
       "244716                 2086                   195.8534   \n",
       "278646                28561                    61.5983   \n",
       "451279                32561                   141.5581   \n",
       "646412                35390                  3378.0289   \n",
       "690018                28561                    61.5983   \n",
       "835472                 2037                    55.7481   \n",
       "944406                51733                  1698.9480   \n",
       "1007332               24562                    69.1495   \n",
       "1032008                2037                    55.7481   \n",
       "1291974                2037                    55.7481   \n",
       "1301232                2037                    55.7481   \n",
       "1357583                2086                   195.8534   \n",
       "1497455                2037                    55.7481   \n",
       "1567900                2037                    55.7481   \n",
       "1950882                2037                    55.7481   \n",
       "2087261                2037                    55.7481   \n",
       "2186077                2037                    55.7481   \n",
       "2450016               28561                    61.5983   \n",
       "2487747               24562                    69.1495   \n",
       "\n",
       "                            hotel_cluster  \n",
       "id                                         \n",
       "40071            {73, 89, 25, 90, 59, 30}  \n",
       "47815         {3, 83, 20, 85, 57, 30, 53}  \n",
       "170559           {73, 25, 89, 90, 59, 30}  \n",
       "244716    {3, 20, 53, 57, 60, 93, 30, 85}  \n",
       "278646           {73, 25, 89, 90, 59, 30}  \n",
       "451279   {40, 41, 44, 83, 86, 90, 60, 31}  \n",
       "646412            {64, 99, 36, 9, 46, 80}  \n",
       "690018           {73, 25, 89, 90, 59, 30}  \n",
       "835472           {73, 89, 25, 90, 59, 30}  \n",
       "944406           {98, 80, 21, 25, 59, 95}  \n",
       "1007332          {98, 68, 70, 25, 28, 95}  \n",
       "1032008          {73, 89, 25, 90, 59, 30}  \n",
       "1291974          {73, 89, 25, 90, 59, 30}  \n",
       "1301232          {73, 89, 25, 90, 59, 30}  \n",
       "1357583   {3, 20, 53, 57, 60, 93, 30, 85}  \n",
       "1497455          {73, 89, 25, 90, 59, 30}  \n",
       "1567900          {73, 89, 25, 90, 59, 30}  \n",
       "1950882          {73, 89, 25, 90, 59, 30}  \n",
       "2087261          {73, 89, 25, 90, 59, 30}  \n",
       "2186077          {73, 89, 25, 90, 59, 30}  \n",
       "2450016          {73, 25, 89, 90, 59, 30}  \n",
       "2487747          {98, 68, 70, 25, 28, 95}  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bads"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scrap\n",
    "Here's an example of how to take in a small number of rows for experimentation with the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.rea"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "chunk=pd.read_csv(datapath+'train.csv',nrows=100,usecols=[5,6,23],dtype={5:np.uint16,6:np.float64,23:np.uint8})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_location_city</th>\n",
       "      <th>orig_destination_distance</th>\n",
       "      <th>hotel_cluster</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>48862</td>\n",
       "      <td>2234.2641</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48862</td>\n",
       "      <td>2234.2641</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>48862</td>\n",
       "      <td>2234.2641</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35390</td>\n",
       "      <td>913.1932</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35390</td>\n",
       "      <td>913.6259</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_location_city  orig_destination_distance  hotel_cluster\n",
       "0               48862                  2234.2641              1\n",
       "1               48862                  2234.2641              1\n",
       "2               48862                  2234.2641              1\n",
       "3               35390                   913.1932             80\n",
       "4               35390                   913.6259             21"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chunk.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}