Network Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network analysis with NetworkX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "\n",
    "from globes import taxi_dir, days_dir, getFullDf, zoneIdToBorough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using 28 cores\n"
     ]
    }
   ],
   "source": [
    "ZONE_COLS = [\n",
    "    \"pickup_day\",\n",
    "    \"pickup_hour\",\n",
    "    \"pickup_zone_taxi\", \n",
    "    \"dropoff_zone_taxi\", \n",
    "    \"pickup_borough\", \n",
    "    \"dropoff_borough\"\n",
    "]\n",
    "\n",
    "zone_nodes = [i for i in range(1, 264) if i not in [57, 104, 105]]\n",
    "\n",
    "df = getFullDf()\n",
    "df = df[ZONE_COLS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "## functions for loading graphs\n",
    "def loadNodes(G):\n",
    "    # load nodes\n",
    "    BoroughDict = zoneIdToBorough()\n",
    "    nodes = {}\n",
    "    for zoneid in zone_nodes:\n",
    "        borough = BoroughDict.get(zoneid) if BoroughDict.get(zoneid) else \"None\"\n",
    "        if nodes.get(borough):\n",
    "            nodes[borough].append(zoneid)\n",
    "        else:\n",
    "            nodes[borough] = [zoneid]\n",
    "    \n",
    "    for borough in nodes:\n",
    "        G.add_nodes_from(nodes[borough], borough=borough)\n",
    "    \n",
    "    return G\n",
    "        \n",
    "def loadDirectedEdges(G):\n",
    "    edges = []\n",
    "    for pickup_zone in zone_nodes:\n",
    "        df_zone = df.loc[df[\"pickup_zone_taxi\"] == pickup_zone]\n",
    "        pickup_group = df_zone.groupby(\"dropoff_zone_taxi\").size()\n",
    "        for dropoff_zone, count in pickup_group.iteritems():\n",
    "            edges.append((pickup_zone, dropoff_zone, count))\n",
    "            \n",
    "    G.add_weighted_edges_from(edges)\n",
    "    return G\n",
    "\n",
    "def loadUndirectedEdges(G):\n",
    "    edgeDict = {}\n",
    "    # load edgeDict: (smaller node, bigger node) => agg flow\n",
    "    for pickup_zone in zone_nodes:\n",
    "        df_zone = df.loc[df[\"pickup_zone_taxi\"] == pickup_zone]\n",
    "        pickup_group = df_zone.groupby(\"dropoff_zone_taxi\").size()\n",
    "        for dropoff_zone, count in pickup_group.iteritems():\n",
    "            edge = (min(dropoff_zone, pickup_zone), max(dropoff_zone, pickup_zone))\n",
    "            if edge in edgeDict:\n",
    "                edgeDict[edge] += count\n",
    "            else:\n",
    "                edgeDict[edge] = count\n",
    "\n",
    "    edges = [(e1, e2, edgeDict[(e1, e2)]) for e1,e2 in edgeDict]\n",
    "    G.add_weighted_edges_from(edges)\n",
    "    return G\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "260\n",
      "260\n"
     ]
    }
   ],
   "source": [
    "## load digraph\n",
    "G = nx.MultiDiGraph()\n",
    "G = loadNodes(G)\n",
    "G = loadDirectedEdges(G)\n",
    "\n",
    "## undirected graph, sum of flow between \n",
    "G_aggflow = nx.MultiGraph()\n",
    "G_aggflow = loadNodes(G_aggflow)\n",
    "G_aggflow = loadUndirectedEdges(G_aggflow)\n",
    "\n",
    "print len(G.nodes())\n",
    "print len(G_aggflow.nodes())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Measures of centrality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def boroughAgg(G, centrality_data):\n",
    "    agg = {}\n",
    "    for zoneid in centrality_data:\n",
    "        borough = G.node[zoneid][\"borough\"]\n",
    "        if not borough in agg:\n",
    "            agg[borough] = [centrality_data[zoneid]]\n",
    "        else:\n",
    "            agg[borough].append(centrality_data[zoneid])\n",
    "    \n",
    "    # get means for each borough\n",
    "    for borough in agg:\n",
    "        agg[borough] = sum(agg[borough])/len(agg[borough])\n",
    "    return agg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'Staten Island': 0.45598455598455595, u'Brooklyn': 1.4281916576998546, u'Bronx': 1.1753614079195474, u'EWR': 0.8918918918918919, u'Manhattan': 1.6707773871952976, u'Queens': 1.2869634340222578}\n"
     ]
    }
   ],
   "source": [
    "deg_centrality = nx.degree_centrality(G)\n",
    "print boroughAgg(G, deg_centrality)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'Staten Island': 0.3790726128126737, u'Brooklyn': 0.4761784180680855, u'Bronx': 0.4857381445962359, u'EWR': 0.40859119396480287, u'Manhattan': 0.37056875737437855, u'Queens': 0.4702500054474006}\n"
     ]
    }
   ],
   "source": [
    "closeness_centrality = nx.closeness_centrality(G, distance=\"weight\")\n",
    "print boroughAgg(G, closeness_centrality)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'Staten Island': 0.0004682492370728995, u'Brooklyn': 0.0013879368041661217, u'Bronx': 0.000454884920034402, u'EWR': 0.00023518068684266527, u'Manhattan': 0.0022433698782293667, u'Queens': 0.0010449833194460766}\n"
     ]
    }
   ],
   "source": [
    "betweenness_centrality = nx.betweenness_centrality(G, weight=\"weight\")\n",
    "print boroughAgg(G, betweenness_centrality)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'Staten Island': 0.38687258687258697, u'Brooklyn': 0.8141654535097158, u'Bronx': 0.6866301517464307, u'EWR': 0.7490347490347491, u'Manhattan': 0.9298104074223474, u'Queens': 0.7621508062684536}\n",
      "{u'Staten Island': 0.5952008448493189, u'Brooklyn': 0.8454540651787196, u'Bronx': 0.7621253250993933, u'EWR': 0.7932217932217932, u'Manhattan': 0.9380565641375687, u'Queens': 0.8152363495637065}\n",
      "{u'Staten Island': 0.00014428834231732475, u'Brooklyn': 0.0008336645781396832, u'Bronx': 0.0002710839050134295, u'EWR': 0.00037929385337888546, u'Manhattan': 0.0017771252999865773, u'Queens': 0.0005991439028553201}\n"
     ]
    }
   ],
   "source": [
    "deg_centrality = nx.degree_centrality(G_aggflow)\n",
    "closeness_centrality = nx.closeness_centrality(G_aggflow)\n",
    "betweenness_centrality = nx.betweenness_centrality(G_aggflow)\n",
    "\n",
    "print boroughAgg(G_aggflow, deg_centrality)\n",
    "print boroughAgg(G_aggflow, closeness_centrality)\n",
    "print boroughAgg(G_aggflow, betweenness_centrality)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}