01-GeoData Download and Prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Geo-Data Data Downloading & Preparation\n",
    "\n",
    "This notebook creates the relevant geo-data files by downloading data from the London Data Store. You should only need to run this notebook once, no matter how many times you wish to work with the rest of the model and its inputs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "mpl.use('TkAgg')\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# For reproducibility\n",
    "import random\n",
    "import numpy as np\n",
    "r_state = 42\n",
    "random.seed(r_state) \n",
    "np.random.seed(r_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pysal as ps\n",
    "import geopandas as gpd\n",
    "import requests\n",
    "import glob\n",
    "import re\n",
    "import os\n",
    "import io\n",
    "import zipfile\n",
    "\n",
    "from io import BytesIO\n",
    "\n",
    "lkp = os.path.join('data','lkp')\n",
    "shp = os.path.join('data','shp')\n",
    "for d in [lkp,shp]:\n",
    "    if not os.path.exists(d):\n",
    "        os.makedirs(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Make sure you always run this!\n",
    "boroughs = ['City of London','Barking and Dagenham','Barnet','Bexley','Brent','Bromley',\n",
    "            'Camden','Croydon','Ealing','Enfield','Greenwich','Hackney','Hammersmith and Fulham',\n",
    "            'Haringey','Harrow','Havering','Hillingdon','Hounslow','Islington',\n",
    "            'Kensington and Chelsea','Kingston upon Thames','Lambeth','Lewisham',\n",
    "            'Merton','Newham','Redbridge','Richmond upon Thames','Southwark','Sutton',\n",
    "            'Tower Hamlets','Waltham Forest','Wandsworth','Westminster']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Downloading geo-data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "shpt = os.path.join(shp,'tmp')\n",
    "if not os.path.exists(shpt):\n",
    "    os.makedirs(shpt)\n",
    "\n",
    "regions2016 = ('https://opendata.arcgis.com/datasets/'\n",
    "               'f99b145881724e15a04a8a113544dfc5_2.zip')\n",
    "gla2015 = ('https://files.datapress.com/london/dataset/'\n",
    "           'statistical-gis-boundary-files-london/2016-10-03T13:52:28/'\n",
    "           'statistical-gis-boundaries-london.zip')\n",
    "#lsoa2011full = ('http://geoportal.statistics.gov.uk/datasets/da831f80764346889837c72508f046fa_1')\n",
    "lsoa2011generalised = ('https://geoportal.statistics.gov.uk/datasets/da831f80764346889837c72508f046fa_2.zip')\n",
    "    \n",
    "for f in [regions2016, gla2015, lsoa2011generalised]:\n",
    "    print(\"Downloading \" + f + \"...\")\n",
    "    r = requests.get(f, stream=True)\n",
    "    z = zipfile.ZipFile(BytesIO(r.content))\n",
    "    z.extractall(shpt)\n",
    "\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selecting Regional Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "regions = glob.glob(os.path.join(shpt,'*Regions*.shp'))[0]\n",
    "\n",
    "print(\"Processing: \" + regions)\n",
    "regions = gpd.read_file(regions)\n",
    "\n",
    "london  = regions[regions.rgn16nm=='London']\n",
    "london.reset_index(inplace=True, drop=True)\n",
    "london.crs = {'init':'epsg:4326'}\n",
    "london = london.to_crs(epsg=27700)\n",
    "london.to_file(os.path.join(shp,'London.shp'))\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selecting Boroughs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "counties = glob.glob(os.path.join(shpt,'statistical-gis-boundaries-london','ESRI','*Borough*.shp'))[0]\n",
    "\n",
    "print(\"Processing: \" + counties)\n",
    "LAs = gpd.read_file(counties)\n",
    "\n",
    "LAs = LAs.loc[LAs.NAME.isin(boroughs)].reset_index(drop=True)\n",
    "LAs.crs = {'init': u'epsg:27700'}\n",
    "#LAs = LAs.to_crs({'init':'epsg:27700'})\n",
    "\n",
    "print(\"\\tSaving to shapefile...\")\n",
    "LAs.to_file(os.path.join(shp,'Boroughs.shp'))\n",
    "\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selecting LSOAs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lsoas = glob.glob(os.path.join(shpt,'statistical-gis-boundaries-london','ESRI','*LSOA*.shp'))\n",
    "\n",
    "for l in lsoas:\n",
    "    print(\"Processing: \" + l)\n",
    "    lsoa_y = gpd.read_file(l)\n",
    "    \n",
    "    # Extract the year as 4 digits\n",
    "    m     = re.search(r'\\d{4}',l)\n",
    "    lyear = l[m.start():m.end()]\n",
    "    \n",
    "    # Set projection\n",
    "    lsoa_y.crs = {'init':'epsg:27700'}\n",
    "    \n",
    "    # Common name\n",
    "    lsoa_y.insert(0, 'lsoacd', \n",
    "                    lsoa_y[[x for x in lsoa_y.columns if 'LSOA' in x and ('CD' in x or 'CODE' in x)][0]])\n",
    "    \n",
    "    print(\"\\tSaving to shapefile...\")\n",
    "    lsoa_y.to_file(os.path.join(shp,'LSOAs ' + str(lyear) + '.shp'))\n",
    "    \n",
    "    print(\"\\tSaving to pickle...\")\n",
    "    lsoa_y.to_pickle(os.path.join(lkp,'LSOAs ' + str(lyear) + '.pkl'))\n",
    "\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dealing with LSOA data issue\n",
    "\n",
    "For some reason the LSOA data from the GLA, while fine to map, doesn't work for calculating the weights matrix -- presumably there is _some_ kind of issue with the shape file provided but I have been unable to work out what it is. To work around this we extract the London data from the ONS download processed above (`lsoa2011generalised`).\n",
    "\n",
    "If all has gone well you should get the following output:\n",
    "- Mean neighbours: 5.93\n",
    "- Max neighbours: 15\n",
    "- Min neighbours: 1\n",
    "- Islands: 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsoag  = glob.glob(os.path.join(shpt,'*2011_Generalised_Clipped*.shp'))[0] # LSOAs Generalised\n",
    "\n",
    "print(\"Processing: \" + lsoag)\n",
    "lsoa_g = gpd.read_file(lsoag)\n",
    "\n",
    "lsoa_g['borough'] = lsoa_g.lsoa11nm.str.extract(\"^(.+?) [0-9A-Z]{4}\") # Extract borough/council names for subsetting\n",
    "lsoa_g = lsoa_g[lsoa_g.borough.isin(boroughs)] # And select only those boroughs that match the array create above\n",
    "print(\"\\tExtracted \" + str(len(lsoa_g)) + \" London LSOAs.\")\n",
    "\n",
    "print(\"\\tSaving to shapefile...\")\n",
    "lsoa_g.to_file(os.path.join(shp,'LSOA-Weights.shp'))\n",
    "\n",
    "w = ps.weights.Queen.from_shapefile(os.path.join(shp,'LSOA-Weights.shp'))\n",
    "print(\"Check these results against above information:\")\n",
    "print(\"\\tMean neighbours: \" + str(w.mean_neighbors))\n",
    "print(\"\\tMax neighbours:  \" + str(w.max_neighbors))\n",
    "print(\"\\tMin neighbours:  \" + str(w.min_neighbors))\n",
    "print(\"\\tNo. Islands:     \" + str(len(w.islands)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selecting and Joining Wards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wards = glob.glob(os.path.join(shpt,'statistical-gis-boundaries-london','ESRI','*Ward*Merged.shp'))[0]\n",
    "\n",
    "print(\"Processing wards...\")\n",
    "ward_geo = gpd.read_file(wards)\n",
    "ward_geo.crs = {'init':'epsg:27700'}\n",
    "    \n",
    "print(\"\\tSaving to shapefile...\")\n",
    "ward_geo.to_file(os.path.join(shp,'Wards.shp'))\n",
    "\n",
    "# Create a mapping for LSOAs to Wards\n",
    "lsoa = gpd.read_file(os.path.join(shp,'LSOAs 2011.shp'))\n",
    "lsoa.crs = {'init':'epsg:27700'}\n",
    "    \n",
    "lsoa_c = lsoa\n",
    "lsoa_c.geometry = lsoa_c.centroid\n",
    "lsoa_c.to_file(os.path.join(shp,'LSOAs 2011 Points.shp'))\n",
    "\n",
    "print(\"\\tJoining Wards to LSOAs...\")\n",
    "t = gpd.sjoin(lsoa_c, ward_geo, how='left')\n",
    "t.rename(columns={\n",
    "    'GSS_CODE':'gss_cd',\n",
    "    'LB_GSS_CD':'lb_gss_cd'\n",
    "}, inplace=True)\n",
    "t[['lsoacd','gss_cd','lb_gss_cd']].to_csv(os.path.join(lkp,'LSOA_WARD_JR.csv'), index=False)\n",
    "\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selecting and Joining Output Areas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "oas = glob.glob(os.path.join(shpt,'statistical-gis-boundaries-london','ESRI','OA_*.shp'))[0]\n",
    "\n",
    "print(\"Processing Output Areas...\")\n",
    "oa_geo = gpd.read_file(oas)\n",
    "oa_geo.crs = {'init':'epsg:27700'}\n",
    "    \n",
    "print(\"\\tSaving to shapefile...\")\n",
    "oa_geo.to_file(os.path.join(shp,'OAs 2011.shp'))\n",
    "\n",
    "# Create a mapping for LSOAs to Wards\n",
    "lsoa = gpd.read_file(os.path.join(shp,'LSOAs 2011.shp'))\n",
    "lsoa.crs = {'init':'epsg:27700'}\n",
    "    \n",
    "oa_c = oa_geo\n",
    "oa_c.geometry = oa_c.centroid\n",
    "\n",
    "print(\"\\tSaving point OAs...\")\n",
    "oa_c.to_file(os.path.join(shp,'OAs 2011 Points.shp'))\n",
    "\n",
    "print(\"\\tOutput Areas to LSOAs...\")\n",
    "\n",
    "oa_geo.rename(columns={\n",
    "        'LSOA11CD':'lsoacd',\n",
    "        'OA11CD':'oacd'\n",
    "    }, inplace=True)\n",
    "oa_geo[['lsoacd','oacd']].to_csv(os.path.join(lkp,'LSOA_OA_JR.csv'), index=False)\n",
    "\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tidying up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import shutil \n",
    "shutil.rmtree(shpt)\n",
    "print(\"Done.\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "ML Gentrification 3",
   "language": "python",
   "name": "mlgent"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}