deepLearningWorkshopDay2CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#the most important imports!\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "#used for generating files for tensorboard\n",
    "import sys\n",
    "import urllib.request\n",
    "import os\n",
    "\n",
    "#used for visualizing\n",
    "import matplotlib.pyplot as plt\n",
    "#used to grab random samples to visualize\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "if sys.version_info[0] >= 3:\n",
    "    from urllib.request import urlretrieve\n",
    "else:\n",
    "    from urllib import urlretrieve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create the log directory for tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOGDIR = './tf_logs/mnist_deep' "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### weight and bias functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weight(shape):\n",
    "    \"\"\"Generates a weight variable of a given shape\"\"\"\n",
    "    return tf.Variable(tf.truncated_normal(shape, stddev=.1), name=\"weight\")\n",
    "\n",
    "def bias(shape):\n",
    "    \"\"\"Generates a bias variable of a given shape\"\"\"\n",
    "    return tf.Variable(tf.constant(.1, shape=shape), name=\"bias\")\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "old_v = tf.logging.get_verbosity()\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "X = tf.placeholder(tf.float32, [None, 784], name=\"X\")\n",
    "_y = tf.placeholder(tf.float32, [None, 10], name=\"labels\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- there are 10000 images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mnist.test.images)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- here we see that each image is stored as a single vector of length 784\n",
    "- we will need to reshape that to (28,28) for visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(784,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.test.images[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mnist.test.images[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets take a look at a sample of the dataset\n",
    "- rerun this block to look at different samples!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeAAAAHVCAYAAAApYyiLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XucVXW9//H3Z4YBFFC5KCGSeAFvpZgTXkvLS8qx0PKYZkblEUstLSvJ0znazTy/vFRqGiaBZVqJppmaSKaZeAFEBVHxggohiKLiBWRmPr8/Zutjvvu7Ye/Ze6+19p79ej4ePJjPh+9e69P4ab6z9vru7zJ3FwAASFdT1gUAANCImIABAMgAEzAAABlgAgYAIANMwAAAZIAJGACADDABAwCQASZgAAAyUNEEbGaHmtkTZvaUmU2qVlHAhtB3SBs9hyRYuTthmVmzpCclHSxpiaQHJR3r7o+t7zW9rY/3Vb+yzof6t1qrVrr75pUcg75Dd6zRm3rH11olx6Dn0F2l/qzrVcE5xkp6yt2fkSQzu1bSeEnrbcq+6qc97cAKTol6dodf91wVDkPfoWT3+8xqHIaeQ7eU+rOukregh0t6oUu8JJcLmNlEM5ttZrPXaW0FpwMk0XdIHz2HRCS+CMvdJ7t7q7u3tqhP0qcDJNF3SB89h+6qZAJeKmlEl3irXA5IEn2HtNFzSEQlE/CDkkaZ2TZm1lvSMZJuqk5ZwHrRd0gbPYdElL0Iy93bzOxUSX+T1CxpirsvqFplQAH0HdJGzyEplayClrvfIumWKtUClIS+Q9roOSSBnbAAAMgAEzAAABlgAgYAIANMwAAAZIAJGACADDABAwCQASZgAAAywAQMAEAGmIABAMgAEzAAABlgAgYAIANMwAAAZIAJGACADDABAwCQgYoeR2hmiyWtltQuqc3dW6tRFLAh9B3SRs8hCRVNwDkfc/eVVThOj9Lyj2FR7ubRt0a5Y579eBCv+shr8cE62qtWVw9S13339hFjg3ijPz+QUSXrt+LkfYL4M1/5ezTmzMHhc+kPPOWr0Zha/N9Wprruuaw9/dO9g3jB534RjRl3/FeCuNff5yRaU9Z4CxoAgAxUOgG7pNvNbI6ZTSw0wMwmmtlsM5u9TmsrPB0gib5D+ug5VF2lb0Hv5+5LzWwLSTPM7HF3v7vrAHefLGmyJG1ig7zC8wESfYf00XOouoomYHdfmvt7hZndIGmspLs3/KqeqXnUtkF87LC7ojHt3hHlrh55RxAfdHD8y3Xvv82usLqepd767p0ZW0e5X4++KIgf/+nm0Zhv3vSFIN7ujPuqW1gXzTtsH+VOOvXGIP7SpoujMfkd/cK4eN4Z/edKKqsN9dZzteiQAx4K4l5qjsYs26dPEI+Ilx30KGW/BW1m/cxswLtfSzpE0vxqFQYUQt8hbfQcklLJFfBQSTeY2bvH+b2731aVqoD1o++QNnoOiSh7Anb3ZyTtVsVagKLoO6SNnkNS+BgSAAAZqMZGHJD0zlabBfEx/V/KqBLUml02ezHKbd2rd14cb8DyqyOuCOILf3ZYNKbthSVl1bT2sA8H8R4/ijc8KLToqpi+S1rKqgeQpD6rsq4gXVwBAwCQASZgAAAywAQMAEAGmIABAMgAi7CAGvXRvu8E8Ynf2ioas+Mv+wZx2+B+0ZhX//utKPevMZcHcUe0p1VsjbdFuTEzTg3i0T+4t+hxgPUZel+4GLGn7+fJFTAAABlgAgYAIANMwAAAZIB7wECdWHjUxXHyqHKP1v3fvfPv90rS6C/FG3gATRtvHOWOG3xPELepPRpj7eFdX+4BAwCAqmMCBgAgA0zAAABkoOgEbGZTzGyFmc3vkhtkZjPMbFHu74HJlolGQ98hC/Qd0lTKIqypki6RdFWX3CRJM939PDOblIvPrH559aPPk+ETb37y8s7RmO8OfiytcnqCqeohfddk8SYXTSW8+dRizUG8roorUko59iEnfCWIR9/2YPUKqF1T1UP6LkttH94hyu3bN9ykZfobm0VjOuY11s/Ioj8F3P1uSa/kpcdLmpb7epqkI6pcFxocfYcs0HdIU7kfQxrq7styX78oaej6BprZREkTJamv4qXpQDfQd8hCSX1Hz6G7Kl6E5e6uDXxcy90nu3uru7e2qE+lpwMk0XfIxob6jp5Dd5V7BbzczIa5+zIzGyZpRTWLqkdtS/8dxPe+vG08iHvAlarLvuvw+PfcUh5+kH9ftpTX/LttbZQ76pEvR7k1s4YE8dZ/ejEa0/eFR/PO37Dqsu+y9NKuG0W5dm/gDlqPcq+Ab5I0Iff1BEk3VqccYIPoO2SBvkMiSvkY0jWSZknawcyWmNkJks6TdLCZLZJ0UC4Gqoa+QxboO6Sp6FvQ7n7sev7pwCrXAryHvkMW6DukiZ2wAADIAE9DqpKm3XYK4klbX5NRJegpxs4+Log3+2X/oq9pXhsvdBnyj7kFRj4ZRPFzaYDyrd7z7aJjvnv9cVFuG81KopyaxRUwAAAZYAIGACADTMAAAGSAe8BVsm5Q+MHzffvwoXNU5quj7w7i61YfHI3p9fKbYWJZvEcE93eRpF7D3hflfjL2+qKv2/43L0W5RutVroABAMgAEzAAABlgAgYAIANMwAAAZIBFWECN+tImLwTxhD/9uuhrxi38TJR75pkdotzoiQ+WXxjQxfLDt4lyn+l3S5S78vWtgrhj8QvRmEbDFTAAABlgAgYAIANMwAAAZKCU5wFPMbMVZja/S+4cM1tqZvNyf8YlWyYaDX2HtNFzSFspi7CmSrpE0lV5+Yvc/fyqVwR0mqoe0nfPfHFklPv+74o/LebcoY8E8Tovfq7bdrohTu4Upybed0AQz/rbB4see+RNr0U5n7OgeFH1Y6p6SM+l6ZV93ylp3CVXHBHEw9bem0Q5daXoFbC73y3plRRqAd5D3yFt9BzSVsk94FPN7JHc2zYD1zfIzCaa2Wwzm71Oays4HSCJvkP66DkkotwJ+DJJ20kaI2mZpAvWN9DdJ7t7q7u3tqhPmacDJNF3SB89h8SUtRGHuy9/92szu0LSzVWrqE61rFoTxA+ujW/YfbiPFT3OqtG9o9zQv5VfV09Sr33XvuCJKDfvuB2D+Kn/7RuN6dPUFsSThjxctZomj/hHEHf819+LvubhL8S5Y+76ShBv95v4KWBNdz3UrdpqSb32XJKahwwO4qkfvTIa01bguUZDH3grsZrqVVlXwGY2rEt4pKT56xsLVAt9h7TRc0hS0StgM7tG0gGShpjZEklnSzrAzMZIckmLJZ2UYI1oQPQd0kbPIW1FJ2B3P7ZAOn7PAagi+g5po+eQNnbCAgAgAzwNqUo65j0WxD947pPRmL+MLr5+47UProtyQ8svCzWq/bEng3ibY+IxD2w7OogPHbVP0eM+//l48ctBOzwe5ZotXCR4zvtmRmM2bQoXBO4Wrw/UwoMvD+K9B30+GrPFG7tEuR62gUdDeXOv7YJ43z4zojF7z/tclBt4z7zEaqpXXAEDAJABJmAAADLABAwAQAa4B1xjRk4vYcd9NIS2ZxYHce+8uJDtC2zaUvxV0qc+f0aU2+XU8COv3xt2WzRmy17hjk/37/H7aMxfr900yl1+9Pgg9oe4J1wvXvpi8Q01Om4YUiC7qPrF1DmugAEAyAATMAAAGWACBgAgA0zAAABkgEVYNWblV+IFDlvyNCQkbNPf3RfllvwujD/71y9FY/45Jl50le+wjVdFuZ9vsVEQF9jjAzWgebN4Ad1fPhxuvvKDlXtGYwb/elZiNfUkXAEDAJABJmAAADJQdAI2sxFmdqeZPWZmC8zstFx+kJnNMLNFub8HJl8uGgV9h7TRc0hbKfeA2ySd4e5zzWyApDlmNkPSFyXNdPfzzGySpEmSzkyu1MYwfNPXolyDbs1B32Xs2fP2DuIFYy4pMCr8Hb7FmqMR6wo1sFkFlSWGnsvz2sE7RrmRve4M4tsu+Gg0ZjNxD7gURa+A3X2Zu8/Nfb1a0kJJwyWNlzQtN2yapCOSKhKNh75D2ug5pK1b94DNbKSk3SXdL2mouy/L/dOL4ql5SAh9h7TRc0hDyROwmfWXNF3S6e7+etd/c3fXet4pNbOJZjbbzGav09qKikXjoe+QNnoOaSlpAjazFnU25NXufn0uvdzMhuX+fZikFYVe6+6T3b3V3Vtb1KfQEKAg+g5po+eQpqKLsMzMJF0paaG7X9jln26SNEHSebm/b0ykwjr12i9HxMmfpV9HvaLvSvPmUeEmCG8PKu1NrXXjXg3i47d/IBozYdPzg7ijhO0yCi246lBHnPTaW1pIzylaHHfMOfETsBaseyeIB133cDSmwH9xFFDKKuh9JR0v6VEzm5fLnaXOZvyjmZ0g6TlJRydTIhoUfYe00XNIVdEJ2N3vkbS+zwwcWN1ygE70HdJGzyFt7IQFAEAGmIABAMgAT0NKSO/V5S1D+MPo66LcUft9NYib7pkXjUFjGnNm2AsXbHlPSa9ryvvdu+BCqTKeUXTn232j3FfuPT7K7fjA00Hc3u0zIQm2xy5BfPJmV0VjRl13ehi/FT9JC6XhChgAgAwwAQMAkAEmYAAAMsA94BrT3+IddLwXvyehsHnnjQnib0yKP0Vz0Zb/rMq51nhblBsz49Qg3n5KfC951D/nRjnu+damFw7eJIifb3srGjP6Ow8Fce1tqVI/+MkOAEAGmIABAMgAEzAAABlgAgYAIAMswkrIRs+uinJ/emNwlPvP/i8H8U6/PSUas81dfNAdhfWbfn8Qzxq2Tzzou+UtwtrxlpODuP8TLdGY0effW9axUZu2mrk6iD+x28nRmG3WPpJWOT0eV8AAAGSACRgAgAwUnYDNbISZ3Wlmj5nZAjM7LZc/x8yWmtm83J9xyZeLRkHfIW30HNJm7hv+GLWZDZM0zN3nmtkASXMkHaHOh1K/4e7nl3qyTWyQ72k8VrNR3eHXzXH31lLG0neohvt9pl73V9b3jN8APYdqKfVnXdFFWO6+TNKy3NerzWyhpOGVlwisH32HtNFzSFu37gGb2UhJu0t6d+nlqWb2iJlNMbOB63nNRDObbWaz12ltRcWiMdF3SBs9hzSUPAGbWX9J0yWd7u6vS7pM0naSxqjzt8YLCr3O3Se7e6u7t7Yo3ucY2BD6Dmmj55CWkiZgM2tRZ0Ne7e7XS5K7L3f3dnfvkHSFpLHJlYlGRN8hbfQc0lTKKmiTdKWkhe5+YZf8sC7DjpQ0v/rloVHRd0gbPYe0lbIT1r6Sjpf0qJnNy+XOknSsmY1R59OoFks6KZEK0ajoO6SNnkOqSlkFfY+kQsv4b6l+OUAn+g5po+eQNnbCAgAgA0zAAABkgAkYAIAMMAEDAJABJmAAADLABAwAQAaKPg2pqicze0nSc5KGSFqZ2omrh7ors7W7b572Seu87+qxZql26qbnykPdlSmp71KdgN87qdnsUh9LV0uou77V4/ehHmuW6rfuaqvX7wN1p4O3oAEAyAATMAAAGchqAp6c0XkrRd31rR6/D/VYs1S/dVdbvX4fqDsFmdwDBgCg0fEWNAAAGWACBgAgA6lPwGZ2qJk9YWZPmdmktM9fKjObYmYrzGx+l9wgM5thZotyfw/MssZ8ZjbCzO40s8fMbIGZnZbL13TdSaPnkkXfFUbfJaen9FyqE7CZNUu6VNJhknZW54Oud06zhm6YKunQvNwkSTPdfZSkmbm4lrRJOsPdd5a0l6RTct/fWq87MfRcKui7PPRd4npEz6V9BTxW0lPu/oy7vyPpWknjU66hJO5+t6RX8tLjJU3LfT1N0hGpFlWEuy9z97m5r1dLWihpuGq87oTRcwmj7wqi7xLUU3ou7Ql4uKQXusRLcrl6MdTdl+W+flHS0CyL2RAzGylpd0n3q47qTgA9lyL67j30XUrquedYhFUm7/z8Vk1+hsvM+kuaLul0d3+967/Vct3YsFr/b0ff9Uy1/N+u3nsu7Ql4qaQRXeKtcrl6sdzMhklS7u8VGdcTMbMWdTbk1e5+fS5d83UniJ5LAX0Xoe8S1hN6Lu0J+EFJo8xsGzPrLekYSTelXEMlbpI0Iff1BEk3ZlhLxMxM0pWSFrr7hV3+qabrThg9lzD6riD6LkE9pufcPdU/ksZJelLS05L+O+3zd6POayQtk7ROnfdvTpA0WJ0r6xZJukPSoKzrzKt5P3W+5fKIpHm5P+Nqve4Uvi/0XLJ103eFvy/0XXI194ieYytKAAAywCIsAAAywAQMAEAGmIABAMgAEzAAABlgAgYAIANMwAAAZIAJGACADFQ0AdfL8y7Rs9B3SBs9hySUvRFH7nmXT0o6WJ27pzwo6Vh3f2x9r+ltfbyv+pV1PtS/1Vq10t03r+QY9B26Y43e1Du+1io5Bj2H7ir1Z12vCs7x3vMuJcnM3n3e5Xqbsq/6aU87sIJTop7d4dc9V4XD0Hco2f0+sxqHoefQLaX+rKvkLeh6f94l6hN9h7TRc0hEJVfAJTGziZImSlJfbZz06QBJ9B3SR8+huyq5Ai7peZfuPtndW929tUV9KjgdIIm+Q/roOSSikgm43p93ifpE3yFt9BwSUfZb0O7eZmanSvqbpGZJU9x9QdUqAwqg75A2eg5JqegesLvfIumWKtUClIS+Q9roOSSBnbAAAMgAEzAAABlgAgYAIANMwAAAZIAJGACADDABAwCQASZgAAAywAQMAEAGmIABAMgAEzAAABlI/HGEDcssSj3/v3tHuWlf/HkQj+3TEo352ILx4aH/b0g0pu+8xUHcvvLlUqpEjXj7iLFRbsWHmsOExz31g2OvDuIj+70SjTlw/lHxse9/X9Fjl8U8Sm170eNRrn3VquqcD6hjXAEDAJABJmAAADLABAwAQAYqugdsZoslrZbULqnN3VurURSwIfQd0kbPIQnVWIT1MXdfWYXj9CgrbxoV5R750MVR7vDHjwzite3xf5IZu0wPE9Pi8+06+WtB/P7v31tClXWtrvvu9c/tFcTTfnJBNGabXn2DuEMdRY9baMSMD/wxyjV9IHzzq5Rjl6KpwJtqd36ub5T7fyceH8TNd86tyvkTVtc9V46mjTeOco9f8IEgfupTl5d0rL3nfTaINzu3XzTG/jWvG9XVP96CBgAgA5VOwC7pdjObY2YTq1EQUAL6Dmmj51B1lb4FvZ+7LzWzLSTNMLPH3f3urgNyzTpRkvoqfjsDKAN9h7TRc6i6iiZgd1+a+3uFmd0gaayku/PGTJY0WZI2sUHxp/R7iOVf3yeIr/zAz6IxP1u1U5TrdULemxDPvxCNGd//Y0HcdFN872TOieH5dtnylGjM6JMejHL1qCf03YsfDe+5bt2rd1nHueDl8H7cc2sGlfS65rwNM9qrtBFH/nEl6baFO0e5Hec9G56/KmdPTk/ouXI8980xUe7JT+WvZYl7p0Px//x/jbk2iFde83Y05ovv3697Bda5st+CNrN+Zjbg3a8lHSJpfrUKAwqh75A2eg5JqeQKeKikG6xzy8Vekn7v7rdVpSpg/eg7pI2eQyLKnoDd/RlJu1WxFqAo+g5po+eQFD6GBABABngaUhma+sYbC+z5+YeC+MG3t4nG3HnErlGuffGzUS4a8/rrQeynbxWN+Z8rw40drjk4/nD82dqj6LmQjlFXrQ0Tnyz+mq8v/WiUW3LkwCBuW/rvSspKxCjFm2zU+qIrdGrvW95ash3uODHKnbvXDUH8vl6vFT3O68fuFeVe2SVc9DXijrXRmJY5i6Jcx+rVRc+XNq6AAQDIABMwAAAZYAIGACADTMAAAGSARVhleOoHu0e53w47P4gPOfdb0ZjNn5pVlfP7gqei3LI1m1bl2EjHq//9VrdfM3v5iCg3ZOmT1SgHKNvMt/tEuR2/Fe/o99sB+wdx2zOLozHP/mTvIL738+dHYzZtylsE+6W4ph3/EO8EuP0374sHZowrYAAAMsAEDABABpiAAQDIAPeAy3DeEVdHuQMeDB8ROvyy6tzvLeStcfETSv4y8rIgnhN/Nh01pCnvqUFNBX4XbrHmvNckWhJQkqa8px8duFH8w+anO8ebBTXdFW5W1Lx9vFnRwi9cmpfZqPsFStqt9eko92ZZR0oWV8AAAGSACRgAgAwwAQMAkIGiE7CZTTGzFWY2v0tukJnNMLNFub8HbugYQHfRd8gCfYc0lbIIa6qkSyRd1SU3SdJMdz/PzCbl4jOrX15teOvTewbxJzaOP9D94xmbJHb+/KcvDTmj+BOUjr3tq1FutB6oWk0pmKoe3HcdHi5k6VBHNGZd3oNoOsp7MA26Z6p6cN9VQ4fyFxDGqwOfPyh+YtzIu8LYl6+Mxux28alB/LuTLorG7NK7+LT13Gvx70hD9FLR16Wt6BWwu98t6ZW89HhJ03JfT5N0RJXrQoOj75AF+g5pKvdjSEPdfVnu6xclDV3fQDObKGmiJPXVxmWeDpBE3yEbJfUdPYfuqngRlru7pPW+Oebuk9291d1bWxTvGQqUg75DFjbUd/QcuqvcK+DlZjbM3ZeZ2TBJK6pZVK1p6xve4+hjLYmdq3mz+KEKq64ZEsQ3bffHaMz3VuwRxDueuTAaE99lrDsN1XeoGfRdSnY+/IkgLuV+74J32qLcZhcNqFpNSSr3CvgmSRNyX0+QdGN1ygE2iL5DFug7JKKUjyFdI2mWpB3MbImZnSDpPEkHm9kiSQflYqBq6Dtkgb5Dmope37v7sev5pwOrXAvwHvoOWaDvkCZ2wgIAIAM8DakEA28NFwbc+YP4Q+ae/6iamfHTQLbs91qUu++WDwbxmZ+7Lhpz3IBlQXzkosOjMR0n9Qvj1U9FY1DfNttoTZRr2jj8uEvHW2+lVQ4awIDie/4UNOoji6Nc+4BwYVSfm+OPal2zza1BXMrC0VPOPC3K9f97vFlSLeIKGACADDABAwCQASZgAAAywD3gErSvWhXEqzs2isac/81fBfH7mt+Ixoxu6R0f/KR/BOHK9rejITv9/ttBvM2N8ZimJ+bFx0aPcstO06PcoR8JH7rR+2+z0yoHDWDoX5+Jco98rz2IxxTYLOPckTdEuWvvHhvEP9zirmhMvo89+p9RbtMvvhnE/V+sj/u9hXAFDABABpiAAQDIABMwAAAZYAIGACADLMKqko/2fScvU2DBVQkefWdglNvhl/8O4rZnnyvr2Kgdb8zaPIibxsS/C7dYcxCvK/AQvNunhIv/dps1IRrTsWCT+IUWHmyzJ+ODb/q7+l3cgupoe3F5lPvh858M4unb3xqN2aklfmLc97d4KIgLbbKRv8nQJse8HNf0aryhUb3iChgAgAwwAQMAkAEmYAAAMlDK84CnmNkKM5vfJXeOmS01s3m5P+OSLRONhr5D2ug5pK2URVhTJV0i6aq8/EXufn7VK8qY9ekT5V779O5BPKbPPQVeGe+Ole//Xt4lyv3+Tx8P4i3/GT/xpvm5h4seuweaqh7cd+8/74EgHtPxtWjMvFMuDuKOEp4N89Dev4mTe8epprzfvZcV2IFt3tlbFD1fvu//NF4EtsU186Ncx+rV3T52CqaqB/dc1pa0hT12+OXficZs9ZN70yqnJhS9Anb3uyW9kkItwHvoO6SNnkPaKrkHfKqZPZJ72yb+7EyOmU00s9lmNnud1lZwOkASfYf00XNIRLkT8GWStpM0RtIySResb6C7T3b3VndvbVH89i7QDfQd0kbPITFlbcTh7u99OtvMrpB0c9Uqythbh+0W5e4+/9IgfqMj/r3luGcPCeKrt7k9GjNz+Q5RbsQPG+ueRyV6Ut95W1sQb3Vu3AenHblvEO/Sb2k05r82jZ9WU46hzfGE8YmNu7/hwWFnXxLlRu/+1Tj31QeiXC3qST1XjraP7xHlznr/5XmZ5mhMIZ+aOzGIG+1+byFlXQGb2bAu4ZGS4lUWQJXRd0gbPYckFb0CNrNrJB0gaYiZLZF0tqQDzGyMJJe0WNJJCdaIBkTfIW30HNJWdAJ292MLpK9MoBbgPfQd0kbPIW3shAUAQAYa/mlIvu+YIL7sZz+Pxqxsbw/iT37/29GYIVfNCeIx134hGnP/nvEv0//xya8Hcd+/1MfiFCTv6Q+Hm7IsHj02GvOb/Q+PcqX43+9MC+J1Hv8o+MhGy4J406bynvB10UG/j3K/OOizQdxyx5xoDNK34uR9gnjady6MxuzSUrwPmmRR7o3l/csvrIfiChgAgAwwAQMAkAEmYAAAMtDw94Bf2Tl8iMLoAvc3dvtluFH+iCvjD5B7Xtzvpk2iMX32aolyg7/9bBC/+dcCH2rvaI9zaDjtTz4d5QYXyJXi0itGFx3z468eF8RvD43v6+1+8MIgnjbyjmjMYRuvinK3nvtYEC+OX4aEvfOJ1ij362//LIh3aol/ZnXk/bQrdL83f4wk9Xq1tA07GglXwAAAZIAJGACADDABAwCQASZgAAAy0PCLsPL9a0286GDrnz0cxB0lHGfwHx6Kk+fGqT9sd1sQf7Jln2iMr2URVr1YedLeUe7MM+KNKKYcG26g4XMWJFZTuTa/bFYQL/9a3Jv5i65aLF5osy5ej6NmK5BEopoGDAjivc+LN/3ZtXf4329J29vRmH5N4aKrwU0bRWMKGXlzfKxGxxUwAAAZYAIGACADTMAAAGSglOcBj5B0laSh6txvYrK7/9zMBkn6g6SR6nxO5tHuHn/ivs6s8QIfPH/zze4faPuRBZL/ijIz394472TcG5PquO8K/Ocb329llHvr6hlBfPGFnyl66M3nvB7lXtoj3vClnNet+kC8suFnh/02iMf0uSca06E+QVzofu9Nbw6Mck9M2iWIeyn7hzHUbc+V6LnfbB3EN29xVTQmvwsOv/w70Zi1Hwjv5T5xQPyQmWtWD41yLQufD2JWtpR2Bdwm6Qx331nSXpJOMbOdJU2SNNPdR0mamYuBaqHvkDZ6DqkqOgG7+zJ3n5v7erWkhZKGSxov6d1nmk2TdERSRaLx0HdIGz2HtHXrY0hmNlLS7pLulzTU3d99YOiL6nzbptBrJkqaKEl9tXGhIcAG0XdIGz2HNJS8CMvM+kuaLul0dw9uKrm7q+DdL8ndJ7t7q7u3tuTdLwKKoe+QNnoOaSnpCtjWS9XcAAAauElEQVTMWtTZkFe7+/W59HIzG+buy8xsmKQVSRWZpv36vhblTp66RxDvdM5L0Zi2xeECg2ePjheeFPL12Z8N4m3WPVLS6xpBT+67YwcsDeOzfxGNacr7/Xj6G0OiMUf2L/4//4Y3tohyn+kfLgzrKGl7meKTyp1v941y3771c1Fu1Mz7Sjhf+npyz+24xfIgbrb4+mv8k4cF8bBZa6Ixfzs1XnSV75wHPhXlRr08t+jrGk3RK2AzM0lXSlro7hd2+aebJE3IfT1B0o3VLw+Nir5D2ug5pK2UK+B9JR0v6VEzm5fLnSXpPEl/NLMTJD0n6ehkSkSDou+QNnoOqSo6Abv7PVKBJy53OrC65QCd6DukjZ5D2tgJCwCADDT805C2+NfLQfyNpfEvuk8cPDmI3zhwbTTmIw+eGMQ3fOiCAmeLF7G0rSjtSSKoD0P/Ea/P2de/HuXGTgyflnXRlv8seuxSFlyV/rru/+69xtui3B1vhZ/IufyEeEevUf+szQVXPVmvrUdEucnbXBvE7R7/7Fm0fPMg/vC5z0RjOvIWgT9f4IlJO/7w1SjHzlcxroABAMgAEzAAABlgAgYAIAMNfw+4/bEng/iFj8b3aXc559QgvvVzP43GPLRn/pNF4uPs89CxUW7H7y0M61lfoagL7U8+HeUGF8jNfXuvIB6z/YeiMY+ceHH1Citi5z98Lcr1fj38/bzXG/Hrtjz/3iBu0kPxIKTO34if4HbHW1sF8X/2fzkaM3+/3xQ99vL28J7vhDPOiMb0W3R/0eOAK2AAADLBBAwAQAaYgAEAyAATMAAAGbDOp2ulYxMb5HsaO7o1qjv8ujnu3pr2eem7xnW/z9Tr/sr6tpdMTC323Msn7h3E3/jWH6MxR+dt2nLxqlHRmOk/OCSI+/+RjVbylfqzjitgAAAywAQMAEAGmIABAMhA0Y04zGyEpKskDZXkkia7+8/N7BxJJ0p6KTf0LHe/JalC0VjoO6Stp/fc4CtmBfFVV8QPbLhKcS5ff3HPt1pK2QmrTdIZ7j7XzAZImmNmM3L/dpG7n59ceWhg9B3SRs8hVUUnYHdfJmlZ7uvVZrZQ0vCkC0Njo++QNnoOaevWPWAzGylpd0nvbvR5qpk9YmZTzGzgel4z0cxmm9nsdYqfowsUQ98hbfQc0lDyBGxm/SVNl3S6u78u6TJJ20kao87fGgs9gV7uPtndW929taXAAwqADaHvkDZ6DmkpaQI2sxZ1NuTV7n69JLn7cndvd/cOSVdIGptcmWhE9B3SRs8hTUUnYDMzSVdKWujuF3bJD+sy7EhJ86tfHhoVfYe00XNIWymroPeVdLykR81sXi53lqRjzWyMOpfrL5Z0UiIVolHRd0gbPYdUlbIK+h5JhfZSrbvPwaF+0HdIGz2HtLETFgAAGWACBgAgA0zAAABkgAkYAIAMMAEDAJABc/f0Tmb2kqTnJA2RtDK1E1cPdVdma3ffPO2T1nnf1WPNUu3UTc+Vh7orU1LfpToBv3dSs9nu3pr6iStE3fWtHr8P9VizVL91V1u9fh+oOx28BQ0AQAaYgAEAyEBWE/DkjM5bKequb/X4fajHmqX6rbva6vX7QN0pyOQeMAAAjY63oAEAyAATMAAAGUh9AjazQ83sCTN7yswmpX3+UpnZFDNbYWbzu+QGmdkMM1uU+3tgljXmM7MRZnanmT1mZgvM7LRcvqbrTho9lyz6rjD6Ljk9pedSnYDNrFnSpZIOk7SzOp+zuXOaNXTDVEmH5uUmSZrp7qMkzczFtaRN0hnuvrOkvSSdkvv+1nrdiaHnUkHf5aHvEtcjei7tK+Cxkp5y92fc/R1J10oan3INJXH3uyW9kpceL2la7utpko5Itagi3H2Zu8/Nfb1a0kJJw1XjdSeMnksYfVcQfZegntJzaU/AwyW90CVeksvVi6Huviz39YuShmZZzIaY2UhJu0u6X3VUdwLouRTRd++h71JSzz3HIqwyeefnt2ryM1xm1l/SdEmnu/vrXf+tluvGhtX6fzv6rmeq5f929d5zaU/ASyWN6BJvlcvVi+VmNkyScn+vyLieiJm1qLMhr3b363Ppmq87QfRcCui7CH2XsJ7Qc2lPwA9KGmVm25hZb0nHSLop5RoqcZOkCbmvJ0i6McNaImZmkq6UtNDdL+zyTzVdd8LouYTRdwXRdwnqMT3n7qn+kTRO0pOSnpb032mfvxt1XiNpmaR16rx/c4KkwepcWbdI0h2SBmVdZ17N+6nzLZdHJM3L/RlX63Wn8H2h55Ktm74r/H2h75KruUf0HFtRAgCQARZhAQCQASZgAAAywAQMAEAGmIABAMgAEzAAABlgAgYAIANMwAAAZIAJGACADFQ0AdfLA6fRs9B3SBs9hySUvRNW7oHTT0o6WJ3blz0o6Vh3f2x9r+ltfbyv+pV1PtS/1Vq10t03r+QY9B26Y43e1Du+1io5Bj2H7ir1Z12vCs7x3gOnJcnM3n3g9Hqbsq/6aU87sIJTop7d4dc9V4XD0Hco2f0+sxqHoefQLaX+rKvkLeiSHjhtZhPNbLaZzV6ntRWcDpBE3yF99BwSkfgiLHef7O6t7t7aoj5Jnw6QRN8hffQcuquSCbjeHziN+kTfIW30HBJRyQRc7w+cRn2i75A2eg6JKHsRlru3mdmpkv4mqVnSFHdfULXKgALoO6SNnkNSKlkFLXe/RdItVaoFKAl9h7TRc0gCO2EBAJABJmAAADLABAwAQAaYgAEAyAATMAAAGWACBgAgA0zAAABkgAkYAIAMMAEDAJABJmAAADJQ0VaUAGrfomkfinJPHHRFEO907SnRmO3OuC+xmpC9XtuOjHKLJg4L4o4Ra4oe55DRC6PcF4f8M8pNmHpaEL//B7Pig7kXPV9PwhUwAAAZYAIGACADTMAAAGSgonvAZrZY0mpJ7ZLa3L21GkUBG0LfIW30HJJQjUVYH3P3lVU4DtAd9F2Jxu0cPzu+Qx1BfOGnrorGXHrG6G6f6+mrd49y2x33ULePU6N6VM+9dHFLlFs45tIqHb05ysyfeEkQ7zDg5GjMdt9qrIV/vAUNAEAGKp2AXdLtZjbHzCYWGmBmE81stpnNXqe1FZ4OkETfIX30HKqu0reg93P3pWa2haQZZva4u9/ddYC7T5Y0WZI2sUGN9SEvJIW+Q9roOVRdRROwuy/N/b3CzG6QNFbS3Rt+VW1b/KO9o9wTX74siNd5ezTmhjcHBfGPLj8uGjPihqVFz9+x8pU4t3p10dc1kp7Yd9XSa/iWUe4D/R5O7fwXjv1DlLtU3b+XXGvqreea+vaNck/8aucgXjTm19GYf6wJ7wtPvPHEaEzLG1b0/O194tzVR/8iiE877NZozM3fGlj02D1J2W9Bm1k/Mxvw7teSDpE0v1qFAYXQd0gbPYekVHIFPFTSDWb27nF+7+63VaUqYP3oO6SNnkMiyp6A3f0ZSbtVsRagKPoOaaPnkBQ+hgQAQAZ4GlKePT72eJTLX3SVv4mBJI3vF34+f/wZP48Pfkacasr7HejkJR+Nxvx91l5BvMOvXo7GtC9cFB8cDefNXYdHuS9temMGlSBLz54Vb4iy6KBwk41l7W9FYyb96NtBvP1vCjyxqATrDtojyu3x+XBzji9f/h/RmOG6t6zz1SuugAEAyAATMAAAGWACBgAgA0zAAABkgEVYeSa+766iY45/9tAot/QXo4L4lZ3i321O+2y8GOaETZ8P4ku2+kd8wv8Mc3ce3j8acsEJnwviprt6zBNoUKH8hX6FNFu8sLAcl46q/12veoJtr3oxyl346fBn1DcHxgs37/jhhUH80U3ilaNbTnk0iDveeCMas/SA3lFu4gvhAtOtLnggGtNo+3dyBQwAQAaYgAEAyAATMAAAGeAecAkmvnBAEL95VPxt6//ifWFc4Dh/mRJ/OP4Pu4wL4i/97IZozGcHLAvij20U33MZ8Jsrg/j7x30pGmOz0nsqDmpHoY1j8rU7v4v3JO1PPRvl7jxi1yD+/SGfiMZc8q1LgvjhM38ZjTn4U58M4tenfSAa892jpke5P+4/Joi9Lf451mj4fx0AABlgAgYAIANMwAAAZKDoBGxmU8xshZnN75IbZGYzzGxR7u+ByZaJRkPfIQv0HdJUyiKsqZIukXRVl9wkSTPd/Twzm5SLz6x+een78eL4CR0tn3k9iNtfXV7WsdteWBLleuflrv1IvFDr9hvDRQ5Xbj0jGtPaJ3xi03P/sXE0ZmR5DzbJylQ1UN+hZkxVD+27/IVZWxRYqHXa2lOCeNYPLonG3LZj3oZCP4nP9cFZX4hyI5bPjwc2uKJXwO5+t6RX8tLjJU3LfT1N0hFVrgsNjr5DFug7pKncjyENdfd3PxvzoqSh6xtoZhMlTZSkvoqvyoBuoO+QhZL6jp5Dd1W8CMvdXRvYwtPdJ7t7q7u3tqhPpacDJNF3yMaG+o6eQ3eVewW83MyGufsyMxsmaUU1i8pS04EvRLn2AuOS0v7SS1HurbbNg7iUzfWH/+OdqtVUQ3ps31XLig+1ZF1CT9QwfTf4ynChyAe3PjUas+CES4se5+s73xnl/tz6sSD22dwTLvcK+CZJE3JfT5AUP+YHqD76Dlmg75CIUj6GdI2kWZJ2MLMlZnaCpPMkHWxmiyQdlIuBqqHvkAX6Dmkq+ha0ux+7nn86sMq1AO+h75AF+g5pYicsAAAywNOQakzbx/eIcl/Z8qogLvR0m8Mf/3QQt9wxp7qFoSY1D90iiE///J+jMaUs2nth3eCq1YQ61tQchF844u/RkI68ReBvebzg88RN48Ws+183OYhP/fLXojG9/t5YP7e4AgYAIANMwAAAZIAJGACADHAPOGNNG4db1g38/nPRmP03eqvocdr/L9wdr0nxPRj0PCsP2y6IJ2zy12hMvGIgdvGfPhnltta95ZaFOtDUt2+Ue2rKjkF8w6DJ0ZhdJ58exFv/+IFozJMXtEa5RUf9Moh/MSV+0MPJp54WxH1vjo/dk3AFDABABpiAAQDIABMwAAAZYAIGACADLMLK2PO/3SaI5247tehrrlk9PMr1nftsEKf5BCdk57VR5b3u169tG8TbTft3NKatvEOjTtiILaPc4/tPCeKd/vlf0Zhtvh8uziv0bMYdz14Y5fbf4aggvuuD10VjLr7kF0F8+ppTojE9aZMhroABAMgAEzAAABko5XGEU8xshZnN75I7x8yWmtm83J9xyZaJRkPfIW30HNJWyhXwVEmHFshf5O5jcn9uqW5ZAH2H1E0VPYcUlfI84LvNbGTypTSmeXvlP+kodsMb4RNv/vDZ+NGkHSvjRQ/1jL4rzZ4fX1DW687/ZzjPjH7mwWqUU9carec2mfpq0TGDbt646JhC2l99Lcr1G/dGEO9/y1HRmPyFWZv8z5JozDsvbB+e64mnyimxJlRyD/hUM3sk97bNwKpVBGwYfYe00XNIRLkT8GWStpM0RtIySResb6CZTTSz2WY2e53Wlnk6QBJ9h/TRc0hMWROwuy9393Z375B0haSxGxg72d1b3b21RX3KrROg75A6eg5JKmsjDjMb5u7LcuGRkuZvaDw6PXXhXlGuxeYF8boCn2r/nz8fE8TbPjyrqnXVi0bvu+bBg6LcVVv/PYjXeWm/U+90wStBXK2NW94+Ip6fNvpz/T7Rpif13FtH7hnE07f+eTRm9we/FMTDb3g0GlPK07UK6gi7bJNjXo6G/N/dOwXx9O1vjcYcdMmRQdz74HILyl7RCdjMrpF0gKQhZrZE0tmSDjCzMercBGWxpJMSrBENiL5D2ug5pK2UVdDHFkhfmUAtwHvoO6SNnkPa2AkLAIAMMAEDAJABnoaUkDWHx4tR7jrq/Ci3zjcK4u+t2CMaM/pXy4KYp9Q0pme+vmOUW+czgrijwBKZh9bGv2fb2neqV1gXLxTYR2r0nxM5Fbpp3cYWxC3WHI1ZvbJfEPs76xKrp9BmHX+aHG4ydOeRo6MxE0aEi1Cv2StuuubHnyvpfFnjChgAgAwwAQMAkAEmYAAAMsA94CrpNXzLIB723XiD8M2b491xXutYE8T/+vGe0Zh+z9xfYXVoZF97LP50zaDnnkzkXD8/8HdR7lLF9/GQvk2vvi+I5/8o3vXnqcMmB/Gu3z41GrPVufdWt7AutrgkPHbznTtEY46//cUwnj41GrPn2adEucG/rr0NjLgCBgAgA0zAAABkgAkYAIAMMAEDAJABFmFVyTMnjgzieSPjJ40UMnbm14N41HUsuEKnpy8In551/WcuLDCqpehxpu86Jcod/flvBfGmv7svGoPG02zhNdku456Ixrxx5RZB3L58RXIFrVwVpa5eHZ7/uAHx+ds2sihXi7gCBgAgA0zAAABkoOgEbGYjzOxOM3vMzBaY2Wm5/CAzm2Fmi3J/D0y+XDQK+g5po+eQtlLuAbdJOsPd55rZAElzzGyGpC9Kmunu55nZJEmTJJ2ZXKm1o9fWI6LcaUffWNaxRn1xTqXl9FR12XevHr93lBvyr2VRru2ZxUWPNW7/sDd2aonv9+ZvqL8u3ltBi9v6Fz1XtTRb/DCIOlKXPVeuk88+Lcr1OS7c5OLOD0yPxsy5rz08zrlfj8YMvqI6m14Uur984aVHB/EnvxM/5Oad/V+PD3ZxVUqqqqJXwO6+zN3n5r5eLWmhpOGSxkualhs2TdIRSRWJxkPfIW30HNLWrXvAZjZS0u6S7pc01N3f/dX+RUlDq1oZkEPfIW30HNJQ8gRsZv0lTZd0ursH1/fu7pIKvPklmdlEM5ttZrPXaW1FxaLx0HdIGz2HtJQ0AZtZizob8mp3vz6XXm5mw3L/PkxSwQ+Duftkd29199YWxQ8jANaHvkPa6DmkqegiLDMzSVdKWujuXXcCuEnSBEnn5f4ubxVSHWgePCiID77l0WjMCZs+H8Qr2+PfgD896VtRbhOxAUIh9dp3m/02XnzSVuax5v7kQ0G827Hxk2Ee3ntaEHcoXgR19slfiXKb3pZM37V7/X6ysV57rlybXRX36roXW4N41PivRmMWHXFZEN9/zqXRmG+cFD/V7fa/fjiIm3aJF0r1v3lAEA/582PRmC1/uyCIHz49XmS48W0DolwtKmUV9L6Sjpf0qJnNy+XOUmcz/tHMTpD0nKSj1/N6oBz0HdJGzyFVRSdgd79H0vr29TqwuuUAneg7pI2eQ9rq9/0iAADqGBMwAAAZ4GlIJVgyYccg/upmf4vG5C992efWb0RjRv+eBVcoXb/p4ZOxVu66Tzwo3ngLKFvL7bODeIe7+0ZjDhz16SCeucv10ZiLhhV4qtt/lfCkt/x+/nHxlxTy2qg4t/mAcGFWx+rV5R28irgCBgAgA0zAAABkgAkYAIAMcA84z8qJ8U21733l6qKv+98V4YfMdzrr6WhMe5QBSrftLx6Pcmd/avcg/uiAJ6IxGz3/WpSjF1GKjjVrotzGnw376dBdvhyNeWtYvBPYqh3CJ3et3fHtouf/wgfj+8bfGzK/6OuaRr4ZJ9trr+u5AgYAIANMwAAAZIAJGACADDABAwCQARZh5Xnj/XHuyH6vBHH8vBnp9ivCTRK2ePneKlYFSO0vvxLl5uwe/g49RzsVeOWTCVUUu3TU6NTOhWy0r1oVxE33rIrGxM8nKpwr5l71jnLj9KECI0Mj9UiUK/RzO2tcAQMAkAEmYAAAMlB0AjazEWZ2p5k9ZmYLzOy0XP4cM1tqZvNyf8YlXy4aBX2HtNFzSFsp94DbJJ3h7nPNbICkOWY2I/dvF7n7+cmVV5t2+1f8wfNtpswN4lq831Bn6DukjZ5DqopOwO6+TNKy3NerzWyhpOFJF4bGRt8hbfQc0tate8BmNlLS7pLe3R/sVDN7xMymmNnA9bxmopnNNrPZ67S2omLRmOg7pI2eQxpKnoDNrL+k6ZJOd/fXJV0maTtJY9T5W+MFhV7n7pPdvdXdW1sU7w8KbAh9h7TRc0hLSROwmbWosyGvdvfrJcndl7t7u7t3SLpC0tjkykQjou+QNnoOaSp6D9jMTNKVkha6+4Vd8sNy90wk6UhJxR9RUQdGfm9WlDv8e3sE8dZ6NBrDoqvqarS+Q/boOaStlFXQ+0o6XtKjZjYvlztL0rFmNkaSS1os6aREKkSjou+QNnoOqSplFfQ9kqzAP91S/XKATvQd0kbPIW3shAUAQAaYgAEAyAATMAAAGWACBgAgA0zAAABkgAkYAIAMmLundzKzlyQ9J2mIpJWpnbh6qLsyW7v75mmftM77rh5rlmqnbnquPNRdmZL6LtUJ+L2Tms1299bUT1wh6q5v9fh9qMeapfqtu9rq9ftA3engLWgAADLABAwAQAaymoAnZ3TeSlF3favH70M91izVb93VVq/fB+pOQSb3gAEAaHS8BQ0AQAaYgAEAyEDqE7CZHWpmT5jZU2Y2Ke3zl8rMppjZCjOb3yU3yMxmmNmi3N8Ds6wxn5mNMLM7zewxM1tgZqfl8jVdd9LouWTRd4XRd8npKT2X6gRsZs2SLpV0mKSd1fmg653TrKEbpko6NC83SdJMdx8laWYuriVtks5w950l7SXplNz3t9brTgw9lwr6Lg99l7ge0XNpXwGPlfSUuz/j7u9IulbS+JRrKIm73y3plbz0eEnTcl9Pk3REqkUV4e7L3H1u7uvVkhZKGq4arzth9FzC6LuC6LsE9ZSeS3sCHi7phS7xklyuXgx192W5r1+UNDTLYjbEzEZK2l3S/aqjuhNAz6WIvnsPfZeSeu45FmGVyTs/v1WTn+Eys/6Spks63d1f7/pvtVw3NqzW/9vRdz1TLf+3q/eeS3sCXippRJd4q1yuXiw3s2GSlPt7Rcb1RMysRZ0NebW7X59L13zdCaLnUkDfRei7hPWEnkt7An5Q0igz28bMeks6RtJNKddQiZskTch9PUHSjRnWEjEzk3SlpIXufmGXf6rpuhNGzyWMviuIvktQj+k5d0/1j6Rxkp6U9LSk/077/N2o8xpJyyStU+f9mxMkDVbnyrpFku6QNCjrOvNq3k+db7k8Imle7s+4Wq87he8LPZds3fRd4e8LfZdczT2i59iKEgCADLAICwCADDABAwCQASZgAAAywAQMAEAGmIABAMgAEzAAABlgAgYAIAP/H/pYi8FsAU8cAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 9 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig=plt.figure(figsize=(8,8)) #8\"x8\" figure\n",
    "columns = 3\n",
    "rows = 3\n",
    "for i in range(1, 10):\n",
    "    img = mnist.test.images[random.randint(0,10000)] #find some random images\n",
    "    img = img.reshape((28,28)) #reshape the vector into a matrix\n",
    "    fig.add_subplot(rows, columns, i)\n",
    "    plt.imshow(img)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NN with softmax "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop in the code here, but not all in one block!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reshape the image to use with the CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"reshape\"):\n",
    "    x_image = tf.reshape(X, [-1, 28, 28, 1])\n",
    "    tf.summary.image('input', x_image, 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convolution Layer\n",
    "- maps a [28,28,1] grayscale image to [32] features\n",
    "- **perhaps explain strides and padding**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('conv1'):\n",
    "    Wc1 = weight([5,5,1,32])\n",
    "    bc1 = bias([32])\n",
    "    xc1 = tf.nn.conv2d(x_image, Wc1, strides=[1,1,1,1], padding=\"SAME\")\n",
    "    hc1 = tf.nn.relu(xc1 + bc1)\n",
    "    tf.summary.histogram(\"weights\", Wc1)\n",
    "    tf.summary.histogram(\"biases\", bc1)\n",
    "    tf.summary.histogram(\"activations\", hc1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pooling Layer\n",
    "- downsamples by 2x\n",
    "- **perhaps explain reshaping**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('pool1'):\n",
    "    hp1 = tf.nn.max_pool(hc1, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')\n",
    "    hp1_image = tf.reshape(hp1, [-1,14,14,1])\n",
    "    tf.summary.image('conv1', hp1_image, 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Second Convolution Layer\n",
    "- maps [32] to [64]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('conv2'):\n",
    "    Wc2 = weight([5,5,32,64])\n",
    "    bc2 = bias([64])\n",
    "    xc2 = tf.nn.conv2d(hp1, Wc2, strides=[1,1,1,1], padding=\"SAME\")\n",
    "    hc2 = tf.nn.relu(xc2 + bc2)\n",
    "    tf.summary.histogram(\"weights\", Wc2)\n",
    "    tf.summary.histogram(\"biases\", bc2)\n",
    "    tf.summary.histogram(\"activations\", hc2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Second Pooling Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('pool2'):\n",
    "    hp2 = tf.nn.max_pool(hc2, ksize=[1,2,2,1], strides=[1,2,2,1], padding=\"SAME\")\n",
    "    hp2_image = tf.reshape(hp2, [-1,7,7,1])\n",
    "    tf.summary.image('conv2', hp2_image, 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- the original 28x28 image with 32 features is now 7x7 with 64 features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('fc1'):\n",
    "    hpf = tf.reshape(hp2, [-1,7*7*64])\n",
    "    Wfc1 = weight([7*7*64, 1024])\n",
    "    bfc1 = bias([1024])\n",
    "    hfc1 = tf.nn.relu(tf.matmul(hpf, Wfc1) + bfc1)\n",
    "    tf.summary.histogram(\"weights\", Wfc1)\n",
    "    tf.summary.histogram(\"biases\", bfc1)\n",
    "    tf.summary.histogram(\"activations\", hfc1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dropouts\n",
    "- prevents overfitting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dropout\"):\n",
    "    Pk = tf.placeholder(tf.float32)\n",
    "    hfc1D = tf.nn.dropout(hfc1, Pk)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Map features to labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('fc-classify'):\n",
    "    Wfc2 = weight([1024, 10])\n",
    "    bfc2 = bias([10])\n",
    "    y = tf.matmul(hfc1D, Wfc2) + bfc2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### define the loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=_y, logits=y),\n",
    "        name=\"cross_entropy\")\n",
    "    tf.summary.scalar('loss', cross_entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### optimizer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('optimizer'):\n",
    "    train_step = tf.train.AdamOptimizer(0.0001).minimize(cross_entropy, name='train_step')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### define accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('accuracy'):\n",
    "    correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(_y,1))\n",
    "    correct_prediction = tf.cast(correct_prediction, tf.float32, name='correct_prediction')\n",
    "    accuracy = tf.reduce_mean(correct_prediction, name='accuracy')\n",
    "    tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### start session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "merged = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = tf.summary.FileWriter(LOGDIR, sess.graph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up embeddings for tensorboard visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "GITHUB_URL ='https://raw.githubusercontent.com/mamcgrath/TensorBoard-TF-Dev-Summit-Tutorial/master/'\n",
    "urlretrieve(GITHUB_URL + 'labels_1024.tsv', os.path.join(LOGDIR, 'labels_1024.tsv'))\n",
    "urlretrieve(GITHUB_URL + 'sprite_1024.png', os.path.join(LOGDIR, 'sprite_1024.png'))\n",
    "embedding = tf.Variable(tf.zeros([1024, 1024]), name=\"test_embedding\")\n",
    "assignment = embedding.assign(hfc1D)\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "config = tf.contrib.tensorboard.plugins.projector.ProjectorConfig()\n",
    "embedding_config = config.embeddings.add()\n",
    "embedding_config.tensor_name = embedding.name\n",
    "embedding_config.sprite.image_path = 'sprite_1024.png'\n",
    "embedding_config.metadata_path = 'labels_1024.tsv'\n",
    "embedding_config.sprite.single_image_dim.extend([28, 28])\n",
    "tf.contrib.tensorboard.plugins.projector.visualize_embeddings(writer, config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0, Training Accuracy 0.06\n",
      "Step 100, Training Accuracy 0.85\n",
      "Step 200, Training Accuracy 0.92\n",
      "Step 300, Training Accuracy 0.93\n",
      "Step 400, Training Accuracy 0.92\n",
      "Step 500, Training Accuracy 0.98\n",
      "Step 600, Training Accuracy 0.94\n",
      "Step 700, Training Accuracy 0.99\n",
      "Step 800, Training Accuracy 0.91\n",
      "Step 900, Training Accuracy 0.94\n",
      "Step 1000, Training Accuracy 0.99\n"
     ]
    }
   ],
   "source": [
    "# Do the training.\n",
    "for i in range(1100):\n",
    "    batch = mnist.train.next_batch(100)\n",
    "    if i % 5 == 0:\n",
    "        summary = sess.run(merged, feed_dict={X: batch[0], _y: batch[1], Pk: 1.0})\n",
    "        writer.add_summary(summary, i)\n",
    "    if i % 100 == 0:\n",
    "        train_accuracy = sess.run(accuracy, feed_dict={X:batch[0], _y: batch[1], Pk: 1.0})\n",
    "        print(\"Step %d, Training Accuracy %g\" % (i, float(train_accuracy)))\n",
    "    if i % 500 == 0:\n",
    "        sess.run(assignment, feed_dict={X: mnist.test.images[:1024], _y: mnist.test.labels[:1024], Pk: 1.0})\n",
    "        saver.save(sess, os.path.join(LOGDIR, \"model.ckpt\"), i)\n",
    "    sess.run(train_step, feed_dict={X: batch[0], _y: batch[1], Pk: 0.5})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See how well the model performed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy 0.9697\n"
     ]
    }
   ],
   "source": [
    "print(\"Test Accuracy %g\" % sess.run(accuracy, feed_dict={X: mnist.test.images,\n",
    "                                                         _y: mnist.test.labels,\n",
    "                                                         Pk: 1.0}))\n",
    "\n",
    "# Close summary writer\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "this looks promising https://github.com/lspvic/jupyter_tensorboard (jupiter notebook hook for tensorboard)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "timothy.s.darrah@vanderbilt.edu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}