keras_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:70% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:70% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "import cv2\n",
    "import numpy as np\n",
    "from enum import Enum\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CameraLabel(Enum):\n",
    "    HTC_1_M7 = 0\n",
    "    LG_Nexus_5x = 1\n",
    "    Motorola_Droid_Maxx = 2\n",
    "    Motorola_Nexus_6 = 3\n",
    "    Motorola_X = 4\n",
    "    Samsung_Galaxy_Note3 = 5\n",
    "    Samsung_Galaxy_S4 = 6\n",
    "    Sony_NEX_7 = 7\n",
    "    iPhone_4s = 8\n",
    "    iPhone_6 = 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(path='data-224', load_ratio=1.0, verbose=False):\n",
    "    print('Loading images')\n",
    "    data = []\n",
    "    labels = []\n",
    "    # Read data from every type of generated data\n",
    "    for data_path in os.listdir(path):\n",
    "        full_data_path = os.path.join(path, data_path)\n",
    "        # Read data from every camera type\n",
    "        for label in CameraLabel:\n",
    "            camera_path = os.path.join(full_data_path, label.name.replace('_', '-'))\n",
    "            if verbose:\n",
    "                print(\"Loading images for \", camera_path)\n",
    "            filenames = os.listdir(camera_path)\n",
    "            last_index = int(load_ratio * len(filenames))\n",
    "            for filename in filenames[:last_index]:\n",
    "                data += [cv2.imread(os.path.join(camera_path, filename))]\n",
    "                labels += [CameraLabel[label.name].value]\n",
    "    return np.asarray(data), np.expand_dims(np.asarray(labels), axis=1)\n",
    "\n",
    "def load_validation(path='data/validation'):\n",
    "    data = []\n",
    "    labels = []\n",
    "    # Read data from every camera type\n",
    "    for label in CameraLabel:\n",
    "        camera_path = os.path.join(path, label.name.replace('_', '-'))\n",
    "        print(\"Loading images for \", camera_path)\n",
    "        for filename in os.listdir(camera_path):\n",
    "            img = cv2.imread(os.path.join(camera_path, filename))\n",
    "            if img.shape[0] != 224 or img.shape[1] != 224 or img.shape[2] != 3:\n",
    "                print('Loaded an image with improper dimensions. Discarding')\n",
    "            else:\n",
    "                data += [img]\n",
    "                labels += [CameraLabel[label.name].value]\n",
    "    return np.asarray(data), np.expand_dims(np.asarray(labels), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading images for  data/validation/HTC-1-M7\n",
      "Loading images for  data/validation/LG-Nexus-5x\n",
      "Loading images for  data/validation/Motorola-Droid-Maxx\n",
      "Loading images for  data/validation/Motorola-Nexus-6\n",
      "Loaded an image with improper dimensions. Discarding\n",
      "Loading images for  data/validation/Motorola-X\n",
      "Loading images for  data/validation/Samsung-Galaxy-Note3\n",
      "Loading images for  data/validation/Samsung-Galaxy-S4\n",
      "Loading images for  data/validation/Sony-NEX-7\n",
      "Loading images for  data/validation/iPhone-4s\n",
      "Loading images for  data/validation/iPhone-6\n"
     ]
    }
   ],
   "source": [
    "valid_images, valid_labels = load_validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#images, labels = load(verbose=True)\n",
    "images, labels = load(path='data-512', load_ratio=0.5, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Good ol' tandem shuffle. \"\"\"\n",
    "def shuffle(a, b, c=None):\n",
    "    assert len(a) == len(b)\n",
    "    state = np.random.get_state()\n",
    "    np.random.shuffle(a)\n",
    "    np.random.set_state(state)\n",
    "    np.random.shuffle(b)\n",
    "    if c is not None:\n",
    "        assert len(c) == len(b)\n",
    "        np.random.set_state(state)\n",
    "        np.random.shuffle(c)\n",
    "  \n",
    "'''\n",
    "Crops a randomly-selected 224x224x3 image from a larger image\n",
    "'''\n",
    "def crop_224(images, crops_per_image=1):\n",
    "    out_images = []\n",
    "    for image in images:\n",
    "        row_index = np.random.randint(0, image.shape[0] - 224)\n",
    "        col_index = np.random.randint(0, image.shape[1] - 224)\n",
    "        out_images += [image[row_index:row_index + 224, col_index:col_index + 224, :]]\n",
    "    return np.asarray(out_images)\n",
    "\n",
    "def center_crop(img, crop_dim=224):\n",
    "    edge = crop_dim // 2\n",
    "    height, width, _ = img.shape\n",
    "    center_height = height // 2\n",
    "    center_width = width // 2\n",
    "    top = center_height - edge\n",
    "    bottom = center_height + edge\n",
    "    left = center_width - edge\n",
    "    right = center_width + edge\n",
    "    return img[top:bottom,left:right]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ntrain_data, train_labels = images[:validation_ratio], label_one_hot[:validation_ratio]\\nvalid_data, valid_labels = images[validation_ratio:], label_one_hot[validation_ratio:]\\ntraining_samples = len(train_data)\\nprint(train_data.shape, train_labels.shape)\\nprint(valid_data.shape, valid_labels.shape)\\n'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# Convert labels to one-hot (using tf for fun):\n",
    "one_hot_graph = tf.Graph()\n",
    "one_hot_session = tf.Session(graph=one_hot_graph)\n",
    "with one_hot_graph.as_default():\n",
    "    label_input = tf.placeholder(tf.int32)\n",
    "    one_hot = tf.one_hot(label_input, 10)\n",
    "label_one_hot = np.asarray(one_hot.eval(feed_dict={label_input: labels}, session=one_hot_session))\n",
    "#########\n",
    "'''\n",
    "label_one_hot = tf.keras.utils.to_categorical(labels, num_classes=10)\n",
    "valid_label_one_hot = tf.keras.utils.to_categorical(valid_labels, num_classes=10)\n",
    "validation_ratio = int(len(images) * 0.8)\n",
    "#shuffle(images, label_one_hot)  # Shuffle before dividing into training/validation sets\n",
    "'''\n",
    "train_data, train_labels = images[:validation_ratio], label_one_hot[:validation_ratio]\n",
    "valid_data, valid_labels = images[validation_ratio:], label_one_hot[validation_ratio:]\n",
    "training_samples = len(train_data)\n",
    "print(train_data.shape, train_labels.shape)\n",
    "print(valid_data.shape, valid_labels.shape)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24750, 224, 224, 3) (24750, 10) (4949, 224, 224, 3) (4949, 10)\n"
     ]
    }
   ],
   "source": [
    "print(images.shape, label_one_hot.shape, valid_images.shape, valid_label_one_hot.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffle(images, label_one_hot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Keras CNN\n",
    "We'll do this twice: once with a Sequential() model, and again with a Functional() model\n",
    "* 2+ Conv layers\n",
    "* 2 pooling\n",
    "* 2x 128-node Fully-Connected Layer\n",
    "* 10-node output layer\n",
    "* Batch Norm before all activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model_filename = 'keras_models/first_submission.h5'\n",
    "model_filename = 'keras_models/first_resnet.h5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def std(inputs):\n",
    "    return 1 / math.sqrt(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading previously-created model, keras_models/first_resnet.h5\n"
     ]
    }
   ],
   "source": [
    "if os.path.isfile(model_filename):\n",
    "    print('Loading previously-created model,', model_filename)\n",
    "    model = keras.models.load_model(model_filename)\n",
    "else:\n",
    "    model = keras.models.Sequential()\n",
    "    # Convolution & Pooling Layers\n",
    "    model.add(keras.layers.Conv2D(filters=64, kernel_size=7, strides=2, padding='same', input_shape=(224, 224, 3), kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(224*224*3))))\n",
    "    model.add(keras.layers.BatchNormalization())\n",
    "    model.add(keras.layers.Activation('relu'))\n",
    "    model.add(keras.layers.MaxPooling2D())  # defaults to 2x2\n",
    "    model.add(keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(5*5*32))))\n",
    "    model.add(keras.layers.BatchNormalization())\n",
    "    model.add(keras.layers.Activation('relu'))\n",
    "    model.add(keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(5*5*32))))\n",
    "    model.add(keras.layers.BatchNormalization())\n",
    "    model.add(keras.layers.Activation('relu'))\n",
    "    model.add(keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(5*5*32))))\n",
    "    model.add(keras.layers.BatchNormalization())\n",
    "    model.add(keras.layers.Activation('relu'))\n",
    "    model.add(keras.layers.MaxPooling2D())  # defaults to 2x2\n",
    "    # Fully-connected Layers\n",
    "    model.add(keras.layers.Flatten())\n",
    "    flatten_shape = model.layers[-1].output_shape[1]\n",
    "    model.add(keras.layers.Dense(128, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(128 * flatten_shape))))\n",
    "    model.add(keras.layers.BatchNormalization())\n",
    "    model.add(keras.layers.Activation('relu'))\n",
    "    model.add(keras.layers.Dense(128, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(128*128))))\n",
    "    model.add(keras.layers.BatchNormalization())\n",
    "    model.add(keras.layers.Activation('relu'))\n",
    "    model.add(keras.layers.Dense(10, activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam', #keras.optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 112, 112, 64) 9472        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 112, 112, 64) 256         conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 112, 112, 64) 0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 56, 56, 64)   0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 56, 56, 64)   4096        max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 56, 56, 64)   256         conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 56, 56, 64)   0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 56, 56, 64)   36864       activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 56, 56, 64)   256         conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 56, 56, 64)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 56, 56, 256)  16384       activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 56, 56, 256)  16384       max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 56, 56, 256)  1024        conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 56, 56, 256)  1024        conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 56, 56, 256)  0           batch_normalization_4[0][0]      \n",
      "                                                                 batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 56, 56, 256)  0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 56, 56, 256)  65536       activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 56, 56, 256)  1024        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 56, 56, 256)  0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 56, 56, 64)   147456      activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 56, 56, 64)   256         conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 56, 56, 64)   0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 56, 56, 256)  16384       activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 56, 56, 256)  1024        conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 56, 56, 256)  0           batch_normalization_8[0][0]      \n",
      "                                                                 activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 56, 56, 256)  0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 56, 56, 256)  65536       activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 56, 56, 256)  1024        conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 56, 56, 256)  0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 56, 56, 64)   147456      activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 56, 56, 64)   256         conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 56, 56, 64)   0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 56, 56, 256)  16384       activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 56, 56, 256)  1024        conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 56, 56, 256)  0           batch_normalization_11[0][0]     \n",
      "                                                                 activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 56, 56, 256)  0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 28, 28, 256)  65536       activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 28, 28, 256)  1024        conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 28, 28, 256)  0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 28, 28, 128)  294912      activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 28, 28, 128)  512         conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 28, 28, 128)  0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 28, 28, 512)  65536       activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 28, 28, 512)  131072      activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 28, 28, 512)  2048        conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 28, 28, 512)  2048        conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 28, 28, 512)  0           batch_normalization_14[0][0]     \n",
      "                                                                 batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 28, 28, 512)  0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 28, 28, 512)  262144      activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 28, 28, 512)  2048        conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 28, 28, 512)  0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 28, 28, 128)  589824      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 28, 28, 128)  512         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 28, 28, 128)  0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 28, 28, 512)  65536       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 28, 28, 512)  2048        conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 28, 28, 512)  0           batch_normalization_18[0][0]     \n",
      "                                                                 activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 28, 28, 512)  0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 28, 28, 512)  262144      activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 28, 28, 512)  2048        conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 28, 28, 512)  0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 28, 28, 128)  589824      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 28, 28, 128)  512         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 28, 28, 128)  0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 28, 28, 512)  65536       activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 28, 28, 512)  2048        conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 28, 28, 512)  0           batch_normalization_21[0][0]     \n",
      "                                                                 activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 28, 28, 512)  0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 28, 28, 512)  262144      activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 28, 28, 512)  2048        conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 28, 28, 512)  0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)              (None, 28, 28, 128)  589824      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 28, 28, 128)  512         conv2d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 28, 28, 128)  0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)              (None, 28, 28, 512)  65536       activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 28, 28, 512)  2048        conv2d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 28, 28, 512)  0           batch_normalization_24[0][0]     \n",
      "                                                                 activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 28, 28, 512)  0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)              (None, 14, 14, 512)  262144      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, 14, 14, 512)  2048        conv2d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 14, 14, 512)  0           batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)              (None, 14, 14, 256)  1179648     activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNo (None, 14, 14, 256)  1024        conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 14, 14, 256)  0           batch_normalization_26[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)              (None, 14, 14, 1024) 262144      activation_24[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)              (None, 14, 14, 1024) 524288      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNo (None, 14, 14, 1024) 4096        conv2d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_28 (BatchNo (None, 14, 14, 1024) 4096        conv2d_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 14, 14, 1024) 0           batch_normalization_27[0][0]     \n",
      "                                                                 batch_normalization_28[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_25 (Activation)      (None, 14, 14, 1024) 0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_29 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_29 (BatchNo (None, 14, 14, 1024) 4096        conv2d_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_26 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_29[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_30 (Conv2D)              (None, 14, 14, 256)  2359296     activation_26[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_30 (BatchNo (None, 14, 14, 256)  1024        conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_27 (Activation)      (None, 14, 14, 256)  0           batch_normalization_30[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_31 (Conv2D)              (None, 14, 14, 1024) 262144      activation_27[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNo (None, 14, 14, 1024) 4096        conv2d_31[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 14, 14, 1024) 0           batch_normalization_31[0][0]     \n",
      "                                                                 activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_28 (Activation)      (None, 14, 14, 1024) 0           add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_32 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_32 (BatchNo (None, 14, 14, 1024) 4096        conv2d_32[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_29 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_32[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_33 (Conv2D)              (None, 14, 14, 256)  2359296     activation_29[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_33 (BatchNo (None, 14, 14, 256)  1024        conv2d_33[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_30 (Activation)      (None, 14, 14, 256)  0           batch_normalization_33[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_34 (Conv2D)              (None, 14, 14, 1024) 262144      activation_30[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_34 (BatchNo (None, 14, 14, 1024) 4096        conv2d_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_34[0][0]     \n",
      "                                                                 activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_31 (Activation)      (None, 14, 14, 1024) 0           add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_35 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_35 (BatchNo (None, 14, 14, 1024) 4096        conv2d_35[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_32 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_35[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_36 (Conv2D)              (None, 14, 14, 256)  2359296     activation_32[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_36 (BatchNo (None, 14, 14, 256)  1024        conv2d_36[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_33 (Activation)      (None, 14, 14, 256)  0           batch_normalization_36[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_37 (Conv2D)              (None, 14, 14, 1024) 262144      activation_33[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_37 (BatchNo (None, 14, 14, 1024) 4096        conv2d_37[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_37[0][0]     \n",
      "                                                                 activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_34 (Activation)      (None, 14, 14, 1024) 0           add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_38 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_38 (BatchNo (None, 14, 14, 1024) 4096        conv2d_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_35 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_38[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_39 (Conv2D)              (None, 14, 14, 256)  2359296     activation_35[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_39 (BatchNo (None, 14, 14, 256)  1024        conv2d_39[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_36 (Activation)      (None, 14, 14, 256)  0           batch_normalization_39[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_40 (Conv2D)              (None, 14, 14, 1024) 262144      activation_36[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_40 (BatchNo (None, 14, 14, 1024) 4096        conv2d_40[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_40[0][0]     \n",
      "                                                                 activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_37 (Activation)      (None, 14, 14, 1024) 0           add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_41 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_41 (BatchNo (None, 14, 14, 1024) 4096        conv2d_41[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_38 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_41[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_42 (Conv2D)              (None, 14, 14, 256)  2359296     activation_38[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_42 (BatchNo (None, 14, 14, 256)  1024        conv2d_42[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_39 (Activation)      (None, 14, 14, 256)  0           batch_normalization_42[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_43 (Conv2D)              (None, 14, 14, 1024) 262144      activation_39[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_43 (BatchNo (None, 14, 14, 1024) 4096        conv2d_43[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_43[0][0]     \n",
      "                                                                 activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_40 (Activation)      (None, 14, 14, 1024) 0           add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_44 (Conv2D)              (None, 7, 7, 1024)   1048576     activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_44 (BatchNo (None, 7, 7, 1024)   4096        conv2d_44[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_41 (Activation)      (None, 7, 7, 1024)   0           batch_normalization_44[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_45 (Conv2D)              (None, 7, 7, 512)    4718592     activation_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_45 (BatchNo (None, 7, 7, 512)    2048        conv2d_45[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_42 (Activation)      (None, 7, 7, 512)    0           batch_normalization_45[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_46 (Conv2D)              (None, 7, 7, 2048)   1048576     activation_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_47 (Conv2D)              (None, 7, 7, 2048)   2097152     activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_46 (BatchNo (None, 7, 7, 2048)   8192        conv2d_46[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_47 (BatchNo (None, 7, 7, 2048)   8192        conv2d_47[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_46[0][0]     \n",
      "                                                                 batch_normalization_47[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_43 (Activation)      (None, 7, 7, 2048)   0           add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_48 (Conv2D)              (None, 7, 7, 2048)   4194304     activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_48 (BatchNo (None, 7, 7, 2048)   8192        conv2d_48[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_44 (Activation)      (None, 7, 7, 2048)   0           batch_normalization_48[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_49 (Conv2D)              (None, 7, 7, 512)    9437184     activation_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_49 (BatchNo (None, 7, 7, 512)    2048        conv2d_49[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_45 (Activation)      (None, 7, 7, 512)    0           batch_normalization_49[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_50 (Conv2D)              (None, 7, 7, 2048)   1048576     activation_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_50 (BatchNo (None, 7, 7, 2048)   8192        conv2d_50[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_50[0][0]     \n",
      "                                                                 activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_46 (Activation)      (None, 7, 7, 2048)   0           add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_51 (Conv2D)              (None, 7, 7, 2048)   4194304     activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_51 (BatchNo (None, 7, 7, 2048)   8192        conv2d_51[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_47 (Activation)      (None, 7, 7, 2048)   0           batch_normalization_51[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_52 (Conv2D)              (None, 7, 7, 512)    9437184     activation_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_52 (BatchNo (None, 7, 7, 512)    2048        conv2d_52[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_48 (Activation)      (None, 7, 7, 512)    0           batch_normalization_52[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_53 (Conv2D)              (None, 7, 7, 2048)   1048576     activation_48[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_53 (BatchNo (None, 7, 7, 2048)   8192        conv2d_53[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_53[0][0]     \n",
      "                                                                 activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_49 (Activation)      (None, 7, 7, 2048)   0           add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_1 (AveragePoo (None, 1, 1, 2048)   0           activation_49[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 2048)         0           average_pooling2d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           20490       flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 62,866,954\n",
      "Trainable params: 62,795,146\n",
      "Non-trainable params: 71,808\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shuffling training data\n",
      "Shuffling training data\n"
     ]
    }
   ],
   "source": [
    "batch_size = 50\n",
    "batches=1000\n",
    "training_samples = len(train_data)\n",
    "index = 0\n",
    "for i in range(batches):\n",
    "        \n",
    "    # Shuffle the data\n",
    "    if (index + 2 * batch_size) > training_samples :  # Just ignore the partial batch, whateverrrr\n",
    "        print('Shuffling training data')\n",
    "        shuffle(train_data, train_labels)\n",
    "        index = 0\n",
    "    else:\n",
    "        index += batch_size\n",
    "    batch = (train_data[index:index+batch_size], train_labels[index:index+batch_size])\n",
    "    model.train_on_batch(batch[0], batch[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need to shuffle once before training so the validation split will get a random sample\n",
    "shuffle(images, label_one_hot, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#loss_and_metrics = model.evaluate(train_data, train_labels, batch_size=50)\n",
    "#model.fit(train_data, train_labels, epochs=5, batch_size=50)\n",
    "\n",
    "''' Crop & Train Idea:\n",
    "To get validation data:\n",
    "Step through all images\n",
    "    center-crop 55 random images from each folder\n",
    "results in 4950 images in validation set -> 750MB\n",
    "\n",
    "1. Load 10000 (first or second half) images, which will take about 8GB of memory \n",
    "2. Center crop 4950 images (55 images from each folder for validation set)\n",
    "3. Random crop 20000 images from data (2 per image)\n",
    "4. Train for an epoch\n",
    "5. Repeat steps 3 & 4 for a few epochs\n",
    "6. Repeat from beginning, using the other half of images\n",
    "'''\n",
    "\n",
    "def train(model, data_path='data-512', load_ratio=0.25, validation_data=None, epochs=1, batch_size=16, crops_per_image=4, bonus_training=False):\n",
    "    \n",
    "    if validation_data is not None:\n",
    "        valid_images = validation_data[0]\n",
    "        valid_label_one_hot = tf.keras.utils.to_categorical(validation_data[1], num_classes=10)\n",
    "    else:\n",
    "        valid_images = None\n",
    "        valid_label_one_hot = None\n",
    "        \n",
    "    for i in range(epochs):\n",
    "        images, labels = load(path=data_path, load_ratio=load_ratio)\n",
    "        train_images = crop_224(images, crops_per_image=crops_per_image)\n",
    "        for i in range(1, crops_per_image):\n",
    "            train_images = np.concatenate((train_images, crop_224(images, crops_per_image=crops_per_image)))\n",
    "        labels = np.reshape(np.tile(labels[:,0], crops_per_image), (crops_per_image * len(labels), 1))\n",
    "        # Need to elementwise duplicate labels for crops_per_image > 1\n",
    "        train_labels = tf.keras.utils.to_categorical(labels, num_classes=10)\n",
    "        #model.fit(images, label_one_hot, epochs=5, batch_size=50, validation_split=0.2)\n",
    "        # using initial_epoch does not work for some reason. May work for functional models\n",
    "        print(train_images.shape, train_labels.shape)\n",
    "        print(valid_images.shape, valid_label_one_hot.shape)\n",
    "        model.fit(train_images, train_labels, epochs=1, batch_size=batch_size, validation_data=(valid_images, valid_label_one_hot), verbose=1)\n",
    "        # get some more mileage outta training before reloading\n",
    "        if bonus_training:\n",
    "            shuffle(train_images, train_labels)\n",
    "            model.fit(train_images, train_labels, epochs=1, batch_size=batch_size, validation_data=(valid_images, valid_label_one_hot), verbose=1)\n",
    "        \n",
    "        del images\n",
    "        del labels\n",
    "        del train_images\n",
    "        del train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading images\n",
      "(24480, 224, 224, 3) (24480, 10)\n",
      "(4949, 224, 224, 3) (4949, 10)\n",
      "Train on 24480 samples, validate on 4949 samples\n",
      "Epoch 1/1\n",
      "24480/24480 [==============================] - 54s 2ms/step - loss: 0.4869 - acc: 0.8312 - val_loss: 1.1828 - val_acc: 0.6937\n",
      "Train on 24480 samples, validate on 4949 samples\n",
      "Epoch 1/1\n",
      "24480/24480 [==============================] - 50s 2ms/step - loss: 0.3214 - acc: 0.8904 - val_loss: 0.6773 - val_acc: 0.7945\n"
     ]
    }
   ],
   "source": [
    "valid_label_one_hot = tf.keras.utils.to_categorical(valid_labels, num_classes=10)\n",
    "train(model, validation_data=(valid_images, valid_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(model_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_filename = 'submissions/fourth_submission.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2641,)\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Guide to making predictions:\n",
    "open file\n",
    "write header\n",
    "For each test image:\n",
    "    load the image\n",
    "    crop the image in multiple plocations\n",
    "    evaluate the image using the trained model and select label with highest confidence from all crops\n",
    "    write filename and category name to file\n",
    "save file\n",
    "close file\n",
    "submit file\n",
    "'''\n",
    "samples = 9\n",
    "test_dir = 'data/test'\n",
    "result = ['fname,camera']\n",
    "for filename in os.listdir(test_dir):\n",
    "    image = cv2.imread(os.path.join(test_dir, filename))\n",
    "    test_images = np.expand_dims(center_crop(image), axis=0)\n",
    "    for i in range(samples):\n",
    "        test_images = np.concatenate((test_images, crop_224([image])))\n",
    "    predictions = model.predict(test_images, batch_size=test_images.shape[0], verbose=0)\n",
    "    category = np.argmax(predictions) % 10\n",
    "    camera_name = CameraLabel(category).name.replace('_','-')\n",
    "    result += [filename + ',' + camera_name]\n",
    "    #print(result)\n",
    "    del image\n",
    "    del test_images\n",
    "result = np.asarray(result)\n",
    "\n",
    "np.savetxt(submission_filename, result, fmt=\"%s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative Predictions (single crop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pick the highest confidence\n",
    "test_dir = 'data/test'\n",
    "result = ['fname,camera']\n",
    "for filename in os.listdir(test_dir):\n",
    "    image = cv2.imread(os.path.join(test_dir, filename))\n",
    "    test_image = np.expand_dims(np.asarray(center_crop(image)), axis=0)\n",
    "    \n",
    "    # I should batch this, but whatever\n",
    "    category = np.argmax(model.predict(test_image, batch_size=1, verbose=0))\n",
    "    camera_name = CameraLabel(category).name.replace('_','-')\n",
    "    result += [filename + ',' + camera_name]\n",
    "    #print(result)\n",
    "    del image\n",
    "    del test_image\n",
    "result = np.asarray(result)\n",
    "\n",
    "np.savetxt(submission_filename, result, fmt=\"%s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functional Model\n",
    "Ok, so we're going to build a functional model and then try to build a resnet "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "The bottleneck block of a resnet has the following structure:\n",
    "    1x1 conv on the input features (no bias, no pad, 1 stride), batch norm w/ scaling, and followed by ReLU activation\n",
    "    3x3 conv w/ no bias, 1 pad, 1 stride, batch norm w/ scaling, ReLU follows\n",
    "    1x1 conv w/ 4x input features (no bias, no pad, 1 stride), batch norm w/ scaling, NO ACTIVATION\n",
    "'''\n",
    "def bottlenet_block(input_net, input_filters=64, down_sampling=False, feature_size=56, middle_filters_divisor=2, output_filters_scale=1):\n",
    "    \n",
    "    first_stride = 2 if down_sampling else 1\n",
    "    # if I want to initialize weights, I need to know incoming size, figure that out\n",
    "    \n",
    "    # First Convolution\n",
    "    block = keras.layers.Conv2D(filters=input_filters, kernel_size=1, strides=first_stride, use_bias=False, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(input_filters * feature_size ** 2)))(input_net)\n",
    "    block = keras.layers.BatchNormalization()(block)\n",
    "    block = keras.layers.Activation('relu')(block)\n",
    "    # Second Convolution\n",
    "    block = keras.layers.Conv2D(filters=input_filters // middle_filters_divisor, kernel_size=3, padding='same', strides=1, use_bias=False, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(input_filters // middle_filters_divisor * (feature_size / first_stride) ** 2)))(block)\n",
    "    block = keras.layers.BatchNormalization()(block)\n",
    "    block = keras.layers.Activation('relu')(block)\n",
    "    # Final Convolution (no activation)\n",
    "    block = keras.layers.Conv2D(filters=input_filters * output_filters_scale, kernel_size=1, strides=1, use_bias=False, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(input_filters * output_filters_scale * (feature_size / first_stride) ** 2)))(block)\n",
    "    block = keras.layers.BatchNormalization()(block)\n",
    "    \n",
    "    return block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "del net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.layers.Input(shape=(224, 224, 3))\n",
    "\n",
    "net = keras.layers.Conv2D(filters=64, kernel_size=7, strides=2, padding='same', input_shape=(224, 224, 3), kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(224*224*3)))(inputs)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "net = keras.layers.MaxPooling2D(pool_size=(3, 3), strides=2, padding='same')(net)\n",
    "\n",
    "\n",
    "# First Set of Bottlenecks\n",
    "\n",
    "bottle_1 = bottlenet_block(net, input_filters=64, feature_size=56, middle_filters_divisor=1, output_filters_scale=4)\n",
    "# Projection Function\n",
    "net = keras.layers.Conv2D(filters=256, kernel_size=1, use_bias=False, strides=1, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(56*56*256)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "# Merge Bottleneck and Projection\n",
    "net = keras.layers.Add()([bottle_1, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "\n",
    "bottle_2 = bottlenet_block(net, input_filters=256, feature_size=56, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_2, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_3 = bottlenet_block(net, input_filters=256, feature_size=56, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_3, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "# Second Set of Bottlenecks\n",
    "bottle_4 = bottlenet_block(net, input_filters=256, down_sampling=True, feature_size=56, middle_filters_divisor=2, output_filters_scale=2)\n",
    "net = keras.layers.Conv2D(filters=512, kernel_size=1, use_bias=False, strides=2, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(28*28*512)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "# Merge Bottleneck and Projection\n",
    "net = keras.layers.Add()([bottle_4, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "\n",
    "bottle_5 = bottlenet_block(net, input_filters=512, feature_size=28, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_5, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_6 = bottlenet_block(net, input_filters=512, feature_size=28, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_6, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_7 = bottlenet_block(net, input_filters=512, feature_size=28, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_7, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "# Third Set of Bottlenecks\n",
    "bottle_8 = bottlenet_block(net, input_filters=512, down_sampling=True, feature_size=28, middle_filters_divisor=2, output_filters_scale=2)\n",
    "net = keras.layers.Conv2D(filters=1024, kernel_size=1, use_bias=False, strides=2, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(14*14*1024)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "# Merge Bottleneck and Projection\n",
    "net = keras.layers.Add()([bottle_8, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_9 = bottlenet_block(net, input_filters=1024, feature_size=14, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_9, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_10 = bottlenet_block(net, input_filters=1024, feature_size=14, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_10, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_11 = bottlenet_block(net, input_filters=1024, feature_size=14, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_11, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_12 = bottlenet_block(net, input_filters=1024, feature_size=14, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_12, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_13 = bottlenet_block(net, input_filters=1024, feature_size=14, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_13, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "# Fourth Set of Bottlenecks\n",
    "bottle_14 = bottlenet_block(net, input_filters=1024, down_sampling=True, feature_size=14, middle_filters_divisor=2, output_filters_scale=2)\n",
    "net = keras.layers.Conv2D(filters=2048, kernel_size=1, use_bias=False, strides=2, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(7*7*2048)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "# Merge Bottleneck and Projection\n",
    "net = keras.layers.Add()([bottle_14, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_15 = bottlenet_block(net, input_filters=2048, feature_size=7, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_15, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "bottle_16 = bottlenet_block(net, input_filters=2048, feature_size=7, middle_filters_divisor=4, output_filters_scale=1)\n",
    "net = keras.layers.Add()([bottle_16, net])\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "# Final Pooling. Should have 2048 outputs\n",
    "net = keras.layers.AveragePooling2D(pool_size=(7,7))(net)\n",
    "\n",
    "# Final Fully-Connected Layers\n",
    "flatten_shape = 2048\n",
    "net = keras.layers.Flatten()(net)\n",
    "#net = keras.layers.Dense(1000, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(128 * flatten_shape)))(net)\n",
    "#net = keras.layers.BatchNormalization()(net)\n",
    "#net = keras.layers.Activation('relu')(net)\n",
    "\n",
    "predictions = keras.layers.Dense(10, activation='softmax', kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(10 * flatten_shape)))(net)\n",
    "\n",
    "model = keras.layers.Model(inputs=inputs, outputs=predictions)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam', #keras.optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 112, 112, 64) 9472        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 112, 112, 64) 256         conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 112, 112, 64) 0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 56, 56, 64)   0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 56, 56, 64)   4096        max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 56, 56, 64)   256         conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 56, 56, 64)   0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 56, 56, 64)   36864       activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 56, 56, 64)   256         conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 56, 56, 64)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 56, 56, 256)  16384       activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 56, 56, 256)  16384       max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 56, 56, 256)  1024        conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 56, 56, 256)  1024        conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 56, 56, 256)  0           batch_normalization_4[0][0]      \n",
      "                                                                 batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 56, 56, 256)  0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 56, 56, 256)  65536       activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 56, 56, 256)  1024        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 56, 56, 256)  0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 56, 56, 64)   147456      activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 56, 56, 64)   256         conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 56, 56, 64)   0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 56, 56, 256)  16384       activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 56, 56, 256)  1024        conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 56, 56, 256)  0           batch_normalization_8[0][0]      \n",
      "                                                                 activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 56, 56, 256)  0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 56, 56, 256)  65536       activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 56, 56, 256)  1024        conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 56, 56, 256)  0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 56, 56, 64)   147456      activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 56, 56, 64)   256         conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 56, 56, 64)   0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 56, 56, 256)  16384       activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 56, 56, 256)  1024        conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 56, 56, 256)  0           batch_normalization_11[0][0]     \n",
      "                                                                 activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 56, 56, 256)  0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 28, 28, 256)  65536       activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 28, 28, 256)  1024        conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 28, 28, 256)  0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 28, 28, 128)  294912      activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 28, 28, 128)  512         conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 28, 28, 128)  0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 28, 28, 512)  65536       activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 28, 28, 512)  131072      activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 28, 28, 512)  2048        conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 28, 28, 512)  2048        conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 28, 28, 512)  0           batch_normalization_14[0][0]     \n",
      "                                                                 batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 28, 28, 512)  0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 28, 28, 512)  262144      activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 28, 28, 512)  2048        conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 28, 28, 512)  0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 28, 28, 128)  589824      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 28, 28, 128)  512         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 28, 28, 128)  0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 28, 28, 512)  65536       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 28, 28, 512)  2048        conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 28, 28, 512)  0           batch_normalization_18[0][0]     \n",
      "                                                                 activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 28, 28, 512)  0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 28, 28, 512)  262144      activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 28, 28, 512)  2048        conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 28, 28, 512)  0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 28, 28, 128)  589824      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 28, 28, 128)  512         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 28, 28, 128)  0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 28, 28, 512)  65536       activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 28, 28, 512)  2048        conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 28, 28, 512)  0           batch_normalization_21[0][0]     \n",
      "                                                                 activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 28, 28, 512)  0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 28, 28, 512)  262144      activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 28, 28, 512)  2048        conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 28, 28, 512)  0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)              (None, 28, 28, 128)  589824      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 28, 28, 128)  512         conv2d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 28, 28, 128)  0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)              (None, 28, 28, 512)  65536       activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 28, 28, 512)  2048        conv2d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 28, 28, 512)  0           batch_normalization_24[0][0]     \n",
      "                                                                 activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 28, 28, 512)  0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)              (None, 14, 14, 512)  262144      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, 14, 14, 512)  2048        conv2d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 14, 14, 512)  0           batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)              (None, 14, 14, 256)  1179648     activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNo (None, 14, 14, 256)  1024        conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 14, 14, 256)  0           batch_normalization_26[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)              (None, 14, 14, 1024) 262144      activation_24[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)              (None, 14, 14, 1024) 524288      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNo (None, 14, 14, 1024) 4096        conv2d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_28 (BatchNo (None, 14, 14, 1024) 4096        conv2d_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 14, 14, 1024) 0           batch_normalization_27[0][0]     \n",
      "                                                                 batch_normalization_28[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_25 (Activation)      (None, 14, 14, 1024) 0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_29 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_29 (BatchNo (None, 14, 14, 1024) 4096        conv2d_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_26 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_29[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_30 (Conv2D)              (None, 14, 14, 256)  2359296     activation_26[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_30 (BatchNo (None, 14, 14, 256)  1024        conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_27 (Activation)      (None, 14, 14, 256)  0           batch_normalization_30[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_31 (Conv2D)              (None, 14, 14, 1024) 262144      activation_27[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNo (None, 14, 14, 1024) 4096        conv2d_31[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 14, 14, 1024) 0           batch_normalization_31[0][0]     \n",
      "                                                                 activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_28 (Activation)      (None, 14, 14, 1024) 0           add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_32 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_32 (BatchNo (None, 14, 14, 1024) 4096        conv2d_32[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_29 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_32[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_33 (Conv2D)              (None, 14, 14, 256)  2359296     activation_29[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_33 (BatchNo (None, 14, 14, 256)  1024        conv2d_33[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_30 (Activation)      (None, 14, 14, 256)  0           batch_normalization_33[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_34 (Conv2D)              (None, 14, 14, 1024) 262144      activation_30[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_34 (BatchNo (None, 14, 14, 1024) 4096        conv2d_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_34[0][0]     \n",
      "                                                                 activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_31 (Activation)      (None, 14, 14, 1024) 0           add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_35 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_35 (BatchNo (None, 14, 14, 1024) 4096        conv2d_35[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_32 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_35[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_36 (Conv2D)              (None, 14, 14, 256)  2359296     activation_32[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_36 (BatchNo (None, 14, 14, 256)  1024        conv2d_36[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_33 (Activation)      (None, 14, 14, 256)  0           batch_normalization_36[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_37 (Conv2D)              (None, 14, 14, 1024) 262144      activation_33[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_37 (BatchNo (None, 14, 14, 1024) 4096        conv2d_37[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_37[0][0]     \n",
      "                                                                 activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_34 (Activation)      (None, 14, 14, 1024) 0           add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_38 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_38 (BatchNo (None, 14, 14, 1024) 4096        conv2d_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_35 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_38[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_39 (Conv2D)              (None, 14, 14, 256)  2359296     activation_35[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_39 (BatchNo (None, 14, 14, 256)  1024        conv2d_39[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_36 (Activation)      (None, 14, 14, 256)  0           batch_normalization_39[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_40 (Conv2D)              (None, 14, 14, 1024) 262144      activation_36[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_40 (BatchNo (None, 14, 14, 1024) 4096        conv2d_40[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_40[0][0]     \n",
      "                                                                 activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_37 (Activation)      (None, 14, 14, 1024) 0           add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_41 (Conv2D)              (None, 14, 14, 1024) 1048576     activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_41 (BatchNo (None, 14, 14, 1024) 4096        conv2d_41[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_38 (Activation)      (None, 14, 14, 1024) 0           batch_normalization_41[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_42 (Conv2D)              (None, 14, 14, 256)  2359296     activation_38[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_42 (BatchNo (None, 14, 14, 256)  1024        conv2d_42[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_39 (Activation)      (None, 14, 14, 256)  0           batch_normalization_42[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_43 (Conv2D)              (None, 14, 14, 1024) 262144      activation_39[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_43 (BatchNo (None, 14, 14, 1024) 4096        conv2d_43[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_43[0][0]     \n",
      "                                                                 activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_40 (Activation)      (None, 14, 14, 1024) 0           add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_44 (Conv2D)              (None, 7, 7, 1024)   1048576     activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_44 (BatchNo (None, 7, 7, 1024)   4096        conv2d_44[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_41 (Activation)      (None, 7, 7, 1024)   0           batch_normalization_44[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_45 (Conv2D)              (None, 7, 7, 512)    4718592     activation_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_45 (BatchNo (None, 7, 7, 512)    2048        conv2d_45[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_42 (Activation)      (None, 7, 7, 512)    0           batch_normalization_45[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_46 (Conv2D)              (None, 7, 7, 2048)   1048576     activation_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_47 (Conv2D)              (None, 7, 7, 2048)   2097152     activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_46 (BatchNo (None, 7, 7, 2048)   8192        conv2d_46[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_47 (BatchNo (None, 7, 7, 2048)   8192        conv2d_47[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_46[0][0]     \n",
      "                                                                 batch_normalization_47[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_43 (Activation)      (None, 7, 7, 2048)   0           add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_48 (Conv2D)              (None, 7, 7, 2048)   4194304     activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_48 (BatchNo (None, 7, 7, 2048)   8192        conv2d_48[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_44 (Activation)      (None, 7, 7, 2048)   0           batch_normalization_48[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_49 (Conv2D)              (None, 7, 7, 512)    9437184     activation_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_49 (BatchNo (None, 7, 7, 512)    2048        conv2d_49[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_45 (Activation)      (None, 7, 7, 512)    0           batch_normalization_49[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_50 (Conv2D)              (None, 7, 7, 2048)   1048576     activation_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_50 (BatchNo (None, 7, 7, 2048)   8192        conv2d_50[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_50[0][0]     \n",
      "                                                                 activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_46 (Activation)      (None, 7, 7, 2048)   0           add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_51 (Conv2D)              (None, 7, 7, 2048)   4194304     activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_51 (BatchNo (None, 7, 7, 2048)   8192        conv2d_51[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_47 (Activation)      (None, 7, 7, 2048)   0           batch_normalization_51[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_52 (Conv2D)              (None, 7, 7, 512)    9437184     activation_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_52 (BatchNo (None, 7, 7, 512)    2048        conv2d_52[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_48 (Activation)      (None, 7, 7, 512)    0           batch_normalization_52[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_53 (Conv2D)              (None, 7, 7, 2048)   1048576     activation_48[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_53 (BatchNo (None, 7, 7, 2048)   8192        conv2d_53[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_53[0][0]     \n",
      "                                                                 activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_49 (Activation)      (None, 7, 7, 2048)   0           add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_1 (AveragePoo (None, 1, 1, 2048)   0           activation_49[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 2048)         0           average_pooling2d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           20490       flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 62,866,954\n",
      "Trainable params: 62,795,146\n",
      "Non-trainable params: 71,808\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading images\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-a3bba7875f7b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrops_per_image\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-14-c1849db61f9e>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, data_path, load_ratio, validation_data, epochs, batch_size, crops_per_image, bonus_training)\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0mtrain_images\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcrop_224\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrops_per_image\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcrops_per_image\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrops_per_image\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m             \u001b[0mtrain_images\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrop_224\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrops_per_image\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcrops_per_image\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m         \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrops_per_image\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mcrops_per_image\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;31m# Need to elementwise duplicate labels for crops_per_image > 1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-ecba662f33f5>\u001b[0m in \u001b[0;36mcrop_224\u001b[0;34m(images, crops_per_image)\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mcol_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m224\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mout_images\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrow_index\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mrow_index\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m224\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcol_index\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mcol_index\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m224\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_images\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcenter_crop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrop_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m224\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/numpy/core/numeric.py\u001b[0m in \u001b[0;36masarray\u001b[0;34m(a, dtype, order)\u001b[0m\n\u001b[1;32m    529\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    530\u001b[0m     \"\"\"\n\u001b[0;32m--> 531\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    532\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train(model, validation_data=(valid_images, valid_labels), epochs=1, crops_per_image=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('keras_models/first_resnet.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This returns a tensor\n",
    "inputs = keras.layers.Input(shape=(224, 224, 3))\n",
    "\n",
    "net = keras.layers.Conv2D(filters=64, kernel_size=7, strides=2, padding='same', input_shape=(224, 224, 3), kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(224*224*3)))(inputs)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "net = keras.layers.MaxPooling2D()(net)  # defaults to 2x2\n",
    "net = keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(5*5*32)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "net = keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(5*5*32)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "net = keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(5*5*32)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "net = keras.layers.MaxPooling2D()(net)  # defaults to 2x2\n",
    "# Fully-connected Layers\n",
    "net = keras.layers.Flatten()(net)\n",
    "#flatten_shape = net.layers[-1].output_shape[1]\n",
    "flatten_shape = 50176  # currently bused\n",
    "net = keras.layers.Dense(128, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(128 * flatten_shape)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "net = keras.layers.Dense(128, kernel_initializer=keras.initializers.TruncatedNormal(stddev=std(128*128)))(net)\n",
    "net = keras.layers.BatchNormalization()(net)\n",
    "net = keras.layers.Activation('relu')(net)\n",
    "predictions = keras.layers.Dense(10, activation='softmax')(net)\n",
    "                          \n",
    "model = keras.layers.Model(inputs=inputs, outputs=predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_17 (Conv2D)           (None, 112, 112, 64)      9472      \n",
      "_________________________________________________________________\n",
      "batch_normalization_21 (Batc (None, 112, 112, 64)      256       \n",
      "_________________________________________________________________\n",
      "activation_21 (Activation)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_9 (MaxPooling2 (None, 56, 56, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_18 (Conv2D)           (None, 56, 56, 64)        36928     \n",
      "_________________________________________________________________\n",
      "batch_normalization_22 (Batc (None, 56, 56, 64)        256       \n",
      "_________________________________________________________________\n",
      "activation_22 (Activation)   (None, 56, 56, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_19 (Conv2D)           (None, 56, 56, 64)        36928     \n",
      "_________________________________________________________________\n",
      "batch_normalization_23 (Batc (None, 56, 56, 64)        256       \n",
      "_________________________________________________________________\n",
      "activation_23 (Activation)   (None, 56, 56, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_20 (Conv2D)           (None, 56, 56, 64)        36928     \n",
      "_________________________________________________________________\n",
      "batch_normalization_24 (Batc (None, 56, 56, 64)        256       \n",
      "_________________________________________________________________\n",
      "activation_24 (Activation)   (None, 56, 56, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_10 (MaxPooling (None, 28, 28, 64)        0         \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 50176)             0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 128)               6422656   \n",
      "_________________________________________________________________\n",
      "batch_normalization_25 (Batc (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "activation_25 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "batch_normalization_26 (Batc (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "activation_26 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 6,562,762\n",
      "Trainable params: 6,561,738\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam', #keras.optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loss_and_metrics = model.evaluate(train_data, train_labels, batch_size=50)\n",
    "#model.fit(train_data, train_labels, epochs=5, batch_size=50)\n",
    "\n",
    "''' Crop & Train Idea:\n",
    "To get validation data:\n",
    "Step through all images\n",
    "    center-crop 55 random images from each folder\n",
    "results in 4950 images in validation set -> 750MB\n",
    "\n",
    "1. Load 10000 (first or second half) images, which will take about 8GB of memory \n",
    "2. Center crop 4950 images (55 images from each folder for validation set)\n",
    "3. Random crop 20000 images from data (2 per image)\n",
    "4. Train for an epoch\n",
    "5. Repeat steps 3 & 4 for a few epochs\n",
    "6. Repeat from beginning, using the other half of images\n",
    "'''\n",
    "\n",
    "def train(model, data_path='data-512', load_ratio=0.25, validation_data=None, epochs=1, crops_per_image=4):\n",
    "    valid_images = validation_data[0]\n",
    "    valid_label_one_hot = tf.keras.utils.to_categorical(validation_data[1], num_classes=10)\n",
    "\n",
    "    for i in range(epochs):\n",
    "        images, labels = load(path=data_path, load_ratio=load_ratio)\n",
    "        train_images = crop_224(images, crops_per_image=crops_per_image)\n",
    "        for i in range(1, crops_per_image):\n",
    "            train_images = np.concatenate((train_images, crop_224(images, crops_per_image=crops_per_image)))\n",
    "        labels = np.reshape(np.tile(labels[:,0], crops_per_image), (crops_per_image * len(labels), 1))\n",
    "        # Need to elementwise duplicate labels for crops_per_image > 1\n",
    "        train_labels = tf.keras.utils.to_categorical(labels, num_classes=10)\n",
    "        #model.fit(images, label_one_hot, epochs=5, batch_size=50, validation_split=0.2)\n",
    "        # using initial_epoch does not work for some reason. May work for functional models\n",
    "        model.fit(train_images, train_labels, epochs=1, batch_size=50, validation_data=(valid_images, valid_label_one_hot), verbose=1)\n",
    "        # get some more mileage outta training before reloading\n",
    "        shuffle(train_images, train_labels)\n",
    "        model.fit(train_images, train_labels, epochs=1, batch_size=50, validation_data=(valid_images, valid_label_one_hot), verbose=1)\n",
    "        del images\n",
    "        del labels\n",
    "        del train_images\n",
    "        del train_labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}