FUNDUS-DEEP-NET-AUGMENTED.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "k-WXDV3Tk6N6"
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3EXwoz-KHtWO"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "print(sys.executable)\n",
    "import tensorflow as tf\n",
    "print(\"Built with CUDA:\", tf.test.is_built_with_cuda())\n",
    "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check gpu availability\n",
    "print(\"GPU Available: \", tf.config.list_physical_devices('GPU'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RX4GbMpwk8Y5"
   },
   "source": [
    "## Prepare the Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OLuUvE7ieuAl"
   },
   "source": [
    "You will prepare the train and test sets a little differently this time. Instead of just normalizing the images, you will also introduce random noise and the generated images will be used as input to your model. The target or label will still be the clean images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ub3k-XfMeTol"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "\n",
    "train_dataset = \"E:\\\\OIA-ODIR\\\\Training Set\\\\Preprocessed_Train_Images\"\n",
    "\n",
    "\n",
    "test_dataset = \"E:\\\\OIA-ODIR\\\\Off-site Test Set\\\\Images\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "extended.to_excel(\"E:\\\\OIA-ODIR\\\\Training Set\\\\Annotation\\\\extended.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "extended = extended.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "annotations = pd.read_excel(\"E:\\\\OIA-ODIR\\\\Training Set\\\\Annotation\\\\training annotation (English).xlsx\")\n",
    "annotations_off_test = pd.read_excel(\"E:\\\\OIA-ODIR\\\\Off-site Test Set\\\\Annotation\\\\off-site test annotation (English).xlsx\")\n",
    "annotations_on_test = pd.read_excel(\"E:\\\\OIA-ODIR\\\\On-site Test Set\\\\Annotation\\\\on-site test annotation (English).xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Classes = [\"Normal\", \"Diabetic Retinopathy\", \"Glaucoma\", \"Cataract\", \"AMD\", \"Hypertension\", \"Myopia\", \"Other\"]\n",
    "y = extended[[\"N\", \"D\", \"G\", \"C\", \"A\", \"H\", \"M\", \"O\"]]\n",
    "y_off_test = annotations_off_test[[\"N\", \"D\", \"G\", \"C\", \"A\", \"H\", \"M\", \"O\"]]\n",
    "y_on_test = annotations_on_test[[\"N\", \"D\", \"G\", \"C\", \"A\", \"H\", \"M\", \"O\"]]\n",
    "# y = old_anno[[\"N\", \"D\", \"G\", \"C\", \"A\", \"H\", \"M\", \"O\"]]\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = extended[['Left-Fundus', \"Right-Fundus\", \"Patient Age\", \"Patient Sex\"]]\n",
    "X_off_test = annotations_off_test[['Left-Fundus', \"Right-Fundus\", \"Patient Age\", \"Patient Sex\"]]\n",
    "X_on_test = annotations_on_test[['Left-Fundus', \"Right-Fundus\", \"Patient Age\", \"Patient Sex\"]]\n",
    "# X_train = old_anno[['Left-Fundus', \"Right-Fundus\"]]\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def generate_batches(dataset_path, batch_size, X_train, y, data_size):\n",
    "    while True:\n",
    "        indices = np.arange(data_size)\n",
    "        np.random.shuffle(indices)\n",
    "        for start in range(0, data_size, batch_size):\n",
    "            end = min(start + batch_size, data_size)\n",
    "            batch_indices = indices[start:end]\n",
    "\n",
    "            batch_x = X_train.iloc[batch_indices]\n",
    "            batch_y = y.iloc[batch_indices]\n",
    "         \n",
    "            left_images = [cv2.cvtColor(cv2.imread(os.path.join(dataset_path, key)), cv2.COLOR_BGR2RGB) for key in batch_x['Left-Fundus']]\n",
    "            right_images = [cv2.cvtColor(cv2.imread(os.path.join(dataset_path, key)), cv2.COLOR_BGR2RGB) for key in batch_x['Right-Fundus']]\n",
    "\n",
    "            ages = batch_x['Patient Age'].values\n",
    "            sexes = (batch_x[\"Patient Sex\"] == 'Male').astype(int).values\n",
    "            \n",
    "            yield (np.array(left_images), np.array(right_images), ages, sexes), np.array(batch_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_images = \"E:\\\\OIA-ODIR\\\\Augmented\\\\Training Set\\\\Images\"\n",
    "off_test_path=\"E:\\\\OIA-ODIR\\\\Off-site Test Set\\\\off_site_preprocessed_images\"\n",
    "off_test_generator = generate_batches(off_test_path, 52, X_off_test, y_off_test, 500)\n",
    "# on_test_generator = generate_batches(on_test_path , 52, X_on_test, y_on_test, 1000)\n",
    "train_generator = generate_batches(preprocessed_images, 52, X_train, y, 18013)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## New Model with HR Net Included"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Layer, Input, Conv2D, BatchNormalization, ReLU, Add, GlobalAveragePooling2D, Dense , Dropout , Concatenate ,Multiply , Flatten\n",
    "from tensorflow.keras.models import Model\n",
    "def conv3x3(out_planes, stride=1):\n",
    "    \"\"\"3x3 convolution with padding\"\"\"\n",
    "    return Conv2D(out_planes, kernel_size=3, strides=stride, padding='same', use_bias=False)\n",
    "\n",
    "def bn_relu(x):\n",
    "    x = BatchNormalization()(x)\n",
    "    x = ReLU()(x)\n",
    "    return x\n",
    "\n",
    "class BasicBlock(Layer):\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, planes, stride=1, downsample=None):\n",
    "        super(BasicBlock, self).__init__()\n",
    "        self.conv1 = conv3x3(planes, stride)\n",
    "        self.bn1 = BatchNormalization()\n",
    "        self.relu = ReLU()\n",
    "        self.conv2 = conv3x3(planes)\n",
    "        self.bn2 = BatchNormalization()\n",
    "        self.downsample = downsample\n",
    "        self.stride = stride\n",
    "\n",
    "    def call(self, x):\n",
    "        identity = x\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "\n",
    "        if self.downsample is not None:\n",
    "            identity = self.downsample(x)\n",
    "\n",
    "        out += identity\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out\n",
    "def make_layer(block, blocks, planes, stride=1):\n",
    "    downsample = None\n",
    "    if stride != 1:\n",
    "        downsample = tf.keras.Sequential([\n",
    "            Conv2D(planes * block.expansion, kernel_size=1, strides=stride, use_bias=False),\n",
    "            BatchNormalization(),\n",
    "        ])\n",
    "\n",
    "    layers = []\n",
    "    layers.append(block(planes, stride, downsample))\n",
    "    for _ in range(1, blocks):\n",
    "        layers.append(block(planes))\n",
    "\n",
    "    return tf.keras.Sequential(layers)\n",
    "\n",
    "class HighResolutionModule(Layer):\n",
    "    def __init__(self, num_blocks, planes):\n",
    "        super(HighResolutionModule, self).__init__()\n",
    "        self.layer1 = make_layer(BasicBlock, num_blocks, planes, stride=1)\n",
    "        self.layer2 = make_layer(BasicBlock, num_blocks, planes, stride=2)\n",
    "\n",
    "    def call(self, x):\n",
    "        high_res_input = x\n",
    "        low_res_input = tf.keras.layers.AveragePooling2D(pool_size=(2, 2), strides=2)(x)\n",
    "\n",
    "        high_res_output = self.layer1(high_res_input)\n",
    "        low_res_output = self.layer2(low_res_input)\n",
    "\n",
    "        # Upsampling to match dimensions\n",
    "        low_res_output = tf.image.resize(low_res_output, (high_res_output.shape[1], high_res_output.shape[2]))\n",
    "\n",
    "        return tf.concat([high_res_output, low_res_output], axis=-1)\n",
    "\n",
    "class HRNet(Model):\n",
    "    def __init__(self):\n",
    "        super(HRNet, self).__init__()\n",
    "        self.initial_layer = Conv2D(64, kernel_size=3, strides=2, padding='same', activation='relu')\n",
    "        self.stage1 = HighResolutionModule(num_blocks=4, planes=64)\n",
    "        self.transition = Conv2D(128, kernel_size=3, strides=2, padding='same', activation='relu')\n",
    "        self.stage2 = HighResolutionModule(num_blocks=3, planes=128)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        x = self.initial_layer(inputs)\n",
    "        x = self.stage1(x)\n",
    "        x = self.transition(x)\n",
    "        x = self.stage2(x)\n",
    "        return x\n",
    "\n",
    "# make the attention block similar to below cell \n",
    "class CustomAttentionBlock(Layer):\n",
    "    def __init__(self, filters):\n",
    "        super(CustomAttentionBlock, self).__init__()\n",
    "        self.ds_conv = Conv2D(filters, kernel_size=(3, 3), padding='same', groups=filters)\n",
    "        self.p_conv = Conv2D(filters=filters, kernel_size=(1, 1), padding='same')\n",
    "        self.bn = BatchNormalization()\n",
    "        self.relu = ReLU()\n",
    "        self.concat = Concatenate(axis=-1)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        ds_features = self.ds_conv(inputs)\n",
    "        p_features = self.p_conv(ds_features)\n",
    "        p_features = self.bn(p_features)\n",
    "        p_features = self.relu(p_features)\n",
    "        concatenated_features = self.concat([inputs, p_features])\n",
    "        return concatenated_features\n",
    "    \n",
    "#define the sener block\n",
    "class SEBlock(Layer):\n",
    "    def __init__(self, reduction=16):\n",
    "        super(SEBlock, self).__init__()\n",
    "        self.reduction = reduction\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.avg_pool = GlobalAveragePooling2D()\n",
    "        self.fc1 = Dense(input_shape[-1] // self.reduction, activation='relu')\n",
    "        self.fc2 = Dense(input_shape[-1], activation='sigmoid')\n",
    "\n",
    "    def call(self, inputs):\n",
    "        x = self.avg_pool(inputs)\n",
    "        x = self.fc1(x)\n",
    "        x = self.fc2(x)\n",
    "        x = tf.reshape(x, [-1, 1, 1, inputs.shape[-1]])\n",
    "        return inputs * x\n",
    "\n",
    "class DRBM(Layer):\n",
    "    def __init__(self, num_visible, num_hidden, num_classes):\n",
    "        super(DRBM, self).__init__()\n",
    "        self.num_visible = num_visible\n",
    "        self.num_hidden = num_hidden\n",
    "        self.num_classes = num_classes\n",
    "        # Rename 'weights' to avoid conflicts with internal Layer properties\n",
    "        self.weights_h = self.add_weight(shape=(num_visible, num_hidden), initializer=\"random_normal\")\n",
    "        self.class_weights_h = self.add_weight(shape=(num_hidden, num_classes), initializer=\"random_normal\")\n",
    "        self.hidden_bias = self.add_weight(shape=(num_hidden,), initializer=\"zeros\")\n",
    "        self.class_bias = self.add_weight(shape=(num_classes,), initializer=\"zeros\")\n",
    "\n",
    "    def call(self, inputs):\n",
    "        # Adjust the matrix multiplication to use the new attribute names\n",
    "        hidden = tf.nn.relu(tf.matmul(inputs, self.weights_h) + self.hidden_bias)\n",
    "        class_logits = tf.matmul(hidden, self.class_weights_h) + self.class_bias\n",
    "        return tf.nn.sigmoid(class_logits)\n",
    "\n",
    "    \n",
    "def create_model(input_shape=(224, 224, 3), num_classes=8):\n",
    "    # Input layers for left and right images\n",
    "    input_left = Input(shape=input_shape, name='input_left')\n",
    "    input_right = Input(shape=input_shape, name='input_right')\n",
    "\n",
    "    # Backbone HRNet for feature extraction\n",
    "    hrnet = HRNet()\n",
    "\n",
    "    # Feature extraction for left and right images\n",
    "    features_left = hrnet(input_left)\n",
    "    features_right = hrnet(input_right)\n",
    "\n",
    "    # Attention blocks for left and right images\n",
    "    attention_block = CustomAttentionBlock(filters=256)  \n",
    "    attention_features_left = attention_block(features_left)\n",
    "    attention_features_right = attention_block(features_right)\n",
    "\n",
    "    # SENet blocks for left and right images\n",
    "    se_block = SEBlock()\n",
    "    se_features_left = se_block(attention_features_left)\n",
    "    se_features_right = se_block(attention_features_right)\n",
    "\n",
    "\n",
    "    multiplied_features = Multiply()([se_features_left, se_features_right]) \n",
    "\n",
    "\n",
    "    flattened_features = GlobalAveragePooling2D()(multiplied_features)\n",
    "    flattened_features = Flatten()(flattened_features)\n",
    "\n",
    "    drbm_output = DRBM(num_visible=flattened_features.shape[-1], num_hidden=1024, num_classes=num_classes)(flattened_features)\n",
    "\n",
    "    model = Model(inputs=[input_left, input_right], outputs=drbm_output)\n",
    "    return model\n",
    "\n",
    "model = create_model()\n",
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile your model\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0006), loss='binary_crossentropy', metrics=['AUC', 'Accuracy'])\n",
    "\n",
    "# Determine the number of steps per epoch (number of batches)\n",
    "num_samples = 18013  # Total number of samples in your dataset\n",
    "batch_size = 25  # Batch size used by your generator\n",
    "steps_per_epoch = num_samples // batch_size\n",
    "val_steps = 500 // batch_size\n",
    "# Train your model using the generator\n",
    "hist = model.fit(train_generator, steps_per_epoch=steps_per_epoch, epochs=10, \n",
    "         validation_data=off_test_generator, validation_steps=val_steps)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Augmentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "images_labels_dict = {}\n",
    "for i in range(len(annotations)):\n",
    "    image_name = annotations.iloc[i, 3]\n",
    "    labels = annotations.iloc[i, 7:].to_numpy()\n",
    "    images_labels_dict[image_name] = labels\n",
    "    \n",
    "images_labels_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import albumentations as A\n",
    "import cv2\n",
    "import os\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "\n",
    "# Define augmentations\n",
    "augmentations = A.Compose([\n",
    "    A.Rotate(limit=40),\n",
    "    A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=0.5),\n",
    "    A.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, p=0.5),\n",
    "    A.HorizontalFlip(),\n",
    "])\n",
    "\n",
    "# Directories\n",
    "images_dir = \"E:\\\\OIA-ODIR\\\\Training Set\\\\train_preprocessed_images\"\n",
    "augmented_images_dir = \"E:\\\\OIA-ODIR\\\\Augmented\\\\Training Set\\\\Images\"\n",
    "\n",
    "# Ensure the augmented images directory exists\n",
    "if not os.path.exists(augmented_images_dir):\n",
    "    os.makedirs(augmented_images_dir)\n",
    "\n",
    "# Initialize a dictionary to keep track of augmented labels\n",
    "augmented_labels_dict = {}\n",
    "\n",
    "# Initialize the class counts\n",
    "initial_class_counts = Counter({i: 0 for i in range(8)})\n",
    "\n",
    "# Calculate initial class representation\n",
    "for labels in images_labels_dict.values():\n",
    "    for i, label in enumerate(labels):\n",
    "        if label == 1:\n",
    "            initial_class_counts[i] += 1\n",
    "\n",
    "# Define target counts for each class to balance the dataset\n",
    "target_counts = np.array([3000] * 8)  # Let's say we want 3000 images per class\n",
    "\n",
    "# Determine how many images are needed to reach the target for each class\n",
    "needed_counts = target_counts - np.array(list(initial_class_counts.values()))\n",
    "\n",
    "# Main augmentation loop\n",
    "new_rows = []\n",
    "for row in annotations.iterrows():\n",
    "    image_file = row[1]['Left-Fundus']\n",
    "    image = cv2.imread(os.path.join(images_dir, image_file))\n",
    "    image_right = cv2.imread(os.path.join(images_dir, image_file.replace(\"left\", \"right\")))\n",
    "    labels = row[1].iloc[7:]\n",
    "    # Determine how many times this image should be augmented for each class\n",
    "    augmentation_counts = [needed_counts[i] if label == 1 else 0 for i, label in enumerate(labels)]\n",
    "    max_augmentations = max(augmentation_counts)\n",
    "\n",
    "    # Perform the necessary augmentations\n",
    "    for i in range(max_augmentations):\n",
    "        # Check if we still need more images for this class\n",
    "        if any(needed_counts[class_idx] > 0 for class_idx, label in enumerate(labels) if label == 1):\n",
    "            \n",
    "            augmented_image = augmentations(image=image)['image']\n",
    "            augmented_image_right = augmentations(image=image_right)['image']\n",
    "            \n",
    "            augmented_filename = f\"{image_file}_{i}_augmented.jpg\"\n",
    "            augmented_filename_right = f\"{image_file.replace('left', 'right')}_{i}_augmented.jpg\"\n",
    "            \n",
    "            cv2.imwrite(os.path.join(augmented_images_dir, augmented_filename), augmented_image)\n",
    "            cv2.imwrite(os.path.join(augmented_images_dir, augmented_filename_right), augmented_image_right)\n",
    "#             augmented_labels_dict[augmented_filename] = labels\n",
    "            row_ = {'Left-Fundus':augmented_filename, \"Right-Fundus\":augmented_filename_right, \n",
    "                   'Patient Age':row[1]['Patient Age'], \"Patient Sex\":row[1][\"Patient Sex\"],\n",
    "                       \"N\":row[1][\"N\"], \"D\":row[1][\"D\"], \"G\":row[1][\"G\"], \"C\":row[1][\"C\"], \"A\":row[1][\"A\"],\n",
    "                        \"H\":row[1][\"H\"], \"M\":row[1][\"M\"], \"O\":row[1][\"O\"]}\n",
    "            new_rows.append(row_)\n",
    "            # Update the counts\n",
    "            for class_idx, count in enumerate(augmentation_counts):\n",
    "                if count > 0:\n",
    "                    needed_counts[class_idx] -= 1\n",
    "                    augmentation_counts[class_idx] -= 1\n",
    "\n",
    "extended = pd.DataFrame(new_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# Count the number of images for each class in the augmented dataset\n",
    "class_counts = np.zeros(8)\n",
    "for labels in extended.iterrows():\n",
    "    # Convert labels to a numpy array if they aren't already one\n",
    "    if not isinstance(labels, np.ndarray):\n",
    "        labels = np.array(labels[1].iloc[2:])\n",
    "    # Make sure the labels are of a numeric type (e.g., integers)\n",
    "    labels = labels.astype(int)\n",
    "    class_counts += labels\n",
    "    \n",
    "# Plot the class distribution\n",
    "Classes = [\"Normal\", \"Diabetic Retinopathy\", \"Glaucoma\", \"Cataract\", \"AMD\", \"Hypertension\", \"Myopia\", \"Other\"]  # Assuming you have defined 'Classes' somewhere\n",
    "\n",
    "plt.figure(figsize=(12, 6))\n",
    "plt.bar(Classes, class_counts)\n",
    "plt.xlabel(\"Classes\")\n",
    "plt.ylabel(\"Number of Images\")\n",
    "plt.title(\"Class Distribution in Augmented Dataset\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model with ResNet feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.applications.resnet import ResNet101, preprocess_input\n",
    "from tensorflow.keras.models import Model\n",
    "# Load ResNet-101 model pre-trained on ImageNet, without the top classification layer\n",
    "#with a shape `(None, None, None, 1024)\n",
    "base_model = ResNet101(weights='imagenet', include_top=False)\n",
    "\n",
    "model_custom = Model(inputs=base_model.input, outputs=base_model.get_layer('conv4_block23_out').output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for layers in model_custom.layers:\n",
    "    layers.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import Model, Input\n",
    "from tensorflow.keras.layers import Layer, DepthwiseConv2D, Reshape, Conv2D, BatchNormalization, ReLU, Concatenate, GlobalAveragePooling2D, Dense, Multiply\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "\n",
    "class CustomAttentionBlock(Layer):\n",
    "    def __init__(self, filters, **kwargs):\n",
    "        super(CustomAttentionBlock, self).__init__(**kwargs)\n",
    "        self.filters = filters\n",
    "        self.ds_conv = DepthwiseConv2D(kernel_size=(3, 3), padding='same')\n",
    "        self.p_conv = Conv2D(filters=filters, kernel_size=(1, 1), padding='same', kernel_regularizer=l2(0.01))\n",
    "        self.bn = BatchNormalization()\n",
    "        self.relu = ReLU()\n",
    "        self.concat = Concatenate(axis=-1)\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super(CustomAttentionBlock, self).get_config()\n",
    "        config.update({'filters': self.filters})\n",
    "        return config\n",
    "\n",
    "class SEBlock(Layer):\n",
    "    def __init__(self, reduction_ratio=16, **kwargs):\n",
    "        super(SEBlock, self).__init__(**kwargs)\n",
    "        self.reduction_ratio = reduction_ratio\n",
    "        self.global_avg_pool = GlobalAveragePooling2D()\n",
    "        self.fc1 = None\n",
    "        self.fc2 = None\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        channels = input_shape[-1]\n",
    "        self.fc1 = Dense(channels // self.reduction_ratio, activation='relu')\n",
    "        self.fc2 = Dense(channels, activation='relu')\n",
    "\n",
    "    def call(self, inputs):\n",
    "        squeeze = self.global_avg_pool(inputs)\n",
    "        excitation = self.fc1(squeeze)\n",
    "        excitation = self.fc2(excitation)\n",
    "        excitation = tf.reshape(excitation, (-1, 1, 1, inputs.shape[-1]))\n",
    "        return inputs * excitation\n",
    "    \n",
    "    def get_config(self):\n",
    "        config = super(SEBlock, self).get_config()\n",
    "        config.update({'reduction_ratio': self.reduction_ratio})\n",
    "        return config\n",
    "\n",
    "class DRBM(Layer):\n",
    "    def __init__(self, visible_units, hidden_units, num_classes, **kwargs):\n",
    "        super(DRBM, self).__init__(**kwargs)\n",
    "        self.visible_units = visible_units\n",
    "        self.hidden_units = hidden_units\n",
    "        self.num_classes = num_classes\n",
    "        self.W = tf.Variable(tf.random.normal(shape=(visible_units, hidden_units), stddev=0.02))\n",
    "        self.v_bias = tf.Variable(tf.random.normal(shape=(visible_units,), stddev=0.02))\n",
    "        self.h_bias = tf.Variable(tf.zeros(shape=(hidden_units,)))\n",
    "        self.class_weights = tf.Variable(tf.random.normal(shape=(hidden_units, num_classes), stddev=0.02))\n",
    "        self.class_bias = tf.Variable(tf.zeros(shape=(num_classes,)))\n",
    "\n",
    "    def call(self, v):\n",
    "        hidden_probs = tf.sigmoid(tf.matmul(v, self.W) + self.h_bias)\n",
    "        class_probs = tf.nn.softmax(tf.matmul(hidden_probs, self.class_weights) + self.class_bias)\n",
    "        return class_probs\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super(DRBM, self).get_config()\n",
    "        config.update({\n",
    "            'visible_units': self.visible_units,\n",
    "            'hidden_units': self.hidden_units,\n",
    "            'num_classes': self.num_classes\n",
    "        })\n",
    "        return config\n",
    "\n",
    "def create_model(visible_units, hidden_units, num_classes):\n",
    "    \n",
    "    # Read Inputs\n",
    "    left_input = Input(shape=(224, 224, 3))\n",
    "    right_input = Input(shape=(224, 224, 3))\n",
    "    input_age = Input(shape=(1,), name='input_age')  # Single value for age\n",
    "    input_sex = Input(shape=(1,), name='input_sex')  # Single value for sex\n",
    "    \n",
    "    # Feature Extraction\n",
    "    left_features = model_custom(left_input)\n",
    "    right_features = model_custom(right_input)\n",
    "    \n",
    "    # Apply Attention Blocks\n",
    "    attention_out_left = CustomAttentionBlock(filters=1024)(left_features)\n",
    "    attention_out_right = CustomAttentionBlock(filters=1024)(right_features)\n",
    "    \n",
    "    # Apply SE Blocks to the outputs of the Attention Blocks\n",
    "    se_output_left = SEBlock()(attention_out_left)\n",
    "    se_output_right = SEBlock()(attention_out_right)\n",
    "    \n",
    "    # Concatenate the outputs of the SE Blocks\n",
    "    concatenated_features = Concatenate(axis=-1)([se_output_left, se_output_right])\n",
    "    \n",
    "    # Processing Demoghraphics data\n",
    "    concatenated_demographics = tf.keras.layers.Concatenate()([input_age, input_sex])\n",
    "\n",
    "    # Feedforward network to process concatenated features\n",
    "    x = Dense(2048, activation='relu')(concatenated_demographics)\n",
    "\n",
    "    # Reshape to prepare for depthwise convolution\n",
    "    x = Reshape((1, 1, 2048))(x)  # Adjusted reshape size\n",
    "\n",
    "    # Combine image features with patient data\n",
    "    combined_features = tf.multiply(concatenated_features, x)\n",
    "    \n",
    "    # Process the concatenated features\n",
    "    processed_features = GlobalAveragePooling2D()(combined_features)\n",
    "    \n",
    "    # Apply the DRBM\n",
    "    output_probs = DRBM(visible_units, hidden_units, num_classes)(processed_features)\n",
    "    \n",
    "    # Define the model\n",
    "    \n",
    "    model = Model(inputs=[left_input, right_input, input_age, input_sex], outputs=output_probs)\n",
    "    \n",
    "    return model\n",
    "\n",
    "visible_units=2048\n",
    "hidden_units=128\n",
    "num_classes=8\n",
    "\n",
    "# Define the model\n",
    "model = create_model(visible_units, hidden_units, num_classes=8)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Compile your model\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0006),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=[tf.keras.metrics.AUC(name='AUC', multi_label=True), 'Accuracy'])\n",
    "\n",
    "# Determine the number of steps per epoch (number of batches)\n",
    "num_samples = 18013  # Total number of samples in your dataset\n",
    "batch_size = 52  # Batch size used by your generator\n",
    "steps_per_epoch = num_samples // (batch_size)\n",
    "val_steps = 500 // (batch_size)\n",
    "# Train your model using the generator\n",
    "hist = model.fit(train_generator, steps_per_epoch=steps_per_epoch, epochs=30, \n",
    "         validation_data=off_test_generator, validation_steps=val_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.evaluate(off_test_generator, steps = 500 // 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"E://OIA-ODIR//Model_multimodal_aug\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Initialize an empty list to store true and predicted labels\n",
    "true_labels = []\n",
    "predicted_labels = []\n",
    "\n",
    "# extended = annotations.sample(frac=1).reset_index(drop=True)\n",
    "# Iterate over each row in the extended DataFrame\n",
    "i = 0\n",
    "for _, row in tqdm(extended.iterrows(), desc='Processing', unit='Images'):\n",
    "    if(i==1000):\n",
    "        break\n",
    "    left = cv2.cvtColor(cv2.imread(os.path.join(preprocessed_images, row['Left-Fundus'])), cv2.COLOR_BGR2RGB)\n",
    "    right = cv2.cvtColor(cv2.imread(os.path.join(preprocessed_images, row['Right-Fundus'])), cv2.COLOR_BGR2RGB)\n",
    "    age = row[\"Patient Age\"]\n",
    "    sex = 1 if row[\"Patient Sex\"] == \"Male\" else 0\n",
    "    \n",
    "    # Make predictions for a single sample\n",
    "    preds = model.predict([np.array([left]), np.array([right]), np.array([age]), np.array([sex])], verbose=0)\n",
    "    \n",
    "    # Choose the predicted class based on the threshold\n",
    "    threshold = 0.05  # Adjust this threshold as needed\n",
    "    predicted_class = np.argmax(preds) if np.max(preds) >= threshold else -1  # Assign -1 for uncertain predictions\n",
    "    \n",
    "    # Append true and predicted labels\n",
    "    true_labels.append(np.argmax(row.iloc[7:]))  # Assuming the class labels start from column index 4\n",
    "    predicted_labels.append(predicted_class)\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the confusion matrix\n",
    "cm = confusion_matrix(true_labels, predicted_labels)\n",
    "\n",
    "# Plot the confusion matrix\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=Classes, yticklabels=Classes)\n",
    "plt.xlabel('Predicted Label')\n",
    "plt.ylabel('True Label')\n",
    "plt.title('Confusion Matrix')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "private_outputs": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}