Building_a_Simple_Multi_Modal_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "authorship_tag": "ABX9TyNOUevVtUSUj9j/jwzKNI2J",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/OneFineStarstuff/OneFineStardust/blob/main/Building_a_Simple_Multi_Modal_Model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import BertModel\n",
        "\n",
        "# Example dimensions and number of classes\n",
        "text_encoder_dim = 768  # Dimension of BERT output\n",
        "image_encoder_dim = 512  # Example dimension for image encoder\n",
        "num_classes = 10  # Number of output classes\n",
        "\n",
        "# Example Text Encoder (BERT)\n",
        "class TextEncoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.bert = BertModel.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "    def forward(self, text):\n",
        "        outputs = self.bert(text)\n",
        "        return outputs.last_hidden_state[:, 0, :]  # Use the CLS token representation\n",
        "\n",
        "# Example Image Encoder (Simple CNN)\n",
        "class ImageEncoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.cnn = nn.Sequential(\n",
        "            nn.Conv2d(3, 16, kernel_size=3, stride=2, padding=1),\n",
        "            nn.ReLU(),\n",
        "            nn.Flatten(),\n",
        "            nn.Linear(16 * 16 * 16, image_encoder_dim)\n",
        "        )\n",
        "\n",
        "    def forward(self, image):\n",
        "        return self.cnn(image)\n",
        "\n",
        "# MultiModal Model\n",
        "class MultiModalModel(nn.Module):\n",
        "    def __init__(self, text_encoder, image_encoder):\n",
        "        super().__init__()\n",
        "        self.text_encoder = text_encoder\n",
        "        self.image_encoder = image_encoder\n",
        "        self.fc = nn.Linear(text_encoder_dim + image_encoder_dim, num_classes)\n",
        "\n",
        "    def forward(self, text, image):\n",
        "        text_embedding = self.text_encoder(text)\n",
        "        image_embedding = self.image_encoder(image)\n",
        "        combined = torch.cat([text_embedding, image_embedding], dim=1)\n",
        "        return self.fc(combined)\n",
        "\n",
        "# Instantiate encoders and multi-modal model\n",
        "text_encoder = TextEncoder()\n",
        "image_encoder = ImageEncoder()\n",
        "model = MultiModalModel(text_encoder, image_encoder)\n",
        "\n",
        "# Example inputs\n",
        "text_input = torch.randint(0, 10000, (8, 64))  # Example tokenized text input\n",
        "image_input = torch.randn(8, 3, 32, 32)  # Example image input\n",
        "\n",
        "# Forward pass\n",
        "outputs = model(text_input, image_input)\n",
        "print(outputs)"
      ],
      "metadata": {
        "id": "bn-XHucfBVx3"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}