Cross_Modal_Training_with_PyTorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "authorship_tag": "ABX9TyPLABf9dD4B7nArwYIvsN6y",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/OneFineStarstuff/OneFineStardust/blob/main/Cross_Modal_Training_with_PyTorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tSJNfCDdqhqb"
      },
      "outputs": [],
      "source": [
        "pip install clip transformers torch"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/openai/CLIP.git"
      ],
      "metadata": {
        "id": "q4QQfh0frFfy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import clip\n",
        "from PIL import Image\n",
        "\n",
        "# Load CLIP model and preprocessing function from the OpenAI repository\n",
        "model, preprocess = clip.load(\"ViT-B/32\", device=\"cpu\")\n",
        "\n",
        "# Preprocess the image\n",
        "image = preprocess(Image.open(\"example.jpg\")).unsqueeze(0)  # Make sure the image path is correct\n",
        "\n",
        "# Tokenize the text\n",
        "text = clip.tokenize([\"A description of the image\"])\n",
        "\n",
        "# Move to the same device as the model\n",
        "image = image.to(\"cpu\")\n",
        "text = text.to(\"cpu\")\n",
        "model = model.to(\"cpu\")\n",
        "\n",
        "# Obtain embeddings\n",
        "with torch.no_grad():\n",
        "    image_features = model.encode_image(image)\n",
        "    text_features = model.encode_text(text)\n",
        "    similarity = (image_features @ text_features.T).softmax(dim=-1)\n",
        "\n",
        "print(\"Image-Text Similarity:\", similarity)"
      ],
      "metadata": {
        "id": "iwujiHVCqrxU"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}