Behavioral_Testing_with_CheckList.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "authorship_tag": "ABX9TyOaQLtuWjn05x81BIeNKxN6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/OneFineStarstuff/OneFineStardust/blob/main/Behavioral_Testing_with_CheckList.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --force-reinstall ipykernel\n",
        "!pip install checklist"
      ],
      "metadata": {
        "id": "VRv6mXM9ImwN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pf4k56MCHiFv"
      },
      "outputs": [],
      "source": [
        "from checklist.test_suite import TestSuite\n",
        "from checklist.editor import Editor\n",
        "from checklist.perturb import Perturb\n",
        "import numpy as np\n",
        "\n",
        "# Assuming 'model' is your pre-trained model and 'tokenizer' is the corresponding tokenizer\n",
        "# Define a prediction function\n",
        "def predict_function(texts):\n",
        "    inputs = tokenizer(texts, padding=True, truncation=True, return_tensors='pt').to(device)\n",
        "    with torch.no_grad():\n",
        "        outputs = model(**inputs)\n",
        "    logits = outputs.logits\n",
        "    predictions = torch.argmax(logits, dim=-1).cpu().numpy()\n",
        "    return predictions\n",
        "\n",
        "# Initialize the test suite\n",
        "suite = TestSuite()\n",
        "\n",
        "# Example of adding tests using Checklist Editor and Perturb functions\n",
        "editor = Editor()\n",
        "suite.add(\n",
        "    suite_name=\"Sentiment Analysis Tests\",\n",
        "    test=editor.template(\n",
        "        \"The movie was {adj}.\",\n",
        "        adj=[\"great\", \"terrible\", \"boring\", \"excellent\", \"awful\"],\n",
        "        labels=[1, 0, 0, 1, 0]  # Adjust labels as per your task (e.g., sentiment analysis)\n",
        "    )\n",
        ")\n",
        "\n",
        "# Adding more perturbation-based tests (e.g., inserting typos, changing names)\n",
        "perturbation_tests = Perturb.perturb([\"The movie was great.\", \"The book was terrible.\"], Perturb.add_typos)\n",
        "suite.add(\n",
        "    suite_name=\"Perturbation Tests\",\n",
        "    test=perturbation_tests.data,\n",
        "    labels=[1, 0] * (len(perturbation_tests.data) // 2)\n",
        ")\n",
        "\n",
        "# Run the tests and summarize results\n",
        "suite.run(predict_function)\n",
        "suite.summary()"
      ]
    }
  ]
}