SCCNN_DCCNN_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "periodic-adolescent",
   "metadata": {},
   "source": [
    "# Load Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "eleven-usage",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Notebook.\n"
     ]
    }
   ],
   "source": [
    "#Needed Modules\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "print(\"Starting Notebook.\")\n",
    "\n",
    "sns.set(font_scale = 1)\n",
    "sns.set_style(\"whitegrid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "killing-metabolism",
   "metadata": {},
   "source": [
    "# Repositories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "piano-hearts",
   "metadata": {},
   "outputs": [],
   "source": [
    "SOURCES = ['Apache', 'Hyperledger', 'IntelDAOS', 'JFrog', 'Jira', \n",
    "           'JiraEcosystem', 'MariaDB', 'Mojang', 'MongoDB', \n",
    "           'Qt', 'RedHat', 'Sakai', 'SecondLife', 'Sonatype', 'Spring']\n",
    "\n",
    "# 'Mindville'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "moderate-performance",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_results(config):\n",
    "    class_rep_dict = {}\n",
    "    valid = []\n",
    "    valid_sources = []\n",
    "\n",
    "    for s in SOURCES:\n",
    "        try:\n",
    "            class_rep = pd.read_csv('data/results/'+config+'-results/class_rep_LT_'+s+'_plus_'+config+'.csv', index_col=0)\n",
    "            valid.append(s)\n",
    "            class_rep_dict[s] = class_rep\n",
    "            print(f'{s.upper()} {t.upper()}')\n",
    "        except:\n",
    "            print(f'project {s.upper()} {t.upper()} NOT in results')\n",
    "    \n",
    "    f1 = []\n",
    "    acc = []\n",
    "\n",
    "    for c in valid:\n",
    "        f1.append(class_rep_dict[c].loc['macro avg']['f1-score'])\n",
    "        acc.append(class_rep_dict[c].loc['accuracy']['support'])\n",
    "\n",
    "    result_dict = {\n",
    "        \"F1\": f1,\n",
    "        \"ACC\": acc,\n",
    "    }\n",
    "    result_df = pd.DataFrame(result_dict, index=valid)\n",
    "    print(result_df)\n",
    "    print(\"Average macro F1-score and accuracy for \"+config)\n",
    "    print(result_df.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "european-israel",
   "metadata": {},
   "source": [
    "## SCCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "regular-lemon",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "project APACHE DCCNN NOT in results\n",
      "HYPERLEDGER DCCNN\n",
      "INTELDAOS DCCNN\n",
      "JFROG DCCNN\n",
      "JIRA DCCNN\n",
      "JIRAECOSYSTEM DCCNN\n",
      "MARIADB DCCNN\n",
      "MOJANG DCCNN\n",
      "MONGODB DCCNN\n",
      "QT DCCNN\n",
      "project REDHAT DCCNN NOT in results\n",
      "SAKAI DCCNN\n",
      "SECONDLIFE DCCNN\n",
      "SONATYPE DCCNN\n",
      "SPRING DCCNN\n",
      "                     F1       ACC\n",
      "Hyperledger    0.506965  0.735761\n",
      "IntelDAOS      0.385647  0.511583\n",
      "JFrog          0.318656  0.554441\n",
      "Jira           0.367311  0.687494\n",
      "JiraEcosystem  0.291433  0.610969\n",
      "MariaDB        0.442583  0.540779\n",
      "Mojang         0.788160  0.903449\n",
      "MongoDB        0.544697  0.640021\n",
      "Qt             0.500539  0.614418\n",
      "Sakai          0.398186  0.542985\n",
      "SecondLife     0.325163  0.581081\n",
      "Sonatype       0.285044  0.598555\n",
      "Spring         0.373687  0.563064\n",
      "Average macro F1-score and accuracy for SCCNN\n",
      "F1     0.425236\n",
      "ACC    0.621892\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print_results('SCCNN')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fatty-ground",
   "metadata": {},
   "source": [
    "## DCCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "soviet-horror",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "project APACHE DCCNN NOT in results\n",
      "HYPERLEDGER DCCNN\n",
      "INTELDAOS DCCNN\n",
      "JFROG DCCNN\n",
      "JIRA DCCNN\n",
      "JIRAECOSYSTEM DCCNN\n",
      "MARIADB DCCNN\n",
      "MOJANG DCCNN\n",
      "MONGODB DCCNN\n",
      "QT DCCNN\n",
      "REDHAT DCCNN\n",
      "SAKAI DCCNN\n",
      "SECONDLIFE DCCNN\n",
      "SONATYPE DCCNN\n",
      "SPRING DCCNN\n",
      "                     F1       ACC\n",
      "Hyperledger    0.475722  0.664020\n",
      "IntelDAOS      0.380226  0.472973\n",
      "JFrog          0.212471  0.454155\n",
      "Jira           0.347157  0.680339\n",
      "JiraEcosystem  0.268515  0.525935\n",
      "MariaDB        0.333761  0.516433\n",
      "Mojang         0.763540  0.891059\n",
      "MongoDB        0.485584  0.588701\n",
      "Qt             0.435178  0.558247\n",
      "RedHat         0.396154  0.559778\n",
      "Sakai          0.340460  0.504153\n",
      "SecondLife     0.327509  0.587838\n",
      "Sonatype       0.253312  0.564499\n",
      "Spring         0.308716  0.514481\n",
      "Average macro F1-score and accuracy for DCCNN\n",
      "F1     0.380593\n",
      "ACC    0.577329\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print_results('DCCNN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "technological-provision",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "APACHE DCCNN\n",
      "HYPERLEDGER DCCNN\n",
      "INTELDAOS DCCNN\n",
      "JFROG DCCNN\n",
      "JIRA DCCNN\n",
      "JIRAECOSYSTEM DCCNN\n",
      "MARIADB DCCNN\n",
      "MOJANG DCCNN\n",
      "MONGODB DCCNN\n",
      "QT DCCNN\n",
      "REDHAT DCCNN\n",
      "SAKAI DCCNN\n",
      "SECONDLIFE DCCNN\n",
      "SONATYPE DCCNN\n",
      "SPRING DCCNN\n",
      "                     F1       ACC\n",
      "Apache         0.563294  0.706092\n",
      "Hyperledger    0.743591  0.849945\n",
      "IntelDAOS      0.716929  0.682785\n",
      "JFrog          0.477771  0.660458\n",
      "Jira           0.733772  0.829008\n",
      "JiraEcosystem  0.529719  0.718857\n",
      "MariaDB        0.697909  0.720633\n",
      "Mojang         0.877596  0.946967\n",
      "MongoDB        0.716492  0.725881\n",
      "Qt             0.663286  0.716990\n",
      "RedHat         0.625128  0.705030\n",
      "Sakai          0.638476  0.683277\n",
      "SecondLife     0.521771  0.750000\n",
      "Sonatype       0.461994  0.655315\n",
      "Spring         0.621171  0.693865\n",
      "Average macro F1-score and accuracy for BERT\n",
      "F1     0.63926\n",
      "ACC    0.73634\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print_results('BERT')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "entitled-gateway",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "APACHE DCCNN\n",
      "HYPERLEDGER DCCNN\n",
      "INTELDAOS DCCNN\n",
      "JFROG DCCNN\n",
      "JIRA DCCNN\n",
      "JIRAECOSYSTEM DCCNN\n",
      "MARIADB DCCNN\n",
      "MOJANG DCCNN\n",
      "MONGODB DCCNN\n",
      "QT DCCNN\n",
      "REDHAT DCCNN\n",
      "SAKAI DCCNN\n",
      "SECONDLIFE DCCNN\n",
      "SONATYPE DCCNN\n",
      "SPRING DCCNN\n",
      "                     F1       ACC\n",
      "Apache         0.538866  0.693484\n",
      "Hyperledger    0.700379  0.809419\n",
      "IntelDAOS      0.645031  0.617021\n",
      "JFrog          0.387185  0.580229\n",
      "Jira           0.699677  0.804974\n",
      "JiraEcosystem  0.475508  0.674061\n",
      "MariaDB        0.668873  0.700852\n",
      "Mojang         0.873800  0.945821\n",
      "MongoDB        0.688878  0.704525\n",
      "Qt             0.639362  0.701870\n",
      "RedHat         0.604857  0.694840\n",
      "Sakai          0.588823  0.652974\n",
      "SecondLife     0.409120  0.628378\n",
      "Sonatype       0.412818  0.605779\n",
      "Spring         0.496925  0.587667\n",
      "Average macro F1-score and accuracy for distillBERT\n",
      "F1     0.588673\n",
      "ACC    0.693460\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print_results('distillBERT')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fallen-duncan",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}