Traditional_MachineLearning_Baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "immediate-musical",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pprint\n",
    "from sklearn import preprocessing\n",
    "import re\n",
    "import pickle\n",
    "import time, datetime\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "\n",
    "## Ignore warnings\n",
    "import warnings\n",
    "def warn(*args, **kwargs):\n",
    "    pass\n",
    "warnings.warn = warn\n",
    "np.warnings.filterwarnings('ignore')\n",
    "\n",
    "from tld.linktypes import fine_linktype_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "japanese-bridal",
   "metadata": {},
   "outputs": [],
   "source": [
    "SOURCES = ['Hyperledger', 'IntelDAOS', 'JFrog', 'Jira', \n",
    "           'JiraEcosystem', 'MariaDB', 'Mindville', 'MongoDB', \n",
    "           'Qt', 'RedHat', 'Sakai', 'SecondLife', 'Sonatype', 'Spring', 'Apache', 'Mojang']\n",
    "\n",
    "# 'Jira', 'RedHat', 'Sonatype', 'Apache'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "adaptive-synthetic",
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = load_data('Qt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "unlike-assumption",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Bug               106804\n",
       "Suggestion         15723\n",
       "Task               12015\n",
       "Technical task      4830\n",
       "Sub-task            4792\n",
       "User Story          3401\n",
       "Epic                 793\n",
       "Change Request       209\n",
       "Improvement           11\n",
       "Research               1\n",
       "Name: type, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x['type'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "authentic-purse",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(source):\n",
    "    #Loading Issues\n",
    "    filename = 'data/processed/issues_'+source.lower()+'.csv'\n",
    "    issue_df = pd.read_csv(filename, encoding=\"UTF-8\", low_memory=False, sep=';')\n",
    "\n",
    "    #Loading Links\n",
    "    filename = 'data/processed/links_plus_'+source.lower()+'.csv'\n",
    "    link_df = pd.read_csv(filename, encoding=\"UTF-8\", low_memory=False, sep=';', index_col=[0]).drop_duplicates()\n",
    "    \n",
    "    link_df['mappedtype'] = link_df['linktype'].map(fine_linktype_map)\n",
    "    \n",
    "    return issue_df, link_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "aquatic-replication",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidfvect_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "sporting-george",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HYPERLEDGER\n",
      "Number of unique labels: 7\n",
      "['Relate', 'Clone', 'Non-Link', 'Epic', 'Subtask', 'Block', 'Duplicate']\n",
      "0.22763241060649037\n",
      "0.29273612862012277\n",
      "0.27551289168331805\n",
      "INTELDAOS\n",
      "Number of unique labels: 8\n",
      "[   'Relate',\n",
      "    'Clone',\n",
      "    'Subtask',\n",
      "    'Non-Link',\n",
      "    'Block',\n",
      "    'Duplicate',\n",
      "    'finish-start',\n",
      "    'Verify']\n",
      "0.344674709392614\n",
      "0.352268218658219\n",
      "0.3700802280450356\n",
      "JFROG\n",
      "Number of unique labels: 7\n",
      "[   'Relate',\n",
      "    'Depend',\n",
      "    'Non-Link',\n",
      "    'Subtask',\n",
      "    'Trigger',\n",
      "    'Incorporate',\n",
      "    'Duplicate']\n",
      "0.2601961274996564\n",
      "0.25636893949322576\n",
      "0.257806468175614\n",
      "JIRA\n",
      "Number of unique labels: 8\n",
      "[   'Relate',\n",
      "    'Detail',\n",
      "    'Cause',\n",
      "    'Non-Link',\n",
      "    'Subtask',\n",
      "    'Incorporate',\n",
      "    'Duplicate',\n",
      "    'Clone']\n",
      "0.2687384019713899\n",
      "0.2852229432116216\n",
      "0.26909365372649\n",
      "JIRAECOSYSTEM\n",
      "Number of unique labels: 11\n",
      "[   'Relate',\n",
      "    'Split',\n",
      "    'Depend',\n",
      "    'Cause',\n",
      "    'Subtask',\n",
      "    'Non-Link',\n",
      "    'Epic',\n",
      "    'Incorporate',\n",
      "    'Duplicate',\n",
      "    'Block',\n",
      "    'Clone']\n",
      "0.1511449589405519\n",
      "0.18082906846051633\n",
      "0.17548892372074326\n",
      "MARIADB\n",
      "Number of unique labels: 8\n",
      "[   'Relate',\n",
      "    'Cause',\n",
      "    'Non-Link',\n",
      "    'Epic',\n",
      "    'Incorporate',\n",
      "    'Subtask',\n",
      "    'Block',\n",
      "    'Duplicate']\n",
      "0.2765140432418024\n",
      "0.27219110844860517\n",
      "0.3143569920969933\n",
      "MINDVILLE\n",
      "Number of unique labels: 5\n",
      "['Relate', 'Non-Link', 'Clone', 'Block', 'Duplicate']\n",
      "0.20505555555555555\n",
      "0.23536446886446885\n",
      "0.18205555555555558\n",
      "MONGODB\n",
      "Number of unique labels: 9\n",
      "[   'Relate',\n",
      "    'Split',\n",
      "    'Depend',\n",
      "    'Cause',\n",
      "    'Subtask',\n",
      "    'Epic',\n",
      "    'Non-Link',\n",
      "    'Duplicate',\n",
      "    'Documented']\n",
      "0.29840264231893265\n",
      "0.34461400643179607\n",
      "0.3073930306739342\n",
      "QT\n",
      "Number of unique labels: 8\n",
      "[   'Relate',\n",
      "    'Replace',\n",
      "    'Split',\n",
      "    'Depend',\n",
      "    'Subtask',\n",
      "    'Non-Link',\n",
      "    'Epic',\n",
      "    'Duplicate']\n",
      "0.2631003482813035\n",
      "0.3061609179385814\n",
      "0.33136252068486793\n",
      "REDHAT\n",
      "Number of unique labels: 9\n",
      "[   'Relate',\n",
      "    'Cause',\n",
      "    'Subtask',\n",
      "    'Non-Link',\n",
      "    'Incorporate',\n",
      "    'Follow',\n",
      "    'Clone',\n",
      "    'Block',\n",
      "    'Duplicate']\n",
      "0.24325201112946188\n",
      "0.26419929605166614\n",
      "0.23173786300317656\n",
      "SAKAI\n",
      "Number of unique labels: 7\n",
      "['Relate', 'Depend', 'Clone', 'Non-Link', 'Incorporate', 'Subtask', 'Duplicate']\n",
      "0.23792391140761526\n",
      "0.21428506726889124\n",
      "0.23114435629016353\n",
      "SECONDLIFE\n",
      "Number of unique labels: 7\n",
      "[   'Relate',\n",
      "    'Depend',\n",
      "    'Non-Link',\n",
      "    'Subtask',\n",
      "    'Parent-Child',\n",
      "    'Incorporate',\n",
      "    'Clone']\n",
      "0.30613904488183985\n",
      "0.34014290640640493\n",
      "0.31390205165077145\n",
      "SONATYPE\n",
      "Number of unique labels: 9\n",
      "[   'Relate',\n",
      "    'Supercede',\n",
      "    'Fix',\n",
      "    'Depend',\n",
      "    'Bonfire Testing',\n",
      "    'Cause',\n",
      "    'Non-Link',\n",
      "    'Subtask',\n",
      "    'Duplicate']\n",
      "0.18939029504787713\n",
      "0.23444054087083313\n",
      "0.22188965625307847\n",
      "SPRING\n",
      "Number of unique labels: 7\n",
      "['Relate', 'Supercede', 'Depend', 'Non-Link', 'Epic', 'Subtask', 'Duplicate']\n",
      "0.2575642975861342\n",
      "0.2813496150373089\n",
      "0.29677218452547305\n",
      "APACHE\n",
      "Number of unique labels: 13\n",
      "[   'Relate',\n",
      "    'Supercede',\n",
      "    'Depend',\n",
      "    'Require',\n",
      "    'Cause',\n",
      "    'Subtask',\n",
      "    'Non-Link',\n",
      "    'Breaks',\n",
      "    'Epic',\n",
      "    'Incorporate',\n",
      "    'Duplicate',\n",
      "    'Block',\n",
      "    'Clone']\n",
      "0.1344694073796485\n",
      "0.14540119493531933\n",
      "0.16524903586344916\n",
      "MOJANG\n",
      "Number of unique labels: 3\n",
      "['Relate', 'Non-Link', 'Duplicate']\n",
      "0.4246854827160452\n",
      "0.5435783154456268\n",
      "0.5297895591674618\n"
     ]
    }
   ],
   "source": [
    "repos = []\n",
    "DTs = []\n",
    "RFs = []\n",
    "SVCs = []\n",
    "\n",
    "# for s in ['Jira', 'RedHat', 'Sonatype', 'Apache']:\n",
    "for s in SOURCES:\n",
    "    repos.append(s)\n",
    "    \n",
    "    print(s.upper())\n",
    "    issue_df, link_df = load_data(s)\n",
    "    \n",
    "    issue_df['title'].fillna(' ', inplace=True)\n",
    "    issue_df['description'].fillna(' ', inplace=True)\n",
    "    \n",
    "    linked_issues = set(link_df['issue_id_1']).union(set(link_df['issue_id_2']))\n",
    "    linked_issue_df = issue_df[issue_df['issue_id'].isin(linked_issues)]\n",
    "    \n",
    "    linked_issue_df['text'] = linked_issue_df['title']+ \" \" + linked_issue_df['description']\n",
    "    \n",
    "    try:\n",
    "        df_tfidfvect = tfidfvect_dict[s]\n",
    "    except:\n",
    "        tfidf_vectorizer = TfidfVectorizer(stop_words='english', min_df=0.01, ngram_range=(1,2))\n",
    "        tfidf_wm = tfidf_vectorizer.fit_transform(list(linked_issue_df['text'].values))\n",
    "        tfidf_tokens = tfidf_vectorizer.get_feature_names()\n",
    "        df_tfidfvect = pd.DataFrame(data = tfidf_wm.toarray(),index = [linked_issue_df['issue_id']],columns = tfidf_tokens)\n",
    "        tfidfvect_dict[s] = df_tfidfvect\n",
    "    \n",
    "    linktypes = (link_df.mappedtype.value_counts()>=len(link_df)*0.01).rename_axis('mappedtype').reset_index(name='valid')\n",
    "    valid_types = set(linktypes[linktypes['valid']==True]['mappedtype'])\n",
    "\n",
    "    link_df = link_df[(link_df[\"mappedtype\"].isin(valid_types))]\n",
    "    \n",
    "    link_df.drop(columns=['name', 'issues', 'mappedtype'])\n",
    "    \n",
    "    all_data = link_df.merge(df_tfidfvect, how='inner', left_on='issue_id_1', right_on='issue_id')\n",
    "    all_data = all_data.merge(df_tfidfvect, how='inner', left_on='issue_id_2', right_on='issue_id')\n",
    "    \n",
    "#     print(all_data['linktype'].unique())\n",
    "    \n",
    "    all_data['label'] = all_data['linktype'].map(fine_linktype_map)\n",
    "    \n",
    "#     print(all_data['label'].unique())\n",
    "#     print(all_data[all_data['label'].isna()])\n",
    "    \n",
    "    all_data.drop(columns=['name', 'mappedtype', 'issue_id_1', 'issue_id_2', 'linktype'], inplace=True)\n",
    "    \n",
    "    try:\n",
    "        all_data.drop(columns=['issues_x'], inplace=True)\n",
    "    except:\n",
    "        print(\"blub\")\n",
    "    \n",
    "    y = all_data[['label']]\n",
    "    \n",
    "    X = all_data.drop(columns=['label'])\n",
    "    \n",
    "#     print(X)\n",
    "    \n",
    "    print(\"Number of unique labels: \"+str(len(set(y['label']))))\n",
    "\n",
    "    labels = list(set(y['label']))\n",
    "\n",
    "    pp.pprint(labels)\n",
    "    \n",
    "    clf = DecisionTreeClassifier(random_state=0, class_weight='balanced')\n",
    "    DT = np.mean(cross_val_score(clf, X, y, cv=5, scoring='f1_macro'))\n",
    "    print(DT)\n",
    "#     DT_perf[s] = DT\n",
    "    \n",
    "    clf = RandomForestClassifier(random_state=0, class_weight='balanced')\n",
    "    RF = np.mean(cross_val_score(clf, X, y, cv=5, scoring='f1_macro'))\n",
    "    print(RF)\n",
    "#     RF_perf[s] = RF\n",
    "    \n",
    "    clf = LinearSVC(random_state=0, tol=1e-5, class_weight='balanced')\n",
    "    SVC = np.mean(cross_val_score(clf, X, y, cv=5, scoring='f1_macro'))\n",
    "    print(SVC)\n",
    "#     SVC_perf[s] = SVC\n",
    "    \n",
    "    DTs.append(DT)\n",
    "    RFs.append(RF)\n",
    "    SVCs.append(SVC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "anonymous-custom",
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_dict = {\n",
    "            'Repository': repos,\n",
    "            'Decision Tree' : DTs,\n",
    "            'Random Forest': RFs,\n",
    "            'Linear SVM': SVCs,\n",
    "          }\n",
    "\n",
    "baseline_df = pd.DataFrame(baseline_dict)   \n",
    "baseline_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "characteristic-bones",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Repository</th>\n",
       "      <th>Decision Tree</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Linear SVM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hyperledger</td>\n",
       "      <td>0.236780</td>\n",
       "      <td>0.298068</td>\n",
       "      <td>0.272101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IntelDAOS</td>\n",
       "      <td>0.361208</td>\n",
       "      <td>0.365818</td>\n",
       "      <td>0.348201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JFrog</td>\n",
       "      <td>0.261597</td>\n",
       "      <td>0.250166</td>\n",
       "      <td>0.266638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Jira</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>JiraEcosystem</td>\n",
       "      <td>0.157573</td>\n",
       "      <td>0.173077</td>\n",
       "      <td>0.170041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>MariaDB</td>\n",
       "      <td>0.257631</td>\n",
       "      <td>0.267917</td>\n",
       "      <td>0.329494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Mindville</td>\n",
       "      <td>0.213849</td>\n",
       "      <td>0.163511</td>\n",
       "      <td>0.199524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MongoDB</td>\n",
       "      <td>0.272820</td>\n",
       "      <td>0.314926</td>\n",
       "      <td>0.248385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Qt</td>\n",
       "      <td>0.233372</td>\n",
       "      <td>0.267493</td>\n",
       "      <td>0.285740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>RedHat</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Sakai</td>\n",
       "      <td>0.221885</td>\n",
       "      <td>0.202446</td>\n",
       "      <td>0.224920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>SecondLife</td>\n",
       "      <td>0.269938</td>\n",
       "      <td>0.374868</td>\n",
       "      <td>0.302194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Sonatype</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Spring</td>\n",
       "      <td>0.224989</td>\n",
       "      <td>0.236483</td>\n",
       "      <td>0.284603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Apache</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Mojang</td>\n",
       "      <td>0.377384</td>\n",
       "      <td>0.475669</td>\n",
       "      <td>0.481904</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Repository  Decision Tree  Random Forest  Linear SVM\n",
       "0     Hyperledger       0.236780       0.298068    0.272101\n",
       "1       IntelDAOS       0.361208       0.365818    0.348201\n",
       "2           JFrog       0.261597       0.250166    0.266638\n",
       "3            Jira            NaN            NaN         NaN\n",
       "4   JiraEcosystem       0.157573       0.173077    0.170041\n",
       "5         MariaDB       0.257631       0.267917    0.329494\n",
       "6       Mindville       0.213849       0.163511    0.199524\n",
       "7         MongoDB       0.272820       0.314926    0.248385\n",
       "8              Qt       0.233372       0.267493    0.285740\n",
       "9          RedHat            NaN            NaN         NaN\n",
       "10          Sakai       0.221885       0.202446    0.224920\n",
       "11     SecondLife       0.269938       0.374868    0.302194\n",
       "12       Sonatype            NaN            NaN         NaN\n",
       "13         Spring       0.224989       0.236483    0.284603\n",
       "14         Apache            NaN            NaN         NaN\n",
       "15         Mojang       0.377384       0.475669    0.481904"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_dict = {\n",
    "            'Repository': repos,\n",
    "            'Decision Tree' : DTs,\n",
    "            'Random Forest': RFs,\n",
    "            'Linear SVM': SVCs,\n",
    "          }\n",
    "\n",
    "baseline_df = pd.DataFrame(baseline_dict)   \n",
    "baseline_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "precise-preview",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}