Repository_Properties.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "constant-friendly",
   "metadata": {},
   "source": [
    "# Load Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f9c666c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.parsing.porter import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#Link type map\n",
    "from tld.linktypes import fine_linktype_map"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "invisible-arrow",
   "metadata": {},
   "source": [
    "## Repositories and Loading Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ce988e24",
   "metadata": {},
   "outputs": [],
   "source": [
    "SOURCES = ['Apache', 'Hyperledger', 'IntelDAOS', 'JFrog', 'Jira', \n",
    "           'JiraEcosystem', 'MariaDB', 'Mindville', 'Mojang', 'MongoDB', \n",
    "           'Qt', 'RedHat', 'Sakai', 'SecondLife', 'Sonatype', 'Spring']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "280903aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(source):\n",
    "    #Loading Issues\n",
    "    filename = 'data/processed/issues_'+source.lower()+'.csv'\n",
    "    issue_df = pd.read_csv(filename, encoding=\"UTF-8\", low_memory=False, sep=';')\n",
    "\n",
    "    #Loading Links\n",
    "    filename = 'data/processed/links_'+source.lower()+'.csv'\n",
    "    link_df = pd.read_csv(filename, encoding=\"UTF-8\", low_memory=False, sep=';')\n",
    "    \n",
    "    return issue_df, link_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9a7a005a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "APACHE\n",
      "HYPERLEDGER\n",
      "INTELDAOS\n",
      "JFROG\n",
      "JIRA\n",
      "JIRAECOSYSTEM\n",
      "MARIADB\n",
      "MINDVILLE\n",
      "MOJANG\n",
      "MONGODB\n",
      "QT\n",
      "REDHAT\n",
      "SAKAI\n",
      "SECONDLIFE\n",
      "SONATYPE\n",
      "SPRING\n"
     ]
    }
   ],
   "source": [
    "# Load Data\n",
    "link_dict = {}\n",
    "issue_dict = {}\n",
    "\n",
    "for s in SOURCES:\n",
    "    print(s.upper())\n",
    "    issues, links = load_data(s)\n",
    "    link_dict[s] = links\n",
    "    issue_dict[s] = issues\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "thermal-swing",
   "metadata": {},
   "source": [
    "## Tables 1: Overview of Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "ec506f63",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_overview(SOURCE):\n",
    "    \n",
    "    issues = issue_dict[SOURCE]\n",
    "    links = link_dict[SOURCE]\n",
    "    \n",
    "    links['mappedtype'] = links['linktype'].map(fine_linktype_map)\n",
    "        \n",
    "    num_links = len(links)\n",
    "    num_issues = len(issues)\n",
    "    \n",
    "    issue_set = set(issues.index.values)\n",
    "    link_set = set(links['issue_id_1']).union(set(links['issue_id_2']))\n",
    "\n",
    "    projects = issues[['projectid', 'issue_id']]\n",
    "    \n",
    "    temp = links.merge(projects, left_on='issue_id_1', right_on='issue_id')\n",
    "    temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
    "    \n",
    "    temp['sameproject'] = (temp[\"projectid_1\"]==temp[\"projectid_2\"])\n",
    "\n",
    "    cp_df = temp.mappedtype.value_counts().rename_axis('mappedtype').reset_index(name='counts')\n",
    "\n",
    "    try:\n",
    "        cross_ratio = round(temp['sameproject'].value_counts().loc[False]/len(temp)*100, 2)\n",
    "    except:\n",
    "        cross_ratio = 0\n",
    "\n",
    "    cp_df['source'] = SOURCE\n",
    "    cp_df['percentages'] = cp_df['counts'].apply(lambda x: round(x/len(links),3)*100)\n",
    "\n",
    "    cp_df['crossproject'] = 0\n",
    "    for i in temp.mappedtype.value_counts().index:\n",
    "        link_temp = temp[temp['mappedtype'] == i]\n",
    "        try:\n",
    "            cross_ratio_temp = round(link_temp['sameproject'].value_counts().loc[False]/len(link_temp)*100, 2)\n",
    "        except:\n",
    "            cross_ratio_temp = 0\n",
    "        link_index = cp_df.index[cp_df['mappedtype']==i]\n",
    "        link_index = list(link_index)\n",
    "        cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
    "        \n",
    "    return num_issues, num_links, len(links.mappedtype.unique()), len(issues.projectid.unique()), round(len(link_set)/len(issue_set), 3), cross_ratio, cp_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "7239ecd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n",
      "<ipython-input-66-89215c31e7e6>:17: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on='issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-66-89215c31e7e6>:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[link_index[0]] = cross_ratio_temp\n"
     ]
    }
   ],
   "source": [
    "overview = pd.DataFrame(columns = ['Source', '#Issues', '#Links', '#Linktypes', '#Projects', '%IssuesWithLinks', \"%CrossProject\"])\n",
    "link_dist = pd.DataFrame(columns=['source', 'linktype', 'counts'])\n",
    "j = 0\n",
    "for s in SOURCES:\n",
    "    i, l, ltu, p, pi, cr, cp_df = print_overview(s)\n",
    "    overview.loc[j] = [s, i, l, ltu, p, pi, cr]\n",
    "    \n",
    "    j+=1\n",
    "    \n",
    "    frames = [link_dist, cp_df]\n",
    "    #concatenate dataframes\n",
    "    link_dist = pd.concat(frames, sort=False)\n",
    "    \n",
    "overview.set_index('Source').to_csv('data/repo_overview.csv', encoding=\"UTF-8\", sep=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "77b4f293",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Source</th>\n",
       "      <th>#Issues</th>\n",
       "      <th>#Links</th>\n",
       "      <th>#Linktypes</th>\n",
       "      <th>#Projects</th>\n",
       "      <th>%IssuesWithLinks</th>\n",
       "      <th>%CrossProject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apache</td>\n",
       "      <td>1014926</td>\n",
       "      <td>255767</td>\n",
       "      <td>16</td>\n",
       "      <td>646</td>\n",
       "      <td>0.285</td>\n",
       "      <td>5.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hyperledger</td>\n",
       "      <td>28146</td>\n",
       "      <td>16304</td>\n",
       "      <td>8</td>\n",
       "      <td>32</td>\n",
       "      <td>0.549</td>\n",
       "      <td>4.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>IntelDAOS</td>\n",
       "      <td>9474</td>\n",
       "      <td>2599</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>0.308</td>\n",
       "      <td>3.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>JFrog</td>\n",
       "      <td>15535</td>\n",
       "      <td>3229</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.286</td>\n",
       "      <td>8.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jira</td>\n",
       "      <td>274545</td>\n",
       "      <td>99819</td>\n",
       "      <td>16</td>\n",
       "      <td>30</td>\n",
       "      <td>0.467</td>\n",
       "      <td>43.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>JiraEcosystem</td>\n",
       "      <td>41866</td>\n",
       "      <td>11398</td>\n",
       "      <td>14</td>\n",
       "      <td>101</td>\n",
       "      <td>0.330</td>\n",
       "      <td>6.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MariaDB</td>\n",
       "      <td>31229</td>\n",
       "      <td>14618</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>0.445</td>\n",
       "      <td>2.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Mindville</td>\n",
       "      <td>2134</td>\n",
       "      <td>44</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>0.040</td>\n",
       "      <td>4.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Mojang</td>\n",
       "      <td>420819</td>\n",
       "      <td>215527</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>0.537</td>\n",
       "      <td>5.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>MongoDB</td>\n",
       "      <td>137172</td>\n",
       "      <td>63821</td>\n",
       "      <td>14</td>\n",
       "      <td>27</td>\n",
       "      <td>0.452</td>\n",
       "      <td>19.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Qt</td>\n",
       "      <td>148579</td>\n",
       "      <td>40105</td>\n",
       "      <td>11</td>\n",
       "      <td>21</td>\n",
       "      <td>0.302</td>\n",
       "      <td>6.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>RedHat</td>\n",
       "      <td>353000</td>\n",
       "      <td>119669</td>\n",
       "      <td>15</td>\n",
       "      <td>241</td>\n",
       "      <td>0.392</td>\n",
       "      <td>23.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Sakai</td>\n",
       "      <td>50550</td>\n",
       "      <td>19803</td>\n",
       "      <td>8</td>\n",
       "      <td>53</td>\n",
       "      <td>0.424</td>\n",
       "      <td>1.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>SecondLife</td>\n",
       "      <td>1867</td>\n",
       "      <td>631</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>0.399</td>\n",
       "      <td>2.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Sonatype</td>\n",
       "      <td>87284</td>\n",
       "      <td>4465</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>0.070</td>\n",
       "      <td>1.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Spring</td>\n",
       "      <td>69156</td>\n",
       "      <td>14462</td>\n",
       "      <td>7</td>\n",
       "      <td>80</td>\n",
       "      <td>0.256</td>\n",
       "      <td>9.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Source  #Issues  #Links  #Linktypes  #Projects  %IssuesWithLinks  \\\n",
       "0          Apache  1014926  255767          16        646             0.285   \n",
       "1     Hyperledger    28146   16304           8         32             0.549   \n",
       "2       IntelDAOS     9474    2599          11          2             0.308   \n",
       "3           JFrog    15535    3229          10         10             0.286   \n",
       "4            Jira   274545   99819          16         30             0.467   \n",
       "5   JiraEcosystem    41866   11398          14        101             0.330   \n",
       "6         MariaDB    31229   14618           8         11             0.445   \n",
       "7       Mindville     2134      44           4          7             0.040   \n",
       "8          Mojang   420819  215527           5          8             0.537   \n",
       "9         MongoDB   137172   63821          14         27             0.452   \n",
       "10             Qt   148579   40105          11         21             0.302   \n",
       "11         RedHat   353000  119669          15        241             0.392   \n",
       "12          Sakai    50550   19803           8         53             0.424   \n",
       "13     SecondLife     1867     631           6          2             0.399   \n",
       "14       Sonatype    87284    4465          11          5             0.070   \n",
       "15         Spring    69156   14462           7         80             0.256   \n",
       "\n",
       "    %CrossProject  \n",
       "0            5.23  \n",
       "1            4.62  \n",
       "2            3.27  \n",
       "3            8.24  \n",
       "4           43.42  \n",
       "5            6.77  \n",
       "6            2.54  \n",
       "7            4.55  \n",
       "8            5.43  \n",
       "9           19.09  \n",
       "10           6.92  \n",
       "11          23.54  \n",
       "12           1.37  \n",
       "13           2.38  \n",
       "14           1.50  \n",
       "15           9.96  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overview"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "written-interim",
   "metadata": {},
   "source": [
    "## Preperations for Table 2: Frequency of Link Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "12e64ddd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_overview_linktypes(SOURCE):  \n",
    "    \n",
    "    issues = issue_dict[SOURCE]\n",
    "    links = link_dict[SOURCE]\n",
    "    \n",
    "    issue_set = set(issues.index.values)\n",
    "    link_set = set(links['issue_id_1']).union(set(links['issue_id_2']))\n",
    "    \n",
    "    links['mappedtype'] = links['linktype'].map(fine_linktype_map)\n",
    "            \n",
    "    projects = issues[['projectid', 'issue_id']]\n",
    "\n",
    "    temp = links.merge(projects, left_on = 'issue_id_1', right_on='issue_id')\n",
    "    temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
    "\n",
    "    temp['sameproject'] = (temp[\"projectid_1\"]==temp[\"projectid_2\"])\n",
    "\n",
    "    cp_df = links.mappedtype.value_counts().rename_axis('mappedtype').reset_index(name='counts')\n",
    "    cp_df['source'] = SOURCE\n",
    "    cp_df['percentageOfLinks'] = cp_df['counts'].apply(lambda x: round(x/len(links),10)*100)\n",
    "    cp_df['crossproject'] = 0\n",
    "    for i in range(len(cp_df)):\n",
    "        mappedType = cp_df['mappedtype'].iloc[i]\n",
    "        link_temp = temp[temp['mappedtype'] == mappedType]\n",
    "        \n",
    "        try:\n",
    "            cross_ratio_temp = round(link_temp['sameproject'].value_counts().loc[False]/len(link_temp)*100, 2)\n",
    "        except:\n",
    "            cross_ratio_temp = 0\n",
    "        \n",
    "        cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
    "        \n",
    "    return len(links.mappedtype.unique()), cp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "2aeb65d9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n",
      "<ipython-input-55-5c7b727312a4>:14: FutureWarning: Passing 'suffixes' which cause duplicate columns {'issue_id_1'} in the result is deprecated and will raise a MergeError in a future version.\n",
      "  temp = temp.merge(projects, left_on = 'issue_id_2', right_on='issue_id', suffixes=('_1', '_2'))\n",
      "<ipython-input-55-5c7b727312a4>:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cp_df['crossproject'].iloc[i] = cross_ratio_temp\n"
     ]
    }
   ],
   "source": [
    "overview_cl = pd.DataFrame(columns=['Source', '#Linktypes'])\n",
    "link_dist_cl = pd.DataFrame(columns=['source', 'mappedtype', 'counts'])\n",
    "j = 0\n",
    "for s in SOURCES:\n",
    "    ltu, cp_df = print_overview_linktypes(s)\n",
    "    overview_cl.loc[j] = [s, ltu]\n",
    "    \n",
    "    j+=1\n",
    "    \n",
    "    frames = [link_dist_cl, cp_df]\n",
    "    #concatenate dataframes\n",
    "    link_dist_cl = pd.concat(frames, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "amino-titanium",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>mappedtype</th>\n",
       "      <th>counts</th>\n",
       "      <th>percentageOfLinks</th>\n",
       "      <th>crossproject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apache</td>\n",
       "      <td>Subtask</td>\n",
       "      <td>83783</td>\n",
       "      <td>32.757549</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Apache</td>\n",
       "      <td>Relate</td>\n",
       "      <td>72342</td>\n",
       "      <td>28.284337</td>\n",
       "      <td>9.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Apache</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>25925</td>\n",
       "      <td>10.136179</td>\n",
       "      <td>2.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Apache</td>\n",
       "      <td>Block</td>\n",
       "      <td>15549</td>\n",
       "      <td>6.079361</td>\n",
       "      <td>12.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Apache</td>\n",
       "      <td>Depend</td>\n",
       "      <td>13009</td>\n",
       "      <td>5.086270</td>\n",
       "      <td>14.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Spring</td>\n",
       "      <td>Depend</td>\n",
       "      <td>1747</td>\n",
       "      <td>12.079934</td>\n",
       "      <td>28.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Spring</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>1745</td>\n",
       "      <td>12.066104</td>\n",
       "      <td>3.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Spring</td>\n",
       "      <td>Epic</td>\n",
       "      <td>1635</td>\n",
       "      <td>11.305490</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Spring</td>\n",
       "      <td>Supercede</td>\n",
       "      <td>478</td>\n",
       "      <td>3.305214</td>\n",
       "      <td>3.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Spring</td>\n",
       "      <td>Clone</td>\n",
       "      <td>14</td>\n",
       "      <td>0.096805</td>\n",
       "      <td>21.43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    source mappedtype counts  percentageOfLinks  crossproject\n",
       "0   Apache    Subtask  83783          32.757549          0.04\n",
       "1   Apache     Relate  72342          28.284337          9.46\n",
       "2   Apache  Duplicate  25925          10.136179          2.53\n",
       "3   Apache      Block  15549           6.079361         12.35\n",
       "4   Apache     Depend  13009           5.086270         14.51\n",
       "..     ...        ...    ...                ...           ...\n",
       "2   Spring     Depend   1747          12.079934         28.91\n",
       "3   Spring  Duplicate   1745          12.066104          3.67\n",
       "4   Spring       Epic   1635          11.305490          0.00\n",
       "5   Spring  Supercede    478           3.305214          3.77\n",
       "6   Spring      Clone     14           0.096805         21.43\n",
       "\n",
       "[159 rows x 5 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "link_dist_cl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "awful-friend",
   "metadata": {},
   "source": [
    "### Get common link types and extract link type frequences per repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "b402842f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Relate', 'Duplicate', 'Subtask', 'Clone', 'Block', 'Depend',\n",
       "       'Split', 'Epic', 'Incorporate', 'Cause', 'Bonfire Testing'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_lt = (link_dist_cl.mappedtype.value_counts()>=7).rename_axis('mappedtype').reset_index(name='valid')\n",
    "common_lt_set = common_lt[common_lt['valid']==True]['mappedtype'].values\n",
    "common_lt_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "26f5aaf3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>counts</th>\n",
       "      <th>percentageOfLinks</th>\n",
       "      <th>crossproject</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mappedtype</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Block</th>\n",
       "      <td>39567</td>\n",
       "      <td>7.040437</td>\n",
       "      <td>10.540000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bonfire Testing</th>\n",
       "      <td>1049</td>\n",
       "      <td>1.344641</td>\n",
       "      <td>3.870000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cause</th>\n",
       "      <td>10687</td>\n",
       "      <td>3.204718</td>\n",
       "      <td>14.441429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Clone</th>\n",
       "      <td>28307</td>\n",
       "      <td>4.006623</td>\n",
       "      <td>14.245714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Depend</th>\n",
       "      <td>38922</td>\n",
       "      <td>8.592757</td>\n",
       "      <td>16.754000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Duplicate</th>\n",
       "      <td>268945</td>\n",
       "      <td>18.449904</td>\n",
       "      <td>3.829333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Epic</th>\n",
       "      <td>39892</td>\n",
       "      <td>14.517931</td>\n",
       "      <td>1.765000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Incorporate</th>\n",
       "      <td>26291</td>\n",
       "      <td>4.430296</td>\n",
       "      <td>6.075000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Relate</th>\n",
       "      <td>255226</td>\n",
       "      <td>34.814910</td>\n",
       "      <td>11.316250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Split</th>\n",
       "      <td>4061</td>\n",
       "      <td>1.132109</td>\n",
       "      <td>12.653333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subtask</th>\n",
       "      <td>137954</td>\n",
       "      <td>20.891880</td>\n",
       "      <td>0.002857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 counts percentageOfLinks crossproject\n",
       "                    sum              mean         mean\n",
       "mappedtype                                            \n",
       "Block             39567          7.040437    10.540000\n",
       "Bonfire Testing    1049          1.344641     3.870000\n",
       "Cause             10687          3.204718    14.441429\n",
       "Clone             28307          4.006623    14.245714\n",
       "Depend            38922          8.592757    16.754000\n",
       "Duplicate        268945         18.449904     3.829333\n",
       "Epic              39892         14.517931     1.765000\n",
       "Incorporate       26291          4.430296     6.075000\n",
       "Relate           255226         34.814910    11.316250\n",
       "Split              4061          1.132109    12.653333\n",
       "Subtask          137954         20.891880     0.002857"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_lt_set = ['Relate', 'Duplicate', 'Subtask', 'Clone', 'Block', 'Depend', 'Epic', 'Split', 'Incorporate', 'Bonfire Testing', 'Cause']\n",
    "commontypes = link_dist_cl[link_dist_cl['mappedtype'].isin(common_lt_set)]\n",
    "commontypes.groupby(['mappedtype']).agg({'counts': ['sum'], 'percentageOfLinks': ['mean'], 'crossproject': ['mean']})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "robust-swiss",
   "metadata": {},
   "source": [
    "## Table 2: Frequency of Link Types per Repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "dcd5bbee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[34.81491028 18.44990405 20.89187953  4.00662292  7.04043665  8.59275746\n",
      " 14.51793117  1.13210925  4.43029637  1.3446408   3.20471755 96.20992807]\n",
      "[14.30033725 21.54601733 13.76280025  5.35862391  8.08474342  7.19685009\n",
      " 12.53953079  2.1463647   2.99236989  2.98841747  1.86400358  3.7068529 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-75-7d4ee6fa522c>:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Coverage'] = pivot_new.sum(axis=1)\n",
      "<ipython-input-75-7d4ee6fa522c>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new.loc['Mean'] = m\n",
      "<ipython-input-75-7d4ee6fa522c>:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new.loc['Standard Deviation'] = s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>mappedtype</th>\n",
       "      <th>Relate</th>\n",
       "      <th>Duplicate</th>\n",
       "      <th>Subtask</th>\n",
       "      <th>Clone</th>\n",
       "      <th>Block</th>\n",
       "      <th>Depend</th>\n",
       "      <th>Epic</th>\n",
       "      <th>Split</th>\n",
       "      <th>Incorporate</th>\n",
       "      <th>Bonfire Testing</th>\n",
       "      <th>Cause</th>\n",
       "      <th>Coverage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Apache</th>\n",
       "      <td>28.284337</td>\n",
       "      <td>10.136179</td>\n",
       "      <td>32.757549</td>\n",
       "      <td>1.689428</td>\n",
       "      <td>6.079361</td>\n",
       "      <td>5.086270</td>\n",
       "      <td>4.889607</td>\n",
       "      <td>0.010166</td>\n",
       "      <td>4.078321</td>\n",
       "      <td>0.030887</td>\n",
       "      <td>1.214387</td>\n",
       "      <td>94.256491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hyperledger</th>\n",
       "      <td>17.216634</td>\n",
       "      <td>3.913150</td>\n",
       "      <td>27.563788</td>\n",
       "      <td>2.925662</td>\n",
       "      <td>8.243376</td>\n",
       "      <td>NaN</td>\n",
       "      <td>39.616045</td>\n",
       "      <td>0.478410</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.042934</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IntelDAOS</th>\n",
       "      <td>39.322816</td>\n",
       "      <td>9.696037</td>\n",
       "      <td>10.542516</td>\n",
       "      <td>1.462101</td>\n",
       "      <td>25.548288</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>86.571758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JFrog</th>\n",
       "      <td>27.376897</td>\n",
       "      <td>19.913286</td>\n",
       "      <td>36.048312</td>\n",
       "      <td>0.836172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.928151</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.362651</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93.465469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jira</th>\n",
       "      <td>63.762410</td>\n",
       "      <td>21.724321</td>\n",
       "      <td>2.477484</td>\n",
       "      <td>2.871197</td>\n",
       "      <td>0.988790</td>\n",
       "      <td>0.165299</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.181328</td>\n",
       "      <td>2.453441</td>\n",
       "      <td>0.227412</td>\n",
       "      <td>1.787235</td>\n",
       "      <td>96.638916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JiraEcosystem</th>\n",
       "      <td>22.925075</td>\n",
       "      <td>15.274610</td>\n",
       "      <td>20.038603</td>\n",
       "      <td>1.763467</td>\n",
       "      <td>5.948412</td>\n",
       "      <td>1.114231</td>\n",
       "      <td>24.249868</td>\n",
       "      <td>1.193192</td>\n",
       "      <td>1.789788</td>\n",
       "      <td>0.894894</td>\n",
       "      <td>3.860326</td>\n",
       "      <td>99.052465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MariaDB</th>\n",
       "      <td>51.060337</td>\n",
       "      <td>9.399371</td>\n",
       "      <td>6.095225</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.990833</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.444110</td>\n",
       "      <td>0.150499</td>\n",
       "      <td>7.894377</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.965248</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mindville</th>\n",
       "      <td>43.181818</td>\n",
       "      <td>38.636364</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.909091</td>\n",
       "      <td>2.272727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mojang</th>\n",
       "      <td>9.454500</td>\n",
       "      <td>90.006820</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.301586</td>\n",
       "      <td>0.110891</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.126202</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MongoDB</th>\n",
       "      <td>39.910061</td>\n",
       "      <td>13.454819</td>\n",
       "      <td>1.391392</td>\n",
       "      <td>0.316510</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.860814</td>\n",
       "      <td>15.902289</td>\n",
       "      <td>1.178296</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.695367</td>\n",
       "      <td>96.709547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qt</th>\n",
       "      <td>22.416158</td>\n",
       "      <td>10.579728</td>\n",
       "      <td>24.445830</td>\n",
       "      <td>0.084777</td>\n",
       "      <td>0.034908</td>\n",
       "      <td>15.609026</td>\n",
       "      <td>13.534472</td>\n",
       "      <td>6.727341</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93.432240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RedHat</th>\n",
       "      <td>25.909801</td>\n",
       "      <td>4.941129</td>\n",
       "      <td>20.830792</td>\n",
       "      <td>15.436746</td>\n",
       "      <td>15.196918</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.135373</td>\n",
       "      <td>8.908740</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.647302</td>\n",
       "      <td>94.006802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sakai</th>\n",
       "      <td>49.038025</td>\n",
       "      <td>9.347069</td>\n",
       "      <td>17.032773</td>\n",
       "      <td>4.792203</td>\n",
       "      <td>0.030298</td>\n",
       "      <td>13.018230</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.736353</td>\n",
       "      <td>0.005050</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SecondLife</th>\n",
       "      <td>29.477021</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49.762282</td>\n",
       "      <td>7.606973</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.437401</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.218700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93.502377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sonatype</th>\n",
       "      <td>39.977604</td>\n",
       "      <td>7.659574</td>\n",
       "      <td>30.078387</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.628219</td>\n",
       "      <td>0.201568</td>\n",
       "      <td>0.134378</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.085106</td>\n",
       "      <td>5.263158</td>\n",
       "      <td>95.027996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spring</th>\n",
       "      <td>47.725073</td>\n",
       "      <td>12.066104</td>\n",
       "      <td>13.421380</td>\n",
       "      <td>0.096805</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.079934</td>\n",
       "      <td>11.305490</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96.694786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean</th>\n",
       "      <td>34.814910</td>\n",
       "      <td>18.449904</td>\n",
       "      <td>20.891880</td>\n",
       "      <td>4.006623</td>\n",
       "      <td>7.040437</td>\n",
       "      <td>8.592757</td>\n",
       "      <td>14.517931</td>\n",
       "      <td>1.132109</td>\n",
       "      <td>4.430296</td>\n",
       "      <td>1.344641</td>\n",
       "      <td>3.204718</td>\n",
       "      <td>96.209928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Standard Deviation</th>\n",
       "      <td>14.300337</td>\n",
       "      <td>21.546017</td>\n",
       "      <td>13.762800</td>\n",
       "      <td>5.358624</td>\n",
       "      <td>8.084743</td>\n",
       "      <td>7.196850</td>\n",
       "      <td>12.539531</td>\n",
       "      <td>2.146365</td>\n",
       "      <td>2.992370</td>\n",
       "      <td>2.988417</td>\n",
       "      <td>1.864004</td>\n",
       "      <td>3.706853</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "mappedtype             Relate  Duplicate    Subtask      Clone      Block  \\\n",
       "source                                                                      \n",
       "Apache              28.284337  10.136179  32.757549   1.689428   6.079361   \n",
       "Hyperledger         17.216634   3.913150  27.563788   2.925662   8.243376   \n",
       "IntelDAOS           39.322816   9.696037  10.542516   1.462101  25.548288   \n",
       "JFrog               27.376897  19.913286  36.048312   0.836172        NaN   \n",
       "Jira                63.762410  21.724321   2.477484   2.871197   0.988790   \n",
       "JiraEcosystem       22.925075  15.274610  20.038603   1.763467   5.948412   \n",
       "MariaDB             51.060337   9.399371   6.095225        NaN  12.990833   \n",
       "Mindville           43.181818  38.636364        NaN  15.909091   2.272727   \n",
       "Mojang               9.454500  90.006820        NaN   0.301586   0.110891   \n",
       "MongoDB             39.910061  13.454819   1.391392   0.316510        NaN   \n",
       "Qt                  22.416158  10.579728  24.445830   0.084777   0.034908   \n",
       "RedHat              25.909801   4.941129  20.830792  15.436746  15.196918   \n",
       "Sakai               49.038025   9.347069  17.032773   4.792203   0.030298   \n",
       "SecondLife          29.477021        NaN  49.762282   7.606973        NaN   \n",
       "Sonatype            39.977604   7.659574  30.078387        NaN        NaN   \n",
       "Spring              47.725073  12.066104  13.421380   0.096805        NaN   \n",
       "Mean                34.814910  18.449904  20.891880   4.006623   7.040437   \n",
       "Standard Deviation  14.300337  21.546017  13.762800   5.358624   8.084743   \n",
       "\n",
       "mappedtype             Depend       Epic     Split  Incorporate  \\\n",
       "source                                                            \n",
       "Apache               5.086270   4.889607  0.010166     4.078321   \n",
       "Hyperledger               NaN  39.616045  0.478410          NaN   \n",
       "IntelDAOS                 NaN        NaN       NaN          NaN   \n",
       "JFrog                7.928151        NaN       NaN     1.362651   \n",
       "Jira                 0.165299        NaN  0.181328     2.453441   \n",
       "JiraEcosystem        1.114231  24.249868  1.193192     1.789788   \n",
       "MariaDB                   NaN   6.444110  0.150499     7.894377   \n",
       "Mindville                 NaN        NaN       NaN          NaN   \n",
       "Mojang                    NaN        NaN       NaN          NaN   \n",
       "MongoDB             22.860814  15.902289  1.178296          NaN   \n",
       "Qt                  15.609026  13.534472  6.727341          NaN   \n",
       "RedHat                    NaN        NaN  0.135373     8.908740   \n",
       "Sakai               13.018230        NaN       NaN     6.736353   \n",
       "SecondLife           4.437401        NaN       NaN     2.218700   \n",
       "Sonatype             3.628219   0.201568  0.134378          NaN   \n",
       "Spring              12.079934  11.305490       NaN          NaN   \n",
       "Mean                 8.592757  14.517931  1.132109     4.430296   \n",
       "Standard Deviation   7.196850  12.539531  2.146365     2.992370   \n",
       "\n",
       "mappedtype          Bonfire Testing     Cause    Coverage  \n",
       "source                                                     \n",
       "Apache                     0.030887  1.214387   94.256491  \n",
       "Hyperledger                0.042934       NaN  100.000000  \n",
       "IntelDAOS                       NaN       NaN   86.571758  \n",
       "JFrog                           NaN       NaN   93.465469  \n",
       "Jira                       0.227412  1.787235   96.638916  \n",
       "JiraEcosystem              0.894894  3.860326   99.052465  \n",
       "MariaDB                         NaN  5.965248  100.000000  \n",
       "Mindville                       NaN       NaN  100.000000  \n",
       "Mojang                     0.126202       NaN  100.000000  \n",
       "MongoDB                         NaN  1.695367   96.709547  \n",
       "Qt                              NaN       NaN   93.432240  \n",
       "RedHat                          NaN  2.647302   94.006802  \n",
       "Sakai                      0.005050       NaN  100.000000  \n",
       "SecondLife                      NaN       NaN   93.502377  \n",
       "Sonatype                   8.085106  5.263158   95.027996  \n",
       "Spring                          NaN       NaN   96.694786  \n",
       "Mean                       1.344641  3.204718   96.209928  \n",
       "Standard Deviation         2.988417  1.864004    3.706853  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#easier to copy for Latex\n",
    "pivot = pd.pivot_table(commontypes, values='percentageOfLinks', index=['source'],\n",
    "                    columns=['mappedtype'], aggfunc=np.sum)\n",
    "pivot_new = pd.DataFrame()\n",
    "pivot_new = pivot[common_lt_set]\n",
    "\n",
    "pivot_freq = pivot_new\n",
    "\n",
    "pivot_new['Coverage'] = pivot_new.sum(axis=1)\n",
    "\n",
    "m = pivot_new.mean().values\n",
    "print(m)\n",
    "s = pivot_new.std().values\n",
    "print(s)\n",
    "\n",
    "# pivot_new.fillna(0, inplace = True)\n",
    "\n",
    "\n",
    "\n",
    "pivot_new.loc['Mean'] = m\n",
    "pivot_new.loc['Standard Deviation'] = s\n",
    "\n",
    "pivot_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "protected-agenda",
   "metadata": {},
   "outputs": [],
   "source": [
    "pivot_new.to_csv('data/repo_frequencies.csv', encoding=\"UTF-8\", sep=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "18dd5c7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def round_nan(x, n):\n",
    "    try:\n",
    "        return str(round(x,1))\n",
    "    except:\n",
    "        return \" \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "8550dc9f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllllllllll}\n",
      "\\toprule\n",
      "mappedtype & Relate & Duplicate & Subtask & Clone & Block & Depend &  Epic & Split & Incorporate & Bonfire Testing & Cause & Coverage \\\\\n",
      "source             &        &           &         &       &       &        &       &       &             &                 &       &          \\\\\n",
      "\\midrule\n",
      "Apache             &   28.3 &      10.1 &    32.8 &   1.7 &   6.1 &    5.1 &   4.9 &   0.0 &         4.1 &             0.0 &   1.2 &     94.3 \\\\\n",
      "Hyperledger        &   17.2 &       3.9 &    27.6 &   2.9 &   8.2 &    nan &  39.6 &   0.5 &         nan &             0.0 &   nan &    100.0 \\\\\n",
      "IntelDAOS          &   39.3 &       9.7 &    10.5 &   1.5 &  25.5 &    nan &   nan &   nan &         nan &             nan &   nan &     86.6 \\\\\n",
      "JFrog              &   27.4 &      19.9 &    36.0 &   0.8 &   nan &    7.9 &   nan &   nan &         1.4 &             nan &   nan &     93.5 \\\\\n",
      "Jira               &   63.8 &      21.7 &     2.5 &   2.9 &   1.0 &    0.2 &   nan &   0.2 &         2.5 &             0.2 &   1.8 &     96.6 \\\\\n",
      "JiraEcosystem      &   22.9 &      15.3 &    20.0 &   1.8 &   5.9 &    1.1 &  24.2 &   1.2 &         1.8 &             0.9 &   3.9 &     99.1 \\\\\n",
      "MariaDB            &   51.1 &       9.4 &     6.1 &   nan &  13.0 &    nan &   6.4 &   0.2 &         7.9 &             nan &   6.0 &    100.0 \\\\\n",
      "Mindville          &   43.2 &      38.6 &     nan &  15.9 &   2.3 &    nan &   nan &   nan &         nan &             nan &   nan &    100.0 \\\\\n",
      "Mojang             &    9.5 &      90.0 &     nan &   0.3 &   0.1 &    nan &   nan &   nan &         nan &             0.1 &   nan &    100.0 \\\\\n",
      "MongoDB            &   39.9 &      13.5 &     1.4 &   0.3 &   nan &   22.9 &  15.9 &   1.2 &         nan &             nan &   1.7 &     96.7 \\\\\n",
      "Qt                 &   22.4 &      10.6 &    24.4 &   0.1 &   0.0 &   15.6 &  13.5 &   6.7 &         nan &             nan &   nan &     93.4 \\\\\n",
      "RedHat             &   25.9 &       4.9 &    20.8 &  15.4 &  15.2 &    nan &   nan &   0.1 &         8.9 &             nan &   2.6 &     94.0 \\\\\n",
      "Sakai              &   49.0 &       9.3 &    17.0 &   4.8 &   0.0 &   13.0 &   nan &   nan &         6.7 &             0.0 &   nan &    100.0 \\\\\n",
      "SecondLife         &   29.5 &       nan &    49.8 &   7.6 &   nan &    4.4 &   nan &   nan &         2.2 &             nan &   nan &     93.5 \\\\\n",
      "Sonatype           &   40.0 &       7.7 &    30.1 &   nan &   nan &    3.6 &   0.2 &   0.1 &         nan &             8.1 &   5.3 &     95.0 \\\\\n",
      "Spring             &   47.7 &      12.1 &    13.4 &   0.1 &   nan &   12.1 &  11.3 &   nan &         nan &             nan &   nan &     96.7 \\\\\n",
      "Mean               &   34.8 &      18.4 &    20.9 &   4.0 &   7.0 &    8.6 &  14.5 &   1.1 &         4.4 &             1.3 &   3.2 &     96.2 \\\\\n",
      "Standard Deviation &   14.3 &      21.5 &    13.8 &   5.4 &   8.1 &    7.2 &  12.5 &   2.1 &         3.0 &             3.0 &   1.9 &      3.7 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-63-969f38075fd8>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Relate'] = pivot_new['Relate'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Subtask'] = pivot_new['Subtask'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Duplicate'] = pivot_new['Duplicate'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Clone'] = pivot_new['Clone'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Depend'] = pivot_new['Depend'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Epic'] = pivot_new['Epic'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Split'] = pivot_new['Split'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Bonfire Testing'] = pivot_new['Bonfire Testing'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Cause'] = pivot_new['Cause'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Block'] = pivot_new['Block'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Incorporate'] = pivot_new['Incorporate'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  pivot_new['Coverage'] = pivot_new['Coverage'].apply(lambda x: round_nan(x,2))\n",
      "<ipython-input-63-969f38075fd8>:17: FutureWarning: In future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n",
      "  print(pivot_new.round(2).to_latex())\n"
     ]
    }
   ],
   "source": [
    "pivot_new.round(1)\n",
    "\n",
    "\n",
    "pivot_new['Relate'] = pivot_new['Relate'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Subtask'] = pivot_new['Subtask'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Duplicate'] = pivot_new['Duplicate'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Clone'] = pivot_new['Clone'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Depend'] = pivot_new['Depend'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Epic'] = pivot_new['Epic'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Split'] = pivot_new['Split'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Bonfire Testing'] = pivot_new['Bonfire Testing'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Cause'] = pivot_new['Cause'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Block'] = pivot_new['Block'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Incorporate'] = pivot_new['Incorporate'].apply(lambda x: round_nan(x,2))\n",
    "pivot_new['Coverage'] = pivot_new['Coverage'].apply(lambda x: round_nan(x,2))\n",
    "\n",
    "print(pivot_new.round(2).to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "3c359007",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>mappedtype</th>\n",
       "      <th>Relate</th>\n",
       "      <th>Duplicate</th>\n",
       "      <th>Subtask</th>\n",
       "      <th>Clone</th>\n",
       "      <th>Block</th>\n",
       "      <th>Depend</th>\n",
       "      <th>Epic</th>\n",
       "      <th>Split</th>\n",
       "      <th>Incorporate</th>\n",
       "      <th>Bonfire Testing</th>\n",
       "      <th>Cause</th>\n",
       "      <th>Coverage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Apache</th>\n",
       "      <td>28.3</td>\n",
       "      <td>10.1</td>\n",
       "      <td>32.8</td>\n",
       "      <td>1.7</td>\n",
       "      <td>6.1</td>\n",
       "      <td>5.1</td>\n",
       "      <td>4.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>94.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hyperledger</th>\n",
       "      <td>17.2</td>\n",
       "      <td>3.9</td>\n",
       "      <td>27.6</td>\n",
       "      <td>2.9</td>\n",
       "      <td>8.2</td>\n",
       "      <td>nan</td>\n",
       "      <td>39.6</td>\n",
       "      <td>0.5</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nan</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IntelDAOS</th>\n",
       "      <td>39.3</td>\n",
       "      <td>9.7</td>\n",
       "      <td>10.5</td>\n",
       "      <td>1.5</td>\n",
       "      <td>25.5</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>86.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JFrog</th>\n",
       "      <td>27.4</td>\n",
       "      <td>19.9</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>nan</td>\n",
       "      <td>7.9</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>1.4</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>93.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jira</th>\n",
       "      <td>63.8</td>\n",
       "      <td>21.7</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1.8</td>\n",
       "      <td>96.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JiraEcosystem</th>\n",
       "      <td>22.9</td>\n",
       "      <td>15.3</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1.8</td>\n",
       "      <td>5.9</td>\n",
       "      <td>1.1</td>\n",
       "      <td>24.2</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1.8</td>\n",
       "      <td>0.9</td>\n",
       "      <td>3.9</td>\n",
       "      <td>99.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MariaDB</th>\n",
       "      <td>51.1</td>\n",
       "      <td>9.4</td>\n",
       "      <td>6.1</td>\n",
       "      <td>nan</td>\n",
       "      <td>13.0</td>\n",
       "      <td>nan</td>\n",
       "      <td>6.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>7.9</td>\n",
       "      <td>nan</td>\n",
       "      <td>6.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mindville</th>\n",
       "      <td>43.2</td>\n",
       "      <td>38.6</td>\n",
       "      <td>nan</td>\n",
       "      <td>15.9</td>\n",
       "      <td>2.3</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mojang</th>\n",
       "      <td>9.5</td>\n",
       "      <td>90.0</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.1</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.1</td>\n",
       "      <td>nan</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MongoDB</th>\n",
       "      <td>39.9</td>\n",
       "      <td>13.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.3</td>\n",
       "      <td>nan</td>\n",
       "      <td>22.9</td>\n",
       "      <td>15.9</td>\n",
       "      <td>1.2</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>1.7</td>\n",
       "      <td>96.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qt</th>\n",
       "      <td>22.4</td>\n",
       "      <td>10.6</td>\n",
       "      <td>24.4</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.6</td>\n",
       "      <td>13.5</td>\n",
       "      <td>6.7</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>93.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RedHat</th>\n",
       "      <td>25.9</td>\n",
       "      <td>4.9</td>\n",
       "      <td>20.8</td>\n",
       "      <td>15.4</td>\n",
       "      <td>15.2</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.1</td>\n",
       "      <td>8.9</td>\n",
       "      <td>nan</td>\n",
       "      <td>2.6</td>\n",
       "      <td>94.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sakai</th>\n",
       "      <td>49.0</td>\n",
       "      <td>9.3</td>\n",
       "      <td>17.0</td>\n",
       "      <td>4.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>6.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>nan</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SecondLife</th>\n",
       "      <td>29.5</td>\n",
       "      <td>nan</td>\n",
       "      <td>49.8</td>\n",
       "      <td>7.6</td>\n",
       "      <td>nan</td>\n",
       "      <td>4.4</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>2.2</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>93.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sonatype</th>\n",
       "      <td>40.0</td>\n",
       "      <td>7.7</td>\n",
       "      <td>30.1</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>3.6</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.1</td>\n",
       "      <td>nan</td>\n",
       "      <td>8.1</td>\n",
       "      <td>5.3</td>\n",
       "      <td>95.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spring</th>\n",
       "      <td>47.7</td>\n",
       "      <td>12.1</td>\n",
       "      <td>13.4</td>\n",
       "      <td>0.1</td>\n",
       "      <td>nan</td>\n",
       "      <td>12.1</td>\n",
       "      <td>11.3</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>96.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean</th>\n",
       "      <td>34.8</td>\n",
       "      <td>18.4</td>\n",
       "      <td>20.9</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.6</td>\n",
       "      <td>14.5</td>\n",
       "      <td>1.1</td>\n",
       "      <td>4.4</td>\n",
       "      <td>1.3</td>\n",
       "      <td>3.2</td>\n",
       "      <td>96.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Standard Deviation</th>\n",
       "      <td>14.3</td>\n",
       "      <td>21.5</td>\n",
       "      <td>13.8</td>\n",
       "      <td>5.4</td>\n",
       "      <td>8.1</td>\n",
       "      <td>7.2</td>\n",
       "      <td>12.5</td>\n",
       "      <td>2.1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.9</td>\n",
       "      <td>3.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "mappedtype         Relate Duplicate Subtask Clone Block Depend  Epic Split  \\\n",
       "source                                                                       \n",
       "Apache               28.3      10.1    32.8   1.7   6.1    5.1   4.9   0.0   \n",
       "Hyperledger          17.2       3.9    27.6   2.9   8.2    nan  39.6   0.5   \n",
       "IntelDAOS            39.3       9.7    10.5   1.5  25.5    nan   nan   nan   \n",
       "JFrog                27.4      19.9    36.0   0.8   nan    7.9   nan   nan   \n",
       "Jira                 63.8      21.7     2.5   2.9   1.0    0.2   nan   0.2   \n",
       "JiraEcosystem        22.9      15.3    20.0   1.8   5.9    1.1  24.2   1.2   \n",
       "MariaDB              51.1       9.4     6.1   nan  13.0    nan   6.4   0.2   \n",
       "Mindville            43.2      38.6     nan  15.9   2.3    nan   nan   nan   \n",
       "Mojang                9.5      90.0     nan   0.3   0.1    nan   nan   nan   \n",
       "MongoDB              39.9      13.5     1.4   0.3   nan   22.9  15.9   1.2   \n",
       "Qt                   22.4      10.6    24.4   0.1   0.0   15.6  13.5   6.7   \n",
       "RedHat               25.9       4.9    20.8  15.4  15.2    nan   nan   0.1   \n",
       "Sakai                49.0       9.3    17.0   4.8   0.0   13.0   nan   nan   \n",
       "SecondLife           29.5       nan    49.8   7.6   nan    4.4   nan   nan   \n",
       "Sonatype             40.0       7.7    30.1   nan   nan    3.6   0.2   0.1   \n",
       "Spring               47.7      12.1    13.4   0.1   nan   12.1  11.3   nan   \n",
       "Mean                 34.8      18.4    20.9   4.0   7.0    8.6  14.5   1.1   \n",
       "Standard Deviation   14.3      21.5    13.8   5.4   8.1    7.2  12.5   2.1   \n",
       "\n",
       "mappedtype         Incorporate Bonfire Testing Cause Coverage  \n",
       "source                                                         \n",
       "Apache                     4.1             0.0   1.2     94.3  \n",
       "Hyperledger                nan             0.0   nan    100.0  \n",
       "IntelDAOS                  nan             nan   nan     86.6  \n",
       "JFrog                      1.4             nan   nan     93.5  \n",
       "Jira                       2.5             0.2   1.8     96.6  \n",
       "JiraEcosystem              1.8             0.9   3.9     99.1  \n",
       "MariaDB                    7.9             nan   6.0    100.0  \n",
       "Mindville                  nan             nan   nan    100.0  \n",
       "Mojang                     nan             0.1   nan    100.0  \n",
       "MongoDB                    nan             nan   1.7     96.7  \n",
       "Qt                         nan             nan   nan     93.4  \n",
       "RedHat                     8.9             nan   2.6     94.0  \n",
       "Sakai                      6.7             0.0   nan    100.0  \n",
       "SecondLife                 2.2             nan   nan     93.5  \n",
       "Sonatype                   nan             8.1   5.3     95.0  \n",
       "Spring                     nan             nan   nan     96.7  \n",
       "Mean                       4.4             1.3   3.2     96.2  \n",
       "Standard Deviation         3.0             3.0   1.9      3.7  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pivot_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "declared-cotton",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}