scripts/10_calculate_confusion_matrices.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalised confusion matrix value calculator\n",
    "\n",
    "### Producing confusion matrices for each tested model, averaged over each imputation fold\n",
    "\n",
    "Shubhayu Bhattacharyay\n",
    "<br>\n",
    "Ari Ercole"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I. Initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import necessary packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fundamental methods\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import glob\n",
    "import random\n",
    "import warnings\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle as cp\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "warnings.filterwarnings(action=\"ignore\")\n",
    "\n",
    "# Keras and tensorflow methods\n",
    "import keras\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers import Input, Activation, Dense, Dropout, Conv2D, Flatten, LSTM, Permute, Reshape, AlphaDropout, BatchNormalization\n",
    "import tensorflow as tf\n",
    "\n",
    "# SciKit-Learn methods\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.model_selection import StratifiedShuffleSplit "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II. Confusion matrices of each model type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load compiled prediction results from each model variant\n",
    "compiled_mnlr_results = pd.read_csv('../repeated_cv/compiled_predictions/mnlr.csv')\n",
    "compiled_polr_results = pd.read_csv('../repeated_cv/compiled_predictions/polr.csv')\n",
    "compiled_deepMN_results = pd.read_csv('../repeated_cv/compiled_predictions/deepMN.csv')\n",
    "compiled_deepOR_results = pd.read_csv('../repeated_cv/compiled_predictions/deepOR.csv')\n",
    "\n",
    "# Removed banned tuning indices from both DeepMN and DeepOR\n",
    "deepMN_banned_tuning_indices = pd.read_csv('../repeated_cv/deepMN_banned_tuning_indices.csv')\n",
    "deepMN_banned_tuning_indices = deepMN_banned_tuning_indices[deepMN_banned_tuning_indices['class'] == 'macro-averaged']\n",
    "\n",
    "deepOR_banned_tuning_indices = pd.read_csv('../repeated_cv/deepOR_banned_tuning_indices.csv')\n",
    "deepOR_banned_tuning_indices = deepOR_banned_tuning_indices[deepOR_banned_tuning_indices['class'] == 'macro-averaged']\n",
    "\n",
    "compiled_deepMN_results = compiled_deepMN_results[~compiled_deepMN_results.tune_idx.isin(deepMN_banned_tuning_indices.tune_idx)]\n",
    "compiled_deepOR_results = compiled_deepOR_results[~compiled_deepOR_results.tune_idx.isin(deepOR_banned_tuning_indices.tune_idx)]\n",
    "\n",
    "# Identify viable columnns for AUROC consideration\n",
    "viable_deepMN_tune_indices = compiled_deepMN_results.tune_idx.unique()\n",
    "viable_deepOR_tune_indices = compiled_deepOR_results.tune_idx.unique()\n",
    "\n",
    "# Identify columns that hold probability scores\n",
    "prob_cols = [col for col in compiled_deepMN_results if col.startswith('prob_GOSE_')]\n",
    "\n",
    "# Initialize empty dataframes to store AUROCs for each viable tuning index on each Repeat and Fold combination\n",
    "deepMN_AUROCs = pd.DataFrame(np.empty((0,4)),columns = ['repeat.name','fold.name','tune_idx','AUROC'])\n",
    "deepOR_AUROCs = pd.DataFrame(np.empty((0,4)),columns = ['repeat.name','fold.name','tune_idx','AUROC'])\n",
    "mnlr_AUROCs = pd.DataFrame(np.empty((0,4)),columns = ['repeat.name','fold.name','SMOTE','AUROC'])\n",
    "polr_AUROCs = pd.DataFrame(np.empty((0,4)),columns = ['repeat.name','fold.name','SMOTE','AUROC'])\n",
    "\n",
    "# First calculate macro-averaged AUROC within each repeat and fold combination for each model type\n",
    "for curr_repeat_idx in compiled_deepMN_results['repeat.name'].unique():\n",
    "    curr_repeat_name = 'Repeat' + str(int(curr_repeat_idx)).zfill(2)\n",
    "    for curr_fold_idx in compiled_deepMN_results[compiled_deepMN_results['repeat.name'] == curr_repeat_idx]['fold.name'].unique():\n",
    "        curr_fold_name = 'Fold' + str(int(curr_fold_idx)).zfill(1)\n",
    "        for curr_deepMN_tune_idx in viable_deepMN_tune_indices:\n",
    "            curr_deepMN_results_idx =  (compiled_deepMN_results['repeat.name'] == curr_repeat_idx) & \\\n",
    "            (compiled_deepMN_results['fold.name'] == curr_fold_idx) & \\\n",
    "            (compiled_deepMN_results['tune_idx'] == curr_deepMN_tune_idx)\n",
    "            curr_deepMN_results = compiled_deepMN_results[curr_deepMN_results_idx]\n",
    "            curr_labels = label_binarize(curr_deepMN_results.true_labels.astype('int').values,classes=[1,3,4,5,6,7,8])\n",
    "            curr_deepMN_auroc = roc_auc_score(y_true = curr_labels, y_score = curr_deepMN_results[prob_cols].values,average='macro',multi_class='ovr')\n",
    "            curr_deepMN_auroc_df = pd.DataFrame({'repeat.name': curr_repeat_name,\\\n",
    "                                                'fold.name': curr_fold_name,\\\n",
    "                                                'tune_idx': [curr_deepMN_tune_idx],\\\n",
    "                                                'AUROC': [curr_deepMN_auroc]})    \n",
    "            deepMN_AUROCs = deepMN_AUROCs.append(curr_deepMN_auroc_df, ignore_index = True)\n",
    "        for curr_deepOR_tune_idx in viable_deepOR_tune_indices:\n",
    "            curr_deepOR_results_idx =  (compiled_deepOR_results['repeat.name'] == curr_repeat_idx) & \\\n",
    "            (compiled_deepOR_results['fold.name'] == curr_fold_idx) & \\\n",
    "            (compiled_deepOR_results['tune_idx'] == curr_deepOR_tune_idx)\n",
    "            curr_deepOR_results = compiled_deepOR_results[curr_deepOR_results_idx]\n",
    "            curr_labels = label_binarize(curr_deepOR_results.true_labels.astype('int').values,classes=[1,3,4,5,6,7,8])\n",
    "            curr_deepOR_auroc = roc_auc_score(y_true = curr_labels, y_score = curr_deepOR_results[prob_cols].values,average='macro',multi_class='ovr')\n",
    "            curr_deepOR_auroc_df = pd.DataFrame({'repeat.name': curr_repeat_name,\\\n",
    "                                                'fold.name': curr_fold_name,\\\n",
    "                                                'tune_idx': [curr_deepOR_tune_idx],\\\n",
    "                                                'AUROC': [curr_deepOR_auroc]})    \n",
    "            deepOR_AUROCs = deepOR_AUROCs.append(curr_deepOR_auroc_df, ignore_index = True)            \n",
    "        for curr_SMOTE in [0,1]:\n",
    "            curr_mnlr_results_idx =  (compiled_mnlr_results['repeat.name'] == curr_repeat_name) & \\\n",
    "            (compiled_mnlr_results['fold.name'] == curr_fold_name) & \\\n",
    "            (compiled_mnlr_results['SMOTE'] == curr_SMOTE)\n",
    "            curr_mnlr_results = compiled_mnlr_results[curr_mnlr_results_idx]\n",
    "            curr_labels = label_binarize(curr_mnlr_results['true.labels'].astype('int').values,classes=[1,3,4,5,6,7,8])\n",
    "            curr_mnlr_auroc = roc_auc_score(y_true = curr_labels, y_score = curr_mnlr_results[prob_cols].values,average='macro',multi_class='ovr')\n",
    "            curr_mnlr_auroc_df = pd.DataFrame({'repeat.name': curr_repeat_name,\\\n",
    "                                                'fold.name': curr_fold_name,\\\n",
    "                                                'SMOTE': [curr_SMOTE],\\\n",
    "                                                'AUROC': [curr_mnlr_auroc]}) \n",
    "            mnlr_AUROCs = mnlr_AUROCs.append(curr_mnlr_auroc_df, ignore_index = True)            \n",
    "            \n",
    "            curr_polr_results_idx =  (compiled_polr_results['repeat.name'] == curr_repeat_name) & \\\n",
    "            (compiled_polr_results['fold.name'] == curr_fold_name) & \\\n",
    "            (compiled_polr_results['SMOTE'] == curr_SMOTE)\n",
    "            curr_polr_results = compiled_polr_results[curr_polr_results_idx]\n",
    "            curr_labels = label_binarize(curr_polr_results['true.labels'].astype('int').values,classes=[1,3,4,5,6,7,8])\n",
    "            curr_polr_auroc = roc_auc_score(y_true = curr_labels, y_score = curr_polr_results[prob_cols].values,average='macro',multi_class='ovr')\n",
    "            curr_polr_auroc_df = pd.DataFrame({'repeat.name': curr_repeat_name,\\\n",
    "                                                'fold.name': curr_fold_name,\\\n",
    "                                                'SMOTE': [curr_SMOTE],\\\n",
    "                                                'AUROC': [curr_polr_auroc]}) \n",
    "            polr_AUROCs = polr_AUROCs.append(curr_polr_auroc_df, ignore_index = True)  \n",
    "\n",
    "            \n",
    "deepMN_AUROCs.to_csv('../metrics/deepMN_repeatedCV_aurocs.csv', index=False)\n",
    "deepOR_AUROCs.to_csv('../metrics/deepOR_repeatedCV_aurocs.csv', index=False)\n",
    "mnlr_AUROCs.to_csv('../metrics/mnlr_repeatedCV_aurocs.csv', index=False)\n",
    "polr_AUROCs.to_csv('../metrics/polr_repeatedCV_aurocs.csv', index=False)\n",
    "\n",
    "# Second, calculate optimal model configuration for each repeat\n",
    "deepMN_repeats_AUROCs = deepMN_AUROCs.groupby(['tune_idx','repeat.name'],as_index=False)[['AUROC']].mean()\n",
    "opt_deepMN_AUROCs = deepMN_repeats_AUROCs[deepMN_repeats_AUROCs.groupby(['repeat.name'])['AUROC'].transform(max) == deepMN_repeats_AUROCs['AUROC']].sort_values(by=['repeat.name']).reset_index(drop=True)\n",
    "\n",
    "deepOR_repeats_AUROCs = deepOR_AUROCs.groupby(['tune_idx','repeat.name'],as_index=False)[['AUROC']].mean()\n",
    "opt_deepOR_AUROCs = deepOR_repeats_AUROCs[deepOR_repeats_AUROCs.groupby(['repeat.name'])['AUROC'].transform(max) == deepOR_repeats_AUROCs['AUROC']].sort_values(by=['repeat.name']).reset_index(drop=True)\n",
    "\n",
    "mnlr_repeats_AUROCs = mnlr_AUROCs.groupby(['SMOTE','repeat.name'],as_index=False)[['AUROC']].mean()\n",
    "opt_mnlr_AUROCs = mnlr_repeats_AUROCs[mnlr_repeats_AUROCs.groupby(['repeat.name'])['AUROC'].transform(max) == mnlr_repeats_AUROCs['AUROC']].sort_values(by=['repeat.name']).reset_index(drop=True)\n",
    "\n",
    "polr_repeats_AUROCs = polr_AUROCs.groupby(['SMOTE','repeat.name'],as_index=False)[['AUROC']].mean()\n",
    "opt_polr_AUROCs = polr_repeats_AUROCs[polr_repeats_AUROCs.groupby(['repeat.name'])['AUROC'].transform(max) == polr_repeats_AUROCs['AUROC']].sort_values(by=['repeat.name']).reset_index(drop=True)\n",
    "\n",
    "opt_deepMN_AUROCs.to_csv('../metrics/deepMN_optimal_repeatedCV_aurocs.csv', index=False)\n",
    "opt_deepOR_AUROCs.to_csv('../metrics/deepOR_optimal_repeatedCV_aurocs.csv', index=False)\n",
    "opt_mnlr_AUROCs.to_csv('../metrics/mnlr_optimal_repeatedCV_aurocs.csv', index=False)\n",
    "opt_polr_AUROCs.to_csv('../metrics/polr_optimal_repeatedCV_aurocs.csv', index=False)\n",
    "\n",
    "# Loop through optimal indices for results to calculate confusion matrix values per repeat\n",
    "deepMN_CMs = pd.DataFrame(np.empty((0,5)),columns = ['true_labels','predicted_labels','cm_prob','repeat.name','tune_idx'])\n",
    "deepOR_CMs = pd.DataFrame(np.empty((0,5)),columns = ['true_labels','predicted_labels','cm_prob','repeat.name','tune_idx'])\n",
    "mnlr_CMs = pd.DataFrame(np.empty((0,5)),columns = ['true_labels','predicted_labels','cm_prob','repeat.name','tune_idx'])\n",
    "polr_CMs = pd.DataFrame(np.empty((0,5)),columns = ['true_labels','predicted_labels','cm_prob','repeat.name','tune_idx'])\n",
    "\n",
    "for curr_repeat_idx in compiled_deepMN_results['repeat.name'].unique():\n",
    "    curr_repeat_name = 'Repeat' + str(int(curr_repeat_idx)).zfill(2)\n",
    "\n",
    "    curr_opt_deepMN_tune_idx = opt_deepMN_AUROCs.tune_idx[opt_deepMN_AUROCs['repeat.name'] == curr_repeat_name].values[0]\n",
    "    curr_deepMN_results = compiled_deepMN_results[(compiled_deepMN_results.tune_idx == curr_opt_deepMN_tune_idx) & (compiled_deepMN_results['repeat.name'] == curr_repeat_idx)]\n",
    "    \n",
    "    curr_deepMN_cm = confusion_matrix(curr_deepMN_results.true_labels, curr_deepMN_results.pred_labels,normalize='true')\n",
    "    curr_deepMN_cm = pd.DataFrame(curr_deepMN_cm)\n",
    "    curr_deepMN_cm.columns = ['GOSE: 1','GOSE: 2 or 3','GOSE: 4','GOSE: 5','GOSE: 6','GOSE: 7','GOSE: 8']\n",
    "    curr_deepMN_cm = curr_deepMN_cm.assign(true_labels=['GOSE: 1','GOSE: 2 or 3','GOSE: 4','GOSE: 5','GOSE: 6','GOSE: 7','GOSE: 8'])\n",
    "    curr_deepMN_cm = curr_deepMN_cm.melt(id_vars=['true_labels'],var_name='predicted_labels',value_name='cm_prob')\n",
    "    curr_deepMN_cm['repeat.name'] = curr_repeat_name\n",
    "    curr_deepMN_cm['tune_idx'] = curr_opt_deepMN_tune_idx\n",
    "    deepMN_CMs = deepMN_CMs.append(curr_deepMN_cm, ignore_index = True)\n",
    "    \n",
    "    curr_opt_deepOR_tune_idx = opt_deepOR_AUROCs.tune_idx[opt_deepOR_AUROCs['repeat.name'] == curr_repeat_name].values[0]\n",
    "    curr_deepOR_results = compiled_deepOR_results[(compiled_deepOR_results.tune_idx == curr_opt_deepOR_tune_idx) & (compiled_deepOR_results['repeat.name'] == curr_repeat_idx)]\n",
    "    \n",
    "    curr_deepOR_cm = confusion_matrix(curr_deepOR_results.true_labels, curr_deepOR_results.pred_labels,normalize='true')\n",
    "    curr_deepOR_cm = pd.DataFrame(curr_deepOR_cm)\n",
    "    curr_deepOR_cm.columns = ['GOSE: 1','GOSE: 2 or 3','GOSE: 4','GOSE: 5','GOSE: 6','GOSE: 7','GOSE: 8']\n",
    "    curr_deepOR_cm = curr_deepOR_cm.assign(true_labels=['GOSE: 1','GOSE: 2 or 3','GOSE: 4','GOSE: 5','GOSE: 6','GOSE: 7','GOSE: 8'])\n",
    "    curr_deepOR_cm = curr_deepOR_cm.melt(id_vars=['true_labels'],var_name='predicted_labels',value_name='cm_prob')\n",
    "    curr_deepOR_cm['repeat.name'] = curr_repeat_name\n",
    "    curr_deepOR_cm['tune_idx'] = curr_opt_deepOR_tune_idx\n",
    "    deepOR_CMs = deepOR_CMs.append(curr_deepOR_cm, ignore_index = True)\n",
    "    \n",
    "    curr_opt_mnlr_tune_idx = opt_mnlr_AUROCs.SMOTE[opt_mnlr_AUROCs['repeat.name'] == curr_repeat_name].values[0]\n",
    "    curr_mnlr_results = compiled_mnlr_results[(compiled_mnlr_results.SMOTE == int(curr_opt_mnlr_tune_idx)) & (compiled_mnlr_results['repeat.name'] == curr_repeat_name)]\n",
    "    \n",
    "    curr_mnlr_cm = confusion_matrix(curr_mnlr_results['true.labels'], curr_mnlr_results['pred.labels'],normalize='true')\n",
    "    curr_mnlr_cm = pd.DataFrame(curr_mnlr_cm)\n",
    "    curr_mnlr_cm.columns = ['GOSE: 1','GOSE: 2 or 3','GOSE: 4','GOSE: 5','GOSE: 6','GOSE: 7','GOSE: 8']\n",
    "    curr_mnlr_cm = curr_mnlr_cm.assign(true_labels=['GOSE: 1','GOSE: 2 or 3','GOSE: 4','GOSE: 5','GOSE: 6','GOSE: 7','GOSE: 8'])\n",
    "    curr_mnlr_cm = curr_mnlr_cm.melt(id_vars=['true_labels'],var_name='predicted_labels',value_name='cm_prob')\n",
    "    curr_mnlr_cm['repeat.name'] = curr_repeat_name\n",
    "    curr_mnlr_cm['SMOTE'] = curr_opt_mnlr_tune_idx\n",
    "    mnlr_CMs = mnlr_CMs.append(curr_mnlr_cm, ignore_index = True)\n",
    "    \n",
    "    curr_opt_polr_tune_idx = opt_polr_AUROCs.SMOTE[opt_polr_AUROCs['repeat.name'] == curr_repeat_name].values[0]\n",
    "    curr_polr_results = compiled_polr_results[(compiled_polr_results.SMOTE == int(curr_opt_polr_tune_idx)) & (compiled_polr_results['repeat.name'] == curr_repeat_name)]\n",
    "    \n",
    "    curr_polr_cm = confusion_matrix(curr_polr_results['true.labels'], curr_polr_results['pred.labels'],normalize='true')\n",
    "    curr_polr_cm = pd.DataFrame(curr_polr_cm)\n",
    "    curr_polr_cm.columns = ['GOSE: 1','GOSE: 2 or 3','GOSE: 4','GOSE: 5','GOSE: 6','GOSE: 7','GOSE: 8']\n",
    "    curr_polr_cm = curr_polr_cm.assign(true_labels=['GOSE: 1','GOSE: 2 or 3','GOSE: 4','GOSE: 5','GOSE: 6','GOSE: 7','GOSE: 8'])\n",
    "    curr_polr_cm = curr_polr_cm.melt(id_vars=['true_labels'],var_name='predicted_labels',value_name='cm_prob')\n",
    "    curr_polr_cm['repeat.name'] = curr_repeat_name\n",
    "    curr_polr_cm['SMOTE'] = curr_opt_polr_tune_idx\n",
    "    polr_CMs = polr_CMs.append(curr_polr_cm, ignore_index = True)\n",
    "    \n",
    "deepMN_CMs.to_csv('../metrics/deepMN_confusionMatrices.csv', index=False)\n",
    "deepOR_CMs.to_csv('../metrics/deepOR_confusionMatrices.csv', index=False)\n",
    "mnlr_CMs.to_csv('../metrics/mnlr_confusionMatrices.csv', index=False)\n",
    "polr_CMs.to_csv('../metrics/polr_confusionMatrices.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sb_venv",
   "language": "python",
   "name": "sb_venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}