From 2a53cf0c9b7ff185f1057451e7f327f852842964 Mon Sep 17 00:00:00 2001
From: Patrick Hall <jpatrickhall@gmail.com>
Date: Mon, 19 Jun 2023 21:23:15 -0400
Subject: [PATCH] try to grade assign 3

---
 .../assignment_3/assign_3_template.ipynb      |    4 +-
 assignments/eval.ipynb                        | 1452 +++++------------
 .../model_eval_2023_06_01_11_08_41.csv        |   26 -
 .../model_eval_2023_06_19_21_04_44.csv        |   26 +
 4 files changed, 456 insertions(+), 1052 deletions(-)
 delete mode 100644 assignments/model_eval_2023_06_01_11_08_41.csv
 create mode 100644 assignments/model_eval_2023_06_19_21_04_44.csv

diff --git a/assignments/assignment_3/assign_3_template.ipynb b/assignments/assignment_3/assign_3_template.ipynb
index abd847a..6ed0394 100644
--- a/assignments/assignment_3/assign_3_template.ipynb
+++ b/assignments/assignment_3/assign_3_template.ipynb
@@ -3378,7 +3378,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "3aae893f",
+   "id": "f1e0ec79",
    "metadata": {},
    "source": [
     "#### Write submission file"
@@ -3387,7 +3387,7 @@
   {
    "cell_type": "code",
    "execution_count": 29,
-   "id": "fe28a2d8",
+   "id": "c2d71d87",
    "metadata": {},
    "outputs": [],
    "source": [
diff --git a/assignments/eval.ipynb b/assignments/eval.ipynb
index c0d162e..8f4e521 100644
--- a/assignments/eval.ipynb
+++ b/assignments/eval.ipynb
@@ -119,25 +119,12 @@
        "      <th></th>\n",
        "      <th>high_priced</th>\n",
        "      <th>fold</th>\n",
-       "      <th>group1_ebm</th>\n",
-       "      <th>group1_glm</th>\n",
-       "      <th>group1_mxgb</th>\n",
-       "      <th>group2_piml_EBM</th>\n",
-       "      <th>group2_piml_XGB2</th>\n",
-       "      <th>group2_piml_reludnn</th>\n",
-       "      <th>group3_piml_EBM</th>\n",
-       "      <th>group3_piml_GAM</th>\n",
-       "      <th>...</th>\n",
-       "      <th>group5_xgb2</th>\n",
-       "      <th>group8_piml_ebm</th>\n",
-       "      <th>group8_piml_gam</th>\n",
-       "      <th>group8_piml_reludnn</th>\n",
-       "      <th>group9_ebm</th>\n",
-       "      <th>group9_glm</th>\n",
-       "      <th>group9_mxgb</th>\n",
-       "      <th>ph_ebm</th>\n",
-       "      <th>ph_glm</th>\n",
-       "      <th>ph_mxgb</th>\n",
+       "      <th>group1_rem_ebm</th>\n",
+       "      <th>group2_rem_ebm</th>\n",
+       "      <th>group3_rem_piml_EBM</th>\n",
+       "      <th>group5_rem_xgb2</th>\n",
+       "      <th>group8_rem_ebm</th>\n",
+       "      <th>ph_rem_ebm</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -145,121 +132,56 @@
        "      <th>0</th>\n",
        "      <td>0.0</td>\n",
        "      <td>2</td>\n",
+       "      <td>0.118787</td>\n",
        "      <td>0.080557</td>\n",
-       "      <td>0.142090</td>\n",
-       "      <td>0.059522</td>\n",
-       "      <td>0.060440</td>\n",
-       "      <td>0.058846</td>\n",
-       "      <td>0.054570</td>\n",
-       "      <td>0.079611</td>\n",
-       "      <td>0.109994</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.077672</td>\n",
-       "      <td>0.073114</td>\n",
-       "      <td>0.066971</td>\n",
-       "      <td>0.066971</td>\n",
-       "      <td>0.080557</td>\n",
-       "      <td>0.142090</td>\n",
-       "      <td>0.086361</td>\n",
-       "      <td>0.080557</td>\n",
-       "      <td>0.142090</td>\n",
-       "      <td>0.071625</td>\n",
+       "      <td>0.920389</td>\n",
+       "      <td>0.078326</td>\n",
+       "      <td>0.223846</td>\n",
+       "      <td>0.219429</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>0.0</td>\n",
        "      <td>1</td>\n",
+       "      <td>0.084506</td>\n",
        "      <td>0.026001</td>\n",
-       "      <td>0.081674</td>\n",
-       "      <td>0.036210</td>\n",
-       "      <td>0.030870</td>\n",
-       "      <td>0.030136</td>\n",
-       "      <td>0.032199</td>\n",
-       "      <td>0.030699</td>\n",
-       "      <td>0.050429</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.035456</td>\n",
-       "      <td>0.029787</td>\n",
-       "      <td>0.034471</td>\n",
-       "      <td>0.034471</td>\n",
-       "      <td>0.026001</td>\n",
-       "      <td>0.081674</td>\n",
-       "      <td>0.033920</td>\n",
-       "      <td>0.026001</td>\n",
-       "      <td>0.081674</td>\n",
-       "      <td>0.036750</td>\n",
+       "      <td>0.969301</td>\n",
+       "      <td>0.035825</td>\n",
+       "      <td>0.053926</td>\n",
+       "      <td>0.053929</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>1.0</td>\n",
        "      <td>4</td>\n",
+       "      <td>0.210389</td>\n",
        "      <td>0.194961</td>\n",
-       "      <td>0.125823</td>\n",
-       "      <td>0.180734</td>\n",
-       "      <td>0.184039</td>\n",
-       "      <td>0.185935</td>\n",
-       "      <td>0.200142</td>\n",
-       "      <td>0.185728</td>\n",
-       "      <td>0.166834</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.198679</td>\n",
-       "      <td>0.178312</td>\n",
-       "      <td>0.183367</td>\n",
-       "      <td>0.183367</td>\n",
-       "      <td>0.194961</td>\n",
-       "      <td>0.125823</td>\n",
-       "      <td>0.183323</td>\n",
-       "      <td>0.194961</td>\n",
-       "      <td>0.125823</td>\n",
-       "      <td>0.168644</td>\n",
+       "      <td>0.814272</td>\n",
+       "      <td>0.195332</td>\n",
+       "      <td>0.143522</td>\n",
+       "      <td>0.133863</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>0.0</td>\n",
        "      <td>1</td>\n",
+       "      <td>0.008529</td>\n",
        "      <td>0.028556</td>\n",
-       "      <td>0.006973</td>\n",
-       "      <td>0.027677</td>\n",
-       "      <td>0.023568</td>\n",
-       "      <td>0.024420</td>\n",
-       "      <td>0.012645</td>\n",
-       "      <td>0.025441</td>\n",
-       "      <td>0.020917</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.019302</td>\n",
-       "      <td>0.019792</td>\n",
-       "      <td>0.014284</td>\n",
-       "      <td>0.014284</td>\n",
-       "      <td>0.028556</td>\n",
-       "      <td>0.006973</td>\n",
-       "      <td>0.030934</td>\n",
-       "      <td>0.028556</td>\n",
-       "      <td>0.006973</td>\n",
-       "      <td>0.029126</td>\n",
+       "      <td>0.974559</td>\n",
+       "      <td>0.022765</td>\n",
+       "      <td>0.009371</td>\n",
+       "      <td>0.014419</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>1.0</td>\n",
        "      <td>2</td>\n",
+       "      <td>0.189933</td>\n",
        "      <td>0.208263</td>\n",
-       "      <td>0.130426</td>\n",
-       "      <td>0.177813</td>\n",
-       "      <td>0.203001</td>\n",
-       "      <td>0.203261</td>\n",
-       "      <td>0.177770</td>\n",
-       "      <td>0.197092</td>\n",
-       "      <td>0.167265</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.196543</td>\n",
-       "      <td>0.182424</td>\n",
-       "      <td>0.177948</td>\n",
-       "      <td>0.177948</td>\n",
-       "      <td>0.208263</td>\n",
-       "      <td>0.130426</td>\n",
-       "      <td>0.178491</td>\n",
-       "      <td>0.208263</td>\n",
-       "      <td>0.130426</td>\n",
-       "      <td>0.179502</td>\n",
+       "      <td>0.802908</td>\n",
+       "      <td>0.193035</td>\n",
+       "      <td>0.151100</td>\n",
+       "      <td>0.156047</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>...</th>\n",
@@ -271,212 +193,95 @@
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19826</th>\n",
        "      <td>0.0</td>\n",
        "      <td>3</td>\n",
+       "      <td>0.163697</td>\n",
        "      <td>0.228342</td>\n",
-       "      <td>0.160032</td>\n",
-       "      <td>0.274767</td>\n",
-       "      <td>0.228772</td>\n",
-       "      <td>0.206548</td>\n",
-       "      <td>0.243380</td>\n",
-       "      <td>0.207749</td>\n",
-       "      <td>0.214998</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.232284</td>\n",
-       "      <td>0.201933</td>\n",
-       "      <td>0.254153</td>\n",
-       "      <td>0.254153</td>\n",
-       "      <td>0.228342</td>\n",
-       "      <td>0.160032</td>\n",
-       "      <td>0.255826</td>\n",
-       "      <td>0.228342</td>\n",
-       "      <td>0.160032</td>\n",
-       "      <td>0.261347</td>\n",
+       "      <td>0.792251</td>\n",
+       "      <td>0.235192</td>\n",
+       "      <td>0.216720</td>\n",
+       "      <td>0.184214</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19827</th>\n",
        "      <td>0.0</td>\n",
        "      <td>1</td>\n",
+       "      <td>0.114999</td>\n",
        "      <td>0.253998</td>\n",
-       "      <td>0.123836</td>\n",
-       "      <td>0.182039</td>\n",
-       "      <td>0.218112</td>\n",
-       "      <td>0.199714</td>\n",
-       "      <td>0.164194</td>\n",
-       "      <td>0.237054</td>\n",
-       "      <td>0.138583</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.236322</td>\n",
-       "      <td>0.212543</td>\n",
-       "      <td>0.167147</td>\n",
-       "      <td>0.167147</td>\n",
-       "      <td>0.253998</td>\n",
-       "      <td>0.123836</td>\n",
-       "      <td>0.176984</td>\n",
-       "      <td>0.253998</td>\n",
-       "      <td>0.123836</td>\n",
-       "      <td>0.185477</td>\n",
+       "      <td>0.762946</td>\n",
+       "      <td>0.235832</td>\n",
+       "      <td>0.161401</td>\n",
+       "      <td>0.141663</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19828</th>\n",
        "      <td>1.0</td>\n",
        "      <td>3</td>\n",
+       "      <td>0.141307</td>\n",
        "      <td>0.213364</td>\n",
-       "      <td>0.169604</td>\n",
-       "      <td>0.212740</td>\n",
-       "      <td>0.212545</td>\n",
-       "      <td>0.211904</td>\n",
-       "      <td>0.227022</td>\n",
-       "      <td>0.252599</td>\n",
-       "      <td>0.232105</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.202498</td>\n",
-       "      <td>0.212675</td>\n",
-       "      <td>0.230183</td>\n",
-       "      <td>0.230183</td>\n",
-       "      <td>0.213364</td>\n",
-       "      <td>0.169604</td>\n",
-       "      <td>0.236894</td>\n",
-       "      <td>0.213364</td>\n",
-       "      <td>0.169604</td>\n",
-       "      <td>0.224887</td>\n",
+       "      <td>0.747401</td>\n",
+       "      <td>0.208723</td>\n",
+       "      <td>0.242814</td>\n",
+       "      <td>0.233266</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19829</th>\n",
        "      <td>0.0</td>\n",
        "      <td>1</td>\n",
+       "      <td>0.007766</td>\n",
        "      <td>0.002176</td>\n",
-       "      <td>0.002538</td>\n",
-       "      <td>0.001323</td>\n",
-       "      <td>0.005655</td>\n",
-       "      <td>0.012688</td>\n",
-       "      <td>0.000296</td>\n",
-       "      <td>0.003545</td>\n",
-       "      <td>0.004071</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.015741</td>\n",
-       "      <td>0.007463</td>\n",
-       "      <td>0.001132</td>\n",
-       "      <td>0.001132</td>\n",
-       "      <td>0.002176</td>\n",
-       "      <td>0.002538</td>\n",
-       "      <td>0.001113</td>\n",
-       "      <td>0.002176</td>\n",
-       "      <td>0.002538</td>\n",
-       "      <td>0.000985</td>\n",
+       "      <td>0.996455</td>\n",
+       "      <td>0.018702</td>\n",
+       "      <td>0.005657</td>\n",
+       "      <td>0.009914</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19830</th>\n",
        "      <td>0.0</td>\n",
        "      <td>0</td>\n",
+       "      <td>0.163946</td>\n",
        "      <td>0.185484</td>\n",
-       "      <td>0.156659</td>\n",
-       "      <td>0.259442</td>\n",
-       "      <td>0.180441</td>\n",
-       "      <td>0.202463</td>\n",
-       "      <td>0.232861</td>\n",
-       "      <td>0.188571</td>\n",
-       "      <td>0.206704</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.212357</td>\n",
-       "      <td>0.192640</td>\n",
-       "      <td>0.246474</td>\n",
-       "      <td>0.246474</td>\n",
-       "      <td>0.185484</td>\n",
-       "      <td>0.156659</td>\n",
-       "      <td>0.233696</td>\n",
-       "      <td>0.185484</td>\n",
-       "      <td>0.156659</td>\n",
-       "      <td>0.256520</td>\n",
+       "      <td>0.811429</td>\n",
+       "      <td>0.215085</td>\n",
+       "      <td>0.167812</td>\n",
+       "      <td>0.155447</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>19831 rows × 23 columns</p>\n",
+       "<p>19831 rows × 8 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
-       "       high_priced  fold  group1_ebm  group1_glm  group1_mxgb  \\\n",
-       "0              0.0     2    0.080557    0.142090     0.059522   \n",
-       "1              0.0     1    0.026001    0.081674     0.036210   \n",
-       "2              1.0     4    0.194961    0.125823     0.180734   \n",
-       "3              0.0     1    0.028556    0.006973     0.027677   \n",
-       "4              1.0     2    0.208263    0.130426     0.177813   \n",
-       "...            ...   ...         ...         ...          ...   \n",
-       "19826          0.0     3    0.228342    0.160032     0.274767   \n",
-       "19827          0.0     1    0.253998    0.123836     0.182039   \n",
-       "19828          1.0     3    0.213364    0.169604     0.212740   \n",
-       "19829          0.0     1    0.002176    0.002538     0.001323   \n",
-       "19830          0.0     0    0.185484    0.156659     0.259442   \n",
-       "\n",
-       "       group2_piml_EBM  group2_piml_XGB2  group2_piml_reludnn  \\\n",
-       "0             0.060440          0.058846             0.054570   \n",
-       "1             0.030870          0.030136             0.032199   \n",
-       "2             0.184039          0.185935             0.200142   \n",
-       "3             0.023568          0.024420             0.012645   \n",
-       "4             0.203001          0.203261             0.177770   \n",
-       "...                ...               ...                  ...   \n",
-       "19826         0.228772          0.206548             0.243380   \n",
-       "19827         0.218112          0.199714             0.164194   \n",
-       "19828         0.212545          0.211904             0.227022   \n",
-       "19829         0.005655          0.012688             0.000296   \n",
-       "19830         0.180441          0.202463             0.232861   \n",
+       "       high_priced  fold  group1_rem_ebm  group2_rem_ebm  group3_rem_piml_EBM  \\\n",
+       "0              0.0     2        0.118787        0.080557             0.920389   \n",
+       "1              0.0     1        0.084506        0.026001             0.969301   \n",
+       "2              1.0     4        0.210389        0.194961             0.814272   \n",
+       "3              0.0     1        0.008529        0.028556             0.974559   \n",
+       "4              1.0     2        0.189933        0.208263             0.802908   \n",
+       "...            ...   ...             ...             ...                  ...   \n",
+       "19826          0.0     3        0.163697        0.228342             0.792251   \n",
+       "19827          0.0     1        0.114999        0.253998             0.762946   \n",
+       "19828          1.0     3        0.141307        0.213364             0.747401   \n",
+       "19829          0.0     1        0.007766        0.002176             0.996455   \n",
+       "19830          0.0     0        0.163946        0.185484             0.811429   \n",
        "\n",
-       "       group3_piml_EBM  group3_piml_GAM  ...  group5_xgb2  group8_piml_ebm  \\\n",
-       "0             0.079611         0.109994  ...     0.077672         0.073114   \n",
-       "1             0.030699         0.050429  ...     0.035456         0.029787   \n",
-       "2             0.185728         0.166834  ...     0.198679         0.178312   \n",
-       "3             0.025441         0.020917  ...     0.019302         0.019792   \n",
-       "4             0.197092         0.167265  ...     0.196543         0.182424   \n",
-       "...                ...              ...  ...          ...              ...   \n",
-       "19826         0.207749         0.214998  ...     0.232284         0.201933   \n",
-       "19827         0.237054         0.138583  ...     0.236322         0.212543   \n",
-       "19828         0.252599         0.232105  ...     0.202498         0.212675   \n",
-       "19829         0.003545         0.004071  ...     0.015741         0.007463   \n",
-       "19830         0.188571         0.206704  ...     0.212357         0.192640   \n",
+       "       group5_rem_xgb2  group8_rem_ebm  ph_rem_ebm  \n",
+       "0             0.078326        0.223846    0.219429  \n",
+       "1             0.035825        0.053926    0.053929  \n",
+       "2             0.195332        0.143522    0.133863  \n",
+       "3             0.022765        0.009371    0.014419  \n",
+       "4             0.193035        0.151100    0.156047  \n",
+       "...                ...             ...         ...  \n",
+       "19826         0.235192        0.216720    0.184214  \n",
+       "19827         0.235832        0.161401    0.141663  \n",
+       "19828         0.208723        0.242814    0.233266  \n",
+       "19829         0.018702        0.005657    0.009914  \n",
+       "19830         0.215085        0.167812    0.155447  \n",
        "\n",
-       "       group8_piml_gam  group8_piml_reludnn  group9_ebm  group9_glm  \\\n",
-       "0             0.066971             0.066971    0.080557    0.142090   \n",
-       "1             0.034471             0.034471    0.026001    0.081674   \n",
-       "2             0.183367             0.183367    0.194961    0.125823   \n",
-       "3             0.014284             0.014284    0.028556    0.006973   \n",
-       "4             0.177948             0.177948    0.208263    0.130426   \n",
-       "...                ...                  ...         ...         ...   \n",
-       "19826         0.254153             0.254153    0.228342    0.160032   \n",
-       "19827         0.167147             0.167147    0.253998    0.123836   \n",
-       "19828         0.230183             0.230183    0.213364    0.169604   \n",
-       "19829         0.001132             0.001132    0.002176    0.002538   \n",
-       "19830         0.246474             0.246474    0.185484    0.156659   \n",
-       "\n",
-       "       group9_mxgb    ph_ebm    ph_glm   ph_mxgb  \n",
-       "0         0.086361  0.080557  0.142090  0.071625  \n",
-       "1         0.033920  0.026001  0.081674  0.036750  \n",
-       "2         0.183323  0.194961  0.125823  0.168644  \n",
-       "3         0.030934  0.028556  0.006973  0.029126  \n",
-       "4         0.178491  0.208263  0.130426  0.179502  \n",
-       "...            ...       ...       ...       ...  \n",
-       "19826     0.255826  0.228342  0.160032  0.261347  \n",
-       "19827     0.176984  0.253998  0.123836  0.185477  \n",
-       "19828     0.236894  0.213364  0.169604  0.224887  \n",
-       "19829     0.001113  0.002176  0.002538  0.000985  \n",
-       "19830     0.233696  0.185484  0.156659  0.256520  \n",
-       "\n",
-       "[19831 rows x 23 columns]"
+       "[19831 rows x 8 columns]"
       ]
      },
      "execution_count": 3,
@@ -613,180 +418,6 @@
    "id": "40fbe608",
    "metadata": {},
    "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Empty DataFrame\n",
-      "Columns: [group1_ebm_rank, group1_glm_rank, group1_mxgb_rank, group2_piml_EBM_rank, group2_piml_XGB2_rank, group2_piml_reludnn_rank, group3_piml_EBM_rank, group3_piml_GAM_rank, group3_piml_XGB_rank, group5_gaminet_rank, group5_reludnn_rank, group5_xgb2_rank, group8_piml_ebm_rank, group8_piml_gam_rank, group8_piml_reludnn_rank, group9_ebm_rank, group9_glm_rank, group9_mxgb_rank, ph_ebm_rank, ph_glm_rank, ph_mxgb_rank]\n",
-      "Index: []\n",
-      "\n",
-      "[0 rows x 21 columns]\n",
-      "    group1_ebm_rank  group1_glm_rank  group1_mxgb_rank  group2_piml_EBM_rank  \\\n",
-      "0               9.0             17.5               2.5                   9.0   \n",
-      "1               2.0             20.0              15.5                   6.5   \n",
-      "2               2.0             20.0              12.0                   7.5   \n",
-      "3               2.5             20.0              15.0                   7.0   \n",
-      "4               5.0             20.0              13.0                   5.0   \n",
-      "5              11.0             11.0              11.0                  11.0   \n",
-      "6               2.5             20.0              16.0                   5.5   \n",
-      "7               4.0             20.0              16.5                   1.5   \n",
-      "8               2.0             20.0              16.5                   4.5   \n",
-      "9               2.5             20.0              15.5                   7.0   \n",
-      "10             14.0             14.0              14.0                   4.5   \n",
-      "11              6.0             20.0              16.5                   2.5   \n",
-      "12              6.0             20.0              17.0                   8.0   \n",
-      "13              7.5             20.0              16.0                   3.5   \n",
-      "14              5.5             20.0              16.0                   5.5   \n",
-      "15             14.5             14.5              14.5                  14.5   \n",
-      "16              3.5             20.0              16.0                   3.5   \n",
-      "17              2.0             20.0              15.0                   5.0   \n",
-      "18              6.0             20.0              16.0                   1.5   \n",
-      "19              5.0             20.0              16.0                   5.0   \n",
-      "20              7.0             18.5              12.5                   1.0   \n",
-      "21              2.0             20.0              16.0                   4.5   \n",
-      "22              5.0             20.0              16.0                   1.0   \n",
-      "23              2.0             20.0              15.0                   4.5   \n",
-      "24              5.0             20.0              12.5                   5.0   \n",
-      "\n",
-      "    group2_piml_XGB2_rank  group2_piml_reludnn_rank  group3_piml_EBM_rank  \\\n",
-      "0                    17.5                      17.5                   2.5   \n",
-      "1                     4.5                      11.0                   4.5   \n",
-      "2                     9.0                      17.0                   4.0   \n",
-      "3                     7.0                      11.0                   2.5   \n",
-      "4                     5.0                      13.0                   5.0   \n",
-      "5                    11.0                      11.0                  11.0   \n",
-      "6                     5.5                      13.0                   2.5   \n",
-      "7                     6.5                      14.0                   1.5   \n",
-      "8                     6.0                      11.5                   4.5   \n",
-      "9                     7.0                      11.5                   2.5   \n",
-      "10                    1.5                       4.5                   4.5   \n",
-      "11                    2.5                      10.0                   6.0   \n",
-      "12                    3.0                      14.5                   9.0   \n",
-      "13                    3.5                      10.0                   3.5   \n",
-      "14                    5.5                      11.0                   5.5   \n",
-      "15                    4.0                       4.0                   4.0   \n",
-      "16                    7.5                      11.0                   3.5   \n",
-      "17                    9.0                      10.0                   5.0   \n",
-      "18                    6.0                      10.5                   1.5   \n",
-      "19                    5.0                      11.5                   5.0   \n",
-      "20                    7.0                      12.5                   3.0   \n",
-      "21                    7.5                      13.0                   4.5   \n",
-      "22                    2.5                      14.0                   2.5   \n",
-      "23                    6.5                      13.0                   4.5   \n",
-      "24                    5.0                      12.5                   5.0   \n",
-      "\n",
-      "    group3_piml_GAM_rank  group3_piml_XGB_rank  group5_gaminet_rank  ...  \\\n",
-      "0                   17.5                   9.0                 17.5  ...   \n",
-      "1                   18.0                   8.5                 14.0  ...   \n",
-      "2                   18.0                   6.0                 10.0  ...   \n",
-      "3                   18.0                   7.0                 17.0  ...   \n",
-      "4                   18.0                   5.0                 17.0  ...   \n",
-      "5                   11.0                  11.0                 11.0  ...   \n",
-      "6                   18.0                   7.0                 10.0  ...   \n",
-      "7                   18.0                   9.0                 10.0  ...   \n",
-      "8                   18.0                   8.0                 14.0  ...   \n",
-      "9                   18.0                   7.0                 15.5  ...   \n",
-      "10                  14.0                   4.5                 14.0  ...   \n",
-      "11                  18.0                   6.0                 11.0  ...   \n",
-      "12                  18.0                   4.0                 10.0  ...   \n",
-      "13                  18.0                   3.5                 14.0  ...   \n",
-      "14                  18.0                   5.5                 13.5  ...   \n",
-      "15                  14.5                  14.5                 14.5  ...   \n",
-      "16                  18.0                   9.0                 14.0  ...   \n",
-      "17                  18.0                   5.0                 11.0  ...   \n",
-      "18                  18.0                   6.0                 14.0  ...   \n",
-      "19                  16.0                   5.0                 16.0  ...   \n",
-      "20                  18.5                   3.0                 18.5  ...   \n",
-      "21                  18.0                   7.5                 14.0  ...   \n",
-      "22                  18.0                   9.0                 12.0  ...   \n",
-      "23                  18.0                   8.5                 15.0  ...   \n",
-      "24                  18.0                   5.0                 16.5  ...   \n",
-      "\n",
-      "    group5_xgb2_rank  group8_piml_ebm_rank  group8_piml_gam_rank  \\\n",
-      "0                9.0                   9.0                   9.0   \n",
-      "1                6.5                   8.5                  11.0   \n",
-      "2                7.5                   5.0                  15.0   \n",
-      "3                7.0                   7.0                  11.0   \n",
-      "4                5.0                   5.0                  13.0   \n",
-      "5               11.0                  11.0                  11.0   \n",
-      "6                8.5                   8.5                  13.0   \n",
-      "7                6.5                   8.0                  14.0   \n",
-      "8                8.0                   8.0                  11.5   \n",
-      "9                7.0                   7.0                  11.5   \n",
-      "10               1.5                  14.0                  14.0   \n",
-      "11               1.0                   9.0                  12.5   \n",
-      "12               1.0                   2.0                  11.5   \n",
-      "13               1.0                   7.5                  12.0   \n",
-      "14               1.0                   5.5                  11.0   \n",
-      "15               4.0                  14.5                   4.0   \n",
-      "16               7.5                   3.5                  12.5   \n",
-      "17               8.0                   7.0                  15.0   \n",
-      "18               6.0                   6.0                  12.5   \n",
-      "19               5.0                   5.0                  11.5   \n",
-      "20               3.0                   7.0                  12.5   \n",
-      "21               6.0                   9.0                  11.5   \n",
-      "22               7.5                   7.5                  12.0   \n",
-      "23               6.5                   8.5                  11.0   \n",
-      "24               5.0                   5.0                  12.5   \n",
-      "\n",
-      "    group8_piml_reludnn_rank  group9_ebm_rank  group9_glm_rank  \\\n",
-      "0                        9.0              9.0             17.5   \n",
-      "1                       11.0              2.0             20.0   \n",
-      "2                       15.0              2.0             20.0   \n",
-      "3                       11.0              2.5             20.0   \n",
-      "4                       13.0              5.0             20.0   \n",
-      "5                       11.0             11.0             11.0   \n",
-      "6                       13.0              2.5             20.0   \n",
-      "7                       14.0              4.0             20.0   \n",
-      "8                       11.5              2.0             20.0   \n",
-      "9                       11.5              2.5             20.0   \n",
-      "10                      14.0             14.0             14.0   \n",
-      "11                      12.5              6.0             20.0   \n",
-      "12                      11.5              6.0             20.0   \n",
-      "13                      12.0              7.5             20.0   \n",
-      "14                      11.0              5.5             20.0   \n",
-      "15                       4.0             14.5             14.5   \n",
-      "16                      12.5              3.5             20.0   \n",
-      "17                      15.0              2.0             20.0   \n",
-      "18                      12.5              6.0             20.0   \n",
-      "19                      11.5              5.0             20.0   \n",
-      "20                      12.5              7.0             18.5   \n",
-      "21                      11.5              2.0             20.0   \n",
-      "22                      12.0              5.0             20.0   \n",
-      "23                      11.0              2.0             20.0   \n",
-      "24                      12.5              5.0             20.0   \n",
-      "\n",
-      "    group9_mxgb_rank  ph_ebm_rank  ph_glm_rank  ph_mxgb_rank  \n",
-      "0                2.5          9.0         17.5           2.5  \n",
-      "1               17.0          2.0         20.0          15.5  \n",
-      "2               15.0          2.0         20.0          11.0  \n",
-      "3               15.0          2.5         20.0          15.0  \n",
-      "4               13.0          5.0         20.0          13.0  \n",
-      "5               11.0         11.0         11.0          11.0  \n",
-      "6               16.0          2.5         20.0          16.0  \n",
-      "7               12.0          4.0         20.0          16.5  \n",
-      "8               16.5          2.0         20.0          15.0  \n",
-      "9               15.5          2.5         20.0          15.5  \n",
-      "10              14.0         14.0         14.0          14.0  \n",
-      "11              15.0          6.0         20.0          16.5  \n",
-      "12              13.0          6.0         20.0          14.5  \n",
-      "13              16.0          7.5         20.0          16.0  \n",
-      "14              16.0          5.5         20.0          16.0  \n",
-      "15              14.5         14.5         14.5          14.5  \n",
-      "16              16.0          3.5         20.0          16.0  \n",
-      "17              12.0          2.0         20.0          17.0  \n",
-      "18              16.0          6.0         20.0          16.0  \n",
-      "19              16.0          5.0         20.0          16.0  \n",
-      "20              18.5          7.0         18.5          12.5  \n",
-      "21              17.0          2.0         20.0          15.0  \n",
-      "22              17.0          5.0         20.0          15.0  \n",
-      "23              17.0          2.0         20.0          15.0  \n",
-      "24              16.5          5.0         20.0          12.5  \n",
-      "\n",
-      "[25 rows x 21 columns]\n"
-     ]
-    },
     {
      "data": {
       "text/html": [
@@ -810,25 +441,18 @@
        "      <th></th>\n",
        "      <th>fold</th>\n",
        "      <th>metric</th>\n",
-       "      <th>group1_ebm</th>\n",
-       "      <th>group1_glm</th>\n",
-       "      <th>group1_mxgb</th>\n",
-       "      <th>group2_piml_EBM</th>\n",
-       "      <th>group2_piml_XGB2</th>\n",
-       "      <th>group2_piml_reludnn</th>\n",
-       "      <th>group3_piml_EBM</th>\n",
-       "      <th>group3_piml_GAM</th>\n",
-       "      <th>...</th>\n",
-       "      <th>group5_xgb2_rank</th>\n",
-       "      <th>group8_piml_ebm_rank</th>\n",
-       "      <th>group8_piml_gam_rank</th>\n",
-       "      <th>group8_piml_reludnn_rank</th>\n",
-       "      <th>group9_ebm_rank</th>\n",
-       "      <th>group9_glm_rank</th>\n",
-       "      <th>group9_mxgb_rank</th>\n",
-       "      <th>ph_ebm_rank</th>\n",
-       "      <th>ph_glm_rank</th>\n",
-       "      <th>ph_mxgb_rank</th>\n",
+       "      <th>group1_rem_ebm</th>\n",
+       "      <th>group2_rem_ebm</th>\n",
+       "      <th>group3_rem_piml_EBM</th>\n",
+       "      <th>group5_rem_xgb2</th>\n",
+       "      <th>group8_rem_ebm</th>\n",
+       "      <th>ph_rem_ebm</th>\n",
+       "      <th>group1_rem_ebm_rank</th>\n",
+       "      <th>group2_rem_ebm_rank</th>\n",
+       "      <th>group3_rem_piml_EBM_rank</th>\n",
+       "      <th>group5_rem_xgb2_rank</th>\n",
+       "      <th>group8_rem_ebm_rank</th>\n",
+       "      <th>ph_rem_ebm_rank</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -836,121 +460,86 @@
        "      <th>0</th>\n",
        "      <td>0.0</td>\n",
        "      <td>acc</td>\n",
-       "      <td>0.901</td>\n",
        "      <td>0.900</td>\n",
-       "      <td>0.902</td>\n",
        "      <td>0.901</td>\n",
        "      <td>0.900</td>\n",
-       "      <td>0.900</td>\n",
-       "      <td>0.902</td>\n",
-       "      <td>0.900</td>\n",
-       "      <td>...</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>17.5</td>\n",
+       "      <td>0.901</td>\n",
+       "      <td>0.901</td>\n",
+       "      <td>0.901</td>\n",
+       "      <td>5.5</td>\n",
+       "      <td>2.5</td>\n",
+       "      <td>5.5</td>\n",
+       "      <td>2.5</td>\n",
        "      <td>2.5</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>17.5</td>\n",
        "      <td>2.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>0.0</td>\n",
        "      <td>auc</td>\n",
+       "      <td>0.781</td>\n",
        "      <td>0.840</td>\n",
-       "      <td>0.775</td>\n",
-       "      <td>0.814</td>\n",
+       "      <td>0.163</td>\n",
        "      <td>0.836</td>\n",
-       "      <td>0.837</td>\n",
-       "      <td>0.822</td>\n",
-       "      <td>0.837</td>\n",
-       "      <td>0.805</td>\n",
-       "      <td>...</td>\n",
-       "      <td>6.5</td>\n",
-       "      <td>8.5</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>17.0</td>\n",
+       "      <td>0.793</td>\n",
+       "      <td>0.791</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
        "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.5</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>0.0</td>\n",
        "      <td>f1</td>\n",
+       "      <td>0.347</td>\n",
        "      <td>0.405</td>\n",
-       "      <td>0.335</td>\n",
-       "      <td>0.379</td>\n",
-       "      <td>0.395</td>\n",
-       "      <td>0.394</td>\n",
-       "      <td>0.373</td>\n",
-       "      <td>0.399</td>\n",
-       "      <td>0.358</td>\n",
-       "      <td>...</td>\n",
-       "      <td>7.5</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>15.0</td>\n",
-       "      <td>15.0</td>\n",
-       "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.0</td>\n",
+       "      <td>0.182</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.342</td>\n",
+       "      <td>0.347</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
        "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>11.0</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>3.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>0.0</td>\n",
        "      <td>logloss</td>\n",
+       "      <td>0.280</td>\n",
        "      <td>0.251</td>\n",
-       "      <td>0.291</td>\n",
-       "      <td>0.263</td>\n",
-       "      <td>0.253</td>\n",
-       "      <td>0.253</td>\n",
-       "      <td>0.259</td>\n",
-       "      <td>0.251</td>\n",
-       "      <td>0.271</td>\n",
-       "      <td>...</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>2.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.0</td>\n",
-       "      <td>2.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.0</td>\n",
+       "      <td>3.257</td>\n",
+       "      <td>0.254</td>\n",
+       "      <td>0.274</td>\n",
+       "      <td>0.275</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>0.0</td>\n",
        "      <td>mse</td>\n",
+       "      <td>0.082</td>\n",
        "      <td>0.077</td>\n",
-       "      <td>0.084</td>\n",
-       "      <td>0.078</td>\n",
-       "      <td>0.077</td>\n",
-       "      <td>0.077</td>\n",
-       "      <td>0.078</td>\n",
+       "      <td>0.773</td>\n",
        "      <td>0.077</td>\n",
        "      <td>0.081</td>\n",
-       "      <td>...</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>13.0</td>\n",
-       "      <td>13.0</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>13.0</td>\n",
+       "      <td>0.081</td>\n",
        "      <td>5.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>13.0</td>\n",
+       "      <td>1.5</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>1.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
@@ -962,115 +551,80 @@
        "      <td>0.906</td>\n",
        "      <td>0.906</td>\n",
        "      <td>0.906</td>\n",
-       "      <td>0.906</td>\n",
-       "      <td>0.906</td>\n",
-       "      <td>...</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
        "      <td>1.0</td>\n",
        "      <td>auc</td>\n",
+       "      <td>0.767</td>\n",
        "      <td>0.828</td>\n",
-       "      <td>0.757</td>\n",
-       "      <td>0.792</td>\n",
-       "      <td>0.826</td>\n",
-       "      <td>0.826</td>\n",
-       "      <td>0.801</td>\n",
-       "      <td>0.828</td>\n",
-       "      <td>0.780</td>\n",
-       "      <td>...</td>\n",
-       "      <td>8.5</td>\n",
-       "      <td>8.5</td>\n",
-       "      <td>13.0</td>\n",
-       "      <td>13.0</td>\n",
-       "      <td>2.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
-       "      <td>2.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
+       "      <td>0.172</td>\n",
+       "      <td>0.822</td>\n",
+       "      <td>0.774</td>\n",
+       "      <td>0.772</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7</th>\n",
        "      <td>1.0</td>\n",
        "      <td>f1</td>\n",
+       "      <td>0.312</td>\n",
        "      <td>0.368</td>\n",
-       "      <td>0.302</td>\n",
-       "      <td>0.337</td>\n",
-       "      <td>0.369</td>\n",
-       "      <td>0.365</td>\n",
-       "      <td>0.341</td>\n",
-       "      <td>0.369</td>\n",
-       "      <td>0.320</td>\n",
-       "      <td>...</td>\n",
-       "      <td>6.5</td>\n",
-       "      <td>8.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>4.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>12.0</td>\n",
+       "      <td>0.172</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.321</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>2.0</td>\n",
        "      <td>4.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.5</td>\n",
+       "      <td>3.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
        "      <td>1.0</td>\n",
        "      <td>logloss</td>\n",
+       "      <td>0.272</td>\n",
        "      <td>0.246</td>\n",
-       "      <td>0.281</td>\n",
-       "      <td>0.264</td>\n",
-       "      <td>0.247</td>\n",
-       "      <td>0.248</td>\n",
-       "      <td>0.257</td>\n",
-       "      <td>0.247</td>\n",
-       "      <td>0.269</td>\n",
-       "      <td>...</td>\n",
-       "      <td>8.0</td>\n",
-       "      <td>8.0</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.5</td>\n",
+       "      <td>3.253</td>\n",
+       "      <td>0.250</td>\n",
+       "      <td>0.270</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
        "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9</th>\n",
        "      <td>1.0</td>\n",
        "      <td>mse</td>\n",
+       "      <td>0.079</td>\n",
        "      <td>0.074</td>\n",
-       "      <td>0.080</td>\n",
-       "      <td>0.078</td>\n",
-       "      <td>0.075</td>\n",
+       "      <td>0.778</td>\n",
        "      <td>0.075</td>\n",
-       "      <td>0.077</td>\n",
-       "      <td>0.074</td>\n",
        "      <td>0.079</td>\n",
-       "      <td>...</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>2.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.5</td>\n",
-       "      <td>2.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.5</td>\n",
+       "      <td>0.079</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
@@ -1079,118 +633,83 @@
        "      <td>0.908</td>\n",
        "      <td>0.908</td>\n",
        "      <td>0.908</td>\n",
-       "      <td>0.909</td>\n",
        "      <td>0.910</td>\n",
-       "      <td>0.909</td>\n",
-       "      <td>0.909</td>\n",
        "      <td>0.908</td>\n",
-       "      <td>...</td>\n",
-       "      <td>1.5</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
-       "      <td>14.0</td>\n",
+       "      <td>0.909</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>2.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
        "      <td>2.0</td>\n",
        "      <td>auc</td>\n",
+       "      <td>0.759</td>\n",
        "      <td>0.825</td>\n",
-       "      <td>0.763</td>\n",
-       "      <td>0.797</td>\n",
-       "      <td>0.826</td>\n",
+       "      <td>0.175</td>\n",
        "      <td>0.826</td>\n",
-       "      <td>0.813</td>\n",
-       "      <td>0.825</td>\n",
-       "      <td>0.792</td>\n",
-       "      <td>...</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>6.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.0</td>\n",
+       "      <td>0.781</td>\n",
+       "      <td>0.780</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>2.0</td>\n",
        "      <td>6.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.5</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>12</th>\n",
        "      <td>2.0</td>\n",
        "      <td>f1</td>\n",
+       "      <td>0.304</td>\n",
        "      <td>0.372</td>\n",
-       "      <td>0.312</td>\n",
-       "      <td>0.344</td>\n",
-       "      <td>0.370</td>\n",
-       "      <td>0.374</td>\n",
-       "      <td>0.346</td>\n",
-       "      <td>0.366</td>\n",
-       "      <td>0.335</td>\n",
-       "      <td>...</td>\n",
+       "      <td>0.169</td>\n",
+       "      <td>0.371</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.323</td>\n",
+       "      <td>5.0</td>\n",
        "      <td>1.0</td>\n",
-       "      <td>2.0</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>6.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>13.0</td>\n",
        "      <td>6.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>14.5</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>3.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13</th>\n",
        "      <td>2.0</td>\n",
        "      <td>logloss</td>\n",
+       "      <td>0.271</td>\n",
        "      <td>0.246</td>\n",
-       "      <td>0.279</td>\n",
-       "      <td>0.259</td>\n",
-       "      <td>0.245</td>\n",
-       "      <td>0.245</td>\n",
-       "      <td>0.250</td>\n",
+       "      <td>3.284</td>\n",
        "      <td>0.245</td>\n",
        "      <td>0.264</td>\n",
-       "      <td>...</td>\n",
+       "      <td>0.264</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>6.0</td>\n",
        "      <td>1.0</td>\n",
-       "      <td>7.5</td>\n",
-       "      <td>12.0</td>\n",
-       "      <td>12.0</td>\n",
-       "      <td>7.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
-       "      <td>7.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14</th>\n",
        "      <td>2.0</td>\n",
        "      <td>mse</td>\n",
+       "      <td>0.078</td>\n",
        "      <td>0.073</td>\n",
-       "      <td>0.079</td>\n",
-       "      <td>0.076</td>\n",
-       "      <td>0.073</td>\n",
+       "      <td>0.781</td>\n",
        "      <td>0.073</td>\n",
-       "      <td>0.074</td>\n",
-       "      <td>0.073</td>\n",
-       "      <td>0.077</td>\n",
-       "      <td>...</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>5.5</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>5.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
-       "      <td>5.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
+       "      <td>0.076</td>\n",
+       "      <td>0.076</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1.5</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>1.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>15</th>\n",
@@ -1200,380 +719,280 @@
        "      <td>0.903</td>\n",
        "      <td>0.903</td>\n",
        "      <td>0.903</td>\n",
-       "      <td>0.904</td>\n",
-       "      <td>0.904</td>\n",
-       "      <td>0.904</td>\n",
        "      <td>0.903</td>\n",
-       "      <td>...</td>\n",
-       "      <td>4.0</td>\n",
-       "      <td>14.5</td>\n",
-       "      <td>4.0</td>\n",
-       "      <td>4.0</td>\n",
-       "      <td>14.5</td>\n",
-       "      <td>14.5</td>\n",
-       "      <td>14.5</td>\n",
-       "      <td>14.5</td>\n",
-       "      <td>14.5</td>\n",
-       "      <td>14.5</td>\n",
+       "      <td>0.903</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16</th>\n",
        "      <td>3.0</td>\n",
        "      <td>auc</td>\n",
+       "      <td>0.772</td>\n",
        "      <td>0.826</td>\n",
-       "      <td>0.755</td>\n",
-       "      <td>0.795</td>\n",
-       "      <td>0.826</td>\n",
-       "      <td>0.825</td>\n",
-       "      <td>0.807</td>\n",
-       "      <td>0.826</td>\n",
-       "      <td>0.790</td>\n",
-       "      <td>...</td>\n",
-       "      <td>7.5</td>\n",
-       "      <td>3.5</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>3.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
-       "      <td>3.5</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
+       "      <td>0.174</td>\n",
+       "      <td>0.823</td>\n",
+       "      <td>0.775</td>\n",
+       "      <td>0.772</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>17</th>\n",
        "      <td>3.0</td>\n",
        "      <td>f1</td>\n",
+       "      <td>0.317</td>\n",
        "      <td>0.371</td>\n",
-       "      <td>0.307</td>\n",
-       "      <td>0.345</td>\n",
-       "      <td>0.368</td>\n",
-       "      <td>0.360</td>\n",
-       "      <td>0.350</td>\n",
-       "      <td>0.368</td>\n",
-       "      <td>0.334</td>\n",
-       "      <td>...</td>\n",
-       "      <td>8.0</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>15.0</td>\n",
-       "      <td>15.0</td>\n",
-       "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>12.0</td>\n",
+       "      <td>0.177</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.328</td>\n",
+       "      <td>0.323</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
        "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>17.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>18</th>\n",
        "      <td>3.0</td>\n",
        "      <td>logloss</td>\n",
+       "      <td>0.276</td>\n",
        "      <td>0.252</td>\n",
-       "      <td>0.288</td>\n",
-       "      <td>0.268</td>\n",
-       "      <td>0.251</td>\n",
-       "      <td>0.252</td>\n",
-       "      <td>0.259</td>\n",
-       "      <td>0.251</td>\n",
-       "      <td>0.271</td>\n",
-       "      <td>...</td>\n",
-       "      <td>6.0</td>\n",
-       "      <td>6.0</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>6.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
+       "      <td>3.254</td>\n",
+       "      <td>0.253</td>\n",
+       "      <td>0.275</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>1.0</td>\n",
        "      <td>6.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19</th>\n",
        "      <td>3.0</td>\n",
        "      <td>mse</td>\n",
+       "      <td>0.081</td>\n",
        "      <td>0.077</td>\n",
-       "      <td>0.082</td>\n",
-       "      <td>0.080</td>\n",
-       "      <td>0.077</td>\n",
-       "      <td>0.077</td>\n",
-       "      <td>0.078</td>\n",
+       "      <td>0.775</td>\n",
        "      <td>0.077</td>\n",
        "      <td>0.080</td>\n",
-       "      <td>...</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
+       "      <td>0.080</td>\n",
        "      <td>5.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.0</td>\n",
+       "      <td>1.5</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>1.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>20</th>\n",
        "      <td>4.0</td>\n",
        "      <td>acc</td>\n",
-       "      <td>0.897</td>\n",
        "      <td>0.895</td>\n",
-       "      <td>0.896</td>\n",
-       "      <td>0.899</td>\n",
        "      <td>0.897</td>\n",
-       "      <td>0.896</td>\n",
+       "      <td>0.895</td>\n",
        "      <td>0.898</td>\n",
        "      <td>0.895</td>\n",
-       "      <td>...</td>\n",
-       "      <td>3.0</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>18.5</td>\n",
-       "      <td>18.5</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>18.5</td>\n",
-       "      <td>12.5</td>\n",
+       "      <td>0.895</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>4.5</td>\n",
+       "      <td>4.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>21</th>\n",
        "      <td>4.0</td>\n",
        "      <td>auc</td>\n",
+       "      <td>0.754</td>\n",
        "      <td>0.831</td>\n",
-       "      <td>0.776</td>\n",
-       "      <td>0.807</td>\n",
-       "      <td>0.830</td>\n",
+       "      <td>0.170</td>\n",
        "      <td>0.828</td>\n",
-       "      <td>0.812</td>\n",
-       "      <td>0.830</td>\n",
-       "      <td>0.798</td>\n",
-       "      <td>...</td>\n",
+       "      <td>0.785</td>\n",
+       "      <td>0.782</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1.0</td>\n",
        "      <td>6.0</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>11.5</td>\n",
-       "      <td>11.5</td>\n",
        "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>17.0</td>\n",
-       "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>22</th>\n",
        "      <td>4.0</td>\n",
        "      <td>f1</td>\n",
+       "      <td>0.323</td>\n",
        "      <td>0.401</td>\n",
-       "      <td>0.358</td>\n",
-       "      <td>0.376</td>\n",
-       "      <td>0.407</td>\n",
-       "      <td>0.402</td>\n",
-       "      <td>0.380</td>\n",
-       "      <td>0.402</td>\n",
-       "      <td>0.369</td>\n",
-       "      <td>...</td>\n",
-       "      <td>7.5</td>\n",
-       "      <td>7.5</td>\n",
-       "      <td>12.0</td>\n",
-       "      <td>12.0</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>17.0</td>\n",
+       "      <td>0.190</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.362</td>\n",
        "      <td>5.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>23</th>\n",
        "      <td>4.0</td>\n",
        "      <td>logloss</td>\n",
+       "      <td>0.296</td>\n",
        "      <td>0.263</td>\n",
-       "      <td>0.300</td>\n",
-       "      <td>0.275</td>\n",
-       "      <td>0.264</td>\n",
-       "      <td>0.265</td>\n",
-       "      <td>0.271</td>\n",
-       "      <td>0.264</td>\n",
-       "      <td>0.284</td>\n",
-       "      <td>...</td>\n",
-       "      <td>6.5</td>\n",
-       "      <td>8.5</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>17.0</td>\n",
+       "      <td>3.200</td>\n",
+       "      <td>0.266</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>0.287</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>6.0</td>\n",
        "      <td>2.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>15.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>4.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>24</th>\n",
        "      <td>4.0</td>\n",
        "      <td>mse</td>\n",
-       "      <td>0.080</td>\n",
        "      <td>0.087</td>\n",
-       "      <td>0.082</td>\n",
        "      <td>0.080</td>\n",
+       "      <td>0.771</td>\n",
        "      <td>0.080</td>\n",
-       "      <td>0.082</td>\n",
-       "      <td>0.080</td>\n",
-       "      <td>0.085</td>\n",
-       "      <td>...</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>12.5</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>16.5</td>\n",
+       "      <td>0.084</td>\n",
+       "      <td>0.084</td>\n",
        "      <td>5.0</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>12.5</td>\n",
+       "      <td>1.5</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>1.5</td>\n",
+       "      <td>3.5</td>\n",
+       "      <td>3.5</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>25 rows × 44 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
-       "    fold   metric  group1_ebm  group1_glm  group1_mxgb  group2_piml_EBM  \\\n",
-       "0    0.0      acc       0.901       0.900        0.902            0.901   \n",
-       "1    0.0      auc       0.840       0.775        0.814            0.836   \n",
-       "2    0.0       f1       0.405       0.335        0.379            0.395   \n",
-       "3    0.0  logloss       0.251       0.291        0.263            0.253   \n",
-       "4    0.0      mse       0.077       0.084        0.078            0.077   \n",
-       "5    1.0      acc       0.906       0.906        0.906            0.906   \n",
-       "6    1.0      auc       0.828       0.757        0.792            0.826   \n",
-       "7    1.0       f1       0.368       0.302        0.337            0.369   \n",
-       "8    1.0  logloss       0.246       0.281        0.264            0.247   \n",
-       "9    1.0      mse       0.074       0.080        0.078            0.075   \n",
-       "10   2.0      acc       0.908       0.908        0.908            0.909   \n",
-       "11   2.0      auc       0.825       0.763        0.797            0.826   \n",
-       "12   2.0       f1       0.372       0.312        0.344            0.370   \n",
-       "13   2.0  logloss       0.246       0.279        0.259            0.245   \n",
-       "14   2.0      mse       0.073       0.079        0.076            0.073   \n",
-       "15   3.0      acc       0.903       0.903        0.903            0.903   \n",
-       "16   3.0      auc       0.826       0.755        0.795            0.826   \n",
-       "17   3.0       f1       0.371       0.307        0.345            0.368   \n",
-       "18   3.0  logloss       0.252       0.288        0.268            0.251   \n",
-       "19   3.0      mse       0.077       0.082        0.080            0.077   \n",
-       "20   4.0      acc       0.897       0.895        0.896            0.899   \n",
-       "21   4.0      auc       0.831       0.776        0.807            0.830   \n",
-       "22   4.0       f1       0.401       0.358        0.376            0.407   \n",
-       "23   4.0  logloss       0.263       0.300        0.275            0.264   \n",
-       "24   4.0      mse       0.080       0.087        0.082            0.080   \n",
-       "\n",
-       "    group2_piml_XGB2  group2_piml_reludnn  group3_piml_EBM  group3_piml_GAM  \\\n",
-       "0              0.900                0.900            0.902            0.900   \n",
-       "1              0.837                0.822            0.837            0.805   \n",
-       "2              0.394                0.373            0.399            0.358   \n",
-       "3              0.253                0.259            0.251            0.271   \n",
-       "4              0.077                0.078            0.077            0.081   \n",
-       "5              0.906                0.906            0.906            0.906   \n",
-       "6              0.826                0.801            0.828            0.780   \n",
-       "7              0.365                0.341            0.369            0.320   \n",
-       "8              0.248                0.257            0.247            0.269   \n",
-       "9              0.075                0.077            0.074            0.079   \n",
-       "10             0.910                0.909            0.909            0.908   \n",
-       "11             0.826                0.813            0.825            0.792   \n",
-       "12             0.374                0.346            0.366            0.335   \n",
-       "13             0.245                0.250            0.245            0.264   \n",
-       "14             0.073                0.074            0.073            0.077   \n",
-       "15             0.904                0.904            0.904            0.903   \n",
-       "16             0.825                0.807            0.826            0.790   \n",
-       "17             0.360                0.350            0.368            0.334   \n",
-       "18             0.252                0.259            0.251            0.271   \n",
-       "19             0.077                0.078            0.077            0.080   \n",
-       "20             0.897                0.896            0.898            0.895   \n",
-       "21             0.828                0.812            0.830            0.798   \n",
-       "22             0.402                0.380            0.402            0.369   \n",
-       "23             0.265                0.271            0.264            0.284   \n",
-       "24             0.080                0.082            0.080            0.085   \n",
-       "\n",
-       "    ...  group5_xgb2_rank  group8_piml_ebm_rank  group8_piml_gam_rank  \\\n",
-       "0   ...               9.0                   9.0                   9.0   \n",
-       "1   ...               6.5                   8.5                  11.0   \n",
-       "2   ...               7.5                   5.0                  15.0   \n",
-       "3   ...               7.0                   7.0                  11.0   \n",
-       "4   ...               5.0                   5.0                  13.0   \n",
-       "5   ...              11.0                  11.0                  11.0   \n",
-       "6   ...               8.5                   8.5                  13.0   \n",
-       "7   ...               6.5                   8.0                  14.0   \n",
-       "8   ...               8.0                   8.0                  11.5   \n",
-       "9   ...               7.0                   7.0                  11.5   \n",
-       "10  ...               1.5                  14.0                  14.0   \n",
-       "11  ...               1.0                   9.0                  12.5   \n",
-       "12  ...               1.0                   2.0                  11.5   \n",
-       "13  ...               1.0                   7.5                  12.0   \n",
-       "14  ...               1.0                   5.5                  11.0   \n",
-       "15  ...               4.0                  14.5                   4.0   \n",
-       "16  ...               7.5                   3.5                  12.5   \n",
-       "17  ...               8.0                   7.0                  15.0   \n",
-       "18  ...               6.0                   6.0                  12.5   \n",
-       "19  ...               5.0                   5.0                  11.5   \n",
-       "20  ...               3.0                   7.0                  12.5   \n",
-       "21  ...               6.0                   9.0                  11.5   \n",
-       "22  ...               7.5                   7.5                  12.0   \n",
-       "23  ...               6.5                   8.5                  11.0   \n",
-       "24  ...               5.0                   5.0                  12.5   \n",
+       "    fold   metric  group1_rem_ebm  group2_rem_ebm  group3_rem_piml_EBM  \\\n",
+       "0    0.0      acc           0.900           0.901                0.900   \n",
+       "1    0.0      auc           0.781           0.840                0.163   \n",
+       "2    0.0       f1           0.347           0.405                0.182   \n",
+       "3    0.0  logloss           0.280           0.251                3.257   \n",
+       "4    0.0      mse           0.082           0.077                0.773   \n",
+       "5    1.0      acc           0.906           0.906                0.906   \n",
+       "6    1.0      auc           0.767           0.828                0.172   \n",
+       "7    1.0       f1           0.312           0.368                0.172   \n",
+       "8    1.0  logloss           0.272           0.246                3.253   \n",
+       "9    1.0      mse           0.079           0.074                0.778   \n",
+       "10   2.0      acc           0.908           0.908                0.908   \n",
+       "11   2.0      auc           0.759           0.825                0.175   \n",
+       "12   2.0       f1           0.304           0.372                0.169   \n",
+       "13   2.0  logloss           0.271           0.246                3.284   \n",
+       "14   2.0      mse           0.078           0.073                0.781   \n",
+       "15   3.0      acc           0.903           0.903                0.903   \n",
+       "16   3.0      auc           0.772           0.826                0.174   \n",
+       "17   3.0       f1           0.317           0.371                0.177   \n",
+       "18   3.0  logloss           0.276           0.252                3.254   \n",
+       "19   3.0      mse           0.081           0.077                0.775   \n",
+       "20   4.0      acc           0.895           0.897                0.895   \n",
+       "21   4.0      auc           0.754           0.831                0.170   \n",
+       "22   4.0       f1           0.323           0.401                0.190   \n",
+       "23   4.0  logloss           0.296           0.263                3.200   \n",
+       "24   4.0      mse           0.087           0.080                0.771   \n",
        "\n",
-       "    group8_piml_reludnn_rank  group9_ebm_rank  group9_glm_rank  \\\n",
-       "0                        9.0              9.0             17.5   \n",
-       "1                       11.0              2.0             20.0   \n",
-       "2                       15.0              2.0             20.0   \n",
-       "3                       11.0              2.5             20.0   \n",
-       "4                       13.0              5.0             20.0   \n",
-       "5                       11.0             11.0             11.0   \n",
-       "6                       13.0              2.5             20.0   \n",
-       "7                       14.0              4.0             20.0   \n",
-       "8                       11.5              2.0             20.0   \n",
-       "9                       11.5              2.5             20.0   \n",
-       "10                      14.0             14.0             14.0   \n",
-       "11                      12.5              6.0             20.0   \n",
-       "12                      11.5              6.0             20.0   \n",
-       "13                      12.0              7.5             20.0   \n",
-       "14                      11.0              5.5             20.0   \n",
-       "15                       4.0             14.5             14.5   \n",
-       "16                      12.5              3.5             20.0   \n",
-       "17                      15.0              2.0             20.0   \n",
-       "18                      12.5              6.0             20.0   \n",
-       "19                      11.5              5.0             20.0   \n",
-       "20                      12.5              7.0             18.5   \n",
-       "21                      11.5              2.0             20.0   \n",
-       "22                      12.0              5.0             20.0   \n",
-       "23                      11.0              2.0             20.0   \n",
-       "24                      12.5              5.0             20.0   \n",
+       "    group5_rem_xgb2  group8_rem_ebm  ph_rem_ebm  group1_rem_ebm_rank  \\\n",
+       "0             0.901           0.901       0.901                  5.5   \n",
+       "1             0.836           0.793       0.791                  5.0   \n",
+       "2             0.392           0.342       0.347                  3.5   \n",
+       "3             0.254           0.274       0.275                  5.0   \n",
+       "4             0.077           0.081       0.081                  5.0   \n",
+       "5             0.906           0.906       0.906                  3.5   \n",
+       "6             0.822           0.774       0.772                  5.0   \n",
+       "7             0.360           0.319       0.321                  5.0   \n",
+       "8             0.250           0.270       0.272                  4.5   \n",
+       "9             0.075           0.079       0.079                  4.0   \n",
+       "10            0.910           0.908       0.909                  4.5   \n",
+       "11            0.826           0.781       0.780                  5.0   \n",
+       "12            0.371           0.315       0.323                  5.0   \n",
+       "13            0.245           0.264       0.264                  5.0   \n",
+       "14            0.073           0.076       0.076                  5.0   \n",
+       "15            0.903           0.903       0.903                  3.5   \n",
+       "16            0.823           0.775       0.772                  4.5   \n",
+       "17            0.365           0.328       0.323                  5.0   \n",
+       "18            0.253           0.275       0.276                  4.5   \n",
+       "19            0.077           0.080       0.080                  5.0   \n",
+       "20            0.898           0.895       0.895                  4.5   \n",
+       "21            0.828           0.785       0.782                  5.0   \n",
+       "22            0.397           0.364       0.362                  5.0   \n",
+       "23            0.266           0.286       0.287                  5.0   \n",
+       "24            0.080           0.084       0.084                  5.0   \n",
        "\n",
-       "    group9_mxgb_rank  ph_ebm_rank  ph_glm_rank  ph_mxgb_rank  \n",
-       "0                2.5          9.0         17.5           2.5  \n",
-       "1               17.0          2.0         20.0          15.5  \n",
-       "2               15.0          2.0         20.0          11.0  \n",
-       "3               15.0          2.5         20.0          15.0  \n",
-       "4               13.0          5.0         20.0          13.0  \n",
-       "5               11.0         11.0         11.0          11.0  \n",
-       "6               16.0          2.5         20.0          16.0  \n",
-       "7               12.0          4.0         20.0          16.5  \n",
-       "8               16.5          2.0         20.0          15.0  \n",
-       "9               15.5          2.5         20.0          15.5  \n",
-       "10              14.0         14.0         14.0          14.0  \n",
-       "11              15.0          6.0         20.0          16.5  \n",
-       "12              13.0          6.0         20.0          14.5  \n",
-       "13              16.0          7.5         20.0          16.0  \n",
-       "14              16.0          5.5         20.0          16.0  \n",
-       "15              14.5         14.5         14.5          14.5  \n",
-       "16              16.0          3.5         20.0          16.0  \n",
-       "17              12.0          2.0         20.0          17.0  \n",
-       "18              16.0          6.0         20.0          16.0  \n",
-       "19              16.0          5.0         20.0          16.0  \n",
-       "20              18.5          7.0         18.5          12.5  \n",
-       "21              17.0          2.0         20.0          15.0  \n",
-       "22              17.0          5.0         20.0          15.0  \n",
-       "23              17.0          2.0         20.0          15.0  \n",
-       "24              16.5          5.0         20.0          12.5  \n",
+       "    group2_rem_ebm_rank  group3_rem_piml_EBM_rank  group5_rem_xgb2_rank  \\\n",
+       "0                   2.5                       5.5                   2.5   \n",
+       "1                   1.0                       6.0                   2.0   \n",
+       "2                   1.0                       6.0                   2.0   \n",
+       "3                   1.0                       6.0                   2.0   \n",
+       "4                   1.5                       6.0                   1.5   \n",
+       "5                   3.5                       3.5                   3.5   \n",
+       "6                   1.0                       6.0                   2.0   \n",
+       "7                   1.0                       6.0                   2.0   \n",
+       "8                   1.0                       6.0                   2.0   \n",
+       "9                   1.0                       6.0                   2.0   \n",
+       "10                  4.5                       4.5                   1.0   \n",
+       "11                  2.0                       6.0                   1.0   \n",
+       "12                  1.0                       6.0                   2.0   \n",
+       "13                  2.0                       6.0                   1.0   \n",
+       "14                  1.5                       6.0                   1.5   \n",
+       "15                  3.5                       3.5                   3.5   \n",
+       "16                  1.0                       6.0                   2.0   \n",
+       "17                  1.0                       6.0                   2.0   \n",
+       "18                  1.0                       6.0                   2.0   \n",
+       "19                  1.5                       6.0                   1.5   \n",
+       "20                  2.0                       4.5                   1.0   \n",
+       "21                  1.0                       6.0                   2.0   \n",
+       "22                  1.0                       6.0                   2.0   \n",
+       "23                  1.0                       6.0                   2.0   \n",
+       "24                  1.5                       6.0                   1.5   \n",
        "\n",
-       "[25 rows x 44 columns]"
+       "    group8_rem_ebm_rank  ph_rem_ebm_rank  \n",
+       "0                   2.5              2.5  \n",
+       "1                   3.0              4.0  \n",
+       "2                   5.0              3.5  \n",
+       "3                   3.0              4.0  \n",
+       "4                   3.5              3.5  \n",
+       "5                   3.5              3.5  \n",
+       "6                   3.0              4.0  \n",
+       "7                   4.0              3.0  \n",
+       "8                   3.0              4.5  \n",
+       "9                   4.0              4.0  \n",
+       "10                  4.5              2.0  \n",
+       "11                  3.0              4.0  \n",
+       "12                  4.0              3.0  \n",
+       "13                  3.5              3.5  \n",
+       "14                  3.5              3.5  \n",
+       "15                  3.5              3.5  \n",
+       "16                  3.0              4.5  \n",
+       "17                  3.0              4.0  \n",
+       "18                  3.0              4.5  \n",
+       "19                  3.5              3.5  \n",
+       "20                  4.5              4.5  \n",
+       "21                  3.0              4.0  \n",
+       "22                  3.0              4.0  \n",
+       "23                  3.0              4.0  \n",
+       "24                  3.5              3.5  "
       ]
      },
      "execution_count": 6,
@@ -1691,27 +1110,12 @@
     {
      "data": {
       "text/plain": [
-       "group3_piml_EBM_rank         4.30\n",
-       "group1_ebm_rank              5.34\n",
-       "ph_ebm_rank                  5.34\n",
-       "group9_ebm_rank              5.34\n",
-       "group2_piml_EBM_rank         5.36\n",
-       "group5_xgb2_rank             5.60\n",
-       "group2_piml_XGB2_rank        6.20\n",
-       "group3_piml_XGB_rank         6.94\n",
-       "group8_piml_ebm_rank         7.52\n",
-       "group2_piml_reludnn_rank    11.70\n",
-       "group8_piml_reludnn_rank    11.84\n",
-       "group8_piml_gam_rank        11.84\n",
-       "group5_reludnn_rank         12.00\n",
-       "group5_gaminet_rank         13.76\n",
-       "ph_mxgb_rank                14.30\n",
-       "group1_mxgb_rank            14.50\n",
-       "group9_mxgb_rank            14.72\n",
-       "group3_piml_GAM_rank        17.34\n",
-       "group9_glm_rank             19.02\n",
-       "group1_glm_rank             19.02\n",
-       "ph_glm_rank                 19.02\n",
+       "group2_rem_ebm_rank         1.60\n",
+       "group5_rem_xgb2_rank        1.90\n",
+       "group8_rem_ebm_rank         3.44\n",
+       "ph_rem_ebm_rank             3.70\n",
+       "group1_rem_ebm_rank         4.70\n",
+       "group3_rem_piml_EBM_rank    5.66\n",
        "dtype: float64"
       ]
      },
diff --git a/assignments/model_eval_2023_06_01_11_08_41.csv b/assignments/model_eval_2023_06_01_11_08_41.csv
deleted file mode 100644
index 7d7b935..0000000
--- a/assignments/model_eval_2023_06_01_11_08_41.csv
+++ /dev/null
@@ -1,26 +0,0 @@
-fold,metric,group1_ebm,group1_glm,group1_mxgb,group2_piml_EBM,group2_piml_XGB2,group2_piml_reludnn,group3_piml_EBM,group3_piml_GAM,group3_piml_XGB,group5_gaminet,group5_reludnn,group5_xgb2,group8_piml_ebm,group8_piml_gam,group8_piml_reludnn,group9_ebm,group9_glm,group9_mxgb,ph_ebm,ph_glm,ph_mxgb,group1_ebm_rank,group1_glm_rank,group1_mxgb_rank,group2_piml_EBM_rank,group2_piml_XGB2_rank,group2_piml_reludnn_rank,group3_piml_EBM_rank,group3_piml_GAM_rank,group3_piml_XGB_rank,group5_gaminet_rank,group5_reludnn_rank,group5_xgb2_rank,group8_piml_ebm_rank,group8_piml_gam_rank,group8_piml_reludnn_rank,group9_ebm_rank,group9_glm_rank,group9_mxgb_rank,ph_ebm_rank,ph_glm_rank,ph_mxgb_rank
-0.0,acc,0.901,0.9,0.902,0.901,0.9,0.9,0.902,0.9,0.901,0.9,0.9,0.901,0.901,0.901,0.901,0.901,0.9,0.902,0.901,0.9,0.902,9.0,17.5,2.5,9.0,17.5,17.5,2.5,17.5,9.0,17.5,17.5,9.0,9.0,9.0,9.0,9.0,17.5,2.5,9.0,17.5,2.5
-0.0,auc,0.84,0.775,0.814,0.836,0.837,0.822,0.837,0.805,0.835,0.819,0.821,0.836,0.835,0.822,0.822,0.84,0.775,0.813,0.84,0.775,0.814,2.0,20.0,15.5,6.5,4.5,11.0,4.5,18.0,8.5,14.0,13.0,6.5,8.5,11.0,11.0,2.0,20.0,17.0,2.0,20.0,15.5
-0.0,f1,0.405,0.335,0.379,0.395,0.394,0.373,0.399,0.358,0.397,0.383,0.375,0.395,0.398,0.374,0.374,0.405,0.335,0.374,0.405,0.335,0.381,2.0,20.0,12.0,7.5,9.0,17.0,4.0,18.0,6.0,10.0,13.0,7.5,5.0,15.0,15.0,2.0,20.0,15.0,2.0,20.0,11.0
-0.0,logloss,0.251,0.291,0.263,0.253,0.253,0.259,0.251,0.271,0.253,0.266,0.26,0.253,0.253,0.259,0.259,0.251,0.291,0.263,0.251,0.291,0.263,2.5,20.0,15.0,7.0,7.0,11.0,2.5,18.0,7.0,17.0,13.0,7.0,7.0,11.0,11.0,2.5,20.0,15.0,2.5,20.0,15.0
-0.0,mse,0.077,0.084,0.078,0.077,0.077,0.078,0.077,0.081,0.077,0.08,0.078,0.077,0.077,0.078,0.078,0.077,0.084,0.078,0.077,0.084,0.078,5.0,20.0,13.0,5.0,5.0,13.0,5.0,18.0,5.0,17.0,13.0,5.0,5.0,13.0,13.0,5.0,20.0,13.0,5.0,20.0,13.0
-1.0,acc,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0,11.0
-1.0,auc,0.828,0.757,0.792,0.826,0.826,0.801,0.828,0.78,0.825,0.805,0.803,0.824,0.824,0.801,0.801,0.828,0.757,0.792,0.828,0.757,0.792,2.5,20.0,16.0,5.5,5.5,13.0,2.5,18.0,7.0,10.0,11.0,8.5,8.5,13.0,13.0,2.5,20.0,16.0,2.5,20.0,16.0
-1.0,f1,0.368,0.302,0.337,0.369,0.365,0.341,0.369,0.32,0.363,0.352,0.346,0.365,0.364,0.341,0.341,0.368,0.302,0.342,0.368,0.302,0.337,4.0,20.0,16.5,1.5,6.5,14.0,1.5,18.0,9.0,10.0,11.0,6.5,8.0,14.0,14.0,4.0,20.0,12.0,4.0,20.0,16.5
-1.0,logloss,0.246,0.281,0.264,0.247,0.248,0.257,0.247,0.269,0.249,0.26,0.257,0.249,0.249,0.257,0.257,0.246,0.281,0.264,0.246,0.281,0.263,2.0,20.0,16.5,4.5,6.0,11.5,4.5,18.0,8.0,14.0,11.5,8.0,8.0,11.5,11.5,2.0,20.0,16.5,2.0,20.0,15.0
-1.0,mse,0.074,0.08,0.078,0.075,0.075,0.077,0.074,0.079,0.075,0.078,0.077,0.075,0.075,0.077,0.077,0.074,0.08,0.078,0.074,0.08,0.078,2.5,20.0,15.5,7.0,7.0,11.5,2.5,18.0,7.0,15.5,11.5,7.0,7.0,11.5,11.5,2.5,20.0,15.5,2.5,20.0,15.5
-2.0,acc,0.908,0.908,0.908,0.909,0.91,0.909,0.909,0.908,0.909,0.908,0.908,0.91,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,14.0,14.0,14.0,4.5,1.5,4.5,4.5,14.0,4.5,14.0,14.0,1.5,14.0,14.0,14.0,14.0,14.0,14.0,14.0,14.0,14.0
-2.0,auc,0.825,0.763,0.797,0.826,0.826,0.813,0.825,0.792,0.825,0.812,0.809,0.828,0.824,0.81,0.81,0.825,0.763,0.798,0.825,0.763,0.797,6.0,20.0,16.5,2.5,2.5,10.0,6.0,18.0,6.0,11.0,14.0,1.0,9.0,12.5,12.5,6.0,20.0,15.0,6.0,20.0,16.5
-2.0,f1,0.372,0.312,0.344,0.37,0.374,0.346,0.366,0.335,0.373,0.358,0.345,0.376,0.375,0.356,0.356,0.372,0.312,0.352,0.372,0.312,0.346,6.0,20.0,17.0,8.0,3.0,14.5,9.0,18.0,4.0,10.0,16.0,1.0,2.0,11.5,11.5,6.0,20.0,13.0,6.0,20.0,14.5
-2.0,logloss,0.246,0.279,0.259,0.245,0.245,0.25,0.245,0.264,0.245,0.253,0.252,0.244,0.246,0.252,0.252,0.246,0.279,0.259,0.246,0.279,0.259,7.5,20.0,16.0,3.5,3.5,10.0,3.5,18.0,3.5,14.0,12.0,1.0,7.5,12.0,12.0,7.5,20.0,16.0,7.5,20.0,16.0
-2.0,mse,0.073,0.079,0.076,0.073,0.073,0.074,0.073,0.077,0.073,0.075,0.075,0.072,0.073,0.074,0.074,0.073,0.079,0.076,0.073,0.079,0.076,5.5,20.0,16.0,5.5,5.5,11.0,5.5,18.0,5.5,13.5,13.5,1.0,5.5,11.0,11.0,5.5,20.0,16.0,5.5,20.0,16.0
-3.0,acc,0.903,0.903,0.903,0.903,0.904,0.904,0.904,0.903,0.903,0.903,0.904,0.904,0.903,0.904,0.904,0.903,0.903,0.903,0.903,0.903,0.903,14.5,14.5,14.5,14.5,4.0,4.0,4.0,14.5,14.5,14.5,4.0,4.0,14.5,4.0,4.0,14.5,14.5,14.5,14.5,14.5,14.5
-3.0,auc,0.826,0.755,0.795,0.826,0.825,0.807,0.826,0.79,0.824,0.803,0.808,0.825,0.826,0.806,0.806,0.826,0.755,0.795,0.826,0.755,0.795,3.5,20.0,16.0,3.5,7.5,11.0,3.5,18.0,9.0,14.0,10.0,7.5,3.5,12.5,12.5,3.5,20.0,16.0,3.5,20.0,16.0
-3.0,f1,0.371,0.307,0.345,0.368,0.36,0.35,0.368,0.334,0.368,0.349,0.347,0.364,0.365,0.345,0.345,0.371,0.307,0.348,0.371,0.307,0.344,2.0,20.0,15.0,5.0,9.0,10.0,5.0,18.0,5.0,11.0,13.0,8.0,7.0,15.0,15.0,2.0,20.0,12.0,2.0,20.0,17.0
-3.0,logloss,0.252,0.288,0.268,0.251,0.252,0.259,0.251,0.271,0.252,0.265,0.259,0.252,0.252,0.26,0.26,0.252,0.288,0.268,0.252,0.288,0.268,6.0,20.0,16.0,1.5,6.0,10.5,1.5,18.0,6.0,14.0,10.5,6.0,6.0,12.5,12.5,6.0,20.0,16.0,6.0,20.0,16.0
-3.0,mse,0.077,0.082,0.08,0.077,0.077,0.078,0.077,0.08,0.077,0.08,0.078,0.077,0.077,0.078,0.078,0.077,0.082,0.08,0.077,0.082,0.08,5.0,20.0,16.0,5.0,5.0,11.5,5.0,16.0,5.0,16.0,11.5,5.0,5.0,11.5,11.5,5.0,20.0,16.0,5.0,20.0,16.0
-4.0,acc,0.897,0.895,0.896,0.899,0.897,0.896,0.898,0.895,0.898,0.895,0.896,0.898,0.897,0.896,0.896,0.897,0.895,0.895,0.897,0.895,0.896,7.0,18.5,12.5,1.0,7.0,12.5,3.0,18.5,3.0,18.5,12.5,3.0,7.0,12.5,12.5,7.0,18.5,18.5,7.0,18.5,12.5
-4.0,auc,0.831,0.776,0.807,0.83,0.828,0.812,0.83,0.798,0.828,0.81,0.817,0.829,0.826,0.815,0.815,0.831,0.776,0.803,0.831,0.776,0.808,2.0,20.0,16.0,4.5,7.5,13.0,4.5,18.0,7.5,14.0,10.0,6.0,9.0,11.5,11.5,2.0,20.0,17.0,2.0,20.0,15.0
-4.0,f1,0.401,0.358,0.376,0.407,0.402,0.38,0.402,0.369,0.398,0.382,0.385,0.399,0.399,0.382,0.382,0.401,0.358,0.371,0.401,0.358,0.378,5.0,20.0,16.0,1.0,2.5,14.0,2.5,18.0,9.0,12.0,10.0,7.5,7.5,12.0,12.0,5.0,20.0,17.0,5.0,20.0,15.0
-4.0,logloss,0.263,0.3,0.275,0.264,0.265,0.271,0.264,0.284,0.266,0.275,0.27,0.265,0.266,0.27,0.27,0.263,0.3,0.277,0.263,0.3,0.275,2.0,20.0,15.0,4.5,6.5,13.0,4.5,18.0,8.5,15.0,11.0,6.5,8.5,11.0,11.0,2.0,20.0,17.0,2.0,20.0,15.0
-4.0,mse,0.08,0.087,0.082,0.08,0.08,0.082,0.08,0.085,0.08,0.083,0.082,0.08,0.08,0.082,0.082,0.08,0.087,0.083,0.08,0.087,0.082,5.0,20.0,12.5,5.0,5.0,12.5,5.0,18.0,5.0,16.5,12.5,5.0,5.0,12.5,12.5,5.0,20.0,16.5,5.0,20.0,12.5
diff --git a/assignments/model_eval_2023_06_19_21_04_44.csv b/assignments/model_eval_2023_06_19_21_04_44.csv
new file mode 100644
index 0000000..d689bc6
--- /dev/null
+++ b/assignments/model_eval_2023_06_19_21_04_44.csv
@@ -0,0 +1,26 @@
+fold,metric,group1_rem_ebm,group2_rem_ebm,group3_rem_piml_EBM,group5_rem_xgb2,group8_rem_ebm,ph_rem_ebm,group1_rem_ebm_rank,group2_rem_ebm_rank,group3_rem_piml_EBM_rank,group5_rem_xgb2_rank,group8_rem_ebm_rank,ph_rem_ebm_rank
+0.0,acc,0.9,0.901,0.9,0.901,0.901,0.901,5.5,2.5,5.5,2.5,2.5,2.5
+0.0,auc,0.781,0.84,0.163,0.836,0.793,0.791,5.0,1.0,6.0,2.0,3.0,4.0
+0.0,f1,0.347,0.405,0.182,0.392,0.342,0.347,3.5,1.0,6.0,2.0,5.0,3.5
+0.0,logloss,0.28,0.251,3.257,0.254,0.274,0.275,5.0,1.0,6.0,2.0,3.0,4.0
+0.0,mse,0.082,0.077,0.773,0.077,0.081,0.081,5.0,1.5,6.0,1.5,3.5,3.5
+1.0,acc,0.906,0.906,0.906,0.906,0.906,0.906,3.5,3.5,3.5,3.5,3.5,3.5
+1.0,auc,0.767,0.828,0.172,0.822,0.774,0.772,5.0,1.0,6.0,2.0,3.0,4.0
+1.0,f1,0.312,0.368,0.172,0.36,0.319,0.321,5.0,1.0,6.0,2.0,4.0,3.0
+1.0,logloss,0.272,0.246,3.253,0.25,0.27,0.272,4.5,1.0,6.0,2.0,3.0,4.5
+1.0,mse,0.079,0.074,0.778,0.075,0.079,0.079,4.0,1.0,6.0,2.0,4.0,4.0
+2.0,acc,0.908,0.908,0.908,0.91,0.908,0.909,4.5,4.5,4.5,1.0,4.5,2.0
+2.0,auc,0.759,0.825,0.175,0.826,0.781,0.78,5.0,2.0,6.0,1.0,3.0,4.0
+2.0,f1,0.304,0.372,0.169,0.371,0.315,0.323,5.0,1.0,6.0,2.0,4.0,3.0
+2.0,logloss,0.271,0.246,3.284,0.245,0.264,0.264,5.0,2.0,6.0,1.0,3.5,3.5
+2.0,mse,0.078,0.073,0.781,0.073,0.076,0.076,5.0,1.5,6.0,1.5,3.5,3.5
+3.0,acc,0.903,0.903,0.903,0.903,0.903,0.903,3.5,3.5,3.5,3.5,3.5,3.5
+3.0,auc,0.772,0.826,0.174,0.823,0.775,0.772,4.5,1.0,6.0,2.0,3.0,4.5
+3.0,f1,0.317,0.371,0.177,0.365,0.328,0.323,5.0,1.0,6.0,2.0,3.0,4.0
+3.0,logloss,0.276,0.252,3.254,0.253,0.275,0.276,4.5,1.0,6.0,2.0,3.0,4.5
+3.0,mse,0.081,0.077,0.775,0.077,0.08,0.08,5.0,1.5,6.0,1.5,3.5,3.5
+4.0,acc,0.895,0.897,0.895,0.898,0.895,0.895,4.5,2.0,4.5,1.0,4.5,4.5
+4.0,auc,0.754,0.831,0.17,0.828,0.785,0.782,5.0,1.0,6.0,2.0,3.0,4.0
+4.0,f1,0.323,0.401,0.19,0.397,0.364,0.362,5.0,1.0,6.0,2.0,3.0,4.0
+4.0,logloss,0.296,0.263,3.2,0.266,0.286,0.287,5.0,1.0,6.0,2.0,3.0,4.0
+4.0,mse,0.087,0.08,0.771,0.08,0.084,0.084,5.0,1.5,6.0,1.5,3.5,3.5