19831 rows × 11 columns
\n", + "19831 rows × 37 columns
\n", "" ], "text/plain": [ - " high_priced fold group1_rem_ebm group2_rem_ebm group2_rem_ebm2 \\\n", - "0 0.0 2 0.118787 0.080557 0.080557 \n", - "1 0.0 1 0.084506 0.026001 0.026001 \n", - "2 1.0 4 0.210389 0.194961 0.194961 \n", - "3 0.0 1 0.008529 0.028556 0.028556 \n", - "4 1.0 2 0.189933 0.208263 0.208263 \n", - "... ... ... ... ... ... \n", - "19826 0.0 3 0.163697 0.228342 0.228342 \n", - "19827 0.0 1 0.114999 0.253998 0.253998 \n", - "19828 1.0 3 0.141307 0.213364 0.213364 \n", - "19829 0.0 1 0.007766 0.002176 0.002176 \n", - "19830 0.0 0 0.163946 0.185484 0.185484 \n", + " high_priced fold group1_ebm group1_glm group1_mxgb group2_ebm \\\n", + "0 0.0 2 0.081364 0.142090 0.062958 0.081364 \n", + "1 0.0 1 0.026177 0.081674 0.037317 0.026177 \n", + "2 1.0 4 0.184662 0.125823 0.171167 0.184662 \n", + "3 0.0 1 0.030049 0.006973 0.030916 0.030049 \n", + "4 1.0 2 0.205948 0.130426 0.174999 0.205948 \n", + "... ... ... ... ... ... ... \n", + "19826 0.0 3 0.227771 0.160032 0.275086 0.227771 \n", + "19827 0.0 1 0.253325 0.123836 0.188414 0.253325 \n", + "19828 1.0 3 0.225811 0.169604 0.211717 0.225811 \n", + "19829 0.0 1 0.001488 0.002538 0.001149 0.001488 \n", + "19830 0.0 0 0.207258 0.156659 0.264629 0.207258 \n", + "\n", + " group2_glm group2_mxgb group3_ebm group3_glm ... group8_ebm \\\n", + "0 0.142090 0.086361 0.081364 0.142090 ... 0.081364 \n", + "1 0.081674 0.033920 0.026177 0.081674 ... 0.026177 \n", + "2 0.125823 0.183323 0.184662 0.125823 ... 0.184662 \n", + "3 0.006973 0.030934 0.030049 0.006973 ... 0.030049 \n", + "4 0.130426 0.178491 0.205948 0.130426 ... 0.205948 \n", + "... ... ... ... ... ... ... \n", + "19826 0.160032 0.255826 0.227771 0.160032 ... 0.227771 \n", + "19827 0.123836 0.176984 0.253325 0.123836 ... 0.253325 \n", + "19828 0.169604 0.236894 0.225811 0.169604 ... 0.225811 \n", + "19829 0.002538 0.001113 0.001488 0.002538 ... 0.001488 \n", + "19830 0.156659 0.233696 0.207258 0.156659 ... 0.207258 \n", "\n", - " group3_rem_piml_EBM group3_rem_piml_EBM2 group5_rem_xgb2 \\\n", - "0 0.920389 0.136749 0.078326 \n", - "1 0.969301 0.053751 0.035825 \n", - "2 0.814272 0.182311 0.195332 \n", - "3 0.974559 0.004065 0.022765 \n", - "4 0.802908 0.211120 0.193035 \n", - "... ... ... ... \n", - "19826 0.792251 0.209322 0.235192 \n", - "19827 0.762946 0.206744 0.235832 \n", - "19828 0.747401 0.246610 0.208723 \n", - "19829 0.996455 0.000268 0.018702 \n", - "19830 0.811429 0.177857 0.215085 \n", + " group8_glm group8_mxgb group9_ebm group9_glm group9_mxgb \\\n", + "0 0.142090 0.078233 0.081364 0.142090 0.059522 \n", + "1 0.081674 0.022467 0.026177 0.081674 0.036210 \n", + "2 0.125823 0.179896 0.184662 0.125823 0.180734 \n", + "3 0.006973 0.010161 0.030049 0.006973 0.027677 \n", + "4 0.130426 0.214328 0.205948 0.130426 0.177813 \n", + "... ... ... ... ... ... \n", + "19826 0.160032 0.236099 0.227771 0.160032 0.274767 \n", + "19827 0.123836 0.255476 0.253325 0.123836 0.182039 \n", + "19828 0.169604 0.224857 0.225811 0.169604 0.212740 \n", + "19829 0.002538 0.001779 0.001488 0.002538 0.001323 \n", + "19830 0.156659 0.185213 0.207258 0.156659 0.259442 \n", "\n", - " group8_rem_ebm group9_rem_xgb ph_rem_ebm \n", - "0 0.223846 0.081792 0.219429 \n", - "1 0.053926 0.110702 0.053929 \n", - "2 0.143522 0.204048 0.133863 \n", - "3 0.009371 0.024038 0.014419 \n", - "4 0.151100 0.170243 0.156047 \n", - "... ... ... ... \n", - "19826 0.216720 0.181403 0.184214 \n", - "19827 0.161401 0.159468 0.141663 \n", - "19828 0.242814 0.138141 0.233266 \n", - "19829 0.005657 0.034570 0.009914 \n", - "19830 0.167812 0.177785 0.155447 \n", + " ph_best_moe ph_ebm ph_glm ph_mxgb \n", + "0 0.073803 0.082841 0.142090 0.059522 \n", + "1 0.028311 0.027079 0.081674 0.036210 \n", + "2 0.200144 0.190718 0.125823 0.180734 \n", + "3 0.008254 0.031069 0.006973 0.027677 \n", + "4 0.209376 0.210361 0.130426 0.177813 \n", + "... ... ... ... ... \n", + "19826 0.222673 0.231624 0.160032 0.274767 \n", + "19827 0.254699 0.254823 0.123836 0.182039 \n", + "19828 0.227556 0.220400 0.169604 0.212740 \n", + "19829 0.001639 0.000993 0.002538 0.001323 \n", + "19830 0.211435 0.222328 0.156659 0.259442 \n", "\n", - "[19831 rows x 11 columns]" + "[19831 rows x 37 columns]" ] }, - "execution_count": 3, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -365,7 +498,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 13, "id": "2eb43506", "metadata": {}, "outputs": [], @@ -414,7 +547,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 14, "id": "fae3756b", "metadata": {}, "outputs": [], @@ -463,7 +596,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 15, "id": "40fbe608", "metadata": {}, "outputs": [ @@ -490,24 +623,25 @@ "25 rows × 72 columns
\n", "" ], "text/plain": [ - " fold metric group1_rem_ebm group2_rem_ebm group2_rem_ebm2 \\\n", - "0 0.0 acc 0.900 0.901 0.901 \n", - "1 0.0 auc 0.781 0.840 0.840 \n", - "2 0.0 f1 0.347 0.405 0.405 \n", - "3 0.0 logloss 0.280 0.251 0.251 \n", - "4 0.0 mse 0.082 0.077 0.077 \n", - "5 1.0 acc 0.906 0.906 0.906 \n", - "6 1.0 auc 0.767 0.828 0.828 \n", - "7 1.0 f1 0.312 0.368 0.368 \n", - "8 1.0 logloss 0.272 0.246 0.246 \n", - "9 1.0 mse 0.079 0.074 0.074 \n", - "10 2.0 acc 0.908 0.908 0.908 \n", - "11 2.0 auc 0.759 0.825 0.825 \n", - "12 2.0 f1 0.304 0.372 0.372 \n", - "13 2.0 logloss 0.271 0.246 0.246 \n", - "14 2.0 mse 0.078 0.073 0.073 \n", - "15 3.0 acc 0.903 0.903 0.903 \n", - "16 3.0 auc 0.772 0.826 0.826 \n", - "17 3.0 f1 0.317 0.371 0.371 \n", - "18 3.0 logloss 0.276 0.252 0.252 \n", - "19 3.0 mse 0.081 0.077 0.077 \n", - "20 4.0 acc 0.895 0.897 0.897 \n", - "21 4.0 auc 0.754 0.831 0.831 \n", - "22 4.0 f1 0.323 0.401 0.401 \n", - "23 4.0 logloss 0.296 0.263 0.263 \n", - "24 4.0 mse 0.087 0.080 0.080 \n", + " fold metric group1_ebm group1_glm group1_mxgb group2_ebm \\\n", + "0 0.0 acc 0.901 0.900 0.902 0.901 \n", + "1 0.0 auc 0.839 0.775 0.812 0.839 \n", + "2 0.0 f1 0.404 0.335 0.376 0.404 \n", + "3 0.0 logloss 0.251 0.291 0.264 0.251 \n", + "4 0.0 mse 0.077 0.084 0.078 0.077 \n", + "5 1.0 acc 0.906 0.906 0.906 0.906 \n", + "6 1.0 auc 0.829 0.757 0.793 0.829 \n", + "7 1.0 f1 0.371 0.302 0.339 0.371 \n", + "8 1.0 logloss 0.246 0.281 0.263 0.246 \n", + "9 1.0 mse 0.074 0.080 0.078 0.074 \n", + "10 2.0 acc 0.908 0.908 0.908 0.908 \n", + "11 2.0 auc 0.827 0.763 0.796 0.827 \n", + "12 2.0 f1 0.376 0.312 0.345 0.376 \n", + "13 2.0 logloss 0.245 0.279 0.260 0.245 \n", + "14 2.0 mse 0.073 0.079 0.076 0.073 \n", + "15 3.0 acc 0.903 0.903 0.903 0.903 \n", + "16 3.0 auc 0.826 0.755 0.794 0.826 \n", + "17 3.0 f1 0.373 0.307 0.340 0.373 \n", + "18 3.0 logloss 0.251 0.288 0.268 0.251 \n", + "19 3.0 mse 0.077 0.082 0.080 0.077 \n", + "20 4.0 acc 0.897 0.895 0.895 0.897 \n", + "21 4.0 auc 0.831 0.776 0.806 0.831 \n", + "22 4.0 f1 0.403 0.358 0.376 0.403 \n", + "23 4.0 logloss 0.263 0.300 0.276 0.263 \n", + "24 4.0 mse 0.080 0.087 0.082 0.080 \n", "\n", - " group3_rem_piml_EBM group3_rem_piml_EBM2 group5_rem_xgb2 \\\n", - "0 0.900 0.901 0.901 \n", - "1 0.163 0.821 0.836 \n", - "2 0.182 0.381 0.392 \n", - "3 3.257 0.262 0.254 \n", - "4 0.773 0.078 0.077 \n", - "5 0.906 0.906 0.906 \n", - "6 0.172 0.810 0.822 \n", - "7 0.172 0.348 0.360 \n", - "8 3.253 0.258 0.250 \n", - "9 0.778 0.077 0.075 \n", - "10 0.908 0.908 0.910 \n", - "11 0.175 0.815 0.826 \n", - "12 0.169 0.354 0.371 \n", - "13 3.284 0.251 0.245 \n", - "14 0.781 0.074 0.073 \n", - "15 0.903 0.903 0.903 \n", - "16 0.174 0.809 0.823 \n", - "17 0.177 0.361 0.365 \n", - "18 3.254 0.262 0.253 \n", - "19 0.775 0.079 0.077 \n", - "20 0.895 0.895 0.898 \n", - "21 0.170 0.818 0.828 \n", - "22 0.190 0.404 0.397 \n", - "23 3.200 0.273 0.266 \n", - "24 0.771 0.082 0.080 \n", + " group2_glm group2_mxgb group3_ebm group3_glm ... group8_ebm_rank \\\n", + "0 0.900 0.902 0.901 0.900 ... 16.0 \n", + "1 0.775 0.813 0.839 0.775 ... 6.0 \n", + "2 0.335 0.374 0.404 0.335 ... 8.0 \n", + "3 0.291 0.263 0.251 0.291 ... 6.0 \n", + "4 0.084 0.078 0.077 0.084 ... 7.5 \n", + "5 0.906 0.906 0.906 0.906 ... 19.0 \n", + "6 0.757 0.792 0.829 0.757 ... 5.5 \n", + "7 0.302 0.342 0.371 0.302 ... 7.0 \n", + "8 0.281 0.264 0.246 0.281 ... 6.5 \n", + "9 0.080 0.078 0.074 0.080 ... 6.0 \n", + "10 0.908 0.908 0.908 0.908 ... 19.5 \n", + "11 0.763 0.798 0.827 0.763 ... 6.5 \n", + "12 0.312 0.352 0.376 0.312 ... 4.0 \n", + "13 0.279 0.259 0.245 0.279 ... 7.5 \n", + "14 0.079 0.076 0.073 0.079 ... 8.5 \n", + "15 0.903 0.903 0.903 0.903 ... 20.0 \n", + "16 0.755 0.795 0.826 0.755 ... 6.0 \n", + "17 0.307 0.348 0.373 0.307 ... 4.5 \n", + "18 0.288 0.268 0.251 0.288 ... 5.5 \n", + "19 0.082 0.080 0.077 0.082 ... 7.5 \n", + "20 0.895 0.895 0.897 0.895 ... 7.0 \n", + "21 0.776 0.803 0.831 0.776 ... 8.0 \n", + "22 0.358 0.371 0.403 0.358 ... 7.0 \n", + "23 0.300 0.277 0.263 0.300 ... 7.0 \n", + "24 0.087 0.083 0.080 0.087 ... 8.5 \n", "\n", - " group8_rem_ebm group9_rem_xgb ph_rem_ebm group1_rem_ebm_rank \\\n", - "0 0.901 0.900 0.901 8.0 \n", - "1 0.793 0.797 0.791 8.0 \n", - "2 0.342 0.357 0.347 6.5 \n", - "3 0.274 0.277 0.275 8.0 \n", - "4 0.081 0.081 0.081 8.0 \n", - "5 0.906 0.906 0.906 5.0 \n", - "6 0.774 0.779 0.772 8.0 \n", - "7 0.319 0.329 0.321 8.0 \n", - "8 0.270 0.271 0.272 7.5 \n", - "9 0.079 0.078 0.079 7.0 \n", - "10 0.908 0.908 0.909 6.0 \n", - "11 0.781 0.772 0.780 8.0 \n", - "12 0.315 0.320 0.323 8.0 \n", - "13 0.264 0.271 0.264 7.5 \n", - "14 0.076 0.077 0.076 8.0 \n", - "15 0.903 0.903 0.903 5.0 \n", - "16 0.775 0.786 0.772 7.5 \n", - "17 0.328 0.343 0.323 8.0 \n", - "18 0.275 0.275 0.276 7.5 \n", - "19 0.080 0.080 0.080 8.0 \n", - "20 0.895 0.896 0.895 7.0 \n", - "21 0.785 0.779 0.782 8.0 \n", - "22 0.364 0.354 0.362 8.0 \n", - "23 0.286 0.291 0.287 8.0 \n", - "24 0.084 0.086 0.084 8.0 \n", + " group8_glm_rank group8_mxgb_rank group9_ebm_rank group9_glm_rank \\\n", + "0 29.0 16.0 16.0 29.0 \n", + "1 31.0 1.0 6.0 31.0 \n", + "2 31.0 3.0 8.0 31.0 \n", + "3 31.0 1.0 6.0 31.0 \n", + "4 31.0 7.5 7.5 31.0 \n", + "5 19.0 1.5 19.0 19.0 \n", + "6 31.0 1.0 5.5 31.0 \n", + "7 31.0 1.0 7.0 31.0 \n", + "8 31.0 1.0 6.5 31.0 \n", + "9 30.0 6.0 6.0 30.0 \n", + "10 19.5 1.5 19.5 19.5 \n", + "11 30.0 1.0 6.5 30.0 \n", + "12 31.0 10.5 4.0 31.0 \n", + "13 30.0 2.0 7.5 30.0 \n", + "14 30.0 1.5 8.5 30.0 \n", + "15 20.0 2.5 20.0 20.0 \n", + "16 31.0 6.0 6.0 31.0 \n", + "17 31.0 10.0 4.5 31.0 \n", + "18 31.0 5.5 5.5 31.0 \n", + "19 30.0 7.5 7.5 30.0 \n", + "20 27.0 7.0 7.0 27.0 \n", + "21 30.0 1.0 8.0 30.0 \n", + "22 30.0 3.0 7.0 30.0 \n", + "23 30.0 1.0 7.0 30.0 \n", + "24 30.0 1.5 8.5 30.0 \n", "\n", - " group2_rem_ebm_rank group2_rem_ebm2_rank group3_rem_piml_EBM_rank \\\n", - "0 3.5 3.5 8.0 \n", - "1 1.5 1.5 9.0 \n", - "2 1.5 1.5 9.0 \n", - "3 1.5 1.5 9.0 \n", - "4 2.0 2.0 9.0 \n", - "5 5.0 5.0 5.0 \n", - "6 1.5 1.5 9.0 \n", - "7 1.5 1.5 9.0 \n", - "8 1.5 1.5 9.0 \n", - "9 1.5 1.5 9.0 \n", - "10 6.0 6.0 6.0 \n", - "11 2.5 2.5 9.0 \n", - "12 1.5 1.5 9.0 \n", - "13 2.5 2.5 9.0 \n", - "14 2.0 2.0 9.0 \n", - "15 5.0 5.0 5.0 \n", - "16 1.5 1.5 9.0 \n", - "17 1.5 1.5 9.0 \n", - "18 1.5 1.5 9.0 \n", - "19 2.0 2.0 9.0 \n", - "20 2.5 2.5 7.0 \n", - "21 1.5 1.5 9.0 \n", - "22 2.5 2.5 9.0 \n", - "23 1.5 1.5 9.0 \n", - "24 2.0 2.0 9.0 \n", + " group9_mxgb_rank ph_best_moe_rank ph_ebm_rank ph_glm_rank ph_mxgb_rank \n", + "0 5.0 29.0 16.0 29.0 5.0 \n", + "1 19.0 11.0 6.0 31.0 19.0 \n", + "2 17.5 2.0 1.0 31.0 17.5 \n", + "3 19.5 11.5 6.0 31.0 19.5 \n", + "4 19.0 7.5 7.5 31.0 19.0 \n", + "5 19.0 19.0 19.0 19.0 19.0 \n", + "6 21.5 11.0 11.0 31.0 21.5 \n", + "7 22.0 2.0 11.5 31.0 22.0 \n", + "8 21.5 6.5 6.5 31.0 21.5 \n", + "9 20.5 6.0 6.0 30.0 20.5 \n", + "10 19.5 1.5 19.5 19.5 19.5 \n", + "11 21.5 6.5 12.0 30.0 21.5 \n", + "12 22.0 14.0 8.5 31.0 22.0 \n", + "13 20.5 1.0 7.5 30.0 20.5 \n", + "14 21.5 1.5 8.5 30.0 21.5 \n", + "15 20.0 2.5 2.5 20.0 20.0 \n", + "16 20.5 13.0 1.0 31.0 20.5 \n", + "17 20.5 11.5 4.5 31.0 20.5 \n", + "18 21.5 12.0 5.5 31.0 21.5 \n", + "19 21.5 7.5 7.5 30.0 21.5 \n", + "20 16.0 7.0 7.0 27.0 16.0 \n", + "21 18.5 2.0 8.0 30.0 18.5 \n", + "22 18.5 2.0 11.0 30.0 18.5 \n", + "23 17.5 2.0 7.0 30.0 17.5 \n", + "24 18.0 1.5 8.5 30.0 18.0 \n", "\n", - " group3_rem_piml_EBM2_rank group5_rem_xgb2_rank group8_rem_ebm_rank \\\n", - "0 3.5 3.5 3.5 \n", - "1 4.0 3.0 6.0 \n", - "2 4.0 3.0 8.0 \n", - "3 4.0 3.0 5.0 \n", - "4 4.0 2.0 6.0 \n", - "5 5.0 5.0 5.0 \n", - "6 4.0 3.0 6.0 \n", - "7 4.0 3.0 7.0 \n", - "8 4.0 3.0 5.0 \n", - "9 4.0 3.0 7.0 \n", - "10 6.0 1.0 6.0 \n", - "11 4.0 1.0 5.0 \n", - "12 4.0 3.0 7.0 \n", - "13 4.0 1.0 5.5 \n", - "14 4.0 2.0 5.5 \n", - "15 5.0 5.0 5.0 \n", - "16 4.0 3.0 6.0 \n", - "17 4.0 3.0 6.0 \n", - "18 4.0 3.0 5.5 \n", - "19 4.0 2.0 6.0 \n", - "20 7.0 1.0 7.0 \n", - "21 4.0 3.0 5.0 \n", - "22 1.0 4.0 5.0 \n", - "23 4.0 3.0 5.0 \n", - "24 4.0 2.0 5.5 \n", - "\n", - " group9_rem_xgb_rank ph_rem_ebm_rank \n", - "0 8.0 3.5 \n", - "1 5.0 7.0 \n", - "2 5.0 6.5 \n", - "3 7.0 6.0 \n", - "4 6.0 6.0 \n", - "5 5.0 5.0 \n", - "6 5.0 7.0 \n", - "7 5.0 6.0 \n", - "8 6.0 7.5 \n", - "9 5.0 7.0 \n", - "10 6.0 2.0 \n", - "11 7.0 6.0 \n", - "12 6.0 5.0 \n", - "13 7.5 5.5 \n", - "14 7.0 5.5 \n", - "15 5.0 5.0 \n", - "16 5.0 7.5 \n", - "17 5.0 7.0 \n", - "18 5.5 7.5 \n", - "19 6.0 6.0 \n", - "20 4.0 7.0 \n", - "21 7.0 6.0 \n", - "22 7.0 6.0 \n", - "23 7.0 6.0 \n", - "24 7.0 5.5 " + "[25 rows x 72 columns]" ] }, - "execution_count": 6, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -1362,26 +1470,52 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 16, "id": "f8ff5fa5", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "group2_rem_ebm_rank 2.28\n", - "group2_rem_ebm2_rank 2.28\n", - "group5_rem_xgb2_rank 2.74\n", - "group3_rem_piml_EBM2_rank 4.14\n", - "group8_rem_ebm_rank 5.74\n", - "group9_rem_xgb_rank 5.96\n", - "ph_rem_ebm_rank 5.96\n", - "group1_rem_ebm_rank 7.46\n", - "group3_rem_piml_EBM_rank 8.44\n", + "group8_mxgb_rank 4.02\n", + "ph_best_moe_rank 7.64\n", + "ph_ebm_rank 8.36\n", + "group1_ebm_rank 8.58\n", + "group5_ebm_rank 8.58\n", + "group3_ebm_rank 8.58\n", + "group8_ebm_rank 8.58\n", + "group2_ebm_rank 8.58\n", + "group9_ebm_rank 8.58\n", + "group7_ebm_rank 8.58\n", + "group6_ebm_rank 8.82\n", + "group7_mxgb2_rank 9.96\n", + "group6_piml_ebm_rank 12.94\n", + "group7_piml_ebm_rank 12.94\n", + "group7_piml_reludnn_rank 16.70\n", + "group7_piml_gaminet_rank 18.96\n", + "group9_mxgb_rank 19.28\n", + "ph_mxgb_rank 19.28\n", + "group3_mxgb_rank 19.28\n", + "group5_mxgb_rank 19.30\n", + "group6_mxgb_rank 20.40\n", + "group2_mxgb_rank 20.40\n", + "group1_mxgb_rank 20.60\n", + "group7_mxgb_rank 20.60\n", + "group6_piml_reludnn_rank 20.66\n", + "group6_piml_gaminet_rank 28.62\n", + "ph_glm_rank 29.02\n", + "group3_glm_rank 29.02\n", + "group6_glm_rank 29.02\n", + "group9_glm_rank 29.02\n", + "group2_glm_rank 29.02\n", + "group7_glm_rank 29.02\n", + "group1_glm_rank 29.02\n", + "group5_glm_rank 29.02\n", + "group8_glm_rank 29.02\n", "dtype: float64" ] }, - "execution_count": 8, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -1389,11 +1523,19 @@ "source": [ "eval_frame[[name for name in eval_frame.columns if name.endswith('rank')]].mean().sort_values()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b72da56", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -1407,7 +1549,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.6.15" } }, "nbformat": 4, diff --git a/assignments/model_eval_2024_09_04_21_10_51.csv b/assignments/model_eval_2024_09_04_21_10_51.csv new file mode 100644 index 0000000..b08372c --- /dev/null +++ b/assignments/model_eval_2024_09_04_21_10_51.csv @@ -0,0 +1,26 @@ +fold,metric,group1_ebm,group1_glm,group1_mxgb,group2_ebm,group2_glm,group2_mxgb,group3_ebm,group3_glm,group3_mxgb,group5_ebm,group5_glm,group5_mxgb,group6_ebm,group6_glm,group6_mxgb,group6_piml_ebm,group6_piml_gaminet,group6_piml_reludnn,group7_ebm,group7_glm,group7_mxgb,group7_mxgb2,group7_piml_ebm,group7_piml_gaminet,group7_piml_reludnn,group8_ebm,group8_glm,group8_mxgb,group9_ebm,group9_glm,group9_mxgb,ph_ebm,ph_glm,ph_mxgb,group1_ebm_rank,group1_glm_rank,group1_mxgb_rank,group2_ebm_rank,group2_glm_rank,group2_mxgb_rank,group3_ebm_rank,group3_glm_rank,group3_mxgb_rank,group5_ebm_rank,group5_glm_rank,group5_mxgb_rank,group6_ebm_rank,group6_glm_rank,group6_mxgb_rank,group6_piml_ebm_rank,group6_piml_gaminet_rank,group6_piml_reludnn_rank,group7_ebm_rank,group7_glm_rank,group7_mxgb_rank,group7_mxgb2_rank,group7_piml_ebm_rank,group7_piml_gaminet_rank,group7_piml_reludnn_rank,group8_ebm_rank,group8_glm_rank,group8_mxgb_rank,group9_ebm_rank,group9_glm_rank,group9_mxgb_rank,ph_ebm_rank,ph_glm_rank,ph_mxgb_rank +0.0,acc,0.901,0.9,0.902,0.901,0.9,0.902,0.901,0.9,0.902,0.901,0.9,0.902,0.901,0.9,0.902,0.901,0.9,0.902,0.901,0.9,0.902,0.9,0.901,0.9,0.901,0.901,0.9,0.901,0.901,0.9,0.902,0.901,0.9,0.902,16.0,28.5,5.0,16.0,28.5,5.0,16.0,28.5,5.0,16.0,28.5,5.0,16.0,28.5,5.0,16.0,28.5,5.0,16.0,28.5,5.0,28.5,16.0,28.5,16.0,16.0,28.5,16.0,16.0,28.5,5.0,16.0,28.5,5.0 +0.0,auc,0.839,0.775,0.812,0.839,0.775,0.813,0.839,0.775,0.814,0.839,0.775,0.814,0.839,0.775,0.813,0.835,0.79,0.814,0.839,0.775,0.812,0.837,0.835,0.82,0.821,0.839,0.775,0.841,0.839,0.775,0.814,0.839,0.775,0.814,6.0,30.0,23.5,6.0,30.0,21.5,6.0,30.0,18.0,6.0,30.0,18.0,6.0,30.0,21.5,12.5,25.0,18.0,6.0,30.0,23.5,11.0,12.5,15.0,14.0,6.0,30.0,1.0,6.0,30.0,18.0,6.0,30.0,18.0 +0.0,f1,0.404,0.335,0.376,0.404,0.335,0.374,0.404,0.335,0.379,0.404,0.335,0.379,0.405,0.335,0.374,0.401,0.353,0.372,0.404,0.335,0.376,0.397,0.401,0.383,0.376,0.404,0.335,0.406,0.404,0.335,0.379,0.408,0.335,0.379,7.0,30.0,20.0,7.0,30.0,22.5,7.0,30.0,16.5,7.0,30.0,16.5,3.0,30.0,22.5,11.5,25.0,24.0,7.0,30.0,20.0,13.0,11.5,14.0,20.0,7.0,30.0,2.0,7.0,30.0,16.5,1.0,30.0,16.5 +0.0,logloss,0.251,0.291,0.264,0.251,0.291,0.263,0.251,0.291,0.263,0.251,0.291,0.263,0.251,0.291,0.263,0.253,0.284,0.264,0.251,0.291,0.264,0.252,0.253,0.262,0.259,0.251,0.291,0.25,0.251,0.291,0.263,0.251,0.291,0.263,6.0,30.0,23.0,6.0,30.0,18.5,6.0,30.0,18.5,6.0,30.0,18.5,6.0,30.0,18.5,12.5,25.0,23.0,6.0,30.0,23.0,11.0,12.5,15.0,14.0,6.0,30.0,1.0,6.0,30.0,18.5,6.0,30.0,18.5 +0.0,mse,0.077,0.084,0.078,0.077,0.084,0.078,0.077,0.084,0.078,0.077,0.084,0.078,0.077,0.084,0.078,0.077,0.083,0.079,0.077,0.084,0.078,0.077,0.077,0.079,0.078,0.077,0.084,0.077,0.077,0.084,0.078,0.077,0.084,0.078,7.0,30.0,18.0,7.0,30.0,18.0,7.0,30.0,18.0,7.0,30.0,18.0,7.0,30.0,18.0,7.0,25.0,23.5,7.0,30.0,18.0,7.0,7.0,23.5,18.0,7.0,30.0,7.0,7.0,30.0,18.0,7.0,30.0,18.0 +1.0,acc,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.906,0.907,0.906,0.906,0.906,0.906,0.906,0.907,0.906,0.906,0.906,0.906,0.906,0.906,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,1.5,18.5,18.5,18.5,18.5,18.5,1.5,18.5,18.5,18.5,18.5,18.5,18.5 +1.0,auc,0.829,0.757,0.793,0.829,0.757,0.792,0.829,0.757,0.792,0.829,0.757,0.792,0.829,0.757,0.792,0.823,0.775,0.787,0.829,0.757,0.793,0.828,0.823,0.802,0.8,0.829,0.757,0.83,0.829,0.757,0.792,0.828,0.757,0.792,5.5,30.0,16.5,5.5,30.0,20.5,5.5,30.0,20.5,5.5,30.0,20.5,5.5,30.0,20.5,12.5,25.0,24.0,5.5,30.0,16.5,10.5,12.5,14.0,15.0,5.5,30.0,1.0,5.5,30.0,20.5,10.5,30.0,20.5 +1.0,f1,0.371,0.302,0.339,0.371,0.302,0.342,0.371,0.302,0.337,0.371,0.302,0.338,0.372,0.302,0.342,0.364,0.322,0.327,0.371,0.302,0.339,0.369,0.364,0.336,0.345,0.371,0.302,0.375,0.371,0.302,0.337,0.369,0.302,0.337,6.0,30.0,17.5,6.0,30.0,15.5,6.0,30.0,21.0,6.0,30.0,19.0,2.0,30.0,15.5,12.5,25.0,24.0,6.0,30.0,17.5,10.5,12.5,23.0,14.0,6.0,30.0,1.0,6.0,30.0,21.0,10.5,30.0,21.0 +1.0,logloss,0.246,0.281,0.263,0.246,0.281,0.264,0.246,0.281,0.264,0.246,0.281,0.264,0.246,0.281,0.264,0.249,0.276,0.265,0.246,0.281,0.263,0.247,0.249,0.26,0.257,0.246,0.281,0.245,0.246,0.281,0.264,0.246,0.281,0.264,6.0,30.0,16.5,6.0,30.0,20.5,6.0,30.0,20.5,6.0,30.0,20.5,6.0,30.0,20.5,12.5,25.0,24.0,6.0,30.0,16.5,11.0,12.5,15.0,14.0,6.0,30.0,1.0,6.0,30.0,20.5,6.0,30.0,20.5 +1.0,mse,0.074,0.08,0.078,0.074,0.08,0.078,0.074,0.08,0.078,0.074,0.08,0.078,0.074,0.08,0.078,0.075,0.081,0.078,0.074,0.08,0.078,0.075,0.075,0.078,0.077,0.074,0.08,0.074,0.074,0.08,0.078,0.074,0.08,0.078,5.5,29.0,19.5,5.5,29.0,19.5,5.5,29.0,19.5,5.5,29.0,19.5,5.5,29.0,19.5,12.0,34.0,19.5,5.5,29.0,19.5,12.0,12.0,19.5,14.0,5.5,29.0,5.5,5.5,29.0,19.5,5.5,29.0,19.5 +2.0,acc,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.908,0.909,0.908,0.908,0.908,0.908,0.908,0.91,0.908,0.908,0.908,0.908,0.908,0.908,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,18.5,2.0,18.5,18.5,18.5,18.5,18.5,1.0,18.5,18.5,18.5,18.5,18.5,18.5 +2.0,auc,0.827,0.763,0.796,0.827,0.763,0.798,0.827,0.763,0.797,0.827,0.763,0.797,0.827,0.763,0.798,0.824,0.762,0.802,0.827,0.763,0.796,0.827,0.824,0.812,0.811,0.827,0.763,0.828,0.827,0.763,0.797,0.826,0.763,0.797,6.0,29.0,23.5,6.0,29.0,17.5,6.0,29.0,20.5,6.0,29.0,20.5,6.0,29.0,17.5,12.5,34.0,16.0,6.0,29.0,23.5,6.0,12.5,14.0,15.0,6.0,29.0,1.0,6.0,29.0,20.5,11.0,29.0,20.5 +2.0,f1,0.376,0.312,0.345,0.376,0.312,0.352,0.376,0.312,0.344,0.376,0.312,0.343,0.374,0.312,0.352,0.37,0.313,0.343,0.376,0.312,0.345,0.371,0.37,0.354,0.354,0.376,0.312,0.371,0.376,0.312,0.344,0.374,0.312,0.344,4.0,30.0,18.5,4.0,30.0,16.5,4.0,30.0,21.0,4.0,30.0,23.5,8.5,30.0,16.5,12.5,25.0,23.5,4.0,30.0,18.5,10.5,12.5,14.5,14.5,4.0,30.0,10.5,4.0,30.0,21.0,8.5,30.0,21.0 +2.0,logloss,0.245,0.279,0.26,0.245,0.279,0.259,0.245,0.279,0.259,0.245,0.279,0.259,0.245,0.279,0.259,0.246,0.282,0.256,0.245,0.279,0.26,0.245,0.246,0.253,0.252,0.245,0.279,0.244,0.245,0.279,0.259,0.245,0.279,0.259,6.5,29.0,23.5,6.5,29.0,19.5,6.5,29.0,19.5,6.5,29.0,19.5,6.5,29.0,19.5,12.5,34.0,16.0,6.5,29.0,23.5,6.5,12.5,15.0,14.0,6.5,29.0,1.0,6.5,29.0,19.5,6.5,29.0,19.5 +2.0,mse,0.073,0.079,0.076,0.073,0.079,0.076,0.073,0.079,0.076,0.073,0.079,0.076,0.073,0.079,0.076,0.073,0.08,0.075,0.073,0.079,0.076,0.073,0.073,0.074,0.074,0.073,0.079,0.072,0.073,0.079,0.076,0.073,0.079,0.076,7.5,29.0,20.5,7.5,29.0,20.5,7.5,29.0,20.5,7.5,29.0,20.5,7.5,29.0,20.5,7.5,34.0,16.0,7.5,29.0,20.5,7.5,7.5,14.5,14.5,7.5,29.0,1.0,7.5,29.0,20.5,7.5,29.0,20.5 +3.0,acc,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.903,0.904,0.903,0.903,0.903,0.903,0.903,0.904,0.903,0.903,0.903,0.904,0.903,0.903,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,19.0,2.0,19.0,19.0,19.0,19.0,19.0,2.0,19.0,19.0,19.0,2.0,19.0,19.0 +3.0,auc,0.826,0.755,0.794,0.826,0.755,0.795,0.826,0.755,0.795,0.826,0.755,0.795,0.826,0.755,0.795,0.825,0.77,0.801,0.826,0.755,0.794,0.823,0.825,0.804,0.81,0.826,0.755,0.826,0.826,0.755,0.795,0.827,0.755,0.795,6.0,30.0,23.5,6.0,30.0,19.5,6.0,30.0,19.5,6.0,30.0,19.5,6.0,30.0,19.5,11.5,25.0,16.0,6.0,30.0,23.5,13.0,11.5,15.0,14.0,6.0,30.0,6.0,6.0,30.0,19.5,1.0,30.0,19.5 +3.0,f1,0.373,0.307,0.34,0.373,0.307,0.348,0.373,0.307,0.345,0.373,0.307,0.345,0.372,0.307,0.348,0.362,0.334,0.342,0.373,0.307,0.34,0.366,0.362,0.348,0.349,0.373,0.307,0.368,0.373,0.307,0.345,0.373,0.307,0.345,4.5,30.0,23.5,4.5,30.0,16.0,4.5,30.0,19.5,4.5,30.0,19.5,9.0,30.0,16.0,12.5,25.0,22.0,4.5,30.0,23.5,11.0,12.5,16.0,14.0,4.5,30.0,10.0,4.5,30.0,19.5,4.5,30.0,19.5 +3.0,logloss,0.251,0.288,0.268,0.251,0.288,0.268,0.251,0.288,0.268,0.251,0.288,0.268,0.251,0.288,0.268,0.252,0.284,0.264,0.251,0.288,0.268,0.254,0.252,0.263,0.258,0.251,0.288,0.251,0.251,0.288,0.268,0.251,0.288,0.268,5.5,30.0,20.5,5.5,30.0,20.5,5.5,30.0,20.5,5.5,30.0,20.5,5.5,30.0,20.5,11.5,25.0,16.0,5.5,30.0,20.5,13.0,11.5,15.0,14.0,5.5,30.0,5.5,5.5,30.0,20.5,5.5,30.0,20.5 +3.0,mse,0.077,0.082,0.08,0.077,0.082,0.08,0.077,0.082,0.08,0.077,0.082,0.08,0.077,0.082,0.08,0.077,0.083,0.079,0.077,0.082,0.08,0.077,0.077,0.079,0.078,0.077,0.082,0.077,0.077,0.082,0.08,0.077,0.082,0.08,7.0,29.0,20.5,7.0,29.0,20.5,7.0,29.0,20.5,7.0,29.0,20.5,7.0,29.0,20.5,7.0,34.0,15.5,7.0,29.0,20.5,7.0,7.0,15.5,14.0,7.0,29.0,7.0,7.0,29.0,20.5,7.0,29.0,20.5 +4.0,acc,0.897,0.895,0.895,0.897,0.895,0.895,0.897,0.895,0.896,0.897,0.895,0.896,0.897,0.895,0.895,0.897,0.895,0.895,0.897,0.895,0.895,0.896,0.897,0.895,0.895,0.897,0.895,0.897,0.897,0.895,0.896,0.897,0.895,0.896,6.5,26.0,26.0,6.5,26.0,26.0,6.5,26.0,15.0,6.5,26.0,15.0,6.5,26.0,26.0,6.5,26.0,26.0,6.5,26.0,26.0,15.0,6.5,26.0,26.0,6.5,26.0,6.5,6.5,26.0,15.0,6.5,26.0,15.0 +4.0,auc,0.831,0.776,0.806,0.831,0.776,0.803,0.831,0.776,0.807,0.831,0.776,0.807,0.831,0.776,0.803,0.825,0.764,0.806,0.831,0.776,0.806,0.832,0.825,0.808,0.815,0.831,0.776,0.836,0.831,0.776,0.807,0.831,0.776,0.807,7.0,29.0,21.0,7.0,29.0,23.5,7.0,29.0,17.5,7.0,29.0,17.5,7.0,29.0,23.5,12.5,34.0,21.0,7.0,29.0,21.0,2.0,12.5,15.0,14.0,7.0,29.0,1.0,7.0,29.0,17.5,7.0,29.0,17.5 +4.0,f1,0.403,0.358,0.376,0.403,0.358,0.371,0.403,0.358,0.376,0.403,0.358,0.376,0.4,0.358,0.371,0.397,0.326,0.374,0.403,0.358,0.376,0.411,0.397,0.371,0.384,0.403,0.358,0.404,0.403,0.358,0.376,0.401,0.358,0.376,6.0,29.0,17.5,6.0,29.0,23.0,6.0,29.0,17.5,6.0,29.0,17.5,11.0,29.0,23.0,12.5,34.0,21.0,6.0,29.0,17.5,1.0,12.5,23.0,14.0,6.0,29.0,2.0,6.0,29.0,17.5,10.0,29.0,17.5 +4.0,logloss,0.263,0.3,0.276,0.263,0.3,0.277,0.263,0.3,0.275,0.263,0.3,0.275,0.263,0.3,0.277,0.266,0.304,0.276,0.263,0.3,0.276,0.264,0.266,0.276,0.271,0.263,0.3,0.261,0.263,0.3,0.275,0.263,0.3,0.275,6.0,29.0,20.5,6.0,29.0,23.5,6.0,29.0,16.5,6.0,29.0,16.5,6.0,29.0,23.5,12.5,34.0,20.5,6.0,29.0,20.5,11.0,12.5,20.5,14.0,6.0,29.0,1.0,6.0,29.0,16.5,6.0,29.0,16.5 +4.0,mse,0.08,0.087,0.082,0.08,0.087,0.083,0.08,0.087,0.082,0.08,0.087,0.082,0.08,0.087,0.083,0.08,0.089,0.083,0.08,0.087,0.082,0.08,0.08,0.083,0.082,0.08,0.087,0.079,0.08,0.087,0.082,0.08,0.087,0.082,7.5,29.0,17.0,7.5,29.0,22.5,7.5,29.0,17.0,7.5,29.0,17.0,7.5,29.0,22.5,7.5,34.0,22.5,7.5,29.0,17.0,7.5,7.5,22.5,17.0,7.5,29.0,1.0,7.5,29.0,17.0,7.5,29.0,17.0 diff --git a/assignments/requirements.txt b/assignments/requirements.txt index 8534fb6..240f3a7 100644 --- a/assignments/requirements.txt +++ b/assignments/requirements.txt @@ -1,12 +1,12 @@ # Python 3.6.9 (default, Jan 26 2021, 15:33:00) # [GCC 8.4.0] on linux - +#Cython==0.29.36 h2o==3.32.1.3 interpret==0.2.4 jupyter==1.0.0 matplotlib==3.3.4 -numpy==1.19.5 +#numpy==1.19.5 pandas==1.1.5 -scikit-learn==0.24.2 +#scikit-learn==0.24.2 seaborn==0.11.1 xgboost==1.4.2 diff --git a/tex/lecture_7.bib b/tex/lecture_7.bib index d89c081..5c6c043 100644 --- a/tex/lecture_7.bib +++ b/tex/lecture_7.bib @@ -66,10 +66,21 @@ @misc{ai2024artificial title={{Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile}}, author={NIST, AI}, year={2024}, - publisher={NIST, Gaithersburg, MD, USA}, + publisher={NIST, Gaithersburg MD}, note={URL: \url{https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf}} } +@inproceedings{li2024task, + title={{Task Contamination: Language Models May Not be Few-shot Anymore}}, + author={Li, Changmao and Flanigan, Jeffrey}, + booktitle={Proceedings of the AAAI Conference on Artificial Intelligence}, + volume={38}, + number={16}, + pages={18471--18480}, + year={2024}, + note={URL: \url{https://ojs.aaai.org/index.php/AAAI/article/view/29808}} +} + @article{hasan2022algorithmic, title={Algorithmic {B}ias and {R}isk {A}ssessments: {L}essons from {P}ractice}, author={Hasan, Ali and Brown, Shea and Davidovic, Jovana and Lange, Benjamin and Regan, Mitt}, diff --git a/tex/lecture_7.pdf b/tex/lecture_7.pdf index 5e8a872..51f7880 100644 Binary files a/tex/lecture_7.pdf and b/tex/lecture_7.pdf differ diff --git a/tex/lecture_7.tex b/tex/lecture_7.tex index d32c476..19be477 100644 --- a/tex/lecture_7.tex +++ b/tex/lecture_7.tex @@ -332,7 +332,7 @@ \end{table} \centering - \scriptsize{Source: "DQI: Measuring Data Quality in NLP,” \\ \url{https://arxiv.org/pdf/2005.00816.pdf}. (\cite{mishra2020dqi})} + \scriptsize{Source: "DQI: Measuring Data Quality in NLP,”\\\url{https://arxiv.org/pdf/2005.00816.pdf}. (\cite{mishra2020dqi})} \end{frame} @@ -346,11 +346,12 @@ \column{0.5\linewidth} \vspace{-5pt} - \begin{itemize} + \begin{itemize}\small \item \textbf{BBQ}: Stereotypes in question answering. \item \textbf{Winogender}: LM output versus employment statistics. \item \textbf{Real toxicity prompts}: 100k prompts to elicit toxic output. \item \textbf{TruthfulQA}: Assess the ability to make true statements. + \item Beware of task contamination (\cite{li2024task}). \end{itemize} \column{0.5\linewidth} \centering @@ -575,46 +576,40 @@ \vspace{5pt} \centering \includegraphics[height=100pt]{../img/buzzer.png} - - \column{0.5\textwidth} - \textbf{YES:} - - \begin{columns} - - \column{0.25\textwidth} - \begin{itemize}\tiny - \item Abuse detection - \item Accessibility - \item Benchmarking - \item Citation - \item Clear instructions - \item Content filters - \item Content provenance - \item Data retention - \item Disclosure of AI interactions - \item Dynamic blocklists - \item Field-testing - \end{itemize} - - \column{0.25\textwidth} - \begin{itemize}\tiny - \item Ground truth training data - \item Kill switches - \item Incident response plans - \item Monitoring - \item Pre-approved responses - \item Rate-limiting/throttling - \item Retrieval augmented generation (RAG) approaches - \item Red-teaming - \item Session limits - \item Strong system prompts - \item User feedback mechanisms - \end{itemize} + - \end{columns} + \column{0.25\textwidth} + \begin{itemize}\tiny + \item Abuse detection + \item Accessibility + \item Benchmarking + \item Citation + \item Clear instructions + \item Content filters + \item Content provenance + \item Data retention + \item Disclosure of AI interactions + \item Dynamic blocklists + \item Field-testing + \end{itemize} + + \column{0.25\textwidth} + \begin{itemize}\tiny + \item Ground truth training data + \item Kill switches + \item Incident response plans + \item Monitoring + \item Pre-approved responses + \item Rate-limiting/throttling + \item Retrieval augmented generation (RAG) approaches + \item Red-teaming + \item Session limits + \item Strong system prompts + \item User feedback mechanisms + \end{itemize} \column{0.25\textwidth} - \textbf{NO:} + \textbf{Restrict:} \begin{itemize}\tiny \item Anonymous use \item Anthropomorphization @@ -626,9 +621,10 @@ \item Undisclosed data collection or secondary use \end{itemize} \vspace{5pt} - \tiny{Various sources, e.g.,\\ \cite{weidinger2022taxonomy}, \cite{ai2024artificial}.} \end{columns} + \vspace{10pt} + \tiny{Various sources, e.g., \cite{weidinger2022taxonomy}, \cite{ai2024artificial}.} \end{frame}