03102022_DDN+OneHot_AlphaD3M.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "17534b9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import math\n",
    "import numpy as np\n",
    "import time\n",
    "from collections import defaultdict\n",
    "\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "from sklearn.neural_network import MLPRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a2c8c0df",
   "metadata": {},
   "outputs": [],
   "source": [
    "AutoML_path = \"\"\n",
    "log_folder = \"alphad3m_log\"\n",
    "\n",
    "log_DDN = AutoML_path+log_folder+'/DDN_single_time'\n",
    "log_pipeline_steps = AutoML_path+log_folder+'/pipeline_steps_class_02272022'\n",
    "\n",
    "combination_name = 'DDN+OneHot'\n",
    "log_target_test_pred_time = AutoML_path+log_folder+'/'+combination_name+'_target_test_pred_time_03102022'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "62870f07",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n",
      "7811\n"
     ]
    }
   ],
   "source": [
    "DDN = pickle.load(open(log_DDN, \"rb\"))\n",
    "print(len(DDN))\n",
    "\n",
    "pipeline_steps_class = pickle.load(open(log_pipeline_steps, 'rb'))\n",
    "print(len(pipeline_steps_class))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "65dba102",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================================\n",
      "0 adult\n",
      "X_train shape: (1526, 37) y_train shape: (1526,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.05 MAE: 0.1123323642799455 MSE: 0.024389497660162088\n",
      "Time: 0.49362921714782715\n",
      "X_train shape: (3405, 37) y_train shape: (3405,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.1 MAE: 0.06914698993994284 MSE: 0.016638473128178764\n",
      "Time: 1.9894590377807617\n",
      "X_train shape: (5415, 37) y_train shape: (5415,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.15000000000000002 MAE: 0.11329970783920833 MSE: 0.02812003687537896\n",
      "Time: 2.40151309967041\n",
      "X_train shape: (5892, 37) y_train shape: (5892,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.2 MAE: 0.12775849858122465 MSE: 0.032847400600078305\n",
      "Time: 2.9129951000213623\n",
      "X_train shape: (6745, 37) y_train shape: (6745,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.25 MAE: 0.16479666403311172 MSE: 0.045430489424858604\n",
      "Time: 4.175504922866821\n",
      "X_train shape: (6931, 37) y_train shape: (6931,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.3 MAE: 0.10877026427213177 MSE: 0.02639556273502164\n",
      "Time: 6.671857833862305\n",
      "X_train shape: (7147, 37) y_train shape: (7147,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.35000000000000003 MAE: 0.11596965520535649 MSE: 0.02945750653931133\n",
      "Time: 8.316960096359253\n",
      "adult_DDN+OneHot_0.35 adult_DDN+OneHot_0.4\n",
      "36 36\n",
      "adult_DDN+OneHot_0.4 adult_DDN+OneHot_0.45\n",
      "36 36\n",
      "========================================\n",
      "1 airlines\n",
      "X_train shape: (5, 37) y_train shape: (5,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.05 MAE: 0.05358525664168222 MSE: 0.004356702333303501\n",
      "Time: 0.08202314376831055\n",
      "X_train shape: (1074, 37) y_train shape: (1074,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.1 MAE: 0.049883693950503204 MSE: 0.003537092418946147\n",
      "Time: 0.3020501136779785\n",
      "X_train shape: (1494, 37) y_train shape: (1494,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.15000000000000002 MAE: 0.038924579823745614 MSE: 0.0021457375081969836\n",
      "Time: 1.6938591003417969\n",
      "X_train shape: (2992, 37) y_train shape: (2992,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.2 MAE: 0.05769281144182798 MSE: 0.0051581443363921025\n",
      "Time: 1.5872738361358643\n",
      "airlines_DDN+OneHot_0.2 airlines_DDN+OneHot_0.25\n",
      "16 16\n",
      "X_train shape: (5174, 37) y_train shape: (5174,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.3 MAE: 0.2171152730396449 MSE: 0.05246013104249147\n",
      "Time: 4.423803091049194\n",
      "X_train shape: (5583, 37) y_train shape: (5583,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.35000000000000003 MAE: 0.05691579658132739 MSE: 0.004233847611707799\n",
      "Time: 1.4859941005706787\n",
      "X_train shape: (6859, 37) y_train shape: (6859,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.4 MAE: 0.11315072173852726 MSE: 0.01606026533508699\n",
      "Time: 6.204416990280151\n",
      "X_train shape: (7630, 37) y_train shape: (7630,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.45 MAE: 0.08464254893141292 MSE: 0.009707353814381234\n",
      "Time: 10.768063306808472\n",
      "========================================\n",
      "2 albert\n",
      "X_train shape: (397, 37) y_train shape: (397,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.05 MAE: 0.01861344136047094 MSE: 0.00042359841747009427\n",
      "Time: 0.18005776405334473\n",
      "X_train shape: (1250, 37) y_train shape: (1250,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.1 MAE: 0.03095527329685599 MSE: 0.0012843280845143928\n",
      "Time: 0.3537919521331787\n",
      "X_train shape: (2102, 37) y_train shape: (2102,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.15000000000000002 MAE: 0.038645317437448944 MSE: 0.001935846186942625\n",
      "Time: 0.6852438449859619\n",
      "X_train shape: (3168, 37) y_train shape: (3168,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.2 MAE: 0.03446887194951616 MSE: 0.0012331974717955507\n",
      "Time: 5.099171161651611\n",
      "albert_DDN+OneHot_0.2 albert_DDN+OneHot_0.25\n",
      "16 16\n",
      "X_train shape: (5350, 37) y_train shape: (5350,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.3 MAE: 0.0536324306606006 MSE: 0.0035938841839343117\n",
      "Time: 4.679501056671143\n",
      "X_train shape: (6171, 37) y_train shape: (6171,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.35000000000000003 MAE: 0.026531774031871725 MSE: 0.0010378752976679504\n",
      "Time: 6.5262510776519775\n",
      "X_train shape: (7095, 37) y_train shape: (7095,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.4 MAE: 0.036811875438767905 MSE: 0.0017144386200965262\n",
      "Time: 14.095168113708496\n",
      "X_train shape: (7806, 37) y_train shape: (7806,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.45 MAE: 0.04883716635003209 MSE: 0.005179574120111318\n",
      "Time: 22.472357034683228\n",
      "========================================\n",
      "3 Amazon_employee_access\n",
      "X_train shape: (1763, 37) y_train shape: (1763,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.07176459768270771 MSE: 0.010594123852162731\n",
      "Time: 1.7926480770111084\n",
      "X_train shape: (3370, 37) y_train shape: (3370,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.06633691656717805 MSE: 0.009283319385651934\n",
      "Time: 5.3545708656311035\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.08849689784987019 MSE: 0.011189136010942399\n",
      "Time: 4.04745078086853\n",
      "X_train shape: (5060, 37) y_train shape: (5060,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.059218212924579856 MSE: 0.00744918428711523\n",
      "Time: 2.962477207183838\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.07830044344106686 MSE: 0.010342538690237898\n",
      "Time: 7.963721036911011\n",
      "Amazon_employee_access_DDN+OneHot_0.25 Amazon_employee_access_DDN+OneHot_0.3\n",
      "28 28\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.08696224665077555 MSE: 0.011159807912291398\n",
      "Time: 3.7565832138061523\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.06193746890840865 MSE: 0.00840664098888957\n",
      "Time: 9.066997051239014\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.07314939159527972 MSE: 0.009783755928356699\n",
      "Time: 6.724735975265503\n",
      "========================================\n",
      "4 APSFailure\n",
      "X_train shape: (1617, 37) y_train shape: (1617,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.05 MAE: 0.18582439848359006 MSE: 0.03694458259468118\n",
      "Time: 0.7746341228485107\n",
      "X_train shape: (2438, 37) y_train shape: (2438,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.1 MAE: 0.0600360092543787 MSE: 0.006544870700367345\n",
      "Time: 1.1568052768707275\n",
      "X_train shape: (4416, 37) y_train shape: (4416,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.15000000000000002 MAE: 0.1756792181012377 MSE: 0.03273685578505952\n",
      "Time: 2.2290329933166504\n",
      "APSFailure_DDN+OneHot_0.15 APSFailure_DDN+OneHot_0.2\n",
      "20 20\n",
      "X_train shape: (5258, 37) y_train shape: (5258,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.25 MAE: 0.19304610390471838 MSE: 0.04024268408634016\n",
      "Time: 2.1821861267089844\n",
      "X_train shape: (6326, 37) y_train shape: (6326,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.3 MAE: 0.1829421367628582 MSE: 0.037314572114276245\n",
      "Time: 7.031955242156982\n",
      "X_train shape: (6343, 37) y_train shape: (6343,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.35000000000000003 MAE: 0.23093459803173294 MSE: 0.05531928687343528\n",
      "Time: 5.113953113555908\n",
      "X_train shape: (7179, 37) y_train shape: (7179,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.4 MAE: 0.16974566290735238 MSE: 0.03627269970974219\n",
      "Time: 8.056227207183838\n",
      "X_train shape: (7365, 37) y_train shape: (7365,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.45 MAE: 0.2078645115940092 MSE: 0.04897818544804619\n",
      "Time: 10.996354103088379\n",
      "========================================\n",
      "5 Australian\n",
      "X_train shape: (2171, 37) y_train shape: (2171,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.10569194542024661 MSE: 0.014741382639509672\n",
      "Time: 1.2930431365966797\n",
      "X_train shape: (3447, 37) y_train shape: (3447,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.1 MAE: 0.06642734211465011 MSE: 0.00817984722992692\n",
      "Time: 2.0732412338256836\n",
      "X_train shape: (5708, 37) y_train shape: (5708,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.08926733031492702 MSE: 0.012622382077752894\n",
      "Time: 3.291149854660034\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.06819570039046231 MSE: 0.00876640307059963\n",
      "Time: 10.82988977432251\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.07558209908404041 MSE: 0.009921762618205678\n",
      "Time: 4.349292993545532\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.08115716140070087 MSE: 0.01075225912884819\n",
      "Time: 15.253896236419678\n",
      "Australian_DDN+OneHot_0.3 Australian_DDN+OneHot_0.35\n",
      "35 35\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.07191831196284616 MSE: 0.009378217702086664\n",
      "Time: 8.53903079032898\n",
      "Australian_DDN+OneHot_0.4 Australian_DDN+OneHot_0.45\n",
      "36 36\n",
      "========================================\n",
      "6 bank-marketing\n",
      "X_train shape: (1758, 37) y_train shape: (1758,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.05 MAE: 0.14587530991510184 MSE: 0.034670562645199965\n",
      "Time: 0.922825813293457\n",
      "X_train shape: (3759, 37) y_train shape: (3759,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.1 MAE: 0.1483711416416738 MSE: 0.03611941538757048\n",
      "Time: 3.184169054031372\n",
      "X_train shape: (4241, 37) y_train shape: (4241,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.15000000000000002 MAE: 0.16985602195744504 MSE: 0.04536382798698901\n",
      "Time: 3.0406200885772705\n",
      "X_train shape: (5719, 37) y_train shape: (5719,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.2 MAE: 0.15621013906074951 MSE: 0.03504398120101273\n",
      "Time: 8.139737844467163\n",
      "X_train shape: (5935, 37) y_train shape: (5935,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.25 MAE: 0.14539324305011264 MSE: 0.032065497271065495\n",
      "Time: 3.785789966583252\n",
      "X_train shape: (6788, 37) y_train shape: (6788,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.3 MAE: 0.1665211385990293 MSE: 0.042258577288814256\n",
      "Time: 9.144449949264526\n",
      "X_train shape: (6974, 37) y_train shape: (6974,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.35000000000000003 MAE: 0.14371715037868416 MSE: 0.03308254564140494\n",
      "Time: 5.196156740188599\n",
      "X_train shape: (7190, 37) y_train shape: (7190,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.4 MAE: 0.15361745611506886 MSE: 0.037501968082685\n",
      "Time: 5.95658016204834\n",
      "bank-marketing_DDN+OneHot_0.4 bank-marketing_DDN+OneHot_0.45\n",
      "36 36\n",
      "========================================\n",
      "7 blood-transfusion-service-center\n",
      "X_train shape: (1798, 37) y_train shape: (1798,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.05 MAE: 0.0594663797839789 MSE: 0.006103872995861032\n",
      "Time: 1.5147531032562256\n",
      "X_train shape: (3677, 37) y_train shape: (3677,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.1 MAE: 0.04751350531816674 MSE: 0.004440294303108389\n",
      "Time: 1.701948881149292\n",
      "X_train shape: (5471, 37) y_train shape: (5471,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.15000000000000002 MAE: 0.06806474652799474 MSE: 0.007006058889053457\n",
      "Time: 3.2649338245391846\n",
      "X_train shape: (6181, 37) y_train shape: (6181,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.2 MAE: 0.06680035737994786 MSE: 0.007008075903353137\n",
      "Time: 8.107470750808716\n",
      "X_train shape: (7017, 37) y_train shape: (7017,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.25 MAE: 0.05954831749642926 MSE: 0.005731954189621112\n",
      "Time: 6.340476989746094\n",
      "X_train shape: (7203, 37) y_train shape: (7203,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.3 MAE: 0.060965472710473555 MSE: 0.006210588055119743\n",
      "Time: 5.761718034744263\n",
      "X_train shape: (7419, 37) y_train shape: (7419,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.35000000000000003 MAE: 0.06319231109222774 MSE: 0.006878586667936795\n",
      "Time: 9.165446996688843\n",
      "blood-transfusion-service-center_DDN+OneHot_0.35 blood-transfusion-service-center_DDN+OneHot_0.4\n",
      "36 36\n",
      "X_train shape: (7427, 37) y_train shape: (7427,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.45 MAE: 0.062287163657745906 MSE: 0.0066466583395255255\n",
      "Time: 10.710065841674805\n",
      "========================================\n",
      "8 car\n",
      "X_train shape: (2121, 37) y_train shape: (2121,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.05 MAE: 0.11753669525876481 MSE: 0.026015009286684404\n",
      "Time: 0.9377398490905762\n",
      "X_train shape: (3613, 37) y_train shape: (3613,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.1 MAE: 0.12022505204851094 MSE: 0.026361621740235472\n",
      "Time: 2.56210994720459\n",
      "X_train shape: (5658, 37) y_train shape: (5658,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.15000000000000002 MAE: 0.08985235951956297 MSE: 0.018510980453329953\n",
      "Time: 4.423593044281006\n",
      "X_train shape: (6078, 37) y_train shape: (6078,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.2 MAE: 0.09876415159171395 MSE: 0.02024291781733523\n",
      "Time: 5.806262969970703\n",
      "X_train shape: (6931, 37) y_train shape: (6931,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.25 MAE: 0.11446639710374122 MSE: 0.02475356790769887\n",
      "Time: 10.837046146392822\n",
      "X_train shape: (7117, 37) y_train shape: (7117,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.3 MAE: 0.13298277252070853 MSE: 0.034081128321597584\n",
      "Time: 6.787929058074951\n",
      "X_train shape: (7333, 37) y_train shape: (7333,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.35000000000000003 MAE: 0.12923162742121472 MSE: 0.03147453315040872\n",
      "Time: 9.79129409790039\n",
      "car_DDN+OneHot_0.35 car_DDN+OneHot_0.4\n",
      "36 36\n",
      "car_DDN+OneHot_0.4 car_DDN+OneHot_0.45\n",
      "36 36\n",
      "========================================\n",
      "9 christine\n",
      "X_train shape: (852, 37) y_train shape: (852,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.1395676660259134 MSE: 0.02938271464482161\n",
      "Time: 0.3468658924102783\n",
      "X_train shape: (2547, 37) y_train shape: (2547,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.05192064056488831 MSE: 0.004019147949935034\n",
      "Time: 4.806427240371704\n",
      "X_train shape: (3581, 37) y_train shape: (3581,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.05851441026730773 MSE: 0.0047778244370780655\n",
      "Time: 7.3764119148254395\n",
      "X_train shape: (4927, 37) y_train shape: (4927,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.05732414938813916 MSE: 0.004716444121547442\n",
      "Time: 3.0256800651550293\n",
      "X_train shape: (6180, 37) y_train shape: (6180,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.06603673468424674 MSE: 0.00825971524003168\n",
      "Time: 3.302182912826538\n",
      "X_train shape: (6913, 37) y_train shape: (6913,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.05471881754312503 MSE: 0.00571729175218048\n",
      "Time: 9.744318008422852\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.07507576679538076 MSE: 0.009594370786217244\n",
      "Time: 7.9975199699401855\n",
      "christine_DDN+OneHot_0.35 christine_DDN+OneHot_0.4\n",
      "37 37\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.06525904718351522 MSE: 0.006513476135244136\n",
      "Time: 9.400846004486084\n",
      "========================================\n",
      "10 cnae-9\n",
      "X_train shape: (1549, 37) y_train shape: (1549,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.05 MAE: 0.1078569417189644 MSE: 0.0331156780599424\n",
      "Time: 0.5833430290222168\n",
      "X_train shape: (4238, 37) y_train shape: (4238,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.1 MAE: 0.09764823450821684 MSE: 0.030166578433462155\n",
      "Time: 2.3507769107818604\n",
      "cnae-9_DDN+OneHot_0.1 cnae-9_DDN+OneHot_0.15\n",
      "20 20\n",
      "X_train shape: (5296, 37) y_train shape: (5296,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.2 MAE: 0.09040910081738758 MSE: 0.027181319307959622\n",
      "Time: 3.0476560592651367\n",
      "X_train shape: (6148, 37) y_train shape: (6148,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.25 MAE: 0.09412155565336282 MSE: 0.027375075814472273\n",
      "Time: 9.009917259216309\n",
      "X_train shape: (6785, 37) y_train shape: (6785,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.3 MAE: 0.08919517678326525 MSE: 0.027362342277024918\n",
      "Time: 10.275615930557251\n",
      "X_train shape: (7006, 37) y_train shape: (7006,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.35000000000000003 MAE: 0.08763892227348696 MSE: 0.026185520639110123\n",
      "Time: 8.556665897369385\n",
      "X_train shape: (7187, 37) y_train shape: (7187,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.4 MAE: 0.09502145241874556 MSE: 0.028323145151372457\n",
      "Time: 7.905894994735718\n",
      "X_train shape: (7403, 37) y_train shape: (7403,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.45 MAE: 0.08597581658439783 MSE: 0.02496499664145131\n",
      "Time: 2.932888984680176\n",
      "========================================\n",
      "11 connect-4\n",
      "X_train shape: (1694, 37) y_train shape: (1694,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.20054716810247106 MSE: 0.04893220691150943\n",
      "Time: 2.7971091270446777\n",
      "X_train shape: (2547, 37) y_train shape: (2547,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.11766350389076756 MSE: 0.0220834117162879\n",
      "Time: 1.8276739120483398\n",
      "X_train shape: (4530, 37) y_train shape: (4530,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.12463729666934913 MSE: 0.02774651087238492\n",
      "Time: 7.2399890422821045\n",
      "X_train shape: (5336, 37) y_train shape: (5336,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.12068532739864096 MSE: 0.022752055141778277\n",
      "Time: 3.902803897857666\n",
      "X_train shape: (6612, 37) y_train shape: (6612,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.13712122535276453 MSE: 0.024103158794523162\n",
      "Time: 3.6697211265563965\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.12394347136091169 MSE: 0.027748770858391874\n",
      "Time: 10.401762008666992\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.18165675408081194 MSE: 0.04056305681724832\n",
      "Time: 5.373494863510132\n",
      "connect-4_DDN+OneHot_0.35 connect-4_DDN+OneHot_0.4\n",
      "37 37\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.19285612492766527 MSE: 0.04783028916761215\n",
      "Time: 7.336377859115601\n",
      "========================================\n",
      "12 covertype\n",
      "X_train shape: (1694, 37) y_train shape: (1694,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.07525517959638571 MSE: 0.013630737173126546\n",
      "Time: 2.044603109359741\n",
      "X_train shape: (1899, 37) y_train shape: (1899,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.08710841762994928 MSE: 0.018581932319156273\n",
      "Time: 2.1647512912750244\n",
      "X_train shape: (4525, 37) y_train shape: (4525,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.09032825890766319 MSE: 0.01810149478765772\n",
      "Time: 6.5947699546813965\n",
      "X_train shape: (5532, 37) y_train shape: (5532,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.09640730205727309 MSE: 0.016814404670596212\n",
      "Time: 6.785830020904541\n",
      "X_train shape: (6672, 37) y_train shape: (6672,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.0807710711293124 MSE: 0.011853268622679082\n",
      "Time: 4.650062799453735\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.09608745595282972 MSE: 0.022078330479863104\n",
      "Time: 7.478471279144287\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.09763553801550978 MSE: 0.018049428294082997\n",
      "Time: 9.659379005432129\n",
      "covertype_DDN+OneHot_0.35 covertype_DDN+OneHot_0.4\n",
      "37 37\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.07481823825401139 MSE: 0.01666512520332744\n",
      "Time: 15.659611701965332\n",
      "========================================\n",
      "13 credit-g\n",
      "X_train shape: (1284, 37) y_train shape: (1284,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.05 MAE: 0.09292189548536844 MSE: 0.014986356697328413\n",
      "Time: 2.4179141521453857\n",
      "X_train shape: (2564, 37) y_train shape: (2564,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.1 MAE: 0.08685176409702691 MSE: 0.016995974211539675\n",
      "Time: 4.486084938049316\n",
      "X_train shape: (4331, 37) y_train shape: (4331,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.15000000000000002 MAE: 0.12017884335932572 MSE: 0.025774133444933227\n",
      "Time: 2.9877099990844727\n",
      "X_train shape: (5770, 37) y_train shape: (5770,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.2 MAE: 0.11682795110338069 MSE: 0.01970901478886841\n",
      "Time: 5.7683000564575195\n",
      "X_train shape: (6923, 37) y_train shape: (6923,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.25 MAE: 0.15712616045767328 MSE: 0.03268571021486175\n",
      "Time: 8.109923124313354\n",
      "X_train shape: (7189, 37) y_train shape: (7189,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.3 MAE: 0.0685026688642351 MSE: 0.007975943686864101\n",
      "Time: 6.314311742782593\n",
      "credit-g_DDN+OneHot_0.3 credit-g_DDN+OneHot_0.35\n",
      "36 36\n",
      "X_train shape: (7197, 37) y_train shape: (7197,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.4 MAE: 0.06697258756481317 MSE: 0.008113464134840334\n",
      "Time: 6.228111982345581\n",
      "credit-g_DDN+OneHot_0.4 credit-g_DDN+OneHot_0.45\n",
      "37 37\n",
      "========================================\n",
      "14 dilbert\n",
      "X_train shape: (1547, 37) y_train shape: (1547,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.09526952613120969 MSE: 0.023273676539871654\n",
      "Time: 0.5767862796783447\n",
      "X_train shape: (4002, 37) y_train shape: (4002,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.09463577047457163 MSE: 0.021834705873778323\n",
      "Time: 4.561247110366821\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.09535277951191842 MSE: 0.023157307769275878\n",
      "Time: 4.040104866027832\n",
      "X_train shape: (5060, 37) y_train shape: (5060,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.09978636591163922 MSE: 0.022904293660621793\n",
      "Time: 2.5750620365142822\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.09791021615147949 MSE: 0.021439175054502088\n",
      "Time: 8.060142993927002\n",
      "X_train shape: (6145, 37) y_train shape: (6145,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.09117715478485541 MSE: 0.021719399444047067\n",
      "Time: 5.729160785675049\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.10134156753497184 MSE: 0.026241396595932623\n",
      "Time: 13.024019002914429\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.09091230976518817 MSE: 0.02015713848488412\n",
      "Time: 6.274775743484497\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.08849322716501759 MSE: 0.02030768220248053\n",
      "Time: 9.627941846847534\n",
      "========================================\n",
      "15 dionis\n",
      "X_train shape: (1490, 37) y_train shape: (1490,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.05 MAE: 0.4590563314234889 MSE: 0.22332986194564092\n",
      "Time: 0.611151933670044\n",
      "X_train shape: (2559, 37) y_train shape: (2559,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.1 MAE: 0.39821758334693774 MSE: 0.16949848815479765\n",
      "Time: 1.8401739597320557\n",
      "X_train shape: (3809, 37) y_train shape: (3809,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.15000000000000002 MAE: 0.4557985055693553 MSE: 0.21794912648143483\n",
      "Time: 2.1198620796203613\n",
      "X_train shape: (5348, 37) y_train shape: (5348,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.2 MAE: 0.4218106204348164 MSE: 0.188122613538145\n",
      "Time: 3.248914957046509\n",
      "X_train shape: (6408, 37) y_train shape: (6408,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.25 MAE: 0.4153016692236779 MSE: 0.1833815231531757\n",
      "Time: 4.274748802185059\n",
      "X_train shape: (7129, 37) y_train shape: (7129,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.3 MAE: 0.4382641694802927 MSE: 0.20349998261166677\n",
      "Time: 3.6702542304992676\n",
      "X_train shape: (7403, 37) y_train shape: (7403,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.35000000000000003 MAE: 0.34423793808398484 MSE: 0.12986565281274556\n",
      "Time: 5.566831827163696\n",
      "dionis_DDN+OneHot_0.35 dionis_DDN+OneHot_0.4\n",
      "37 37\n",
      "X_train shape: (7607, 37) y_train shape: (7607,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.45 MAE: 0.4331913859713162 MSE: 0.1973834470425489\n",
      "Time: 4.063852071762085\n",
      "========================================\n",
      "16 fabert\n",
      "X_train shape: (665, 37) y_train shape: (665,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.05 MAE: 0.13546537198888292 MSE: 0.0262237647034157\n",
      "Time: 0.3056797981262207\n",
      "X_train shape: (1919, 37) y_train shape: (1919,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.1 MAE: 0.12537842943344657 MSE: 0.021176386535802474\n",
      "Time: 0.7480959892272949\n",
      "X_train shape: (2977, 37) y_train shape: (2977,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.15000000000000002 MAE: 0.1399262473617496 MSE: 0.026218409900715316\n",
      "Time: 1.650205135345459\n",
      "fabert_DDN+OneHot_0.15 fabert_DDN+OneHot_0.2\n",
      "15 15\n",
      "X_train shape: (4963, 37) y_train shape: (4963,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.25 MAE: 0.10172613882774427 MSE: 0.016829509406675457\n",
      "Time: 5.031810998916626\n",
      "X_train shape: (5784, 37) y_train shape: (5784,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.3 MAE: 0.11147415623879474 MSE: 0.01928996947234828\n",
      "Time: 5.503465175628662\n",
      "X_train shape: (6912, 37) y_train shape: (6912,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.35000000000000003 MAE: 0.14210575526275906 MSE: 0.02719674570269031\n",
      "Time: 4.474220037460327\n",
      "X_train shape: (7623, 37) y_train shape: (7623,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.4 MAE: 0.1168289891321436 MSE: 0.021103032587914395\n",
      "Time: 10.490339994430542\n",
      "fabert_DDN+OneHot_0.4 fabert_DDN+OneHot_0.45\n",
      "38 38\n",
      "========================================\n",
      "17 fashion_mnist\n",
      "X_train shape: (1762, 37) y_train shape: (1762,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.186481817216705 MSE: 0.06710313857993015\n",
      "Time: 0.8485770225524902\n",
      "X_train shape: (3857, 37) y_train shape: (3857,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.16677488600604062 MSE: 0.05672815653706059\n",
      "Time: 2.117798089981079\n",
      "X_train shape: (5442, 37) y_train shape: (5442,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.1877925995918028 MSE: 0.06075577751616946\n",
      "Time: 5.420635938644409\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.15458187913090263 MSE: 0.04989590708374121\n",
      "Time: 4.809509754180908\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.19345535232178002 MSE: 0.0655962861825217\n",
      "Time: 7.7003350257873535\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.19023933832704418 MSE: 0.06598955232895877\n",
      "Time: 9.270923137664795\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.19341124188172326 MSE: 0.06506123795737792\n",
      "Time: 6.0040602684021\n",
      "fashion_mnist_DDN+OneHot_0.35 fashion_mnist_DDN+OneHot_0.4\n",
      "36 36\n",
      "fashion_mnist_DDN+OneHot_0.4 fashion_mnist_DDN+OneHot_0.45\n",
      "36 36\n",
      "========================================\n",
      "18 guillermo\n",
      "X_train shape: (836, 37) y_train shape: (836,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.05 MAE: 0.12477765328013488 MSE: 0.018871652349256075\n",
      "Time: 0.33595705032348633\n",
      "X_train shape: (2090, 37) y_train shape: (2090,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.1 MAE: 0.03130481085237061 MSE: 0.0017119698421464544\n",
      "Time: 0.5791101455688477\n",
      "X_train shape: (3148, 37) y_train shape: (3148,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.15000000000000002 MAE: 0.04805996537285664 MSE: 0.003801793908613078\n",
      "Time: 4.940329074859619\n",
      "X_train shape: (3544, 37) y_train shape: (3544,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.2 MAE: 0.05875152609653439 MSE: 0.004882508018729731\n",
      "Time: 5.336280107498169\n",
      "X_train shape: (5134, 37) y_train shape: (5134,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.25 MAE: 0.05299939216049498 MSE: 0.00408226807100372\n",
      "Time: 9.0887610912323\n",
      "X_train shape: (6171, 37) y_train shape: (6171,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.3 MAE: 0.05317398924932997 MSE: 0.004431058356472608\n",
      "Time: 9.157286167144775\n",
      "X_train shape: (6897, 37) y_train shape: (6897,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.35000000000000003 MAE: 0.06160949266339971 MSE: 0.005166386235020922\n",
      "Time: 3.7335739135742188\n",
      "X_train shape: (7794, 37) y_train shape: (7794,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.4 MAE: 0.06003298091197918 MSE: 0.005876358279849578\n",
      "Time: 7.914870738983154\n",
      "guillermo_DDN+OneHot_0.4 guillermo_DDN+OneHot_0.45\n",
      "38 38\n",
      "========================================\n",
      "19 helena\n",
      "X_train shape: (8, 37) y_train shape: (8,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.15000000000000002 MAE: 0.15037937441644847 MSE: 0.02637470666160187\n",
      "Time: 0.0472872257232666\n",
      "helena_DDN+OneHot_0.15 helena_DDN+OneHot_0.2\n",
      "1 1\n",
      "X_train shape: (224, 37) y_train shape: (224,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.25 MAE: 0.3437922496731777 MSE: 0.1224368322019159\n",
      "Time: 0.1380300521850586\n",
      "X_train shape: (410, 37) y_train shape: (410,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.3 MAE: 0.3396923721848842 MSE: 0.11923606019963566\n",
      "Time: 0.19714713096618652\n",
      "X_train shape: (1246, 37) y_train shape: (1246,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.35000000000000003 MAE: 0.33173080076114436 MSE: 0.11258140013723428\n",
      "Time: 0.354992151260376\n",
      "X_train shape: (1263, 37) y_train shape: (1263,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.4 MAE: 0.34013931202082026 MSE: 0.11829743857537774\n",
      "Time: 0.36428093910217285\n",
      "X_train shape: (2331, 37) y_train shape: (2331,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.45 MAE: 0.3339746305785937 MSE: 0.11392684994338549\n",
      "Time: 0.6717891693115234\n",
      "========================================\n",
      "20 higgs\n",
      "X_train shape: (642, 37) y_train shape: (642,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.11598801436418515 MSE: 0.015605307172317642\n",
      "Time: 0.22709989547729492\n",
      "X_train shape: (1675, 37) y_train shape: (1675,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.12508715003393092 MSE: 0.01981105067387812\n",
      "Time: 0.4969468116760254\n",
      "X_train shape: (2949, 37) y_train shape: (2949,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.1362952811232345 MSE: 0.020568723854938\n",
      "Time: 3.230396032333374\n",
      "higgs_DDN+OneHot_0.15 higgs_DDN+OneHot_0.2\n",
      "15 15\n",
      "X_train shape: (4935, 37) y_train shape: (4935,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.08482077345788346 MSE: 0.00936113502714427\n",
      "Time: 5.54935097694397\n",
      "X_train shape: (5560, 37) y_train shape: (5560,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.11458635042383933 MSE: 0.015230282607836453\n",
      "Time: 2.481088161468506\n",
      "X_train shape: (6824, 37) y_train shape: (6824,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.11874433895172919 MSE: 0.016127848247846584\n",
      "Time: 5.3227152824401855\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.11392464185720262 MSE: 0.015183831174638639\n",
      "Time: 6.914562940597534\n",
      "higgs_DDN+OneHot_0.4 higgs_DDN+OneHot_0.45\n",
      "38 38\n",
      "========================================\n",
      "21 jannis\n",
      "X_train shape: (637, 37) y_train shape: (637,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.07493060934262236 MSE: 0.007011000058576388\n",
      "Time: 0.28641200065612793\n",
      "X_train shape: (1675, 37) y_train shape: (1675,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.09636985522021897 MSE: 0.012298324089221692\n",
      "Time: 0.881537675857544\n",
      "X_train shape: (2949, 37) y_train shape: (2949,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.15000000000000002 MAE: 0.11415176992418649 MSE: 0.014355625608524488\n",
      "Time: 1.642387866973877\n",
      "jannis_DDN+OneHot_0.15 jannis_DDN+OneHot_0.2\n",
      "15 15\n",
      "X_train shape: (4935, 37) y_train shape: (4935,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.1521211316236691 MSE: 0.02684424543133441\n",
      "Time: 6.853058099746704\n",
      "X_train shape: (5756, 37) y_train shape: (5756,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.06653596102486421 MSE: 0.006726100448668422\n",
      "Time: 7.669067144393921\n",
      "X_train shape: (6884, 37) y_train shape: (6884,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.08336891830721861 MSE: 0.008833742644905865\n",
      "Time: 7.894777774810791\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.08859817680913419 MSE: 0.010211428574734194\n",
      "Time: 8.428193092346191\n",
      "jannis_DDN+OneHot_0.4 jannis_DDN+OneHot_0.45\n",
      "38 38\n",
      "========================================\n",
      "22 jasmine\n",
      "X_train shape: (1274, 37) y_train shape: (1274,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.10569600903721928 MSE: 0.017336487529823134\n",
      "Time: 0.5645618438720703\n",
      "X_train shape: (3240, 37) y_train shape: (3240,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.08223427166322253 MSE: 0.010833842493550929\n",
      "Time: 2.627603054046631\n",
      "X_train shape: (4934, 37) y_train shape: (4934,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.11315236778013175 MSE: 0.015950226220775465\n",
      "Time: 4.288399696350098\n",
      "X_train shape: (6396, 37) y_train shape: (6396,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.10727474592646961 MSE: 0.01572007161975679\n",
      "Time: 3.074254035949707\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.09282676579935446 MSE: 0.01206998849235734\n",
      "Time: 6.414793014526367\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.08876853834573924 MSE: 0.011014472185557943\n",
      "Time: 6.539883852005005\n",
      "jasmine_DDN+OneHot_0.3 jasmine_DDN+OneHot_0.35\n",
      "36 36\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.09880857634698746 MSE: 0.013033649133112554\n",
      "Time: 5.842039108276367\n",
      "jasmine_DDN+OneHot_0.4 jasmine_DDN+OneHot_0.45\n",
      "37 37\n",
      "========================================\n",
      "23 jungle_chess_2pcs_raw_endgame_complete\n",
      "X_train shape: (1478, 37) y_train shape: (1478,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.07237322439846985 MSE: 0.008955936095602654\n",
      "Time: 2.5903658866882324\n",
      "X_train shape: (2758, 37) y_train shape: (2758,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.07286073484563213 MSE: 0.00975455782589067\n",
      "Time: 5.22308087348938\n",
      "X_train shape: (4525, 37) y_train shape: (4525,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07672423016544055 MSE: 0.010580270027392416\n",
      "Time: 5.27967381477356\n",
      "X_train shape: (5964, 37) y_train shape: (5964,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.0968789941616554 MSE: 0.016336784526966568\n",
      "Time: 4.458581209182739\n",
      "X_train shape: (7117, 37) y_train shape: (7117,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.07224199376072589 MSE: 0.008491975716968483\n",
      "Time: 9.152125835418701\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.11081601040331808 MSE: 0.02187387350908374\n",
      "Time: 7.341161012649536\n",
      "jungle_chess_2pcs_raw_endgame_complete_DDN+OneHot_0.3 jungle_chess_2pcs_raw_endgame_complete_DDN+OneHot_0.35\n",
      "36 36\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.07656513832041979 MSE: 0.009491769215853025\n",
      "Time: 5.523070812225342\n",
      "jungle_chess_2pcs_raw_endgame_complete_DDN+OneHot_0.4 jungle_chess_2pcs_raw_endgame_complete_DDN+OneHot_0.45\n",
      "37 37\n",
      "========================================\n",
      "24 kc1\n",
      "X_train shape: (2171, 37) y_train shape: (2171,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.09705929784970067 MSE: 0.01226461350646311\n",
      "Time: 1.0255382061004639\n",
      "X_train shape: (3663, 37) y_train shape: (3663,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.04840901051459626 MSE: 0.004940781303445856\n",
      "Time: 2.4368908405303955\n",
      "X_train shape: (5708, 37) y_train shape: (5708,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07836160376115783 MSE: 0.009110795398821888\n",
      "Time: 2.7685649394989014\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.052806487891430734 MSE: 0.005074171533647991\n",
      "Time: 6.625546932220459\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.06376945916798915 MSE: 0.0076658434654327985\n",
      "Time: 2.836390972137451\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.0827850794914995 MSE: 0.009684348833161872\n",
      "Time: 7.1804039478302\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.07830628784064787 MSE: 0.008971470273066625\n",
      "Time: 12.3333261013031\n",
      "kc1_DDN+OneHot_0.35 kc1_DDN+OneHot_0.4\n",
      "36 36\n",
      "kc1_DDN+OneHot_0.4 kc1_DDN+OneHot_0.45\n",
      "36 36\n",
      "========================================\n",
      "25 KDDCup09_appetency\n",
      "X_train shape: (1196, 37) y_train shape: (1196,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.05 MAE: 0.16680883459787813 MSE: 0.02943875700210983\n",
      "Time: 0.5392560958862305\n",
      "X_train shape: (2449, 37) y_train shape: (2449,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.1 MAE: 0.057218566778642975 MSE: 0.005285145857428381\n",
      "Time: 1.15218186378479\n",
      "X_train shape: (3579, 37) y_train shape: (3579,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.15000000000000002 MAE: 0.06770458195980489 MSE: 0.0068726628383948135\n",
      "Time: 1.9834649562835693\n",
      "X_train shape: (4427, 37) y_train shape: (4427,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.2 MAE: 0.14933719593427355 MSE: 0.023491968795639518\n",
      "Time: 2.101673126220703\n",
      "X_train shape: (5269, 37) y_train shape: (5269,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.25 MAE: 0.13857584761046643 MSE: 0.020604500969171054\n",
      "Time: 2.2016377449035645\n",
      "X_train shape: (6337, 37) y_train shape: (6337,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.3 MAE: 0.11426366680106988 MSE: 0.013908365062970175\n",
      "Time: 8.791774988174438\n",
      "KDDCup09_appetency_DDN+OneHot_0.3 KDDCup09_appetency_DDN+OneHot_0.35\n",
      "28 28\n",
      "X_train shape: (7190, 37) y_train shape: (7190,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.4 MAE: 0.09165481220698 MSE: 0.00958938447651086\n",
      "Time: 8.612123012542725\n",
      "X_train shape: (7376, 37) y_train shape: (7376,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.45 MAE: 0.12131498495753858 MSE: 0.016625829107995153\n",
      "Time: 8.947980165481567\n",
      "========================================\n",
      "26 kr-vs-kp\n",
      "X_train shape: (505, 37) y_train shape: (505,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.05 MAE: 0.14469902941461177 MSE: 0.05187264374346584\n",
      "Time: 0.30634403228759766\n",
      "X_train shape: (1781, 37) y_train shape: (1781,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.1 MAE: 0.14203718211015834 MSE: 0.044388088110180295\n",
      "Time: 0.6915593147277832\n",
      "X_train shape: (3104, 37) y_train shape: (3104,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.15000000000000002 MAE: 0.11292988030166429 MSE: 0.025152353795096723\n",
      "Time: 1.023494005203247\n",
      "X_train shape: (4168, 37) y_train shape: (4168,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.2 MAE: 0.1401901027833471 MSE: 0.0463476894946485\n",
      "Time: 2.2426047325134277\n",
      "X_train shape: (4384, 37) y_train shape: (4384,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.25 MAE: 0.12011243287374129 MSE: 0.035969134472679694\n",
      "Time: 1.5305709838867188\n",
      "X_train shape: (5658, 37) y_train shape: (5658,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.3 MAE: 0.1484263054738434 MSE: 0.04749430459849408\n",
      "Time: 2.3480920791625977\n",
      "X_train shape: (6078, 37) y_train shape: (6078,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.35000000000000003 MAE: 0.12842742251738076 MSE: 0.04397798566833418\n",
      "Time: 5.264671087265015\n",
      "X_train shape: (6931, 37) y_train shape: (6931,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.4 MAE: 0.14988957253669685 MSE: 0.04072701574396188\n",
      "Time: 10.58340573310852\n",
      "X_train shape: (7117, 37) y_train shape: (7117,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.45 MAE: 0.12499472673440735 MSE: 0.040906086900307614\n",
      "Time: 4.716324090957642\n",
      "========================================\n",
      "27 mfeat-factors\n",
      "X_train shape: (1203, 37) y_train shape: (1203,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.07364852203060532 MSE: 0.014563340603694852\n",
      "Time: 0.7181389331817627\n",
      "X_train shape: (2240, 37) y_train shape: (2240,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.06921996294174507 MSE: 0.012148447589771303\n",
      "Time: 0.8908650875091553\n",
      "X_train shape: (3822, 37) y_train shape: (3822,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07395916137179658 MSE: 0.014497203114193846\n",
      "Time: 3.2583189010620117\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.0844505627032165 MSE: 0.01765633659824906\n",
      "Time: 2.491962194442749\n",
      "X_train shape: (5060, 37) y_train shape: (5060,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.06597605439045894 MSE: 0.012866231580689972\n",
      "Time: 2.3278470039367676\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.0627671299386029 MSE: 0.013101228675975076\n",
      "Time: 5.999495029449463\n",
      "mfeat-factors_DDN+OneHot_0.3 mfeat-factors_DDN+OneHot_0.35\n",
      "28 28\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.07387078474076095 MSE: 0.014767897762845674\n",
      "Time: 10.507073163986206\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.06767801592744509 MSE: 0.01276315456916272\n",
      "Time: 11.30379867553711\n",
      "========================================\n",
      "28 MiniBooNE\n",
      "X_train shape: (2171, 37) y_train shape: (2171,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.10439295413810405 MSE: 0.013871480810772299\n",
      "Time: 1.1392021179199219\n",
      "X_train shape: (3447, 37) y_train shape: (3447,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.06869884310357609 MSE: 0.008402430089691784\n",
      "Time: 2.2975308895111084\n",
      "X_train shape: (5708, 37) y_train shape: (5708,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.05372021692825491 MSE: 0.007560405991628131\n",
      "Time: 2.44209623336792\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.06928969352097919 MSE: 0.008408632318211392\n",
      "Time: 6.569070100784302\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.06084326474786663 MSE: 0.008062699909237995\n",
      "Time: 3.463298797607422\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.07033458594517593 MSE: 0.008542102389322695\n",
      "Time: 7.899287939071655\n",
      "MiniBooNE_DDN+OneHot_0.3 MiniBooNE_DDN+OneHot_0.35\n",
      "35 35\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.09617584167251977 MSE: 0.012224887169063637\n",
      "Time: 10.294243097305298\n",
      "MiniBooNE_DDN+OneHot_0.4 MiniBooNE_DDN+OneHot_0.45\n",
      "36 36\n",
      "========================================\n",
      "29 nomao\n",
      "X_train shape: (1565, 37) y_train shape: (1565,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.054050529030146434 MSE: 0.008571714011475561\n",
      "Time: 0.6547482013702393\n",
      "X_train shape: (3154, 37) y_train shape: (3154,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.06129026218307539 MSE: 0.007805198370837491\n",
      "Time: 1.422673225402832\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.04785827417961809 MSE: 0.00760499984651472\n",
      "Time: 3.6654300689697266\n",
      "X_train shape: (4434, 37) y_train shape: (4434,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.04884676980289062 MSE: 0.007066611797721302\n",
      "Time: 2.7501132488250732\n",
      "X_train shape: (5708, 37) y_train shape: (5708,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.046258565150247725 MSE: 0.0070156437101933046\n",
      "Time: 5.069075107574463\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.05946633848494959 MSE: 0.008345337380601673\n",
      "Time: 6.236254930496216\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.05134197057465109 MSE: 0.00864595141763128\n",
      "Time: 10.334859132766724\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.05856189726727869 MSE: 0.009336303988349741\n",
      "Time: 7.098556041717529\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.048848320642657 MSE: 0.008294229493798233\n",
      "Time: 7.087906122207642\n",
      "========================================\n",
      "30 numerai28.6\n",
      "X_train shape: (186, 37) y_train shape: (186,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.026377560974737417 MSE: 0.001032145001311401\n",
      "Time: 0.09250879287719727\n",
      "X_train shape: (1047, 37) y_train shape: (1047,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.04317691840865174 MSE: 0.0028178359193382916\n",
      "Time: 0.3794090747833252\n",
      "X_train shape: (1467, 37) y_train shape: (1467,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.16055823187264057 MSE: 0.02824848481860268\n",
      "Time: 2.0559141635894775\n",
      "X_train shape: (2945, 37) y_train shape: (2945,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.13400029615359813 MSE: 0.022422711487282383\n",
      "Time: 1.6221592426300049\n",
      "X_train shape: (3161, 37) y_train shape: (3161,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.151858211111732 MSE: 0.02637945762801788\n",
      "Time: 1.4417011737823486\n",
      "X_train shape: (4707, 37) y_train shape: (4707,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.10876621360213957 MSE: 0.015614509587460444\n",
      "Time: 6.708508729934692\n",
      "X_train shape: (5548, 37) y_train shape: (5548,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.12815008984791543 MSE: 0.01978211713232237\n",
      "Time: 2.5954620838165283\n",
      "X_train shape: (6824, 37) y_train shape: (6824,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.09180269554652509 MSE: 0.01114932860029277\n",
      "Time: 5.634044885635376\n",
      "========================================\n",
      "31 phoneme\n",
      "X_train shape: (1762, 37) y_train shape: (1762,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.05438528874678203 MSE: 0.005741810182926321\n",
      "Time: 0.901475191116333\n",
      "X_train shape: (3425, 37) y_train shape: (3425,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.04442872913815038 MSE: 0.0033880650007258534\n",
      "Time: 2.135906219482422\n",
      "X_train shape: (5435, 37) y_train shape: (5435,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.049167814461955214 MSE: 0.00419899213960232\n",
      "Time: 3.9668588638305664\n",
      "X_train shape: (6145, 37) y_train shape: (6145,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.051990169836322576 MSE: 0.004734536634882092\n",
      "Time: 5.383110046386719\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.052514466429058784 MSE: 0.0050983581136426264\n",
      "Time: 2.8910109996795654\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.04831433228597336 MSE: 0.004194550240817137\n",
      "Time: 10.07320785522461\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.049547459202681825 MSE: 0.004543821109536839\n",
      "Time: 9.727535963058472\n",
      "phoneme_DDN+OneHot_0.35 phoneme_DDN+OneHot_0.4\n",
      "36 36\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.45 MAE: 0.05112977444627499 MSE: 0.005265452786174086\n",
      "Time: 6.356205940246582\n",
      "========================================\n",
      "32 riccardo\n",
      "X_train shape: (1771, 37) y_train shape: (1771,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.05 MAE: 0.09552329030998875 MSE: 0.024018496236244906\n",
      "Time: 1.103269100189209\n",
      "X_train shape: (2396, 37) y_train shape: (2396,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.1 MAE: 0.09665336444909962 MSE: 0.023809098474688328\n",
      "Time: 1.1652050018310547\n",
      "X_train shape: (4374, 37) y_train shape: (4374,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.15000000000000002 MAE: 0.098282103267515 MSE: 0.02464643106191432\n",
      "Time: 2.4609522819519043\n",
      "riccardo_DDN+OneHot_0.15 riccardo_DDN+OneHot_0.2\n",
      "20 20\n",
      "X_train shape: (5432, 37) y_train shape: (5432,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.25 MAE: 0.0931239972138079 MSE: 0.022786291909156906\n",
      "Time: 3.072545051574707\n",
      "X_train shape: (6284, 37) y_train shape: (6284,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.3 MAE: 0.1083892566169383 MSE: 0.028941725887446486\n",
      "Time: 7.876476049423218\n",
      "X_train shape: (6921, 37) y_train shape: (6921,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.35000000000000003 MAE: 0.09635713545900533 MSE: 0.025538363747344375\n",
      "Time: 6.7019898891448975\n",
      "X_train shape: (7142, 37) y_train shape: (7142,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.4 MAE: 0.10084980268078407 MSE: 0.02395651371918562\n",
      "Time: 9.010563135147095\n",
      "X_train shape: (7323, 37) y_train shape: (7323,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.45 MAE: 0.09748881044724461 MSE: 0.02388762857739157\n",
      "Time: 5.9210240840911865\n",
      "========================================\n",
      "33 robert\n",
      "X_train shape: (420, 37) y_train shape: (420,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.15000000000000002 MAE: 0.1666629486343035 MSE: 0.03446418772703792\n",
      "Time: 0.3928971290588379\n",
      "X_train shape: (606, 37) y_train shape: (606,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.2 MAE: 0.20723110926513455 MSE: 0.04919884216397974\n",
      "Time: 0.47290587425231934\n",
      "X_train shape: (1459, 37) y_train shape: (1459,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.25 MAE: 0.3279466906667503 MSE: 0.11025576614940164\n",
      "Time: 0.9028949737548828\n",
      "robert_DDN+OneHot_0.25 robert_DDN+OneHot_0.3\n",
      "9 9\n",
      "X_train shape: (2527, 37) y_train shape: (2527,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.35000000000000003 MAE: 0.4216568236444991 MSE: 0.1829382113949962\n",
      "Time: 1.8625149726867676\n",
      "X_train shape: (3369, 37) y_train shape: (3369,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.4 MAE: 0.3181131974514002 MSE: 0.10467422404413249\n",
      "Time: 1.859515905380249\n",
      "X_train shape: (3765, 37) y_train shape: (3765,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.45 MAE: 0.2556352943350345 MSE: 0.06949754960420709\n",
      "Time: 1.5670690536499023\n",
      "========================================\n",
      "34 segment\n",
      "X_train shape: (1565, 37) y_train shape: (1565,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.07860692062688343 MSE: 0.013148231986544825\n",
      "Time: 0.7661559581756592\n",
      "X_train shape: (3154, 37) y_train shape: (3154,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.07889127913373926 MSE: 0.011532202422242023\n",
      "Time: 1.4624249935150146\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07589107371378968 MSE: 0.01023821441209037\n",
      "Time: 3.0605649948120117\n",
      "X_train shape: (4434, 37) y_train shape: (4434,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.06725953280473229 MSE: 0.010043886486947349\n",
      "Time: 3.1627557277679443\n",
      "X_train shape: (5912, 37) y_train shape: (5912,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.0780220252098814 MSE: 0.013662854996431564\n",
      "Time: 8.741522073745728\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.07659584443478862 MSE: 0.013029120074452604\n",
      "Time: 10.117765188217163\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.07223713777631849 MSE: 0.011806534492781186\n",
      "Time: 11.534343957901001\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.0747353290766951 MSE: 0.011139189455371368\n",
      "Time: 6.598075866699219\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.07219364589768244 MSE: 0.01097460438969732\n",
      "Time: 7.654994249343872\n",
      "========================================\n",
      "35 shuttle\n",
      "X_train shape: (999, 37) y_train shape: (999,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.05 MAE: 0.06876804824147349 MSE: 0.007903370187697325\n",
      "Time: 0.587597131729126\n",
      "X_train shape: (2252, 37) y_train shape: (2252,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.1 MAE: 0.06736961912908455 MSE: 0.006463997775103432\n",
      "Time: 1.1141271591186523\n",
      "X_train shape: (3166, 37) y_train shape: (3166,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.15000000000000002 MAE: 0.08744448652370707 MSE: 0.009514144865189282\n",
      "Time: 1.2793080806732178\n",
      "X_train shape: (4230, 37) y_train shape: (4230,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.2 MAE: 0.04281549218846573 MSE: 0.003664098996116925\n",
      "Time: 3.8810391426086426\n",
      "X_train shape: (5072, 37) y_train shape: (5072,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.25 MAE: 0.044964233331161 MSE: 0.0039758422816020605\n",
      "Time: 2.360275983810425\n",
      "X_train shape: (5924, 37) y_train shape: (5924,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.3 MAE: 0.043839716932687275 MSE: 0.003533603665269497\n",
      "Time: 6.170809984207153\n",
      "X_train shape: (6140, 37) y_train shape: (6140,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.35000000000000003 MAE: 0.048129083739606726 MSE: 0.003814083211244144\n",
      "Time: 4.308168888092041\n",
      "X_train shape: (6993, 37) y_train shape: (6993,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.4 MAE: 0.05697293907527469 MSE: 0.0048884697992991385\n",
      "Time: 6.477644920349121\n",
      "X_train shape: (7179, 37) y_train shape: (7179,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.45 MAE: 0.06494992449545464 MSE: 0.00593700680290821\n",
      "Time: 6.96119499206543\n",
      "========================================\n",
      "36 sylvine\n",
      "X_train shape: (1469, 37) y_train shape: (1469,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.06341833843022576 MSE: 0.011554898548049072\n",
      "Time: 0.7711849212646484\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.06501467899241875 MSE: 0.009979843902194009\n",
      "Time: 3.056950092315674\n",
      "sylvine_DDN+OneHot_0.1 sylvine_DDN+OneHot_0.15\n",
      "20 20\n",
      "X_train shape: (5492, 37) y_train shape: (5492,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.08424210896321185 MSE: 0.012488363730916293\n",
      "Time: 2.739567756652832\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.06077235188715849 MSE: 0.009996468146753631\n",
      "Time: 10.053106784820557\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.07008964788734662 MSE: 0.011092116570997314\n",
      "Time: 6.106781959533691\n",
      "X_train shape: (6986, 37) y_train shape: (6986,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.07525133080350106 MSE: 0.010952471199530451\n",
      "Time: 10.137291193008423\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.09474759687968509 MSE: 0.014507087934848333\n",
      "Time: 8.399434089660645\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.06259911882131901 MSE: 0.010191031780851213\n",
      "Time: 5.383616209030151\n",
      "========================================\n",
      "37 vehicle\n",
      "X_train shape: (1694, 37) y_train shape: (1694,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.12343161713928093 MSE: 0.03218890349228144\n",
      "Time: 4.6975390911102295\n",
      "X_train shape: (2547, 37) y_train shape: (2547,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.1 MAE: 0.1354151680379846 MSE: 0.036426356740787105\n",
      "Time: 5.916474103927612\n",
      "X_train shape: (4530, 37) y_train shape: (4530,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.28066264129723495 MSE: 0.09745956650608495\n",
      "Time: 6.202378988265991\n",
      "X_train shape: (5120, 37) y_train shape: (5120,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.17526553374907555 MSE: 0.05286029259450623\n",
      "Time: 9.583697319030762\n",
      "X_train shape: (6612, 37) y_train shape: (6612,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.22697466669031902 MSE: 0.07291554822869081\n",
      "Time: 9.01286792755127\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.1743961431579391 MSE: 0.05084612090608828\n",
      "Time: 9.038412094116211\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.21558628766566018 MSE: 0.06758248614301408\n",
      "Time: 10.374309301376343\n",
      "vehicle_DDN+OneHot_0.35 vehicle_DDN+OneHot_0.4\n",
      "37 37\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.26049847413205246 MSE: 0.0905043792740913\n",
      "Time: 10.428576946258545\n",
      "========================================\n",
      "38 volkert\n",
      "X_train shape: (637, 37) y_train shape: (637,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.08530380127644187 MSE: 0.012741509736188102\n",
      "Time: 0.28487205505371094\n",
      "X_train shape: (1675, 37) y_train shape: (1675,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.05641493054808007 MSE: 0.006256234609833077\n",
      "Time: 0.5385870933532715\n",
      "X_train shape: (2949, 37) y_train shape: (2949,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.09442328969577175 MSE: 0.014711534877269051\n",
      "Time: 5.3807761669158936\n",
      "volkert_DDN+OneHot_0.15 volkert_DDN+OneHot_0.2\n",
      "15 15\n",
      "X_train shape: (4935, 37) y_train shape: (4935,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.05680512147081567 MSE: 0.007102474132934713\n",
      "Time: 3.3408918380737305\n",
      "X_train shape: (5756, 37) y_train shape: (5756,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.08780731331327997 MSE: 0.012965317099243805\n",
      "Time: 9.90477204322815\n",
      "X_train shape: (6884, 37) y_train shape: (6884,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.07987797200205717 MSE: 0.01120662503410688\n",
      "Time: 6.292514085769653\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.07762240513180713 MSE: 0.011042077615903612\n",
      "Time: 7.579829931259155\n",
      "volkert_DDN+OneHot_0.4 volkert_DDN+OneHot_0.45\n",
      "38 38\n"
     ]
    }
   ],
   "source": [
    "target_test_pred_time = {}\n",
    "data_names = list(DDN.keys())\n",
    "data_embed = DDN\n",
    "\n",
    "for data_index, data_name in enumerate(data_names):\n",
    "    print(\"=\"*40)\n",
    "    print(data_index, data_name)\n",
    "    \n",
    "    task_test = [data_name]\n",
    "    candidate_score = DDN[data_name]\n",
    "    prev_key = None\n",
    "    dict_key = None\n",
    "    len_task_train = 0\n",
    "    prev_len_task_train = 0\n",
    "    for threshold in np.arange(0.05, 0.5, 0.05):\n",
    "        prev_key = dict_key\n",
    "        dict_key = '_'.join([data_name, combination_name, str(round(threshold,2))])\n",
    "        \n",
    "        start = time.time()\n",
    "        task_train = [i for i in data_names if i!=data_name and abs(DDN[i]-candidate_score)<threshold]\n",
    "        prev_len_task_train = len_task_train\n",
    "        len_task_train = len(task_train)\n",
    "        if len_task_train == 0:\n",
    "            target_test_pred_time[dict_key] = 'N.A.'\n",
    "            pickle.dump(target_test_pred_time, open(log_target_test_pred_time, \"wb\"))\n",
    "            continue\n",
    "            \n",
    "        if len_task_train == prev_len_task_train:\n",
    "            print(prev_key, dict_key)\n",
    "            print(prev_len_task_train, len_task_train)\n",
    "            target_test_pred_time[dict_key] =  target_test_pred_time[prev_key]\n",
    "            continue\n",
    "\n",
    "        X_train = np.array([np.hstack((data_embed[i[0]], i[1])) for i in pipeline_steps_class if i[0] in task_train])\n",
    "        y_train = np.array([i[3] for i in pipeline_steps_class if i[0] in task_train])\n",
    "        X_test = np.array([np.hstack((data_embed[i[0]], i[1])) for i in pipeline_steps_class if i[0] in task_test])\n",
    "        y_test = np.array([i[3] for i in pipeline_steps_class if i[0] in task_test])\n",
    "        print(\"X_train shape:\", X_train.shape, \"y_train shape:\", y_train.shape)\n",
    "        print(\"X_test shape:\", X_test.shape, \"y_test shape:\", y_test.shape)\n",
    "\n",
    "        model = MLPRegressor(hidden_layer_sizes=(256,128,64), learning_rate='adaptive', random_state=19)\n",
    "        model.fit(X_train, y_train)\n",
    "        end = time.time()\n",
    "        duration = end-start\n",
    "\n",
    "        y_pred = model.predict(X_test)\n",
    "        mae = mean_absolute_error(y_test, y_pred)\n",
    "        mse = mean_squared_error(y_test, y_pred)\n",
    "        print(\"Threshold\", threshold, \"MAE:\", mae, \"MSE:\", mse)\n",
    "        print(\"Time:\", duration)\n",
    "        target_test_pred_time[dict_key] = (y_test, y_pred, duration, mae, mse)\n",
    "        pickle.dump(target_test_pred_time, open(log_target_test_pred_time, 'wb'))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e9ee2c4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}