rfe-scratchpad.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%run default-imports.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unpickling object...\n"
     ]
    }
   ],
   "source": [
    "rfe_experiments = unpickle('./experiments/_experiments_rfe_mimic.d') or {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_experiment = {exp_id:experiment for exp_id, experiment in rfe_experiments.items() if experiment['parameters']['algorithm'] == algorithms.RF}\n",
    "gbdt_experiment = {exp_id:experiment for exp_id, experiment in rfe_experiments.items() if experiment['parameters']['algorithm'] == algorithms.GBDT}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_mimic = list(rf_experiment.values())[0]['weighted_explanations']\n",
    "gbdt_mimic = list(gbdt_experiment.values())[0]['weighted_explanations']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "[('SODIUM (WHOLE BLOOD)_DTS_2.0', 0.18037641542180127, 2, 0.09999999999999998), ('WBC COUNT_DTS_3.0', 0.19062148427152775, 2, 0.09999999999999998), ('SODIUM (WHOLE BLOOD)_DTS_3.0', 0.22152636552126703, 2, 0.09999999999999998), ('CHRONIC_PULMONARY', 0.36919849559487117, 3, 0.55), ('CHLORIDE (WHOLE BLOOD)_DTS_1.0', 0.3972840322018451, 3, 0.55), ('BLD CREA_DTS_2.0', 0.41919946442946365, 3, 0.55), ('BLD CREA_DTS_1.0', 0.5384068013717567, 3, 0.55), ('HEMATOCRIT (CALCULATED)_DTS_2.0', 0.5713505058097198, 4, 1.0), ('SODIUM_DTS_3.0', 0.5773239464719524, 4, 1.0), ('PULMONARY_CIRCULATION', 0.5782476118744765, 4, 1.0), ('PERIPHERAL_VASCULAR', 0.5791543024035244, 4, 1.0), ('ANION GAP_DTS_1.0', 0.58499927950758, 4, 1.0), ('HEMATOCRIT (CALCULATED)_DTS_3.0', 0.5853425780498953, 4, 1.0), ('HEMOGLOBIN_DTS_1.0', 0.5860810263849293, 4, 1.0), ('PT_DTS_2.0', 0.586845102006732, 4, 1.0), ('SODIUM (WHOLE BLOOD)_DTS_1.0', 0.5870680875378229, 4, 1.0), ('BLD CREA_DTS_3.0', 0.5872708135376746, 4, 1.0), ('HEMOGLOBIN (BLOOD GAS)_DTS_2.0', 0.5877096254714393, 4, 1.0), ('WHITE BLOOD CELLS_DTS_2.0', 0.5880645029336444, 4, 1.0), ('SODIUM_DTS_1.0', 0.5887116160303263, 4, 1.0), ('HYPOTHYROIDISM', 0.5899597977162533, 4, 1.0), ('WEIGHT_LOSS', 0.5902149542597162, 4, 1.0), ('LYMPHOMA', 0.5905138731720339, 4, 1.0), ('SOLID_TUMOR', 0.5918764792392105, 4, 1.0), ('CHLORIDE (WHOLE BLOOD)_DTS_2.0', 0.5923779961716268, 4, 1.0), ('METASTATIC_CANCER', 0.5925931818010837, 4, 1.0), ('INR(PT)_DTS_2.0', 0.5930594445326072, 4, 1.0), ('INR(PT)_DTS_1.0', 0.593649685078193, 4, 1.0), ('ANION GAP_DTS_3.0', 0.5939278312992541, 4, 1.0), ('ALBUMIN_DTS_3.0', 0.5940613768615645, 4, 1.0), ('ALBUMIN_DTS_2.0', 0.5943881341899634, 4, 1.0), ('HEMOGLOBIN (BLOOD GAS)_DTS_3.0', 0.594672852148422, 4, 1.0), ('HEMOGLOBIN (BLOOD GAS)_DTS_1.0', 0.5954189155373202, 4, 1.0), ('ALBUMIN_DTS_1.0', 0.5957825537176662, 4, 1.0), ('BLOOD_LOSS_ANEMIA', 0.5960127188648144, 4, 1.0), ('POTASSIUM WHOLE BLOOD_DTS_2.0', 0.5960582893659432, 4, 1.0), ('GLUCOSE_DTS_3.0', 0.5965469947089116, 4, 1.0), ('POTASSIUM WHOLE BLOOD_DTS_3.0', 0.5968994665216496, 4, 1.0), ('POTASSIUM_DTS_1.0', 0.5976568521477774, 4, 1.0), ('GLUCOSE (BLOOD GAS)_DTS_1.0', 0.5977855492755596, 4, 1.0), ('HEMATOCRIT_DTS_2.0', 0.5979186457897648, 4, 1.0), ('PTT_DTS_3.0', 0.5979630675637406, 4, 1.0), ('DIABETES_UNCOMPLICATED', 0.5980569958468995, 4, 1.0), ('INR(PT)_DTS_3.0', 0.5984030823553514, 4, 1.0), ('GLUCOSE (BLOOD GAS)_DTS_3.0', 0.5988823129485366, 4, 1.0), ('LACTATE_DTS_3.0', 0.5989876494043453, 4, 1.0), ('RHEUMATOID_ARTHRITIS', 0.5990682580892801, 4, 1.0), ('PLATELET COUNT_DTS_3.0', 0.5994370535836974, 4, 1.0), ('HEMATOCRIT_DTS_3.0', 0.5995222203408211, 4, 1.0), ('HEMATOCRIT_DTS_1.0', 0.6009772420455746, 4, 1.0), ('DEFICIENCY_ANEMIAS', 0.6018022491670603, 4, 1.0), ('PARALYSIS', 0.6025869692490611, 4, 1.0), ('DEPRESSION', 0.6038051810229915, 4, 1.0), ('HEMATOCRIT (CALCULATED)_DTS_1.0', 0.6042180967982665, 4, 1.0), ('ANION GAP_DTS_2.0', 0.6045219548316285, 4, 1.0), ('AGE_AT_ADMISSION', 0.604573295336578, 4, 1.0), ('LACTATE_DTS_2.0', 0.605493431648638, 4, 1.0), ('DIABETES_COMPLICATED', 0.6057009198379292, 4, 1.0), ('GLUCOSE_DTS_1.0', 0.6058181444971478, 4, 1.0), ('CHLORIDE (WHOLE BLOOD)_DTS_3.0', 0.6060061927156253, 4, 1.0), ('POTASSIUM_DTS_3.0', 0.6065259605479497, 4, 1.0), ('GLUCOSE_DTS_2.0', 0.6071019335313957, 4, 1.0), ('PLATELET COUNT_DTS_2.0', 0.6077362530979545, 4, 1.0), ('ALCOHOL_ABUSE', 0.6082588036587588, 4, 1.0), ('POTASSIUM WHOLE BLOOD_DTS_1.0', 0.6088908182041655, 4, 1.0), ('SODIUM_DTS_2.0', 0.611001533109988, 4, 1.0), ('HEMOGLOBIN_DTS_2.0', 0.6112810623268521, 4, 1.0), ('CHLORIDE_DTS_1.0', 0.6118663574484896, 4, 1.0), ('PTT_DTS_2.0', 0.6124154330445504, 4, 1.0), ('GLUCOSE (BLOOD GAS)_DTS_2.0', 0.6125875267046968, 4, 1.0), ('PEPTIC_ULCER', 0.6129987637547074, 4, 1.0), ('DRUG_ABUSE', 0.6131424962300277, 4, 1.0), ('PTT_DTS_1.0', 0.6140625145136058, 4, 1.0), ('HEMOGLOBIN_DTS_3.0', 0.6141673727574221, 4, 1.0), ('WHITE BLOOD CELLS_DTS_3.0', 0.6144469096492974, 4, 1.0), ('OTHER_NEUROLOGICAL', 0.6177074513040492, 4, 1.0), ('CHLORIDE_DTS_2.0', 0.6178234824823502, 4, 1.0), ('WHITE BLOOD CELLS_DTS_1.0', 0.62578397353109, 4, 1.0), ('PLATELET COUNT_DTS_1.0', 0.6271540863712561, 4, 1.0), ('BILIRUBIN_DTS_2.0', 0.6294474403207349, 4, 1.0), ('PSYCHOSES', 0.6314747666265117, 4, 1.0), ('GENDER', 0.6326503028289518, 4, 1.0), ('PT_DTS_3.0', 0.6340727527257801, 4, 1.0), ('POTASSIUM_DTS_2.0', 0.6357899568978573, 4, 1.0), ('BILIRUBIN_DTS_1.0', 0.6359236649602806, 4, 1.0), ('BUN_DTS_3.0', 0.6417880773747808, 4, 1.0), ('PT_DTS_1.0', 0.6434905672645292, 4, 1.0), ('HYPERTENSION', 0.6435080821697543, 4, 1.0), ('CHLORIDE_DTS_3.0', 0.6443726521088508, 4, 1.0), ('LACTATE_DTS_1.0', 0.6463278742615005, 4, 1.0), ('OBESITY', 0.6660373277658679, 4, 1.0), ('CONGESTIVE_HEART_FAILURE', 0.6696603389506195, 4, 1.0), ('BILIRUBIN_DTS_3.0', 0.6704023284623936, 4, 1.0), ('BUN_DTS_1.0', 0.6794528839464279, 4, 1.0), ('LIVER_DISEASE', 0.7073906381827205, 4, 1.0), ('AIDS', 0.724555455083373, 4, 1.0), ('CARDIAC_ARRHYTHMIAS', 0.7289939841333126, 4, 1.0), ('VALVULAR_DISEASE', 0.7370261595513858, 4, 1.0), ('RENAL_FAILURE', 0.7392650041090106, 4, 1.0), ('BUN_DTS_2.0', 0.748206831584104, 4, 1.0), ('COAGULOPATHY', 0.7583844053348038, 4, 1.0), ('ELIXHAUSER_SCORE', 0.768994094757506, 4, 1.0), ('FLUID_ELECTROLYTE', 0.9018339610500634, 4, 1.0)]"
      ],
      "text/plain": [
       "[('SODIUM (WHOLE BLOOD)_DTS_2.0', 0.18037641542180127, 2, 0.09999999999999998),\n",
       " ('WBC COUNT_DTS_3.0', 0.19062148427152775, 2, 0.09999999999999998),\n",
       " ('SODIUM (WHOLE BLOOD)_DTS_3.0', 0.22152636552126703, 2, 0.09999999999999998),\n",
       " ('CHRONIC_PULMONARY', 0.36919849559487117, 3, 0.55),\n",
       " ('CHLORIDE (WHOLE BLOOD)_DTS_1.0', 0.3972840322018451, 3, 0.55),\n",
       " ('BLD CREA_DTS_2.0', 0.41919946442946365, 3, 0.55),\n",
       " ('BLD CREA_DTS_1.0', 0.5384068013717567, 3, 0.55),\n",
       " ('HEMATOCRIT (CALCULATED)_DTS_2.0', 0.5713505058097198, 4, 1.0),\n",
       " ('SODIUM_DTS_3.0', 0.5773239464719524, 4, 1.0),\n",
       " ('PULMONARY_CIRCULATION', 0.5782476118744765, 4, 1.0),\n",
       " ('PERIPHERAL_VASCULAR', 0.5791543024035244, 4, 1.0),\n",
       " ('ANION GAP_DTS_1.0', 0.58499927950758, 4, 1.0),\n",
       " ('HEMATOCRIT (CALCULATED)_DTS_3.0', 0.5853425780498953, 4, 1.0),\n",
       " ('HEMOGLOBIN_DTS_1.0', 0.5860810263849293, 4, 1.0),\n",
       " ('PT_DTS_2.0', 0.586845102006732, 4, 1.0),\n",
       " ('SODIUM (WHOLE BLOOD)_DTS_1.0', 0.5870680875378229, 4, 1.0),\n",
       " ('BLD CREA_DTS_3.0', 0.5872708135376746, 4, 1.0),\n",
       " ('HEMOGLOBIN (BLOOD GAS)_DTS_2.0', 0.5877096254714393, 4, 1.0),\n",
       " ('WHITE BLOOD CELLS_DTS_2.0', 0.5880645029336444, 4, 1.0),\n",
       " ('SODIUM_DTS_1.0', 0.5887116160303263, 4, 1.0),\n",
       " ('HYPOTHYROIDISM', 0.5899597977162533, 4, 1.0),\n",
       " ('WEIGHT_LOSS', 0.5902149542597162, 4, 1.0),\n",
       " ('LYMPHOMA', 0.5905138731720339, 4, 1.0),\n",
       " ('SOLID_TUMOR', 0.5918764792392105, 4, 1.0),\n",
       " ('CHLORIDE (WHOLE BLOOD)_DTS_2.0', 0.5923779961716268, 4, 1.0),\n",
       " ('METASTATIC_CANCER', 0.5925931818010837, 4, 1.0),\n",
       " ('INR(PT)_DTS_2.0', 0.5930594445326072, 4, 1.0),\n",
       " ('INR(PT)_DTS_1.0', 0.593649685078193, 4, 1.0),\n",
       " ('ANION GAP_DTS_3.0', 0.5939278312992541, 4, 1.0),\n",
       " ('ALBUMIN_DTS_3.0', 0.5940613768615645, 4, 1.0),\n",
       " ('ALBUMIN_DTS_2.0', 0.5943881341899634, 4, 1.0),\n",
       " ('HEMOGLOBIN (BLOOD GAS)_DTS_3.0', 0.594672852148422, 4, 1.0),\n",
       " ('HEMOGLOBIN (BLOOD GAS)_DTS_1.0', 0.5954189155373202, 4, 1.0),\n",
       " ('ALBUMIN_DTS_1.0', 0.5957825537176662, 4, 1.0),\n",
       " ('BLOOD_LOSS_ANEMIA', 0.5960127188648144, 4, 1.0),\n",
       " ('POTASSIUM WHOLE BLOOD_DTS_2.0', 0.5960582893659432, 4, 1.0),\n",
       " ('GLUCOSE_DTS_3.0', 0.5965469947089116, 4, 1.0),\n",
       " ('POTASSIUM WHOLE BLOOD_DTS_3.0', 0.5968994665216496, 4, 1.0),\n",
       " ('POTASSIUM_DTS_1.0', 0.5976568521477774, 4, 1.0),\n",
       " ('GLUCOSE (BLOOD GAS)_DTS_1.0', 0.5977855492755596, 4, 1.0),\n",
       " ('HEMATOCRIT_DTS_2.0', 0.5979186457897648, 4, 1.0),\n",
       " ('PTT_DTS_3.0', 0.5979630675637406, 4, 1.0),\n",
       " ('DIABETES_UNCOMPLICATED', 0.5980569958468995, 4, 1.0),\n",
       " ('INR(PT)_DTS_3.0', 0.5984030823553514, 4, 1.0),\n",
       " ('GLUCOSE (BLOOD GAS)_DTS_3.0', 0.5988823129485366, 4, 1.0),\n",
       " ('LACTATE_DTS_3.0', 0.5989876494043453, 4, 1.0),\n",
       " ('RHEUMATOID_ARTHRITIS', 0.5990682580892801, 4, 1.0),\n",
       " ('PLATELET COUNT_DTS_3.0', 0.5994370535836974, 4, 1.0),\n",
       " ('HEMATOCRIT_DTS_3.0', 0.5995222203408211, 4, 1.0),\n",
       " ('HEMATOCRIT_DTS_1.0', 0.6009772420455746, 4, 1.0),\n",
       " ('DEFICIENCY_ANEMIAS', 0.6018022491670603, 4, 1.0),\n",
       " ('PARALYSIS', 0.6025869692490611, 4, 1.0),\n",
       " ('DEPRESSION', 0.6038051810229915, 4, 1.0),\n",
       " ('HEMATOCRIT (CALCULATED)_DTS_1.0', 0.6042180967982665, 4, 1.0),\n",
       " ('ANION GAP_DTS_2.0', 0.6045219548316285, 4, 1.0),\n",
       " ('AGE_AT_ADMISSION', 0.604573295336578, 4, 1.0),\n",
       " ('LACTATE_DTS_2.0', 0.605493431648638, 4, 1.0),\n",
       " ('DIABETES_COMPLICATED', 0.6057009198379292, 4, 1.0),\n",
       " ('GLUCOSE_DTS_1.0', 0.6058181444971478, 4, 1.0),\n",
       " ('CHLORIDE (WHOLE BLOOD)_DTS_3.0', 0.6060061927156253, 4, 1.0),\n",
       " ('POTASSIUM_DTS_3.0', 0.6065259605479497, 4, 1.0),\n",
       " ('GLUCOSE_DTS_2.0', 0.6071019335313957, 4, 1.0),\n",
       " ('PLATELET COUNT_DTS_2.0', 0.6077362530979545, 4, 1.0),\n",
       " ('ALCOHOL_ABUSE', 0.6082588036587588, 4, 1.0),\n",
       " ('POTASSIUM WHOLE BLOOD_DTS_1.0', 0.6088908182041655, 4, 1.0),\n",
       " ('SODIUM_DTS_2.0', 0.611001533109988, 4, 1.0),\n",
       " ('HEMOGLOBIN_DTS_2.0', 0.6112810623268521, 4, 1.0),\n",
       " ('CHLORIDE_DTS_1.0', 0.6118663574484896, 4, 1.0),\n",
       " ('PTT_DTS_2.0', 0.6124154330445504, 4, 1.0),\n",
       " ('GLUCOSE (BLOOD GAS)_DTS_2.0', 0.6125875267046968, 4, 1.0),\n",
       " ('PEPTIC_ULCER', 0.6129987637547074, 4, 1.0),\n",
       " ('DRUG_ABUSE', 0.6131424962300277, 4, 1.0),\n",
       " ('PTT_DTS_1.0', 0.6140625145136058, 4, 1.0),\n",
       " ('HEMOGLOBIN_DTS_3.0', 0.6141673727574221, 4, 1.0),\n",
       " ('WHITE BLOOD CELLS_DTS_3.0', 0.6144469096492974, 4, 1.0),\n",
       " ('OTHER_NEUROLOGICAL', 0.6177074513040492, 4, 1.0),\n",
       " ('CHLORIDE_DTS_2.0', 0.6178234824823502, 4, 1.0),\n",
       " ('WHITE BLOOD CELLS_DTS_1.0', 0.62578397353109, 4, 1.0),\n",
       " ('PLATELET COUNT_DTS_1.0', 0.6271540863712561, 4, 1.0),\n",
       " ('BILIRUBIN_DTS_2.0', 0.6294474403207349, 4, 1.0),\n",
       " ('PSYCHOSES', 0.6314747666265117, 4, 1.0),\n",
       " ('GENDER', 0.6326503028289518, 4, 1.0),\n",
       " ('PT_DTS_3.0', 0.6340727527257801, 4, 1.0),\n",
       " ('POTASSIUM_DTS_2.0', 0.6357899568978573, 4, 1.0),\n",
       " ('BILIRUBIN_DTS_1.0', 0.6359236649602806, 4, 1.0),\n",
       " ('BUN_DTS_3.0', 0.6417880773747808, 4, 1.0),\n",
       " ('PT_DTS_1.0', 0.6434905672645292, 4, 1.0),\n",
       " ('HYPERTENSION', 0.6435080821697543, 4, 1.0),\n",
       " ('CHLORIDE_DTS_3.0', 0.6443726521088508, 4, 1.0),\n",
       " ('LACTATE_DTS_1.0', 0.6463278742615005, 4, 1.0),\n",
       " ('OBESITY', 0.6660373277658679, 4, 1.0),\n",
       " ('CONGESTIVE_HEART_FAILURE', 0.6696603389506195, 4, 1.0),\n",
       " ('BILIRUBIN_DTS_3.0', 0.6704023284623936, 4, 1.0),\n",
       " ('BUN_DTS_1.0', 0.6794528839464279, 4, 1.0),\n",
       " ('LIVER_DISEASE', 0.7073906381827205, 4, 1.0),\n",
       " ('AIDS', 0.724555455083373, 4, 1.0),\n",
       " ('CARDIAC_ARRHYTHMIAS', 0.7289939841333126, 4, 1.0),\n",
       " ('VALVULAR_DISEASE', 0.7370261595513858, 4, 1.0),\n",
       " ('RENAL_FAILURE', 0.7392650041090106, 4, 1.0),\n",
       " ('BUN_DTS_2.0', 0.748206831584104, 4, 1.0),\n",
       " ('COAGULOPATHY', 0.7583844053348038, 4, 1.0),\n",
       " ('ELIXHAUSER_SCORE', 0.768994094757506, 4, 1.0),\n",
       " ('FLUID_ELECTROLYTE', 0.9018339610500634, 4, 1.0)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbdt_mimic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rfe_experiments_feat' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-064213267686>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#sorted_experiments = sorted([exp for exp in rfe_experiments.values()], key=lambda x: x['performance']['cv'][metric]['mean'], reverse=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msorted_experiments_feat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mexp\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mexp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrfe_experiments_feat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'parameters'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'datetime'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mexperiment_feat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted_experiments_feat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0msorted_experiments_exp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mexp\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mexp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrfe_experiments_exp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'parameters'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'datetime'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'rfe_experiments_feat' is not defined"
     ]
    }
   ],
   "source": [
    "#sorted_experiments = sorted([exp for exp in rfe_experiments.values()], key=lambda x: x['performance']['cv'][metric]['mean'], reverse=True)\n",
    "sorted_experiments_feat = sorted([exp for exp in rfe_experiments_feat.values()], key=lambda x: x['parameters']['datetime'], reverse=True)\n",
    "experiment_feat = sorted_experiments_feat[0]\n",
    "\n",
    "sorted_experiments_exp = sorted([exp for exp in rfe_experiments_exp.values()], key=lambda x: x['parameters']['datetime'], reverse=True)\n",
    "experiment_exp = sorted_experiments_exp[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_description_to_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = {'MIMIC' : \"~/cohorts/hs_mimic.csv\", 'SINAI' : \"~/cohorts/hs_sinai_preprocessed.csv\"}    \n",
    "    \n",
    "''' first load and split data '''\n",
    "data = Load().execute(filename=filenames['SINAI'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from morpher.utils import data_description_to_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Method .ptp is deprecated and will be removed in a future version. Use numpy.ptp instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved description to sinai_description.csv.\n",
      "### Data description:\n",
      "# Rows: 25799\n",
      "# Columns: 84\n",
      "# Column names: ['GENDER' 'AGE_AT_ADMISSION' 'CONGESTIVE_HEART_FAILURE'\n",
      " 'FLUID_ELECTROLYTE' 'LIVER_DISEASE' 'RHEUMATOID_ARTHRITIS'\n",
      " 'ALCOHOL_ABUSE' 'BLOOD_LOSS_ANEMIA' 'CARDIAC_ARRHYTHMIAS'\n",
      " 'CHRONIC_PULMONARY' 'COAGULOPATHY' 'DEFICIENCY_ANEMIAS' 'DEPRESSION'\n",
      " 'DIABETES_COMPLICATED' 'DIABETES_UNCOMPLICATED' 'DRUG_ABUSE' 'LYMPHOMA'\n",
      " 'METASTATIC_CANCER' 'OBESITY' 'OTHER_NEUROLOGICAL' 'PARALYSIS'\n",
      " 'PEPTIC_ULCER' 'PERIPHERAL_VASCULAR' 'PSYCHOSES' 'PULMONARY_CIRCULATION'\n",
      " 'RENAL_FAILURE' 'SOLID_TUMOR' 'VALVULAR_DISEASE' 'WEIGHT_LOSS'\n",
      " 'BUN_DTS_3.0' 'BUN_DTS_2.0' 'BUN_DTS_1.0' 'BLD CREA_DTS_3.0'\n",
      " 'BLD CREA_DTS_2.0' 'BLD CREA_DTS_1.0' 'ANION GAP_DTS_3.0'\n",
      " 'ANION GAP_DTS_2.0' 'ANION GAP_DTS_1.0' 'BILIRUBIN_DTS_3.0'\n",
      " 'BILIRUBIN_DTS_2.0' 'BILIRUBIN_DTS_1.0' 'ALBUMIN_DTS_3.0'\n",
      " 'ALBUMIN_DTS_2.0' 'ALBUMIN_DTS_1.0' 'CHLORIDE_DTS_3.0' 'CHLORIDE_DTS_2.0'\n",
      " 'CHLORIDE_DTS_1.0' 'GLUCOSE_DTS_3.0' 'GLUCOSE_DTS_2.0' 'GLUCOSE_DTS_1.0'\n",
      " 'HEMATOCRIT_DTS_3.0' 'HEMATOCRIT_DTS_2.0' 'HEMATOCRIT_DTS_1.0'\n",
      " 'HEMOGLOBIN_DTS_3.0' 'HEMOGLOBIN_DTS_2.0' 'HEMOGLOBIN_DTS_1.0'\n",
      " 'PLATELET COUNT_DTS_3.0' 'PLATELET COUNT_DTS_2.0'\n",
      " 'PLATELET COUNT_DTS_1.0' 'POTASSIUM_DTS_3.0' 'POTASSIUM_DTS_2.0'\n",
      " 'POTASSIUM_DTS_1.0' 'SODIUM_DTS_3.0' 'SODIUM_DTS_2.0' 'SODIUM_DTS_1.0'\n",
      " 'WHITE BLOOD CELLS_DTS_3.0' 'WHITE BLOOD CELLS_DTS_2.0'\n",
      " 'WHITE BLOOD CELLS_DTS_1.0' 'INR(PT)_DTS_3.0' 'INR(PT)_DTS_2.0'\n",
      " 'INR(PT)_DTS_1.0' 'PTT_DTS_3.0' 'PTT_DTS_2.0' 'PTT_DTS_1.0' 'PT_DTS_3.0'\n",
      " 'PT_DTS_2.0' 'PT_DTS_1.0' 'LACTATE_DTS_3.0' 'LACTATE_DTS_2.0'\n",
      " 'LACTATE_DTS_1.0' 'AKI' 'HYPERTENSION' 'HYPOTHYROIDISM'\n",
      " 'ELIXHAUSER_SCORE']\n",
      "### Descriptive Statistics for Features Used\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Column</th>\n",
       "      <th>All(n=25799)</th>\n",
       "      <th>Missing</th>\n",
       "      <th>Missing(%)</th>\n",
       "      <th>P</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GENDER</td>\n",
       "      <td>15331 (59.42%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AGE_AT_ADMISSION</td>\n",
       "      <td>66.66+/-15.95</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CONGESTIVE_HEART_FAILURE</td>\n",
       "      <td>12427 (48.17%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FLUID_ELECTROLYTE</td>\n",
       "      <td>10637 (41.23%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LIVER_DISEASE</td>\n",
       "      <td>812 (3.15%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RHEUMATOID_ARTHRITIS</td>\n",
       "      <td>768 (2.98%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ALCOHOL_ABUSE</td>\n",
       "      <td>253 (0.98%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>BLOOD_LOSS_ANEMIA</td>\n",
       "      <td>525 (2.03%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CARDIAC_ARRHYTHMIAS</td>\n",
       "      <td>6291 (24.38%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CHRONIC_PULMONARY</td>\n",
       "      <td>8467 (32.82%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>COAGULOPATHY</td>\n",
       "      <td>3755 (14.55%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>DEFICIENCY_ANEMIAS</td>\n",
       "      <td>1050 (4.07%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>DEPRESSION</td>\n",
       "      <td>2047 (7.93%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>DIABETES_COMPLICATED</td>\n",
       "      <td>60 (0.23%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>DIABETES_UNCOMPLICATED</td>\n",
       "      <td>3124 (12.11%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>DRUG_ABUSE</td>\n",
       "      <td>327 (1.27%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>LYMPHOMA</td>\n",
       "      <td>353 (1.37%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>METASTATIC_CANCER</td>\n",
       "      <td>908 (3.52%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>OBESITY</td>\n",
       "      <td>1155 (4.48%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>OTHER_NEUROLOGICAL</td>\n",
       "      <td>820 (3.18%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>PARALYSIS</td>\n",
       "      <td>254 (0.98%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>PEPTIC_ULCER</td>\n",
       "      <td>126 (0.49%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>PERIPHERAL_VASCULAR</td>\n",
       "      <td>5403 (20.94%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>PSYCHOSES</td>\n",
       "      <td>238 (0.92%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>PULMONARY_CIRCULATION</td>\n",
       "      <td>6784 (26.3%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>RENAL_FAILURE</td>\n",
       "      <td>5143 (19.93%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>SOLID_TUMOR</td>\n",
       "      <td>1524 (5.91%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>VALVULAR_DISEASE</td>\n",
       "      <td>18606 (72.12%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>WEIGHT_LOSS</td>\n",
       "      <td>2928 (11.35%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>BUN_DTS_3.0</td>\n",
       "      <td>23.8+/-15.59</td>\n",
       "      <td>15691</td>\n",
       "      <td>60.8</td>\n",
       "      <td>0.801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>HEMOGLOBIN_DTS_2.0</td>\n",
       "      <td>10.43+/-1.76</td>\n",
       "      <td>14389</td>\n",
       "      <td>55.8</td>\n",
       "      <td>0.991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>HEMOGLOBIN_DTS_1.0</td>\n",
       "      <td>10.69+/-1.62</td>\n",
       "      <td>8292</td>\n",
       "      <td>32.1</td>\n",
       "      <td>0.172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>PLATELET COUNT_DTS_3.0</td>\n",
       "      <td>182.42+/-101.0</td>\n",
       "      <td>15722</td>\n",
       "      <td>60.9</td>\n",
       "      <td>0.005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>PLATELET COUNT_DTS_2.0</td>\n",
       "      <td>196.12+/-101.15</td>\n",
       "      <td>14480</td>\n",
       "      <td>56.1</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>PLATELET COUNT_DTS_1.0</td>\n",
       "      <td>198.79+/-97.42</td>\n",
       "      <td>8333</td>\n",
       "      <td>32.3</td>\n",
       "      <td>0.458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>POTASSIUM_DTS_3.0</td>\n",
       "      <td>4.21+/-0.5</td>\n",
       "      <td>15682</td>\n",
       "      <td>60.8</td>\n",
       "      <td>0.689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>POTASSIUM_DTS_2.0</td>\n",
       "      <td>4.15+/-0.5</td>\n",
       "      <td>14592</td>\n",
       "      <td>56.6</td>\n",
       "      <td>0.923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>POTASSIUM_DTS_1.0</td>\n",
       "      <td>4.21+/-0.43</td>\n",
       "      <td>8695</td>\n",
       "      <td>33.7</td>\n",
       "      <td>0.961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>SODIUM_DTS_3.0</td>\n",
       "      <td>137.29+/-3.67</td>\n",
       "      <td>15692</td>\n",
       "      <td>60.8</td>\n",
       "      <td>0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>SODIUM_DTS_2.0</td>\n",
       "      <td>137.39+/-3.61</td>\n",
       "      <td>14598</td>\n",
       "      <td>56.6</td>\n",
       "      <td>0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>SODIUM_DTS_1.0</td>\n",
       "      <td>138.41+/-3.41</td>\n",
       "      <td>8695</td>\n",
       "      <td>33.7</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>WHITE BLOOD CELLS_DTS_3.0</td>\n",
       "      <td>10.49+/-4.6</td>\n",
       "      <td>15629</td>\n",
       "      <td>60.6</td>\n",
       "      <td>0.187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>WHITE BLOOD CELLS_DTS_2.0</td>\n",
       "      <td>9.8+/-4.49</td>\n",
       "      <td>14384</td>\n",
       "      <td>55.8</td>\n",
       "      <td>0.014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>WHITE BLOOD CELLS_DTS_1.0</td>\n",
       "      <td>10.61+/-5.11</td>\n",
       "      <td>8299</td>\n",
       "      <td>32.2</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>INR(PT)_DTS_3.0</td>\n",
       "      <td>1.4+/-0.58</td>\n",
       "      <td>19173</td>\n",
       "      <td>74.3</td>\n",
       "      <td>0.612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>INR(PT)_DTS_2.0</td>\n",
       "      <td>1.39+/-0.56</td>\n",
       "      <td>18274</td>\n",
       "      <td>70.8</td>\n",
       "      <td>0.052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>INR(PT)_DTS_1.0</td>\n",
       "      <td>1.4+/-0.52</td>\n",
       "      <td>13637</td>\n",
       "      <td>52.9</td>\n",
       "      <td>0.165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>PTT_DTS_3.0</td>\n",
       "      <td>37.58+/-11.29</td>\n",
       "      <td>19696</td>\n",
       "      <td>76.3</td>\n",
       "      <td>0.059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>PTT_DTS_2.0</td>\n",
       "      <td>37.68+/-12.07</td>\n",
       "      <td>18942</td>\n",
       "      <td>73.4</td>\n",
       "      <td>0.093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>PTT_DTS_1.0</td>\n",
       "      <td>37.18+/-10.77</td>\n",
       "      <td>14369</td>\n",
       "      <td>55.7</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>PT_DTS_3.0</td>\n",
       "      <td>16.93+/-4.47</td>\n",
       "      <td>19177</td>\n",
       "      <td>74.3</td>\n",
       "      <td>0.632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>PT_DTS_2.0</td>\n",
       "      <td>16.84+/-4.55</td>\n",
       "      <td>18280</td>\n",
       "      <td>70.9</td>\n",
       "      <td>0.021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>PT_DTS_1.0</td>\n",
       "      <td>17.07+/-4.28</td>\n",
       "      <td>13639</td>\n",
       "      <td>52.9</td>\n",
       "      <td>0.534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>LACTATE_DTS_3.0</td>\n",
       "      <td>1.26+/-1.22</td>\n",
       "      <td>24252</td>\n",
       "      <td>94.0</td>\n",
       "      <td>0.081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>LACTATE_DTS_2.0</td>\n",
       "      <td>1.75+/-1.97</td>\n",
       "      <td>25101</td>\n",
       "      <td>97.3</td>\n",
       "      <td>0.153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>LACTATE_DTS_1.0</td>\n",
       "      <td>2.09+/-1.56</td>\n",
       "      <td>18837</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>AKI</td>\n",
       "      <td>1163 (4.51%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>HYPERTENSION</td>\n",
       "      <td>17097 (66.27%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>HYPOTHYROIDISM</td>\n",
       "      <td>2789 (10.81%)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>ELIXHAUSER_SCORE</td>\n",
       "      <td>11.26+/-10.91</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>84 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Column     All(n=25799) Missing  Missing(%)      P\n",
       "0                      GENDER   15331 (59.42%)       0         0.0  0.757\n",
       "1            AGE_AT_ADMISSION    66.66+/-15.95       0         0.0  0.506\n",
       "2    CONGESTIVE_HEART_FAILURE   12427 (48.17%)       0         0.0  0.314\n",
       "3           FLUID_ELECTROLYTE   10637 (41.23%)       0         0.0  0.032\n",
       "4               LIVER_DISEASE      812 (3.15%)       0         0.0  0.032\n",
       "5        RHEUMATOID_ARTHRITIS      768 (2.98%)       0         0.0  0.716\n",
       "6               ALCOHOL_ABUSE      253 (0.98%)       0         0.0  0.232\n",
       "7           BLOOD_LOSS_ANEMIA      525 (2.03%)       0         0.0  0.014\n",
       "8         CARDIAC_ARRHYTHMIAS    6291 (24.38%)       0         0.0  0.002\n",
       "9           CHRONIC_PULMONARY    8467 (32.82%)       0         0.0  0.372\n",
       "10               COAGULOPATHY    3755 (14.55%)       0         0.0  0.127\n",
       "11         DEFICIENCY_ANEMIAS     1050 (4.07%)       0         0.0  0.005\n",
       "12                 DEPRESSION     2047 (7.93%)       0         0.0  0.653\n",
       "13       DIABETES_COMPLICATED       60 (0.23%)       0         0.0  0.001\n",
       "14     DIABETES_UNCOMPLICATED    3124 (12.11%)       0         0.0  0.128\n",
       "15                 DRUG_ABUSE      327 (1.27%)       0         0.0  0.953\n",
       "16                   LYMPHOMA      353 (1.37%)       0         0.0  0.002\n",
       "17          METASTATIC_CANCER      908 (3.52%)       0         0.0  0.483\n",
       "18                    OBESITY     1155 (4.48%)       0         0.0  0.021\n",
       "19         OTHER_NEUROLOGICAL      820 (3.18%)       0         0.0  0.630\n",
       "20                  PARALYSIS      254 (0.98%)       0         0.0  0.134\n",
       "21               PEPTIC_ULCER      126 (0.49%)       0         0.0  0.312\n",
       "22        PERIPHERAL_VASCULAR    5403 (20.94%)       0         0.0  0.689\n",
       "23                  PSYCHOSES      238 (0.92%)       0         0.0  0.064\n",
       "24      PULMONARY_CIRCULATION     6784 (26.3%)       0         0.0  0.362\n",
       "25              RENAL_FAILURE    5143 (19.93%)       0         0.0  0.000\n",
       "26                SOLID_TUMOR     1524 (5.91%)       0         0.0  0.087\n",
       "27           VALVULAR_DISEASE   18606 (72.12%)       0         0.0  0.460\n",
       "28                WEIGHT_LOSS    2928 (11.35%)       0         0.0  0.137\n",
       "29                BUN_DTS_3.0     23.8+/-15.59   15691        60.8  0.801\n",
       "..                        ...              ...     ...         ...    ...\n",
       "54         HEMOGLOBIN_DTS_2.0     10.43+/-1.76   14389        55.8  0.991\n",
       "55         HEMOGLOBIN_DTS_1.0     10.69+/-1.62    8292        32.1  0.172\n",
       "56     PLATELET COUNT_DTS_3.0   182.42+/-101.0   15722        60.9  0.005\n",
       "57     PLATELET COUNT_DTS_2.0  196.12+/-101.15   14480        56.1  0.000\n",
       "58     PLATELET COUNT_DTS_1.0   198.79+/-97.42    8333        32.3  0.458\n",
       "59          POTASSIUM_DTS_3.0       4.21+/-0.5   15682        60.8  0.689\n",
       "60          POTASSIUM_DTS_2.0       4.15+/-0.5   14592        56.6  0.923\n",
       "61          POTASSIUM_DTS_1.0      4.21+/-0.43    8695        33.7  0.961\n",
       "62             SODIUM_DTS_3.0    137.29+/-3.67   15692        60.8  0.001\n",
       "63             SODIUM_DTS_2.0    137.39+/-3.61   14598        56.6  0.001\n",
       "64             SODIUM_DTS_1.0    138.41+/-3.41    8695        33.7  0.000\n",
       "65  WHITE BLOOD CELLS_DTS_3.0      10.49+/-4.6   15629        60.6  0.187\n",
       "66  WHITE BLOOD CELLS_DTS_2.0       9.8+/-4.49   14384        55.8  0.014\n",
       "67  WHITE BLOOD CELLS_DTS_1.0     10.61+/-5.11    8299        32.2  0.000\n",
       "68            INR(PT)_DTS_3.0       1.4+/-0.58   19173        74.3  0.612\n",
       "69            INR(PT)_DTS_2.0      1.39+/-0.56   18274        70.8  0.052\n",
       "70            INR(PT)_DTS_1.0       1.4+/-0.52   13637        52.9  0.165\n",
       "71                PTT_DTS_3.0    37.58+/-11.29   19696        76.3  0.059\n",
       "72                PTT_DTS_2.0    37.68+/-12.07   18942        73.4  0.093\n",
       "73                PTT_DTS_1.0    37.18+/-10.77   14369        55.7  0.000\n",
       "74                 PT_DTS_3.0     16.93+/-4.47   19177        74.3  0.632\n",
       "75                 PT_DTS_2.0     16.84+/-4.55   18280        70.9  0.021\n",
       "76                 PT_DTS_1.0     17.07+/-4.28   13639        52.9  0.534\n",
       "77            LACTATE_DTS_3.0      1.26+/-1.22   24252        94.0  0.081\n",
       "78            LACTATE_DTS_2.0      1.75+/-1.97   25101        97.3  0.153\n",
       "79            LACTATE_DTS_1.0      2.09+/-1.56   18837        73.0  0.696\n",
       "80                        AKI     1163 (4.51%)       0         0.0  0.000\n",
       "81               HYPERTENSION   17097 (66.27%)       0         0.0  0.006\n",
       "82             HYPOTHYROIDISM    2789 (10.81%)       0         0.0  0.772\n",
       "83           ELIXHAUSER_SCORE    11.26+/-10.91       0         0.0  0.000\n",
       "\n",
       "[84 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_description_to_csv(data,'AKI', 'sinai_description.csv',print_summary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}