run-experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%run default-imports.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_experiment(params):\n",
    "           \n",
    "    filenames = {'MIMIC' : \"~/cohorts/hs_mimic.csv\", 'SINAI' : \"~/cohorts/hs_sinai_preprocessed.csv\"}    \n",
    "    \n",
    "    ''' first load and split data '''\n",
    "    data = Load().execute(filename=filenames[params['cohort']])\n",
    "    \n",
    "    ''' keep only features that were explicitly chosen or all features '''\n",
    "    if params.get('features_to_select'):\n",
    "        for feature in data.columns:\n",
    "            if feature not in [params['target']] + params['features_to_select']:\n",
    "                data.drop(feature, axis=1, inplace=True)\n",
    "    else:\n",
    "        params['features_to_select'] = 'all'\n",
    "    \n",
    "    train, test = Split().execute(data,test_size=params['test_size'])\n",
    "    train, imputer = Impute().execute(train.copy(), imputation_method=params['imputation_method'])\n",
    "    \n",
    "    scaler=None\n",
    "    if params.get('scaling_method'):\n",
    "        train, scaler = Scale().execute(train.copy(), params['target'],transform_method=params['scaling_method'])\n",
    "    \n",
    "    if params.get('sampling_method'):\n",
    "        train = Sample().execute(train.copy(), params['target'],sampling_method=params['sampling_method'], sampling_strategy=params.get('sampling_strategy'))        \n",
    "    \n",
    "    runtime = {}\n",
    "    \n",
    "    crossval_metrics = {}\n",
    "    with Timer() as t:\n",
    "        ''' fit models '''\n",
    "        if params.get('crossval'):\n",
    "            models, crossval_metrics = Train().execute(train, target=params['target'], algorithms=[params['algorithm']], optimize=params['optimize_mode'], crossval=params.get('crossval'))\n",
    "        else:                    \n",
    "            models = Train().execute(train, target=params['target'], algorithms=[params['algorithm']], optimize=params['optimize_mode'])\n",
    "    \n",
    "    runtime['train'] = t.interval    \n",
    "    \n",
    "    ''' evaluate models '''\n",
    "    test, _ = Impute().execute(test.copy(), imputer=imputer)    \n",
    "    \n",
    "    if params.get('scale_method'):\n",
    "        test,_ = Scale().execute(test.copy(), params['target'], scaler=scaler)\n",
    "    \n",
    "    results = Evaluate().execute(test.copy(), target=params['target'], models=models)\n",
    "    \n",
    "    ''' obtain performance metrics '''\n",
    "    performance = {}\n",
    "    performance['crossval_metrics'] = crossval_metrics.get(params['algorithm'])\n",
    "    performance['discrimination'] = get_discrimination_metrics(**results[params['algorithm']])\n",
    "    performance['calibration'] = get_calibration_metrics(results[params['algorithm']]['y_true'],results[params['algorithm']]['y_probs'])\n",
    "    performance['clinical_usefulness'] = get_clinical_usefulness_metrics(performance['discrimination'])\n",
    "    \n",
    "    ''' save pipeline for later reproducibility '''\n",
    "    pipeline = None\n",
    "    if params.get('save_pipeline'):\n",
    "        pipeline = {'model': models[params['algorithm']], 'imputer' : imputer, 'scaler': scaler, 'results': results}\n",
    "\n",
    "    ''' interpret explanations '''\n",
    "    explanations = {}     \n",
    "    weighted_explanations = {}\n",
    "    feature_importances = None\n",
    "       \n",
    "    with Timer() as t:\n",
    "        if params.get('explainers'):\n",
    "            explanations = Explain().execute(train, models=models, target=params['target'], explainers=params['explainers'], exp_kwargs={'test':test, 'sample_size':200, 'mimic': params.get('mimic'), 'num_features': params.get('num_features'), 'num_exps_desired':20})\n",
    "            weighted_explanations = get_weighted_explanations(explanations[params['algorithm']])\n",
    "            feature_importances = list(sorted([(k,v) for k,v in explanations[params['algorithm']]['FeatContribExplainer'].items()], key=lambda x: x[1]))            \n",
    "            \n",
    "    runtime['explain'] = t.interval    \n",
    "    params['datetime'] = datetime.now()\n",
    "    \n",
    "    ''' summarize results '''\n",
    "    experiment = {'parameters' : params,\n",
    "                  'pipeline': pipeline,\n",
    "                  'performance' : performance,\n",
    "                  'explanations' : explanations, \n",
    "                  'weighted_explanations' : weighted_explanations,\n",
    "                  'feature_importances' : feature_importances,\n",
    "                  'runtime' : runtime}\n",
    "    \n",
    "    return experiment \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}