strats_notebook_mimic_iv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "driven-termination",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Dense, Dropout, BatchNormalization, Lambda\n",
    "from tensorflow.keras.models import Model\n",
    "import pickle\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from sklearn.metrics import roc_auc_score, precision_recall_curve, auc\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras.callbacks import Callback, EarlyStopping\n",
    "import pandas as pd\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "recreational-record",
   "metadata": {},
   "source": [
    "## Load forecast dataset into matrices."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "executive-annual",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "95780it [00:00, 847375.25it/s]\n",
      "100%|██████████| 26/26 [03:34<00:00,  8.24s/it]\n"
     ]
    }
   ],
   "source": [
    "pred_window = 2 # hours\n",
    "obs_windows = range(20, 124, 4)\n",
    "\n",
    "# Read data.\n",
    "data_path = 'mimic_iv_preprocessed.pkl'\n",
    "data, oc, train_ind, valid_ind, test_ind = pickle.load(open(data_path, 'rb'))\n",
    "data.fillna(0, inplace=True) \n",
    "# Remove test patients.\n",
    "data = data.merge(oc[['ts_ind', 'subject_id']], on='ts_ind', how='left')\n",
    "test_sub = oc.loc[oc.ts_ind.isin(test_ind)].subject_id.unique()\n",
    "data = data.loc[~data.subject_id.isin(test_sub)]\n",
    "oc = oc.loc[~oc.subject_id.isin(test_sub)]\n",
    "data.drop(columns=['subject_id', 'TABLE'], inplace=True)\n",
    "# Fix age.\n",
    "data.loc[(data.variable=='age')&(data.value>200), 'value'] = 91.4\n",
    "# Get static data with mean fill and missingness indicator.\n",
    "static_varis = ['age', 'gender']\n",
    "ii = data.variable.isin(static_varis)\n",
    "static_data = data.loc[ii]\n",
    "data = data.loc[~ii]\n",
    "def inv_list(l, start=0):\n",
    "    d = {}\n",
    "    for i in range(len(l)):\n",
    "        d[l[i]] = i+start\n",
    "    return d\n",
    "static_var_to_ind = inv_list(static_varis)\n",
    "D = len(static_varis)\n",
    "N = data.ts_ind.max()+1\n",
    "demo = np.zeros((N, D))\n",
    "for row in tqdm(static_data.itertuples()):\n",
    "    demo[row.ts_ind, static_var_to_ind[row.variable]] = row.value\n",
    "# Normalize static data.\n",
    "means = demo.mean(axis=0, keepdims=True)\n",
    "stds = demo.std(axis=0, keepdims=True)\n",
    "stds = (stds==0)*1 + (stds!=0)*stds\n",
    "demo = (demo-means)/stds\n",
    "# Get variable indices.\n",
    "varis = sorted(list(set(data.variable)))\n",
    "V = len(varis)\n",
    "var_to_ind = inv_list(varis, start=1)\n",
    "data['vind'] = data.variable.map(var_to_ind)\n",
    "data = data[['ts_ind', 'vind', 'hour', 'value']].sort_values(by=['ts_ind', 'vind', 'hour'])\n",
    "# Find max_len.\n",
    "fore_max_len = 880\n",
    "# Get forecast inputs and outputs.\n",
    "fore_times_ip = []\n",
    "fore_values_ip = []\n",
    "fore_varis_ip = []\n",
    "fore_op = []\n",
    "fore_inds = []\n",
    "def f(x):\n",
    "    mask = [0 for i in range(V)]\n",
    "    values = [0 for i in range(V)]\n",
    "    for vv in x:\n",
    "        v = int(vv[0])-1\n",
    "        mask[v] = 1\n",
    "        values[v] = vv[1]\n",
    "    return values+mask\n",
    "def pad(x):\n",
    "    return x+[0]*(fore_max_len-len(x))\n",
    "for w in tqdm(obs_windows):\n",
    "    pred_data = data.loc[(data.hour>=w)&(data.hour<=w+pred_window)]\n",
    "    pred_data = pred_data.groupby(['ts_ind', 'vind']).agg({'value':'first'}).reset_index()\n",
    "    pred_data['vind_value'] = pred_data[['vind', 'value']].values.tolist()\n",
    "    pred_data = pred_data.groupby('ts_ind').agg({'vind_value':list}).reset_index()\n",
    "    pred_data['vind_value'] = pred_data['vind_value'].apply(f)    \n",
    "    obs_data = data.loc[(data.hour<w)&(data.hour>=w-24)]\n",
    "    obs_data = obs_data.loc[obs_data.ts_ind.isin(pred_data.ts_ind)]\n",
    "    obs_data = obs_data.groupby('ts_ind').head(fore_max_len)\n",
    "    obs_data = obs_data.groupby('ts_ind').agg({'vind':list, 'hour':list, 'value':list}).reset_index()\n",
    "    obs_data = obs_data.merge(pred_data, on='ts_ind')\n",
    "    for col in ['vind', 'hour', 'value']:\n",
    "        obs_data[col] = obs_data[col].apply(pad)\n",
    "    fore_op.append(np.array(list(obs_data.vind_value)))\n",
    "    fore_inds.append(np.array(list(obs_data.ts_ind)))\n",
    "    fore_times_ip.append(np.array(list(obs_data.hour)))\n",
    "    fore_values_ip.append(np.array(list(obs_data.value)))\n",
    "    fore_varis_ip.append(np.array(list(obs_data.vind)))\n",
    "del data\n",
    "fore_times_ip = np.concatenate(fore_times_ip, axis=0)\n",
    "fore_values_ip = np.concatenate(fore_values_ip, axis=0)\n",
    "fore_varis_ip = np.concatenate(fore_varis_ip, axis=0)\n",
    "fore_op = np.concatenate(fore_op, axis=0)\n",
    "fore_inds = np.concatenate(fore_inds, axis=0)\n",
    "fore_demo = demo[fore_inds]\n",
    "# Get train and valid ts_ind for forecast task.\n",
    "train_sub = oc.loc[oc.ts_ind.isin(train_ind)].subject_id.unique()\n",
    "valid_sub = oc.loc[oc.ts_ind.isin(valid_ind)].subject_id.unique()\n",
    "rem_sub = oc.loc[~oc.subject_id.isin(np.concatenate((train_ind, valid_ind)))].subject_id.unique()\n",
    "bp = int(0.8*len(rem_sub))\n",
    "train_sub = np.concatenate((train_sub, rem_sub[:bp]))\n",
    "valid_sub = np.concatenate((valid_sub, rem_sub[bp:]))\n",
    "train_ind = oc.loc[oc.subject_id.isin(train_sub)].ts_ind.unique() # Add remaining ts_ind s of train subjects.\n",
    "valid_ind = oc.loc[oc.subject_id.isin(valid_sub)].ts_ind.unique() # Add remaining ts_ind s of train subjects.\n",
    "# Generate 3 sets of inputs and outputs.\n",
    "train_ind = np.argwhere(np.in1d(fore_inds, train_ind)).flatten()\n",
    "valid_ind = np.argwhere(np.in1d(fore_inds, valid_ind)).flatten()\n",
    "fore_train_ip = [ip[train_ind] for ip in [fore_demo, fore_times_ip, fore_values_ip, fore_varis_ip]]\n",
    "fore_valid_ip = [ip[valid_ind] for ip in [fore_demo, fore_times_ip, fore_values_ip, fore_varis_ip]]\n",
    "del fore_times_ip, fore_values_ip, fore_varis_ip, demo, fore_demo\n",
    "fore_train_op = fore_op[train_ind]\n",
    "fore_valid_op = fore_op[valid_ind]\n",
    "del fore_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "realistic-control",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(34961, (444837, 880), (169128, 2))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rem_sub), fore_train_ip[1].shape, fore_valid_ip[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "accessory-concern",
   "metadata": {},
   "outputs": [],
   "source": [
    "# m = data.groupby('ts_ind').agg({'hour':'max'}).hour\n",
    "# for q in [0.5,0.6,0.7,0.8,0.9, 0.95, 0.99, 1]:\n",
    "#     print (q, m.quantile(q))\n",
    "# import matplotlib.pyplot as plt\n",
    "# plt.plot(sorted(list(m)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "massive-indiana",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "communist-hanging",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "imperial-concern",
   "metadata": {},
   "source": [
    "## Load target dataset into matrices."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "stretch-hybrid",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29172 7222 9172\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "91132it [00:00, 661295.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_len 880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18653168it [00:27, 668403.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29172\n",
      "7222\n",
      "9172\n"
     ]
    }
   ],
   "source": [
    "# Read data.\n",
    "data_path = 'mimic_iv_preprocessed.pkl'\n",
    "data, oc, train_ind, valid_ind, test_ind = pickle.load(open(data_path, 'rb'))\n",
    "data.fillna(0, inplace=True) \n",
    "print(len(train_ind),len(valid_ind),len(test_ind))\n",
    "# Filter labeled data in first 24h.\n",
    "data = data.loc[data.ts_ind.isin(np.concatenate((train_ind, valid_ind, test_ind), axis=-1))]\n",
    "data = data.loc[(data.hour>=0)&(data.hour<=24)]\n",
    "oc = oc.loc[oc.ts_ind.isin(np.concatenate((train_ind, valid_ind, test_ind), axis=-1))]\n",
    "# Fix age.\n",
    "data.loc[(data.variable=='age')&(data.value>200), 'value'] = 91.4\n",
    "# Get y and N.\n",
    "y = np.array(oc.sort_values(by='ts_ind')['in_hospital_mortality']).astype('float32')\n",
    "\n",
    "N = data.ts_ind.max() + 1\n",
    "# Get static data with mean fill and missingness indicator.\n",
    "static_varis = ['age', 'gender']\n",
    "ii = data.variable.isin(static_varis)\n",
    "static_data = data.loc[ii]\n",
    "data = data.loc[~ii]\n",
    "def inv_list(l, start=0):\n",
    "    d = {}\n",
    "    for i in range(len(l)):\n",
    "        d[l[i]] = i+start\n",
    "    return d\n",
    "static_var_to_ind = inv_list(static_varis)\n",
    "D = len(static_varis)\n",
    "demo = np.zeros((N, D))\n",
    "for row in tqdm(static_data.itertuples()):\n",
    "    demo[row.ts_ind, static_var_to_ind[row.variable]] = row.value\n",
    "# Normalize static data.\n",
    "means = demo.mean(axis=0, keepdims=True)\n",
    "stds = demo.std(axis=0, keepdims=True)\n",
    "stds = (stds==0)*1 + (stds!=0)*stds\n",
    "demo = (demo-means)/stds\n",
    "# Trim to max len.\n",
    "data = data.sample(frac=1)\n",
    "data = data.groupby('ts_ind').head(880)\n",
    "# Get N, V, var_to_ind.\n",
    "N = data.ts_ind.max() + 1\n",
    "varis = sorted(list(set(data.variable)))\n",
    "V = len(varis)\n",
    "def inv_list(l, start=0):\n",
    "    d = {}\n",
    "    for i in range(len(l)):\n",
    "        d[l[i]] = i+start\n",
    "    return d\n",
    "var_to_ind = inv_list(varis, start=1)\n",
    "data['vind'] = data.variable.map(var_to_ind)\n",
    "data = data[['ts_ind', 'vind', 'hour', 'value']].sort_values(by=['ts_ind', 'vind', 'hour'])\n",
    "# Add obs index.\n",
    "data = data.sort_values(by=['ts_ind']).reset_index(drop=True)\n",
    "data = data.reset_index().rename(columns={'index':'obs_ind'})\n",
    "data = data.merge(data.groupby('ts_ind').agg({'obs_ind':'min'}).reset_index().rename(columns={ \\\n",
    "                                                            'obs_ind':'first_obs_ind'}), on='ts_ind')\n",
    "data['obs_ind'] = data['obs_ind'] - data['first_obs_ind']\n",
    "# Find max_len.\n",
    "max_len = data.obs_ind.max()+1\n",
    "print ('max_len', max_len)\n",
    "# Generate times_ip and values_ip matrices.\n",
    "times_inp = np.zeros((N, max_len), dtype='float32')\n",
    "values_inp = np.zeros((N, max_len), dtype='float32')\n",
    "varis_inp = np.zeros((N, max_len), dtype='int32')\n",
    "for row in tqdm(data.itertuples()):\n",
    "    ts_ind = row.ts_ind\n",
    "    l = row.obs_ind\n",
    "    times_inp[ts_ind, l] = row.hour\n",
    "    values_inp[ts_ind, l] = row.value\n",
    "    varis_inp[ts_ind, l] = row.vind\n",
    "data.drop(columns=['obs_ind', 'first_obs_ind'], inplace=True)\n",
    "# Generate 3 sets of inputs and outputs.\n",
    "train_ip = [ip[train_ind] for ip in [demo, times_inp, values_inp, varis_inp]]\n",
    "valid_ip = [ip[valid_ind] for ip in [demo, times_inp, values_inp, varis_inp]]\n",
    "test_ip = [ip[test_ind] for ip in [demo, times_inp, values_inp, varis_inp]]\n",
    "del times_inp, values_inp, varis_inp\n",
    "train_op = y[train_ind]\n",
    "valid_op = y[valid_ind]\n",
    "test_op = y[test_ind]\n",
    "\n",
    "train_op_ind_df = pd.DataFrame({\n",
    "    'train_ind': train_ind,\n",
    "    'train_op': y[train_ind]\n",
    "})\n",
    "\n",
    "# For Valid Data\n",
    "valid_op_ind_df = pd.DataFrame({\n",
    "    'valid_ind': valid_ind,\n",
    "    'valid_op': y[valid_ind]\n",
    "})\n",
    "\n",
    "# For Test Data\n",
    "test__op_ind_df = pd.DataFrame({\n",
    "    'test_ind': test_ind,\n",
    "    'test_op': y[test_ind]\n",
    "})\n",
    "\n",
    "#print(valid_op)\n",
    "print(len(train_op))\n",
    "\n",
    "print(len(valid_op))\n",
    "\n",
    "print(len(test_op))\n",
    "#print(test_ip)\n",
    "#print(test_ip.shape,test_op.shape)\n",
    "del y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "de2f5468",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45566\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "45566"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.concatenate((train_op,valid_op,test_op))\n",
    "print(len(y))\n",
    "y.sum()/len(y)\n",
    "len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "purple-collect",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# l = data.groupby('ts_ind').size()\n",
    "# print (sorted(l)[-500:], l.quantile(0.99), len(l))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "convinced-vault",
   "metadata": {},
   "source": [
    "## Define metrics and losses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "familiar-weight",
   "metadata": {},
   "outputs": [],
   "source": [
    "#def get_res(y_true, y_pred):\n",
    "    #if len(set(y_true)) > 1:\n",
    "        #return roc_auc_score(y_true, y_pred)\n",
    "    #else:\n",
    "        #print(\"Warning: ROC AUC score is undefined for single class validation set.\")\n",
    "        #return None\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def get_res(y_true, y_pred):\n",
    "    #df = pd.DataFrame({\n",
    "    #'y_true': y_true,\n",
    "    #'y_pred': y_pred\n",
    "#})\n",
    "\n",
    "# Save DataFrame to CSV\n",
    "    #df.to_csv('y_true_y_pred.csv', index=False)\n",
    "    precision, recall, thresholds = precision_recall_curve(y_true, y_pred)\n",
    "    \n",
    "    #if len(set(y_true)) > 1:\n",
    "    pr_auc = auc(recall, precision)\n",
    "    minrp = np.minimum(precision, recall).max()\n",
    "    roc_auc = roc_auc_score(y_true, y_pred)\n",
    "    return [roc_auc, pr_auc, minrp]\n",
    "    #else:\n",
    "        #print(\"Warning: ROC AUC score is undefined for single class validation set.\")\n",
    "        #return None\n",
    "    \n",
    "class_weights = compute_class_weight(class_weight='balanced', classes=[0,1], y=train_op)\n",
    "def mortality_loss(y_true, y_pred):\n",
    "    sample_weights = (1-y_true)*class_weights[0] + y_true*class_weights[1]\n",
    "    bce = K.binary_crossentropy(y_true, y_pred)\n",
    "    return K.mean(sample_weights*bce, axis=-1)\n",
    "\n",
    "# var_weights = np.sum(fore_train_op[:, V:], axis=0)\n",
    "# var_weights[var_weights==0] = var_weights.max()\n",
    "# var_weights = var_weights.max()/var_weights\n",
    "# var_weights = var_weights.reshape((1, V))\n",
    "def forecast_loss(y_true, y_pred):\n",
    "    return K.sum(y_true[:,V:]*(y_true[:,:V]-y_pred)**2, axis=-1)\n",
    "\n",
    "def get_min_loss(weight):\n",
    "    def min_loss(y_true, y_pred):\n",
    "        return weight*y_pred\n",
    "    return min_loss\n",
    "\n",
    "class CustomCallback(Callback):\n",
    "    def __init__(self, validation_data, batch_size):\n",
    "        self.val_x, self.val_y = validation_data\n",
    "        self.batch_size = batch_size\n",
    "        super(Callback, self).__init__()\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        y_pred = self.model.predict(self.val_x, verbose=0, batch_size=self.batch_size)\n",
    "        if type(y_pred)==type([]):\n",
    "            y_pred = y_pred[0]\n",
    "        precision, recall, thresholds = precision_recall_curve(self.val_y, y_pred)\n",
    "        pr_auc = auc(recall, precision)\n",
    "        roc_auc = roc_auc_score(self.val_y, y_pred)\n",
    "        logs['custom_metric'] = pr_auc + roc_auc\n",
    "        print ('val_aucs:', pr_auc, roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "favorite-swing",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "metallic-forth",
   "metadata": {},
   "source": [
    "## Define model architecture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "subsequent-deputy",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras.layers import Embedding, Activation, Dropout, Softmax, Layer, InputSpec, Input, Dense, Lambda, TimeDistributed, Concatenate, Add\n",
    "from tensorflow.keras import initializers, regularizers, constraints, Model\n",
    "from tensorflow.python.keras.utils import tf_utils\n",
    "from tensorflow.python.ops import array_ops\n",
    "from tensorflow import nn\n",
    "\n",
    "    \n",
    "class CVE(Layer):\n",
    "    def __init__(self, hid_units, output_dim):\n",
    "        self.hid_units = hid_units\n",
    "        self.output_dim = output_dim\n",
    "        super(CVE, self).__init__()\n",
    "        \n",
    "    def build(self, input_shape): \n",
    "        self.W1 = self.add_weight(name='CVE_W1',\n",
    "                            shape=(1, self.hid_units),\n",
    "                            initializer='glorot_uniform',\n",
    "                            trainable=True)\n",
    "        self.b1 = self.add_weight(name='CVE_b1',\n",
    "                            shape=(self.hid_units,),\n",
    "                            initializer='zeros',\n",
    "                            trainable=True)\n",
    "        self.W2 = self.add_weight(name='CVE_W2',\n",
    "                            shape=(self.hid_units, self.output_dim),\n",
    "                            initializer='glorot_uniform',\n",
    "                            trainable=True)\n",
    "        super(CVE, self).build(input_shape)\n",
    "        \n",
    "    def call(self, x):\n",
    "        x = K.expand_dims(x, axis=-1)\n",
    "        x = K.dot(K.tanh(K.bias_add(K.dot(x, self.W1), self.b1)), self.W2)\n",
    "        return x\n",
    "        \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape + (self.output_dim,)\n",
    "    \n",
    "    \n",
    "class Attention(Layer):\n",
    "    \n",
    "    def __init__(self, hid_dim):\n",
    "        self.hid_dim = hid_dim\n",
    "        super(Attention, self).__init__()\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        d = input_shape.as_list()[-1]\n",
    "        self.W = self.add_weight(shape=(d, self.hid_dim), name='Att_W',\n",
    "                                 initializer='glorot_uniform',\n",
    "                                 trainable=True)\n",
    "        self.b = self.add_weight(shape=(self.hid_dim,), name='Att_b',\n",
    "                                 initializer='zeros',\n",
    "                                 trainable=True)\n",
    "        self.u = self.add_weight(shape=(self.hid_dim,1), name='Att_u',\n",
    "                                 initializer='glorot_uniform',\n",
    "                                 trainable=True)\n",
    "        super(Attention, self).build(input_shape)\n",
    "        \n",
    "    def call(self, x, mask, mask_value=-1e30):\n",
    "        attn_weights = K.dot(K.tanh(K.bias_add(K.dot(x,self.W), self.b)), self.u)\n",
    "        mask = K.expand_dims(mask, axis=-1)\n",
    "        attn_weights = mask*attn_weights + (1-mask)*mask_value\n",
    "        attn_weights = K.softmax(attn_weights, axis=-2)\n",
    "        return attn_weights\n",
    "        \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[:-1] + (1,)\n",
    "    \n",
    "    \n",
    "class Transformer(Layer):\n",
    "    \n",
    "    def __init__(self, N=2, h=8, dk=None, dv=None, dff=None, dropout=0):\n",
    "        self.N, self.h, self.dk, self.dv, self.dff, self.dropout = N, h, dk, dv, dff, dropout\n",
    "        self.epsilon = K.epsilon() * K.epsilon()\n",
    "        super(Transformer, self).__init__()\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        d = input_shape.as_list()[-1]\n",
    "        if self.dk==None:\n",
    "            self.dk = d//self.h\n",
    "        if self.dv==None:\n",
    "            self.dv = d//self.h\n",
    "        if self.dff==None:\n",
    "            self.dff = 2*d\n",
    "        self.Wq = self.add_weight(shape=(self.N, self.h, d, self.dk), name='Wq',\n",
    "                                 initializer='glorot_uniform', trainable=True)\n",
    "        self.Wk = self.add_weight(shape=(self.N, self.h, d, self.dk), name='Wk',\n",
    "                                 initializer='glorot_uniform', trainable=True)\n",
    "        self.Wv = self.add_weight(shape=(self.N, self.h, d, self.dv), name='Wv',\n",
    "                                 initializer='glorot_uniform', trainable=True)\n",
    "        self.Wo = self.add_weight(shape=(self.N, self.dv*self.h, d), name='Wo',\n",
    "                                 initializer='glorot_uniform', trainable=True)\n",
    "        self.W1 = self.add_weight(shape=(self.N, d, self.dff), name='W1',\n",
    "                                 initializer='glorot_uniform', trainable=True)\n",
    "        self.b1 = self.add_weight(shape=(self.N, self.dff), name='b1',\n",
    "                                 initializer='zeros', trainable=True)\n",
    "        self.W2 = self.add_weight(shape=(self.N, self.dff, d), name='W2',\n",
    "                                 initializer='glorot_uniform', trainable=True)\n",
    "        self.b2 = self.add_weight(shape=(self.N, d), name='b2',\n",
    "                                 initializer='zeros', trainable=True)\n",
    "        self.gamma = self.add_weight(shape=(2*self.N,), name='gamma',\n",
    "                                 initializer='ones', trainable=True)\n",
    "        self.beta = self.add_weight(shape=(2*self.N,), name='beta',\n",
    "                                 initializer='zeros', trainable=True)\n",
    "        super(Transformer, self).build(input_shape)\n",
    "        \n",
    "    def call(self, x, mask, mask_value=-1e-30):\n",
    "        mask = K.expand_dims(mask, axis=-2)\n",
    "        for i in range(self.N):\n",
    "            # MHA\n",
    "            mha_ops = []\n",
    "            for j in range(self.h):\n",
    "                q = K.dot(x, self.Wq[i,j,:,:])\n",
    "                k = K.permute_dimensions(K.dot(x, self.Wk[i,j,:,:]), (0,2,1))\n",
    "                v = K.dot(x, self.Wv[i,j,:,:])\n",
    "                A = K.batch_dot(q,k)\n",
    "                # Mask unobserved steps.\n",
    "                A = mask*A + (1-mask)*mask_value\n",
    "                # Mask for attention dropout.\n",
    "                def dropped_A():\n",
    "                    dp_mask = K.cast((K.random_uniform(shape=array_ops.shape(A))>=self.dropout), K.floatx())\n",
    "                    return A*dp_mask + (1-dp_mask)*mask_value\n",
    "                A = tf_utils.smart_cond(K.learning_phase(), dropped_A, lambda: array_ops.identity(A))\n",
    "                A = K.softmax(A, axis=-1)\n",
    "                mha_ops.append(K.batch_dot(A,v))\n",
    "            conc = K.concatenate(mha_ops, axis=-1)\n",
    "            proj = K.dot(conc, self.Wo[i,:,:])\n",
    "            # Dropout.\n",
    "            proj = tf_utils.smart_cond(K.learning_phase(), lambda: array_ops.identity(nn.dropout(proj, rate=self.dropout)),\\\n",
    "                                       lambda: array_ops.identity(proj))\n",
    "            # Add & LN\n",
    "            x = x+proj\n",
    "            mean = K.mean(x, axis=-1, keepdims=True)\n",
    "            variance = K.mean(K.square(x - mean), axis=-1, keepdims=True)\n",
    "            std = K.sqrt(variance + self.epsilon)\n",
    "            x = (x - mean) / std\n",
    "            x = x*self.gamma[2*i] + self.beta[2*i]\n",
    "            # FFN\n",
    "            ffn_op = K.bias_add(K.dot(K.relu(K.bias_add(K.dot(x, self.W1[i,:,:]), self.b1[i,:])), \n",
    "                           self.W2[i,:,:]), self.b2[i,:,])\n",
    "            # Dropout.\n",
    "            ffn_op = tf_utils.smart_cond(K.learning_phase(), lambda: array_ops.identity(nn.dropout(ffn_op, rate=self.dropout)),\\\n",
    "                                       lambda: array_ops.identity(ffn_op))\n",
    "            # Add & LN\n",
    "            x = x+ffn_op\n",
    "            mean = K.mean(x, axis=-1, keepdims=True)\n",
    "            variance = K.mean(K.square(x - mean), axis=-1, keepdims=True)\n",
    "            std = K.sqrt(variance + self.epsilon)\n",
    "            x = (x - mean) / std\n",
    "            x = x*self.gamma[2*i+1] + self.beta[2*i+1]            \n",
    "        return x\n",
    "        \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape\n",
    "\n",
    "\n",
    "def build_strats(D, max_len, V, d, N, he, dropout, forecast=False):\n",
    "    demo = Input(shape=(D,))\n",
    "    demo_enc = Dense(2*d, activation='tanh')(demo)\n",
    "    demo_enc = Dense(d, activation='tanh')(demo_enc)\n",
    "    varis = Input(shape=(max_len,))\n",
    "    values = Input(shape=(max_len,))\n",
    "    times = Input(shape=(max_len,))\n",
    "    varis_emb = Embedding(V+1, d)(varis)\n",
    "    cve_units = int(np.sqrt(d))\n",
    "    values_emb = CVE(cve_units, d)(values)\n",
    "    times_emb = CVE(cve_units, d)(times)\n",
    "    comb_emb = Add()([varis_emb, values_emb, times_emb]) # b, L, d\n",
    "#     demo_enc = Lambda(lambda x:K.expand_dims(x, axis=-2))(demo_enc) # b, 1, d\n",
    "#     comb_emb = Concatenate(axis=-2)([demo_enc, comb_emb]) # b, L+1, d\n",
    "    mask = Lambda(lambda x:K.clip(x,0,1))(varis) # b, L\n",
    "#     mask = Lambda(lambda x:K.concatenate((K.ones_like(x)[:,0:1], x), axis=-1))(mask) # b, L+1\n",
    "    cont_emb = Transformer(N, he, dk=None, dv=None, dff=None, dropout=dropout)(comb_emb, mask=mask)\n",
    "    attn_weights = Attention(2*d)(cont_emb, mask=mask)\n",
    "    fused_emb = Lambda(lambda x:K.sum(x[0]*x[1], axis=-2))([cont_emb, attn_weights])\n",
    "    conc = Concatenate(axis=-1)([fused_emb, demo_enc])\n",
    "    fore_op = Dense(V)(conc)\n",
    "    op = Dense(1, activation='sigmoid')(fore_op)\n",
    "    model = Model([demo, times, values, varis], op)\n",
    "    if forecast:\n",
    "        fore_model = Model([demo, times, values, varis], fore_op)\n",
    "        return [model, fore_model]\n",
    "    return model\n",
    "\n",
    "# To tune:\n",
    "# 1. Transformer parameters. (N, h, dropout)\n",
    "# 2. Normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "quick-fetish",
   "metadata": {},
   "source": [
    "## Pretrain on forecasting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "temporal-grain",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/anand/miniconda3/envs/strats/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "WARNING:tensorflow:From /home/anand/miniconda3/envs/strats/lib/python3.7/site-packages/tensorflow_core/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 880)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            [(None, 880)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            [(None, 880)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 880, 50)      6400        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "cve (CVE)                       (None, 880, 50)      364         input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "cve_1 (CVE)                     (None, 880, 50)      364         input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 880, 50)      0           embedding[0][0]                  \n",
      "                                                                 cve[0][0]                        \n",
      "                                                                 cve_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "lambda (Lambda)                 (None, 880)          0           input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "transformer (Transformer)       (None, 880, 50)      39508       add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            [(None, 2)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "attention (Attention)           (None, 880, 1)       5200        transformer[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 100)          300         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 50)           0           transformer[0][0]                \n",
      "                                                                 attention[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 50)           5050        dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 100)          0           lambda_1[0][0]                   \n",
      "                                                                 dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 127)          12827       concatenate[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 70,013\n",
      "Trainable params: 70,013\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/31 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/anand/miniconda3/envs/strats/lib/python3.7/site-packages/tensorflow_core/python/ops/math_grad.py:1375: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-04 22:36:20.814305: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
      "2024-01-04 22:36:20.847243: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties: \n",
      "name: NVIDIA GeForce RTX 2080 Ti major: 7 minor: 5 memoryClockRate(GHz): 1.545\n",
      "pciBusID: 0000:19:00.0\n",
      "2024-01-04 22:36:20.847546: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 1 with properties: \n",
      "name: NVIDIA GeForce RTX 2080 Ti major: 7 minor: 5 memoryClockRate(GHz): 1.545\n",
      "pciBusID: 0000:1a:00.0\n",
      "2024-01-04 22:36:20.847829: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 2 with properties: \n",
      "name: NVIDIA GeForce RTX 2080 Ti major: 7 minor: 5 memoryClockRate(GHz): 1.545\n",
      "pciBusID: 0000:67:00.0\n",
      "2024-01-04 22:36:20.848108: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 3 with properties: \n",
      "name: NVIDIA GeForce RTX 2080 Ti major: 7 minor: 5 memoryClockRate(GHz): 1.545\n",
      "pciBusID: 0000:68:00.0\n",
      "2024-01-04 22:36:20.848296: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2024-01-04 22:36:20.849448: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
      "2024-01-04 22:36:20.850421: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
      "2024-01-04 22:36:20.850707: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
      "2024-01-04 22:36:20.852023: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
      "2024-01-04 22:36:20.852994: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
      "2024-01-04 22:36:20.856025: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2024-01-04 22:36:20.858103: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0, 1, 2, 3\n",
      "2024-01-04 22:36:20.859009: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 AVX512F FMA\n",
      "2024-01-04 22:36:20.884870: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 3499910000 Hz\n",
      "2024-01-04 22:36:20.886061: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56521519b3d0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2024-01-04 22:36:20.886086: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2024-01-04 22:36:21.293710: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties: \n",
      "name: NVIDIA GeForce RTX 2080 Ti major: 7 minor: 5 memoryClockRate(GHz): 1.545\n",
      "pciBusID: 0000:19:00.0\n",
      "2024-01-04 22:36:21.293988: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 1 with properties: \n",
      "name: NVIDIA GeForce RTX 2080 Ti major: 7 minor: 5 memoryClockRate(GHz): 1.545\n",
      "pciBusID: 0000:1a:00.0\n",
      "2024-01-04 22:36:21.294232: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 2 with properties: \n",
      "name: NVIDIA GeForce RTX 2080 Ti major: 7 minor: 5 memoryClockRate(GHz): 1.545\n",
      "pciBusID: 0000:67:00.0\n",
      "2024-01-04 22:36:21.294468: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 3 with properties: \n",
      "name: NVIDIA GeForce RTX 2080 Ti major: 7 minor: 5 memoryClockRate(GHz): 1.545\n",
      "pciBusID: 0000:68:00.0\n",
      "2024-01-04 22:36:21.294514: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2024-01-04 22:36:21.294524: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
      "2024-01-04 22:36:21.294532: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
      "2024-01-04 22:36:21.294541: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
      "2024-01-04 22:36:21.294549: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
      "2024-01-04 22:36:21.294557: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
      "2024-01-04 22:36:21.294566: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2024-01-04 22:36:21.296347: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0, 1, 2, 3\n",
      "2024-01-04 22:36:21.296385: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2024-01-04 22:36:21.297312: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2024-01-04 22:36:21.297322: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165]      0 1 2 3 \n",
      "2024-01-04 22:36:21.297327: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0:   N N N N \n",
      "2024-01-04 22:36:21.297330: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 1:   N N N N \n",
      "2024-01-04 22:36:21.297333: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 2:   N N N N \n",
      "2024-01-04 22:36:21.297336: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 3:   N N N N \n",
      "2024-01-04 22:36:21.298816: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10306 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:19:00.0, compute capability: 7.5)\n",
      "2024-01-04 22:36:21.299646: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 10306 MB memory) -> physical GPU (device: 1, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:1a:00.0, compute capability: 7.5)\n",
      "2024-01-04 22:36:21.300418: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:2 with 10306 MB memory) -> physical GPU (device: 2, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:67:00.0, compute capability: 7.5)\n",
      "2024-01-04 22:36:21.301226: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:3 with 6414 MB memory) -> physical GPU (device: 3, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:68:00.0, compute capability: 7.5)\n",
      "2024-01-04 22:36:21.303146: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x565213956b60 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2024-01-04 22:36:21.303159: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA GeForce RTX 2080 Ti, Compute Capability 7.5\n",
      "2024-01-04 22:36:21.303163: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (1): NVIDIA GeForce RTX 2080 Ti, Compute Capability 7.5\n",
      "2024-01-04 22:36:21.303167: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (2): NVIDIA GeForce RTX 2080 Ti, Compute Capability 7.5\n",
      "2024-01-04 22:36:21.303170: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (3): NVIDIA GeForce RTX 2080 Ti, Compute Capability 7.5\n",
      "2024-01-04 22:36:22.936190: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
      "0.509583: 100%|██████████| 31/31 [00:06<00:00,  4.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 134s 815us/sample - loss: 15.3906\n",
      "Epoch 0 loss 16.155353223663017 val loss 15.390635881742385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.533016: 100%|██████████| 31/31 [00:03<00:00,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:31 - loss: 16.6957"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 823us/sample - loss: 14.7036\n",
      "Epoch 1 loss 16.898247001097374 val loss 14.703593796818696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.465561: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:26 - loss: 16.4715"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 14.3671\n",
      "Epoch 2 loss 14.759737938949742 val loss 14.367115212091045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.432360: 100%|██████████| 31/31 [00:03<00:00,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 16.5812"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 14.2765\n",
      "Epoch 3 loss 13.7071483179466 val loss 14.276493298308415\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.436252: 100%|██████████| 31/31 [00:03<00:00,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 16.3258"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 14.0746\n",
      "Epoch 4 loss 13.830525034481717 val loss 14.074550019571788\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.451743: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:35 - loss: 16.1870"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 826us/sample - loss: 14.0181\n",
      "Epoch 5 loss 14.321638189886034 val loss 14.018079650491922\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.498388: 100%|██████████| 31/31 [00:03<00:00,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 16.1384"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 826us/sample - loss: 14.1965\n",
      "Epoch 6 loss 15.800438005899645 val loss 14.196548616142863\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.439580: 100%|██████████| 31/31 [00:03<00:00,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:30 - loss: 16.1193"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.8907\n",
      "Epoch 7 loss 13.936058186009987 val loss 13.89072126619291\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.441804: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 16.1306"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.8461\n",
      "Epoch 8 loss 14.006565746818621 val loss 13.846117809271439\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.427244: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 16.1524"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.8557\n",
      "Epoch 9 loss 13.544966329987517 val loss 13.855738082123812\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.437858: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:30 - loss: 16.2995"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.7637\n",
      "Epoch 10 loss 13.881468373721408 val loss 13.763683108155785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.471329: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 16.0232"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.8964\n",
      "Epoch 11 loss 14.942589199420103 val loss 13.896432739883062\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.413395: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 16.2775"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.7250\n",
      "Epoch 12 loss 13.105909115506202 val loss 13.72503128103152\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.469608: 100%|██████████| 31/31 [00:03<00:00,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:31 - loss: 16.1446"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.6611\n",
      "Epoch 13 loss 14.888011484047802 val loss 13.661114588313845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.448202: 100%|██████████| 31/31 [00:03<00:00,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 15.4449"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.5324\n",
      "Epoch 14 loss 14.20937493707716 val loss 13.532373841720506\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.428674: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:24 - loss: 15.6199"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 13.4196\n",
      "Epoch 15 loss 13.59028446158183 val loss 13.419561691151296\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.406491: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 16.0487"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 13.3639\n",
      "Epoch 16 loss 12.88702560896726 val loss 13.363872146949072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.430252: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 15.3299"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 13.2612\n",
      "Epoch 17 loss 13.640313878010229 val loss 13.261208889751765\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.425796: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:21 - loss: 15.3098"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 13.1661\n",
      "Epoch 18 loss 13.49906512388249 val loss 13.166139878175535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.527285: 100%|██████████| 31/31 [00:03<00:00,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:31 - loss: 16.0533"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 12.9789\n",
      "Epoch 19 loss 16.716561669418493 val loss 12.97886230580706\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.433062: 100%|██████████| 31/31 [00:03<00:00,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:24 - loss: 15.2488"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.8282\n",
      "Epoch 20 loss 13.729398023467702 val loss 12.828230982699905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.371129: 100%|██████████| 31/31 [00:03<00:00,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 14.3810"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.9788\n",
      "Epoch 21 loss 11.765940542319386 val loss 12.97877809753055\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.385542: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 15.1915"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.7349\n",
      "Epoch 22 loss 12.222877486710695 val loss 12.734870977271743\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.382063: 100%|██████████| 31/31 [00:03<00:00,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:23 - loss: 14.5733"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.7363\n",
      "Epoch 23 loss 12.11259008977831 val loss 12.736298450772363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.409298: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:26 - loss: 15.1702"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.6050\n",
      "Epoch 24 loss 12.976000363064795 val loss 12.605027842260355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.416534: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:27 - loss: 14.2627"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.5357\n",
      "Epoch 25 loss 13.20542539616221 val loss 12.53571465257074\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.386076: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 14.8533"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.4329\n",
      "Epoch 26 loss 12.239809803127013 val loss 12.432923157332693\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.384618: 100%|██████████| 31/31 [00:03<00:00,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:19 - loss: 14.4029"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.5825\n",
      "Epoch 27 loss 12.19357405200447 val loss 12.582514035187955\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.377194: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:30 - loss: 13.9715"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.2976\n",
      "Epoch 28 loss 11.95821533203125 val loss 12.29761424257538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.374384: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:31 - loss: 14.6155"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 826us/sample - loss: 12.3900\n",
      "Epoch 29 loss 11.86913382933312 val loss 12.390022971097663\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.360635: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:35 - loss: 14.3451"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 826us/sample - loss: 12.2310\n",
      "Epoch 30 loss 11.433246982220522 val loss 12.23102576932685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.371641: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 13.4767"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 12.1348\n",
      "Epoch 31 loss 11.782174446656532 val loss 12.13481254390349\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.376487: 100%|██████████| 31/31 [00:03<00:00,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 13.6671"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.8881\n",
      "Epoch 32 loss 11.935806683412531 val loss 11.888127693472875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.385418: 100%|██████████| 31/31 [00:03<00:00,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 13.5288"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.9385\n",
      "Epoch 33 loss 12.218937415683392 val loss 11.938478970501894\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.347354: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 13.6696"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 11.6976\n",
      "Epoch 34 loss 11.01220621325306 val loss 11.697611344608822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.377664: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:29 - loss: 13.8896"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.6984\n",
      "Epoch 35 loss 11.973112440600838 val loss 11.698384129320587\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.352415: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:30 - loss: 13.6524"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.8125\n",
      "Epoch 36 loss 11.172653622971367 val loss 11.812542824397399\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.367957: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 13.2544"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.5266\n",
      "Epoch 37 loss 11.665374504168009 val loss 11.52657783711655\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.357299: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 13.0314"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.4264\n",
      "Epoch 38 loss 11.327468588917526 val loss 11.426367306683535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.365967: 100%|██████████| 31/31 [00:03<00:00,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:30 - loss: 13.2825"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 11.5319\n",
      "Epoch 39 loss 11.602285436256645 val loss 11.531853572185808\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.372570: 100%|██████████| 31/31 [00:03<00:00,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:31 - loss: 13.3577"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.3623\n",
      "Epoch 40 loss 11.81161857683634 val loss 11.362326800164146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.373464: 100%|██████████| 31/31 [00:03<00:00,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 12.6347"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.4758\n",
      "Epoch 41 loss 11.839951694134584 val loss 11.475809438876041\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.381880: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 13.7227"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.4866\n",
      "Epoch 42 loss 12.106775028189434 val loss 11.486592375791853\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.363228: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 13.0719"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 11.4650\n",
      "Epoch 43 loss 11.515442067569063 val loss 11.464966784277225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.357675: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:18 - loss: 12.3950"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.2657\n",
      "Epoch 44 loss 11.339415874677835 val loss 11.265714987990366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.327372: 100%|██████████| 31/31 [00:03<00:00,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 13.0436"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.3035\n",
      "Epoch 45 loss 10.3787142724106 val loss 11.303453650964935\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.334220: 100%|██████████| 31/31 [00:03<00:00,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 12.7610"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.1934\n",
      "Epoch 46 loss 10.595818258069226 val loss 11.19338705311989\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.336590: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 12.8442"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.1682\n",
      "Epoch 47 loss 10.670934287788942 val loss 11.168206681526774\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.388384: 100%|██████████| 31/31 [00:03<00:00,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 12.8542"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.1842\n",
      "Epoch 48 loss 12.312968570669902 val loss 11.184167906908373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.369577: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:30 - loss: 12.6714"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 11.2597\n",
      "Epoch 49 loss 11.716734707232604 val loss 11.25973941515588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.350165: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:22 - loss: 12.5291"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.1493\n",
      "Epoch 50 loss 11.101328600067454 val loss 11.149257828424783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.350272: 100%|██████████| 31/31 [00:03<00:00,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 12.6883"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 11.1190\n",
      "Epoch 51 loss 11.104703230710374 val loss 11.119015704163314\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.335847: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:26 - loss: 12.2282"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 11.1485\n",
      "Epoch 52 loss 10.647377847887807 val loss 11.148495337733017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.360119: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:35 - loss: 12.6086"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.9473\n",
      "Epoch 53 loss 11.41687093518444 val loss 10.947301481772211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.318061: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 12.3290"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 10.8890\n",
      "Epoch 54 loss 10.083517896514579 val loss 10.888959506329245\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.348762: 100%|██████████| 31/31 [00:03<00:00,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:35 - loss: 12.0293"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.9151\n",
      "Epoch 55 loss 11.056834395890384 val loss 10.915118930550744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.402915: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:31 - loss: 11.6674"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.9558\n",
      "Epoch 56 loss 12.773653718614087 val loss 10.955843612504122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.373460: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:31 - loss: 12.0267"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.7918\n",
      "Epoch 57 loss 11.83982526641531 val loss 10.791751099781079\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.360264: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:31 - loss: 11.6775"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.9233\n",
      "Epoch 58 loss 11.421482943505357 val loss 10.923268658824675\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.337460: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 12.2882"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.8153\n",
      "Epoch 59 loss 10.69853466859798 val loss 10.81534376902473\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.303517: 100%|██████████| 31/31 [00:03<00:00,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 11.9451"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 10.8401\n",
      "Epoch 60 loss 9.622442438184601 val loss 10.840052501852234\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.336564: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 12.1273"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.8552\n",
      "Epoch 61 loss 10.670107623228093 val loss 10.855245061914973\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.332711: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:35 - loss: 12.3148"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.7691\n",
      "Epoch 62 loss 10.547980617247905 val loss 10.769086919744565\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.369090: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:21 - loss: 11.5450"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.6909\n",
      "Epoch 63 loss 11.70128335854442 val loss 10.690874644498033\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.445026: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 11.5552"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 10.9446\n",
      "Epoch 64 loss 14.108692837744645 val loss 10.944607746556395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.334631: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 12.0039"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.7112\n",
      "Epoch 65 loss 10.608852504454937 val loss 10.711176126748626\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.305490: 100%|██████████| 31/31 [00:03<00:00,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 11.7591"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.6017\n",
      "Epoch 66 loss 9.684980270543049 val loss 10.601715108691373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.332134: 100%|██████████| 31/31 [00:03<00:00,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:57 - loss: 11.3412"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.6370\n",
      "Epoch 67 loss 10.529672524363724 val loss 10.636982497036422\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.333409: 100%|██████████| 31/31 [00:03<00:00,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 11.2090"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.5270\n",
      "Epoch 68 loss 10.570103895049733 val loss 10.527000940840498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.330279: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 3:06 - loss: 11.4922"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.4344\n",
      "Epoch 69 loss 10.470860841102207 val loss 10.434430847276314\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.348651: 100%|██████████| 31/31 [00:03<00:00,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:30 - loss: 11.5506"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.5230\n",
      "Epoch 70 loss 11.05331691466656 val loss 10.523038616714503\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.340562: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:24 - loss: 11.4397"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.4922\n",
      "Epoch 71 loss 10.796872844892679 val loss 10.492165149174825\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.344153: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:34 - loss: 10.8559"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.5149\n",
      "Epoch 72 loss 10.910700705616744 val loss 10.514865938977536\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.392039: 100%|██████████| 31/31 [00:03<00:00,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 12.0618"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.6968\n",
      "Epoch 73 loss 12.428857075799371 val loss 10.696763389111057\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.305099: 100%|██████████| 31/31 [00:03<00:00,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:33 - loss: 10.6082"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 825us/sample - loss: 10.7927\n",
      "Epoch 74 loss 9.672586264069547 val loss 10.792684867771007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.339839: 100%|██████████| 31/31 [00:03<00:00,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    32/164778 [..............................] - ETA: 2:32 - loss: 11.6411"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164778/164778 [==============================] - 136s 824us/sample - loss: 10.5296\n",
      "Epoch 75 loss 10.773958501127577 val loss 10.529612213410442\n"
     ]
    }
   ],
   "source": [
    "lr, batch_size, samples_per_epoch, patience = 0.0005, 32, 970, 5\n",
    "d, N, he, dropout = 50, 2, 4, 0.2\n",
    "model, fore_model =  build_strats(D, fore_max_len, V, d, N, he, dropout, forecast=True)\n",
    "print (fore_model.summary())\n",
    "fore_model.compile(loss=forecast_loss, optimizer=Adam(lr))\n",
    "\n",
    "# Pretrain fore_model.\n",
    "best_val_loss = np.inf\n",
    "N_fore = len(fore_train_op)\n",
    "fore_savepath = 'mimic_iv_24h_strats_no_interp_with_ss_fore.h5'\n",
    "best_epoch = None\n",
    "for e in range(1000):\n",
    "    e_indices = np.random.choice(range(N_fore), size=samples_per_epoch, replace=False)\n",
    "    e_loss = 0\n",
    "    pbar = tqdm(range(0, len(e_indices), batch_size))\n",
    "    for start in pbar:\n",
    "        ind = e_indices[start:start+batch_size]\n",
    "        e_loss += fore_model.train_on_batch([ip[ind] for ip in fore_train_ip], fore_train_op[ind])\n",
    "        pbar.set_description('%f'%(e_loss/(start+1)))\n",
    "    val_loss = fore_model.evaluate(fore_valid_ip, fore_valid_op, batch_size=batch_size, verbose=1)\n",
    "    print ('Epoch', e, 'loss', e_loss*batch_size/samples_per_epoch, 'val loss', val_loss)\n",
    "    if best_epoch is None or val_loss < best_val_loss:\n",
    "        best_val_loss = val_loss\n",
    "        fore_model.save_weights(fore_savepath)\n",
    "        best_epoch = e\n",
    "    if best_epoch is not None and (e-best_epoch)>patience:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "wrapped-girlfriend",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "western-theme",
   "metadata": {},
   "source": [
    "## Train on different % of labeled data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "czech-reality",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{10: 10, 20: 10, 30: 10, 40: 10, 50: 10}\n",
      "Repeat 0 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat0_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5729val_aucs: 0.4216640589984951 0.8377942576490633\n",
      "2917/2917 [==============================] - 17s 6ms/sample - loss: 0.5724\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5112val_aucs: 0.43626429505401154 0.8425889888638816\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.5121\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4987val_aucs: 0.4441764134720048 0.8432395803328632\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4980\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4842val_aucs: 0.45799297316156873 0.8512794574986939\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4836\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4497val_aucs: 0.4710901078084527 0.85278608009608\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4519\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4500val_aucs: 0.4681046927600799 0.8547195953697962\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4495\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4562val_aucs: 0.4802371013041965 0.857019660302008\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4557\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4414val_aucs: 0.47839666061938163 0.8571844807131803\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4411\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4422val_aucs: 0.48880921006018535 0.8576797230860864\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4420\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4182val_aucs: 0.4567035928666831 0.8526421551635753\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4178\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4266val_aucs: 0.4366980408735863 0.8413443408892295\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4269\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4184val_aucs: 0.4575030612603337 0.8516965859326561\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4177\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4097val_aucs: 0.47810976148711626 0.8545516504010662\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4097\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3917val_aucs: 0.45759211562407454 0.8476711793740146\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3911\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4037val_aucs: 0.43085142627586176 0.8416163726816028\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4035\n",
      "Epoch 16/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3766val_aucs: 0.44913002279222153 0.8433223811081438\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3765\n",
      "Epoch 17/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3623val_aucs: 0.45598680843702144 0.849214906092399\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3628\n",
      "Epoch 18/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3731val_aucs: 0.4574406681562598 0.8502112493836321\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3731\n",
      "Epoch 19/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3500val_aucs: 0.4422354886081802 0.84288689589853\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3501\n",
      "Test res 0.857711332284826 0.45282346469918255 0.4741966893865628\n",
      "Repeat 1 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat1_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5384val_aucs: 0.4163216504026391 0.826177152649283\n",
      "2917/2917 [==============================] - 18s 6ms/sample - loss: 0.5383\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4891val_aucs: 0.43396454778613097 0.8331003910579069\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4890\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4629val_aucs: 0.4338384150535289 0.8423925323074369\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4626\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4655val_aucs: 0.442706183983323 0.8474294167329822\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4653\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4501val_aucs: 0.4390385945492126 0.8445909514765975\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4495\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4320val_aucs: 0.43662283890901765 0.8404889932577906\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4320\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4126val_aucs: 0.43565033868890973 0.8393442334825635\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4124\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4140val_aucs: 0.43251506866047046 0.8461602605099864\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4140\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3994val_aucs: 0.43527865843841756 0.8410189963335268\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3991\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3944val_aucs: 0.4417221488239635 0.845836087663368\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3941\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3734val_aucs: 0.428888930350679 0.840316361452724\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3732\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3786val_aucs: 0.43514329130554835 0.8463504679513154\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3792\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3586val_aucs: 0.422402733280924 0.8371954166646325\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3583\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3714val_aucs: 0.4240873242504824 0.843962720122639\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3713\n",
      "Test res 0.8461274056643416 0.42316653696086853 0.4404296875\n",
      "Repeat 2 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat2_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5903val_aucs: 0.4369687386872238 0.8378604592123187\n",
      "2917/2917 [==============================] - 18s 6ms/sample - loss: 0.5919\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5447val_aucs: 0.4505531697331797 0.8486077655019553\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.5444\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5092val_aucs: 0.46744205889115387 0.8529032510044964\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.5097\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4909val_aucs: 0.44824820445340463 0.841751216868705\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4908\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4831val_aucs: 0.4691580843702948 0.8530311625795176\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4841\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4866val_aucs: 0.4494479088540587 0.8461131968617723\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4875\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4689val_aucs: 0.45628737059598085 0.8539080891865899\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4688\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4690val_aucs: 0.45774280532620576 0.8533163760990875\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4695\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4378val_aucs: 0.4544208299033009 0.8513983859707365\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4391\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4475val_aucs: 0.4424827641686676 0.8458927202691026\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4469\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4277val_aucs: 0.46103795323562524 0.8557007064429353\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4270\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4177val_aucs: 0.4361041278109885 0.848395002660756\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4173\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4036val_aucs: 0.44303576645037857 0.8468598684756553\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.4033\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3913val_aucs: 0.43593846556244065 0.8422389407749878\n",
      "2917/2917 [==============================] - 15s 5ms/sample - loss: 0.3924\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3837val_aucs: 0.4220402525147625 0.8428853336197512\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3836\n",
      "Test res 0.8466180231231917 0.44514170923798757 0.4657534246575342\n",
      "Repeat 3 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat3_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5412val_aucs: 0.436601799377637 0.8365877878620703\n",
      "2917/2917 [==============================] - 19s 7ms/sample - loss: 0.5405\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4898val_aucs: 0.44977483554536446 0.8457985929726747\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4906\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4677val_aucs: 0.45060945184200135 0.8464145213812498\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4678\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4502val_aucs: 0.46227526858409 0.8480175170508082\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4530\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4458val_aucs: 0.4530093119203869 0.847866757148646\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4477\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4664val_aucs: 0.46311723905955915 0.8504818165396502\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4660\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4225val_aucs: 0.4477940317538271 0.8470704831835335\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4230\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4097val_aucs: 0.4563907308259772 0.8493978879943758\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4094\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4001val_aucs: 0.40300910018785924 0.8323532312319057\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4001\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3966val_aucs: 0.4312789734254866 0.8414109330221795\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3965\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3827val_aucs: 0.446794458566682 0.8466082439498314\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3827\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4078val_aucs: 0.44482550418903627 0.8481935663407038\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4091\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3798val_aucs: 0.43833669853891083 0.8462107416430291\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3799\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3569val_aucs: 0.4319750850629446 0.8405360569060045\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3567\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3496val_aucs: 0.414252389972972 0.8347688071513312\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3506\n",
      "Epoch 16/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3610val_aucs: 0.4125704315146084 0.8330123175917472\n",
      "2917/2917 [==============================] - 16s 6ms/sample - loss: 0.3609\n",
      "Test res 0.8436303170734636 0.44151585206835964 0.45401174168297453\n",
      "Repeat 4 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat4_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5890val_aucs: 0.423441511530586 0.834489745104453\n",
      "2917/2917 [==============================] - 20s 7ms/sample - loss: 0.5902\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5535val_aucs: 0.454317074121988 0.8437838392024567\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5531\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5416val_aucs: 0.43562615039303787 0.8438891953776075\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5414\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5149val_aucs: 0.4350524756856102 0.8439595955650812\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5150\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5018val_aucs: 0.4480267131689917 0.8445054167134537\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5020\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4911val_aucs: 0.4609013213834258 0.851899975101182\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4919\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4885val_aucs: 0.4224251969966782 0.8370470978230623\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4886\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4749val_aucs: 0.4424592797632617 0.8430939954791558\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4751\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4576val_aucs: 0.45345465562141873 0.8447467887847911\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4573\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4433val_aucs: 0.4610529195286198 0.8405012962031744\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4437\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4493val_aucs: 0.44463703436021906 0.8425355784581285\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4486\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4335val_aucs: 0.4533665618424088 0.8467043240947326\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4333\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4263val_aucs: 0.41715881472790917 0.842978484491942\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4271\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4237val_aucs: 0.4296478896150817 0.8374087653603737\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4238\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4337val_aucs: 0.4452885309396405 0.8466100015134577\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4336\n",
      "Epoch 16/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4041val_aucs: 0.4494017465787888 0.8406315511963638\n",
      "2917/2917 [==============================] - 16s 6ms/sample - loss: 0.4041\n",
      "Test res 0.8500218505756786 0.4395454866298488 0.46775745909528393\n",
      "Repeat 5 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat5_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5703val_aucs: 0.44828818939840026 0.8383852872395999\n",
      "2917/2917 [==============================] - 20s 7ms/sample - loss: 0.5697\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5225val_aucs: 0.4570026447445349 0.8429343501164386\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5218\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5036val_aucs: 0.4577990800287001 0.8428400275351635\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5035\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4943val_aucs: 0.45713366834659547 0.8414852389066001\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4942\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4845val_aucs: 0.44061177303835625 0.8477360139433382\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4846\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4855val_aucs: 0.46247795345145165 0.8463592557694468\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4848\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4858val_aucs: 0.46145716531056175 0.8438949562806048\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4855\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4622val_aucs: 0.4731966489944742 0.8510509742272822\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4617\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4448val_aucs: 0.46838912379258196 0.8489214906092399\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4449\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4393val_aucs: 0.47351468035846767 0.8533956617471158\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4407\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4320val_aucs: 0.4810589064867111 0.847998672063038\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4344\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4444val_aucs: 0.44787391366384005 0.839155393035166\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4439\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4246val_aucs: 0.45794986705774676 0.8420036225339186\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4243\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4143val_aucs: 0.4510761990246311 0.8330848659125417\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4139\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4027val_aucs: 0.45714366894351616 0.8407323181776017\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4043\n",
      "Epoch 16/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4353val_aucs: 0.4469294123651805 0.8304431501398728\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4351\n",
      "Epoch 17/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3939val_aucs: 0.4469802713775762 0.8386972547832582\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3937\n",
      "Epoch 18/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3881val_aucs: 0.4459113699386588 0.8401863993868057\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3874\n",
      "Epoch 19/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3795val_aucs: 0.4440089586417034 0.8289385780333839\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3797\n",
      "Epoch 20/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4177val_aucs: 0.4812187735694641 0.8525185398551962\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4172\n",
      "Epoch 21/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3710val_aucs: 0.4429335805940132 0.8293204575523974\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3705\n",
      "Epoch 22/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3590val_aucs: 0.42858254628328935 0.8203299337496156\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3585\n",
      "Epoch 23/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3696val_aucs: 0.4495667456014393 0.8387058473165421\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3693\n",
      "Epoch 24/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3343val_aucs: 0.4290570794379872 0.818860415273228\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3343\n",
      "Epoch 25/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3335val_aucs: 0.41635263051903487 0.81361545484282\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3340\n",
      "Epoch 26/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3082val_aucs: 0.42159661410061655 0.8197487660438708\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3080\n",
      "Epoch 27/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3054val_aucs: 0.38686197127932453 0.7946759492064112\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3053\n",
      "Epoch 28/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3278val_aucs: 0.403216438570431 0.8087915285433214\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3286\n",
      "Epoch 29/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3264val_aucs: 0.4070338311135819 0.8148593216780827\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3265\n",
      "Epoch 30/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3322val_aucs: 0.41141987711660444 0.8178896542969989\n",
      "2917/2917 [==============================] - 17s 6ms/sample - loss: 0.3316\n",
      "Test res 0.8440153434262182 0.4489054360800942 0.458252427184466\n",
      "Repeat 6 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat6_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5591val_aucs: 0.43290563976790886 0.8289656249847434\n",
      "2917/2917 [==============================] - 21s 7ms/sample - loss: 0.5586\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5174val_aucs: 0.4575749666061374 0.8387904056554492\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5170\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4938val_aucs: 0.46775674025160985 0.8418921148860757\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4932\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4710val_aucs: 0.45580091140004286 0.8412457220413124\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4708\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4638val_aucs: 0.46581696020512176 0.8475550825322585\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4636\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4581val_aucs: 0.46985806987441703 0.8477473404644851\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4577\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4415val_aucs: 0.48525652817699894 0.8499669480395843\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4412\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4258val_aucs: 0.48059625984113297 0.8475383856778093\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4255\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4317val_aucs: 0.4699802215469822 0.851080071669539\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4312\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4211val_aucs: 0.4759154089177066 0.8502194513472214\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4206\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4237val_aucs: 0.4806299834409444 0.8399554750548018\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4256\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3992val_aucs: 0.4661350767286798 0.8411826450356152\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4000\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3839val_aucs: 0.4629626962271252 0.8439959185466901\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3840\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3735val_aucs: 0.4629109076371211 0.8418981687163438\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3733\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3733val_aucs: 0.45026686941852884 0.839619194547647\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3734\n",
      "Epoch 16/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3670val_aucs: 0.429138347010088 0.8318345546773162\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3667\n",
      "Epoch 17/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3559val_aucs: 0.4256545641932136 0.8362184065732879\n",
      "2917/2917 [==============================] - 17s 6ms/sample - loss: 0.3563\n",
      "Test res 0.8529607530044542 0.4600679751821325 0.4717898832684825\n",
      "Repeat 7 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat7_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5358val_aucs: 0.44966738013547713 0.8395641242206914\n",
      "2917/2917 [==============================] - 21s 7ms/sample - loss: 0.5385\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4822val_aucs: 0.4599176449067229 0.8468533264332686\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4817\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4673val_aucs: 0.4540561229861929 0.8489524432575464\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4667\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4379val_aucs: 0.45549068505645995 0.847366535012132\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4375\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4412val_aucs: 0.4463880219936416 0.8456915768763211\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4412\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4295val_aucs: 0.44155276419278333 0.8406665071840415\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4292\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4043val_aucs: 0.4465553221443673 0.8434975516162264\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4043\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4086val_aucs: 0.46591648630320903 0.8525037958492205\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4083\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3856val_aucs: 0.4591002955245728 0.849808572028375\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3871\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3931val_aucs: 0.4471514394724452 0.8461809607038066\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3925\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3617val_aucs: 0.44653187416566137 0.8462334923277466\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3611\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3605val_aucs: 0.4607562957847616 0.8518735140043646\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3603\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3560val_aucs: 0.4256238866551568 0.8367823892124651\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3559\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3396val_aucs: 0.4460452145015729 0.8468585014817238\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3400\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3522val_aucs: 0.4417229199783657 0.8473983664422519\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3524\n",
      "Epoch 16/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3252val_aucs: 0.44221311500049704 0.8412521664412753\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3254\n",
      "Epoch 17/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3126val_aucs: 0.43974052471398245 0.8405090099546451\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3120\n",
      "Epoch 18/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3043val_aucs: 0.4256780512402166 0.8390415419691548\n",
      "2917/2917 [==============================] - 17s 6ms/sample - loss: 0.3057\n",
      "Test res 0.8463700431008608 0.43386910112187205 0.4461839530332681\n",
      "Repeat 8 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat8_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5738val_aucs: 0.43938160487938144 0.8315227824185052\n",
      "2917/2917 [==============================] - 22s 8ms/sample - loss: 0.5734\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5192val_aucs: 0.43809648950964075 0.8341463367003695\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5188\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4959val_aucs: 0.4426085663363257 0.8404097076097623\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4954\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4860val_aucs: 0.4486414213042928 0.848490594593539\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4862\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4734val_aucs: 0.43443561949466253 0.8349171259929015\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4730\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4498val_aucs: 0.45888855652679156 0.8500593177723856\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4497\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4557val_aucs: 0.4628387189667657 0.854436041771429\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4554\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4269val_aucs: 0.45394477407432515 0.8534832470011571\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4276\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4498val_aucs: 0.4565677140165574 0.8502298014441315\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4495\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4409val_aucs: 0.44508581803169145 0.8480113655781165\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4409\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3967val_aucs: 0.45387852822572433 0.8531605387908939\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3963\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3955val_aucs: 0.45017178669852986 0.8488187707795283\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3950\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3950val_aucs: 0.41408463798374145 0.8323628978318501\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3944\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4018val_aucs: 0.4267527428660468 0.8390878244779791\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4021\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3753val_aucs: 0.39251780435593364 0.8394168794457817\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3748\n",
      "Epoch 16/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3609val_aucs: 0.40108050661675815 0.828071025098985\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3603\n",
      "Epoch 17/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3567val_aucs: 0.4401396630042169 0.8472167515342067\n",
      "2917/2917 [==============================] - 17s 6ms/sample - loss: 0.3566\n",
      "Test res 0.8515331420407477 0.44494534185165324 0.4662756598240469\n",
      "Repeat 9 ld 10\n",
      "Num train: 2917 Num valid: 722\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat9_10ld.h5\n",
      "Train on 2917 samples\n",
      "Epoch 1/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.6333val_aucs: 0.4287898281974731 0.828896689433625\n",
      "2917/2917 [==============================] - 23s 8ms/sample - loss: 0.6328\n",
      "Epoch 2/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5440val_aucs: 0.4536557634690556 0.8425289387733182\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5434\n",
      "Epoch 3/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.5181val_aucs: 0.46300483514828644 0.85120798324456\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.5177\n",
      "Epoch 4/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4843val_aucs: 0.4657767618669207 0.8504073153703822\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4848\n",
      "Epoch 5/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4634val_aucs: 0.46014217904919347 0.8541893970092125\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4645\n",
      "Epoch 6/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4486val_aucs: 0.3485510893665216 0.8168420487333335\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4488\n",
      "Epoch 7/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4596val_aucs: 0.4615832746300579 0.8565720674318578\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4599\n",
      "Epoch 8/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4300val_aucs: 0.45617152480386686 0.8525359202066114\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4300\n",
      "Epoch 9/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4517val_aucs: 0.4518943387250734 0.8495795029024211\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4513\n",
      "Epoch 10/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4153val_aucs: 0.4317678204432932 0.8457070043792628\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4150\n",
      "Epoch 11/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3927val_aucs: 0.43140878074229805 0.8416714430085585\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3930\n",
      "Epoch 12/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.4065val_aucs: 0.4522273163041769 0.8476953946950871\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.4060\n",
      "Epoch 13/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3853val_aucs: 0.4498756072826152 0.8504475440489385\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3853\n",
      "Epoch 14/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3723val_aucs: 0.44786610781536373 0.8457935155666434\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3721\n",
      "Epoch 15/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3696val_aucs: 0.42800626784793266 0.83927754370719\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3695\n",
      "Epoch 16/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3604val_aucs: 0.3726079370356012 0.8244143163321602\n",
      "2917/2917 [==============================] - 16s 5ms/sample - loss: 0.3611\n",
      "Epoch 17/1000\n",
      "2912/2917 [============================>.] - ETA: 0s - loss: 0.3582val_aucs: 0.43769165889124845 0.8431182108002284\n",
      "2917/2917 [==============================] - 17s 6ms/sample - loss: 0.3581\n",
      "Test res 0.8594874719364173 0.46356156855069663 0.470703125\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)]}\n",
      "Repeat 0 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat0_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5133val_aucs: 0.4331099127360165 0.8292470304497899\n",
      "5834/5834 [==============================] - 33s 6ms/sample - loss: 0.5130\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4836val_aucs: 0.4732894619340622 0.8517212894658472\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4834\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4566val_aucs: 0.46781133691397253 0.8498943020763662\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4570\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4403val_aucs: 0.4842066384223008 0.8564750108627196\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4405\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4306val_aucs: 0.4857339460646848 0.8590208417753346\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4307\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4259val_aucs: 0.48503541572420195 0.8532198077420678\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4257\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4206val_aucs: 0.4859112140224388 0.8594529095001197\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4206\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4166val_aucs: 0.4909745371905067 0.8592464934164596\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4166\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4106val_aucs: 0.47272276172830524 0.8543711095596815\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4104\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4106val_aucs: 0.4828574446619369 0.8573450048577105\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4104\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3886val_aucs: 0.48669191296674474 0.8538185510840749\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3885\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3850val_aucs: 0.4817675393655173 0.8570967001742917\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3850\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3684val_aucs: 0.470811241095699 0.8529233653437746\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3683\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3664val_aucs: 0.48545192005690163 0.8551759760580777\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3662\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3712val_aucs: 0.4270546997563681 0.8383250418641892\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3713\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3507val_aucs: 0.44630569811712584 0.8482276435465681\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3506\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3549val_aucs: 0.46394706689180887 0.852674963017932\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3548\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3469val_aucs: 0.47782011623196696 0.8559088800902217\n",
      "5834/5834 [==============================] - 27s 5ms/sample - loss: 0.3469\n",
      "Test res 0.8601845893412412 0.4846748040205914 0.49122807017543857\n",
      "Repeat 1 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat1_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5133val_aucs: 0.44237384815104075 0.8355883200132794\n",
      "5834/5834 [==============================] - 34s 6ms/sample - loss: 0.5127\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4764val_aucs: 0.46402969475393263 0.8497825991436759\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4780\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4731val_aucs: 0.47319496472292283 0.8560922525618931\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4737\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4408val_aucs: 0.4691658046212741 0.8553689174872698\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4411\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4362val_aucs: 0.4824892417004309 0.8586598577349887\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4367\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4358val_aucs: 0.4937415148961088 0.8573000893428177\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4353\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4169val_aucs: 0.45826309992032727 0.84826191603728\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4164\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4187val_aucs: 0.4733848013593381 0.8544721694681905\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4184\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3972val_aucs: 0.4701974431424533 0.856405880026754\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3966\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4027val_aucs: 0.49806140533399734 0.859414633670037\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4044\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3884val_aucs: 0.4899322346237209 0.8584891787783956\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3882\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3737val_aucs: 0.48902127500038883 0.8553723349720987\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3737\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3844val_aucs: 0.45870052817214946 0.8499897963667252\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3842\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3763val_aucs: 0.47381234395581157 0.8551339898158953\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3759\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3623val_aucs: 0.4438143283951156 0.8520576676154258\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3623\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3625val_aucs: 0.47743644947196623 0.8575552289958943\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3622\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3643val_aucs: 0.4697466535682286 0.8528218172231472\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3640\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3485val_aucs: 0.4729836767956007 0.8561588446948429\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3480\n",
      "Epoch 19/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3424val_aucs: 0.48350143015002817 0.8573338736214111\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3423\n",
      "Epoch 20/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3385val_aucs: 0.4069483744256938 0.8363329411362649\n",
      "5834/5834 [==============================] - 28s 5ms/sample - loss: 0.3383\n",
      "Test res 0.8572707190280094 0.47857714902964793 0.4684159378036929\n",
      "Repeat 2 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat2_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5546val_aucs: 0.4060894113706068 0.8290849440264807\n",
      "5834/5834 [==============================] - 34s 6ms/sample - loss: 0.5550\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5200val_aucs: 0.4390684235606741 0.840326125695092\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.5200\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5077val_aucs: 0.45158221337016446 0.8477414819190643\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.5071\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4845val_aucs: 0.4533129589205034 0.8481291223410747\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4845\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4727val_aucs: 0.4653587693656543 0.8577556888917098\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4733\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4682val_aucs: 0.4646292838199314 0.852386136728686\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4680\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4593val_aucs: 0.46437531484557604 0.8584309838938822\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4591\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4434val_aucs: 0.48345885024029883 0.8603088429861006\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4435\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4376val_aucs: 0.4695940829296081 0.8578683682486365\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4374\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4251val_aucs: 0.46766069884812367 0.8528608741926192\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4247\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4193val_aucs: 0.4659686827077726 0.8570208320110921\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4188\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4252val_aucs: 0.4565810791054395 0.8560681348832442\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4252\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4125val_aucs: 0.420723835362763 0.8518198106713406\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4121\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4011val_aucs: 0.4526534901019665 0.8472739699944831\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4011\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4076val_aucs: 0.4635549445951826 0.8561563059918273\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4074\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3949val_aucs: 0.4365113913788133 0.8415766322151648\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3959\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3859val_aucs: 0.4316885429314109 0.8406448305659845\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3862\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3818val_aucs: 0.421158511084852 0.8407879743590996\n",
      "5834/5834 [==============================] - 28s 5ms/sample - loss: 0.3823\n",
      "Test res 0.8629070270010686 0.48161788949396644 0.47847358121330724\n",
      "Repeat 3 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat3_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5557val_aucs: 0.4461059496846125 0.8445054167134537\n",
      "5834/5834 [==============================] - 35s 6ms/sample - loss: 0.5555\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5222val_aucs: 0.45667452972032124 0.8483040975643098\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.5218\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4897val_aucs: 0.47240586521246586 0.8591693559017523\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4903\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4738val_aucs: 0.47616320064228895 0.861844660668167\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4733\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4654val_aucs: 0.48151461554277747 0.86375288655415\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4655\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4526val_aucs: 0.4772620502949039 0.8634206093863662\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4529\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4462val_aucs: 0.47627257822548374 0.8648650337598679\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4461\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4387val_aucs: 0.4643219946146777 0.8591642784957209\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4386\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4264val_aucs: 0.44216514490200964 0.8570454379018597\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4263\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4245val_aucs: 0.48202696713101584 0.8667616401974331\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4240\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4189val_aucs: 0.48468787792700774 0.8646931830941907\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4186\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4155val_aucs: 0.46866493887504385 0.8542190803060113\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4155\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4092val_aucs: 0.47015377416474197 0.8607887554984891\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4094\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3959val_aucs: 0.4728011357835214 0.8605378144696306\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3961\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4057val_aucs: 0.47777343509268966 0.8601012551933563\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4061\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3958val_aucs: 0.48361430423842144 0.8677221487191756\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3965\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3850val_aucs: 0.45481448745062675 0.8570728754229138\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3852\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3872val_aucs: 0.48582545324366816 0.8611898705749674\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3875\n",
      "Epoch 19/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3783val_aucs: 0.4618871473511456 0.8563052106879397\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3788\n",
      "Epoch 20/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3818val_aucs: 0.46344933283367534 0.8545225529588095\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3820\n",
      "Epoch 21/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3781val_aucs: 0.45842545024265163 0.8537430734905701\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3779\n",
      "Epoch 22/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3547val_aucs: 0.43550667381717584 0.8428616065107969\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3552\n",
      "Epoch 23/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3656val_aucs: 0.4398820703872508 0.8495535300177219\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3653\n",
      "Epoch 24/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3503val_aucs: 0.4225271109236841 0.8470191232686778\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3502\n",
      "Epoch 25/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3484val_aucs: 0.45216163563917916 0.8520306206640661\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3482\n",
      "Epoch 26/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3410val_aucs: 0.45030974926861406 0.8481943474800931\n",
      "5834/5834 [==============================] - 28s 5ms/sample - loss: 0.3411\n",
      "Test res 0.8682477519119253 0.4731242884979232 0.4726027397260274\n",
      "Repeat 4 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat4_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5546val_aucs: 0.45872658339228634 0.8432223952662952\n",
      "5834/5834 [==============================] - 36s 6ms/sample - loss: 0.5549\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4988val_aucs: 0.468786149674951 0.8462836805335183\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4983\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4737val_aucs: 0.47880211301641135 0.8579990138115208\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4745\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4635val_aucs: 0.48591101992697316 0.861055612242407\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4632\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4508val_aucs: 0.48538087688110587 0.8609240878977098\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4515\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4616val_aucs: 0.48977484738809074 0.8600680567693051\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4614\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4325val_aucs: 0.4865174640361796 0.8624188957618306\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4324\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4194val_aucs: 0.48699865571520445 0.8594142431003422\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4225\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4071val_aucs: 0.4931244272273437 0.8599557679820728\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4069\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4064val_aucs: 0.49069115030550947 0.8620701170244448\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4058\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4048val_aucs: 0.5049811661688525 0.8686094254231579\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4048\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3991val_aucs: 0.502034447671772 0.8639659423226204\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4009\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3878val_aucs: 0.4948032892060088 0.8649419759897281\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3885\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3806val_aucs: 0.4923025583347011 0.8613879870526148\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3802\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3755val_aucs: 0.4726729281069279 0.8554868695350756\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3752\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3573val_aucs: 0.48555451389092436 0.8608300582437058\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3577\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3674val_aucs: 0.489521795148903 0.8639208315228801\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3670\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3584val_aucs: 0.4637480611066738 0.848183997383183\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3582\n",
      "Epoch 19/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3442val_aucs: 0.4847827527037264 0.8603973070219549\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3437\n",
      "Epoch 20/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3423val_aucs: 0.4809673521798214 0.8565230509351703\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3420\n",
      "Epoch 21/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3360val_aucs: 0.49992208805824534 0.8601247870174634\n",
      "5834/5834 [==============================] - 28s 5ms/sample - loss: 0.3358\n",
      "Test res 0.8689009880782298 0.4902280268563286 0.49217221135029354\n",
      "Repeat 5 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat5_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5587val_aucs: 0.4413088603586532 0.8440709079280767\n",
      "5834/5834 [==============================] - 36s 6ms/sample - loss: 0.5588\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4955val_aucs: 0.46186743748705755 0.8501312802386382\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4959\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4703val_aucs: 0.47571750291191695 0.8526388353211702\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4698\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4592val_aucs: 0.4824769300697461 0.8595079798270753\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4589\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4498val_aucs: 0.4851231462796358 0.8613284251741697\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4493\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4328val_aucs: 0.49104724696075447 0.8621747897026301\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4327\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4248val_aucs: 0.49737848388505473 0.8654737366290907\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4252\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4162val_aucs: 0.5017946966123346 0.8645818707311953\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4164\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4126val_aucs: 0.48903300910880293 0.8638097144447321\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4124\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4039val_aucs: 0.4976801147668242 0.8616789614751817\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4035\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3919val_aucs: 0.5003339831390005 0.8652191828305562\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3919\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3846val_aucs: 0.5061832070916488 0.8650781871707619\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3846\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3774val_aucs: 0.5031030104739432 0.8607304629715519\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3791\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3768val_aucs: 0.48620854866446744 0.8582321839192693\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3771\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3717val_aucs: 0.5016188094892379 0.8623782765135797\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3712\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3740val_aucs: 0.485139930800436 0.8608324016618741\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3737\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3592val_aucs: 0.47760632157012145 0.8526042699031876\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3594\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3548val_aucs: 0.49635680682006156 0.8510794858149969\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3555\n",
      "Epoch 19/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3553val_aucs: 0.49233160725401043 0.8574637380449057\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3547\n",
      "Epoch 20/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3472val_aucs: 0.49326141827500025 0.8577041336920066\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3473\n",
      "Epoch 21/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3360val_aucs: 0.47102064965573576 0.8436254631912473\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3357\n",
      "Epoch 22/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3446val_aucs: 0.47896683953246866 0.8565161183230889\n",
      "5834/5834 [==============================] - 28s 5ms/sample - loss: 0.3443\n",
      "Test res 0.8691948903269182 0.4984916270687977 0.4931506849315068\n",
      "Repeat 6 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat6_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5644val_aucs: 0.42509553299226405 0.8363686782633318\n",
      "5834/5834 [==============================] - 37s 6ms/sample - loss: 0.5642\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5109val_aucs: 0.4680847884667538 0.8560496804651686\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.5106\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4909val_aucs: 0.48055545393681603 0.8603744586948138\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4907\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4775val_aucs: 0.4788608934577063 0.8613823237920412\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4771\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4633val_aucs: 0.4575245494166544 0.8600969589267144\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4647\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4661val_aucs: 0.4521885231144539 0.8514102983464256\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4660\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4548val_aucs: 0.46344320668236966 0.860310893476998\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4545\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4392val_aucs: 0.4953725875814561 0.8672400880734662\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4389\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4417val_aucs: 0.46879988673263534 0.8544793950075429\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4421\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4313val_aucs: 0.49978653446763494 0.8633393708898641\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4311\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4138val_aucs: 0.48183993694065197 0.8571073431984728\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4138\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4183val_aucs: 0.47949278336874446 0.8624136230709519\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4183\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3991val_aucs: 0.4932148422314191 0.8606488339053552\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3991\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3984val_aucs: 0.48383325479513634 0.8546531985216939\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3981\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3954val_aucs: 0.4746954314269509 0.8501657480141972\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3954\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3782val_aucs: 0.471929863536451 0.8496335968051398\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3782\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3803val_aucs: 0.48700102707222853 0.8585798885899945\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3810\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3732val_aucs: 0.4758797711498731 0.8547994668723667\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3730\n",
      "Epoch 19/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3639val_aucs: 0.46181121186063395 0.8503250028072197\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3638\n",
      "Epoch 20/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3638val_aucs: 0.4756989563257189 0.8533936112562186\n",
      "5834/5834 [==============================] - 29s 5ms/sample - loss: 0.3638\n",
      "Test res 0.8626630088963058 0.4936739803789682 0.48490749756572543\n",
      "Repeat 7 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat7_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5547val_aucs: 0.4369929929175535 0.8366235249891373\n",
      "5834/5834 [==============================] - 37s 6ms/sample - loss: 0.5553\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5064val_aucs: 0.4470402595412362 0.8417489710929604\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.5065\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4848val_aucs: 0.4667788919452598 0.8518684365983332\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4843\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4754val_aucs: 0.4660659734172207 0.8481984484618877\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4754\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4585val_aucs: 0.4684780002397796 0.8592587963618431\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4587\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4601val_aucs: 0.4732295199030375 0.8609668552792817\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4601\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4433val_aucs: 0.47179437380867834 0.8613885729071569\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4430\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4355val_aucs: 0.4834071856450492 0.8643700843142329\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4356\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4291val_aucs: 0.4704466154004224 0.862448383773782\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4285\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4259val_aucs: 0.48318861113297573 0.8669910998930814\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4259\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4145val_aucs: 0.4712642526595152 0.8610845143998165\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4140\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4142val_aucs: 0.4684833646361267 0.8618580376802114\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4138\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4111val_aucs: 0.45603401968411633 0.8605675954088532\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4113\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4017val_aucs: 0.4626734635758623 0.8553292746632556\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4020\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3982val_aucs: 0.46623509896541787 0.857948923248173\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3981\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4034val_aucs: 0.44390245466571027 0.8572909109549918\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4030\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3884val_aucs: 0.4726237152802831 0.8630746622792669\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3881\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3892val_aucs: 0.4484845057015816 0.8595068081179911\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3888\n",
      "Epoch 19/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3829val_aucs: 0.435370410053058 0.8526982995571916\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3827\n",
      "Epoch 20/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3701val_aucs: 0.4584826832833436 0.859822583716173\n",
      "5834/5834 [==============================] - 29s 5ms/sample - loss: 0.3696\n",
      "Test res 0.8656506549169798 0.48574406705194006 0.4882583170254403\n",
      "Repeat 8 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat8_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5557val_aucs: 0.42784980273095097 0.8379379872967205\n",
      "5834/5834 [==============================] - 38s 7ms/sample - loss: 0.5560\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5193val_aucs: 0.44755973051210746 0.8433948317865145\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.5189\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4855val_aucs: 0.4625031453786516 0.8509708097974409\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4852\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4737val_aucs: 0.4687100703915294 0.8548631297326064\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4732\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4729val_aucs: 0.4783217693469355 0.8596852008260548\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4726\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4605val_aucs: 0.46289150032283055 0.8586770428015564\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4607\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4495val_aucs: 0.46788852615047954 0.8563541295422035\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4490\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4455val_aucs: 0.4797066278031511 0.8612017829506565\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4466\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4451val_aucs: 0.4790853914837626 0.8626153523182751\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4446\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4269val_aucs: 0.475182207485623 0.8588246781461609\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4267\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4243val_aucs: 0.4589688823608055 0.8532144374087652\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4241\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4185val_aucs: 0.4429076261916402 0.8517857334654759\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4184\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4117val_aucs: 0.47465106025594567 0.8563072611788369\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.4120\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4031val_aucs: 0.48395863162209907 0.8588818966064375\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4031\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3905val_aucs: 0.46540070786046217 0.8522302994204923\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.3905\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3835val_aucs: 0.4627232143897863 0.8533647090988092\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.3836\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3861val_aucs: 0.46942850591128915 0.8488931743063726\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.3860\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3711val_aucs: 0.45693563066424553 0.8433087111688286\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.3713\n",
      "Epoch 19/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3662val_aucs: 0.4463745939678259 0.8439039393835834\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3660\n",
      "Epoch 20/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3654val_aucs: 0.4599049970328316 0.8504692206669954\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3652\n",
      "Epoch 21/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3494val_aucs: 0.4529302488911144 0.8420301812731595\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.3495\n",
      "Epoch 22/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3333val_aucs: 0.45915044308922104 0.8438116672932053\n",
      "5834/5834 [==============================] - 26s 5ms/sample - loss: 0.3333\n",
      "Epoch 23/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3430val_aucs: 0.44967185987019914 0.8376735716133945\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3430\n",
      "Epoch 24/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3269val_aucs: 0.4537802726799239 0.8485898969384219\n",
      "5834/5834 [==============================] - 29s 5ms/sample - loss: 0.3269\n",
      "Test res 0.863856686636332 0.477227414655119 0.4634858812074002\n",
      "Repeat 9 ld 20\n",
      "Num train: 5834 Num valid: 1444\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat9_20ld.h5\n",
      "Train on 5834 samples\n",
      "Epoch 1/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5543val_aucs: 0.4663174085607438 0.8496189504415879\n",
      "5834/5834 [==============================] - 39s 7ms/sample - loss: 0.5541\n",
      "Epoch 2/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.5033val_aucs: 0.474389400049799 0.8556600871946844\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.5028\n",
      "Epoch 3/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4815val_aucs: 0.4863735837302043 0.8586879787530084\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4812\n",
      "Epoch 4/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4772val_aucs: 0.465496073473955 0.8568659711271354\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4775\n",
      "Epoch 5/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4672val_aucs: 0.4807582464002664 0.8575381415717501\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4666\n",
      "Epoch 6/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4638val_aucs: 0.4821777061265391 0.8582222243920539\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4637\n",
      "Epoch 7/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4641val_aucs: 0.4843925934974946 0.8572776315853711\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4637\n",
      "Epoch 8/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4518val_aucs: 0.4783737818340439 0.8585714913415581\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4515\n",
      "Epoch 9/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4371val_aucs: 0.4841061179008334 0.8564273613599637\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4369\n",
      "Epoch 10/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4344val_aucs: 0.4826533944215721 0.8603998457249706\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4344\n",
      "Epoch 11/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4230val_aucs: 0.4893084907310448 0.8602094429987942\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4237\n",
      "Epoch 12/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4334val_aucs: 0.47261296802719355 0.8535129302979559\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4334\n",
      "Epoch 13/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4239val_aucs: 0.4951303915226353 0.8635956822520249\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4238\n",
      "Epoch 14/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4145val_aucs: 0.47095796671452556 0.8570748282713873\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4142\n",
      "Epoch 15/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4003val_aucs: 0.4733158961006355 0.8524089850558271\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4003\n",
      "Epoch 16/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4124val_aucs: 0.48202593317372944 0.8551341851007426\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4119\n",
      "Epoch 17/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.4037val_aucs: 0.47980947120345113 0.8522851744626005\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.4032\n",
      "Epoch 18/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3857val_aucs: 0.5041515988148549 0.8573084865912544\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3861\n",
      "Epoch 19/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3948val_aucs: 0.46576736927788537 0.8545612193585871\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3945\n",
      "Epoch 20/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3859val_aucs: 0.4697889728761365 0.8530268663128755\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3869\n",
      "Epoch 21/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3783val_aucs: 0.4601912589995769 0.8478195958580084\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3780\n",
      "Epoch 22/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3701val_aucs: 0.4725412086921967 0.8569286575631381\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3703\n",
      "Epoch 23/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3603val_aucs: 0.4424785417571726 0.8418134150925896\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3601\n",
      "Epoch 24/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3493val_aucs: 0.4609305966655378 0.8498886388157927\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3490\n",
      "Epoch 25/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3517val_aucs: 0.43830920288722836 0.8355897846496345\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3519\n",
      "Epoch 26/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3601val_aucs: 0.45114333213214336 0.849556264005585\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3596\n",
      "Epoch 27/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3467val_aucs: 0.4360406711338159 0.8420436559276274\n",
      "5834/5834 [==============================] - 26s 4ms/sample - loss: 0.3465\n",
      "Epoch 28/1000\n",
      "5824/5834 [============================>.] - ETA: 0s - loss: 0.3398val_aucs: 0.46414168286366014 0.85220901337213\n",
      "5834/5834 [==============================] - 29s 5ms/sample - loss: 0.3399\n",
      "Test res 0.8668935564813369 0.4824635543817102 0.4873046875\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)]}\n",
      "Repeat 0 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat0_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5137val_aucs: 0.46676548224327097 0.8483646358669915\n",
      "8751/8751 [==============================] - 49s 6ms/sample - loss: 0.5135\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4811val_aucs: 0.47782056170593784 0.8582655776281678\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4814\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4576val_aucs: 0.47666091823886086 0.8610405753091603\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4574\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4468val_aucs: 0.4924016450456031 0.8642643375693871\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4467\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4289val_aucs: 0.49282046881733416 0.8656093619555825\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4289\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4211val_aucs: 0.47502516318651244 0.8581224338350526\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4217\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4181val_aucs: 0.49043921798738255 0.8628733236016385\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4178\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4078val_aucs: 0.49420163849639237 0.8647693441846613\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4078\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4028val_aucs: 0.49481173762765024 0.8641334967216556\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4025\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3926val_aucs: 0.49534835243191755 0.8629602253587139\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3923\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3930val_aucs: 0.4868883295525469 0.8605964975662627\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3935\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3842val_aucs: 0.4742328494475579 0.8577060865404802\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3848\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3809val_aucs: 0.48157019630756526 0.860925552534065\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3808\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3773val_aucs: 0.4797656267482126 0.8570645758169009\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3773\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3738val_aucs: 0.49022921198514113 0.8610044476123987\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3736\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3658val_aucs: 0.47885145863894457 0.8540027046951358\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3668\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3652val_aucs: 0.49663815588689064 0.8627419945417886\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3650\n",
      "Epoch 18/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3558val_aucs: 0.4813519254233593 0.8616264298512418\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3556\n",
      "Epoch 19/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3476val_aucs: 0.4814345969068331 0.8571284339619879\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3475\n",
      "Epoch 20/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3363val_aucs: 0.48313972579137243 0.858786792885773\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3367\n",
      "Epoch 21/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3414val_aucs: 0.46123783687709186 0.846903905208735\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3413\n",
      "Epoch 22/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3340val_aucs: 0.4812175276203068 0.8579895424964238\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3336\n",
      "Epoch 23/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3279val_aucs: 0.46379655333301567 0.8469251912570975\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3276\n",
      "Epoch 24/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3239val_aucs: 0.4606275721928283 0.8498280028706872\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3237\n",
      "Epoch 25/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3240val_aucs: 0.4473261174784061 0.8470808332804437\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3238\n",
      "Epoch 26/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3287val_aucs: 0.45308628656330446 0.8539562269014641\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3291\n",
      "Epoch 27/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3174val_aucs: 0.4950791915798085 0.8614242123918\n",
      "8751/8751 [==============================] - 39s 4ms/sample - loss: 0.3174\n",
      "Test res 0.8690113214795963 0.4896981086086969 0.4892367906066536\n",
      "Repeat 1 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat1_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5209val_aucs: 0.46369582339365656 0.8519143285374631\n",
      "8751/8751 [==============================] - 50s 6ms/sample - loss: 0.5204\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4870val_aucs: 0.4564527053623344 0.8470204902626094\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4880\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4680val_aucs: 0.4745145056413327 0.8567724296852497\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4687\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4503val_aucs: 0.48327978893682555 0.8597306045530663\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4500\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4463val_aucs: 0.48229872869147894 0.8587229347406862\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4462\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4329val_aucs: 0.4952177078515633 0.8619662254856489\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4334\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4335val_aucs: 0.4757754240993599 0.8551707033671991\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4330\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4163val_aucs: 0.4996922768299873 0.8653979661083148\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4161\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4206val_aucs: 0.4922209907098716 0.8626946379663036\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4205\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4073val_aucs: 0.4954055098342522 0.8665548335440783\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4070\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4037val_aucs: 0.46136675721716053 0.8506592328234772\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4035\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3965val_aucs: 0.47948412155168696 0.8626181839485619\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3967\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3904val_aucs: 0.4704914707793647 0.8579289065513184\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3904\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3887val_aucs: 0.4832739293594782 0.8599331149397791\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3886\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3768val_aucs: 0.48048191462627154 0.8618984616436149\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3766\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3688val_aucs: 0.4661860299191867 0.8559377822476311\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3687\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3620val_aucs: 0.46496344022358993 0.8534347187165879\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3640\n",
      "Epoch 18/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3557val_aucs: 0.47305638464027183 0.8567882477578859\n",
      "8751/8751 [==============================] - 40s 5ms/sample - loss: 0.3558\n",
      "Test res 0.8698032247607841 0.49310486232230977 0.5009784735812133\n",
      "Repeat 2 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat2_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5379val_aucs: 0.4727272259978418 0.8534666477891314\n",
      "8751/8751 [==============================] - 51s 6ms/sample - loss: 0.5375\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4892val_aucs: 0.47956159905714785 0.8547328747394168\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4891\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4687val_aucs: 0.4900207824209105 0.8633517714776715\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4695\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4602val_aucs: 0.48230471970721356 0.8625710226579245\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4605\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4491val_aucs: 0.4934625631027057 0.8665694799076302\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4499\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4440val_aucs: 0.48636484350492654 0.8609937069457937\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4442\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4343val_aucs: 0.4869181990113785 0.8568454662181625\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4342\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4311val_aucs: 0.5000868353512848 0.8620533225275716\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4307\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4245val_aucs: 0.5014114767638983 0.8650272178256007\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4243\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4206val_aucs: 0.49249369889488104 0.8630027974554384\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4209\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4192val_aucs: 0.48981231511594114 0.8612379106474182\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4195\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4096val_aucs: 0.49375827096060754 0.8624110843679362\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4098\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4080val_aucs: 0.4950488659219586 0.862402882404347\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4081\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4024val_aucs: 0.46146902408267715 0.8556944573278198\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4022\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3990val_aucs: 0.48513516106077414 0.8570193673747369\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3989\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3912val_aucs: 0.4692612336148085 0.8576840193527284\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3921\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3757val_aucs: 0.49515357375745606 0.8582437057252635\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3760\n",
      "Epoch 18/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3746val_aucs: 0.4807896651046689 0.855283968578668\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3745\n",
      "Epoch 19/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3651val_aucs: 0.4763602275122529 0.8490881662264621\n",
      "8751/8751 [==============================] - 40s 5ms/sample - loss: 0.3651\n",
      "Test res 0.8750683730925769 0.5038957411167312 0.5024390243902439\n",
      "Repeat 3 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat3_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5309val_aucs: 0.46109413168777647 0.8471310214862153\n",
      "8751/8751 [==============================] - 51s 6ms/sample - loss: 0.5312\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5061val_aucs: 0.475326308319723 0.8577431906614786\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.5067\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4708val_aucs: 0.47935735331897833 0.8596425310869066\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4716\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4705val_aucs: 0.4827285693343131 0.8621019484545644\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4702\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4571val_aucs: 0.47195769020653966 0.8578009949762972\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4572\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4519val_aucs: 0.4926525897439167 0.8662228493035653\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4518\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4428val_aucs: 0.49413819952887567 0.865415932314272\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4425\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4357val_aucs: 0.4917717851904502 0.8656256682403373\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4361\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4306val_aucs: 0.4652087324053486 0.8538654194474414\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4307\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4271val_aucs: 0.5035288826453321 0.8683032187824967\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4270\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4198val_aucs: 0.4987871985093088 0.866665169482837\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4203\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4198val_aucs: 0.4988678124191525 0.8646629139428498\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4194\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4144val_aucs: 0.4867486553534498 0.8634779254890664\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4146\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4053val_aucs: 0.47423661602820605 0.8582066016042651\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4055\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4031val_aucs: 0.49206877131521115 0.8660050090563349\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4030\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4019val_aucs: 0.47649915873618803 0.8595183299239852\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4022\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3923val_aucs: 0.4846510141571984 0.8620607433517715\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3928\n",
      "Epoch 18/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3873val_aucs: 0.4819271744710939 0.860206123156389\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3871\n",
      "Epoch 19/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3810val_aucs: 0.4690485181792093 0.8565284212684728\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3807\n",
      "Epoch 20/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3756val_aucs: 0.4731432475340022 0.8585815485111972\n",
      "8751/8751 [==============================] - 40s 5ms/sample - loss: 0.3755\n",
      "Test res 0.8666457565461682 0.49036567694729083 0.49071358748778104\n",
      "Repeat 4 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat4_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5505val_aucs: 0.46321211979687815 0.8498306392161267\n",
      "8751/8751 [==============================] - 52s 6ms/sample - loss: 0.5499\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5137val_aucs: 0.47542315237666566 0.8589089435577969\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.5136\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4873val_aucs: 0.4823936758588917 0.8637489808572029\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4874\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4742val_aucs: 0.48876790658322145 0.8651816881398631\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4744\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4607val_aucs: 0.4865320538300709 0.8657623676334895\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4617\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4518val_aucs: 0.49845901904961015 0.8694514936849763\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4526\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4452val_aucs: 0.4861855023211592 0.8670627694320628\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4452\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4476val_aucs: 0.4893365524003672 0.8681374219470877\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4475\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4302val_aucs: 0.5067775889809503 0.8727080637995597\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4299\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4333val_aucs: 0.49871157867325916 0.8684381606120228\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4339\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4185val_aucs: 0.4949119772060275 0.8611673151750973\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4190\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4123val_aucs: 0.4519889634755494 0.850253430910662\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4126\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4174val_aucs: 0.49601839629692934 0.8684925474420125\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4171\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4020val_aucs: 0.5033940877975706 0.867218411455409\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4018\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3956val_aucs: 0.49006673775757414 0.8630783726913669\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3955\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3945val_aucs: 0.47473574098341365 0.8602126651987756\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3943\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3814val_aucs: 0.4769025275284915 0.8594024283670769\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3815\n",
      "Epoch 18/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3749val_aucs: 0.43541174265516974 0.8531758686514118\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3744\n",
      "Epoch 19/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3730val_aucs: 0.4874314741023072 0.859526727172422\n",
      "8751/8751 [==============================] - 41s 5ms/sample - loss: 0.3737\n",
      "Test res 0.8713919537055934 0.5001011587229169 0.5039138943248532\n",
      "Repeat 5 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat5_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5330val_aucs: 0.4606575498657668 0.8493438917340806\n",
      "8751/8751 [==============================] - 52s 6ms/sample - loss: 0.5329\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5062val_aucs: 0.48275173735524324 0.858467990372457\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.5057\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4773val_aucs: 0.4888795363787808 0.8630535715157523\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4770\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4738val_aucs: 0.47695260176984383 0.8559514521869461\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4736\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4492val_aucs: 0.4861625425236444 0.8651194899159789\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4492\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4443val_aucs: 0.5007758772622634 0.8657289739245906\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4436\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4440val_aucs: 0.5075216617019139 0.8685467389871551\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4434\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4377val_aucs: 0.49944309312739743 0.8670706784683812\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4383\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4244val_aucs: 0.5025617983178456 0.8633164249202995\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4240\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4188val_aucs: 0.494680071665095 0.8672705525096543\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4187\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4079val_aucs: 0.49951117711845794 0.8657774045667361\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4076\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4103val_aucs: 0.5035253190672347 0.8694452445698607\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4098\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3982val_aucs: 0.48704416731467476 0.8628842595530907\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3978\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3949val_aucs: 0.5018132937307689 0.865021652207451\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3955\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3807val_aucs: 0.49700853018558533 0.8608044759287015\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3805\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3712val_aucs: 0.48382991906839806 0.8631893921270913\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3711\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3736val_aucs: 0.47765599502103107 0.8543114500388128\n",
      "8751/8751 [==============================] - 41s 5ms/sample - loss: 0.3733\n",
      "Test res 0.8707851199980791 0.5022224476327378 0.5009784735812133\n",
      "Repeat 6 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat6_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5392val_aucs: 0.4683881530772358 0.8517165049870868\n",
      "8751/8751 [==============================] - 53s 6ms/sample - loss: 0.5394\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4958val_aucs: 0.481899212388084 0.8601776115686744\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4953\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4773val_aucs: 0.4871888654016887 0.8616279921300206\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4777\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4584val_aucs: 0.4939616404825469 0.8645033662225564\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4579\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4472val_aucs: 0.4886972108277282 0.865572550761855\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4470\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4426val_aucs: 0.49299938176217833 0.863459763998262\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4440\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4415val_aucs: 0.5025741389642108 0.867873787403151\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4421\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4312val_aucs: 0.4709761262847841 0.8547574806301841\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4310\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4342val_aucs: 0.4922870191062147 0.8681092032866439\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4339\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4286val_aucs: 0.49524032349606817 0.8669635647296037\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4281\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4190val_aucs: 0.4836965760316132 0.8687504210829522\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4187\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4119val_aucs: 0.4860876301635389 0.8649579893472115\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4117\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4078val_aucs: 0.48654799488004585 0.8701785391716994\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4075\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3983val_aucs: 0.48121450966819757 0.863230401945037\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3980\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3894val_aucs: 0.49334055693698364 0.867955025899653\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3896\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3850val_aucs: 0.4670763954787931 0.8596253460203388\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3856\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3862val_aucs: 0.4940927605913816 0.8643615894233727\n",
      "8751/8751 [==============================] - 41s 5ms/sample - loss: 0.3864\n",
      "Test res 0.8677577947726698 0.501850558814944 0.490234375\n",
      "Repeat 7 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat7_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5409val_aucs: 0.46941206764551324 0.8485106112903935\n",
      "8751/8751 [==============================] - 54s 6ms/sample - loss: 0.5406\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5024val_aucs: 0.4713340665275686 0.852498816085613\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.5021\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4686val_aucs: 0.48605064286061744 0.8632129239511983\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4686\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4622val_aucs: 0.4784453071899777 0.8623293576593158\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4624\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4501val_aucs: 0.4702384914857883 0.8577869344672874\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4506\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4491val_aucs: 0.4847213080376258 0.8654747130533272\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4487\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4419val_aucs: 0.4951806669992037 0.8687994375796397\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4418\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4346val_aucs: 0.496683590239226 0.8647574318089725\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4344\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4388val_aucs: 0.49989555770402155 0.8698629588583648\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4389\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4214val_aucs: 0.510265300867096 0.871615738005849\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4226\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4165val_aucs: 0.4911393832584849 0.8638684951837874\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4176\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4184val_aucs: 0.4845232107013407 0.8618876233345864\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.4179\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4080val_aucs: 0.4665674669768732 0.8582724125978255\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4077\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4030val_aucs: 0.4842140116376566 0.8665093321746433\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4029\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3958val_aucs: 0.5026791304355712 0.8671588495769642\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3955\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3812val_aucs: 0.47158346898003 0.8554766170805892\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3813\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3876val_aucs: 0.4904914285608616 0.8651399948249516\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3888\n",
      "Epoch 18/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3882val_aucs: 0.4755122625656381 0.8607412036381566\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3882\n",
      "Epoch 19/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3693val_aucs: 0.47800768501818913 0.8593960816095378\n",
      "8751/8751 [==============================] - 36s 4ms/sample - loss: 0.3692\n",
      "Epoch 20/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3700val_aucs: 0.43443592814509546 0.8604572594700945\n",
      "8751/8751 [==============================] - 41s 5ms/sample - loss: 0.3702\n",
      "Test res 0.8706144573973802 0.499206573164437 0.49319066147859925\n",
      "Repeat 8 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat8_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5506val_aucs: 0.458841873795243 0.8519880485673416\n",
      "8751/8751 [==============================] - 55s 6ms/sample - loss: 0.5500\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5023val_aucs: 0.47273082101291536 0.858765702122258\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.5018\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4776val_aucs: 0.47150855859758245 0.8623251590350975\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4773\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4841val_aucs: 0.4725037991867917 0.8551783194762462\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4842\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4643val_aucs: 0.4809813275540831 0.8640887764916101\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4639\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4535val_aucs: 0.493106729672247 0.8668145623910677\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4532\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4501val_aucs: 0.47944613894646526 0.864397521835287\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4515\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4416val_aucs: 0.49194065815245863 0.8657926367848303\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4417\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4464val_aucs: 0.4498909847524879 0.8604777643790674\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4464\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4344val_aucs: 0.5048969163468143 0.8696879836351297\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4342\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4308val_aucs: 0.49818514235596534 0.8663654072421385\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4306\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4243val_aucs: 0.48830648489581735 0.8650122785347778\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4245\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4164val_aucs: 0.48985694516239353 0.8633816500593177\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4163\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4119val_aucs: 0.44833882588027496 0.861764691523173\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4121\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4032val_aucs: 0.4784708926525742 0.8653796093326628\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4034\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4055val_aucs: 0.4675651446423588 0.8540279940828691\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4050\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3961val_aucs: 0.46083368479479775 0.8571455213861318\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3962\n",
      "Epoch 18/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3897val_aucs: 0.46494738846454653 0.8578062676671758\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3897\n",
      "Epoch 19/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3840val_aucs: 0.46131148933247146 0.857587060426014\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3843\n",
      "Epoch 20/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3782val_aucs: 0.4539644178779076 0.8561709523553793\n",
      "8751/8751 [==============================] - 41s 5ms/sample - loss: 0.3780\n",
      "Test res 0.8739670800667524 0.5054471259543175 0.49416342412451364\n",
      "Repeat 9 ld 30\n",
      "Num train: 8751 Num valid: 2166\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat9_30ld.h5\n",
      "Train on 8751 samples\n",
      "Epoch 1/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5452val_aucs: 0.4510875036377346 0.8403216341436026\n",
      "8751/8751 [==============================] - 55s 6ms/sample - loss: 0.5455\n",
      "Epoch 2/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.5072val_aucs: 0.47310780202696695 0.8563763920148024\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.5069\n",
      "Epoch 3/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4870val_aucs: 0.4830476859110559 0.8618566706862798\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4880\n",
      "Epoch 4/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4755val_aucs: 0.4883293102659181 0.8631992540118831\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4770\n",
      "Epoch 5/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4662val_aucs: 0.497521438693658 0.8678284813185634\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4670\n",
      "Epoch 6/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4640val_aucs: 0.49342107460359486 0.8656163922100875\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4638\n",
      "Epoch 7/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4615val_aucs: 0.506007565492115 0.8719849240097838\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4612\n",
      "Epoch 8/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4503val_aucs: 0.5014357991208134 0.8688941507306094\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4501\n",
      "Epoch 9/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4423val_aucs: 0.5114482938518476 0.8727267135024824\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4430\n",
      "Epoch 10/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4427val_aucs: 0.5168376888445506 0.8715822466545267\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4422\n",
      "Epoch 11/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4365val_aucs: 0.5109225444929808 0.8695890718599418\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4378\n",
      "Epoch 12/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4275val_aucs: 0.47919236810825067 0.8677061353616919\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4277\n",
      "Epoch 13/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4287val_aucs: 0.4930896251980175 0.8685150051994591\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4285\n",
      "Epoch 14/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4282val_aucs: 0.500466973531672 0.8664693964233582\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4279\n",
      "Epoch 15/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4172val_aucs: 0.4956319036816467 0.8648017614693232\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4169\n",
      "Epoch 16/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4085val_aucs: 0.4974635230981941 0.8706308188781862\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4085\n",
      "Epoch 17/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3944val_aucs: 0.5020254606029882 0.8713420462922731\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3949\n",
      "Epoch 18/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.4062val_aucs: 0.49371307549576476 0.8674990357810662\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.4066\n",
      "Epoch 19/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3902val_aucs: 0.4977946177441217 0.8613911116101725\n",
      "8751/8751 [==============================] - 37s 4ms/sample - loss: 0.3901\n",
      "Epoch 20/1000\n",
      "8736/8751 [============================>.] - ETA: 0s - loss: 0.3935val_aucs: 0.461056518668841 0.859765169971049\n",
      "8751/8751 [==============================] - 41s 5ms/sample - loss: 0.3933\n",
      "Test res 0.8743446628168033 0.5063953655864138 0.4931506849315068\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554), (0.49922876188707954, 0.0057904772677334635)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554), (0.49922876188707954, 0.0057904772677334635), (0.49589993895065787, 0.005290317714153174)]}\n",
      "Repeat 0 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat0_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5214val_aucs: 0.47093761685966457 0.8544517622016414\n",
      "11668/11668 [==============================] - 66s 6ms/sample - loss: 0.5210\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4827val_aucs: 0.4743308805187881 0.8564470851295471\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4821\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4715val_aucs: 0.49154657960952225 0.8665763148772878\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4715\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4511val_aucs: 0.49374297122854466 0.8640772546856158\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4510\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4507val_aucs: 0.49832937812088196 0.8677685288704237\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4507\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4383val_aucs: 0.5035317698838128 0.8674656420721676\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4383\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4274val_aucs: 0.5126254161233738 0.8713500529710149\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4283\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4233val_aucs: 0.5090255164627787 0.8704395373701964\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4247\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4173val_aucs: 0.5077130734875124 0.8714814796732885\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4172\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4171val_aucs: 0.5106495789277272 0.8684655004906532\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4168\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4051val_aucs: 0.5007264515976588 0.8625454403429202\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4049\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4081val_aucs: 0.5103183897265655 0.8719368839373332\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4078\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4033val_aucs: 0.4956636647008574 0.8660004198624218\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4028\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3980val_aucs: 0.503961387289203 0.8661107558011806\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3980\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3888val_aucs: 0.49320880061406075 0.860886202637322\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3885\n",
      "Epoch 16/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3832val_aucs: 0.5089967171454645 0.8715766810363768\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3832\n",
      "Epoch 17/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3801val_aucs: 0.5006991466186127 0.8692459563831294\n",
      "11668/11668 [==============================] - 52s 4ms/sample - loss: 0.3799\n",
      "Test res 0.8744508542134394 0.5048081527967334 0.49477682811016144\n",
      "Repeat 1 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat1_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5127val_aucs: 0.4603069618368054 0.8493641037157823\n",
      "11668/11668 [==============================] - 66s 6ms/sample - loss: 0.5135\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4800val_aucs: 0.470296733678436 0.8569190886056173\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4798\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4637val_aucs: 0.4917272574533341 0.8648182630389252\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4642\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4481val_aucs: 0.49321676839145356 0.8663540807209917\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4476\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4327val_aucs: 0.5063939406326641 0.8706265226115442\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4327\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4284val_aucs: 0.5204175664236226 0.8732607199175898\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4280\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4292val_aucs: 0.509335884121346 0.8692156872317883\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4286\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4199val_aucs: 0.5125064006323394 0.8715328395881443\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4197\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4098val_aucs: 0.49179271106948086 0.8667516806702176\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4094\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4139val_aucs: 0.5155697562500089 0.8698904940218427\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4134\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4018val_aucs: 0.5068050961502325 0.8718242045804061\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4017\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3918val_aucs: 0.5104123687305729 0.8677406031372512\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3914\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3852val_aucs: 0.5042033277193453 0.8685719307324646\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3852\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3831val_aucs: 0.4966412594201025 0.8673314813820309\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3834\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3848val_aucs: 0.4958743455282491 0.8690273349965092\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3855\n",
      "Epoch 16/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3677val_aucs: 0.49108737775174327 0.864080574528021\n",
      "11668/11668 [==============================] - 52s 4ms/sample - loss: 0.3675\n",
      "Test res 0.8754421139831678 0.5122700751858337 0.49706457925636005\n",
      "Repeat 2 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat2_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5364val_aucs: 0.47313425410470494 0.8544545938319281\n",
      "11668/11668 [==============================] - 67s 6ms/sample - loss: 0.5360\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4885val_aucs: 0.48277398782779923 0.8612270723383897\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4885\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4673val_aucs: 0.48514180295836706 0.8653168252542365\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4671\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4579val_aucs: 0.4742568235453539 0.8607173788867787\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4578\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4534val_aucs: 0.49768784126014926 0.8701980676564355\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4532\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4489val_aucs: 0.4998427871754581 0.8698545616099282\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4492\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4370val_aucs: 0.4978085026850185 0.8703843694008172\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4370\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4318val_aucs: 0.48868708489221746 0.8675652373443214\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4318\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4296val_aucs: 0.5083792298714556 0.8726918551572288\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4293\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4172val_aucs: 0.4954958240490148 0.8652901688725718\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4171\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4145val_aucs: 0.5143831104993247 0.8715859570666262\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4140\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4154val_aucs: 0.5113407799989307 0.8721181082756835\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4151\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4083val_aucs: 0.5039706100106985 0.8721044383363683\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4080\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4060val_aucs: 0.5027068287791875 0.8666928022887385\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4060\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3922val_aucs: 0.507325596877037 0.8693464304370964\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3921\n",
      "Epoch 16/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3921val_aucs: 0.4846154792006683 0.8613874011980726\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3925\n",
      "Epoch 17/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3837val_aucs: 0.5020144414557972 0.8702802825771742\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3838\n",
      "Epoch 18/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3810val_aucs: 0.49495279640200535 0.8653635959751792\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3810\n",
      "Epoch 19/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3675val_aucs: 0.5028575064386198 0.8668592826211131\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3686\n",
      "Epoch 20/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3730val_aucs: 0.5055289305355617 0.867976116663168\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3735\n",
      "Epoch 21/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3661val_aucs: 0.5087691683636225 0.8733651973109278\n",
      "11668/11668 [==============================] - 52s 4ms/sample - loss: 0.3660\n",
      "Test res 0.874414896810056 0.5009808411544783 0.4936708860759494\n",
      "Repeat 3 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat3_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5388val_aucs: 0.45817029792582836 0.8457382499548404\n",
      "11668/11668 [==============================] - 68s 6ms/sample - loss: 0.5384\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4920val_aucs: 0.47807409742056295 0.8568099243759427\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4916\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4823val_aucs: 0.4858457834538657 0.8630929214124954\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4821\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4644val_aucs: 0.4976524414221907 0.8680468097779123\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4639\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4519val_aucs: 0.4933131637440502 0.86892520102134\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4527\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4412val_aucs: 0.5000027082878169 0.871471520146073\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4410\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4388val_aucs: 0.4983100224781975 0.8682237378496208\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4387\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4301val_aucs: 0.5096542776768971 0.8702172055714769\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4300\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4342val_aucs: 0.5092249909812139 0.8711797645841164\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4344\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4218val_aucs: 0.5131857835993711 0.8729648633738386\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4216\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4204val_aucs: 0.501957811467108 0.8680480791294201\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4201\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4175val_aucs: 0.4967567396183792 0.8668989254451275\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4178\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4109val_aucs: 0.482331341500502 0.866270303521474\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4110\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4057val_aucs: 0.5061233235607184 0.8700230924332003\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4054\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4129val_aucs: 0.514439807889139 0.8698266358767558\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4127\n",
      "Epoch 16/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3978val_aucs: 0.5026420899133841 0.8681917111346539\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3977\n",
      "Epoch 17/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3982val_aucs: 0.4980018279536882 0.8688670061368262\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3981\n",
      "Epoch 18/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3884val_aucs: 0.49861816720777696 0.867758374058361\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3882\n",
      "Epoch 19/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3875val_aucs: 0.4948629120341472 0.8677011555980844\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3873\n",
      "Epoch 20/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3759val_aucs: 0.49025416399054067 0.864396935980745\n",
      "11668/11668 [==============================] - 52s 4ms/sample - loss: 0.3758\n",
      "Test res 0.8783784951916727 0.5129874743751146 0.4975609756097561\n",
      "Repeat 4 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat4_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5395val_aucs: 0.44769890697871473 0.848416679278813\n",
      "11668/11668 [==============================] - 69s 6ms/sample - loss: 0.5395\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4945val_aucs: 0.48229685536855316 0.8615234170942592\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4954\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4783val_aucs: 0.4712396373850498 0.8589077718487129\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4780\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4616val_aucs: 0.4835796728546618 0.8631631263151214\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4615\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4598val_aucs: 0.4799622655070686 0.8633685659745447\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4598\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4611val_aucs: 0.49199800033588303 0.8679926182327699\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4613\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4477val_aucs: 0.4948205390801398 0.8670455843654951\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4481\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4441val_aucs: 0.5035273330839443 0.868446948430154\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4437\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4317val_aucs: 0.4842555755051801 0.8629258552255783\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4318\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4274val_aucs: 0.5037330589299974 0.8695655400358349\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4275\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4214val_aucs: 0.4838054910696972 0.8630707565823198\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4215\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4198val_aucs: 0.5018712844012857 0.8662703035214738\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4204\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4146val_aucs: 0.502457149913174 0.8660345947107099\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4143\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4127val_aucs: 0.5053603797940344 0.867466423211557\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4131\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4078val_aucs: 0.486382922918165 0.862491151155354\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4079\n",
      "Epoch 16/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4051val_aucs: 0.5039158728171985 0.8679017131363235\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4048\n",
      "Epoch 17/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3936val_aucs: 0.4735291941687804 0.8602231129381092\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3937\n",
      "Epoch 18/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4006val_aucs: 0.4969856422646318 0.865547554301393\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4003\n",
      "Epoch 19/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3889val_aucs: 0.48596426798590214 0.8679271001664803\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3892\n",
      "Epoch 20/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3823val_aucs: 0.48483942636428284 0.8604052160582729\n",
      "11668/11668 [==============================] - 53s 5ms/sample - loss: 0.3824\n",
      "Test res 0.8757834391845654 0.5063693140914902 0.4941747572815534\n",
      "Repeat 5 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat5_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5335val_aucs: 0.45397058809994667 0.8515040350731586\n",
      "11668/11668 [==============================] - 69s 6ms/sample - loss: 0.5340\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5053val_aucs: 0.466971758822823 0.8562783590214277\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.5052\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4749val_aucs: 0.4811588625929429 0.8628708825410464\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4756\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4718val_aucs: 0.48655124660632965 0.8652977849816189\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4715\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4660val_aucs: 0.49990515404835933 0.8707461345805525\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4660\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4536val_aucs: 0.48836428315642966 0.863320525902094\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4537\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4464val_aucs: 0.4864814318175382 0.861369239707268\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4464\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4506val_aucs: 0.4842280494657397 0.8649811306016238\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4502\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4359val_aucs: 0.4783220322578584 0.8605260973787892\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4361\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4359val_aucs: 0.4909426485026595 0.8673812790181078\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4356\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4283val_aucs: 0.485815531069767 0.8630548408672601\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4281\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4253val_aucs: 0.49166392175732965 0.8665001537868173\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4262\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4165val_aucs: 0.49582419034675973 0.8694257160851246\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4169\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4123val_aucs: 0.48567000548594874 0.8652848961816931\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4122\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4086val_aucs: 0.4992208845776475 0.865334303248075\n",
      "11668/11668 [==============================] - 53s 5ms/sample - loss: 0.4085\n",
      "Test res 0.8701402878993433 0.49287760865418306 0.48336594911937375\n",
      "Repeat 6 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat6_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5344val_aucs: 0.4647858649984375 0.8513196861772503\n",
      "11668/11668 [==============================] - 70s 6ms/sample - loss: 0.5342\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5000val_aucs: 0.4716194043606693 0.8572680626278506\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.5003\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4737val_aucs: 0.4706129448060906 0.8622710651323787\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4735\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4583val_aucs: 0.48348523135734806 0.8652136172124065\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4584\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4560val_aucs: 0.4857866217937765 0.8635509620219792\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4558\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4498val_aucs: 0.4836841774421366 0.8633318524232408\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4499\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4438val_aucs: 0.5068216943476677 0.8676270449985111\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4434\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4342val_aucs: 0.5033555804540267 0.8680089245175244\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4339\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4276val_aucs: 0.4991855335499143 0.8700026851666511\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4279\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4279val_aucs: 0.5033422491588777 0.8708247367316151\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4285\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4176val_aucs: 0.4996388391276121 0.8696688457200884\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4173\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4185val_aucs: 0.4900294520714984 0.8649833763773684\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4183\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4107val_aucs: 0.502573034409487 0.8689270562273896\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4107\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4042val_aucs: 0.5044360328476519 0.8677682359431527\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4043\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4001val_aucs: 0.5043984881412932 0.868986520463411\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.4000\n",
      "Epoch 16/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3957val_aucs: 0.4753725243569315 0.8596153864931235\n",
      "11668/11668 [==============================] - 47s 4ms/sample - loss: 0.3956\n",
      "Epoch 17/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3862val_aucs: 0.5076243107269849 0.8616449819117411\n",
      "11668/11668 [==============================] - 53s 5ms/sample - loss: 0.3863\n",
      "Test res 0.8709586639933729 0.5019016090757128 0.49853372434017595\n",
      "Repeat 7 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat7_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5402val_aucs: 0.4711138645042343 0.8530538156218114\n",
      "11668/11668 [==============================] - 71s 6ms/sample - loss: 0.5409\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4900val_aucs: 0.4799406071667564 0.8613061627015706\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4900\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4850val_aucs: 0.4810593272253098 0.8623723203257352\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4859\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4584val_aucs: 0.48951247813642507 0.8668266700516039\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4588\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4524val_aucs: 0.48363587263631214 0.8629944002070019\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4522\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4421val_aucs: 0.5040242626843265 0.8701332330871117\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4425\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4436val_aucs: 0.4909441337059493 0.8666702468888683\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4433\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4367val_aucs: 0.48688520747086605 0.8666421258708484\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4374\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4316val_aucs: 0.49233576432488274 0.864569763070659\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4315\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4325val_aucs: 0.4977599061083674 0.8720899872576637\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4323\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4205val_aucs: 0.4808275240616306 0.8662169907581447\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4202\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4180val_aucs: 0.4991403696603068 0.8682635759584825\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4176\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4171val_aucs: 0.5031870635019655 0.8689314501364552\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4173\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4087val_aucs: 0.5002390874213114 0.8686600041986242\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4083\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4067val_aucs: 0.49143240759977913 0.8672590307036601\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4065\n",
      "Epoch 16/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3963val_aucs: 0.4953788869005703 0.8640092955587343\n",
      "11668/11668 [==============================] - 54s 5ms/sample - loss: 0.3961\n",
      "Test res 0.8720456701043304 0.49131197836097834 0.4839650145772595\n",
      "Repeat 8 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat8_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5177val_aucs: 0.4712891735183483 0.8530828154216444\n",
      "11668/11668 [==============================] - 72s 6ms/sample - loss: 0.5176\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4782val_aucs: 0.48068186123566337 0.8570204414413976\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4779\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4627val_aucs: 0.48672982771273365 0.8659180096568357\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4638\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4475val_aucs: 0.4870035342677134 0.8658908650630526\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4475\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4427val_aucs: 0.48614829803597226 0.8656323079251472\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4428\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4326val_aucs: 0.5050386317473698 0.8712301480747355\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4323\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4247val_aucs: 0.5045098996440885 0.8704462746974306\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4260\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4283val_aucs: 0.4965418526923733 0.8691682330138799\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4282\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4212val_aucs: 0.4984372826761414 0.8696650376655649\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4216\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4136val_aucs: 0.4941142251291699 0.866500544356512\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4141\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4141val_aucs: 0.5014035679638362 0.8708997261130016\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4139\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4011val_aucs: 0.49975108906111054 0.8676263615015453\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4008\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3963val_aucs: 0.5017260973431035 0.8674505074964971\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.3958\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3937val_aucs: 0.49904262288965956 0.869777424095221\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.3937\n",
      "Epoch 15/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3900val_aucs: 0.492660028400334 0.8687322595921477\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.3909\n",
      "Epoch 16/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3846val_aucs: 0.4920687270971804 0.8680296247113446\n",
      "11668/11668 [==============================] - 54s 5ms/sample - loss: 0.3844\n",
      "Test res 0.873141320399073 0.5004279698413016 0.5039138943248532\n",
      "Repeat 9 ld 40\n",
      "Num train: 11668 Num valid: 2888\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat9_40ld.h5\n",
      "Train on 11668 samples\n",
      "Epoch 1/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.5451val_aucs: 0.47076344084856636 0.8523704162984735\n",
      "11668/11668 [==============================] - 72s 6ms/sample - loss: 0.5452\n",
      "Epoch 2/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4770val_aucs: 0.4586742282889225 0.8570653569562904\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4765\n",
      "Epoch 3/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4614val_aucs: 0.4872945009054897 0.8648025426087127\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4611\n",
      "Epoch 4/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4510val_aucs: 0.5014807981237569 0.8691595428381724\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4508\n",
      "Epoch 5/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4434val_aucs: 0.47267178553665706 0.8611811803992598\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4435\n",
      "Epoch 6/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4346val_aucs: 0.4906458596323076 0.8645123493255349\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4344\n",
      "Epoch 7/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4268val_aucs: 0.4953770225416257 0.8682628924615167\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4266\n",
      "Epoch 8/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4194val_aucs: 0.5012418786716761 0.8668663128756182\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4192\n",
      "Epoch 9/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4177val_aucs: 0.49570136591077374 0.8651765130914079\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4172\n",
      "Epoch 10/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4059val_aucs: 0.4894938257073808 0.8632521762055179\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4057\n",
      "Epoch 11/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4064val_aucs: 0.48503429533995934 0.8611028711754682\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4063\n",
      "Epoch 12/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.4071val_aucs: 0.47334485975944696 0.8568343349818629\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.4067\n",
      "Epoch 13/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3958val_aucs: 0.47254040387239776 0.8600896357449385\n",
      "11668/11668 [==============================] - 48s 4ms/sample - loss: 0.3959\n",
      "Epoch 14/1000\n",
      "11648/11668 [============================>.] - ETA: 0s - loss: 0.3968val_aucs: 0.4893383853607171 0.8653038388118871\n",
      "11668/11668 [==============================] - 54s 5ms/sample - loss: 0.3967\n",
      "Test res 0.866551811076561 0.48703501642430264 0.48140900195694714\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554), (0.49922876188707954, 0.0057904772677334635), (0.49589993895065787, 0.005290317714153174)], 40: [(0.8731307552855583, 0.00318481506214208)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554), (0.49922876188707954, 0.0057904772677334635), (0.49589993895065787, 0.005290317714153174)], 40: [(0.8731307552855583, 0.00318481506214208), (0.5010970039960129, 0.008185009808080705)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554), (0.49922876188707954, 0.0057904772677334635), (0.49589993895065787, 0.005290317714153174)], 40: [(0.8731307552855583, 0.00318481506214208), (0.5010970039960129, 0.008185009808080705), (0.49284356106523897, 0.007075296181590279)]}\n",
      "Repeat 0 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat0_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5214val_aucs: 0.47110839270368204 0.851447109540153\n",
      "14586/14586 [==============================] - 84s 6ms/sample - loss: 0.5213\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4765val_aucs: 0.4963689850295862 0.866095914152781\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4765\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4580val_aucs: 0.49967835254655085 0.8682450238979831\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4581\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4462val_aucs: 0.4973902649432623 0.8642877717510704\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4460\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4318val_aucs: 0.5045736294936953 0.8715176073700501\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4318\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4366val_aucs: 0.5106640646648776 0.8714148875403386\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4366\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4199val_aucs: 0.500117805206555 0.8628809397106854\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4202\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4200val_aucs: 0.5202942555597284 0.8739938192345811\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4201\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4117val_aucs: 0.5147339955211102 0.8721980774206777\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4121\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4155val_aucs: 0.5208152632972374 0.8717398415263462\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4151\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4087val_aucs: 0.5327522251659202 0.8741340337549858\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4083\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4024val_aucs: 0.5241416500454851 0.8759999804715152\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4022\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3938val_aucs: 0.5113030883771672 0.8681485531833871\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3938\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3937val_aucs: 0.5112037903833867 0.8679776789419467\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3936\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3931val_aucs: 0.5128403909634299 0.8666764960039839\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3930\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3886val_aucs: 0.5357976588324895 0.8711268423904819\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3887\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3860val_aucs: 0.5187783914705176 0.8666133213558628\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3857\n",
      "Epoch 18/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3846val_aucs: 0.5267994961409198 0.8689169990577505\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3846\n",
      "Epoch 19/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3772val_aucs: 0.5183110824203091 0.8651860820489287\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3770\n",
      "Epoch 20/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3731val_aucs: 0.5320406560377609 0.8703234405284408\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3728\n",
      "Epoch 21/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3670val_aucs: 0.5211199260116649 0.8677443135493508\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3666\n",
      "Epoch 22/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3688val_aucs: 0.5070650916318763 0.8606728539415806\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3691\n",
      "Epoch 23/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3633val_aucs: 0.5199989904942423 0.8671529910315434\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3629\n",
      "Epoch 24/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3591val_aucs: 0.5246601540468515 0.8685098301510039\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3588\n",
      "Epoch 25/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3593val_aucs: 0.5116961380749769 0.8633392732474405\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3593\n",
      "Epoch 26/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3492val_aucs: 0.5223825778057166 0.8678599221789882\n",
      "14586/14586 [==============================] - 65s 4ms/sample - loss: 0.3491\n",
      "Test res 0.8778752116024156 0.5262459310300751 0.5058708414872799\n",
      "Repeat 1 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat1_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5160val_aucs: 0.4809630645303373 0.8605685718330901\n",
      "14586/14586 [==============================] - 84s 6ms/sample - loss: 0.5157\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4702val_aucs: 0.48821988734639876 0.8658313031846077\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4698\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4550val_aucs: 0.49475152148039764 0.8661308701404585\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4550\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4426val_aucs: 0.5108250766017693 0.8705468463938212\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4429\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4428val_aucs: 0.5046833710829286 0.871593670818097\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4430\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4287val_aucs: 0.521340321982081 0.8752633660272716\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4286\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4236val_aucs: 0.5108389832946626 0.8726824814845553\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4236\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4209val_aucs: 0.5067711144714256 0.8728943655439415\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4207\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4122val_aucs: 0.5056246540171209 0.8718390462288055\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4124\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4065val_aucs: 0.5119554135872767 0.8739442168833514\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4060\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4046val_aucs: 0.5178901404287191 0.8739758530286239\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4047\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4062val_aucs: 0.5326646543385959 0.8795289729481666\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4059\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3972val_aucs: 0.5253848816013237 0.8776673224982792\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3981\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3910val_aucs: 0.5118388891871568 0.8740363913313055\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3908\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3834val_aucs: 0.5230333336732337 0.8742447602634393\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3834\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3820val_aucs: 0.510946778109294 0.8728855777258103\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3819\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3788val_aucs: 0.5152502590910106 0.8695228702966864\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3791\n",
      "Epoch 18/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3827val_aucs: 0.5088021235381969 0.8710688427908158\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3826\n",
      "Epoch 19/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3757val_aucs: 0.5251011610030566 0.8778879943757965\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3757\n",
      "Epoch 20/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3698val_aucs: 0.5171077362256447 0.8748907625385077\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3696\n",
      "Epoch 21/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3692val_aucs: 0.5185486523010526 0.8735032636980116\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3693\n",
      "Epoch 22/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3658val_aucs: 0.5080620649940133 0.871725976302184\n",
      "14586/14586 [==============================] - 65s 4ms/sample - loss: 0.3657\n",
      "Test res 0.879905394210798 0.5220614835776498 0.5083088954056696\n",
      "Repeat 2 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat2_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5086val_aucs: 0.4750188562242967 0.8583952467668152\n",
      "14586/14586 [==============================] - 85s 6ms/sample - loss: 0.5086\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4690val_aucs: 0.4904079594863471 0.8638954444927233\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4688\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4489val_aucs: 0.4913847030675721 0.8683949050183325\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4492\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4477val_aucs: 0.4876038944962507 0.8653348891026172\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4474\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4424val_aucs: 0.4997791985072281 0.869712980095592\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4425\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4324val_aucs: 0.5081649264465768 0.8711290881662264\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4321\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4171val_aucs: 0.5035529266159331 0.8703687466130285\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4176\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4151val_aucs: 0.5127365995335336 0.8708061846711158\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4150\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4087val_aucs: 0.5184163904324752 0.8722530501052097\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4094\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4043val_aucs: 0.5218711279488664 0.8743859512080809\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4039\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4020val_aucs: 0.511110391390451 0.8703480464192084\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4020\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3934val_aucs: 0.5154777994461192 0.8704330929702339\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3936\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3925val_aucs: 0.5223932075652438 0.8764313646993347\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3925\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3845val_aucs: 0.5060212710771111 0.8708020836893213\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3850\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3819val_aucs: 0.5048017486469276 0.8685758364294118\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3821\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3721val_aucs: 0.5154584204402142 0.8691775090441294\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3721\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3768val_aucs: 0.5033971076502116 0.8683893394001825\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3764\n",
      "Epoch 18/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3705val_aucs: 0.5107080322882538 0.8702243334684053\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3704\n",
      "Epoch 19/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3652val_aucs: 0.5174014282087646 0.8722659389051355\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3656\n",
      "Epoch 20/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3624val_aucs: 0.5174041686386847 0.8678277001791739\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3620\n",
      "Epoch 21/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3564val_aucs: 0.4829616207078521 0.86292722221951\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3565\n",
      "Epoch 22/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3537val_aucs: 0.520406853973306 0.8710938392512779\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.3540\n",
      "Epoch 23/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3453val_aucs: 0.49583105007703376 0.865001537868173\n",
      "14586/14586 [==============================] - 65s 4ms/sample - loss: 0.3451\n",
      "Test res 0.8749866135209441 0.5051828936652114 0.4931506849315068\n",
      "Repeat 3 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat3_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5196val_aucs: 0.4664787335526761 0.8551793935429065\n",
      "14586/14586 [==============================] - 86s 6ms/sample - loss: 0.5196\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4702val_aucs: 0.4838470687498714 0.8596299352142518\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4708\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4560val_aucs: 0.5039327468226517 0.8686943743317596\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4555\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4475val_aucs: 0.5118571793426959 0.8700785533298508\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4475\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4353val_aucs: 0.5023833799906734 0.8688698377671129\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4348\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4310val_aucs: 0.5072757572471527 0.8692171518681437\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4309\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4281val_aucs: 0.5103276272752453 0.8721841145540915\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4280\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4225val_aucs: 0.5098765670814571 0.8728463254714909\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4227\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4166val_aucs: 0.5055913294777994 0.8728045345141557\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4165\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4142val_aucs: 0.5147571735225053 0.8744486376440836\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4137\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4065val_aucs: 0.48942757394473485 0.8708093092286737\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4064\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4064val_aucs: 0.49667846069836674 0.8714161568918465\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4064\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3961val_aucs: 0.4976980031315847 0.8725014524310521\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3959\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3961val_aucs: 0.5146206057709631 0.8743401569113749\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3960\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3880val_aucs: 0.5069406131691978 0.8726010477032061\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3882\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3836val_aucs: 0.5013163300655522 0.8741727001547633\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3833\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3775val_aucs: 0.4992161276729463 0.8727252488661273\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3775\n",
      "Epoch 18/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3761val_aucs: 0.4838534880185825 0.870430847194489\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3757\n",
      "Epoch 19/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3737val_aucs: 0.475860288156101 0.8635424671311192\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3742\n",
      "Epoch 20/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3672val_aucs: 0.4708615082428531 0.8622534894961165\n",
      "14586/14586 [==============================] - 66s 5ms/sample - loss: 0.3672\n",
      "Test res 0.8789266805133684 0.5144396856772124 0.5\n",
      "Repeat 4 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat4_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5224val_aucs: 0.45963835178642093 0.8513056256682403\n",
      "14586/14586 [==============================] - 86s 6ms/sample - loss: 0.5224\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4731val_aucs: 0.49344725456701505 0.8640732513462449\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4730\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4581val_aucs: 0.4962178700105083 0.8678515249305517\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4582\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4504val_aucs: 0.4935372011302665 0.8669389588388363\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4500\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4447val_aucs: 0.5006690541327282 0.8687888921978821\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4448\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4331val_aucs: 0.4997718924742493 0.8682032329406479\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4338\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4223val_aucs: 0.499681371334321 0.8717792890655133\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4220\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4195val_aucs: 0.5116089052989838 0.8717988175502493\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4202\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4185val_aucs: 0.5233945573539799 0.8753231231905638\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4186\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4090val_aucs: 0.49859619468454214 0.8701941619594882\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4092\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4059val_aucs: 0.5146341089927112 0.8750354686104018\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4062\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4066val_aucs: 0.503228193924721 0.8734116751045993\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4066\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3925val_aucs: 0.505535723236201 0.8703890562371539\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3922\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3899val_aucs: 0.5170253985092884 0.8762187971429827\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3898\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3841val_aucs: 0.5111000579687157 0.8732279120632332\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3839\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3803val_aucs: 0.5012979342898239 0.8706706569870478\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3800\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3797val_aucs: 0.5042404544104346 0.8686018093141108\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3798\n",
      "Epoch 18/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3726val_aucs: 0.5028808357937595 0.8703494134131398\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3725\n",
      "Epoch 19/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3686val_aucs: 0.5046788914568657 0.8709049011614565\n",
      "14586/14586 [==============================] - 66s 5ms/sample - loss: 0.3686\n",
      "Test res 0.8796798050256325 0.5217560312666731 0.5102439024390244\n",
      "Repeat 5 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat5_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5130val_aucs: 0.46796581217296906 0.8510826103725546\n",
      "14586/14586 [==============================] - 87s 6ms/sample - loss: 0.5126\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4665val_aucs: 0.4830044404554131 0.8613569367618844\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4668\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4507val_aucs: 0.4911108129585115 0.8654083162052248\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4503\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4388val_aucs: 0.4895640930703888 0.8658686025904535\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4391\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4316val_aucs: 0.504752144252327 0.8709069516523541\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4313\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4292val_aucs: 0.5041270624050812 0.870648589799296\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4289\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4196val_aucs: 0.5033649033502892 0.8675620151443402\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4200\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4126val_aucs: 0.5153211966619679 0.8693893931035157\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4123\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4139val_aucs: 0.5099631966203644 0.8727572755810945\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4138\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4084val_aucs: 0.47678511144905455 0.866266983679069\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4081\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4038val_aucs: 0.493370913544566 0.8673717100605871\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4038\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3983val_aucs: 0.49307260891341703 0.8600028316302867\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3983\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3911val_aucs: 0.5062310311085851 0.8700731829965482\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3909\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3928val_aucs: 0.4822807980130716 0.8664050500661528\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3925\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3865val_aucs: 0.5028655136434619 0.8672020075282307\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3865\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3800val_aucs: 0.49292013333120704 0.8632449506661654\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3809\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3791val_aucs: 0.5120994993594593 0.8652335362668372\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3789\n",
      "Epoch 18/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3680val_aucs: 0.5072394055727628 0.8635640461067525\n",
      "14586/14586 [==============================] - 66s 5ms/sample - loss: 0.3678\n",
      "Test res 0.8667304575414501 0.49604057691577885 0.498046875\n",
      "Repeat 6 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat6_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5177val_aucs: 0.46623932929029205 0.8514046350858522\n",
      "14586/14586 [==============================] - 88s 6ms/sample - loss: 0.5174\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4748val_aucs: 0.49331863129413994 0.8649575987775168\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4749\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4575val_aucs: 0.5044802191630848 0.8668586967665711\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4573\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4575val_aucs: 0.500978558894886 0.868041146517339\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4582\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4470val_aucs: 0.5055681488664782 0.8710192404395862\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4470\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4385val_aucs: 0.5058356939979414 0.8743052985661212\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4384\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4316val_aucs: 0.5025635816916485 0.8655239248348622\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4312\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4250val_aucs: 0.5181884551965468 0.8759491087687779\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4255\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4241val_aucs: 0.5219766940174541 0.8768016247699298\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4241\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4171val_aucs: 0.5184860039823219 0.8739588632469034\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4170\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4120val_aucs: 0.5086335500595425 0.8729672067920069\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4114\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4073val_aucs: 0.526042208851135 0.8753524159176679\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4075\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4048val_aucs: 0.5267474557703214 0.8734708464133497\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4043\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4002val_aucs: 0.49223846442535596 0.8664585581143295\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4001\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3930val_aucs: 0.5095299603088238 0.8699201773186414\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3929\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3911val_aucs: 0.5237249957093528 0.8747833558724595\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3907\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3870val_aucs: 0.5147804346244376 0.8732031108876183\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3873\n",
      "Epoch 18/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3827val_aucs: 0.5183790991207664 0.8740077821011675\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3825\n",
      "Epoch 19/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3781val_aucs: 0.5219082942022718 0.875373409038759\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3781\n",
      "Epoch 20/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3714val_aucs: 0.5113989763555643 0.870914567761401\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3713\n",
      "Epoch 21/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3634val_aucs: 0.5148940252696163 0.8717689389686031\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3638\n",
      "Epoch 22/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3724val_aucs: 0.4908495445564882 0.8693669353460691\n",
      "14586/14586 [==============================] - 67s 5ms/sample - loss: 0.3721\n",
      "Test res 0.8821390753124514 0.5345680518025648 0.5146771037181996\n",
      "Repeat 7 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat7_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5203val_aucs: 0.4818535299817492 0.8589830541573702\n",
      "14586/14586 [==============================] - 88s 6ms/sample - loss: 0.5206\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4746val_aucs: 0.48410994021332227 0.8626165240273593\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4749\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4641val_aucs: 0.47337726712034056 0.8645100059073667\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4640\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4505val_aucs: 0.49552405019973056 0.8657424485790585\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4505\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4429val_aucs: 0.5029962705796586 0.8677206840828203\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4445\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4418val_aucs: 0.5029592824510911 0.8683942215213667\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4421\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4332val_aucs: 0.5214157179239018 0.8762306118762481\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4332\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4281val_aucs: 0.5086080984780516 0.8707034648414043\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4277\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4291val_aucs: 0.5103342207438687 0.8732050637360922\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4296\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4195val_aucs: 0.5032932593135452 0.8700283651240791\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4193\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4157val_aucs: 0.48322443058982 0.8672197784493407\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4154\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4086val_aucs: 0.49471533699882514 0.8665276889502952\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4081\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4062val_aucs: 0.49361116212155204 0.8692011385106602\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4059\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4084val_aucs: 0.5002546165794125 0.8681021730321391\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4081\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4006val_aucs: 0.49891390997548196 0.8689252010213397\n",
      "14586/14586 [==============================] - 58s 4ms/sample - loss: 0.4002\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3983val_aucs: 0.5156953498520174 0.8717451142172252\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3981\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3902val_aucs: 0.483174161478895 0.8628301656503718\n",
      "14586/14586 [==============================] - 66s 5ms/sample - loss: 0.3902\n",
      "Test res 0.8775499741875068 0.5165988659526786 0.5117416829745597\n",
      "Repeat 8 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat8_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5212val_aucs: 0.47145502667384 0.8529413315497317\n",
      "14586/14586 [==============================] - 89s 6ms/sample - loss: 0.5212\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4814val_aucs: 0.48214802000234 0.8612375200777234\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4815\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4622val_aucs: 0.48739091502938925 0.8664085651934051\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4618\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4561val_aucs: 0.5047714335636521 0.8690724457962495\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4559\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4461val_aucs: 0.5059841787764299 0.8691554418563777\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4459\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4463val_aucs: 0.517967011798307 0.873024034682589\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4465\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4330val_aucs: 0.5180131352125541 0.8718077030108042\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4328\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4281val_aucs: 0.516409468279514 0.8741088420096763\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4281\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4232val_aucs: 0.5053631474975335 0.8716210106967275\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4232\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4196val_aucs: 0.4801139593830718 0.8673955348119652\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4195\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4136val_aucs: 0.4904258836449509 0.8635974398156511\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4137\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4065val_aucs: 0.5022320212415927 0.87047146644274\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4064\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4096val_aucs: 0.5154371289198318 0.8673012122306901\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4091\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4010val_aucs: 0.5050426019966896 0.869779572228542\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4008\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3941val_aucs: 0.510258921029905 0.8714545303643527\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3940\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3881val_aucs: 0.5106010240828632 0.8682104584800003\n",
      "14586/14586 [==============================] - 67s 5ms/sample - loss: 0.3881\n",
      "Test res 0.8799537776283721 0.5232969349814791 0.5097847358121331\n",
      "Repeat 9 ld 50\n",
      "Num train: 14586 Num valid: 3611\n",
      "random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat9_50ld.h5\n",
      "Train on 14586 samples\n",
      "Epoch 1/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.5277val_aucs: 0.46918771694144723 0.8540795492825722\n",
      "14586/14586 [==============================] - 90s 6ms/sample - loss: 0.5274\n",
      "Epoch 2/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4792val_aucs: 0.4730623815915115 0.8595831644933088\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4794\n",
      "Epoch 3/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4701val_aucs: 0.47616219486953615 0.8631354935092199\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4697\n",
      "Epoch 4/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4611val_aucs: 0.4888289633713475 0.8664708610597133\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4608\n",
      "Epoch 5/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4505val_aucs: 0.49224317699807857 0.8671979065464362\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4506\n",
      "Epoch 6/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4427val_aucs: 0.4840246411939292 0.858806321370509\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4423\n",
      "Epoch 7/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4397val_aucs: 0.4963524044576404 0.8674261945330007\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4396\n",
      "Epoch 8/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4423val_aucs: 0.4884457626327698 0.8655821197193756\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4429\n",
      "Epoch 9/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4309val_aucs: 0.4879076256052067 0.8628467648623975\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4306\n",
      "Epoch 10/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4266val_aucs: 0.5047121060698839 0.8630530833036337\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4265\n",
      "Epoch 11/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4252val_aucs: 0.4972271035352634 0.8645233829194108\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4250\n",
      "Epoch 12/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4178val_aucs: 0.5061419596236371 0.8681667146741916\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4181\n",
      "Epoch 13/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4109val_aucs: 0.5032254355077055 0.8690660990387103\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4111\n",
      "Epoch 14/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4068val_aucs: 0.48214013060927396 0.8654231578536242\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4068\n",
      "Epoch 15/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.4064val_aucs: 0.5095369941382151 0.8688441578096853\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.4059\n",
      "Epoch 16/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3976val_aucs: 0.4877126741368868 0.8696541017141127\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3973\n",
      "Epoch 17/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3951val_aucs: 0.5018376743520704 0.8701845930019674\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3951\n",
      "Epoch 18/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3885val_aucs: 0.49286929186765277 0.8632060889815406\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3883\n",
      "Epoch 19/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3843val_aucs: 0.457025899486691 0.8592486415497806\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3840\n",
      "Epoch 20/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3777val_aucs: 0.48991445858413074 0.8640151541041552\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3778\n",
      "Epoch 21/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3714val_aucs: 0.49432444150861127 0.8650730121223069\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3718\n",
      "Epoch 22/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3704val_aucs: 0.47376717486372494 0.8595622690146416\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3704\n",
      "Epoch 23/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3699val_aucs: 0.4656726131738753 0.8555284652075634\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3704\n",
      "Epoch 24/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3655val_aucs: 0.4668706079789288 0.8605671071967348\n",
      "14586/14586 [==============================] - 59s 4ms/sample - loss: 0.3652\n",
      "Epoch 25/1000\n",
      "14560/14586 [============================>.] - ETA: 0s - loss: 0.3587val_aucs: 0.48058773986376213 0.8626314633181826\n",
      "14586/14586 [==============================] - 67s 5ms/sample - loss: 0.3587\n",
      "Test res 0.8753077689601767 0.5174683134097293 0.5043988269794721\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554), (0.49922876188707954, 0.0057904772677334635), (0.49589993895065787, 0.005290317714153174)], 40: [(0.8731307552855583, 0.00318481506214208), (0.5010970039960129, 0.008185009808080705), (0.49284356106523897, 0.007075296181590279)], 50: [(0.8773054758503115, 0.004086390228060627)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554), (0.49922876188707954, 0.0057904772677334635), (0.49589993895065787, 0.005290317714153174)], 40: [(0.8731307552855583, 0.00318481506214208), (0.5010970039960129, 0.008185009808080705), (0.49284356106523897, 0.007075296181590279)], 50: [(0.8773054758503115, 0.004086390228060627), (0.5177658768279052, 0.01030545828543718)]}\n",
      "gen_res {10: [(0.84984756822302, 0.005257012380309644), (0.44535424723826955, 0.011354694681146936), (0.4615354050632619, 0.010848903361120925)], 20: [(0.8645769872618347, 0.0037371300288093563), (0.4845822801434993, 0.007376174752995785), (0.48199996084988317, 0.010083512242458466)], 30: [(0.8709389744636404, 0.0026779029142823554), (0.49922876188707954, 0.0057904772677334635), (0.49589993895065787, 0.005290317714153174)], 40: [(0.8731307552855583, 0.00318481506214208), (0.5010970039960129, 0.008185009808080705), (0.49284356106523897, 0.007075296181590279)], 50: [(0.8773054758503115, 0.004086390228060627), (0.5177658768279052, 0.01030545828543718), (0.5056223548747845, 0.006414330494625141)]}\n"
     ]
    }
   ],
   "source": [
    "repeats = {k:10 for k in [10,20,30,40,50]}\n",
    "print(repeats)\n",
    "lds = [10,20,30,40,50]\n",
    "batch_size, lr, patience = 32, 0.0005, 10\n",
    "d, N, he, dropout = 50,2,4,0.2\n",
    "fore_savepath = 'mimic_iv_24h_strats_no_interp_with_ss_fore.h5'\n",
    "f = open('log.csv', 'a+')\n",
    "f.write('\\nTraining on different % of labeled data\\n')\n",
    "\n",
    "#print(\"train op\", train_op)\n",
    "#print(\"val op\",valid_op)\n",
    "#print(\"train op len\", len(train_op))\n",
    "#print(\"val op len\",len(valid_op))\n",
    "\n",
    "train_inds = np.arange(len(train_op))\n",
    "valid_inds = np.arange(len(valid_op))\n",
    "\n",
    "#print(\"train ind len\", len(train_inds))\n",
    "#print(\"val ind len\", len(valid_inds))\n",
    "gen_res = {}\n",
    "\n",
    "np.random.seed(2022)\n",
    "for ld in lds:\n",
    "    np.random.shuffle(train_inds)\n",
    "    np.random.shuffle(valid_inds)\n",
    "    train_starts = [int(i) for i in np.linspace(0, len(train_inds)-int(ld*len(train_inds)/100), repeats[ld])]\n",
    "    valid_starts = [int(i) for i in np.linspace(0, len(valid_inds)-int(ld*len(valid_inds)/100), repeats[ld])]\n",
    "    #print(\"train_starts\",train_starts)\n",
    "    #print(\"valid_starts\",valid_starts)\n",
    "    f.write('Training on '+str(ld)+' % of labaled data+\\n'+'val_metric,roc_auc,pr_auc,min_rp,savepath\\n')\n",
    "    all_test_res = []\n",
    "    for i in range(repeats[ld]):\n",
    "        print ('Repeat', i, 'ld', ld)\n",
    "        # Get train and validation data.\n",
    "        curr_train_ind = train_inds[np.arange(train_starts[i], train_starts[i]+int(ld*len(train_inds)/100))]\n",
    "        #print(\"curr_train_ind\",curr_train_ind)\n",
    "        curr_valid_ind = valid_inds[np.arange(valid_starts[i], valid_starts[i]+int(ld*len(valid_inds)/100))]\n",
    "        #print(\"curr_valid_ind\",curr_valid_ind)\n",
    "        curr_train_ip = [ip[curr_train_ind] for ip in train_ip]\n",
    "        #print(\"curr_train_ip\",curr_train_ip)\n",
    "        curr_valid_ip = [ip[curr_valid_ind] for ip in valid_ip]\n",
    "        #print(\"curr_valid_ip\",curr_valid_ip)\n",
    "        curr_train_op = train_op[curr_train_ind]\n",
    "        #print(\"curr_train_op\",curr_train_op)\n",
    "        curr_valid_op = valid_op[curr_valid_ind]\n",
    "        #print(\"curr_valid_op\",curr_valid_op)\n",
    "        print ('Num train:',len(curr_train_op),'Num valid:',len(curr_valid_op))\n",
    "        # Construct save_path.\n",
    "        savepath = 'random_seed_2_mimic_iv_24hm_strats_no_interp_with_ss_repeat'+str(i)+'_'+str(ld)+'ld'+'.h5'\n",
    "        print (savepath)\n",
    "        # Build and compile model.\n",
    "        model, fore_model =  build_strats(D, max_len, V, d, N, he, dropout, forecast=True)\n",
    "        model.compile(loss=mortality_loss, optimizer=Adam(lr))\n",
    "        fore_model.compile(loss=forecast_loss, optimizer=Adam(lr))\n",
    "        # Load pretrained weights here.\n",
    "        fore_model.load_weights(fore_savepath)\n",
    "        # Train model.\n",
    "        es = EarlyStopping(monitor='custom_metric', patience=patience, mode='max', \n",
    "                           restore_best_weights=True)\n",
    "        cus = CustomCallback(validation_data=(valid_ip, valid_op), batch_size=batch_size)\n",
    "        his = model.fit(curr_train_ip, curr_train_op, batch_size=batch_size, epochs=1000,\n",
    "                        verbose=1, callbacks=[cus, es]).history\n",
    "        model.save_weights(savepath)\n",
    "\n",
    "        #if ld == lds[-1] and i == repeats[ld] - 1:\n",
    "            #val_predicted = model.predict(valid_ip, verbose=0, batch_size=batch_size)\n",
    "            #val_true_op = valid_op\n",
    "            #valid_op_ind_df['val_predicted'] = val_predicted \n",
    "            #valid_op_ind_df['val_true_op'] = valid_op # Assuming this is what you mean by val_y\n",
    "            \n",
    "            #valid_op_ind_df.to_csv(f'final_val_results{ld}ld_repeat{i}.csv', index=False)\n",
    "        # Test and write to log.\n",
    "        #with warnings.catch_warnings():\n",
    "            #warnings.filterwarnings('error')\n",
    "            #try:\n",
    "        a = model.predict(test_ip, verbose=0, batch_size=batch_size)\n",
    "        rocauc, prauc, minrp = get_res(test_op, a )\n",
    "        #rocauc, prauc, minrp = get_res(test_op, model.predict(test_ip, verbose=0, batch_size=batch_size))\n",
    "\n",
    "        if i == repeats[ld] - 1:\n",
    "            test__op_ind_df['test_predicted'] = a\n",
    "            test__op_ind_df['test_true_op'] = test_op\n",
    "            test__op_ind_df.to_csv(f'mimic iv final_test_results{ld}ld_repeat{i}.csv', index=False)\n",
    "\n",
    "        f.write(str(np.min(his['custom_metric']))+str(rocauc)+str(prauc)+str(minrp)+savepath+'\\n')\n",
    "        print ('Test res', rocauc, prauc, minrp)\n",
    "        all_test_res.append([rocauc, prauc, minrp])\n",
    "            #except (ValueError, RuntimeWarning) as e:\n",
    "               # print(f\"Skipping ROC AUC calculation due to error or warning: {e}\")\n",
    "                #all_test_res.append([None, None, None])\n",
    "                #f.write(\"ROC AUC calculation skipped due to error or warning.\\n\")\n",
    "        \n",
    "        \n",
    "    gen_res[ld] = []\n",
    "\n",
    "    #if len(all_test_res) > 0:\n",
    "    for i in range(len(all_test_res[0])):\n",
    "        nums = [test_res[i] for test_res in all_test_res]\n",
    "        gen_res[ld].append((np.mean(nums), np.std(nums)))\n",
    "        print ('gen_res', gen_res)\n",
    "    #else\n",
    "        #print(\"No test results available.\")\n",
    "\n",
    "\n",
    "    #for i in range(len(all_test_res[0])):\n",
    "        #nums = [test_res[i] for test_res in all_test_res]\n",
    "        #gen_res[ld].append((np.mean(nums), np.std(nums)))\n",
    "    #print ('gen_res', gen_res)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "infinite-theology",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAn9klEQVR4nO3deZyVZf3/8ddbFDcQNzQRDTFFzRRxMC2zzFS03CoT3BBNMjF3A/uqmWY/s8wNkkARVzQXwn3JNNeUQTaBSEQUZGTRwl1k+Pz+uG7kMMxwGJgz95yZ9/PxOI9z7uu+73M+537MzGeu5b4uRQRmZmYra428AzAzs/LixGFmZvXixGFmZvXixGFmZvXixGFmZvWyZt4BNIZNN900OnXqlHcYZmZlZcyYMfMjon3N8haRODp16kRlZWXeYZiZlRVJb9ZW7qYqMzOrFycOMzOrFycOMzOrFycOMzOrFycOMzOrFycOM7PmqqoKvv1teOedBn1bJw4zs+bq0kvhuefScwNy4jAza46eeQaGDIHFi+Gmmxq01uHEYWbWXHz8Mdx8M+y9d2qiqq5O5dXVDVrrcOIwMyt348ZBv37QoQOccELq21izYGKQhQsbtNbhxGFmVo4++ACGDoU99oDddoMbb4Qf/ACefhoOOADWqPHnvQFrHS1iriozs2YhAkaPTgljxAj46CPYeWe45ho49ljYeON03JlnplpGoYUL4YUXGiQMJw4zs6buf/+D225LCWPCBFhvPejZE04+Gb7+dZCWPX7s2JKG48RhZtYURcDzz6dkcffd8Mkn0K0bDB4MvXrBBhvkFpoTh5lZUzJ/PtxyC9xwA0yZAm3bQu/eqXbRrVve0QFOHGZm+Vu8GJ56KtUuRo5M/RF77QXDhsFPfgLrr593hMtw4jAzy8s778Dw4al28frrsNFGcMopqXax8855R1cnJw4zs8ZUXQ2PP55qFw88AIsWpZv1fvMb+NGPYJ118o6wKCcOM7PGMHNmanoaNgzeegvat4ezzoKf/hS23z7v6OrFicPMrFQ+/xweeijVLh59NPVl7L8/XHklHHootG6dd4SrxInDzKyhTZ+e7uS+6aY0/ccWW8D558NJJ8E22+Qd3Wpz4jAzawgLF8Lf/pZqF3//e5ry4+CDU0f3wQcvO3dUmSvpXFWSekiaKmmapAG17G8n6QFJ4yVNktQnK+8iaVzB431JZ2b7Lpb0dsG+g0v5HczMVmjqVDj3XNhySzjqKHjtNbjkEnjzzdT5feihzSppQAlrHJJaAYOA/YFZwGhJ90fE5ILD+gGTI+IQSe2BqZJuj4ipQNeC93kbGFlw3lUR8cdSxW5mtkKffAL33ptqF888kxLDoYdC377wve9Bq1Z5R1hSpUyDewDTImI6gKQ7gcOAwsQRQFtJAtoA7wGLarzPfsDrEfFmCWM1Mytu4sSULG69Nc0f9ZWvwOWXp6nMN9887+gaTSkTx5bAzILtWcDXaxwzELgfmA20BY6KiMU1jukJjKhRdpqk44FK4JyI+G/ND5fUF+gLsPXWW6/qdzCzlu7DD+Guu1LCeOmlNBLqRz9KfRff/vby05e3AKX8xqqlLGpsHwiMAzqQmqYGSvpi5i5JrYFDgbsLzrke2DY7vgq4srYPj4ghEVERERXt27dftW9gZi3XmDHpLu4OHdK9Fu+/D3/6E7z9NtxxB+y7b4tMGlDaGscsYKuC7Y6kmkWhPsDlERHANElvADsAL2f7DwJeiYg5S04ofC1pKPBgCWI3s5ZowYKUFIYOTVOTr7tumivq5JPhG99YfvryFqqUiWM0sJ2kbUid2z2Bo2sc8xapD+NZSZsDXYDpBft7UaOZStIWEVGVbR4BvFqC2M2spYiAf/0LhgyBv/41rdvdtSsMGgRHHw0bbph3hE1OyRJHRCySdBrwGNAKGBYRkySdku0fDFwKDJc0kdS01T8i5gNIWo80IutnNd76CkldSc1eM2rZb2ZW3HvvpU7uoUNh0iRo0waOOSaNjNp9d9cuVkCplah5q6ioiMrKyrzDMLO8RcA//5mSxb33wmefpTW7Tz45rajXpk3eETYpksZEREXN8uZ1V4qZWW3mzl06fflrr0G7dilZnHwy7LJL3tGVHScOM2ueFi9OU38MGQKjRqXpy/feGy64AH7847Rut60SJw4za17efjtNLnjjjTBjBmyyCZx+ehpSu+OOeUfXLDhxmFn5W7QIHnkk9V089FCqbey3X7qr+/DDYe21846wWWmZd6+YWfmpqkp3ar/zztKyGTPgwguhU6c0V9To0dC/P0yblpqpjjrKSaMEXOMws/Jw6aXw3HNw8cVpMaShQ9MSrAA9esB118EPfgBrrZVrmC2Bh+OaWdM3eTLstlta82KJjh3Twkgnngiej64kPBzXzMrDRx/BK6+kZqclj9dfX7pfSgsjjRrV7Kcvb6qcOMwsPwsXwoQJyyaJyZNT5zakmsTOO6dFkRZlKy5EwD/+AfPmwZe+lF/sLZgTh5k1jurqtFpeYZIYN25p89Omm0L37mnK8u7doaIirXFx6qnLz0JbXZ36PAYNavSvYU4cZlYKEWnEU2GSGDMmrW0B0LZtmg/qjDNSkujeHb785drnh3rxxWX7NiBtv/BCyb+G1c6Jw8xW35w5SxPEyy9DZSXMn5/2rb12mm32hBOWJokuXVZ+LYuxY0sVta0iJw4zq58FC1JiKKxNzMwW+1xjDfjqV9M9FUuSxNe+llbNs2bDicPM6vbJJ+k//sIk8Z//LN3/la+k+Z+WJInddoP1188vXmsUThxmlnz+eVqXojBJTJyYOqIhLaHavTscf/zSzuuNN843ZsuFE4dZS7R4cZpevDBJjB0Ln36a9m+0UUoOAwYsrU106JBvzNZkOHGYNXcRMGvWsp3XY8akvgpI04t36wY//3la1Kh7d+jc2SvgWZ2cOMyam/nzl61JjB6dRj1Bmsdpl12gV6+lNYkdd4Q1/afAVp5/WszK2QcfpNpDYZKYMSPtk1JS6NFjaZLYZRdYZ51cQ7by58RhVi4++wzGj182SUyZkpqiIE0t3r17utO6e/d0g13btrmGbM2TE4dZXqqqoGdPuOuu5edcqq5OczYVJokJE9LIJ4DNNkv9EUcdtXSEU/v2jf8drEVy4jDLy5L1JS65BM45Z9nO61degY8/TsdtsEFKDGefvbTJaaut3HltufF6HGZ5qKpKTUs152BaZ510E92SBNG9O2y33cpPz2HWgLweh1lTMWMG7Lvv0qSxxhrwrW/B1Ven6Tq8gp01cf43xqyxfPop/Pa3sMMOS0c+QboZ7+WXUz+Hk4aVAScOs8bw6KNpsr8LL4Qtt1x+0r8l60uYlQEnDrNSevNN+OEP4aCDUpPUY4+lzm6vL2FlrKSJQ1IPSVMlTZM0oJb97SQ9IGm8pEmS+mTlXSSNK3i8L+nMbN/Gkp6Q9Fr2vFEpv4PZKvnsM/jd79INeI8+ml5PmAAHHJDmhIpY/uF1J6xMlCxxSGoFDAIOAnYCeknaqcZh/YDJEbEr8B3gSkmtI2JqRHSNiK7A7sDHwMjsnAHAkxGxHfBktm3WdDz+eLpD+//+L9U0/v1vOP/8tKCRWTNQyhrHHsC0iJgeEQuBO4HDahwTQFtJAtoA7wGLahyzH/B6RLyZbR8G3Jy9vhk4vASxm9XfzJlw5JFw4IGpw/uRR+Dee2HrrfOOzKxBlTJxbAnMLNielZUVGgjsCMwGJgJnRMTiGsf0BEYUbG8eEVUA2fNmtX24pL6SKiVVzps3b9W/hVkxCxfC73+fRks99FAaOfXqq2mOKLNmqJSJo7bbWmvebXggMA7oAHQFBkra4Is3kFoDhwJ31/fDI2JIRFREREV7T8VgpfLkk6lZasCA1H8xeXJqonKzlDVjpUwcs4CtCrY7kmoWhfoA90UyDXgD2KFg/0HAKxExp6BsjqQtALLnuQ0euVkxs2aleaK+9z1YtCjVNEaOTHeDmzVzpUwco4HtJG2T1Rx6AvfXOOYtUh8GkjYHugDTC/b3YtlmKrL36J297g2MauC4zeq2cCH84Q+pWer++9M8U6++CgcfnHdkZo2mZFOORMQiSacBjwGtgGERMUnSKdn+wcClwHBJE0lNW/0jYj6ApPWA/YGf1Xjry4G/SjqJlHiOLNV3MFvGU09Bv35pKvNDDklThHTunHdUZo2upHNVRcTDwMM1ygYXvJ4NHFDHuR8Dm9RS/i5ZLcWsUcyeDeeeCyNGwDbbpJrGIYfkHZVZbnznuFldPv8c/vQn6NIF7rsPfv1rmDTJScNaPM+Oa1abf/4zNUtNmpT6L669FrbdNu+ozJoE1zjMClVVwbHHwne+Ax9+CKNGwYMPOmmYFXDiMIM0pPbqq1Oz1N13p1lsJ0+GQw/1SntmNbipyuzZZ1Oz1MSJabqQ665Lq+6ZWa1c47CWa84c6N0b9tkHFixIHeCPPOKkYVaEE4e1PIsWpVrF9tunIba/+lVqljriCDdLma0EN1VZy/L886lZavx42H9/GDgwJRAzW2mucVjLMHcu9OkDe+8N774L99yTVuNz0jCrNycOa96qq2HQoDRa6vbb0yy2U6bAj37kZimzVeSmKmu+XnwxNUuNHQv77ZeapXbYofh5ZrZCrnFY8zNvHpx0EnzjG2nk1F13wRNPOGmYNRAnDms+qqvh+utTs9Qtt8B556X1vn/yEzdLmTUgN1VZ8/Dyy3DqqTBmDOy7b2qW2mmnvKMya5Zc47DyNn8+9O0Le+6Zpj8fMSIt5+qkYVYyThxWnqqrYciQ1Cw1bBicfXZqlurZ081SZiXmpiorP6NHp9FSo0fDt7+dmqV23jnvqMxaDNc4rHy8+y6ccgp8/eswcybcdltaztVJw6xROXFY07d4MdxwQ2qWuuEGOOOM1Cx1zDFuljLLgZuqrGkbMyY1S730UpouZNAg2GWXvKMya9FWWOOQ1F3SQbWUHypp99KFZS3ef/+bEkb37vDGG+m+jGeecdIwawKKNVX9AZhSS/nkbJ9Zw1q8GG66KU0+OHgw/OIXMHUqHHecm6XMmohiiWOTiJhRszAipgGblCQia7nGjk3NUSeemBLHmDFwzTWw4YZ5R2ZmBYoljnVXsG/9hgzEWrD//S/VLCoqYNo0GD48LefatWvOgZlZbYoljr9Lukxato1A0m+Af5QuLGsRFi+Gm29Oo6X+/Gf4+c9Ts1Tv3rCGB/yZNVXFRlWdA9wATJM0LivbFagEflrCuKy5Gz8+dX4//3yaLuSRR6Bbt7yjMrOVsMLEEREfAb0kdQa+mhVPiojpJY/MmqcFC+Cii9Ld3htvDDfeCCec4BqGWRkpNhy3m6RuwIbA28As4POVfXNJPSRNlTRN0oBa9reT9ICk8ZImSepTsG9DSfdI+rekKZL2ysovlvS2pHHZ4+CVjcdyFAG33pqapa67Dn72s9QsdeKJThpmZaZYU9WVtZRtLKk10CsixtV1oqRWwCBgf1LCGS3p/oiYXHBYP2ByRBwiqT0wVdLtEbEQuAZ4NCJ+nH3eegXnXRURfyz67axpmDgxNUs9+yzssQc8+GDqCDezslSsqWrf2solVQDXAvus4PQ9gGlLmrUk3QkcRroH5IuPANpmne9tgPeARZI2yN77hCyOhcDClfg+1pS8/z5cfDFce20aUjt0qGsYZs3AKv0GR0Ql6Q/9imwJzCzYnpWVFRoI7AjMBiYCZ0TEYqAzMA+4SdJYSTdIKhz+e5qkCZKGSdqotg+X1FdSpaTKefPmrfyXs1VXVZVmq62qgjvuSM1SV1+dlnGdOhV++lMnDbNmYJV+iyVtTqotrPCwWspqnnMgMA7oAHQFBma1jTWBbsD1EbEb8BGwpI/kemDb7Pgqam9OIyKGRERFRFS0b9++SKjWIC69NDVHVVSkCQg7doR//Qv+8hfYxPeLmjUXK2yqknQdy/+x3xj4BnBGkfeeBWxVsN2RVLMo1Ae4PCKCNOT3DWAH4C1gVkS8lB13D1niiIg5BfENBR4sEoc1hqqqNHNtRFqJ7/e/h3POgVat8o7MzBpYsc7xyhrbAbwLnB0Rc4ucOxrYTtI2pBFZPYGjaxzzFrAf8GxWi+kCTI+I+ZJmSuoSEVOzYyYDSNoiIqqy848AXi0ShzWGs8+Gz7MBd61bw5tvOmmYNVNK/+zX8yRpK6BnRKxwosNsqOzVQCtgWERcJukUgIgYLKkDMBzYgtS0dXlE3Jad25V082FrYDrQJyL+K+lWUjNVADOAnxUkklpVVFREZWXNHGgNZsYM6Nw51TaWWHddmD4dvvSl3MIys9UjaUxELDcEcqUTh6RNgSOBXqRO7pERcW6DRlkiThwltuuuMGHCsmWtW6fO8EGD8onJzFZbXYmjWB9HW1Jz0NHA9sBIoHNEdCxJlFZ+Hn98+aQBsHAhvPBC48djZiVXrI9jLvAycAHwXESEpCNKH5aVhblz4fjjYaedoLIyNU+ZWbNXbDjur4B1SENgz5e0belDsrIQke7P+N//YMQIJw2zFmSFiSMiroqIrwOHkjqv/wZ0kNRf0vaNEJ81VX/+c5o65IorvJyrWQuzUjcARsT0iLgsIr4GdAfaAY+UNDJrul59Nd2jcdBBaQEmM2tRVuXO8R9FxK8iws1WLdEnn0CvXmnuqeHDvQ64WQu0Konj0AaPwspH//6pxjF8OGy2Wd7RmFkOViVx+F/Mluqhh9JaGmeeCT165B2NmeVkVRLH7g0ehTV977yTVurbdVe4/PK8ozGzHBVbAfCKJVOELBERiyWdJen3pQ3NmozFi6F3b/jwwzRd+tpr5x2RmeWoWI3jB8CQWsqvAb7f8OFYk3TNNekO8auuSjf7mVmLVixxRLawUs3Cxbivo2UYOzZ1iB92WFon3MxavGKJ42NJ29UszMo+KU1I1mR8/DEcfTS0b5/W2vDQWzOj+FxVFwGPSPotMCYrqwDOB84sYVzWFJx9dlry9YknYNNN847GzJqIFSaOiHhE0uHAecCSW4QnkW4CnFji2CxPI0emJV9/+UvYb7+8ozGzJqRYjYOIeBXoLalN2oyPSh+W5ertt9NaGrvvntYRNzMrUPQ+DkmnSnoLeBN4S9Kbkk4tfWiWi+pqOO44+PTTNPS2deu8IzKzJqbYQk4XAN8AvhMR07OyzsA1kjaOiN82QozWmP74R3jqKbjxRtjeEyCb2fKK1TiOA364JGlAmikX+AlwfCkDsxyMHg0XXABHHgl9+uQdjZk1UUWbqiLi01rKPgGWu7/DytiHH6aht1tskTrFPfTWzOpQLHHMkrTckBpJ3wWqShOS5eL00+H11+G222CjjfKOxsyasGKjqk4HRkl6jnQfR5AWcvomcFiJY7PGctddcNNNqZlqn33yjsbMmrhiS8dOAnYGngE6AZ2z1ztn+6zcvflmmkpkzz3hoovyjsbMysDK3MfxKTCssExSK0nHRMTtJYvMSq+6Go49Ns1+e/vtsNZaeUdkZmWg2LTqG0g6X9JASfsrOQ1YMrLKytnvfgfPPQd//jN07px3NGZWJorVOG4F/gu8CJwM/BJoDRwWEeNKG5qV1Isvwm9+A8cck2odZmYrqVji6BwRXwOQdAMwH9g6Ij4oeWRWOgsWpKG3W20FgwblHY2ZlZliw3E/X/IiIqqBN+qTNCT1kDRV0jRJA2rZ307SA5LGS5okqU/Bvg0l3SPp35KmSNorK99Y0hOSXsuePXa0vvr1g5kz05Qi7drlHY2ZlZliiWNXSe9njw+AXZa8lvT+ik6U1AoYBBwE7AT0klRz+bh+wOSI2BX4DnClpCWTI10DPBoROwC7AlOy8gHAkxGxHfBktm0r67bbUkf4r38Ne+2VdzRmVoaKTaveajXeew9gWsEcV3eS7v2YXPgRQFtJAtoA7wGLJG0A7AOckMWxEFiYnXMYKckA3Aw8DfRfjThbjunT4dRTYe+94Ve/yjsaMytTRaccWQ1bAjMLtmdlZYUGAjsCs4GJwBnZsrSdgXnATZLGSrpB0vrZOZtHRBVA9rxZbR8uqa+kSkmV8+bNa7AvVbY+/zz1a6yxRqp1tFqd/wnMrCUrZeKobbKjqLF9IDAO6AB0BQZmtY01gW7A9RGxG/AR9WySioghEVERERXt27evZ+jN0CWXwEsvwZAh8OUv5x2NmZWxUiaOWcBWBdsdSTWLQn2A+yKZBrwB7JCdOysiXsqOu4eUSADmSNoCIHueW6L4m49nnoHLLoMTToCf+PYbM1s9pUwco4HtJG2TdXj3BO6vccxbwH4AkjYHugDTI+IdYKakLtlx+7G0b+R+oHf2ujcwqnRfoRn473/TfRrbbgvXXpt3NGbWDBSdcmRVRcSi7C7zx4BWwLCImCTplGz/YOBSYLikiaSmrf4RMT97i18At2dJZzqpdgJwOfBXSSeREs+RpfoOZS8C+vaFqip44QVo2zbviMysGShZ4gCIiIeBh2uUDS54PRs4oI5zxwEVtZS/S1ZLsSJuugnuuQcuvxy6d887GjNrJkrZVGV5+s9/0hob++4L552XdzRm1ow4cTRHCxemobdrrw233JKG4JqZNZCSNlVZTi68EMaMgfvug44d847GzJoZ/yva3Dz5JPzhD6lT/Igj8o7GzJohJ47mZP58OP546NIF/vSnvKMxs2bKTVXNRQT89KcpeTz4IKy/fvFzzMxWgRNHczFkCIwaBVdeCbvtlnc0ZtaMuamqOZg8Gc46Cw44AM48M+9ozKyZc+Iod59+mobetmkDN9/sobdmVnJuqip3558P48enfo0vfSnvaMysBfC/p+Xs0Ufh6qvhtNPg+9/POxozayGcOMrVnDnQuzfsvDNccUXe0ZhZC+KmqnIUASeeCAsWpBv+1l0374jMrAVx4ihHAwfCww/DddelGoeZWSNyU1W5mTAhzXb7/e9Dv355R2NmLZATRzn55BPo1Qs22iittaHalnU3MystN1WVk/POSzf7PfYYtG+fdzRm1kK5xlEuHngABg2Cs89Od4ibmeXEiaMczJ4NffpA167wu9/lHY2ZtXBOHE3d4sXpfo2PP4YRI9KqfmZmOXIfR1N31VXw97/DX/4CO+yQdzRmZq5xNGmvvJLmojriCDj55LyjMTMDnDiaro8+SkNvN9sMhg710FszazLcVNVUnXUWvPZamlJkk03yjsbM7AuucTRF996bahn9+8O+++YdjZnZMpw4mpqZM1N/RvfucMkleUdjZrYcJ46mpLoajj8eFi6EO+6AtdbKOyIzs+WUNHFI6iFpqqRpkgbUsr+dpAckjZc0SVKfgn0zJE2UNE5SZUH5xZLezsrHSTq4lN+hUV1xBTz9dJr99itfyTsaM7NalaxzXFIrYBCwPzALGC3p/oiYXHBYP2ByRBwiqT0wVdLtEbEw279vRMyv5e2viog/lir2XLz0Elx4IRx1VLrhz8ysiSpljWMPYFpETM8SwZ3AYTWOCaCtJAFtgPeARSWMqWn64AM4+mjYcksYPNhDb82sSStl4tgSmFmwPSsrKzQQ2BGYDUwEzoiIxdm+AB6XNEZS3xrnnSZpgqRhkjaq7cMl9ZVUKaly3rx5q/1lSuoXv4AZM+C222DDDfOOxsxshUqZOGr7tzlqbB8IjAM6AF2BgZI2yPZ9MyK6AQcB/STtk5VfD2ybHV8FXFnbh0fEkIioiIiK9k15CvIRI+Dmm+GCC+Bb38o7GjOzokqZOGYBWxVsdyTVLAr1Ae6LZBrwBrADQETMzp7nAiNJTV9ExJyIqM5qJkOXlJelGTPglFNgr71S/4aZWRkoZeIYDWwnaRtJrYGewP01jnkL2A9A0uZAF2C6pPUltc3K1wcOAF7NtrcoOP+IJeVlZ9EiOPbY9Pr222FN38RvZuWhZH+tImKRpNOAx4BWwLCImCTplGz/YOBSYLikiaSmrf4RMV9SZ2Bk6jNnTeCOiHg0e+srJHUlNXvNAH5Wqu9QUpddBs8/n5LGNtvkHY2Z2UpTRM1uh+anoqIiKisrix/YWJ5/HvbZB445Bm65Je9ozMxqJWlMRFTULPed441twYKUMDp1Sjf6mZmVGTesN6aI1Bk+axY89xxssEHxc8zMmhgnjsZ0661w553w29/CnnvmHY2Z2SpxU1Vjef116Ncv9W0MWG7aLjOzsuHE0Rg+/zxNKbLmmqnW0apV3hGZma0yN1U1hosvhpdfhrvvhq23zjsaM7PV4hpHqT39NPy//wcnnQQ//nHe0ZiZrTYnjlJ67z047ri0tsbVV+cdjZlZg3BTValEpCVg58yBF1+ENm3yjsjMrEE4cZTKjTfCffelVf123z3vaMzMGoybqkph6lQ44wzYbz8455y8ozEza1BOHA3ts8+gVy9Yd920zsYavsRm1ry4qaqhXXABjB0Lf/tbWgrWzKyZ8b/DDenvf4c//jHNR3VYzeXVzcyaByeOhjJvHhx/POy4I1xZ62q2ZmbNgpuqGkJEusHv3XfhkUdgvfXyjsjMrGScOBrC9dfDAw+km/x23TXvaMzMSspNVatr0qQ05LZHDzj99LyjMTMrOSeO1fHpp2no7QYbwPDhkNZINzNr1txUtTr694eJE+Ghh2DzzfOOxsysUbjGsaoefhiuvTY1Tx18cN7RmJk1GieOVTFnDvTpA1/7Gvz+93lHY2bWqNxUVV+LF8MJJ8D778M//gHrrJN3RGZmjcqJo76uuw4efRQGDYKvfjXvaMzMGp2bqupj/Hj45S/hkEPg5z/POxozs1w4caysjz9OQ2832QSGDfPQWzNrsdxUtbLOOQemTIHHH4dNN807GjOz3JS0xiGph6SpkqZJGlDL/naSHpA0XtIkSX0K9s2QNFHSOEmVBeUbS3pC0mvZ80al/A4AjBoFgwfDuefC/vuX/OPMzJqykiUOSa2AQcBBwE5AL0k71TisHzA5InYFvgNcKal1wf59I6JrRFQUlA0AnoyI7YAns+3SqKqCPfdMo6i6dYPLLivZR5mZlYtS1jj2AKZFxPSIWAjcCdRcpCKAtpIEtAHeAxYVed/DgJuz1zcDhzdYxDVdcgm89BJ88AHccQe0bl38HDOzZq6UiWNLYGbB9qysrNBAYEdgNjAROCMiFmf7Anhc0hhJfQvO2TwiqgCy581q+3BJfSVVSqqcN29e/aOvqoIbbkiv11gD2rWr/3uYmTVDpUwctQ07ihrbBwLjgA5AV2CgpA2yfd+MiG6kpq5+kvapz4dHxJCIqIiIivbt29crcAAuvXTpa2nZbTOzFqyUiWMWsFXBdkdSzaJQH+C+SKYBbwA7AETE7Ox5LjCS1PQFMEfSFgDZ89wGj7yqCm66CRZlrWYLF6btd95p8I8yMys3pUwco4HtJG2TdXj3BO6vccxbwH4AkjYHugDTJa0vqW1Wvj5wAPBqds79QO/sdW9gVINHfumlaWqRQtXVrnWYmVHC+zgiYpGk04DHgFbAsIiYJOmUbP9g4FJguKSJpKat/hExX1JnYGTqM2dN4I6IeDR768uBv0o6iZR4jmzw4F98MdUyCi1cCC+80OAfZWZWbhRRs9uh+amoqIjKysriB5qZ2RckjalxOwTgKUfMzKyenDjMzKxenDjMzKxenDjMzKxenDjMzKxeWsSoKknzgDdX8fRNgfkNGE5DcVz147jqx3HVT1ONC1Yvti9HxHJTb7SIxLE6JFXWNhwtb46rfhxX/Tiu+mmqcUFpYnNTlZmZ1YsTh5mZ1YsTR3FD8g6gDo6rfhxX/Tiu+mmqcUEJYnMfh5mZ1YtrHGZmVi9OHGZmVi9OHAUkDZM0V9KrBWUbS3pC0mvZ80ZNJK6LJb0taVz2ODiHuLaS9JSkKZImSTojK8/1mq0grlyvmaR1JL0saXwW12+y8ryvV11x5f4zlsXRStJYSQ9m27n/TtYRV+7XS9IMSROzz6/Myhr8ejlxLGs40KNG2QDgyYjYDngy225sw1k+LoCrIqJr9ni4kWMCWAScExE7AnuSlvjdifyvWV1xQb7X7DPguxGxK2mp5B6S9iT/61VXXJD/zxjAGcCUgu28r9cSNeOCpnG99s0+f8m9Gw1+vZw4CkTEM8B7NYoPA27OXt8MHN6YMUGdceUuIqoi4pXs9QekX6ItyfmarSCuXGVLJH+Yba6VPYL8r1ddceVOUkfg+8ANBcW5/07WEVdT1eDXy4mjuM0jogrSHyRgs5zjKXSapAlZU1Yu1fUlJHUCdgNeogldsxpxQc7XLGveGAfMBZ6IiCZxveqIC/L/Gbsa+CVQuJZz7terjrgg/+sVwOOSxkjqm5U1+PVy4ihf1wPbkpoWqoAr8wpEUhvgXuDMiHg/rzhqqiWu3K9ZRFRHRFegI7CHpJ0bO4ba1BFXrtdL0g+AuRExpjE/t5gVxJX7zxfwzYjoBhxEaqLdpxQf4sRR3BxJWwBkz3NzjgeAiJiT/bIvBoYCe+QRh6S1SH+cb4+I+7Li3K9ZbXE1lWuWxfI/4GlS31Xu16u2uJrA9fomcKikGcCdwHcl3Ub+16vWuJrA9SIiZmfPc4GRWQwNfr2cOIq7H+idve4NjMoxli8s+UHIHAG8WtexJYxBwI3AlIj4U8GuXK9ZXXHlfc0ktZe0YfZ6XeB7wL/J/3rVGlfe1ysizo+IjhHRCegJ/CMijiXn61VXXHlfL0nrS2q75DVwQBZDw1+viPAjewAjSFXMz4FZwEnAJqSRCK9lzxs3kbhuBSYCE7IfjC1yiGtvUpvqBGBc9jg472u2grhyvWbALsDY7PNfBS7KyvO+XnXFlfvPWEGM3wEebArXawVx5f3z1RkYnz0mAf9XquvlKUfMzKxe3FRlZmb14sRhZmb14sRhZmb14sRhZmb14sRhZmb14sRhLUZ2v8Jzkl6VdHhB+ShJHeo452JJ5xZ53+GSflyPODqpYKbjlTyn6GeszPtmxxxdn882q8mJw1qSXqRJ3vYCzgOQdAjwSmR33LYAnQAnDlstThzWknwOrAusDSyWtCZwJvCHlTlZ0smSRmfrVtwrab2C3d+T9Kyk/2RzGS2ZOPAP2TkTJP2slves9RglAyVNlvQQdUxMJ2n3LJ4XgX4F5Z2yeF7JHt/Idl0OfEtpvYazVnCcWZ2cOKwluQM4EHgUuBg4FbglIj5eyfPvi4jukdatmEK6g3+JTsC3SVNtD5a0TrZ/QUR0B7oDJ0vapsZ71nXMEUAX4GvAyUBdf9BvAk6PiL1qlM8F9o804d1RwLVZ+QDg2UjrNVy1guPM6rRm3gGYNZaIWED6w0425XV/4IeShgIbAVdGxIsreIudJf0W2BBoAzxWsO+vkSa3e03SdGAH0lxBuxT0TbQDtgP+U3BeXcfsA4yIiGpgtqR/1AxGUjtgw4j4Z1Z0K2lWVEhragyU1BWoBrav4zut7HFmX3DisJbqIuAyUr/HGFJtZBSw7wrOGQ4cHhHjJZ1AmqdoiZpz9wQg4BcRUZhglqwR8sVmHcccXMt71qQVHHMWMAfYldSy8OlqHmf2BTdVWYsjaTugQ/af+nqkxXgCWKfIqW2BqmzK9mNq7DtS0hqStiVNNjeVVCP5eXY8krbPZi0tVNcxzwA9sz6QLagloUWaAn2BpL2zosKY2gFVWS3oOKBVVv5B9j2KHWdWJ9c4rCW6DPi/7PUI4G+k9aMvKnLehaSVBN8kzYJa+Ad4KvBPYHPglIj4VNINpL6PV7Kp3uex/LKddR0zEvhu9jn/yd67Nn2AYZI+Ztmmsz8D90o6EngK+CgrnwAskjSeVIOq6zizOnl2XDMzqxc3VZmZWb04cZiZWb04cZiZWb04cZiZWb04cZiZWb04cZiZWb04cZiZWb38fxZjMqeARw13AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEHCAYAAAC0pdErAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmaklEQVR4nO3deZgU1bnH8e+bEeKOQYgkLAJe1BBUNA3q1WjcInpdI+Yixt0gQcQFIxqXKLjGJW54EZVFRblGBAlRxKuJmqgJAwKCyCISGRiFaFA2HWbmvX+cQpumZ2Omprp7fp/n6We6TlV1vVNKv3NOncXcHRERkUzfSjoAERHJTUoQIiKSlRKEiIhkpQQhIiJZKUGIiEhWShAiIpLVNnF+uJn1Au4DioBH3f32jP0/AZ4HPoyKnnP3oWbWHngcaANUAiPd/b6arteqVSvv2LFjg8UvIlLoZsyY8S93b51tX2wJwsyKgOHAMUAJMN3MJrv7exmHvuHuJ2SUlQOD3X2mme0EzDCzl7Ocu5mOHTtSXFzcUL+CiEjBM7N/VrUvziamnsBid1/i7mXAeODk2pzo7qXuPjN6vwaYD7SNLVIREdlCnAmiLbAsbbuE7F/yB5vZbDN70cx+mLnTzDoC+wN/jyVKERHJKs5nEJalLHNej5nA7u6+1syOByYBXb7+ALMdgQnAZe7+RdaLmPUD+gF06NChAcIWERGItwZRArRP224HrEg/wN2/cPe10fsXgGZm1grAzJoRksM4d3+uqou4+0h3T7l7qnXrrM9ZRERkK8SZIKYDXcysk5k1B/oAk9MPMLM2ZmbR+55RPJ9GZY8B8939nhhjFBGRKsSWINy9HBgIvER4yPyMu88zs/5m1j86rDcw18xmA/cDfTxML3sIcBZwpJnNil7HxxWriEjeKi2Fww+Hjz9u8I+2QpruO5VKubq5ikiTMmAAPPww9O8Pw4fX+XQzm+HuqWz7NJJaRCRflZbCY49BZSWMHt3gtQglCBGRfLRiBRx2GJSVhe2KChg2rEEvoQQhIpJPNm6Eu++GLl1g8eJvysvKGrwWoQQhIpIv/vxn6N4drrwSWrWCZs0239/AtQglCBGRXLd8OfTpA0ceCevXw/PPQ8uWoTaRrqwM3nyzwS4b62yuIiJSD2VlcO+9MHQolJfDb38LQ4bAdtvBSSfFfnklCBGRXPTKKzBwILz/Ppx4YkgUnTs3aghqYhIRySXLlsHPfw5HHx1qEFOmwOTJjZ4cQAlCRCQ3lJXB7bfD3nvDH/8YmpXmzYP/+q/EQlITk4hI0qZNg0sugYUL4ZRT4Pe/hxxYHVM1CBGRpHz0EZx2Ghx7bBgN/eKLMHFiTiQHUIIQEWl8X30Ft94ampNefBFuvhnmzoVevZKObDNqYhIRaUxTp8KgQbBoEfzsZ3DPPbD77klHlZVqECIijWHpUjj1VDjuODCDl16CCRNyNjmAEoSISLy+/DJMf/GDH4SH0bfdBnPmwE9/mnRkNVITk4hIXP70J7j0UvjgAzj99DDJXvv2NZ+XI1SDEBFpaEuWhKkwTjgBttkGXn4Znnkmr5IDKEGIiDScDRvgxhuha1d49VW4447QnHT00UlHtlViTRBm1svMFpjZYjO7Osv+n5jZ52nrTt+Qtm+Uma00s7lxxigi0iD++Ef44Q/hppvCYLf334erroLmzZOObKvFliDMrAgYDhwHdAXOMLOuWQ59w927R6+haeVjgNzqFCwikumDD0JT0kknhVlWX30Vxo+Hdu2Sjqze4qxB9AQWu/sSdy8DxgMn1/Zkd38d+Cyu4ERE6mX9erjhhlBreO01uOsumDULjjgi6cgaTJwJoi2wLG27JCrLdLCZzTazF83shzHGIyJSf+4waVJ4zjBsWJgqY8ECGDx4yxXe8lyc3VwtS5lnbM8Ednf3tWZ2PDAJ6FKni5j1A/oBdOjQYSvCFBGppUWLwijoqVOhWzf4y1/g8MOTjio2cdYgSoD0Pl3tgBXpB7j7F+6+Nnr/AtDMzFrV5SLuPtLdU+6eat26dX1jFhHZ0rp1cO21ISn87W9httWZMws6OUC8NYjpQBcz6wQsB/oAfdMPMLM2wCfu7mbWk5CwPo0xJhGR2nMPs6tefnmYefUXv4Df/Q6+972kI2sUsdUg3L0cGAi8BMwHnnH3eWbW38z6R4f1Buaa2WzgfqCPuzuAmT0NvAXsZWYlZnZBXLGKiGxh4cIwu+ppp0GLFvD66/DEE00mOQBY9H1cEFKplBcXFycdhojks3XrwvTbd98duq0OGwYDBoQR0QXIzGa4eyrbvsL8jUVE6sodnn0WrrgCSkrgnHPCSOjddks6ssRoqg0RkfffD7Or/vzn0KoV/PWvMGZMk04OoAQhIk3Z2rUwZAjsuy8UF8ODD4afhxySdGQ5QU1MItL0uIfZVQcPhuXL4fzzwzoN3/1u0pHlFNUgRKRpmTcPjjoK+vQJTUhvvQWPPabkkIUShIg0DWvWwJVXQvfuYc6khx6Cf/wDDjoo6chylpqYRKSwucPTT4fk8PHHcMEFoTmpVZ0mbWiSlCBEpHDNnQsDB4bZVlOpMMlez55JR5U31MQkIoXn88/D9Bjdu8O778LDD8Pbbys51JFqECJSONxh3LjQnLRyJfTrB7fcArvumnRkeUkJQkQKw5w5oTnpjTdCTWHKlNCsJFtNTUwikt9Wr4ZLL4UDDoD33oNHHgldV5Uc6k01CBHJT5WVYXbVq66CVaugf/8wyV7LlklHVjCUIEQk/8yaBRdfDG++GcYxvPhiqEFIg1ITk4jkj9Wr4ZJL4Ec/Cst/jhoVVnhTcoiFahAikvsqK2Hs2DCx3qefhvUZhg6F73wn6cgKmmoQIpLbZs4Ms6uefz7suSfMmAEPPKDk0AiUIEQkt5SWwuGHw/z5oaaQSsGSJaEG8cYbYfCbNAo1MYlIbhk6NCSC/feHjRvDM4ebboJddkk6siYn1gRhZr2A+4Ai4FF3vz1j/0+A54EPo6Ln3H1obc4VkRz35Zfw73+H12efffM+85W+71//Cl1WISSHl1+GI49M9vdowmJLEGZWBAwHjgFKgOlmNtnd38s49A13P2ErzxWROH31VfVf6NXt+/LL6j+7RYvwHGHTq2vXsFbDZ59BRQVssw1MmKAEkaA4axA9gcXuvgTAzMYDJwO1+ZKvz7kikq6srHZf6NnKN2yo/rN32umbL/iWLWHvvTf/0t9Unlm2yy5QVLT5Z5WWQufOITlsinv0aLj+emjTJpZbI9WLM0G0BZalbZcAB2Y57mAzmw2sAK5093l1OFck95WWhtXL/vd/t/6LbuPGMAagtn+9p7/Wrav+s3fccfMv7//4j+xf6plf9rvsEv7KbyjDhoXurOkqKkL58OENdx2ptTgThGUp84ztmcDu7r7WzI4HJgFdanluuIhZP6AfQIcOHbY6WJHYDBsGf/1reNA6bFjdm2r+/W9Yu7b6a2y//eZf3p071/xX/KZXs2aNcx9q8tZbodaQrqwsjJaWRMSZIEqA9mnb7Qi1hK+5+xdp718ws4fMrFVtzk07byQwEiCVSmVNIiKJef75sBZBZSWMGBFeVdluu82/uDt2DD15atNk07x5o/1KsXnnnaQjkAxxJojpQBcz6wQsB/oAfdMPMLM2wCfu7mbWkzAu41NgdU3niuS02bPh2mvhT3/6pqyoKMwbdNFF2b/sv/3t5OIVySK2BOHu5WY2EHiJ0FV1lLvPM7P+0f4RQG/gV2ZWDmwA+ri7A1nPjStWkQazaBHccAOMHw877xza6MvLw76KijAq+Jhj9NBV8oKF7+PCkEqlvLi4OOkwpCkqKQkDvEaNCjWByy6Djz+GJ5/cvF29eXO48EI9dJWcYWYz3D3r4hkaSS1SH6tWwe23hy/8ysowNcS118Juu4XnB3roKnlMCUJka3zxBdxzD9x9N6xfD2efDb/9bXiwvIkeukqeU4IQqYsNG+Chh+C228K006edFpqWunZNOjKRBqfZXEVqY+NGGDkSunSBK68MC9ZMnw7PPqvkIAVLCUKkOpWV8PTTIQlcdBF06AB//jO89FKYhlqkgClBiGTjDlOmhAfNffuGQWyTJ4flLX/yk6SjE2kUShAimV57DQ49FE48McxjNG4czJoVti3bLDAihUkJQmSTGTOgV69QQ1i6NEyLMX9+qEF8S/9UpOnR//Ui8+dD797hmcL06XDnnbB4cXjmkCsT2YkkQN1cpen65z/DDKtjx4bZUG+4AQYPDlNkiIgShDRBn3wCt94ampDM4NJL4ZproHXrpCMTySlKENJ0rF4Nd90F994blsM877xQa2jfvqYzRZokJQgpfOvXwwMPwB13hMV3/vu/w+jnPfdMOjKRnKaH1FK4ysrCtBh77AFXXw0HHxzmRxo/XslBpBZUg5DCU1EBTz0VJs/78EP48Y/hD38IYxtEpNZUg5DC4Q6TJsF++4XZVXfZBV588ZuBbyJSJ0oQUhheeSUs53nqqWFivWeegeLiMPBNo59FtooShOS3v/8djjoKjj4aSkvhscdg3jw4/XSNfhapp1j/BZlZLzNbYGaLzezqao7rYWYVZtY7rexSM5trZvPM7LI445Q8NHcunHJKqDW8+27ourpwIZx/flgHWkTqLbYEYWZFwHDgOKArcIaZbTFxfnTcHcBLaWXdgF8CPYH9gBPMrEtcsUoeWbIEzjoL9t03TLs9bBh88EEY7LbttklHJ1JQ4qxB9AQWu/sSdy8DxgMnZznuEmACsDKt7AfA2+6+3t3LgdeAU2OMVXJdaWlY73mvvcIiPVdeGZLFddfBTjslHZ1IQYozQbQFlqVtl0RlXzOztoQv/hEZ584FDjOzXc1se+B4QMNdm6LPPoMhQ8JYhkcegQsvDDWG3/0Odt016ehEClqcjbXZuo54xva9wBB3r7C0nibuPt/M7gBeBtYCs4HyrBcx6wf0A+jQoUP9o5bcsHZteK5w552wZg2ceSbceGNIFCLSKOKsQZSw+V/97YAVGcekgPFmthToDTxkZqcAuPtj7n6Aux8GfAYsynYRdx/p7il3T7XWZGv576uv4L77oHNnuP56OOIImD0bnnhCyUGkkcVZg5gOdDGzTsByoA/QN/0Ad++06b2ZjQGmuPukaPu77r7SzDoAPwMOjjFWSVp5OTz+eJh++6OPQmK49dbQS0lEEhFbgnD3cjMbSOidVASMcvd5ZtY/2p/53CHTBDPbFdgIXOzu/44rVklQZSU891yoLbz/PvToEcYyHHWUBriJJCzWDuPu/gLwQkZZ1sTg7udmbP84vsgkce4wbRr85jcwcyZ07RoSxSmnKDGI5AgNNZXG97e/hXWfe/UKvZTGjoU5c8I0GUoOIjlDCUIaz+zZcMIJYeK8BQvgwQfDz7PPhqKipKMTkQzVJohoCozjspSfZGY/ii8sKSiLFsEZZ0D37qH2cNttYSzDxRdD8+ZJRyciVaipBnEnMD9L+XvRPpGqlZRAv37wgx/A5MnhecOSJWHxnh12SDo6EalBTQ+pd3X3pZmF7r446mEksqVVq+D222H48NBLacCAkBzatEk6MhGpg5oSxHbV7NOfgLK5L76Ae+6Bu+8O60CffXZY1a1jx6QjE5GtUFMT0/+Z2S1mm3ctMbObgFfjC0vyyoYNISl07hwGuh17bJiCe/RoJQeRPFZTghgMdAYWm9mE6LUY2Au4IvboJDeVlsLhh8OyZTByJHTpEmZX/dGPYPr0MNtq1y1mdheRPFNtE5O7ryOs49AZ+GFUPM/dl8QemeSuoUPhjTegW7fQrHTQQfDkk2Fsg4gUjGoThJkdkLa5nDAb68ZYI5Lc9tFHodbgHmZZHTs2LOCjAW4iBaemh9R3ZylraWbNgTPcfVbDhyQ57dRTQ88kgGbNwprQZ5+dbEwiEouampiOyFZuZingfuCwOIKSHDVxYpg3aZOysvAg+vrr1YVVpABt1VQb7l4M7NjAsUguW706e02hoiKsCy0iBWerEoSZ7caWq8NJIbv44rDKW6ayMnjzzcaPR0RiV9ND6gfYMhG0BP4TuDSuoCTHjBsHTz0VagrXXZd0NCLSSGp6SF2cse3Ap8AV7r4ynpAkp3z4YZgq49BD4Zprko5GRBpRTQ+px2YrN7P2ZvZrd9eEfYWsvDx0YYWwJrSm5BZpUmq9opyZtQJOB84A2gIT4wpKcsRtt4XpuceN05QZIk1QTetB7GRmZ5vZVOAfwH8And19D3e/sqYPN7NeZrbAzBab2dXVHNfDzCrMrHda2eVmNs/M5prZ02a2bR1+L6mvt98O8yqdeSb07Zt0NCKSgJp6Ma0ELgBuAfZw98FAWW0+2MyKgOHAcUBXwpQdW0zQEx13B/BSWllbYBCQcvduQBHQpzbXlQawZk1IDO3ahSm7RaRJqilB/AbYFvgf4Boz26MOn90TWOzuS9y9DBgPnJzluEuACYRklG4bYDsz2wbYHlhRh2tLfQwaBEuXhvmVWrRIOhoRSUi1CcLdf+/uBwInAQZMAr5vZkPMbM8aPrstsCxtuyQq+1pUUzgVGJFx3eXAXcBHQCnwubtPq/G3kfr7wx9gzBi49trQc0lEmqxaDZSLagG3uPs+QA+gBfBiDadlm70tc0zFvcAQd6/Y7ESz7xBqG52A7wM7mNkvsl7ErJ+ZFZtZ8apVq2r+ZaRqy5aFJUIPPDBMnyEiTVqtezGlOc3df0NofqpOCdA+bbsdWzYTpYDx0XpErYDjzawcaAZ86O6rAMzsOcLgvCczL+LuI4GRAKlUSqO7t1ZFRZhKo7w89Fpq1izpiEQkYVsz1cZJtTxuOtDFzDpFs7/2ASanH+Dundy9o7t3BJ4FBrj7JELT0kFmtn20mt1RwPytiFVq6+674S9/gfvvhz3q8qhJRArV1tQgajXxv7uXm9lAQu+kImCUu88zs/7R/hHVnPt3M3sWmAmUA+8Q1RIkBjNmhCk0eveGc89NOhoRyRHmXrdWGTP7FiFJ9HH3cbFEtZVSqZQXF2fODiLVWrcuLBW6di3MmQMtWyYdkYg0IjOb4e6pbPtqGii3s5ldY2YPmtlPo+aeAcAS4OcxxCqNbfBgWLgwTKWh5CAiaWpqYnoC+DfwFnAh8GugOXCyVpMrAM8/Dw8/DFddBUdkXRtKRJqwmhJE56hrK2b2KPAvoIO7r4k9MolXaSlccAEccIAW/BGRrGrqxbRx05torMKHSg4FoLIyPIxevz50aW3ePOmIRCQH1VSD2M/MvuCbnkvbpW27u+8ca3QSj/vvh2nTYMQI2HvvpKMRkRxV03oQWgCg0MyZA0OGwEknhVHTIiJVqGnJ0W2B/oRpvucQxjKUN0ZgEoMNG8IsrS1bwqOPgtVqSIuINFE1NTGNJTyHeAM4HvghWos6f119NcydC1OnQuvWSUcjIjmupgTRNa0X02OERYMkH734Ynj2cOmlcOyxSUcjInmgLr2Y1LSUr1auhPPOg27d4Pbbk45GRPJEbXsxQei5pF5M+cY9jHdYvRpefhm21cqtIlI76sVU6EaMgClT4L77YJ99ko5GRPLI1kz3Lfli/ny44gro1QsuuSTpaEQkzyhBFKqvvoK+fWHHHWH0aHVpFZE625r1ICQfXHcdzJoFkydDmzZJRyMieUg1iEL0yitw113wq1/BiScmHY2I5CkliELz6adwzjlhjqW77ko6GhHJY2piKiTucNFFYdzDH/8I22+fdEQikseUIArJ6NEwYQL87new//5JRyMieS7WJiYz62VmC8xssZldXc1xPcyswsx6R9t7mdmstNcXZnZZnLHmvUWLYNAgOPLIsIyoiEg9xVaDMLMiYDhwDFACTDezye7+Xpbj7gBe2lTm7guA7mn7lwMT44o1723cGGZpbd4cxo6Fb+nRkojUX5zfJD2Bxe6+xN3LgPHAyVmOuwSYAKys4nOOAj5w93/GE2YBuOkmmD4dHnkE2rVLOhoRKRBxJoi2wLK07ZKo7Gtm1hY4FRhRzef0AZ5u8OgKxeuvw623wvnnw2mnJR2NiBSQOBNEtqG7nrF9LzAkWu96yw8waw6cBPyhyouY9TOzYjMrXrVq1dbGmp9Wr4azzoI99ghzLYmINKA4ezGVAO3TttsBKzKOSQHjLUwD0Qo43szK3X1StP84YKa7f1LVRdx9JDASIJVKZSagwuUeBsItXw5vvhmm1BARaUBxJojpQBcz60R4yNwH6Jt+gLt32vTezMYAU9KSA8AZqHkpu3HjYPx4uPlm6Nkz6WhEpADFliDcvdzMBhJ6JxUR1rOeZ2b9o/3VPXfAzLYn9IC6KK4Y89aHH8KAAXDooWEZURGRGMQ6UM7dXwBeyCjLmhjc/dyM7fXArrEFl6/Ky8NzBzN48kko0pIdIhIPjaTON7fdBn/7Gzz1FOy+e9LRiEgB04iqfPL222HMw5lnwhlnJB2NiBQ4JYh8sWZNSAzt2sHw4UlHIyJNgJqY8sWgQbB0Kbz2GrRokXQ0ItIEqAaRD555BsaMgWuvDT2XREQagRJErlu2LKzxcOCBcP31SUcjIk2IEkQuq6gIXVrLy8PAuGbNko5IRJoQPYPIZXfdFZ45jB4d5lsSEWlEqkHkqhkz4Lrr4PTTwxrTIiKNTAkiF61bF7q0tmkDI0aEUdMiIo1MTUy5aPBgWLgQXnkFWrZMOhoRaaJUg8g1zz8PDz8Mv/41HHFE0tGISBOmBJFLSkvhggvggANg2LCkoxGRJk4JIldUVsK558L69aFLa/PmSUckIk2cnkHkivvvh2nTwkPpvfdOOhoREdUgcsKcOTBkCJx0EvTrl3Q0IiKAEkTyNmyAvn1Db6VHH1WXVhHJGWpiStqQITBvHkydCq1bJx2NiMjXYq1BmFkvM1tgZovNrMrFk82sh5lVmFnvtLJdzOxZM3vfzOab2cFxxpqIF16ABx6Ayy6DY49NOhoRkc3EliDMrAgYDhwHdAXOMLOuVRx3B/BSxq77gKnuvjewHzA/rlgTsXIlnHce7LNPWEZURCTHxFmD6Aksdvcl7l4GjAdOznLcJcAEYOWmAjPbGTgMeAzA3cvcfXWMsTYu9zDe4fPPw9rS226bdEQiIluIM0G0BZalbZdEZV8zs7bAqcCIjHM7A6uA0Wb2jpk9amY7xBhr4xoxAqZMgTvvhG7dko5GRCSrOBNEtu44nrF9LzDE3SsyyrcBDgD+x933B9YBWZ9hmFk/Mys2s+JVq1bVM+RGMH8+XHEF9OoFAwcmHY2ISJXi7MVUArRP224HrMg4JgWMt9C1sxVwvJmVA28DJe7+9+i4Z6kiQbj7SGAkQCqVykxAueWrr0KX1h13DGs8qEuriOSwOBPEdKCLmXUClgN9gL7pB7h7p03vzWwMMMXdJ0Xby8xsL3dfABwFvBdjrI3juutg1iyYPDlM5S0iksNiSxDuXm5mAwm9k4qAUe4+z8z6R/sznztkugQYZ2bNgSXAeXHF2ij+7//CCnG/+hWceGLS0YiI1Mjcc7tVpi5SqZQXFxcnHcaWPv0U9t0Xdt45rBS3/fZJRyQiAoCZzXD3VLZ9GkkdN3f45S9h1arQc0nJQUTyhBJE3EaNgokTQ5fW/fdPOhoRkVrTZH1xWrQIBg2CI48MXVtFRPKIEkRcNm6EM8+Eb38bHn8cvqVbLSL5RU1McbnpJpg+HZ59Ftq2rfl4EZEcoz9r4/D663DrrXD++XDaaUlHIyKyVZQgGtrq1XDWWbDHHnDffUlHIyKy1dTE1JDcw0C45cvhzTfDlBoiInlKCaIhjRsH48fDzTdDz55JRyMiUi9qYmooH34IAwbAoYfC1VUunicikjeUIBpCeTn84hdhdtYnn4SioqQjEhGpNzUxNYRbbw3PHJ56CnbfPeloREQahGoQ9fXWWzB0aKhBnHFG0tGIiDQYJYj6WLMmJIb27eHBB5OORkSkQamJqT4GDYKlS8PAuBYtko5GRKRBqQaxtZ55BsaMCavEHXJI0tGIiDQ4JYitsWwZXHQRHHggXH990tGIiMRCCaKuKirCVBrl5WFg3DZqpRORwhRrgjCzXma2wMwWm1mVo8fMrIeZVZhZ77SypWb2rpnNMrPcWUf0rrvgtdfggQfCfEsiIgUqtj9/zawIGA4cA5QA081ssru/l+W4O4CXsnzMEe7+r7hirLMZM8Izh9NPh3POSToaEZFYxVmD6Aksdvcl7l4GjAdOznLcJcAEYGWMsdTfunXQty+0aQMjRoRR0yIiBSzOBNEWWJa2XRKVfc3M2gKnAiOynO/ANDObYWb9Youytq64Iiwh+vjj0LJl0tGIiMQuzies2f7E9ozte4Eh7l5hW/5Ffoi7rzCz7wIvm9n77v76FhcJyaMfQIcOHeofdTaTJsHIkTBkCBxxRDzXEBHJMXHWIEqA9mnb7YAVGcekgPFmthToDTxkZqcAuPuK6OdKYCKhyWoL7j7S3VPunmrdunWD/gIArFgBF14IBxwQptQQEWki4kwQ04EuZtbJzJoDfYDJ6Qe4eyd37+juHYFngQHuPsnMdjCznQDMbAfgp8DcGGPNrrISzjsP1q8PE/E1b97oIYiIJCW2JiZ3LzezgYTeSUXAKHefZ2b9o/3ZnjtsshswMWp22gZ4yt2nxhVrle6/H6ZNg4cfhr32avTLi4gkydwzHwvkr1Qq5cXFDTRkYs4c6NEDjjsOJk5UryURKUhmNsPdU9n2aSR1Nhs2hC6tLVvCI48oOYhIk6R5IrIZMgTmzYOpUyGOB98iInlANYhML7wQptG47DI49tikoxERSYwSRLqVK0OvpX32gdtuSzoaEZFEqYlpE3c4/3z4/HN45RXYdtukIxIRSZQSBEBpKRx2GCxeHLq2duuWdEQiIolTExPA5ZeH5NC+PQwcmHQ0IiI5QQli6dKwfCjAqlXwySeJhiMikiuUIG6++ZtxDpWVMGxYsvGIiOSIpp0gSkvDsqGVlWG7rAxGj4aPP042LhGRHNC0E8SwYd8kh00qKlSLEBGhqSeIt94KtYZ0ZWXw5pvJxCMikkOadjfXd95JOgIRkZzVtGsQIiJSJSUIERHJSglCRESyUoIQEZGslCBERCSrglpy1MxWAf/cytNbAf9qwHAaiuKqG8VVN4qrbgoxrt3dPevKaAWVIOrDzIqrWpc1SYqrbhRX3SiuumlqcamJSUREslKCEBGRrJQgvjEy6QCqoLjqRnHVjeKqmyYVl55BiIhIVqpBiIhIVk0yQZjZKDNbaWZz08pamtnLZrYo+vmdHInrRjNbbmazotfxjRxTezP7s5nNN7N5ZnZpVJ7o/aomrqTv17Zm9g8zmx3FdVNUnvT9qiquRO9XWnxFZvaOmU2JthP/91hFXLlyv5aa2btRDMVRWYPfsyaZIIAxQK+MsquBV9y9C/BKtN3YxrBlXAC/d/fu0euFRo6pHBjs7j8ADgIuNrOuJH+/qooLkr1fXwFHuvt+QHegl5kdRPL3q6q4INn7tcmlwPy07aTv1yaZcUFu3C+AI6IYNnVvbfB71iQThLu/DnyWUXwyMDZ6PxY4pTFjgirjSpS7l7r7zOj9GsI/lrYkfL+qiStRHqyNNptFLyf5+1VVXIkzs3bAfwGPphUn/u+xirhyWYPfsyaZIKqwm7uXQvjyAb6bcDzpBprZnKgJKpGqNoCZdQT2B/5ODt2vjLgg4fsVNUvMAlYCL7t7TtyvKuKC5P//uhe4Ckhf3jHx+1VFXJD8/YKQ3KeZ2Qwz6xeVNfg9U4LIff8D7EFoFigF7k4iCDPbEZgAXObuXyQRQzZZ4kr8frl7hbt3B9oBPc2sW2PHkE0VcSV6v8zsBGClu89ozOvWpJq4Ev//K3KIux8AHEdoXj0sjosoQXzjEzP7HkD0c2XC8QDg7p9E/7ArgUeAno0dg5k1I3wJj3P356LixO9Xtrhy4X5t4u6rgb8Qnislfr+yxZUD9+sQ4CQzWwqMB440sydJ/n5ljSsH7hcA7r4i+rkSmBjF0eD3TAniG5OBc6L35wDPJxjL1zb9B4+cCsyt6tiYrm/AY8B8d78nbVei96uquHLgfrU2s12i99sBRwPvk/z9yhpX0vfL3a9x93bu3hHoA7zq7r8g4ftVVVxJ3y8AM9vBzHba9B74aRRHw98zd29yL+BpQvVwI1ACXADsSnjyvyj62TJH4noCeBeYE/0P8L1GjulQQnvnHGBW9Do+6ftVTVxJ3699gXei688FbojKk75fVcWV6P3KiPEnwJRcuF/VxJX4/QI6A7Oj1zzg2rjumUZSi4hIVmpiEhGRrJQgREQkKyUIERHJSglCRESyUoIQEZGslCCk4ER9/v9qZnPN7JS08ufN7PtVnHOjmV1Zw+eOMbPedYijo6XNzFvLc2q8Rm0+Nzqmb12uLZJJCUIK0RmEycoOBn4NYGYnAjM9GoHaBHQElCCkXpQgpBBtBLYDvg1Umtk2wGXAnbU52cx+aWbTo7UTJpjZ9mm7jzazN8xsYTRfz6ZJ8O6MzpljZhdl+cysx1jwoJm9Z2Z/oooJ1szsR1E8bwEXp5V3jOKZGb3+M9p1O/BjC+sFXF7NcSJVUoKQQvQUcCwwFbgRGAA87u7ra3n+c+7ew8PaCfMJI9o36QgcTpgGeoSZbRvt/9zdewA9gF+aWaeMz6zqmFOBvYB9gF8CVX1xjwYGufvBGeUrgWM8TNz238D9UfnVwBse1gv4fTXHiVRpm6QDEGlo7v454QucaDrmIcDPzOwR4DvA3e7+VjUf0c3MbgZ2AXYEXkrb94yHidoWmdkSYG/CXDj7pj07aAF0ARamnVfVMYcBT7t7BbDCzF7NDMbMWgC7uPtrUdEThFk8Iazr8KCZdQcqgD2r+J1qe5zI15QgpNDdANxCeC4xg1C7eB44oppzxgCnuPtsMzuXMBfPJplz0zhgwCXunp5INq1T8fVmFcccn+UzM1k1x1wOfALsR2gR+LKex4l8TU1MUrDMrAvw/egv7+0JC784sG0Np+4ElEbTiZ+Zse90M/uWme1BmDRtAaGG8avoeMxsz2iWzXRVHfM60Cd6RvE9siQuD9Nzf25mh0ZF6TG1AEqjWs1ZQFFUvib6PWo6TqRKqkFIIbsFuDZ6/zQwibDG8A01nHc9YXW6fxJm7kz/ol0AvAbsBvR39y/N7FHCs4mZ0TTkq9hyuceqjpkIHBldZ2H02dmcB4wys/Vs3uT1EDDBzE4H/gysi8rnAOVmNptQI6rqOJEqaTZXERHJSk1MIiKSlRKEiIhkpQQhIiJZKUGIiEhWShAiIpKVEoSIiGSlBCEiIlkpQYiISFb/D1Ei5ixclfkIAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlDklEQVR4nO3deZgU1bnH8e8rguKKCi4BzRDlanDBZXAlMcYliiKoRHFJRA2oEdTEDZMLmKDGNW7BIEFARVFBZFFcUIOiojIsImBw54qMgEZEBBxn5r1/nB5pmp5hGrqmevl9nqef6a6u7v5RQL9zzqk6x9wdERGRVJvEHUBERHKTCoSIiKSlAiEiImmpQIiISFoqECIiktamcQfIpubNm3tJSUncMURE8sb06dO/cPcW6Z4rqAJRUlJCWVlZ3DFERPKGmS2o7Tl1MYmISFoqECIikpYKhIiIpKUCISIiaalAiIhIWioQIiL5rLwcjjwSPv8862+tAiEiks8GDIBXXw0/s0wFQkQkX5WXw/33Q3U1DBuW9VaECoSISD5yh9NOg4qK8LiqKuutCBUIEZF889//wq9+BVOnrtlWUZH1VoQKhIhIPnnjDTjgAHjhBWjUaO3nstyKUIEQEckH7nDHHfCzn8Emm0CbNqEgJKuogNdfz9pHFtRkfSIiBemrr+C882DcOOjSBYYOhe22i/xj1YIQEcll06bBgQfC00+HFsSYMQ1SHEAFQkQkN7nD3XfDEUeE01inTIHLLwezBougAiEikmuWLYOuXeGyy+D442HmTDj00AaPoQIhIpJLpk+Hgw4K4w233hp+br99LFFUIEREcoE7DBwIhx8ezkZ65RW48soG7VJKpQIhIhK3r7+GM86AXr3gmGNg1qxQKGKmAiEiEqeZM0OX0pgxcNNNMGEC7LBD3KkAFQgRkXi4w6BBcNhhsHo1TJ4M11wTLoLLEbmTRESkWHzzDZx1Flx8MfziF6EV0aFD3KnWoQIhItKQ3n47dCk9/jjccANMnAgtWsSdKi1NtSEi0hDcYcgQuPTScCX0Sy+FleBymFoQIiJRW7ECfvMb6NkzTLY3a1bOFwdQgRARidY770BpKYwcCX/9KzzzDOy4Y9yp6kVdTCIiUXAPC/j06gXbbBPWbzjqqLhTZUQtCBGRbPv2W+jeHS64IJzGOmtW3hUHUIEQEcmuuXOhfXt46CHo3x+efx523jnuVBtEXUwiItnywAPw+9/DVluFwnDMMXEn2iiRtiDM7Hgzm29mH5hZnzTP/8LMvjazWYlbv/q+VkQkZ6xcCeefH7qVDj44dCnleXGACFsQZtYIGAgcCywEppnZeHefl7LrFHc/aQNfKyISr3ffhV//GubNg//939CttGlhdM5E2YI4GPjA3T9y9wrgUaBzA7xWRKRhjBgRxhsWL4Znn4UBAwqmOEC0BaIl8GnS44WJbakOM7O3zewZM9s7w9diZj3NrMzMypYuXZqN3CIidVu1Cnr0CBe/HXhg6FI67ri4U2VdlAUi3SoXnvJ4BvBjd28H3AOMzeC1YaP7YHcvdffSFjk6n4mIFJD58+GQQ8K0GddeG6bMaJn299e8F2WBWAjsmvS4FbAoeQd3X+7uKxL3JwKNzax5fV4rItLgRo4MV0UvWhQm2bvxxoLqUkoVZYGYBrQxs9Zm1gToBoxP3sHMdjYL6+mZ2cGJPF/W57UiIg1m9Wq46KIwRXe7dqFL6YQT4k4VuchKn7tXmlkv4DmgETDU3eea2UWJ5wcBXYGLzawSWAV0c3cH0r42qqwiIrV6//1wltLbb8PVV8P110PjxnGnahAWvo8LQ2lpqZeVlcUdQ0QKxeOPw+9+FwrCAw/ASSet/zV5xsymu3tpuuc01YaISKrVq+GSS+CMM2CffcKKbwVYHNZHBUJEJNmHH8IRR8C998IVV8DLL8Nuu8WdKhaFO/wuIpKpJ54IU2ZssgmMGwcnnxx3olipBSEi8t13YSnQrl1hr71Cl1KRFwdQgRCRYvfxx9ChA9xzD1x2GUyZAiUlcafKCepiEpHi9eSTcN554f6YMXDKKfHmyTFqQYhI8amogD/8AU49FfbYA2bMUHFIQy0IESkun3wSTl99662wXvRtt8Fmm8WdKiepQIhI8Rg/Hs49F6qrYdSoMCgttVIXk4gUvu+/hyuvhM6doXVrmD5dxaEeVCBEolZeDkceCZ9/HneS4vTpp+H43347XHwxvP56GHeQ9VKBEInagAHw6qvhpzSsiRNh//1hzhx49NFwdfTmm8edKm9oDEIkSnPnwn33hT7v++6DbbcN0zjstx+0agWWbm0s2Wjffw99+8LNN4fpuUeNgjZt4k6Vd1QgRKIydy4cfngoDgBVVfC3v615vlmzUCiSb/vsA1tuGUvcgrFwIXTrBq+9Bj17wp13QtOmcafKSyoQIlGYMCF8Sa1cufb2zTeHxx4LX2KzZ4fb8OGwYkV43iz0j6cWjpKSMD+Q1O3ZZ8M60atWwcMPhwV+ZIOpQIhkkzvcdBP8+c+www5QWRkuyqpRXQ3PPQcDB6697ZNP1hSMmtuYMeH9ALbaCvbdNxSLdu3Cz333hW22adA/Xs6qrIT+/cMSoPvuG7qU9twz7lR5TwsGiWTLypVwwQVhMLRbN5g3L3zRp9p//zAZ3PqsWBG6qVILx7Jla/YpKVm3tbHHHtCoUZb+UHlg0SI480x45ZVw/O++G7bYIu5UeaOuBYPUghDJhk8/hS5dwhf/3/4G11yz8QPQW20FhxwSbjXc1+6emj07LIX51FNrxjqaNoW9917T0qhpbeyww8blyUWTJsHZZ8O338KDD4buJckatSBENtbrr4c5fVauDP3enTo1fIbVq9e0WJILxxdfrNmnZct1Wxt77pmf6ytXVcFf/hLWh27bNnQp/fSncafKS2pBiERl2DC46CLYdVd46aXwZRWHzTeHAw8MtxrusHjxukXjhRfCaaAATZqEzKmFY6ed4vlz1Ed5eRh8njwZuneHf/xDZ35FRAVCZENUVsJVV4VTKI8+Oixuv/32cadamxnsvHO4HXfcmu0VFTB//tqF44UXQhdNjR13XLtgtGsXfkOPe1K7F18MXUrLl4fi3L17vHkKnLqYRDL13/+GQehJk8IqZLffDpsWwO9aX3wB77wTWhk1hWPu3NB9BWHge6+91m1ttGwZ/QV/VVWhO+kvfwkZRo0K4yyy0erqYlKBEMnEu++GpSgXLIB//jOcNVPIKivhgw/WPZNqwYI1+2y33dotjf32C1/e2TqTaPHi0Gp48cUwCH3vvWEAX7JCYxAi2fD00+F0yqZN4d//DlNmFLpNNw2/se+1F5x++prty5aF+Y2SWxtDh4aziSC0KNq0SX/BXyatjcmTwzFftgyGDIHzz9f0JA1IBUJkfdzhllvg2mvDNQxjx8Juu8WdKl7NmoV1nDt0WLOtujqs75zc0pg1C0aPXrPP1luHU26TT8HdZ5+1L/grLw8L+hx2WFjMp00beP758DppUOpiEqnLqlXwu9/BI4+E36CHDdNFWJlasSK0NlK7qb7+es0+rVuvKRhvvBHGdyCcrTRoUCgsEgmNQYhsiM8+Cxe/lZWFAdI//UndG9niHi4uTC0a//nPmulFGjcOYx277BJv1gKnMQiRTL3xRljEfsWK0KXUuXPciQqLWeim2203OOmkNdt79gyTF37/fdjn+uvXnrdKGpSmhxRJ9cADYQWypk1h6lQVh4ZSXg4PPbTmIr6KitClp5X4YqMCIVKjshKuuCJcfHXEETBtWhhAlYYxYMCa+aRqVFVpJb4YqYtJBMJplN26ham4e/WCv/89P+coymdTp649NTqEx6+/Hk8eUYEQYf78cPHbxx/D4MHQo0fciYpTfaZAlwalAiHF7ZlnQsths83Clbo/+1nciURyhsYgpDi5h4uwTjwxnIM/bZqKg0gKFQgpPqtXw29/G2ZjPe20sLj9j38cdyqRnKMCIcVl0aJwCuuIEfDXv4ZpurWWgEhakRYIMzvezOab2Qdm1qeO/dqbWZWZdU3adpmZzTGzuWZ2eZQ5pUi89RaUloYprMeMgb59dWW0SB0iKxBm1ggYCJwAtAXONLN1lttK7Hcz8FzStn2AHsDBQDvgJDNrE1VWKQIjRsDPfx4Go6dODVdJi0idomxBHAx84O4fuXsF8CiQ7pLU3sATwJKkbT8F3nD3le5eCbwM6H+0ZK6qCq6+OqwjcOihYTBas4KK1EuUBaIl8GnS44WJbT8ws5aEL/5BKa+dA/zczHYwsy2AjsCu6T7EzHqaWZmZlS1dujRr4aUAfP01dOoEt94Kv/99mCG0efO4U4nkjSivg0jXuZs6deydwDXuXmVJfcHu/q6Z3QxMAlYAbwOV6T7E3QcDgyHM5rrxsaUgvPdeuPjtww/Dym8XXRR3IpG8E2WBWMjav/W3Ahal7FMKPJooDs2BjmZW6e5j3f1+4H4AM7sx8X4i6/fcc2HBmcaN4YUXwllLIpKxKLuYpgFtzKy1mTUBugHjk3dw99buXuLuJcBo4PfuPhbAzHZM/NwNOBUYGWFWKQTuYQ6ljh3DdQ3Tpqk4iGyEyFoQ7l5pZr0IZyc1Aoa6+1wzuyjxfOq4Q6onzGwH4HvgEnf/KqqsUgBWrw7dSA88AKeeGn5qYXuRjRLpXEzuPhGYmLItbWFw9+4pjzXvgdRPeXkoCm+8Af37Q79+sImuARXZWPUuEIkunyOAHwGrCGcalbl7dZ0vFInStGnhmoavvoLRo8PUGSKSFestEGZ2FNAH2B6YSbheYXOgC7C7mY0Gbnf35RHmFFnXI4/ABRfATjuFNQPatYs7kUhBqU8LoiPQw93/L/UJM9sUOAk4lnCxm0j0qqrgz3+Gm28OV0ePHg0tWsSdSqTgrLdAuPtVZraJmZ3u7o+nPFcJjI0qnMg6li+Hs86Cp5+GCy+Eu++GJk3iTiVSkOo1kpcYZ+gdcRaRur3/fpgu49ln4d57YdAgFQeRCGVyFtPzZnYl8Bjwbc1Gd/9v1lOJpJo0CU4/HRo1CvePOiruRCIFL5MCcX7i5yVJ2xz4SfbiiKRwD91If/wjtG0L48bBT/RPTqQh1LtAuHvrKIOIrOO778Ike0OHQpcu8OCDsPXWcacSKRrrHYMws0PM7G0zW2FmU83spw0RTIrc55/DL38ZikPfvvDEEyoOIg2sPi2IgcCVwCvAyYQZWH8VYSYpdjNmQOfO8OWXYUnQX/867kQiRak+ZzFt4u6T3P07dx8F6IRzic5jj0GHDmEp0NdeU3EQiVF9WhDNzOzU2h67+5jsx5KiU10dupJuvBGOOCKsGb3jjnGnEilq9SkQLwOdannsgAqEbJzly+Gcc2DCBPjd72DgQF3fIJID6nMl9XkNEUSK1IcfhpXf5s+He+6BSy4J3UsiErsNnu7bzDoDn7v7m1nMI8XkxRfDGINZWAXu6KPjTiQiSTZm0vxDgP81s2eyFUaKhHtoLfzqV7DLLvDWWyoOIjlog1sQ7v6nbAaRIlFREbqRhgyBTp1gxAjYZpu4U4lIGvVuQZjZFmbW18z+lXjcxsxOii6aFJwlS0JLYciQMF332LEqDiI5LJMWxDBgOnBY4vFCYBTwVLZDSQGaNSsMRn/xBYwcCd26xZ1IRNYjkzGI3d39FuB7AHdfBeh0E1m/UaPg8MPD2MOrr6o4iOSJTApEhZk1JVz7gJntDnwXSSopDNXV0K9fmKb7gAOgrAwOPDDuVCJST5l0MfUHngV2NbOHgSOA7lGEkgLwzTfw29+GcYbzzw8L/Gy2WdypRCQDmUz3PcnMZgCHErqWLgO2jCqY5LGPPgqT7c2bB3fdBb176+I3kTxUrwJhZocBLYFX3P1pM9sPuBv4GbBrhPkk3/z739C1axhvePZZOPbYuBOJyAaqz3oQtwJDgdOAp82sPzAJeBNoE208yRvuoRvp2GNhp53gzTdVHETyXH1aECcCB7j7ajPbDlgE7Ofu70cbTfJGRQVceincdx+ceCI88oiubxApAPU5i2mVu68GcPevgPkqDvKDpUvhmGNCcejTJ6wZreIgUhDq04LY3czGJz0uSX7s7idnP5bktPLycC1Dv35wwQWweDE8/DCcdVbcyUQki+pTIDqnPL49iiCSRwYMgClTwmR7O+8c7peWxp1KRLKsPutBvNwQQSRPlJfDv/4VBqWrq+Hpp6Fdu7hTiUgE6nMW0wQz62RmjdM89xMz+6uZnR9NPMk5Z58NlZXhfuPGMHhwvHlEJDL1GaTuQbje4T9mNs3MJprZS2b2EXAfMN3dh0aaUnLDuHHhOocaFRUwbBh8/nl8mUQkMvXpYvocuBq42sxKgF2AVcB77r4y2niSMxYsSD/JXlVVGJMYOLDhM4lIpDJaMMjdPwE+iSSJ5K4VK8JU3RUV6z5XUQGvv97wmUQkcpksGHSamb1vZl+b2XIz+8bMlkcZTnJAdTWccw7MmQPPPBMGp1NvM2fGnVJEIpBJC+JmoJO7vxtVGMlBffuGsYe77oLjjos7jYg0oEzWg1icaXEws+PNbL6ZfWBmferYr72ZVZlZ16RtfzCzuWY2x8xGmtnmmXy2ZMHDD8ONN0LPnmFGVhEpKpm0IMrM7DFgLEkLBbn7mHQ7m1kjYCBwLGF50mlmNt7d56XZ72bguaRtLYFLgbbuvsrMHge6AcMzyCsb4803w1XSRx4J99yj6bpFilAmBWIbYCWQ3M/gQNoCARwMfODuHwGY2aOEq7LnpezXG3gCaJ8mW1Mz+x7YgjBJoDSEhQuhSxdo2RJGj4YmTeJOJCIxyGTBoPMyfO+WwKdJjxcChyTvkGgpnAL8kqQC4e6fmdltwP8RTql93t2fT/chZtYT6Amw2267ZRhR1rFyZVjs59tv4YUXoHnzuBOJSEzWWyDM7Gp3v8XM7iGxHnUyd7+0tpem2Zb6+juBa9y9ypK6MBLTincGWgPLgFFmdo67j0jz+YOBwQClpaXr5JMMVFdD9+7hrKQJE2DvveNOJCIxqk8LomZguow0BaIOC1l7tblWrNtNVAo8migOzYGOZlYJNAY+dvelAGY2BjgcWKdASBYNGACjRsGtt4Z1HUSkqNXnSuoJibvzgD8BJUmvc+DBWl46DWhjZq2BzwiDzGvNB+3urWvum9lw4Cl3H2tmhwCHmtkWhC6mowkFSqIyahRcdx2cey5ccUXcaUQkB2QySD0CuAp4B6he387uXmlmvQhnJzUChrr7XDO7KPH8oDpe+6aZjQZmAJXATBLdSBKB6dNDYTj88LDwj85YEhHA3OvXa2Rmr7p7h4jzbJTS0lIvK1NDIyPl5dC+PTRqBG+9FdaTFpGiYWbT3T3tgi6ZtCD6m9kQ4EXqcR2E5IFVq8LprMuWwWuvqTiIyFoyKRDnAXsRBpBrupjqug5Ccpk79OgRWg1PPqlFf0RkHZkUiHbuvm9kSaRh3XRTmErjhhtCK0JEJEUmczG9YWZtI0siDWfcOPjTn+DMM+Haa+NOIyI5KpMWRAfgXDP7mDAGYYC7+36RJJNozJ4dlg1t3x7uv19nLIlIrTIpEMdHlkIaxpIl0KkTNGsWWhFNm8adSERyWCZzMS2IMohE7Lvv4NRTYelSmDIFdtkl7kQikuMyWnJU8pQ7XHxxOJX1scfgoIPiTiQieSCTQWrJV3fcAcOGQf/+cPrpcacRkTyhAlHoJk6Eq66Crl2hX7+404hIHlGBKGTz5kG3buEiuOHDYRP9dYtI/ekbo1B9+WU4Y2nLLWH8+PBTRCQDGqQuRBUVoUvps8/g5ZehVau4E4lIHlKBKDTu0Ls3TJ4MDz0Ehxyy3peIiKSjLqZCM3AgDB4MffrAOefEnUZE8pgKRCGZNAkuvxw6dw6T8ImIbAQViELx3nvhGoe2bUPXks5YEpGNpG+RQvDVV+GMpcaNwxlLW28ddyIRKQAapM53lZVwxhnw8cfw0ktQUhJ3IhEpECoQ+e6PfwxjD/ffDx1yeslwEckz6mLKZ/fdB/fcE4rE+efHnUZECowKRL6aPBl69YITToBbbok7jYgUIBWIfPThh3DaadCmDYwcCY0axZ1IRAqQCkS+Wb48nLEE4YylbbeNN4+IFCwNUueTqio480x4/314/nnYY4+4E4lIAVOByCd9+oT1HQYNgqOOijuNiBQ4dTHli+HD4bbbwsD0hRfGnUZEioAKRD547TXo2ROOOSYsHyoi0gBUIHLdggVwyinhCunHH4dN1SsoIg1DBSKXrVgBJ58cFgCaMAG22y7uRCJSRPTraK6qrg7rOcyZA888A3vuGXciESkyKhC5qm9fGDcO7roLjjsu7jQiUoTUxZSLHnkEbrwxDEz37h13GhEpUioQuebNN8PEe0ceGSbiM4s7kYgUKRWIXLJwIXTpAi1bwujR0KRJ3IlEpIhpDCJXrFwZ1pL+9lt44QVo3jzuRCJS5CJtQZjZ8WY238w+MLM+dezX3syqzKxr4vGeZjYr6bbczC6PMmusqquhe3eYOTPMzrr33nEnEhGJrgVhZo2AgcCxwEJgmpmNd/d5afa7GXiuZpu7zwf2T3r+M+DJqLLGbsAAGDUKbr0VTjwx7jQiIkC0LYiDgQ/c/SN3rwAeBTqn2a838ASwpJb3ORr40N0XRBMzZqNGwXXXhRbEFVfEnUZE5AdRFoiWwKdJjxcmtv3AzFoCpwCD6nifbsDI2p40s55mVmZmZUuXLt2IuDGYMQPOPRcOPzzM0KozlkQkh0RZINJ923nK4zuBa9y9Ku0bmDUBTgZG1fYh7j7Y3UvdvbRFixYbmrXhlZeHaTRatIAxY2CzzeJOJCKylijPYloI7Jr0uBWwKGWfUuBRC785Nwc6mlmlu49NPH8CMMPdF0eYs+GtWhVOZ122LMzUutNOcScSEVlHlAViGtDGzFoTBpm7AWcl7+DurWvum9lw4Kmk4gBwJnV0L+Uld+jRA956C558Etq1izuRiEhakRUId680s16Es5MaAUPdfa6ZXZR4vq5xB8xsC8IZUIW1Os5NN8HDD8MNN4RWhIhIjjL31GGB/FVaWuplZWVxx6jduHGhKJx1FowYoUFpEYmdmU1399J0z2mqjYYyezacfTYcfDAMGaLiICI5TwWiISxZEs5YatYMxo6Fpk3jTiQisl6aiylq330Hp54aisSUKbDLLnEnEhGpFxWIKLnDxReHU1kfewwOOijuRCIi9aYupijdcQcMGwb9+8Ppp8edRkQkIyoQUZk4Ea66Crp2hX794k4jIpIxFYgozJsH3bqFi+CGD4dNdJhFJP/omyvbvvwSOnWCLbeE8ePDTxGRPKRB6myqqAhdSp99Bi+/DK1axZ1IRGSDqUBkizv07g2TJ4erpA85JO5EIiIbRV1M2TJwIAweDNdeG66YFhHJcyoQ2TBpElx+OXTuDNdfH3caEZGsUIHYWO+9F65xaNsWHnpIZyyJSMHQt9nG+OqrcMZS48bhjKWtt447kYhI1miQekNVVsIZZ8DHH8NLL0FJSdyJRESySgViQ/3xj2Hs4f77oUOHuNOIiGSdupg2xH33wT33hCJx/vlxpxERiYQKRKYmT4ZeveCEE+CWW+JOIyISGRWITHz0EZx2GrRpAyNHQqNGcScSEYmMCkR9LV8ezliCcMbSttvGm0dEJGIapK6Pqio488xwzcPzz8Mee8SdSEQkcioQ9dGnT1jfYdAgOOqouNOIiDQIdTGtz/DhcNttYWD6wgvjTiMi0mBUIOry2mvQsyccc0xYPlREpIioQNRmwQI45ZRwhfTjj8Om6o0TkeKiApHOihVw8slhAaAJE2C77eJOJCLS4PRrcarqavjNb2Du3DAwveeecScSEYmFCkSqvn1h7Fi4+2447ri404iIxEZdTMkeeQRuvDEMTPfqFXcaEZFYqQUBUF4OHTvCvHlw5JFhIj6zuFOJiMRKLQiAa66BWbNgs81g9Gho0iTuRCIisVOB+PBDGDEi3K+oCAsBiYiICgQ337ymO8kdBgyIN4+ISI4o7gJRXg4PPRRObYXQghg2DD7/PN5cIiI5oLgLxIABa4pDjaoqtSJERCj2AjF1amg1JKuogNdfjyePiEgOibRAmNnxZjbfzD4wsz517NfezKrMrGvStmZmNtrM/mNm75rZYVkPOHNmGHdIvc2cmfWPEhHJN5EVCDNrBAwETgDaAmeaWdta9rsZeC7lqbuAZ919L6Ad8G5UWUVEZF1RtiAOBj5w94/cvQJ4FOicZr/ewBPAkpoNZrYN8HPgfgB3r3D3ZRFmFRGRFFEWiJbAp0mPFya2/cDMWgKnAINSXvsTYCkwzMxmmtkQM9sy3YeYWU8zKzOzsqVLl2YvvYhIkYuyQKSbq8JTHt8JXOPuVSnbNwUOBP7p7gcA3wJpxzDcfbC7l7p7aYsWLTYysoiI1IhyLqaFwK5Jj1sBi1L2KQUetXChWnOgo5lVAm8AC939zcR+o6mlQIiISDSiLBDTgDZm1hr4DOgGnJW8g7u3rrlvZsOBp9x9bOLxp2a2p7vPB44G5q3vA6dPn/6FmS3YwLzNgS828LVRUq7MKFdmlCszhZjrx7U9EVmBcPdKM+tFODupETDU3eea2UWJ51PHHVL1Bh42sybAR8B59fjMDe5jMrMydy/d0NdHRbkyo1yZUa7MFFuuSKf7dveJwMSUbWkLg7t3T3k8i9AFJSIiMSjuK6lFRKRWKhBrDI47QC2UKzPKlRnlykxR5TL31DNPRURE1IIQEZFaqECIiEhaRVkgzGyomS0xszlJ27Y3s0lm9n7i53Y5kus6M/vMzGYlbh0bONOuZvbvxIy6c83sssT2WI9XHbniPl6bm9lbZvZ2ItdfEtvjPl615Yr1eCXla5SYVuepxOPY/z/WkitXjtcnZvZOIkNZYlvWj1lRFghgOHB8yrY+wIvu3gZ4kXiu3B7OurkA7nD3/RO3iWmej1IlcIW7/xQ4FLgkMStv3MertlwQ7/H6Dvilu7cD9geON7NDif941ZYL4j1eNS5j7Rmb4z5eNVJzQW4cL4CjEhlqLgfI+jErygLh7q8A/03Z3Bl4IHH/AaBLQ2aCWnPFyt3L3X1G4v43hP8sLYn5eNWRK1YerEg8bJy4OfEfr9pyxc7MWgEnAkOSNsf+/7GWXLks68esKAtELXZy93IIXz7AjjHnSdbLzGYnuqBiaWoDmFkJcADwJjl0vFJyQczHK9EtMYswhf2kxJxisR+vWnJB/P++7gSuBpLX/439eNWSC+I/XhCK+/NmNt3Meia2Zf2YqUDkvn8CuxO6BcqB2+MIYWZbEdbtuNzdl8eRIZ00uWI/Xu5e5e77EyaoPNjM9mnoDOnUkivW42VmJwFL3H16Q37u+tSRK/Z/XwlHuPuBhAXZLjGzn0fxISoQayw2s10AEj+XrGf/BuHuixP/sauBfxEWYmpQZtaY8CX8sLuPSWyO/Xily5ULx6tGYpGryYRxpdiPV7pcOXC8jgBONrNPCIuK/dLMRhD/8UqbKweOFwDuvijxcwnwZCJH1o+ZCsQa44FzE/fPBcbFmOUHNX/hCacAc2rbN6LPN8LKfu+6+9+Tnor1eNWWKweOVwsza5a43xQ4BvgP8R+vtLniPl7ufq27t3L3EsKMzy+5+znEfLxqyxX38QIwsy3NbOua+8BxiRzZP2buXnQ3YCShefg9Yd2KC4AdCCP/7yd+bp8juR4C3gFmJ/4B7NLAmToQ+jtnA7MSt45xH686csV9vPYDZiY+fw7QL7E97uNVW65Yj1dKxl8QpvyP/XjVkSv240VYcfPtxG0u8Oeojpmm2hARkbTUxSQiImmpQIiISFoqECIikpYKhIiIpKUCISIiaalASMFJnPP/qpnNMbMuSdvHmdmPannNdWZ25Xred7iZdc0gR4klzcxbz9es9zPq876Jfc7K5LNFUqlASCE6kzBZ2WHAVQBm1gmY4YkrUItACaACIRtFBUIK0fdAU2AzoNrMNgUuB26tz4vNrIeZTUusnfCEmW2R9PQxZjbFzN5LzNdTMwnerYnXzDazC9O8Z9p9LPiHmc0zs6epZYI1MzsokWcqcEnS9pJEnhmJ2+GJp24CfmZhvYA/1LGfSK1UIKQQPQL8CngWuA74PfCgu6+s5+vHuHt7D2snvEu4or1GCXAkYRroQWa2eeL5r929PdAe6GFmrVPes7Z9TgH2BPYFegC1fXEPAy5198NSti8BjvUwcdsZwN2J7X2AKR7WC7ijjv1EarVp3AFEss3dvyZ8gZOYjvka4FQz+xewHXC7u0+t4y32MbPrgWbAVsBzSc897mGitvfN7CNgL8JcOPsljR1sC7QB3kt6XW37/BwY6e5VwCIzeyk1jJltCzRz95cTmx4izOIJYV2Hf5jZ/kAV8D+1/Jnqu5/ID1QgpND1A24gjEtMJ7QuxgFH1fGa4UAXd3/bzLoT5uKpkTo3jQMG9Hb35EJSs07FDw9r2adjmvdMZXXs8wdgMdCO0COweiP3E/mBupikYJlZG+BHid+8tyAs/OLA5ut56dZAeWI68bNTnvu1mW1iZrsTJk2bT2hhXJzYHzP7n8Qsm8lq2+cVoFtijGIX0hQuD9Nzf21mHRKbkjNtC5QnWjW/ARoltn+T+HOsbz+RWqkFIYXsBuDPifsjgbGENYb7red1fQmr0y0gzNyZ/EU7H3gZ2Am4yN1Xm9kQwtjEjMQ05EtZd7nH2vZ5Evhl4nPeS7x3OucBQ81sJWt3ed0LPGFmvwb+DXyb2D4bqDSztwktotr2E6mVZnMVEZG01MUkIiJpqUCIiEhaKhAiIpKWCoSIiKSlAiEiImmpQIiISFoqECIiktb/A1B0Rc7PqeE2AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "x= range(10,51,10)\n",
    "for i,metric in enumerate(['ROC-AUC', 'PR-AUC', 'min(Re,Pr)']):\n",
    "    plt.figure()\n",
    "    y = [gen_res[ld][i][0] for ld in x]\n",
    "    plt.plot(x, y, color='r', marker='^')\n",
    "    plt.xlabel('% labeled data')\n",
    "    plt.ylabel(metric)\n",
    "    plt.savefig(f'{metric}-mimiciv with ss random seed 2.pdf', format='pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "58ba67db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test op [0. 0. 0. 0. 0. 1. 0. 0. 1. 0. 0. 1. 1. 1. 0. 0. 0. 0.]\n",
      "test ip [array([[ 0.09277487,  1.14354375],\n",
      "       [ 0.38677976, -0.87447463],\n",
      "       [ 0.23977732, -0.87447463],\n",
      "       [ 0.97478953, -0.87447463],\n",
      "       [ 0.68078464,  1.14354375],\n",
      "       [ 0.97478953, -0.87447463],\n",
      "       [ 0.31327854, -0.87447463],\n",
      "       [ 0.31327854, -0.87447463],\n",
      "       [ 1.19529319,  1.14354375],\n",
      "       [-1.74475565, -0.87447463],\n",
      "       [-1.74475565, -0.87447463],\n",
      "       [-1.59775321, -0.87447463],\n",
      "       [-0.34823245,  1.14354375],\n",
      "       [-0.34823245,  1.14354375],\n",
      "       [ 1.37169612,  1.14354375],\n",
      "       [-1.30374833, -0.87447463],\n",
      "       [-1.30374833, -0.87447463],\n",
      "       [-2.03876054,  1.14354375]]), array([[ 9.133333  , 10.133333  , 12.133333  , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       [18.383333  , 13.9       , 15.383333  , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       [ 1.0833334 ,  3.5833333 ,  3.3333333 , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       ...,\n",
      "       [ 2.0333333 ,  2.95      ,  3.95      , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       [ 0.21666667, 20.733334  , 10.3       , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       [19.333334  , 19.416666  , 20.183332  , ..., 13.35      ,\n",
      "        15.683333  , 15.866667  ]], dtype=float32), array([[-0.5313693 , -0.15907255, -0.7175177 , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       [ 0.2132242 , -0.3946206 ,  0.585521  , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       [ 0.02707583,  0.39937258,  0.39937258, ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       ...,\n",
      "       [ 0.64815795,  0.585521  ,  0.585521  , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       [ 0.40689883, -0.4131337 , -0.588522  , ...,  0.        ,\n",
      "         0.        ,  0.        ],\n",
      "       [ 0.02674642, -0.11098031,  0.05612939, ...,  0.9600761 ,\n",
      "         0.18834372, -0.19201468]], dtype=float32), array([[ 85,  85,  85, ...,   0,   0,   0],\n",
      "       [ 85,  90,  85, ...,   0,   0,   0],\n",
      "       [ 85,  85,  85, ...,   0,   0,   0],\n",
      "       ...,\n",
      "       [ 87,  85,  85, ...,   0,   0,   0],\n",
      "       [104,  88,  90, ...,   0,   0,   0],\n",
      "       [ 84,  84,  84, ...,  60,  60,  60]], dtype=int32)]\n",
      "test ind [ 3  6  9 20 22 25 43 44 47 52 53 65 69 70 80 85 86 87]\n"
     ]
    }
   ],
   "source": [
    "print(\"test op\",test_op)\n",
    "print(\"test ip\",test_ip)\n",
    "print(\"test ind\",test_ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "44c7f033",
   "metadata": {},
   "outputs": [],
   "source": [
    "test__op_ind_df.rename(columns={'test_ind': 'ind','test_op':'op','test_predicted':'pred','test_true_op':'true'}, inplace=True)\n",
    "#valid_op_ind_df.rename(columns={'valid_ind': 'ind','valid_op':'op','val_predicted':'pred','val_true_op':'true'}, inplace=True)\n",
    "#roc_test = pd.concat([valid_op_ind_df, test__op_ind_df], ignore_index=True)\n",
    "#len(roc_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abc86fd1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d1e6f85e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABJCElEQVR4nO3dd3gU5fbA8e9JAkmA0EJRmvQiCKihqSAKSBFFfxYsFy9evRKaDRUvNqyIBaUjguK14bUhAopiAURFQKogVYQoIjVASCDl/P6YgSwhWRbI7mST83mefXb6nJ3dnTPzzsz7iqpijDHG5CXC6wCMMcYUbJYojDHG+GWJwhhjjF+WKIwxxvhlicIYY4xfliiMMcb4ZYmikBORX0SkvddxFBQiMkREJnm07iki8pQX685vInKziHxxivPabzLMWKIIIRHZLCKpInJARP5ydxylgrlOVW2sqt8Gcx1HiEi0iAwTkS3u51wvIveLiIRi/bnE015EknyHqeozqnp7kNYnInKniKwSkRQRSRKR90XknGCs71SJyFAReet0lqGqb6vqZQGs67jkeKq/SREp7sa+3t2+m0XkNRGpebLLMifHEkXoXaGqpYDmwLnAf7wN5+SJSFQeo94HOgDdgDigF3AHMDIIMYiIFLTf70jgLuBOoDxQH5gGXJ7fK/LzHQSdh+v+ALgSuAkoAzQDluD85k6Kl9svLKmqvUL0AjYDHX36nwNm+vS3Br4H9gLLgfY+48oDrwN/AnuAaT7jugPL3Pm+B5rmXCdQBUgFyvuMOxfYCRRz+/8FrHGXPxs4y2daBfoD64HfcvlsHYA0oHqO4a2ATKCu2/8tMAz4CUgGPskRk79t8C3wNLDA/Sx1gVvdmPcDm4A+7rQl3WmygAPuqwowFHjLnaam+7n+CWxxt8VDPuuLBd5wt8ca4AEgKY/vtp77OVv6+f6nAGOBmW68C4E6PuNHAluBfTg7wLY+44bi7CjfcsffDrQEfnC31TZgDFDcZ57GwJfAbmA7MAToAhwG0t1tstydtgww2V3OH8BTQKQ7rre7zV9yl/WUO+w7d7y44/52v9MVQBOcg4R0d30HgE9z/g+ASDeuje42WUKO35A7XUf3+zxunJ//V27f9W3udz0P+BwYkGMZy4H/c7sb+my/tcD1Xu9DvHp5HkBReuX4g1QDVgIj3f6qwC6co/EIoJPbX9EdPxN4DygHFAMudoef5/5BW7l/un+664nOZZ1fA//2ied5YILbfRWwAWgERAEPA9/7TKvun6Y8EJvLZ3sWmJvH5/6d7B34tzg7oiY4O/MPff7MJ9oG37p/8sZujMVwjtbr4OysLgYOAue507cnx449j53HqzhJoRlwCGjk+5ncbV4NZweYV6JIBH4/wfc/xd3ptHTjfxuY6jP+H0C8O24Q8BcQ4xN3uvs9Rbjxno+TWKPcz7IGuNudPg5npz8IiHH7W+XcBj7rnga84n4nlXAS+ZHvrDeQAQx01xXLsYmiM84Ovqz7PTQCzvT5zE/5+R/cj/M/aODO2wyIP5nfV27L9fNd/9f9jLHALcACn+nPxkm60e40W3EORKJw/mc7gcZe70e8eBW0U/eiYJqI7Mf5Ef4NPOYO/wcwS1VnqWqWqn4JLAa6iciZQFcgUVX3qGq6qs515/s38IqqLlTVTFV9A2dn1zqXdb8D3AhO0Q1wgzsMoA8wTFXXqGoG8AzQXETO8pl/mKruVtXUXJZdAWfHlJtt7vgj3lTVVaqaAjwCXC8ikf62gc+8U1T1F1XNcLfDTFXdqI65wBdA2zziyMvjqpqqqstxjiibucOvB55xt3kSMMrPMuL9fH5fH6nqT+42fhunCBIAVX1LVXe5n+1FnB1WA595f1DVae62SVXVJar6ozv9Zpwd/cXutN2Bv1T1RVVNU9X9qrowt4BEpDLO7+tuVU1R1b9xzhBu8JnsT1Ud7a4r5/efjpOIGgLi/oYC2RbgnBk9rKpr3e9wuaruymW6QLfviQx1P2Mq8DHH/sZvxvl+DuFsv82q+rr7mX/GOai5Nh9iCDuWKELvKlWNwznabUj2DvQs4DoR2XvkBVwEnAlUB3ar6p5clncWMCjHfNVxilly+gBoIyJVgHY4R1jzfZYz0mcZu3GO8Kr6zL/Vz+fa6caamzPd8bkt53ecM4MK+N8GucYgIl1F5EcR2e1O341jk1Ig/vLpPggcucGgSo71+fv8u8j78weyLkRkkIisEZFk97OU4djPkvOz1xeRGe6NEftwkvuR6avjFOcE4iyc72Cbz3Z/BefMItd1+1LVr3GKvcYC20VkooiUDnDdgcYZ6PY9kaOfQ1X345ypH0mIN+Akb3C2Sascv8WbgTPyIYawY4nCI+7R7xTgBXfQVpwj7bI+r5Kq+qw7rryIlM1lUVuBp3PMV0JV381lnXtxjrivx7kg+K6qqs9y+uRYTqyqfu+7CD8faQ7OH6u670ARaYmzM/jaZ7DvNDVwjkh3nmAbHBeDiETjHOW9AFRW1bLALJwEd6J4A7ENp8gpt7hz+gqoJiIJp7IiEWkLDMb5bsq5nyWZ7M8Cx3+e8cCvQD1VLY1T1n9k+q04RXK5ybmcrThnoRV8tntpVW3sZ55jF6g6SlXPxykWrI9TpHTC+U4Qp685QEsRqeZnmhSghE9/bjv1nPG8C9woIm1wiqO+8Ylrbo7fYilV7RtArIWOJQpvvQx0EpHmOBcprxCRziISKSIx7u2d1dzT+M+AcSJSTkSKiUg7dxmvAoki0sq9E6ikiFwuInF5rPMdnLLZa8gudgKYAPxHRBoDiEgZEbku0A+iqnNwdpYfikhj9zO0xjlCG6+q630m/4eInC0iJYAngA9UNdPfNshjtcVximd2ABki0hXwvWVzOxAvImUC/Rw5/A9nm5QTkarAgLwmdD/fOOBdN+bibvw3iMiDAawrDuc6wA4gSkQeBU50VB6Hc2H7gIg0BHx3YjOAM0TkbnFuW44TkVbuuO1AzSN3jbm/ry+AF0WktIhEiEgdEbmYAIhIC/f3VwxnZ52Gc2H/yLpq+5l9EvCkiNRzf79NRSQ+50Tu7+tL4GMROV9EotzPlCgi/3InWwbc4P4/EgismGgWztnDE8B7qprlDp8B1BeRXu7yirmfs1EAyyx0LFF4SFV34Fxce0RVtwI9cI4Kd+Ac0dxP9nfUC+fI+1ecaxt3u8tYjHOdYgzO3TkbcC405mU6zh06290y+SOxfAwMB6a6xRircMqtT8Y1OEdkn+Pc5fIWzp00A3NM9ybO2dRfOBda73RjONE2OIZbdHAnzg59D85Z0nSf8b/iHDFucosPciuO8+cJIAn4DeeI9gOcI++83El2EcxenCKVq4FPA1jXbJyDgXU4xXFp+C/qArgP5zPvxzlgeO/ICHfbdAKuwNnO64FL3NHvu++7RORnt/sWnMS7GmdbfkDgRT2l3fXvcWPfRfaZ8mTgbHf7T8tl3hE4398XOElvMs6RfW6uxdmxv4dztrUKSMD5bsC53lXHjeNxjj0QypV7PeIjnLuq3vEZvh/noOMGnDsN/8L5f0SfaJmFkWSXPBgTfCLyLc6dKJ48HX06RKQvcIOqBnSkbUxhYWcUxuRBRM4UkQvdopgGOLeafux1XMaEmj2daEzeiuPc/VMLpyhpKs51CGOKFCt6MsYY45cVPRljjPEr7IqeKlSooDVr1vQ6DGOMCStLlizZqaoVT2XesEsUNWvWZPHixV6HYYwxYUVEfj/Vea3oyRhjjF+WKIwxxvhlicIYY4xfliiMMcb4ZYnCGGOMX5YojDHG+BW0RCEir4nI3yKyKo/xIiKjRGSDiKwQkfOCFYsxxphTF8znKKbgVLn83zzGd8Wp7roeTnvP4913Y4wxvtL2Qlb6KcyosHMVh1Nza704cEFLFKo6T0Rq+pmkB/Bft4W1H0WkrIiceRJt7RpjzPEy0+Hwfp8BCjtXwuED8Md3EO22B7X9Z0hPgag8mr/Y8DHEVefYRgY9sH/Lac0+cn4rJi08vQIbL5/MrsqxDbMkucOOSxQicgdwB0CNGjVCEpwxJp9kZTg75CP2boDUnc5Ou1heDTH6OPg3bF8EKdsgZTvEHtcAXrZD+yBt1+nHfMT+E7UdFWKxJ9scPDSrtofV00+p5o6jvEwUuaXpXKuyVdWJwESAhIQEq+7WmPyQlQGZh5wd8b7TO2rl75/hwJ/Oe2wlZ9iGjyGmHKT8dfqx+jq8L7DpYspld6fthWIloWIzKFYCKrtNm6fthurtIbJ47suILgtlA2nSO8iKl4GYsgFNunVrMjNmrKNv3xYAtAc2DNpD7dpPnPLqvUwUSRzbWH01nCYHjTGnQrMg8zCkH4Rdq3OOdI7gv3/MKU45vM85qg823yRR3C3yyTzkvM5s5RT7VG5x4uWk74fyDaFEZah0LkQUy3taEShd03kvQjIyshg1aiGPPvoNKSnpNGlSibZtzwKgVq1yJ5jbPy8TxXRggIhMxbmInWzXJ4zxI2W7U9b+91L4fQ6UOhPWvA3lGjgXOvesC2w5yZuO7Y+KhYxUqNbu1GNTdYppzuoI8WdDiTOyl12hCZStDWJ34wfLwoVJ9Okzg+XLtwNwzTWNqF379JKDr6AlChF5F+esp4KIJAGPAcUAVHUCTiPp3YANwEHg1mDFYkyB5tt4WEYabP0akubBouehQmNnB7tjRd7z7/rl2P7IaOeI/YwWEOFTpJJ12Ble6XxocT9IJJSrazvwMLZnTypDhnzFK68sQRVq1izLmDFdufzy+vm6nmDe9XTjCcYr0D9Y6zfGM6m7YOcq5+g/Mjrv6dL2wPzB/pe1M5fHkMrUdl5la8OZrZ1y9HL1nHHlGkCkn2IZU6g8/vhcJkxYQlRUBPfd14ZHHrmYEiXy//sPu/YojPFEZjrsXuMc/f+1CFL+hO+HwhkJzrDti3Huz8iney2iSkD1i6FGB6jR0RlWtjYUD+AuIVOoZWRkERXlnAU+/HA7fvttL08/fSlNmlQK2jotURjjz9a58L/2eY//a5FPT44kUfIMiG8CZWrmPf+hZKhzBZzd6zSCNEVBWloGw4d/x7Rpa1m48HaKF4+kQoUSfPLJDUFftyUKY3LavRZeb5j7uHL1AYGMg86tltXbQ9W2gDp35Rw54rdyf5OPvvpqE337zmT9+t0AzJ69gSuuaBCy9VuiMObwfvj9K8hMg1k3O7eZ5nThU9BqSJG75dJ4a/v2Awwa9AVvv70SgEaNKjB+/OVcfHHNkMZhicIUflkZzq2bezfCxk+dC8x/zHfu8d+3Oe/5LhoG590FxfKo4sGYIHrrrRUMHPgZe/emERMTxaOPtmPQoAsoXjwy5LFYojCFU3oKvNPGufMoUFGxUPsKSN0B18y2u4eMp7KylL170+jSpS5jx3bL1+ciTpYlClM4ZByCLXNgyzew5MXcp4mr4VSwVrUt1L7cKXKq3h4qNocSJ1+HjjH56cCBw/zww1Y6dXKqDOnVqylVqsTRoUMtxOMiT0sUJnwdPgDLxjoPqf0wNPdpyjeCLlOg8vkQEfpTdmMCMW3arwwc+Bk7dqSwalU/6tYtj4jQsWNtr0MDLFGYcLPqddif5Jw9JM07fnx0GYiJdy48N7zBqQjOmALq99/3cuednzN9+loAEhKqcOhQhsdRHc8ShSm49m2FjZ/AykmwY3ne01VOgJqdnVo+m1hNMKbgS0/P5OWXf2To0LkcPJhOXFxxnnmmA337JhAZWfBurbZEYQqGfVucO5IyDsK8B5yH1fxVT93yP07tp20eg7iqoYvTmHxw552fMWHCEgCuv74xL73UmSpVCu5T95YojLeSN8Pc+2D9h8cO900Spao6ZwpndYaKTbNbKDMmTN19d2vmzv2dESM606VLXa/DOSFLFCb0sjKds4EvbodNM44dV6oqNLjeqeuocW+n+osI+5ma8KWqvPXWCmbN2sA77/wfIkKDBhVYtaofERHh8QCn/QNN6PyxAKZd4dSamlOjf0D7F6FE8Co2MybU1q7dSd++M/nmm82Ac8trt25OTb/hkiTAEoUJhcx0eDmPpibL1YfLJkO1i0IbkzFBlJqazrBh3zF8+AIOH84kPj6WF1+8jK5dC34xU24sUZjgUHXOIN5re/y4ev8HXd902i42ppCZM2cTiYkz2LjROXO+7bZzGT68I/Hx4ft7t0Rh8o+q0xznhmkw/8Hjx5eoDH393MlkTCHw/fdb2bhxD40bV2TChO5cdFENr0M6bZYoTP44sA1eqZL7uEY3Q7vnnTaejSlkMjOz2LBhNw0aONXADB58IRUqlOD228/zpAK/YLBEYU7doWT45i745b8c12hPdFk4/x5o/YhVzW0KraVLt5GYOJNNm/awdu0AypePJTo6in79WngdWr6yRGFO3h8L4I/vci9euuBxaPNo6GMyJoT27z/Eo49+w6hRP5GVpVStGsfGjbspX75wPvxpicIE7tA+GFPm+OElKkGLwdCsr7XdYAo1VeWjj9Zw112f88cf+4mIEO65pzWPP96euLhor8MLGksUJjBr34cZ1x87rMEN0OIBqHyuNzEZE2J33/05o0b9BECLFlV45ZXunHtu4b/2ZonC+KcKbzaHHSuyh5WuCf9aC5F5PBthTCF19dWNeOON5TzzTAf69Dm/QFbgFwyWKMzxDu+HH5+CRc8dP+5f66BcvdDHZIwHvvtuC9988xuPPHIxAO3b12TLlnsoXbrwFjPlxhKFAc2CkbHOnUoH/857uoH7oXipkIVljFd27TrI4MFzmDx5KQAdOtTmgguqAxS5JAGWKEzqLhjnNgOaM0lEFINrPoeKzSA2PvSxGRNiqsp//7uc++77kp07D1KsWAQPPngR5557htehecoSRVG2bSG80/rYYbf/5lStYZXzmSJmzZod9O07k7lzfwfgkktqMm7c5TRsaO2pW6IoalTh76WwfAKsfDV7eP1roft7IEXj4pwxOY0Y8QNz5/5OxYolGDGiMzfffA5iD4sCliiKlkPJMKbs8cO7vQONbgx5OMZ4LTk5jTJlYgAYNqwjJUsW59FHL6Z8eXseyJcdPhYFm2fDO21yJAmBmHLQ+xdLEqbI+fPP/fTs+QGtW0/m8OFMACpUKMHLL3exJJELO6Mo7Ba/6DQ16qvts9BysDfxGOOhzMwsxo1bxEMPfc3+/YcpUaIYP/+8jdatq3kdWoFmiaIw+2vRsUmiVlfoMsUuVJsiacmSP+nTZwZLlmwD4MorGzB6dFdq1MilWhpzjKAmChHpAowEIoFJqvpsjvFlgLeAGm4sL6jq68GMqcj47TP4qFt2/z+WQOXzvIvHGA8NHfotTz45j6wspXr10owe3ZUePRp6HVbYCFqiEJFIYCzQCUgCFonIdFVd7TNZf2C1ql4hIhWBtSLytqoeDlZcRYZvkrhomCUJU6TVrl0OERg0qA1Dh7anVCmrfuZkBPOMoiWwQVU3AYjIVKAH4JsoFIgT5x60UsBuICOIMRV+6akwyqfJxe7/gwbXeRePMR7YtGkPixb9Qc+eTQDo1asprVpVPdq4kDk5wbzrqSqw1ac/yR3mawzQCPgTWAncpapZORckIneIyGIRWbxjx45gxRv+Ns44NkmA83yEMUXE4cOZPPPMfBo3Hsc//zmNDRt2AyAiliROQzDPKHJ7UiVHM2h0BpYBlwJ1gC9FZL6q7jtmJtWJwESAhISEnMsourIy4fcv4LfPYemoY8dVaAK9llnrcqbImDfvdxITZ7BmzU4Abr75nCJZL1MwBDNRJAHVffqr4Zw5+LoVeFZVFdggIr8BDYGfghhX4ZCeAqPyqKCvxzSo2yOk4RjjlZ07D3L//V8yZcoyAOrVK8/48ZfToUNtbwMrRIKZKBYB9USkFvAHcANwU45ptgAdgPkiUhloAGwKYkyFQ1bG8Umi+iVw2atQto43MRnjkcTEGXz44RqioyMZMqQtDzxwITExdud/fgra1lTVDBEZAMzGuT32NVX9RUQS3fETgCeBKSKyEqeoarCq7gxWTIXC/iSY6HOiVqcHXDXNs3CM8UJWlhIR4RSrPv30paSmZvDyy52pV89qOQ4GcUp9wkdCQoIuXrzY6zC8kTQf3muX3R/fGHqv8i4eY0Ls4MF0nnxyLsuWbWfWrJus0r6TICJLVDXhVOa187NwcfDvY5NEs0ToON67eIwJsZkz1zFgwGds3rwXEfjppz9o1cqq3ggFSxThYN9WeLVGdn+PT6Duld7FY0wIJSXt4667Puejj9YA0KxZZSZM6G5JIoQsURR06z+C6ddk9zf5lyUJU2SMG7eIwYPncODAYUqWLMaTT17CwIGtiIqyiq9DyRJFQXZw57FJonFv6DzZs3CMCbWdOw9y4MBhrr66ISNHdqF6davAzwuWKAqqw/thfMXs/uu+ghqXehePMSGwd28av/6682i134MHX0jLllXp0qWux5EVbXb+VhAtfhFGl87ubzPUkoQp1FSVqVNX0ajRWK688l12704FIDo6ypJEAWBnFAXJ3o0wOcef4sInofXD3sRjTAhs2LCb/v1n8cUXGwG44ILqJCenWUtzBYglCq9pFkysAQf+OH5c/91Oc6XGFEKHDmXw3HMLePrp+Rw6lEm5cjE891wn/vWvc48+TGcKhoAThYiUVNWUYAZTJI2IPH5Yu+cg4T6r0M8Uaj17fsAnn6wF4JZbmvH8852oVKmkx1GZ3JwwUYjIBcAknPYiaohIM6CPqvYLdnCF3p4N2d2lqsC/1kNkcYiwEz1T+N19d2vWrt3FuHHduOSSWl6HY/wIZI/0Ek514NMBVHW5iLTzP4s5odcbwe5fs/v75FL0ZEwhkZWlvPbaUtas2cGLL3YGoH37mqxa1ZfISLunpqAL6NBVVbfmqFMlMzjhFBGrphybJKwqDlOIrVy5ncTEmXz/vdOO2S23NKNZszMALEmEiUASxVa3+ElFpDhwJ7AmuGEVcrNvze6+MwWKlch7WmPCVErKYR5/fC4jRvxAZqZyxhmlePnlzjRtWtnr0MxJCiRRJAIjcZoxTQK+AOz6xKlI2wNjy2f3t37YkoQplD79dC0DBnzGli3JiED//i14+ulLKVMmxuvQzCkIJFE0UNWbfQeIyIXAguCEVEj9PBq+ufPYYW0e8yYWY4Js2rRf2bIlmXPPPYNXXulOixZVvQ7JnIZAEsVo4LwAhpm8ZKYfmyRqXw5Xz/AuHmPyWUZGFn/8sY+zzioLwPDhnTj33DNJTEywCvwKgTwThYi0AS4AKorIvT6jSuO0WGcC9XLx7O7eayC+oXexGJPPfvwxicTEGRw6lMny5YkULx5JhQolGDCgpdehmXziL9UXx3l2IgqI83ntA64NfmiFRNJ32d21u1uSMIXGnj2p9O07gwsumMzy5dtJS8tg8+a9XodlgiDPMwpVnQvMFZEpqvp7CGMqPBY8Cj8+md1/9afexWJMPlFV3n13FffcM5u//04hKiqC+++/gIcfbkeJEsW8Ds8EQSDXKA6KyPNAY+DoLQuqatWZ+vNOG9j2Y3Z/+xHexWJMPrr55o94912nrfa2bWswfvzlNG5cyeOoTDAFcpXpbeBXoBbwOLAZWBTEmMLf/P8cmyT67oDz7/EuHmPyUZcudYmPj+W1167k2297W5IoAkRV/U8gskRVzxeRFara1B02V1UvDkmEOSQkJOjixYu9WHVgVr8Jn92S3T9gL0Rbq1wmfM2Zs4mNG3fTp08C4BQ97dlj1YCHG3dfnnAq8wZS9JTuvm8TkcuBPwFr1Tw3066CjZ9k9/debUnChK3t2w9w771f8M47K4mOjqRjx9rUqVMeEbEkUcQEkiieEpEywCCc5ydKA3cHM6iwtPOXY5PEjd9DfCPv4jHmFGVlKRMnLuHBB+eQnHyImJgoHn20nbVXXYSdMFGo6pEnw5KBS+Dok9nG1xtNsrvvSbeqwk1YWr78L/r0mcHChU5txl271mXMmG7Urm0NaBVl/h64iwSux6nj6XNVXSUi3YEhQCxwbmhCDAOrpmR3d33TkoQJWw88MIeFC/+gSpU4Ro7swjXXNEKsAa0iz98ebTJQHfgJGCUivwNtgAdVdVoIYgsfvrXBnv0P7+Iw5iSpKgcPplOypFN7wKhRXZgwYTGPP34JpUtHexydKSj8JYoEoKmqZolIDLATqKuqf4UmtDCx5evs7q5veheHMSfp99/3MnDgZ6SkpDNnTi9EhAYNKvDSS128Ds0UMP4SxWFVzQJQ1TQRWWdJIoc96+H9Dtn9jW7Oe1pjCoj09ExeeulHHn98LgcPphMXV5z163dTv36816GZAspfomgoIivcbgHquP0C6JFnKoq01+pnd9/wHVhZringFizYQmLiTFat+huAnj0bM2JEZ6pUifM4MlOQ+UsUdm+nP4f3Z3c3/idUtRvBTME2cOAsxoxxKlWoXbscY8d2o0uXuh5HZcKBv0oBrSLAvKjC6NLZ/V2meBaKMYGqWLEkxYpFMHjwhQwZ0pbYWKvAzwQmqC2KiEgXEVkrIhtE5ME8pmkvIstE5BcRmRvMePJFViaM8Nls1TypycSYE/r115188cXGo/2DB1/IihV9efLJSy1JmJMStBv+3ecwxgKdcNraXiQi01V1tc80ZYFxQBdV3SIiBbt2saxMeMlnk8XEw/Vf5z29MR5ITU3nmWfmM3z4AsqWjeHXXwdQvnws0dFRNGxYwevwTBgKKFGISCxQQ1XXnsSyWwIbVHWTu4ypQA9gtc80NwEfqeoWAFX9+ySWH3rf52jjuv9Ob+IwJg9ffLGRfv1msnHjHgCuvLKB3WNhTtsJi55E5ApgGfC5299cRKYHsOyqwFaf/iR3mK/6QDkR+VZElojILRRUmYdh4dPZ/fdmeheLMTls27afG274gM6d32Ljxj00blyR+fNvZdKkKylXzirwM6cnkDOKoThnB98CqOoyEakZwHy5HcfkrNM8Cjgf6IBTLcgPIvKjqq47ZkEidwB3ANSoUSOAVQfBtB7Z3b1Xg1iD8abg+L//+x8//phEbGwUQ4e25557WlOsmDVtb/JHIHu7DFVNPoVlJ+FUAXJENZwqynNO87mqpqjqTmAe0CznglR1oqomqGpCxYoVTyGUfLD5c+c9toLVCmsKBN+2ZJ59tgPdu9dn9er+PPDAhZYkTL4KJFGsEpGbgEgRqScio4HvA5hvEVBPRGqJSHHgBiBnkdUnQFsRiRKREkArYM1JxB8a396X3d1rmWdhGAOwf/8h7rnnc/r0mXF02MUX1+TTT2+kZs2y3gVmCq1AEsVAnPayDwHv4FQ3fveJZlLVDGAAMBtn5/8/Vf1FRBJFJNGdZg3OtY8VOJUPTlLVVafwOYJryYvZ3XE5L7MYExqqyocfrqZRo7G8/PJCXn99GZs37/U6LFMEBHKNooGqPgQ8dLILV9VZwKwcwybk6H8eeP5klx0yKydnd/ec510cpkj77bc9DBjwGbNmrQegZcuqTJhwuZ1BmJAIJFGMEJEzgfeBqar6S5BjKjhU4Yvbs/urtfUuFlMkqSrPPbeAxx+fS2pqBmXKRDNsWAfuuON8IiPthgoTGoG0cHeJiJyB04jRRBEpDbynqk8FPTqvTbsyu/uqT72LwxRZIsK6dbtITc3gxhubMGJEZ844o5TXYZkiRnzvnDjhxCLnAA8APVW1eNCi8iMhIUEXL14c/BWteRdm3ZTdPyjw7WTM6di58yB//XWAJk0qHe1funQbnTrV8TgyE85EZImqJpzKvIE8cNdIRIaKyCpgDM4dT9VOZWVhQ/XYJHHH1rynNSafqCpTpiyjYcMxXHfd+xw+7DzUWaFCCUsSxlOBXKN4HXgXuExVcz4HUTjt86k496rpEFe486Lx3po1O0hMnMm8ec5vr1mzM9izJ5XKla2YyXgvkGsUrUMRSIGyeXZ2d50rvIvDFHoHD6bz9NPzeP7570lPz6JixRKMGNGZm28+B7FKmkwBkWeiEJH/qer1IrKSY6veKNwt3GkWzEl0uqPLehqKKdxUlUsvfYOFC/8AoE+f8xk2rIPVzWQKHH9nFHe5791DEUiB8Wqt7O7uU72LwxR6IkK/fi04eDCdV17pTps21U88kzEeyPNitqpuczv7qervvi+gX2jCC7Ffp8L+Ldn9NTt7F4spdDIzsxg9eiEjRvxwdFivXk1ZsuQOSxKmQAvkiZ1OuQzrmt+BeG7NOzDzxuz+u1K9i8UUOosX/0mrVpO4887PGTLkK/7802lzXUSsAj9T4Pm7RtEX58yhtois8BkVBywIdmAhN+vm7O5eyyAqxrNQTOGRnJzGww9/zdixi1CF6tVLM3p0V6pUifM6NGMC5u8axTvAZ8AwwLe96/2qujuoUYXaiz53l/xjCVQ6rqZzY06KqvL++6u5++7P2bbtAJGRwj33tOaxx9pTqpQnz6oac8r8JQpV1c0i0j/nCBEpX2iSxYEcj4ZUPs+bOEyh88orS9i27QCtW1djwoTLadbsDK9DMuaUnOiMojuwBOf2WN+buhWoHcS4QucVn2rDrXlTcxoOHcpg7940KlcuhYgwblw3vv12M//+9/lERNgzESZ85ZkoVLW7+14rr2nCXubh7O5GN1vzpuaUzZ27mcTEmVSpEsecOb0QERo0qECDBhW8Ds2Y0xZIXU8XikhJt/sfIjJCRDxquDqfzfSpz6nbW97FYcLWjh0p9O49jfbt3+DXX3eydWsy27eneB2WMfkqkEPo8cBBEWmGU3Ps78CbQY0qFDLSYP2HXkdhwlRWljJ58s80bDiWN95YTnR0JI8/3p4VK/paNeCm0AmkUsAMVVUR6QGMVNXJIvLPYAcWdCN9qkmw2mHNSVBVOnd+izlzNgHQsWNtxo3rRr168R5HZkxwBJIo9ovIf4BeQFsRiQSKBTesIJvj82B5k9usdlhzUkSEtm1rsHLldl56qTM33NDEKvAzhdoJGy5yW7e7CVikqvPd6xPtVfW/oQgwp3xpuGhMOTi01+m2BolMAGbOXEd6ehZXXdUQcO5wSk3NoGxZezDThIfTabgokGrG/xKRt4EWItId+MmrJJEvkuZnJ4kuU7yMxISBpKR93HXX53z00RoqVChBu3ZnUb58LNHRUURHB3JCbkz4C+Sup+uBn4DrcNrNXigi1wY7sKBI3Q3vtcvub3CDd7GYAi0jI4uXXvqBRo3G8tFHayhZshhDhlxE6dLRXodmTMgFckj0ENBCVf8GEJGKwBzgg2AGFhTjK2Z3dxgHUfanN8f76ac/6NNnBsuW/QXA1Vc3ZOTILlSvXsbjyIzxRiCJIuJIknDtIrDbagsWzXJeAGVqQfO+3sZjCqSsLOXWWz9h9eod1KhRhjFjunLFFQ28DssYTwWSKD4Xkdk47WYD9ARmBS+kINn+c3b3rb96F4cpcFSVQ4cyiYmJIiJCGDu2G599tp5HH72YkiWtAj9jArmYfb+I/B9wEU59TxNV9eOgR5bftv3ovBcvDZH25zeODRt206/fTKpXL83kyT0AaN++Ju3b1/Q2MGMKEH/tUdQDXgDqACuB+1T1j1AFlu/2/e68ly4ctY+Y03PoUAbDhy/gmWfmc+hQJuXLx/LccweJjy/hdWjGFDj+rjW8BswArsGpQXZ0SCIKlsUvOO/VL/U2DuO5r7/+jaZNJ/DYY99y6FAm//xnM379tb8lCWPy4K/oKU5VX3W714rIz36mDR/1w/POXnP6MjOzuPXWT3jzTafBxgYN4pkwobsVMxlzAv4SRYyInEt2OxSxvv2qGj6JY51P5X9VL/IuDuOpyMgIoqIiiImJ4uGH23LffRfYQ3PGBCDPKjxE5Bs/86mqelKGc0pVePg2dWpVdhQpK1duJy0tgxYtnAaqdu06yN69adSpU97jyIwJraBU4aGql5x6SAXIaJ+HpFo/7F0cJqRSUg4zdOi3vPTSj9SrF8/y5YkULx5JfHwJuxZhzEkq3OfdmgWH92X3X/ikd7GYkJk+fS0DB37Gli3JiEDHjrVIT8+kePFIr0MzJiwF9QlrEekiImtFZIOIPOhnuhYikpnvdUj95VNEdW9Wvi7aFDxbtiRz1VVT6dFjKlu2JHPeeWfy00//ZvTobvbgnDGnIWhnFG67FWOBTkASsEhEpqvq6lymGw7Mzvcg1vg0b2rtBRRqmZlZtG8/hd9+20tcXHGeeupS+vVrQVRU+NU2Y0xBc8JEIU6LLDcDtVX1Cbc9ijNU9acTzNoS2KCqm9zlTAV6AKtzTDcQ+BBocbLBn9BS99GPml3yfdGmYFBVRITIyAiGDm3Pp5+u4+WXO1O1ammvQzOm0AjkcGsc0Aa40e3fj3OmcCJVAd82RpPcYUeJSFXgamCCvwWJyB0islhEFu/YsSOAVQNpe7K7m/cPbB4TNvbsSSUxcQbPPDP/6LBevZry/vvXWZIwJp8FUvTUSlXPE5GlAKq6R0QCKfDNrawn572pLwODVTXTX1OSqjoRmAjO7bEBrBt+HpndXad7QLOYgk9Veeedldx77xf8/XcKcXHFGTCgJWXKxFhzpMYESSCJIt29jqBwtD2KQK4MJwHVffqrAX/mmCYBmOr+wSsA3UQkQ1WnBbB8/zbNcN4rnHPaizIFw7p1u+jXbyZfffUbAG3b1mD8+MspU8aaIzUmmAJJFKOAj4FKIvI0cC0QyAMJi4B6IlIL+AO4Aaft7aNUtdaRbhGZAszIlyQBsGO5817bzibCXUZGFk89NY9hw77j8OFM4uNjef75TvTu3dzOIowJgUCqGX9bRJYAHXCKk65S1TUBzJchIgNw7maKBF5T1V9EJNEd7/e6xGnLynDe610d1NWY4IuMFObP38Lhw5n861/NGT68ExUq2ENzxoRKnlV4HJ3AucvpOKq6JSgRnUBAVXh89zAsfNrpvicdIgr3c4WF0fbtB0hLy+Css8oCsH79LrZtO0C7dmd5G5gxYSooVXj4mIlzfUKAGKAWsBZofCorDIl1Ps15W5IIK1lZysSJS3jwwTkkJFThyy97ISLUqxdPvXrxXodnTJEUSNHTMVeDReQ8oE/QIsoPe9Y675eM8jYOc1KWLfuLxMQZLFzotI9VvHgkBw4cJi4u2uPIjCnaTvpwW1V/FpH8fzguv6jPDVlVL/AuDhOw/fsP8dhj3zJy5EKyspQqVeIYObIL11zTyC5WG1MABPJk9r0+vRHAeUCAT7154Kfh2d2VzvUuDhOQw4czOe+8iWzYsJuICOGuu1rxxBOXULq0nUUYU1AEckYR59OdgXPN4sM8pvXeoueyu8Xq+SnoihePpFevpnz66TomTLic88+v4nVIxpgc/CYK90G7Uqp6f4jiOX2H9jrv13zuaRgmd+npmbz00o/UqFGGG25oAsCDD17EQw+1JTLSErsxBVGeiUJEotxnIc4LZUCnJf1gdnflgnsZpahasGALiYkzWbXqbypWLEH37vUpVaq4tRNhTAHn74ziJ5zrEctEZDrwPpByZKSqfhTk2E7e0jHZ3bHW1GVBsXt3KoMHf8mkSUsBqF27HOPGdaNUKWsjwphwEMg1ivLALuBSsp+nUKDgJYoM94wippy3cRjAqcDvzTdXMGjQF+zceZBixSIYPPhChgxpS2xsMa/DM8YEyF+iqOTe8bSK7ARxRGA1uIbaysnOe/OB3sZhAEhPz2LYsO/YufMgF198FuPHX06jRhW9DssYc5L8JYpIoBSBVRdeMBxIct6jrDZRr6SmpnP4cCZlysRQvHgkEyd2Z9OmPdxySzN7JsKYMOUvUWxT1SdCFsnp+vGp7O5zbvMujiJs9uwN9Os3i/btz2Ly5B4AtG17Fm3bWv1MxoQzf4kivA7/FjyS3V2ikndxFEHbtu3nnntm8957vwBQsmQxDh5Mp0QJuw5hTGHg78b1DiGLIj9El3XeLx3taRhFSWZmFmPG/ETDhmN5771fiI2NYvjwjixZcoclCWMKkTzPKFR1dygDOS1ZmdkP2jW8ye+kJn+kpWXQrt3rLFrkNFrYvXt9Ro/uSs2aZb0NzBiT7wpHHdy/f5Hdbc9PhERMTBRNmlRi27YDjBrVhauuamgXq40ppApHolj8gtcRFHqqykcfraFy5VJcdJHTltWIEZ2JjBSrBtyYQq5wJIptPznvFc7xP505Jb/9tocBAz5j1qz1NGxYgWXL+hAdHUXZsnYbsjFFQeFIFOkHnPdLRnobRyFz+HAmL774PU8+OY/U1AzKlInmrrtaERVllfcZU5SEf6LY/0d2d4Um3sVRyMyf/zuJiTNZvdppeuSmm87hxRcv44wzSnkcmTEm1MI/UexYnt1dwqqHyA+pqelce+37/P13CnXrlmfcuG506lTH67CMMR4J/0SxfbHzXr29p2GEO1UlM1OJioogNrYYI0Zcxrp1u/jPf9oSExP+PxNjzKkL/z3AnnXOu1ibBqdq9eodJCbOoFOn2jzyyMUA3HxzU4+jMsYUFOF/VTL5N+fd2sc+aQcPpjNkyFc0azaB+fO3MGnSUg4dyvA6LGNMARP+ZxQpfznvpWt6Gka4+eyz9fTvP4vfftsLQJ8+5zNsWAeio8P/J2GMyV/hv1dI3uS8V7RnKAKRknKY3r0/4YMPVgPQtGllJky4nDZtqnscmTGmoArvRJG2J7u7ygXexRFGSpQoxu7dqZQsWYzHH2/PXXe1tucijDF+hXeiOPBndndEeH+UYFq8+E/Klo2hbt3yiAiTJl1BZGQENWqU8To0Y0wYCO9Dycw0572i3aGTm+TkNAYOnEXLlq+SmDgDVadhwlq1ylmSMMYELLwPw488lR1Vwts4ChhV5X//+4W7757NX38dIDJSOO+8M8nIyKJYMbuN2BhzcsI7UWiW837gD//TFSEbN+6mf/9ZzJ69EYA2baoxYUJ3mjat7HFkxphwFd6J4i+31li7kA3A/v2HSEh4lb170yhbNobhwzty++3nERFh7UQYY05dUBOFiHQBRgKRwCRVfTbH+JuBwW7vAaCvqi4nUDtXOe+pO/Mh2vAXFxfNPfe0ZsOG3bzwwmVUqlTS65CMMYVA0BKFiEQCY4FOQBKwSESmq+pqn8l+Ay5W1T0i0hWYCLQ66ZXV6ZEPEYefHTtSuP/+L+nQoRa9ejUD4JFH2llLc8aYfBXMu55aAhtUdZOqHgamAsfs0VX1e1U98jDEj0C1k1rDpk+d9/L1TzfWsJKVpUya9DMNGozhjTeW89BDX5OenglgScIYk++CWfRUFdjq05+E/7OF24DPchshIncAdwDUqOE0w8m+37MnKFd0EsWqVX+TmDiDBQucTduxY23GjetmdzMZY4ImmIkit0NbzXVCkUtwEsVFuY1X1Yk4xVIkJCQ4y1j/UfYEZWqdVqDhIDU1naFDv2XEiB/JyMiicuWSvPRSZ264oYmdRRhjgiqYiSIJ8K1AqBrwZ86JRKQpMAnoqqq7Al76n9+7S734dGIMGxERwvTp68jMzKJfvwSefrqDtVltjAmJYCaKRUA9EakF/AHcANzkO4GI1AA+Anqp6rqTWnpsJec9/ux8CLVgSkraR4kSxShfPpbo6CimTHEu8bRqdXKXcowx5nQE7WK2qmYAA4DZwBrgf6r6i4gkikiiO9mjQDwwTkSWicjigFfwy2vOe+WE/Ay7QMjIyOKll36gUaOx3H//F0eHt2pVzZKEMSbkgvocharOAmblGDbBp/t24PZTWniGW89TdOlTjq8gWrgwiT59ZrB8+XYAkpMPkZGRZTW8GmM8E55PZmdlZnef1cm7OPLR3r1pDBnyFRMmLEYVzjqrDGPGdKN796JzR5cxpmAKz0SRsi27Ozr8a0HdsyeVs88ex19/HSAqKoJBg9rwyCPtKFmyuNehGWNMmCaK3z73OoJ8Va5cLF271mXdul2MH38555xjFfgZYwqO8EwUm93n8kqF54XdQ4cyGD58ARdffBYXX1wTgDFjuhETE2UV+BljCpzwTBRb5zrvjW7yP10B9PXXv9G370zWrdtFo0YVWLmyL5GREZQoUczr0IwxJlfhmSiy0p33mp29jeMk/P13CoMGfcFbb60AoGHDCowbdzmRkXY3kzGmYAvPRHGkJpDK53sbRgCOVOA3ePAc9u5NIyYmiocfbsv9919I8eJWP5MxpuALz0RxeL/zHhntbRwBSE5O46GHvmbv3jQ6d67D2LHdqFOnvNdhGWNMwMIvUajPMxQFNFGkpBwmKiqC6OgoypWLZcKEy8nMVK677myrwM8YE3bCr4A8Mz27uwDudKdPX8vZZ4/juecWHB12zTVnc/31jS1JGGPCUvglCs1w3mPKeRtHDlu2JHPVVVPp0WMqW7YkM3v2RrKycq1V3Rhjwkr4JYojdTzFVvQ2Dld6eiYvvPA9jRqN5ZNP1hIXV5yRI7swd25veybCGFMohN81iszDzntMvLdxADt3HqRDh/+yYoVTgd91153NSy91pmrVwlVRoTGmaAu/RJHlJoricd7GAcTHx1KhQglq1SrLmDHd6NatntchmQIkPT2dpKQk0tLSvA7FFCExMTFUq1aNYsXy7yHeMEwU7sXssrVDvmpV5e23V9KyZVXq149HRHjrraspUybGnqw2x0lKSiIuLo6aNWvajQwmJFSVXbt2kZSURK1a+ddEdPhdozi0z3mv2jakq127dicdO75Jr14f06/fTFSdC9VnnhlnScLkKi0tjfj4eEsSJmREhPj4+Hw/iw2/M4ojKjUPyWrS0jIYNmw+zz67gMOHM4mPj+Uf/2gaknWb8GdJwoRaMH5z4ZsoyjcM+irmzNlE374z2bBhNwD/+ldznnuuE/HxJYK+bmOMKSjCr+jpCAlu6Nu3H6B793fYsGE3Z59dkXnzejN5cg9LEiasREZG0rx5c5o0acIVV1zB3r17j4775ZdfuPTSS6lfvz716tXjySefPFqkCvDZZ5+RkJBAo0aNaNiwIffdd58Hn8C/pUuXcvvtp9aacigcOnSInj17UrduXVq1asXmzZtzne7dd9/lnHPOoWnTpnTp0oWdO3cCsGXLFi655BLOPfdcmjZtyqxZTsvSO3bsoEuXLqH6GM7Fj3B6nV8N1ckNNBgyM7M0KyvraP/w4d/psGHz9dChjKCszxRuq1ev9joELVmy5NHuW265RZ966ilVVT148KDWrl1bZ8+eraqqKSkp2qVLFx0zZoyqqq5cuVJr166ta9asUVXV9PR0HTt2bL7Glp6eftrLuPbaa3XZsmUhXefJGDt2rPbp00dVVd999129/vrrc42pYsWKumPHDlVVvf/++/Wxxx5TVdV///vfOm7cOFVV/eWXX/Sss846Ol/v3r31u+++y3W9uf32gMV6ivvd8Cx62rc53xe5bNlfJCbOoH//FvTq1QyABx64MN/XY4qoF4N0rWJQ4E//t2nThhUrnGru33nnHS688EIuu+wyAEqUKMGYMWNo3749/fv357nnnuOhhx6iYUOniDcqKop+/fodt8wDBw4wcOBAFi9ejIjw2GOPcc0111CqVCkOHDgAwAcffMCMGTOYMmUKvXv3pnz58ixdupTmzZvz8ccfs2zZMsqWLQtA3bp1WbBgARERESQmJrJlyxYAXn75ZS688Nj/4/79+1mxYgXNmjn/159++om7776b1NRUYmNjef3112nQoAFTpkxh5syZpKWlkZKSwqeffsrAgQNZuXIlGRkZDB06lB49erB582Z69epFSkoKAGPGjOGCCy4IePvm5pNPPmHo0KEAXHvttQwYMABVPeY6wpGdcUpKCvHx8ezbt4+6desCzvWGffucG3iSk5OpUqXK0fmuuuoq3n777eO2SzCEZ6IoXTPfFrV//yEee+xbRo5cSFaWcuhQJv/4R1O7CGkKlczMTL766ituu+02wCl2Ov/8Y6vpr1OnDgcOHGDfvn2sWrWKQYMGnXC5Tz75JGXKlGHlypUA7Nmz54TzrFu3jjlz5hAZGUlWVhYff/wxt956KwsXLqRmzZpUrlyZm266iXvuuYeLLrqILVu20LlzZ9asWXPMchYvXkyTJk2O9jds2JB58+YRFRXFnDlzGDJkCB9++CEAP/zwAytWrKB8+fIMGTKESy+9lNdee429e/fSsmVLOnbsSKVKlfjyyy+JiYlh/fr13HjjjSxevPi4+Nu2bcv+/fuPG/7CCy/QsWPHY4b98ccfVK9eHXCSbZkyZdi1axcVKlQ4Ok2xYsUYP34855xzDiVLlqRevXqMHTsWgKFDh3LZZZcxevRoUlJSmDNnztH5EhISePjhh0+4vfNDeCaKM1ue9iJUlWnTfuXOOz8nKWkfERHCXXe14oknLrEkYfLfSRz556fU1FSaN2/O5s2bOf/88+nUqRPAcUe1vk7m9z9nzhymTp16tL9cuRPXwXbdddcRGem0xdKzZ0+eeOIJbr31VqZOnUrPnj2PLnf16tVH59m3bx/79+8nLi77Qdtt27ZRsWJ2VT7Jycn885//ZP369YgI6enZFYh26tSJ8uWd6v2/+OILpk+fzgsvvAA4tzFv2bKFKlWqMGDAAJYtW0ZkZCTr1q3LNf758+ef8DMeoXr8955z+6anpzN+/HiWLl1K7dq1GThwIMOGDePhhx/m3XffpXfv3gwaNIgffviBXr16sWrVKiIiIqhUqRJ//vlnwLGcjvBMFKdZvfjOnQe59dZPmDHD+SEkJFThlVe6c955Z+ZHdMYUGLGxsSxbtozk5GS6d+/O2LFjufPOO2ncuDHz5s07ZtpNmzZRqlQp4uLiaNy4MUuWLDlarJOXvBKO77Cc9/SXLFnyaHebNm3YsGEDO3bsYNq0aUePkLOysvjhhx+IjY31+9l8l/3II49wySWX8PHHH7N582bat2+f6zpVlQ8//JAGDRocs7yhQ4dSuXJlli9fTlZWFjExMbmu92TOKKpVq8bWrVupVq0aGRkZJCcnH01YRyxbtgxwzugArr/+ep599lkAJk+ezOeff350W6WlpbFz504qVapEWlqa3+2Tn8LzrqcydU5r9ri44mzYsJvSpaMZM6YrP/54myUJU6iVKVOGUaNG8cILL5Cens7NN9/Md999d7QoIzU1lTvvvJMHHngAgPvvv59nnnnm6FF1VlYWI0aMOG65l112GWPGjDnaf6ToqXLlyqxZs+Zo0VJeRISrr76ae++9l0aNGhEfH5/rco/sTH01atSIDRs2HO1PTk6matWqAEyZMiXPdXbu3JnRo0cfPdpfunTp0fnPPPNMIiIiePPNN8nMzMx1/vnz57Ns2bLjXjmTBMCVV17JG2+8ATjXai699NLjEmvVqlVZvXo1O3bsAODLL7+kUaNGANSoUYOvvvoKgDVr1pCWlnb0LGrdunXHFL0FU3gmioiTb0J0wYIt7Np1EIDo6CimTr2GX3/tT//+La3dalMknHvuuTRr1oypU6cSGxvLJ598wlNPPUWDBg0455xzaNGiBQMGDACgadOmvPzyy9x44400atSIJk2asG3btuOW+fDDD7Nnzx6aNGlCs2bN+OabbwB49tln6d69O5deeilnnun/IKxnz5689dZbR4udAEaNGsXixYtp2rQpZ599NhMmTDhuvoYNG5KcnHz06P6BBx7gP//5DxdeeGGeO3lwzjzS09Np2rQpTZo04ZFHHgGgX79+vPHGG7Ru3Zp169YdcxZyqm677TZ27dpF3bp1GTFixNEzBYDmzZsDUKVKFR577DHatWtH06ZNWbZsGUOGDAHgxRdf5NVXX6VZs2bceOONTJky5Wii+eabb7j88stPO8ZASG5laAVZQnXRxR8+Cy0HBzT9rl0HefDBOUyatJTbbjuXSZOuDHKExjjWrFlz9MjQBMdLL71EXFxcgX6WIljatWvHJ598kut1odx+eyKyRFUTTmVd4XkoHUDNsarKG28so2HDsUyatJRixSKoUiUu14tLxpjw1LdvX6KjC2aTyMG0Y8cO7r333oBuHsgP4Xkxu1x9v6N//XUniYkzmDv3dwDat6/J+PGX07BhBb/zGWPCS0xMDL169fI6jJCrWLEiV111VcjWF56Jwk/1HUlJ+2jWbAKHD2dSoUIJXnzxMnr1sucijDf83YZqTDAEo9Sk0CWKatVK06tXUyIihGef7Uj58qG5fcyYnGJiYti1a5dVNW5CRt32KPK6tfdUhWmiyL7radu2/dxzz2wSExNo374mABMnXmHtVRvPVatWjaSkpKO3PRoTCkdauMtPYZooIsjMzGL8+MU89NDX7Nt3iA0bdrNo0b8REUsSpkAoVqxYvrYyZoxXgnrXk4h0EZG1IrJBRB7MZbyIyCh3/AoROS+Q5f686gCtW09m4MDP2LfvEFdcUZ8PP7zeTu+NMSYIgnZGISKRwFigE5AELBKR6aq62meyrkA999UKGO++52nr3tK06PYjWVnO9YjRo7vSo0cDSxLGGBMkwTyjaAlsUNVNqnoYmAr0yDFND+C/bnXpPwJlRcTvY5y7D8YiItx7b2vWrOnPVVc1tCRhjDFBFMxrFFWBrT79SRx/tpDbNFWBY+oKEJE7gDvc3kPw2KoRIyCXqmeKmgrATq+DKCBsW2SzbZHNtkW2BieeJHfBTBS5HebnvME3kGlQ1YnARAARWXyqj6EXNrYtstm2yGbbIptti2wicnzjGgEKZtFTElDdp78akLPy9ECmMcYY46FgJopFQD0RqSUixYEbgOk5ppkO3OLe/dQaSFbV46uoNMYY45mgFT2paoaIDABmA5HAa6r6i4gkuuMnALOAbsAG4CBwawCLnhikkMORbYtsti2y2bbIZtsi2ylvi7CrZtwYY0xohWc148YYY0LGEoUxxhi/CmyiCFb1H+EogG1xs7sNVojI9yLSzIs4Q+FE28JnuhYikiki14YyvlAKZFuISHsRWSYiv4jI3FDHGCoB/EfKiMinIrLc3RaBXA8NOyLymoj8LSKr8hh/avtNVS1wL5yL3xuB2kBxYDlwdo5pugGf4TyL0RpY6HXcHm6LC4BybnfXorwtfKb7GudmiWu9jtvD30VZYDVQw+2v5HXcHm6LIcBwt7sisBso7nXsQdgW7YDzgFV5jD+l/WZBPaMISvUfYeqE20JVv1fVPW7vjzjPoxRGgfwuAAYCHwJ/hzK4EAtkW9wEfKSqWwBUtbBuj0C2hQJx4tT3UwonUWSENszgU9V5OJ8tL6e03yyoiSKvqj1OdprC4GQ/5204RwyF0Qm3hYhUBa4GJoQwLi8E8ruoD5QTkW9FZImI3BKy6EIrkG0xBmiE80DvSuAuVc0KTXgFyintNwtqexT5Vv1HIRDw5xSRS3ASxUVBjcg7gWyLl4HBqppZyCuLDGRbRAHnAx2AWOAHEflRVdcFO7gQC2RbdAaWAZcCdYAvRWS+qu4LcmwFzSntNwtqorDqP7IF9DlFpCkwCeiqqrtCFFuoBbItEoCpbpKoAHQTkQxVnRaSCEMn0P/ITlVNAVJEZB7QDChsiSKQbXEr8Kw6BfUbROQ3oCHwU2hCLDBOab9ZUIuerPqPbCfcFiJSA/gI6FUIjxZ9nXBbqGotVa2pqjWBD4B+hTBJQGD/kU+AtiISJSIlcGpvXhPiOEMhkG2xBefMChGpjFOT6qaQRlkwnNJ+s0CeUWjwqv8IOwFui0eBeGCceySdoYWwxswAt0WREMi2UNU1IvI5sALIAiapaq63TYazAH8XTwJTRGQlTvHLYFUtdNWPi8i7QHuggogkAY8BxeD09ptWhYcxxhi/CmrRkzHGmALCEoUxxhi/LFEYY4zxyxKFMcYYvyxRGGOM8csShSmQ3Jpfl/m8avqZ9kA+rG+KiPzmrutnEWlzCsuYJCJnu91Dcoz7/nRjdJdzZLuscmtDLXuC6ZuLSLf8WLcpuuz2WFMgicgBVS2V39P6WcYUYIaqfiAilwEvqGrT01jeacd0ouWKyBvAOlV92s/0vYEEVR2Q37GYosPOKExYEJFSIvKVe7S/UkSOqzVWRM4UkXk+R9xt3eGXicgP7rzvi8iJduDzgLruvPe6y1olIne7w0qKyEy3bYNVItLTHf6tiCSIyLNArBvH2+64A+77e75H+O6ZzDUiEikiz4vIInHaCegTwGb5AbdCNxFpKU5bJEvd9wbuU8pPAD3dWHq6sb/mrmdpbtvRmON4XX+6veyV2wvIxKnEbRnwMU4tAqXdcRVwniw9ckZ8wH0fBDzkdkcCce6084CS7vDBwKO5rG8KbtsVwHXAQpwK9VYCJXGqpv4FOBe4BnjVZ94y7vu3OEfvR2PymeZIjFcDb7jdxXFq8owF7gAedodHA4uBWrnEecDn870PdHH7SwNRbndH4EO3uzcwxmf+Z4B/uN1lcep9Kun1922vgv0qkFV4GAOkqmrzIz0iUgx4RkTa4VRHURWoDPzlM88i4DV32mmqukxELgbOBha41ZsUxzkSz83zIvIwsAOnFt4OwMfqVKqHiHwEtAU+B14QkeE4xVXzT+JzfQaMEpFooAswT1VT3eKuppLdIl8ZoB7wW475Y0VkGVATWAJ86TP9GyJSD6c20GJ5rP8y4EoRuc/tjwFqUDjrgDL5xBKFCRc347RMdr6qpovIZpyd3FGqOs9NJJcDb4rI88Ae4EtVvTGAddyvqh8c6RGRjrlNpKrrROR8nDpzhonIF6r6RCAfQlXTRORbnGqvewLvHlkdMFBVZ59gEamq2lxEygAzgP7AKJy6jL5R1avdC//f5jG/ANeo6tpA4jUG7BqFCR9lgL/dJHEJcFbOCUTkLHeaV4HJOE1C/ghcKCJHrjmUEJH6Aa5zHnCVO09JnGKj+SJSBTioqm8BL7jrySndPbPJzVScytja4lRkh/ve98g8IlLfXWeuVDUZuBO4z52nDPCHO7q3z6T7cYrgjpgNDBT39EpEzs1rHcYcYYnChIu3gQQRWYxzdvFrLtO0B5aJyFKc6wgjVXUHzo7zXRFZgZM4GgayQlX9GefaxU841ywmqepS4BzgJ7cI6CHgqVxmnwisOHIxO4cvcNo2nqNO053gtCWyGvhZRFYBr3CCM343luU41Wo/h3N2swDn+sUR3wBnH7mYjXPmUcyNbZXbb4xfdnusMcYYv+yMwhhjjF+WKIwxxvhlicIYY4xfliiMMcb4ZYnCGGOMX5YojDHG+GWJwhhjjF//D8mfoIyElRuSAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "# Assuming y_true and y_pred are your data\n",
    "# y_true = true binary labels (either 0 or 1)\n",
    "# y_pred = predicted probabilities that each entry in y_true is a 1\n",
    "test_y_true = test__op_ind_df['true']\n",
    "test_y_pred = test__op_ind_df['pred']\n",
    "# Compute ROC curve and ROC area (AUC)\n",
    "fpr, tpr, _ = roc_curve(test_y_true, test_y_pred)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "\n",
    "plt.figure()\n",
    "lw = 2  # Line width\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver Operating Characteristic Curve')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.savefig(f'ROC_curve.pdf-mimiciv with ss random seed 2', format='pdf')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8d8998cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAArf0lEQVR4nO3deXTc5XX/8ffVZsmWvOB9l8EGywbbGGP2zWw2TQu0SQMhG0lLaZI2XX49yenpr0mbLml62iY5ISUUCM1JGn4ECDFlcRIggDFgMNjGC2BjvMiLjCXvu6z7++N+hcayPBrJGo1m9Hmd8z2e+c4zM898Jc/Vs93H3B0REZGTKcp1BUREpGdToBARkbQUKEREJC0FChERSUuBQkRE0lKgEBGRtBQopF1mdpuZ/TKDcneb2f/tjjp1BzNbb2bXJLe/bmY/znWdRHJBgSLPJV9mB81sn5nVmdkPzayyK9/D3X/i7tdlUO5Od/9GV753MzNzM9uffM7NZvbvZlacjfcqdGb2gJk1mtmoNs7/Q6tz1cm1L0k59wkzez35WWw1s6fM7NJO1OPPzWybme02s/vNrE+asqk//31mdm/KY58xsyVmtsfMas3sW6n1lVOnQFEYftvdK4GZwPnA37QuUCD/caYnn/MK4OPA53Jcny7VHT8jM+sH/B6wG7itE8//C+DbwD8Bw4FxwPeBGzv4OtcDXwWuBqqB04G/a+dp0929Mjn+IOV8X+DPgCHABclr/p+O1EfSU6AoIO6+GXgKOBs+/Cvsi2a2BliTnPuImS01s11mtsjMpjU/38zGmtmjZvaBmdWb2feS8581s4XJbTOz/zCz7clfgsvNrPn9jvuL1Mz+0MzWmlmDmc1P/Qs2qdudZrbGzHaa2V1mZhl+zrXAS8CMlNfrzOc6w8yeTc7tMLOfmNnADl725ve4MXn/PWb2npnNTc5/2H2V3P+wCyvlr/XPm9lG4Fkze9rMvtTqtZeZ2e8mtyeb2a+Sa/qOmf1+B6v6e8Au4O+Bz3TwMw5InvdFd3/U3fe7+1F3f9zd/6qD9fgMcJ+7r3T3ncA3gM928DUAcPf/dPcX3f1I8n/gJ8AlnXktaZsCRQExs7HADcCbKadvIv7KmmJmM4H7gT8CBgM/AOabWZ+kG+d/gQ3EX3ijgQfbeJvrgMuBM4GBxF/29W3UZQ7wz8DvAyOT1239eh8hWkDTk3LXZ/g5JwOXAWuT+539XJbUcRRQA4wFvp5JHVrVZzbwI+CviGtyObC+Ay9xRfL+1wP/A9ya8tpTgPHAE0lr4FdJmWFJue+b2dSk7CfMbHk77/UZ4KfENZicXLtMXQSUAz8/WYGkDrvSHOOSolOBZSlPXQYMN7PBad7/haSr6lEzq05T7nJgZWYfSTLi7jry+CC+kPYRfyVuILoBKpLHHJiTUvY/gW+0ev47xBfVRcAHQEkb7/FZYGFyew7wLnAhUNSq3APAPyS37wO+lfJYJXAUqE6p26Upjz8EfDXN53RgD7A/uf1ToM+pfK423uMm4M1W1/aa5PbXgR+f5Hk/AP4jzc/nmpT7H74OEbgcOD3l8arkM45P7v8jcH9y++PAi22899cy/F0ZBzQBM5L7C4DvtPXzSznXXMcSoqtqWxf93r4HzE25X5q8T/VJyl8OlBGB+HvAipP8rt4O1AJDuvr/Wm8+1KIoDDe5+0B3H+/uX3D3gymPbUq5PR74y9S/8Ii/okcl/25w98Z0b+TuzxL/Ue8C6szsHjPr30bRUUTgan7ePqLlMTqlzLaU2weIYIKZrUwZtLwspczMpMzHiVZSv1P5XGY2zMwetBgc3wP8mOjn7qixxBdfZ334M3L3vcATwC3JqVuIrhSIz3lBq895GzAiw/f5FLDa3Zcm938CfMLMSpP7jcQXdqpSIrg0ET+/IdY1Yyn7gNTfm+bbe9sq7O4veHQt7QK+DEwgWmEfMrObgG8C89x9RxfUURIKFIUvNT3wJuAfk6DSfPR1958mj43L5EvA3b/r7ucR3QdnEl0urW0hvtiADwdRBwObM3j9qd4yaPliq8fc3R8CXgb+9hQ/1z8T12eau/cHPkl0R3XUJuCMkzy2nxhsbdbWl3rrFM4/BW41s4uACuC5lPd5vtXnrHT3P86wnp8GTk+6b7YB/04ExnnJ4xuJFkSqCcAmd28irvkhouXVJoup1PvSHM1dTyuJLsdm04E6dz+hG/MknJSfVTIm9F/ExI63MnwNyZACRe/yX8CdZnaBhX5m9ltmVgUsBrYC30zOl5vZCQOCZnZ+8vxS4kvwEHCsjff6H+B2M5thMe3xn4BX3X19F32WbwJ3mNmIU/hcVSTddmY2mrYDXibuIz7r1WZWZGajk3EUgKXALWZWamazgI9m8HpPEkH274H/l3xJQ4y1nGlmn0perzT5edSc9JUSSdA5A5hNTAKYQUx6+B9aBrUfAX7LzK4zs2KLyQd/QzKm4+67ieB8l5ndZGZ9kzrMM7NvJWV+khLk2zo2Ju/1I+DzZjbFzAYl7/PASeo+Nfk9KraY+v1vxB8cq5PH5xCto99z98XtX17psFz3fek4tYNWfeCtHnNgYqtzc4HXiDGNrcDPgKrksXHAY0QXww7gu8n5z9IyRnE1sJz4gt1B/AetTB57gJQ+buBOokumgfiSG3OyurV+boaf5Sng307hc00FliSfZSnwl0BtW9eWNGMUyeM3J9dlLzHIfn1y/nTg1eQ9ngC+y4ljFG31td+XPHZ+q/NnJa/zQfJ5nqVlzOE2YOVJ6nc38Egb52cDh4HTkvu/nVyT3UTX4b+SjHmlPOc24HXiD4VtSX0u7sTv7l8AdcTY0w9JxpxSfrZ/ndyeQ4w57Qe2Jz/LSSllnyO6zfalHE/l+v9mIR2WXGgREZE2qetJRETSUqAQEZG0FChERCQtBQoREUkr7xLFDRkyxKurq3NdDRGRvLJkyZId7j60M8/Nu0BRXV3N66+/nutqiIjkFTPb0H6ptqnrSURE0lKgEBGRtBQoREQkLQUKERFJS4FCRETSUqAQEZG0shYozOx+i32VV5zkcTOz71rsqby8g1syiohIN8lmi+IBIvXzycwDJiXHHcR2lhlpamo5lPxWRCS7srbgzt1faGcD9BuBH3nkOX/FzAaa2Uh335rudfftgxdT9jwzg1mzoG/fkz9HREQ6L5crs0dz/H7Otcm5EwKFmd1BtDoYNqya9euhtBSOHoXt22HMGDj99O6osohI75PLQNHW3sRtdiS5+z3APQA1NbN84kQoL4/WxZ492ayiiIjkctZTLTA25f4YYEuO6iIiIieRy0AxH/h0MvvpQmB3e+MTIiLS/bLW9WRmPwWuBIaYWS3wNaAUwN3vBp4EbiA2oj8A3J6tuoiISOdlc9bTre087sAXs/X+IiLSNbQyW0RE0lKgEBGRtAo+UBw5AkuWwLvv5romIiL5Ke+2Qm2Le6TzKGoV9j74ADZsgPXrob4ehg+HAQNyUkURkbxVEIFi7Vqoq4OLL245V1sLmzfDqlURMCCCiYiIdExBdD1t3w7LlkVKD4igUVsLK1bAsWNwzjlQUZHbOoqI5Ku8DxTucOhQBASAAwdg48ZoSQDMmAGHD7eUExGRjsnrQHHgAOzfH/meiosjGKxeDe+9B42NMG1aZJeFyAvVHDxERCRzeR0oiosjIBw8GAPZ69fDzp3RFTV1asvg9vDh0K9fHCIi0jF5PZg9eDBMnx4ti507Y2bT++9DdXVkl01VVnbirCgREWlf3n91jh4dmxY1NsbspsZGGDXq+DLHjsWhtRQiIh2X94ECYmzCHbZtgwkTWsYlmpWXx8K7gwdzUz8RkXyW111PzfbujTUSjY0wZMiJj5tFy+PIke6vm4hIviuIQAHRtXTaaSe2JpodOhRjGa2tWBHrL2pqThzXEBGRAul6gggCI0akL3P4cEydrauL+0uWRHfVq6/G2gsRETlRQbQozjoLBg2CqqqTlxk6NALE2rWwdWs8Z+9eWLMGGho0I0pE5GQKIlAUFbXfmjhyBPr0idQeu3bFDKlNm2KNRVNTTK/duzd9sBER6Y16zd/Rhw/HrKft2yMwbNkCY8fCsGHx+KZNsGBBSyoQEREJBdGiyER1dQSLkpJYwV1RETOh6utjoHvHjhjnOHo0VnyLiEjoNYGipCTSegCMGROL9KBlf4rt2yN41NXB+PG5qaOISE/Ua7qeUvXr1zKNtqQE5syBWbOia2rv3tzWTUSkp+mVgaItzd1NS5bEhkciIhIUKBKnnRYzng4fhqee0qC2iEgzBYoUZWUxTXbfvjhERESB4jjnnx/TZQ8ejFlQIiKiQHEcs5hGW1UVU2YPH45EgyIivVmvmR6bqWPHImX5G29Eq+LwYbjwwphOW6KrJSK9kL76Wjl2DHbvjpQfzXtYNDRErqjLLotxDBGR3kRdT60MGQJnnBEBoq4u1lXU1cHKlTHQLSLS26hF0YYzz4yxiuLiyAm1YkUkFHTPdc1ERLqfWhQnUVYWgWLs2NjUqLERFi/Oda1ERLqfAkUGhg+PQFFXFynKRUR6k6wGCjOba2bvmNlaM/tqG48PMLPHzWyZma00s9uzWZ/OqqiIldv19fDcc7mujYhI98paoDCzYuAuYB4wBbjVzKa0KvZFYJW7TweuBP7NzHrkvKJp0yIF+fbtETBERHqLbLYoZgNr3X2dux8BHgRubFXGgSozM6ASaAB65BK3vn1jRlRDAyxfnuvaiIh0n2wGitHAppT7tcm5VN8DaoAtwFvAl929qfULmdkdZva6mb2+a9cH2apvu2bMiG1XNftJRHqTbAYKa+Nc66/Y64GlwChgBvA9M+t/wpPc73H3We4+a+DAoV1dz4w1NUXLYv36uC0i0htkM1DUAmNT7o8hWg6pbgce9bAWeB+YnMU6nZLy8sgHdeAAvPiigoWI9A7ZDBSvAZPMbEIyQH0LML9VmY3A1QBmNhw4C1iXxTqdsv79o+vprbfgnXdyXRsRkezLWqBw90bgS8ACYDXwkLuvNLM7zezOpNg3gIvN7C3gGeAr7t6jE3yXl8OePdGqWLQoUn1AtC72789t3UREsiGrKTzc/UngyVbn7k65vQW4Lpt16GpnnAEDB8Jrr0Xup8WLYfToSCRYXw/nnhsJBEVECoVyPXXC4MEwbhzU1sL778OaNdEltWlTZJwdORIGDIARI6BfvxjXaLZ3b6zyHjQod/UXEekIBYpOmjQpZj9t3QqlpdGiaGqKVsa2bREcBg6Mozk9+apVUa62FubOjQ2SRER6OgWKTiotjS/7AwciBfm4cTHAXV/fMlaxa1ckFty+HSZOjLGN996L8wcOKFCISH5QoDgFRUVQWQkXXBD33SMd+eDBsQFSXR0sXRotjaNHI1BUVMRRWxvltGueiPR0yh7bhUaNii9/iJbEqFFw3XXRwti6Fc46K8Ym9u2LPS6eeCK39RURyYT+ns2ykhKYNy8Gufv0iTGLVauiRXHgQIxpDBx4/IC3iEhPohZFNzCLIAHRXTVvXoxpHDwIv/41bNiQ2/qJiKSjQJEjVVVw+HAEidraXNdGROTkFChyZMwYmDo1Zk+tXx9dUyIiPZECRY6YtWyxWl8Pv/gFLFyY61qJiJxIg9k5VFIC55wT6UDefx8+SLbaKCuLGVOjRsWYhohILulrKMcGD46FewMHxjTadevgzTfhV7+CtWu1SZKI5J5aFD3EmWfGHhdbt8YCvaamaGns2BEzphobI2icd16Ma4iIdBcFih6iqiqmzbrHKu7nnouV3fv3R6A4dCiCx9atMHt2JB2srMx1rUWkN1DXUw9iFmMSffpEd1RxcaT9qK2NLLT19ZGhdtEiePJJdUuJSPdQi6IHu/TSSPfR3HLo1w/efjsGvpt32tOKbhHJNrUoerjU7qVx4yJ31PDhsVjv3XdzVy8R6T3UoshDxcURKF54AcaPj/tvvx1jGzNmxP1UBw/Gor4DB+K+WezEp9aIiGRCgSIP1dTEl/7evfDYY7EV6/bt0NAQYxznnhvdUvX1kSKksTH2wThypGX21Nq1cNFFMYheXh6HiEhbFCjy1DnnRELBhoYIEocORSBYtSr2wjh8ODZI2rEjHmtoiPODB8e53bvj8fHjo+yoUVGutLRlj4wBA2DChBNbKCLSuyhQ5KmyMrjkkggM554bLYKFC2OW1LJlMa32yJGYUjt1Ksyadfzz33gjyq5aFV1WW7dGF1VVVQSU5rUby5bB1VfHgkAR6Z0UKPLYgAHRfdRs2jR4+WUYOzaCQ//+J08BMm1aTLsdMyb2+D50KMpv2RLdWn36REtj5054/PHo7mpqgilTtH5DpLcxz7PJ+DU1s/zee19Xn3o3Wbgwup6KiyNQVFXBzTdrv2+RfGNmS9x9VvslT6TpsZLWzJnRghg5MsY56urgkUdgyRJ4551c105EuoO6niStvn1j4BxidtVvfhMD4UuXxlhGQwNUV8eYyJAhGssQKUQKFJKxoiK4/PJYv7FvXxyrVsGaNbEmo7ERPvlJJS0UKTQKFNIhJSUwZ07cXrMmjqKiOIqLo1vq8stjuq2IFAYFCum0SZPigJg19dxzMZPqySfhc5/TpksihUL/laVLlJdHmvRBg2Ih3+uv57pGItJV1KKQLjVxYmzAtHx5pAo566wY/D5wAEaMiEAiIvlFgUK6VGVlpAWprY1Wxdtvx0K+XbtifGPgwFjp3bdvrmsqIplSoJAuN2VKS8ti795Y+X30aDxWVxervc86qyWHlFms11A2W5GeKaNAYWaXAF8HxifPMcDd/fR2njcX+A5QDNzr7t9so8yVwLeBUmCHu1+Rce2lxyori5ZDKvdY6b1tW0ytbd54qaQETjstEhCKSM+TaYviPuDPgSXAsUyeYGbFwF3AtUAt8JqZzXf3VSllBgLfB+a6+0YzG9aBukueMYPLLouFetu3x7qLvXsjFfqbb8YOfkOGaLaUSE+TaaDY7e5PdfC1ZwNr3X0dgJk9CNwIrEop8wngUXffCODu2zv4HpKHiosjJQhEq2LLlhjT+OUvY/e+006L8+6R4HDEiNzWV6S3yzRQPGdm/wo8ChxuPunub6R5zmhgU8r9WuCCVmXOBErN7DdAFfAdd/9RhnWSAmAGV10FCxZE62L79hjobmqKQLF8Odx2mzZWEsmlTANF8xd8auZBB+akeU5bQ5OtU9WWAOcBVwMVwMtm9oq7H7cbtJndAdwBMGLEuAyrLPmiuBhuuCFuHzwYA94VFbBuXezS9/DDMHlydFk1NcXMqgEDovyIETEeIiLZk1GgcPerOvHatcDYlPtjgC1tlNnh7vuB/Wb2AjAdOC5QuPs9wD0QacY7URfJExUVkWQQYOhQePbZCBx79kSgKCuLvTLMYixj0CC49to4JyLZkemspwHA14DLk1PPA3/v7rvTPO01YJKZTQA2A7cQYxKpfgF8z8xKgDKi5fIfmVdfCllREVxzTcySKimJ4LBsWaQLKSqKrql9+yK/VHU1XHCBtm0VyYZMu57uB1YAv5/c/xTwQ+B3T/YEd280sy8BC4jpsfe7+0ozuzN5/G53X21mTwPLgSZiCu2Kzn0UKVSpg9lzUjo7N2+Gt96KxXy7d8OmTbFjX3FxZLAdNiwGx9U1JXJqMtrhzsyWuvuM9s51B+1wJ601JyQsLY2BcLOWLqqhQ2HuXE25FTmVHe4ybVEcNLNL3X1h8oaXAAc784YiXa05ISFEV9SePZH+/IMPYMeOWCV+5pm5raNIPss0UPwx8N/JWIUBDcBns1Upkc6qrIxj1KgIGi++CIsWKVCInIpMZz0tBaabWf/k/p5sVkqkK1RWwuDBsU3rb34Ds2crGaFIZ6QNFGb2SXf/sZn9RavzALj7v2exbiKnrKYm8ku9804MeFdUwIwZ0V3V1HTiHt9Hj7bMsBKR0F6Lol/yb1W2KyKSDVVVMW321VcjUJSXx7qMoqIICCNGRMujsTGCw969cb66OtKMaMaUSDuBwt1/kPz7d91THZGud9ppMdh95Ag880wMdpeWRnDYvz/WY+zfH+cOHIhWx9q1MWvqkkuU1VYk0wV33wL+gZjp9DSxevrP3P3HWaybSJcqK2uZHQUxM2rLlkg8eMYZEUiqqmDJkphy29zyUKCQ3i7T2eXXJQPYHyHSbpwJ/FXWaiXSDSZNgiuugNNPjwAxeHAEk4suggsvjDUYe/fC+vXR+hDprTINFKXJvzcAP3X3hizVR6THGD06Vn0/8wz84hcKFtJ7ZbqO4nEze5voevqCmQ0FDmWvWiK5N3x4bKb0wgsxEL5gQbQyzGDaNCUilN4joxaFu38VuAiY5e5Hgf3EJkQiBa2yEq68Mga5166FxYtjBtXDD0fwEOkN2ltHMcfdnzWz3005l1rk0WxVTKSn6Nu3ZdZUUxO89FKkPH/sMfid34lU5yKFrL2upyuAZ4HfbuMxR4FCepHmNRVXXw2rVsHGjTB/fnRDzZihRXpSuNpbR/G15N/bu6c6IvmhpiZ242togFdeiS1ca2qiq6q0NGZRiRSKTNdR/BPwLXffldwfBPylu/9NFusm0mOZwcyZcPhw7MJXVBT7YZSXx2OzZ8faDKU3l0KQ6a/xvOYgAeDuO4mpsiK9llkEhiuuaNlx74MPYt3FSy/FgPcbb+S6liKnLtPpscVm1sfdDwOYWQWgyYEixBTamTNb7q9fD6tXx5jGjh0RPC66CPr3z1kVRU5JpoHix8AzZvZDYhD7c8B/Z61WInmsujqOpiZ4+uloaWzbFivAp09XwJD8k+l+FN8ys+XANcTGRd9w9wVZrZlInisqguuvj42Ttm2DnTujtfHRj0biQZF8kWmLAmA10OjuvzazvmZW5e57s1UxkUJQXAyXXRbrLhYvhq1b4ZFH4GMf08puyR8ZDWab2R8CDwM/SE6NBh7LUp1ECk5xcYxTlJVFosGHHoL3349uKZGeLtNZT18ELgH2ALj7GmBYtiolUqhmzowB7s2b4de/jtXdy5dHwGjeF2P//lzXUuR4mXY9HXb3I83pO8yshBjUFpEO6NcPbrghgsHzz8cmSnV1sGFDbLC0a1fshTFsWAyIjx2b6xqLZB4onjezvwYqzOxa4AvA49mrlkhhaw4YR4/Cr34VgaOqKsYyGhpiQ6V3341AMWdObM8qkiuZ/vp9BfgD4C3gj4AngXuzVSmR3qK0NAJGU9Pxq7jr6mKx3pEjETgmTIj9MUaMUNCQ7tfur5yZFQHL3f1s4L+yXyWR3qd1qo/hw+Haa6O1sWtXpDRfvTqCxFlnRfkhQ6KLSlNtJdvaDRTu3mRmy8xsnLtv7I5KiUgEhXnzorWxciXU18cue3v3xsB3eXmUGT06khGWlERX1TBNM5EulmkjdiSw0swWE5sWAeDuv5OVWonIh4qK4Jxz4nZTU8yaOnAAamvj3x07YvptWVnMoBo2LPbQmDpVQUO6RqaB4u+yWgsRyUhRUcuXf3V1y/kjR2Lwe9OmGNOAWKdx3nkwaVIEDpHOam+Hu3LgTmAiMZB9n7tri3mRHqasDM4+Ow6AdesiUCxcCEuXwq23tmy8JNJR7bUo/hs4CrwIzAOmAF/OdqVE5NScfjqMGwfLlkXX1IMPRsti9OgYy6iogIEDtSufZKa9QDHF3c8BMLP7gMXZr5KIdIWSkuh6Wro0Up3v2gVvvx0D4cXF0R118cURPETSaS9QHG2+4e6Npj8/RPLOjBkRHLZvj/UZ7rHfd9++kdF26FAYNSpaF8XFsRjwtNPivP7LC7QfKKab2Z7kthErs/ckt93d02bWN7O5wHeAYuBed//mScqdD7wCfNzdH+7IBxCR9pnF2ozhw+P+jBkxzXbRolijsWlTnC8tjbGM4uIIFGPGxOD5yJE5q7r0AGkDhbsXd/aFzawYuAu4FqgFXjOz+e6+qo1y/wJofwuRblRVFftlNGewPXIkWhxbtkRL4+BBWLs2xjMuvRRqatTC6K2ymQxgNrDW3dcBmNmDwI3Aqlbl/gR4BDg/i3URkZNo/vLv0ycGwMeNa3ls377Y/3vhQlizBqZNi4V+amH0LtkMFKOBTSn3a4ELUguY2WjgZmAOaQKFmd0B3AEwYsS4kxUTkS5WWRlJCZ9/Prqqtm2LbqnJk2NB36BBua6hdIdsBoq2GqmtU5N/G/iKux9LN1Du7vcA9wDU1MxSenORblRaCtdcE2MZH3wA773X0i110UVw5plRTt1ShSubgaIWSM2mPwbY0qrMLODBJEgMAW4ws0Z3fyyL9RKRThgwII6JE6Mb6r33oltq6dLojmrukhoxIhIWtk50KPkrm4HiNWCSmU0ANgO3AJ9ILeDuE5pvm9kDwP8qSIj0fJMmxZTaxYtjQR9Ey2PDhpg11bcvfOQjSh1SKLIWKJJ1F18iZjMVA/e7+0ozuzN5/O5svbeIZF+/fnDVVS339+2Dd96J9RrFxfCzn8VYxnnntWz1qjQi+SmrW6C4+5PEJkep59oMEO7+2WzWRUSyq7IyggLA+vXRPXXoUNyurIzbQ4ZEmcrKXNZUOkp7ZYlIl6uujq6pl16KPTSaB8L794/AcfbZMWPKHcaP1659PZ1+PCKSFWVlx3dNQcyU2rgRXnklgkRJSbQuPvaxWMchPZMChYh0m4kT49i/PwLD889Ha2PhwggqminVM+nHIiLdrl+/aE1cfHEEjE2bYP782LFPeh4FChHJmYqKSBmyaVNMrX300RjXaN4XXHoGdT2JSE6NHx+L9F58MQa69+2LxXzDh8PgwXGMHBkL+iQ3FChEJOf69Ik0IY2N8OabsYhv925oaopWR9++schv0KAIHKWlkf1WaUO6hwKFiPQYJSVwfkp60MZGWLIkZkrt3h0BoqIiAsSwYZH6fMwYBYxsU6AQkR6rpAQuSHJO79sH774b4xnuUF8ft8vLYzvXqqpYCd6vX27rXIgUKEQkL1RWwsyZLfd37IjV3zt2xIZLffrAihXwqU9pmm1XU6AQkbw0ZEgcEGMZr74as6UefxyuvlppQrqSAoWI5L2iohjb+OUv4dgx+PnPYwxjwoQIJiUl0V3VvH5DOkaXTEQKQkkJzJ0bGWzr6iKL7YYNMfjdnL22Tx8499xogfTvH2Mb0j4FChEpGEVFMROqpiaCwRtvRHdUWVkkJSwujkFw95hBNXFiJCgcMEDrNNJRoBCRglRUBLNmHX/u2LEIIEePxgrwVaui1VFSAjffHDOn5EQKFCLSaxQXx1FaCtdeG62M1avhyBF47LEY12heDT50aIxpaI2GAoWI9GJDh8axbh3U1sa4RmlpBIjS0hgInzdPwUKBQkR6vdNPjwOidfHWW7BtG+zcGXmozj23dwcLLUsREUlRVhbbtV55ZawGf+MNWLAg9tDordSiEBFpQ3l5bKb0zDMx+L1jRyzi698fxo6NbqlBg3Jdy+6hQCEichJ9+sQYxZIl0NAQx9GjMaZRUhK5pSZObFkhXqgUKERE0jA7fpptY2N0R23bFjvyrVwZweKMM6KFUYhJCRUoREQ6oKQEZs+OgLFrF7z2WgyAr1kTM6WuuSYGwIuLc13TrqNAISLSCSUlLdNnt26NPTMaGmLgu7QUpk+PdRmDBkUXVj5ToBAROUUjR8axe3dMrT10KLqlSkpiFtW550ZakXxtZShQiIh0kQED4NJLI1XIhg0xU2r79sg3tXRpjGUMHx5H3765rm3mFChERLpYcXHLIj53ePll2Lw5EhJWVERX1E03RWDJBwoUIiJZZAYXXxy3jxyBN9+MQfCf/zwy17ZOXNgTKVCIiHSTsrLYA3zrVli+HA4fjr0zrroqWho9NU2IAoWISDcbOTLGKX75yxj03rkzZlCdf35kru1pFChERHKgqCh25Hv//VjpXV8f4xhjxsSK7yFDes7iPQUKEZEcmjAhjo0bYxvXvXth/frINXXDDZEGPdeymj3WzOaa2TtmttbMvtrG47eZ2fLkWGRm07NZHxGRnmrcuNhMafbsmDW1dSs88URsrpRrWQsUZlYM3AXMA6YAt5rZlFbF3geucPdpwDeAe7JVHxGRfNCnTyzQmzw51mA8+WQMfK9fH0Gjqan765TNrqfZwFp3XwdgZg8CNwKrmgu4+6KU8q8AY7JYHxGRvDF2bASN5ctjf++iouiOKi+PVd7TpnVfXbIZKEYDm1Lu1wIXpCn/eeCpth4wszuAOwBGjBjXVfUTEenRhg2LJIMHDsQ02q1bI2vt7t2RlHDmzO6pRzYDRVszgr3NgmZXEYHi0rYed/d7SLqlampmtfkaIiKFqm/flkHvI0fg2Wcja+2oUZGpNtuyOZhdC4xNuT8G2NK6kJlNA+4FbnT3+izWR0Qk75WVRT6p+npYtKj98l0hm4HiNWCSmU0wszLgFmB+agEzGwc8CnzK3d/NYl1ERApGZWWst6iri42Tsi1rgcLdG4EvAQuA1cBD7r7SzO40szuTYn8LDAa+b2ZLzez1bNVHRKSQjB4dYxUvvQRPPw0HD2bvvcw9v7r8a2pm+b33vk55ea5rIiKSW4cPw/PPxxjGoEEwZ060NNpiZkvcvVMpCLO64E5ERLKnTx+47rrID7V5c+SO2rOn699HgUJEJM/V1MTWq3V1MH8+vPde176+cj2JiBSA4cMjDcjGjbHD3oQJsUivKyhQiIgUiMmTI+Pshg2xIK+srGteV11PIiIFxCyCxOOPx2B3V1CgEBEpIGPGwKFD0ap4+OGueU0FChGRAjN3bsyIqq+HLSfkw+g4BQoRkQJ09tmwbx8sWBAtjFOhQCEiUoCa03zs3AkPPXRqr6VAISJSoKZOhYoKaGgAKO70970ChYhIAZs0qXlXPGtr64eMKFCIiPQKnV9+p0AhIlLAKitjhzzo0+nldwoUIiIFrLQUzj0XoLi4s6+hQCEiUuBOdbtUBQoREUlLgUJERNJSoBARkbQUKEREJC0FChERSUuBQkRE0lKgEBGRtBQoREQkLQUKERFJS4FCRETSUqAQEZG0FChERCQtBQoREUlLgUJERNJSoBARkbQUKEREJC0FChERSUuBQkRE0spqoDCzuWb2jpmtNbOvtvG4mdl3k8eXm9nMbNZHREQ6LmuBwsyKgbuAecAU4FYzm9Kq2DxgUnLcAfxntuojIiKdU5LF154NrHX3dQBm9iBwI7AqpcyNwI/c3YFXzGygmY10963pXvjQoWxVWUREWstmoBgNbEq5XwtckEGZ0cBxgcLM7iBaHABHrrmm/zpw79rq5qOjg6B0Z65r0TPoWrTQtWiha9HiwPjOPjObgcLaONf6yz2TMrj7PcA9AGb2uvueWadevfwX1+KQrgW6Fql0LVroWrQws9c7+9xsDmbXAmNT7o8BtnSijIiI5FA2A8VrwCQzm2BmZcAtwPxWZeYDn05mP10I7G5vfEJERLpX1rqe3L3RzL4ELACKgfvdfaWZ3Zk8fjfwJHADsBY4ANyewUvfk6Uq5yNdixa6Fi10LVroWrTo9LUw15iwiIikoZXZIiKSlgKFiIik1WMDhdJ/tMjgWtyWXIPlZrbIzKbnop7dob1rkVLufDM7ZmYf7c76dadMroWZXWlmS81spZk939117C4Z/B8ZYGaPm9my5FpkMh6ad8zsfjPbbmYrTvJ457433b3HHcTg93vA6UAZsAyY0qrMDcBTxFqMC4FXc13vHF6Li4FBye15vflapJR7lpgs8dFc1zuHvxcDiUwI45L7w3Jd7xxei78G/iW5PRRoAMpyXfcsXIvLgZnAipM83qnvzZ7aovgw/Ye7HwGa03+k+jD9h7u/Agw0s5HdXdFu0O61cPdF7t68+vQVYj1KIcrk9wLgT4BHgO3dWblulsm1+ATwqLtvBHD3Qr0emVwLB6rMzIBKIlA0dm81s8/dXyA+28l06nuzpwaKk6X26GiZQtDRz/l54i+GQtTutTCz0cDNwN3dWK9cyOT34kxgkJn9xsyWmNmnu6123SuTa/E9oIZY0PsW8GV3b+qe6vUonfrezGYKj1PRZek/CkDGn9PMriICxaVZrVHuZHItvg18xd2PxR+PBSuTa1ECnAdcDVQAL5vZK+7+brYr180yuRbXA0uBOcAZwK/M7EV335PluvU0nfre7KmBQuk/WmT0Oc1sGnAvMM/d67upbt0tk2sxC3gwCRJDgBvMrNHdH+uWGnafTP+P7HD3/cB+M3sBmA4UWqDI5FrcDnzTo6N+rZm9D0wGFndPFXuMTn1v9tSuJ6X/aNHutTCzccCjwKcK8K/FVO1eC3ef4O7V7l4NPAx8oQCDBGT2f+QXwGVmVmJmfYnszau7uZ7dIZNrsZFoWWFmw4GzgHXdWsueoVPfmz2yReHZS/+RdzK8Fn8LDAa+n/wl3ejuBZcxM8Nr0Stkci3cfbWZPQ0sB5qAe929zWmT+SzD34tvAA+Y2VtE98tX3H1HziqdJWb2U+BKYIiZ1QJfA0rh1L43lcJDRETS6qldTyIi0kMoUIiISFoKFCIikpYChYiIpKVAISIiaSlQiLQhyTy71MxWJFlHB3bx6683syHJ7X1d+doiXU2BQqRtB919hrufTSRZ+2KuKySSKwoUIu17mSRxmpmdYWZPJ0n2XjSzycn54Wb282S/g2VmdnFy/rGk7EozuyOHn0Gk03rkymyRnsLMionUD/clp+4B7nT3NWZ2AfB9ItHcd4Hn3f3m5DmVSfnPuXuDmVUAr5nZIwWci0sKlAKFSNsqzGwpUA0sIbKNVhKbRP0sJTNtn+TfOcCnAdz9GLA7Of+nZnZzcnssMAlQoJC8okAh0raD7j7DzAYA/0uMUTwA7HL3GZm8gJldCVwDXOTuB8zsN0B5Niorkk0aoxBJw913A38K/B/gIPC+mX0MPtx/uHl/8meAP07OF5tZf2AAsDMJEpOJrSdF8o4ChUg73P1NYh/mW4DbgM+b2TJgJS1bbn4ZuCrJTroEmAo8DZSY2XIie+kr3V13ka6g7LEiIpKWWhQiIpKWAoWIiKSlQCEiImkpUIiISFoKFCIikpYChYiIpKVAISIiaf1/Q4ezute3LloAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve, average_precision_score\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Assuming y_true and y_pred are your data\n",
    "# y_true = true binary labels (either 0 or 1)\n",
    "# y_pred = predicted probabilities that each entry in y_true is a 1\n",
    "test_y_true = test__op_ind_df['true']\n",
    "test_y_pred = test__op_ind_df['pred']\n",
    "\n",
    "# Compute Precision-Recall curve and the AUC-PR\n",
    "precision, recall, _ = precision_recall_curve(test_y_true, test_y_pred)\n",
    "average_precision = average_precision_score(test_y_true, test_y_pred)\n",
    "\n",
    "# Plotting the Precision-Recall Curve\n",
    "plt.figure()\n",
    "lw = 2  # Line width\n",
    "plt.step(recall, precision, color='b', alpha=0.2, where='post')\n",
    "plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('Precision-Recall curve: AUC={0:0.2f}'.format(average_precision))\n",
    "plt.savefig(f'PR AUC curve.pdf mimic iv with ss random seed 2', format='pdf')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fa681352",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_ind</th>\n",
       "      <th>op</th>\n",
       "      <th>pred</th>\n",
       "      <th>true</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007687</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.097661</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.020064</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.794653</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.809801</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9167</th>\n",
       "      <td>45499</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011544</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9168</th>\n",
       "      <td>45508</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.924047</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9169</th>\n",
       "      <td>45548</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004260</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9170</th>\n",
       "      <td>45555</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.835929</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9171</th>\n",
       "      <td>45556</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.827372</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9172 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ts_ind   op      pred  true\n",
       "0          0  0.0  0.007687   0.0\n",
       "1          8  0.0  0.097661   0.0\n",
       "2          9  0.0  0.020064   0.0\n",
       "3         14  0.0  0.794653   0.0\n",
       "4         20  1.0  0.809801   1.0\n",
       "...      ...  ...       ...   ...\n",
       "9167   45499  0.0  0.011544   0.0\n",
       "9168   45508  1.0  0.924047   1.0\n",
       "9169   45548  0.0  0.004260   0.0\n",
       "9170   45555  0.0  0.835929   0.0\n",
       "9171   45556  0.0  0.827372   0.0\n",
       "\n",
       "[9172 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test__op_ind_df.rename(columns={'ind': 'ts_ind'},inplace=True)\n",
    "test__op_ind_df\n",
    "#test__op_ind_df.to_csv('mimic iv final true pred with ind',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "70f863b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_ind</th>\n",
       "      <th>op</th>\n",
       "      <th>pred</th>\n",
       "      <th>true</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>hadm_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007687</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10002013</td>\n",
       "      <td>23581541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.097661</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10002930</td>\n",
       "      <td>25696644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.020064</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10003046</td>\n",
       "      <td>26048429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.794653</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10004235</td>\n",
       "      <td>24181354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.809801</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10004720</td>\n",
       "      <td>22081550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9167</th>\n",
       "      <td>45499</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011544</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19989302</td>\n",
       "      <td>21980453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9168</th>\n",
       "      <td>45508</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.924047</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19990427</td>\n",
       "      <td>29695607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9169</th>\n",
       "      <td>45548</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004260</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19997752</td>\n",
       "      <td>29452285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9170</th>\n",
       "      <td>45555</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.835929</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19998591</td>\n",
       "      <td>24349193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9171</th>\n",
       "      <td>45556</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.827372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19998591</td>\n",
       "      <td>24349193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9172 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ts_ind   op      pred  true  subject_id   hadm_id\n",
       "0          0  0.0  0.007687   0.0    10002013  23581541\n",
       "1          8  0.0  0.097661   0.0    10002930  25696644\n",
       "2          9  0.0  0.020064   0.0    10003046  26048429\n",
       "3         14  0.0  0.794653   0.0    10004235  24181354\n",
       "4         20  1.0  0.809801   1.0    10004720  22081550\n",
       "...      ...  ...       ...   ...         ...       ...\n",
       "9167   45499  0.0  0.011544   0.0    19989302  21980453\n",
       "9168   45508  1.0  0.924047   1.0    19990427  29695607\n",
       "9169   45548  0.0  0.004260   0.0    19997752  29452285\n",
       "9170   45555  0.0  0.835929   0.0    19998591  24349193\n",
       "9171   45556  0.0  0.827372   0.0    19998591  24349193\n",
       "\n",
       "[9172 rows x 6 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = 'mimic_iv_preprocessed.pkl'\n",
    "data_2, oc_2, train_ind_2, valid_ind_2, test_ind_2 = pickle.load(open(data_path, 'rb'))\n",
    "oc_2.sort_values(by='ts_ind')\n",
    "\n",
    "merged_df=test__op_ind_df.merge(oc_2[['ts_ind','subject_id','hadm_id']],on='ts_ind')\n",
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "941013a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_ind</th>\n",
       "      <th>op</th>\n",
       "      <th>pred</th>\n",
       "      <th>true</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>hadm_id</th>\n",
       "      <th>race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007687</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10002013</td>\n",
       "      <td>23581541</td>\n",
       "      <td>OTHER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.097661</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10002930</td>\n",
       "      <td>25696644</td>\n",
       "      <td>BLACK/AFRICAN AMERICAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.020064</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10003046</td>\n",
       "      <td>26048429</td>\n",
       "      <td>WHITE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.794653</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10004235</td>\n",
       "      <td>24181354</td>\n",
       "      <td>BLACK/CAPE VERDEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.809801</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10004720</td>\n",
       "      <td>22081550</td>\n",
       "      <td>WHITE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9167</th>\n",
       "      <td>45499</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011544</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19989302</td>\n",
       "      <td>21980453</td>\n",
       "      <td>WHITE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9168</th>\n",
       "      <td>45508</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.924047</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19990427</td>\n",
       "      <td>29695607</td>\n",
       "      <td>WHITE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9169</th>\n",
       "      <td>45548</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004260</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19997752</td>\n",
       "      <td>29452285</td>\n",
       "      <td>WHITE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9170</th>\n",
       "      <td>45555</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.835929</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19998591</td>\n",
       "      <td>24349193</td>\n",
       "      <td>WHITE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9171</th>\n",
       "      <td>45556</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.827372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19998591</td>\n",
       "      <td>24349193</td>\n",
       "      <td>WHITE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9172 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ts_ind   op      pred  true  subject_id   hadm_id  \\\n",
       "0          0  0.0  0.007687   0.0    10002013  23581541   \n",
       "1          8  0.0  0.097661   0.0    10002930  25696644   \n",
       "2          9  0.0  0.020064   0.0    10003046  26048429   \n",
       "3         14  0.0  0.794653   0.0    10004235  24181354   \n",
       "4         20  1.0  0.809801   1.0    10004720  22081550   \n",
       "...      ...  ...       ...   ...         ...       ...   \n",
       "9167   45499  0.0  0.011544   0.0    19989302  21980453   \n",
       "9168   45508  1.0  0.924047   1.0    19990427  29695607   \n",
       "9169   45548  0.0  0.004260   0.0    19997752  29452285   \n",
       "9170   45555  0.0  0.835929   0.0    19998591  24349193   \n",
       "9171   45556  0.0  0.827372   0.0    19998591  24349193   \n",
       "\n",
       "                        race  \n",
       "0                      OTHER  \n",
       "1     BLACK/AFRICAN AMERICAN  \n",
       "2                      WHITE  \n",
       "3         BLACK/CAPE VERDEAN  \n",
       "4                      WHITE  \n",
       "...                      ...  \n",
       "9167                   WHITE  \n",
       "9168                   WHITE  \n",
       "9169                   WHITE  \n",
       "9170                   WHITE  \n",
       "9171                   WHITE  \n",
       "\n",
       "[9172 rows x 7 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mimic_data_dir = '/home/anand/UHIF/UniHPF-master/UniHPF-master/mimic4/'\n",
    "adm = pd.read_csv(mimic_data_dir+'admissions.csv', usecols=['subject_id','hadm_id', 'hospital_expire_flag','race'])\n",
    "merged_df_eth = merged_df.merge(adm[['subject_id','hadm_id','race']],on=['subject_id','hadm_id'])\n",
    "merged_df_eth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2f84c56a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openpyxl in /home/anand/miniconda3/envs/strats/lib/python3.7/site-packages (3.1.2)\n",
      "Requirement already satisfied: et-xmlfile in /home/anand/miniconda3/envs/strats/lib/python3.7/site-packages (from openpyxl) (1.1.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install openpyxl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bf91ba4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the mapping dictionary\n",
    "racial_groupings = {\n",
    "    'White': ['white', 'white - brazilian','white - eastern european', 'white - other european','white - russian','portuguese'],\n",
    "    'Black': ['black/african', 'black/african american', 'black/cape verdean','black/caribbean island','south american'],\n",
    "    'Hispanic/Latino': ['hispanic or latino','hispanic/latino - central american', 'hispanic/latino - columbian','hispanic/latino - cuban', 'hispanic/latino - dominican','hispanic/latino - guatemalan', 'hispanic/latino - honduran','hispanic/latino - mexican', 'hispanic/latino - puerto rican','hispanic/latino - salvadoran'],\n",
    "    'Asian': ['asian', 'asian - asian indian','asian - chinese', 'asian - korean', 'asian - south east asian'],\n",
    "    'Other': ['native hawaiian or other pacific islander','other','patient declined to answer', 'unable to obtain', 'unknown','american indian/alaska native','multiple race/ethnicity']\n",
    "}\n",
    "# Load your dataset into a pandas dataframe\n",
    "#df = pd.read_csv('your_dataset.csv')\n",
    "#df_admission_unique[]\n",
    "# Replace the original race values with the new groupings\n",
    "\n",
    "merged_df_eth['race'] = merged_df_eth['race'].str.strip().str.lower()\n",
    "#print(df_admission_unique['race'])\n",
    "\n",
    "def get_race_sub_group(race):\n",
    "    for group, races in racial_groupings.items():\n",
    "        if race in races:\n",
    "            return group\n",
    "    return 'Unknown'\n",
    "\n",
    "merged_df_eth['race_sub_group'] = merged_df_eth['race'].apply(get_race_sub_group)\n",
    "merged_df_eth.to_csv(f'mimic iv final_test_results with ethn and racial sub groups 50ld random seed 2.csv', index=False)\n",
    "\n",
    "#df_admission_unique['race_sub_group'] = df_admission_unique['race'].replace(racial_groupings, regex=True)\n",
    "#df_admission_unique['race_sub_group'] = df_admission_unique['race_sub_group'].map(racial_groupings)\n",
    "# Check the new groupings\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9ac09adf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_ind</th>\n",
       "      <th>op</th>\n",
       "      <th>pred</th>\n",
       "      <th>true</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>hadm_id</th>\n",
       "      <th>race</th>\n",
       "      <th>race_sub_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007687</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10002013</td>\n",
       "      <td>23581541</td>\n",
       "      <td>other</td>\n",
       "      <td>Other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.097661</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10002930</td>\n",
       "      <td>25696644</td>\n",
       "      <td>black/african american</td>\n",
       "      <td>Black</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.020064</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10003046</td>\n",
       "      <td>26048429</td>\n",
       "      <td>white</td>\n",
       "      <td>White</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.794653</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10004235</td>\n",
       "      <td>24181354</td>\n",
       "      <td>black/cape verdean</td>\n",
       "      <td>Black</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.809801</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10004720</td>\n",
       "      <td>22081550</td>\n",
       "      <td>white</td>\n",
       "      <td>White</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ts_ind   op      pred  true  subject_id   hadm_id                    race  \\\n",
       "0       0  0.0  0.007687   0.0    10002013  23581541                   other   \n",
       "1       8  0.0  0.097661   0.0    10002930  25696644  black/african american   \n",
       "2       9  0.0  0.020064   0.0    10003046  26048429                   white   \n",
       "3      14  0.0  0.794653   0.0    10004235  24181354      black/cape verdean   \n",
       "4      20  1.0  0.809801   1.0    10004720  22081550                   white   \n",
       "\n",
       "  race_sub_group  \n",
       "0          Other  \n",
       "1          Black  \n",
       "2          White  \n",
       "3          Black  \n",
       "4          White  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df_eth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c8d89f2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Demographic Parity: {'Other': 0.24680511182108625, 'Black': 0.18979591836734694, 'White': 0.1861124232404346, 'Asian': 0.18972332015810275, 'Hispanic/Latino': 0.19047619047619047}\n",
      "Equalized Odds (TPR): {'Other': 0.7635467980295566, 'Black': 0.7216494845360825, 'White': 0.651840490797546, 'Asian': 0.6578947368421053, 'Hispanic/Latino': 0.625}\n",
      "Equalized Odds (FPR): {'Other': 0.14680648236415633, 'Black': 0.13137032842582105, 'White': 0.13283032110896648, 'Asian': 0.10697674418604651, 'Hispanic/Latino': 0.14473684210526316}\n",
      "False Negative Rate: {'Other': 0.23645320197044334, 'Black': 0.27835051546391754, 'White': 0.348159509202454, 'Asian': 0.34210526315789475, 'Hispanic/Latino': 0.375}\n",
      "False Omission Rate: {'Other': 0.8531935176358436, 'Black': 0.8686296715741789, 'White': 0.8671696788910335, 'Asian': 0.8930232558139535, 'Hispanic/Latino': 0.8552631578947368}\n",
      "Disparate Impact: {'Other_Black': 1.3003710192723898, 'Other_White': 1.3261076693534, 'Other_Asian': 1.3008686102236422, 'Other_Hispanic/Latino': 1.295726837060703, 'Black_Other': 0.7690112938379235, 'Black_White': 1.0197917745778513, 'Black_Asian': 1.0003826530612245, 'Black_Hispanic/Latino': 0.9964285714285714, 'White_Other': 0.7540865821910165, 'White_Black': 0.980592337503365, 'White_Asian': 0.980967564163124, 'White_Hispanic/Latino': 0.9770902220122817, 'Asian_Other': 0.7687171418703711, 'Asian_Black': 0.9996174933061328, 'Asian_White': 1.0194016973977247, 'Asian_Hispanic/Latino': 0.9960474308300395, 'Hispanic/Latino_Other': 0.7717676067190631, 'Hispanic/Latino_Black': 1.003584229390681, 'Hispanic/Latino_White': 1.023446942228668, 'Hispanic/Latino_Asian': 1.003968253968254}\n"
     ]
    }
   ],
   "source": [
    "## performing the grouped fairness metrics for racial subgroups\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "# Assuming df contains your data\n",
    "threshold = 0.5\n",
    "merged_df_eth['pred_label'] = (merged_df_eth['pred'] > threshold).astype(int)\n",
    "\n",
    "# Initialize empty dictionaries to hold calculated values for each group\n",
    "demographic_parity = {}\n",
    "equalized_odds_tpr = {}\n",
    "equalized_odds_fpr = {}\n",
    "disparate_impact = {}\n",
    "false_negative_rate = {}\n",
    "false_omission_rate = {}\n",
    "\n",
    "# Loop over each demographic group to compute the metrics\n",
    "for group in merged_df_eth['race_sub_group'].unique():\n",
    "    sub_df = merged_df_eth[merged_df_eth['race_sub_group'] == group]\n",
    "    total = len(sub_df)\n",
    "    total_positives = sum(sub_df['true'] == 1)\n",
    "    total_negatives = total - total_positives\n",
    "\n",
    "    # For Demographic Parity\n",
    "    demographic_parity[group] = sum(sub_df['pred_label'] == 1) / total\n",
    "\n",
    "    # For Equalized Odds\n",
    "    true_positives = sum((sub_df['true'] == 1) & (sub_df['pred_label'] == 1))\n",
    "    false_positives = sum((sub_df['true'] == 0) & (sub_df['pred_label'] == 1))\n",
    "\n",
    "    equalized_odds_tpr[group] = true_positives / total_positives if total_positives > 0 else 0\n",
    "    equalized_odds_fpr[group] = false_positives / total_negatives if total_negatives > 0 else 0\n",
    "\n",
    "    # For False Negative Rate (FNR)\n",
    "    false_negatives = sum((sub_df['true'] == 1) & (sub_df['pred_label'] == 0))\n",
    "    false_negative_rate[group] = false_negatives / total_positives if total_positives > 0 else 0\n",
    "\n",
    "    # For False Omission Rate (FOR)\n",
    "    false_negatives = sum((sub_df['true'] == 0) & (sub_df['pred_label'] == 0))\n",
    "    false_omission_rate[group] = false_negatives / total_negatives if total_negatives > 0 else 0\n",
    "\n",
    "# For Disparate Impact (use two groups for demonstration)\n",
    "#group_a, group_b = 'WHITE', 'BLACK/AFRICAN AMERICAN'\n",
    "#disparate_impact[f\"{group_a}_{group_b}\"] = demographic_parity[group_a] / demographic_parity[group_b] if demographic_parity[group_b] > 0 else float('inf')\n",
    "\n",
    "#reference_group = 'White'\n",
    "#for group in merged_df_eth['race_sub_group'].unique():\n",
    "#    if group != reference_group:\n",
    "#        key = f\"{reference_group}_{group}\"\n",
    "#        disparate_impact[key] = demographic_parity[reference_group] / demographic_parity[group] if demographic_parity[group] > 0 else float('inf')\n",
    "\n",
    "# Calculate Disparate Impact for all pairs of groups\n",
    "for group_a in merged_df_eth['race_sub_group'].unique():\n",
    "    for group_b in merged_df_eth['race_sub_group'].unique():\n",
    "        if group_a != group_b:\n",
    "            key = f\"{group_a}_{group_b}\"\n",
    "            if demographic_parity[group_b] > 0:\n",
    "                disparate_impact[key] = demographic_parity[group_a] / demographic_parity[group_b]\n",
    "            else:\n",
    "                disparate_impact[key] = float('inf')  # Handle division by zero\n",
    "\n",
    "print(f\"Demographic Parity: {demographic_parity}\")\n",
    "print(f\"Equalized Odds (TPR): {equalized_odds_tpr}\")\n",
    "print(f\"Equalized Odds (FPR): {equalized_odds_fpr}\")\n",
    "print(f\"False Negative Rate: {false_negative_rate}\")\n",
    "print(f\"False Omission Rate: {false_omission_rate}\")\n",
    "print(f\"Disparate Impact: {disparate_impact}\")\n",
    "\n",
    "# Create DataFrames from dictionaries\n",
    "demographic_parity_df = pd.DataFrame(list(demographic_parity.items()), columns=['Group', 'Demographic Parity'])\n",
    "equalized_odds_tpr_df = pd.DataFrame(list(equalized_odds_tpr.items()), columns=['Group', 'Equalized Odds TPR'])\n",
    "equalized_odds_fpr_df = pd.DataFrame(list(equalized_odds_fpr.items()), columns=['Group', 'Equalized Odds FPR'])\n",
    "false_negative_rate_df = pd.DataFrame(list(false_negative_rate.items()), columns=['Group', 'False Negative Rate'])\n",
    "false_omission_rate_df = pd.DataFrame(list(false_omission_rate.items()), columns=['Group', 'False Omission Rate'])\n",
    "disparate_impact_df = pd.DataFrame(list(disparate_impact.items()), columns=['Group Pair', 'Disparate Impact'])\n",
    "\n",
    "# Save DataFrames to different sheets in the same Excel file\n",
    "with pd.ExcelWriter('fairness_metrics for race_subgroup 50ld randomseed 2.xlsx') as writer:\n",
    "    demographic_parity_df.to_excel(writer, sheet_name='Demographic Parity', index=False)\n",
    "    equalized_odds_tpr_df.to_excel(writer, sheet_name='Equalized Odds TPR', index=False)\n",
    "    equalized_odds_fpr_df.to_excel(writer, sheet_name='Equalized Odds FPR', index=False)\n",
    "    false_negative_rate_df.to_excel(writer, sheet_name='False Negative Rate', index=False)\n",
    "    false_omission_rate_df.to_excel(writer, sheet_name='False Omission Rate', index=False)\n",
    "    disparate_impact_df.to_excel(writer, sheet_name='Disparate Impact', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "bc8d3530",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_ind</th>\n",
       "      <th>op</th>\n",
       "      <th>pred</th>\n",
       "      <th>true</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>hadm_id</th>\n",
       "      <th>race</th>\n",
       "      <th>race_sub_group</th>\n",
       "      <th>pred_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007687</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10002013</td>\n",
       "      <td>23581541</td>\n",
       "      <td>other</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.097661</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10002930</td>\n",
       "      <td>25696644</td>\n",
       "      <td>black/african american</td>\n",
       "      <td>Black</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.020064</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10003046</td>\n",
       "      <td>26048429</td>\n",
       "      <td>white</td>\n",
       "      <td>White</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.794653</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10004235</td>\n",
       "      <td>24181354</td>\n",
       "      <td>black/cape verdean</td>\n",
       "      <td>Black</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.809801</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10004720</td>\n",
       "      <td>22081550</td>\n",
       "      <td>white</td>\n",
       "      <td>White</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ts_ind   op      pred  true  subject_id   hadm_id                    race  \\\n",
       "0       0  0.0  0.007687   0.0    10002013  23581541                   other   \n",
       "1       8  0.0  0.097661   0.0    10002930  25696644  black/african american   \n",
       "2       9  0.0  0.020064   0.0    10003046  26048429                   white   \n",
       "3      14  0.0  0.794653   0.0    10004235  24181354      black/cape verdean   \n",
       "4      20  1.0  0.809801   1.0    10004720  22081550                   white   \n",
       "\n",
       "  race_sub_group  pred_label  \n",
       "0          Other           0  \n",
       "1          Black           0  \n",
       "2          White           0  \n",
       "3          Black           1  \n",
       "4          White           1  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df_eth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b492d74c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Contingency Table:\n",
      "pred_label          0     1\n",
      "race_sub_group             \n",
      "Asian             205    48\n",
      "Black             794   186\n",
      "Hispanic/Latino   272    64\n",
      "Other             943   309\n",
      "White            5169  1182\n",
      "Chi2 value: 24.853746368646966\n",
      "p-value: 5.3833477599663564e-05\n",
      "Degrees of freedom: 4\n",
      "Expected table:\n",
      "[[ 203.65231138   49.34768862]\n",
      " [ 788.85085041  191.14914959]\n",
      " [ 270.46314871   65.53685129]\n",
      " [1007.7972089   244.2027911 ]\n",
      " [5112.23648059 1238.76351941]]\n"
     ]
    }
   ],
   "source": [
    "#Chi-Square Test for Independence: This is useful to check if the model's predictions are independent of the ethnic groups. Essentially, you'd look to see if the model is more or less likely to, say, predict ICU mortality for one group over another.\n",
    "\n",
    "from scipy.stats import chi2_contingency\n",
    "\n",
    "# Create a contingency table\n",
    "contingency_table = pd.crosstab(merged_df_eth['race_sub_group'], merged_df_eth['pred_label'])\n",
    "print(\"Contingency Table:\")\n",
    "print(contingency_table)\n",
    "\n",
    "# Perform the chi-square test\n",
    "chi2, p, dof, expected = chi2_contingency(contingency_table)\n",
    "print(f\"Chi2 value: {chi2}\")\n",
    "print(f\"p-value: {p}\")\n",
    "print(f\"Degrees of freedom: {dof}\")\n",
    "print(\"Expected table:\")\n",
    "print(expected)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6a436d0",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}