mvp_scipspacy_appendix.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Scispacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "### chose to do this from cli\n",
    "#%pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.4.0/en_core_sci_sm-0.4.0.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/dragonzord/Documents/SchoolDocs/codeup-data-science/exercises/methodologies2/capstone/nlp-capstone\n",
      "/Users/dragonzord/.vscode/extensions/ms-toolsai.jupyter-2022.2.1030672458/pythonFiles\n",
      "/Users/dragonzord/.vscode/extensions/ms-toolsai.jupyter-2022.2.1030672458/pythonFiles/lib/python\n",
      "/opt/miniconda3/envs/scispacy/lib/python39.zip\n",
      "/opt/miniconda3/envs/scispacy/lib/python3.9\n",
      "/opt/miniconda3/envs/scispacy/lib/python3.9/lib-dynload\n",
      "\n",
      "/opt/miniconda3/envs/scispacy/lib/python3.9/site-packages\n",
      "/opt/miniconda3/envs/scispacy/lib/python3.9/site-packages/IPython/extensions\n",
      "/Users/dragonzord/.ipython\n",
      "/Users/dragonzord/.local/lib/python3.9/site-packages\n",
      "/Users/dragonzord/Documents/SchoolDocs/codeup-data-science/exercises/methodologies2/capstone/nlp-capstone/src\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import scispacy\n",
    "import spacy\n",
    "\n",
    "from scispacy.linking import EntityLinker\n",
    "\n",
    "import sys\n",
    "directory_path = \"/Users/dragonzord/Documents/SchoolDocs/codeup-data-science/exercises/methodologies2/capstone/nlp-capstone/src\"\n",
    "sys.path.append(directory_path)\n",
    "for p in sys.path:\n",
    "    print(p)\n",
    "    \n",
    "# these imports must come after adding to path\n",
    "import prepare_jag\n",
    "from casenums import UniqueDataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   pn_num  case_num                                         pn_history\n",
      "0       0         0  17-year-old male, has come to the student heal...\n",
      "1       1         0  17 yo male with recurrent palpitations for the...\n",
      "2       2         0  Dillon Cleveland is a 17 y.o. male patient wit...\n",
      "3       3         0  a 17 yo m c/o palpitation started 3 mos ago; \\...\n",
      "4       4         0  17yo male with no pmh here for evaluation of p...\n",
      "5       5         0  17 yo m, presenting with palpitations/increase...\n",
      "6       6         0  17 yo male presents with 3-4 month history of ...\n",
      "7       7         0  17 yo male presents with a 2-3 month history o...\n",
      "8       8         0  HPI: 17 YO complains of intermittent heart pal...\n",
      "9       9         0  Mr. Cleveland is a 17 yo M who presents to cli...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_num</th>\n",
       "      <th>case_num</th>\n",
       "      <th>feature_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Family-history-of-MI-OR-Family-history-of-myoc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Family-history-of-thyroid-disorder</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Chest-pressure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>Intermittent-symptoms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>Lightheaded</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>No-hair-changes-OR-no-nail-changes-OR-no-tempe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Adderall-use</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>Shortness-of-breath</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>Caffeine-use</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>heart-pounding-OR-heart-racing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   feature_num  case_num                                       feature_text\n",
       "0            0         0  Family-history-of-MI-OR-Family-history-of-myoc...\n",
       "1            1         0                 Family-history-of-thyroid-disorder\n",
       "2            2         0                                     Chest-pressure\n",
       "3            3         0                              Intermittent-symptoms\n",
       "4            4         0                                        Lightheaded\n",
       "5            5         0  No-hair-changes-OR-no-nail-changes-OR-no-tempe...\n",
       "6            6         0                                       Adderall-use\n",
       "7            7         0                                Shortness-of-breath\n",
       "8            8         0                                       Caffeine-use\n",
       "9            9         0                     heart-pounding-OR-heart-racing"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "patient_notes = pd.read_csv(\"data/patient_notes.csv\")\n",
    "features = pd.read_csv(\"data/features.csv\")\n",
    "\n",
    "print(patient_notes[patient_notes[\"case_num\"] == 0][:10])\n",
    "features[features[\"case_num\"] == 0][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"17-year-old male, has come to the student health clinic complaining of heart pounding. Mr. Cleveland's mother has given verbal consent for a history, physical examination, and treatment\\r\\n-began 2-3 months ago,sudden,intermittent for 2 days(lasting 3-4 min),worsening,non-allev/aggrav\\r\\n-associated with dispnea on exersion and rest,stressed out about school\\r\\n-reports fe feels like his heart is jumping out of his chest\\r\\n-ros:denies chest pain,dyaphoresis,wt loss,chills,fever,nausea,vomiting,pedal edeam\\r\\n-pmh:non,meds :aderol (from a friend),nkda\\r\\n-fh:father had MI recently,mother has thyroid dz\\r\\n-sh:non-smoker,mariguana 5-6 months ago,3 beers on the weekend, basketball at school\\r\\n-sh:no std\\n17 yo male with recurrent palpitations for the past 3 mo lasting about 3 - 4 min, it happened about 5 - 6 times since the beginning. One time durign a baskeball game two days ago light headedness, pressure in the chest, catching breath, but no fainting. During teh episodes no sweating. No diarrhea, no h\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "case_num_zero = patient_notes[\"pn_history\"][patient_notes[\"case_num\"] == 0][:1500]\n",
    "whole_case_num_zero = \"\\n\".join(case_num_zero.to_list())\n",
    "len(whole_case_num_zero)\n",
    "whole_case_num_zero[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "968544"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_note = prepare_jag.basic_clean3(whole_case_num_zero)\n",
    "clean_note = prepare_jag.remove_stopwords(clean_note, exclude_words=[\"no\"])\n",
    "len(clean_note)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### run this always"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "works, yay!\n"
     ]
    }
   ],
   "source": [
    "sci_nlp = spacy.load(\"en_ner_bc5cdr_md\")\n",
    "# This line takes a while, because we have to download ~1GB of data\n",
    "# and load a large JSON file (the knowledge base). Be patient!\n",
    "# Thankfully it should be faster after the first time you use it, because\n",
    "# the downloads are cached.\n",
    "print(\"works, yay!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From https://gbnegrini.com/post/biomedical-text-nlp-scispacy-named-entity-recognition-medical-records/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = sci_nlp(clean_note)\n",
    "print(\"TEXT\", \"START\", \"END\", \"ENTITY TYPE\")\n",
    "for ent in doc.ents:\n",
    "    print(ent.text, ent.start_char, ent.end_char, ent.label_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- case num 0 patient notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pn_num</th>\n",
       "      <th>case_num</th>\n",
       "      <th>pn_history</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17-year-old male, has come to the student heal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17 yo male with recurrent palpitations for the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Dillon Cleveland is a 17 y.o. male patient wit...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pn_num  case_num                                         pn_history\n",
       "0       0         0  17-year-old male, has come to the student heal...\n",
       "1       1         0  17 yo male with recurrent palpitations for the...\n",
       "2       2         0  Dillon Cleveland is a 17 y.o. male patient wit..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "case_num_zero_notes = patient_notes[patient_notes[\"case_num\"] == 0]\n",
    "case_num_zero_notes.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- case num 0 feature text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_num</th>\n",
       "      <th>case_num</th>\n",
       "      <th>feature_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Family-history-of-MI-OR-Family-history-of-myoc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Family-history-of-thyroid-disorder</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Chest-pressure</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   feature_num  case_num                                       feature_text\n",
       "0            0         0  Family-history-of-MI-OR-Family-history-of-myoc...\n",
       "1            1         0                 Family-history-of-thyroid-disorder\n",
       "2            2         0                                     Chest-pressure"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "case_num_zero_features = features[features[\"case_num\"] == 0]\n",
    "case_num_zero_features.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merge of the notes and features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create function to repeat this for each note in the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_doc_ents(clean_text):\n",
    "    \n",
    "    sci_doc = sci_nlp(clean_text)\n",
    "    # print(\"TEXT\", \"START\", \"END\", \"ENTITY TYPE\")\n",
    "    return {ent.text: [ent.start_char, ent.end_char] for ent in sci_doc.ents}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "case_num_feature_text = {}\n",
    "for i in range(10):\n",
    "    case_num_feature_text[i] = features[\"feature_text\"][features[\"case_num\"] == i].to_list()\n",
    "case_num_feature_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>case_num</th>\n",
       "      <th>feature_num</th>\n",
       "      <th>feature_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Family-history-of-MI-OR-Family-history-of-myoc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Family-history-of-thyroid-disorder</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>Chest-pressure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Intermittent-symptoms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>Lightheaded</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>9</td>\n",
       "      <td>912</td>\n",
       "      <td>Family-history-of-migraines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>9</td>\n",
       "      <td>913</td>\n",
       "      <td>Female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>9</td>\n",
       "      <td>914</td>\n",
       "      <td>Photophobia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>9</td>\n",
       "      <td>915</td>\n",
       "      <td>No-known-illness-contacts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>9</td>\n",
       "      <td>916</td>\n",
       "      <td>Subjective-fever</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>143 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     case_num  feature_num                                       feature_text\n",
       "0           0            0  Family-history-of-MI-OR-Family-history-of-myoc...\n",
       "1           0            1                 Family-history-of-thyroid-disorder\n",
       "2           0            2                                     Chest-pressure\n",
       "3           0            3                              Intermittent-symptoms\n",
       "4           0            4                                        Lightheaded\n",
       "..        ...          ...                                                ...\n",
       "138         9          912                        Family-history-of-migraines\n",
       "139         9          913                                             Female\n",
       "140         9          914                                        Photophobia\n",
       "141         9          915                          No-known-illness-contacts\n",
       "142         9          916                                   Subjective-fever\n",
       "\n",
       "[143 rows x 3 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features[[\"case_num\", \"feature_num\", \"feature_text\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### creating dictionary of dictionaries\n",
    "- first dictionary key is the case_num\n",
    "- first dictionary value (second dictionary key) is the feature_num\n",
    "- second dictionary value is the feature_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [ 0  1  2  3  4  5  6  7  8  9 10 11 12]\n",
      "1 [100 101 102 103 104 105 106 107 108 109 110 111 112]\n",
      "2 [200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216]\n",
      "3 [300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315]\n",
      "4 [400 401 402 403 404 405 406 407 408 409]\n",
      "5 [500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517]\n",
      "6 [600 601 602 603 604 605 606 607 608 609 610 611]\n",
      "7 [700 701 702 703 704 705 706 707 708]\n",
      "8 [800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817]\n",
      "9 [900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916]\n"
     ]
    }
   ],
   "source": [
    "# looking at feature_num per case_num\n",
    "for i in features[\"case_num\"].unique():\n",
    "    print(i, features[\"feature_num\"][features[\"case_num\"] == i].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Family-history-of-MI-OR-Family-history-of-myoc...\n",
      "Family-history-of-thyroid-disorder\n",
      "Chest-pressure\n",
      "Intermittent-symptoms\n",
      "Lightheaded\n",
      "No-hair-changes-OR-no-nail-changes-OR-no-tempe...\n",
      "Adderall-use\n",
      "Shortness-of-breath\n",
      "Caffeine-use\n",
      "heart-pounding-OR-heart-racing\n",
      "Few-months-duration\n",
      "17-year\n",
      "Male\n"
     ]
    }
   ],
   "source": [
    "# testing how to get only strings\n",
    "for a in features[\"feature_num\"][features[\"case_num\"] == 0]:\n",
    "    print(features[\"feature_text\"][features[\"feature_num\"] == a].to_string()[5:].strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {}\n",
    "for i in features[\"case_num\"].unique():\n",
    "    d[i] = {a: features[\"feature_text\"][features[\"feature_num\"] == a].to_string()[5:].strip() for a in features[\"feature_num\"][features[\"case_num\"] == i]}\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>case_num</th>\n",
       "      <th>features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>{100: 'No-vaginal-discharge', 101: 'Weight-los...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>{200: 'Prior-normal-periods', 201: 'Last-Pap-s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>{300: 'FHx-of-PUD-OR-Family-history-of-peptic-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>{400: 'Lack-of-other-thyroid-symptoms', 401: '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>{500: 'Onset-5-years-ago', 501: 'Female', 502:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>{600: 'Subjective-fevers', 601: 'Male', 602: '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>{700: 'Female', 701: 'Weight-Gain', 702: 'heav...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>{800: 'Increased-appetite', 801: 'Son-died-3-w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  case_num                                           features\n",
       "0        0  {0: 'Family-history-of-MI-OR-Family-history-of...\n",
       "1        1  {100: 'No-vaginal-discharge', 101: 'Weight-los...\n",
       "2        2  {200: 'Prior-normal-periods', 201: 'Last-Pap-s...\n",
       "3        3  {300: 'FHx-of-PUD-OR-Family-history-of-peptic-...\n",
       "4        4  {400: 'Lack-of-other-thyroid-symptoms', 401: '...\n",
       "5        5  {500: 'Onset-5-years-ago', 501: 'Female', 502:...\n",
       "6        6  {600: 'Subjective-fevers', 601: 'Male', 602: '...\n",
       "7        7  {700: 'Female', 701: 'Weight-Gain', 702: 'heav...\n",
       "8        8  {800: 'Increased-appetite', 801: 'Son-died-3-w...\n",
       "9        9  {900: 'No-relief-with-Motrin-OR-no-relief-with..."
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data=[d.keys(), d.values()]).T\n",
    "df.columns = [\"case_num\", \"features\"]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pn_num</th>\n",
       "      <th>case_num</th>\n",
       "      <th>pn_history</th>\n",
       "      <th>features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17-year-old male, has come to the student heal...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17 yo male with recurrent palpitations for the...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Dillon Cleveland is a 17 y.o. male patient wit...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>a 17 yo m c/o palpitation started 3 mos ago; \\...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>17yo male with no pmh here for evaluation of p...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42141</th>\n",
       "      <td>95330</td>\n",
       "      <td>9</td>\n",
       "      <td>Ms. Madden is a 20 yo female presenting w/ the...</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42142</th>\n",
       "      <td>95331</td>\n",
       "      <td>9</td>\n",
       "      <td>A 20 YO F CAME COMPLAIN A DULL 8/10 HEADACHE T...</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42143</th>\n",
       "      <td>95332</td>\n",
       "      <td>9</td>\n",
       "      <td>Ms. Madden is a 20yo female who presents with ...</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42144</th>\n",
       "      <td>95333</td>\n",
       "      <td>9</td>\n",
       "      <td>Stephanie madden is a 20 year old woman compla...</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42145</th>\n",
       "      <td>95334</td>\n",
       "      <td>9</td>\n",
       "      <td>patient is a 20 yo F who presents with a heada...</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>42146 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       pn_num case_num                                         pn_history  \\\n",
       "0           0        0  17-year-old male, has come to the student heal...   \n",
       "1           1        0  17 yo male with recurrent palpitations for the...   \n",
       "2           2        0  Dillon Cleveland is a 17 y.o. male patient wit...   \n",
       "3           3        0  a 17 yo m c/o palpitation started 3 mos ago; \\...   \n",
       "4           4        0  17yo male with no pmh here for evaluation of p...   \n",
       "...       ...      ...                                                ...   \n",
       "42141   95330        9  Ms. Madden is a 20 yo female presenting w/ the...   \n",
       "42142   95331        9  A 20 YO F CAME COMPLAIN A DULL 8/10 HEADACHE T...   \n",
       "42143   95332        9  Ms. Madden is a 20yo female who presents with ...   \n",
       "42144   95333        9  Stephanie madden is a 20 year old woman compla...   \n",
       "42145   95334        9  patient is a 20 yo F who presents with a heada...   \n",
       "\n",
       "                                                features  \n",
       "0      {0: 'Family-history-of-MI-OR-Family-history-of...  \n",
       "1      {0: 'Family-history-of-MI-OR-Family-history-of...  \n",
       "2      {0: 'Family-history-of-MI-OR-Family-history-of...  \n",
       "3      {0: 'Family-history-of-MI-OR-Family-history-of...  \n",
       "4      {0: 'Family-history-of-MI-OR-Family-history-of...  \n",
       "...                                                  ...  \n",
       "42141  {900: 'No-relief-with-Motrin-OR-no-relief-with...  \n",
       "42142  {900: 'No-relief-with-Motrin-OR-no-relief-with...  \n",
       "42143  {900: 'No-relief-with-Motrin-OR-no-relief-with...  \n",
       "42144  {900: 'No-relief-with-Motrin-OR-no-relief-with...  \n",
       "42145  {900: 'No-relief-with-Motrin-OR-no-relief-with...  \n",
       "\n",
       "[42146 rows x 4 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes_features2 = patient_notes.merge(right=df, left_on=\"case_num\", right_on=\"case_num\")\n",
    "notes_features2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3    9753\n",
       "5    6909\n",
       "4    5405\n",
       "9    5151\n",
       "8    4196\n",
       "7    4101\n",
       "0    2268\n",
       "2    1958\n",
       "6    1597\n",
       "1     808\n",
       "Name: case_num, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes_features2.case_num.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### redo cleaning and finding entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pn_num</th>\n",
       "      <th>case_num</th>\n",
       "      <th>pn_history</th>\n",
       "      <th>features</th>\n",
       "      <th>clean_pn_history</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17-year-old male, has come to the student heal...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>17-year-old male come student health clinic co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17 yo male with recurrent palpitations for the...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>17 yo male recurrent palpitations past 3 mo la...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Dillon Cleveland is a 17 y.o. male patient wit...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>dillon cleveland 17 male patient no significan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>a 17 yo m c/o palpitation started 3 mos ago; \\...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>17 yo c/o palpitation started 3 mos ago nothin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>17yo male with no pmh here for evaluation of p...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>17yo male no pmh evaluation palpitations state...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pn_num case_num                                         pn_history  \\\n",
       "0       0        0  17-year-old male, has come to the student heal...   \n",
       "1       1        0  17 yo male with recurrent palpitations for the...   \n",
       "2       2        0  Dillon Cleveland is a 17 y.o. male patient wit...   \n",
       "3       3        0  a 17 yo m c/o palpitation started 3 mos ago; \\...   \n",
       "4       4        0  17yo male with no pmh here for evaluation of p...   \n",
       "\n",
       "                                            features  \\\n",
       "0  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "1  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "2  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "3  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "4  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "\n",
       "                                    clean_pn_history  \n",
       "0  17-year-old male come student health clinic co...  \n",
       "1  17 yo male recurrent palpitations past 3 mo la...  \n",
       "2  dillon cleveland 17 male patient no significan...  \n",
       "3  17 yo c/o palpitation started 3 mos ago nothin...  \n",
       "4  17yo male no pmh evaluation palpitations state...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes_features2[\"clean_pn_history\"] = notes_features2[\"pn_history\"].apply(prepare_jag.basic_clean3).apply(prepare_jag.remove_stopwords, exclude_words=[\"no\"])\n",
    "notes_features2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pn_num</th>\n",
       "      <th>case_num</th>\n",
       "      <th>pn_history</th>\n",
       "      <th>features</th>\n",
       "      <th>clean_pn_history</th>\n",
       "      <th>note_entities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17-year-old male, has come to the student heal...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>17-year-old male come student health clinic co...</td>\n",
       "      <td>{'heart pounding': [56, 70], 'dispnea': [257, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17 yo male with recurrent palpitations for the...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>17 yo male recurrent palpitations past 3 mo la...</td>\n",
       "      <td>{'palpitations': [21, 33], 'chest catching bre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Dillon Cleveland is a 17 y.o. male patient wit...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>dillon cleveland 17 male patient no significan...</td>\n",
       "      <td>{'complaints heart pounding': [61, 86], 'pain'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>a 17 yo m c/o palpitation started 3 mos ago; \\...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>17 yo c/o palpitation started 3 mos ago nothin...</td>\n",
       "      <td>{'palpitation': [10, 21], 'nausea vomiting hea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>17yo male with no pmh here for evaluation of p...</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>17yo male no pmh evaluation palpitations state...</td>\n",
       "      <td>{'palpitations': [28, 40], '1-2 times month pe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pn_num case_num                                         pn_history  \\\n",
       "0       0        0  17-year-old male, has come to the student heal...   \n",
       "1       1        0  17 yo male with recurrent palpitations for the...   \n",
       "2       2        0  Dillon Cleveland is a 17 y.o. male patient wit...   \n",
       "3       3        0  a 17 yo m c/o palpitation started 3 mos ago; \\...   \n",
       "4       4        0  17yo male with no pmh here for evaluation of p...   \n",
       "\n",
       "                                            features  \\\n",
       "0  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "1  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "2  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "3  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "4  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "\n",
       "                                    clean_pn_history  \\\n",
       "0  17-year-old male come student health clinic co...   \n",
       "1  17 yo male recurrent palpitations past 3 mo la...   \n",
       "2  dillon cleveland 17 male patient no significan...   \n",
       "3  17 yo c/o palpitation started 3 mos ago nothin...   \n",
       "4  17yo male no pmh evaluation palpitations state...   \n",
       "\n",
       "                                       note_entities  \n",
       "0  {'heart pounding': [56, 70], 'dispnea': [257, ...  \n",
       "1  {'palpitations': [21, 33], 'chest catching bre...  \n",
       "2  {'complaints heart pounding': [61, 86], 'pain'...  \n",
       "3  {'palpitation': [10, 21], 'nausea vomiting hea...  \n",
       "4  {'palpitations': [28, 40], '1-2 times month pe...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes_features2[\"note_entities\"] = notes_features2[\"clean_pn_history\"].apply(get_doc_ents)\n",
    "notes_features2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_records_show = int(input(\"Please select how many records to show. Must be less than 1000, but the smaller the better due to timing and length: \"))\n",
    "\n",
    "case_num_one = notes_features2[\"pn_history\"][notes_features2[\"case_num\"] == 1][:num_records_show]\n",
    "whole_case_num_one = \"\\n\".join(case_num_one.to_list())\n",
    "\n",
    "sci_doc = sci_nlp(whole_case_num_one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy.displacy.render(sci_doc, style='ent', jupyter=True);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From Scispacy GitHub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/scispacy/lib/python3.9/site-packages/sklearn/base.py:329: UserWarning: Trying to unpickle estimator TfidfTransformer from version 0.20.3 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n",
      "/opt/miniconda3/envs/scispacy/lib/python3.9/site-packages/sklearn/base.py:329: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 0.20.3 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<scispacy.linking.EntityLinker at 0x7f900d41d8e0>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# NOTE: The resolve_abbreviations parameter is optional, and requires that\n",
    "# the AbbreviationDetector pipe has already been added to the pipeline. Adding\n",
    "# the AbbreviationDetector pipe and setting resolve_abbreviations to True means\n",
    "# that linking will only be performed on the long form of abbreviations.\n",
    "sci_nlp1 = spacy.load(\"en_ner_bc5cdr_md\")\n",
    "sci_nlp1.add_pipe(\"scispacy_linker\", config={\"linker_name\": \"umls\"}) # {\"resolve_abbreviations\": True,} "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### with case_num == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: abdominal pain\n"
     ]
    }
   ],
   "source": [
    "sci_doc1 = sci_nlp(whole_case_num_one)\n",
    "# Let's look at a random entity!\n",
    "# sci_doc1 comes from above\n",
    "entity1 = sci_doc1.ents[1]\n",
    "print(f\"Name: {entity1}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Each entity is linked to UMLS with a score\n",
    "# (currently just char-3gram matching).\n",
    "linker = sci_nlp1.get_pipe(\"scispacy_linker\")\n",
    "for umls_ent in entity1._.kb_ents:\n",
    "\tprint(linker.kb.cui_to_entity[umls_ent[0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### with case_num == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "case_num_zero = notes_features2[\"pn_history\"][notes_features2[\"case_num\"] == 0][:num_records_show]\n",
    "whole_case_num_zero = \"\\n\".join(case_num_zero.to_list())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: dispnea\n"
     ]
    }
   ],
   "source": [
    "sci_doc1 = sci_nlp(whole_case_num_zero)\n",
    "# Let's look at a random entity!\n",
    "# sci_doc1 comes from above\n",
    "entity1 = sci_doc1.ents[1]\n",
    "print(f\"Name: {entity1}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "entities = sci_doc1.ents[0]\n",
    "linker = sci_nlp1.get_pipe(\"scispacy_linker\")\n",
    "for umls_ent in entities._.kb_ents:\n",
    "    print(linker.kb.cui_to_entity[umls_ent[0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### comparing locations to features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>case_num</th>\n",
       "      <th>features</th>\n",
       "      <th>note_entities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>{'heart pounding': [56, 70], 'dispnea': [257, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>{'palpitations': [21, 33], 'chest catching bre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>{'complaints heart pounding': [61, 86], 'pain'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>{'palpitation': [10, 21], 'nausea vomiting hea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>{0: 'Family-history-of-MI-OR-Family-history-of...</td>\n",
       "      <td>{'palpitations': [28, 40], '1-2 times month pe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42141</th>\n",
       "      <td>9</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "      <td>{'pain': [378, 382], 'phonophobia nausea': [19...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42142</th>\n",
       "      <td>9</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "      <td>{'headache': [138, 146], 'vomiting': [67, 75],...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42143</th>\n",
       "      <td>9</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "      <td>{'headache': [92, 100], 'throbbing': [76, 85],...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42144</th>\n",
       "      <td>9</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "      <td>{'headache pain': [47, 60], 'ache': [113, 117]...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42145</th>\n",
       "      <td>9</td>\n",
       "      <td>{900: 'No-relief-with-Motrin-OR-no-relief-with...</td>\n",
       "      <td>{'headache': [25, 33], 'pain': [231, 235], 'ib...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>42146 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      case_num                                           features  \\\n",
       "0            0  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "1            0  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "2            0  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "3            0  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "4            0  {0: 'Family-history-of-MI-OR-Family-history-of...   \n",
       "...        ...                                                ...   \n",
       "42141        9  {900: 'No-relief-with-Motrin-OR-no-relief-with...   \n",
       "42142        9  {900: 'No-relief-with-Motrin-OR-no-relief-with...   \n",
       "42143        9  {900: 'No-relief-with-Motrin-OR-no-relief-with...   \n",
       "42144        9  {900: 'No-relief-with-Motrin-OR-no-relief-with...   \n",
       "42145        9  {900: 'No-relief-with-Motrin-OR-no-relief-with...   \n",
       "\n",
       "                                           note_entities  \n",
       "0      {'heart pounding': [56, 70], 'dispnea': [257, ...  \n",
       "1      {'palpitations': [21, 33], 'chest catching bre...  \n",
       "2      {'complaints heart pounding': [61, 86], 'pain'...  \n",
       "3      {'palpitation': [10, 21], 'nausea vomiting hea...  \n",
       "4      {'palpitations': [28, 40], '1-2 times month pe...  \n",
       "...                                                  ...  \n",
       "42141  {'pain': [378, 382], 'phonophobia nausea': [19...  \n",
       "42142  {'headache': [138, 146], 'vomiting': [67, 75],...  \n",
       "42143  {'headache': [92, 100], 'throbbing': [76, 85],...  \n",
       "42144  {'headache pain': [47, 60], 'ache': [113, 117]...  \n",
       "42145  {'headache': [25, 33], 'pain': [231, 235], 'ib...  \n",
       "\n",
       "[42146 rows x 3 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes_features2[[\"case_num\", \"features\", \"note_entities\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[dict_keys([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),\n",
       " dict_keys([100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112]),\n",
       " dict_keys([200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216]),\n",
       " dict_keys([300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315]),\n",
       " dict_keys([400, 401, 402, 403, 404, 405, 406, 407, 408, 409]),\n",
       " dict_keys([500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517]),\n",
       " dict_keys([600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611]),\n",
       " dict_keys([700, 701, 702, 703, 704, 705, 706, 707, 708]),\n",
       " dict_keys([800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817]),\n",
       " dict_keys([900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916])]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d_keys = []\n",
    "for dic in notes_features2[\"features\"]:\n",
    "    if dic.keys() not in d_keys:\n",
    "        d_keys.append(dic.keys())\n",
    "d_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "224266041085884618f6085fcfb69664e3b25455f6c771e6653be1f9fbf573ea"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}