Project_Task2_GroupC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import pandas as pd\n",
    "import huggingface_hub\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import spacy\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from spacy.scorer import Scorer\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ground_truth(file_name):\n",
    "    ''' Read the ground truth from the json file and return the entities\n",
    "        Input: file_name: the name of the json file'''\n",
    "    \n",
    "    file = open(file_name, \"r\")\n",
    "    annotations = json.load(file)\n",
    "    return annotations['annotations'][0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_ner_metric(predicted_entities, ground_truth_entities):\n",
    "    ''' Compute the precision, recall, f1-score and accuracy for the predicted entities\n",
    "        Input: predicted_entities: the list of predicted entities, tuples (word, entity)\n",
    "               ground_truth_entities: the list of ground truth entities, tuples (word, entity)'''\n",
    "    \n",
    "    # Initialize counters for TP, FP, and FN\n",
    "    tp, fp, fn = 0, 0, 0\n",
    "\n",
    "    # Create sets of predicted entities and ground truth entities\n",
    "    predicted_entities_set = set(predicted_entities)\n",
    "    ground_truth_entities_set = set(ground_truth_entities)\n",
    "\n",
    "    # Calculate True Positives (TP)\n",
    "    tp = len(predicted_entities_set.intersection(ground_truth_entities_set))\n",
    "\n",
    "    # Calculate False Positives (FP)\n",
    "    fp = len(predicted_entities_set - ground_truth_entities_set)\n",
    "\n",
    "    # Calculate False Negatives (FN)\n",
    "    fn = len(ground_truth_entities_set - predicted_entities_set)\n",
    "\n",
    "    # Calculate Precision, Recall, and F1-Score\n",
    "    precision = tp / (tp + fp) if tp + fp > 0 else 0\n",
    "    recall = tp / (tp + fn) if tp + fn > 0 else 0\n",
    "    f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0\n",
    "\n",
    "    return precision, recall, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ner_ent(index, dataset_p, nlp, med7):\n",
    "    ''' Get the entities from the text and display them\n",
    "        Input: index: the index of the text in the dataset\n",
    "               dataset_p: the dataset\n",
    "               nlp: the nlp model\n",
    "               med7: the med7 model '''\n",
    "    \n",
    "    print('ID: ', index, '\\n')\n",
    "    # List for the colours of the entities to be displayed\n",
    "    col_dict = {}\n",
    "    colours = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf', \n",
    "               '#aec7e8', '#ffbb78', '#98df8a', '#ff9896', '#c5b0d5', '#c49c94', '#f7b6d2', '#c7c7c7', '#dbdb8d', '#9edae5', \n",
    "               '#7f7f7f', '#dbdb8d', '#7fc97f', '#beaed4', '#fdc086', '#ffff99', '#386cb0', '#f0027f', '#bf5b17']\n",
    "\n",
    "\n",
    "    # Get the entities list from the models\n",
    "    nlp_ner = list(nlp.pipe_labels['ner'])\n",
    "    med7_ner = list(med7.pipe_labels['ner'])\n",
    "    combined_ner = nlp_ner + med7_ner\n",
    "\n",
    "    # Create a dictionary with the entities and the colours\n",
    "    l_combined_ner = len(combined_ner)\n",
    "    for label, colour in zip(combined_ner, colours[:l_combined_ner]):\n",
    "        col_dict[label] = colour\n",
    "\n",
    "    # Create the options for the display\n",
    "    options = {'entities': combined_ner, 'colors':col_dict}\n",
    "\n",
    "    # Get the text entities from the dataset\n",
    "    text = dataset_p.loc[index, 'tokens']\n",
    "    doc_web_sm = nlp(text)\n",
    "\n",
    "    doc_med7 = med7(text)\n",
    "    docs = [doc_web_sm, doc_med7]\n",
    "\n",
    "    # Display the entities\n",
    "    spacy.displacy.render(docs, style='ent', jupyter=True, options=options)\n",
    "\n",
    "    #  Get the entities from the text\n",
    "    combined_entities = []\n",
    "    for doc in docs:         \n",
    "        combined_entities.extend([(ent.text, ent.label_) for ent in doc.ents])\n",
    "\n",
    "    return combined_entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_text(text):\n",
    "    ''' Process the text: lowercasing, lemmatization, stopwords removal, \n",
    "        and punctuation removal\n",
    "        Input: text: the text to be processed'''\n",
    "    text = text.lower()\n",
    "    text = text.replace('\\n', ' ')\n",
    "    text = text.replace('\\t', ' ')\n",
    "    text = text.replace('\\r', ' ')\n",
    "    text = text.replace('  ', ' ')\n",
    "\n",
    "    # Word tokenization \n",
    "    tokens = word_tokenize(text)\n",
    "\n",
    "    # Normalization (lowercasing and lemmatization)\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    tokens = [lemmatizer.lemmatize(token) for token in tokens]\n",
    "\n",
    "    # Token filtering (stopwords removal)\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    tokens = [token for token in tokens if token not in stop_words]\n",
    "\n",
    "    # Remove punctuation tokens\n",
    "    tokens = [token for token in tokens if token.isalpha()]\n",
    "\n",
    "    # Join the tokens back into a string\n",
    "    text = ' '.join(tokens)\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Task 1 recap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we are going to set the environment that we created in Task 1 for the preprocessing of the text using the pipeline provided in our Task 1 notebook. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the dataset\n",
    "dataset = load_dataset(\"argilla/medical-domain\")\n",
    "\n",
    "# preprocess the data\n",
    "dataset_p = pd.DataFrame(columns=['tokens'])\n",
    "for i in range(len(dataset['train'])):\n",
    "    text = dataset['train'][i]['text']\n",
    "    text_p = process_text(text)\n",
    "    dataset_p.loc[i] = [text_p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>preoperative diagnosis iron deficiency postope...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>clinical indication normal stress procedure pe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>finding scan performed reformatted image obtai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>preoperative diagnosis blood loss postoperativ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>reason visit elevated psa nocturia occasional ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              tokens\n",
       "0  preoperative diagnosis iron deficiency postope...\n",
       "1  clinical indication normal stress procedure pe...\n",
       "2  finding scan performed reformatted image obtai...\n",
       "3  preoperative diagnosis blood loss postoperativ...\n",
       "4  reason visit elevated psa nocturia occasional ..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_p.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### NER Recognition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The standard NER types can be obtained from the \"en_core_web_sm\" model. And are the following:\n",
    "\n",
    "CARDINAL, DATE, EVENT, FAC, GPE, LANGUAGE, LAW, LOC, MONEY, NORP, ORDINAL, ORG, PERCENT, PERSON, PRODUCT, QUANTITY, TIME, WORK_OF_ART."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CARDINAL',\n",
       " 'DATE',\n",
       " 'EVENT',\n",
       " 'FAC',\n",
       " 'GPE',\n",
       " 'LANGUAGE',\n",
       " 'LAW',\n",
       " 'LOC',\n",
       " 'MONEY',\n",
       " 'NORP',\n",
       " 'ORDINAL',\n",
       " 'ORG',\n",
       " 'PERCENT',\n",
       " 'PERSON',\n",
       " 'PRODUCT',\n",
       " 'QUANTITY',\n",
       " 'TIME',\n",
       " 'WORK_OF_ART']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load nlp english model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "list(nlp.pipe_labels['ner'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our documents are medical related, so we should look for NER classes that can explain some medical terms as medication or treatments to extend the standard NER types.\n",
    "By manual inspection we identified following potential new NER types:\n",
    "- Healthcare profession \n",
    "- Administrative event (e.g. hospitalization)\n",
    "- Care environment\n",
    "- Demographic\n",
    "- Anatomy\n",
    "- Diseases\n",
    "- Symptom\n",
    "- Examination\n",
    "- Measurement unit\n",
    "- Measurement value\n",
    "- Treatment\n",
    "- Medication\n",
    "- Medication Dosage\n",
    "- Medication form (e.g. tablet)\n",
    "- Medication route\n",
    "\n",
    "To extend our NER classification we can't use all of them, because we are restricted to labels which our classifiers are trained on. In order to extend our classes we are going to use the med7 NER model (https://github.com/kormilitzin/med7) composed from the following NER labels:\n",
    "DOSAGE, DRUG, DURATION, FORM, FREQUENCY, ROUTE, STRENGTH"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To install the med7 model follow the instructions on the git repository (we chose the option of downloading it from dropbox and install it with pip)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DOSAGE', 'DRUG', 'DURATION', 'FORM', 'FREQUENCY', 'ROUTE', 'STRENGTH']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "med7 = spacy.load(\"en_core_med7_lg\")\n",
    "list(med7.pipe_labels['ner'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Apply NER types and evaluate."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To apply the NER classifier to our data we have implemented the get_ner_ent function. Will be applied after together with the evaluation of our samples.\n",
    "\n",
    "To evaluate the NER classifier we first need to create our GT data. For that we are going to manually annonate 4 texts with the standard NER types and the extended classes. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The GT annotations are been done exporting the NER types from the four text using NER Annotator (https://tecoholic.github.io/ner-annotator/). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Select 4 random samples: these random samples id are then stored \n",
    "in the idx array for further replication.'''\n",
    "# l = len(dataset_p)\n",
    "# select 10 random samples\n",
    "# idx = np.random.randint(0, l, 9)\n",
    "idx = np.array([2906, 4146, 4671, 1074])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2906  -->  preoperative diagnosis tympanic membrane atelectasis chronic eustachian tube postoperative diagnosis tympanic membrane atelectasis chronic eustachian tube operative procedure bilateral myringotomy insertion santa barbara anesthesia general finding patient white female chronic eustachian tube dysfunction tm atelectasis wa taken operating room tube time surgery ha extruding right santa barbara severe left tm atelectasis retraction wa scant amount fluid middle ear description procedure patient wa taken operating room placed supine position general mask anesthesia wa established right ear wa draped normal sterile fashion cerumen wa removed external canal extruding santa barbara wa identified atraumatically removed fresh santa barbara wa atraumatically inserted ciloxan drop attention wa directed left side severe tm atelectasis wa identified mask anesthetic eardrum elevated radial incision wa made inferior aspect tympanic membrane middle ear fluid aspirated santa barbara wa inserted without difficulty drop ciloxan solution applied anesthesia wa reversed patient taken recovery room satisfactory condition\n",
      "idx:  4146  -->  procedure endoscopic retrograde cholangiopancreatography brush cytology indication procedure patient history chronic abdominal pain ct showing evidence chronic pancreatitis recent upper endoscopy showing medication general risk procedure made aware patient consisted medication reaction bleeding perforation aspiration post ercp description procedure informed consent appropriate sedation duodenoscope wa inserted oropharynx esophagus stomach scope wa advanced pylorus ampulla ampulla markedly abnormal appearance wa enlarged prominent extended outward almost polypoid shape appeared mucosa tip also wa ulceration noted tip ampulla biliary pancreatic orifice identified wa located tip ampulla rather towards base cannulation wa performed tritome sphincterotome easy cannulation biliary tree common bile duct wa mildly dilated measuring approximately mm intrahepatic duct minimally dilated filling defect identified wa felt possible stricture within distal common bile duct likely represented anatomic variant given abnormal shape ampulla patient ha evidence obstruction based lab work clinically nevertheless wa decided proceed brush cytology segment wa done without complication wa adequate drainage biliary tree noted throughout procedure multiple effort made access pancreatic ductal anatomy however shape ampulla wa unsuccessful effort made proceed long scope position still unsuccessful next biopsy obtained ampulla away biliary orifice four biopsy taken wa minor oozing ceased end procedure stomach wa decompressed endoscope wa finding abnormal papilla bulging polypoid appearance look adenomatous ulceration tip biopsy cholangiogram reveals mildly dilated common bile duct measuring mm possible distal cbd stricture although think likely anatomic variant brush cytology unable access pancreatic recommendation npo except ice chip proceed mrcp better delineate pancreatic ductal follow biopsy cytology\n",
      "idx:  4671  -->  preoperative diagnosis adenotonsillar hypertrophy chronic otitis postoperative diagnosis adenotonsillar hypertrophy chronic otitis procedure performed tympanostomy tube anesthesia general description procedure patient wa taken operating room prepped draped usual fashion induction general endotracheal anesthesia mcivor mouth gag wa placed oral cavity tongue depressor applied two red rubber robinson catheter placed nasal passage brought oral cavity clamped dental gauze roll placed upper lip provide soft palate attention wa directed nasopharynx bovie set coag suction bovie tip suction hose adenoid bed wa fulgurated beginning posterosuperior aspect nasopharynx apex choana placing tip suction cautery deep root adenoid next roof nasopharynx linear fashion making serial passage base adenoid fossa parallel line entire nasopharynx adenoid bed fulgurated moving posterior anterior mcivor wa relaxed attention wa directed left external auditory canal wa examined operating microscope cleaned ceruminous anteroinferior quadrant tympanostomy incision wa made fluid wa suctioned middle ear space tympanostomy tube wa placed level incision pushed position rosen needle cortisporin ear drop instilled canal cotton ball wa placed external similar procedure opposite tympanostomy tube placement patient tolerated procedure well left operating room good condition\n",
      "idx:  1074  -->  general evaluation twin b fetal cardiac activity normal bpm fetal lie longitudinal maternal fetal presentation placenta fused posterior placenta grade uterus normal cervix adnexa seen amniotic fluid afi single ap biometry consistent day gestation hc consistent week day gestation ac consistent week day gestation fl\n"
     ]
    }
   ],
   "source": [
    "# test data txt: ground truth annotations\n",
    "for i in idx:\n",
    "    print('idx: ', i, ' --> ', dataset_p.loc[i]['tokens'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID:  2906 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">preoperative diagnosis \n",
       "<mark class=\"entity\" style=\"background: #c5b0d5; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    tympanic\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PRODUCT</span>\n",
       "</mark>\n",
       " membrane atelectasis chronic eustachian tube postoperative diagnosis \n",
       "<mark class=\"entity\" style=\"background: #c5b0d5; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    tympanic\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PRODUCT</span>\n",
       "</mark>\n",
       " membrane atelectasis chronic eustachian tube operative procedure bilateral myringotomy insertion \n",
       "<mark class=\"entity\" style=\"background: #9467bd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    santa barbara\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       " anesthesia general finding patient white female chronic eustachian tube dysfunction tm atelectasis wa taken operating room tube time surgery ha extruding right \n",
       "<mark class=\"entity\" style=\"background: #9467bd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    santa barbara\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       " severe left tm atelectasis retraction wa scant amount fluid middle ear description procedure patient wa taken operating room placed supine position general mask anesthesia wa established right ear wa draped normal sterile fashion cerumen wa removed external canal extruding \n",
       "<mark class=\"entity\" style=\"background: #9467bd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    santa barbara\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       " wa identified atraumatically removed fresh \n",
       "<mark class=\"entity\" style=\"background: #9467bd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    santa barbara\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       " wa atraumatically inserted ciloxan drop attention wa directed left side severe tm atelectasis wa identified mask anesthetic eardrum elevated radial incision wa made inferior aspect \n",
       "<mark class=\"entity\" style=\"background: #c5b0d5; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    tympanic\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PRODUCT</span>\n",
       "</mark>\n",
       " membrane middle ear fluid aspirated \n",
       "<mark class=\"entity\" style=\"background: #9467bd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    santa barbara\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       " wa inserted without difficulty drop ciloxan solution applied \n",
       "<mark class=\"entity\" style=\"background: #9467bd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    anesthesia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       " wa reversed patient taken recovery room satisfactory condition</div>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">preoperative diagnosis tympanic membrane atelectasis chronic eustachian tube postoperative diagnosis tympanic membrane atelectasis chronic eustachian tube operative procedure bilateral myringotomy insertion santa barbara anesthesia general finding patient white female chronic eustachian tube dysfunction tm atelectasis wa taken operating room tube time surgery ha extruding right santa barbara severe left tm atelectasis retraction wa scant amount fluid middle ear description procedure patient wa taken operating room placed supine position general mask anesthesia wa established right ear wa draped normal sterile fashion cerumen wa removed external canal extruding santa barbara wa identified atraumatically removed fresh santa barbara wa atraumatically \n",
       "<mark class=\"entity\" style=\"background: #dbdb8d; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    inserted\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">FORM</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: #9edae5; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    ciloxan\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DRUG</span>\n",
       "</mark>\n",
       " drop attention wa directed left side severe tm atelectasis wa identified mask anesthetic eardrum elevated radial incision wa made inferior aspect tympanic membrane middle ear fluid aspirated santa barbara wa inserted without difficulty drop \n",
       "<mark class=\"entity\" style=\"background: #9edae5; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    ciloxan\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DRUG</span>\n",
       "</mark>\n",
       " solution applied \n",
       "<mark class=\"entity\" style=\"background: #9edae5; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    anesthesia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DRUG</span>\n",
       "</mark>\n",
       " wa reversed patient taken recovery room satisfactory condition</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER Entities:  [('tympanic', 'PRODUCT'), ('tympanic', 'PRODUCT'), ('santa barbara', 'GPE'), ('santa barbara', 'GPE'), ('santa barbara', 'GPE'), ('santa barbara', 'GPE'), ('tympanic', 'PRODUCT'), ('santa barbara', 'GPE'), ('anesthesia', 'GPE'), ('inserted', 'FORM'), ('ciloxan', 'DRUG'), ('ciloxan', 'DRUG'), ('anesthesia', 'DRUG')]\n",
      "Ground Truth:  [('santa barbara', 'GPE'), ('time', 'TIME'), ('ciloxan', 'DRUG'), ('santa barbara', 'GPE'), ('ciloxan', 'DRUG')]\n",
      "\n",
      "ID:  4146 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">procedure endoscopic retrograde cholangiopancreatography brush cytology indication procedure patient history chronic abdominal pain ct showing evidence chronic pancreatitis recent upper endoscopy showing medication general risk procedure made aware patient consisted medication reaction bleeding perforation aspiration post ercp description procedure informed consent appropriate sedation duodenoscope wa inserted oropharynx esophagus stomach scope wa advanced pylorus \n",
       "<mark class=\"entity\" style=\"background: #ff9896; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    ampulla ampulla\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
       "</mark>\n",
       " markedly abnormal appearance wa enlarged prominent extended outward almost polypoid shape appeared mucosa tip also wa ulceration noted tip ampulla biliary pancreatic orifice identified wa located tip ampulla rather towards base cannulation wa performed tritome sphincterotome easy cannulation biliary tree common bile duct wa mildly dilated measuring approximately mm intrahepatic duct minimally dilated filling defect identified wa felt possible stricture within distal common bile duct likely represented anatomic variant given abnormal shape \n",
       "<mark class=\"entity\" style=\"background: #ff9896; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    ampulla\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
       "</mark>\n",
       " patient ha evidence obstruction based lab work clinically nevertheless wa decided proceed brush cytology segment wa done without complication wa adequate drainage biliary tree noted throughout procedure multiple effort made access pancreatic ductal anatomy however shape ampulla wa unsuccessful effort made proceed long scope position still unsuccessful next biopsy obtained ampulla away biliary orifice \n",
       "<mark class=\"entity\" style=\"background: #1f77b4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    four\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CARDINAL</span>\n",
       "</mark>\n",
       " biopsy taken wa minor oozing ceased end procedure stomach wa decompressed endoscope wa finding abnormal papilla bulging polypoid appearance look adenomatous ulceration tip biopsy cholangiogram reveals mildly dilated common bile duct measuring mm possible distal cbd stricture although think likely anatomic variant brush cytology unable access pancreatic recommendation npo except ice chip proceed mrcp better delineate pancreatic ductal follow biopsy cytology</div>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">procedure endoscopic retrograde cholangiopancreatography brush cytology indication procedure patient history chronic abdominal pain ct showing evidence chronic pancreatitis recent upper endoscopy showing medication general risk procedure made aware patient consisted medication reaction bleeding perforation aspiration post ercp description procedure informed consent appropriate sedation duodenoscope wa inserted oropharynx esophagus stomach scope wa advanced pylorus ampulla ampulla markedly abnormal appearance wa enlarged prominent extended outward almost polypoid shape appeared mucosa tip also wa ulceration noted tip ampulla biliary pancreatic orifice identified wa located tip ampulla rather towards base cannulation wa performed \n",
       "<mark class=\"entity\" style=\"background: #9edae5; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    tritome\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DRUG</span>\n",
       "</mark>\n",
       " sphincterotome easy cannulation biliary tree common bile duct wa mildly dilated measuring approximately mm intrahepatic duct minimally dilated filling defect identified wa felt possible stricture within distal common bile duct likely represented anatomic variant given abnormal shape ampulla patient ha evidence obstruction based lab work clinically nevertheless wa decided proceed brush cytology segment wa done without complication wa adequate drainage biliary tree noted throughout procedure multiple effort made access pancreatic ductal anatomy however shape ampulla wa unsuccessful effort made proceed long scope position still unsuccessful next biopsy obtained ampulla away biliary orifice four biopsy taken wa minor oozing ceased end procedure stomach wa decompressed endoscope wa finding abnormal papilla bulging polypoid appearance look adenomatous ulceration tip biopsy cholangiogram reveals mildly dilated common bile duct measuring mm possible distal cbd stricture although think likely anatomic variant brush cytology unable access pancreatic recommendation npo except ice chip proceed mrcp better delineate pancreatic ductal follow biopsy cytology</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER Entities:  [('ampulla ampulla', 'PERSON'), ('ampulla', 'PERSON'), ('four', 'CARDINAL'), ('tritome', 'DRUG')]\n",
      "Ground Truth:  [('ct', 'PRODUCT'), ('medication', 'DRUG'), ('medication', 'DRUG'), ('ercp', 'PRODUCT'), ('pylorus', 'DRUG'), ('ampulla', 'DOSAGE'), ('ampulla', 'DOSAGE'), ('ampulla', 'DOSAGE'), ('ampulla', 'DOSAGE'), ('tritome', 'DRUG'), ('mm', 'QUANTITY'), ('ampulla', 'DOSAGE'), ('ampulla', 'DOSAGE'), ('ampulla', 'DOSAGE'), ('four', 'CARDINAL'), ('mm', 'QUANTITY')]\n",
      "\n",
      "ID:  4671 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">preoperative diagnosis adenotonsillar hypertrophy chronic otitis postoperative diagnosis adenotonsillar hypertrophy chronic otitis procedure performed tympanostomy tube anesthesia general description procedure patient wa taken operating room prepped draped usual fashion induction general endotracheal anesthesia mcivor mouth gag wa placed oral cavity tongue depressor applied \n",
       "<mark class=\"entity\" style=\"background: #1f77b4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    two\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CARDINAL</span>\n",
       "</mark>\n",
       " red rubber robinson catheter placed nasal passage brought oral cavity clamped dental gauze roll placed upper lip provide soft palate attention wa directed \n",
       "<mark class=\"entity\" style=\"background: #1f77b4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    nasopharynx\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CARDINAL</span>\n",
       "</mark>\n",
       " bovie set coag suction bovie tip suction \n",
       "<mark class=\"entity\" style=\"background: #ff9896; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hose adenoid\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
       "</mark>\n",
       " bed wa fulgurated beginning posterosuperior aspect \n",
       "<mark class=\"entity\" style=\"background: #ffbb78; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    nasopharynx apex choana\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " placing tip suction cautery deep root adenoid next roof \n",
       "<mark class=\"entity\" style=\"background: #1f77b4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    nasopharynx\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CARDINAL</span>\n",
       "</mark>\n",
       " linear fashion making serial passage base adenoid fossa parallel line entire nasopharynx adenoid bed fulgurated moving posterior anterior mcivor wa relaxed attention wa directed left external auditory canal wa examined operating microscope cleaned ceruminous anteroinferior quadrant tympanostomy incision wa made fluid wa suctioned middle ear space tympanostomy tube wa placed level incision pushed position rosen needle cortisporin ear drop instilled canal cotton ball wa placed external similar procedure opposite tympanostomy tube placement patient tolerated procedure well left operating room good condition</div>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">preoperative diagnosis adenotonsillar hypertrophy chronic otitis postoperative diagnosis adenotonsillar hypertrophy chronic otitis procedure performed tympanostomy tube anesthesia general description procedure patient wa taken operating room prepped draped usual fashion induction general \n",
       "<mark class=\"entity\" style=\"background: #beaed4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    endotracheal\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ROUTE</span>\n",
       "</mark>\n",
       " anesthesia mcivor mouth gag wa placed oral cavity tongue depressor applied \n",
       "<mark class=\"entity\" style=\"background: #dbdb8d; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    two\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DOSAGE</span>\n",
       "</mark>\n",
       " red rubber robinson catheter placed nasal passage brought oral cavity clamped dental gauze roll placed upper lip provide soft palate attention wa directed nasopharynx bovie set coag suction bovie tip suction hose adenoid bed wa fulgurated beginning posterosuperior aspect nasopharynx apex choana placing tip suction cautery deep root adenoid next roof nasopharynx linear fashion making serial passage base adenoid fossa parallel line entire nasopharynx adenoid bed fulgurated moving posterior anterior mcivor wa relaxed attention wa directed left external auditory canal wa examined operating microscope cleaned ceruminous anteroinferior quadrant tympanostomy incision wa made fluid wa suctioned middle ear space tympanostomy tube wa placed level incision pushed position rosen needle \n",
       "<mark class=\"entity\" style=\"background: #9edae5; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    cortisporin\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DRUG</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: #beaed4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    ear\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ROUTE</span>\n",
       "</mark>\n",
       " drop instilled canal cotton ball wa placed external similar procedure opposite tympanostomy tube placement patient tolerated procedure well left operating room good condition</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER Entities:  [('two', 'CARDINAL'), ('nasopharynx', 'CARDINAL'), ('hose adenoid', 'PERSON'), ('nasopharynx apex choana', 'ORG'), ('nasopharynx', 'CARDINAL'), ('endotracheal', 'ROUTE'), ('two', 'DOSAGE'), ('cortisporin', 'DRUG'), ('ear', 'ROUTE')]\n",
      "Ground Truth:  [('oral', 'ROUTE'), ('nasal', 'ROUTE'), ('oral', 'ROUTE'), ('cortisporin', 'DRUG'), ('drop', 'DOSAGE')]\n",
      "\n",
      "ID:  1074 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">general evaluation twin b fetal cardiac activity \n",
       "<mark class=\"entity\" style=\"background: #ff9896; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    normal bpm fetal\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
       "</mark>\n",
       " lie longitudinal maternal fetal presentation placenta fused posterior \n",
       "<mark class=\"entity\" style=\"background: #9467bd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    placenta\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       " grade uterus normal cervix adnexa seen \n",
       "<mark class=\"entity\" style=\"background: #ffbb78; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    amniotic\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " fluid \n",
       "<mark class=\"entity\" style=\"background: #ffbb78; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    afi\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " single ap biometry consistent day gestation hc consistent \n",
       "<mark class=\"entity\" style=\"background: #ff7f0e; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    week day\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DATE</span>\n",
       "</mark>\n",
       " gestation \n",
       "<mark class=\"entity\" style=\"background: #ffbb78; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    ac\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " consistent \n",
       "<mark class=\"entity\" style=\"background: #ff7f0e; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    week day\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DATE</span>\n",
       "</mark>\n",
       " gestation fl</div>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">general evaluation twin b fetal cardiac activity normal bpm fetal lie longitudinal maternal fetal presentation placenta fused posterior placenta grade uterus normal cervix adnexa seen amniotic fluid afi \n",
       "<mark class=\"entity\" style=\"background: #dbdb8d; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    single\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DOSAGE</span>\n",
       "</mark>\n",
       " ap biometry consistent day gestation hc consistent week day gestation \n",
       "<mark class=\"entity\" style=\"background: #7fc97f; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    ac consistent week day\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">FREQUENCY</span>\n",
       "</mark>\n",
       " gestation \n",
       "<mark class=\"entity\" style=\"background: #dbdb8d; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    fl\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">DOSAGE</span>\n",
       "</mark>\n",
       "</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER Entities:  [('normal bpm fetal', 'PERSON'), ('placenta', 'GPE'), ('amniotic', 'ORG'), ('afi', 'ORG'), ('week day', 'DATE'), ('ac', 'ORG'), ('week day', 'DATE'), ('single', 'DOSAGE'), ('ac consistent week day', 'FREQUENCY'), ('fl', 'DOSAGE')]\n",
      "Ground Truth:  [('bpm', 'QUANTITY'), ('day', 'DURATION'), ('week day', 'DATE'), ('week day', 'DATE')]\n",
      "\n",
      "Precision: 0.37 (+/-) 0.17\n",
      "Recall: 0.27 (+/-) 0.16\n",
      "F1-Score: 0.28 (+/-) 0.12\n"
     ]
    }
   ],
   "source": [
    "# retrieve all test data metrics\n",
    "precission_all = []\n",
    "recall_all = []\n",
    "f1_score_all = []\n",
    "for i in idx:\n",
    "    # ner entities: spacy model\n",
    "    ents_ne = get_ner_ent(i, dataset_p, nlp, med7)\n",
    "\n",
    "    # gt annotations\n",
    "    file_name = 'gt_annotations/{:d}_annotations.json'.format(i)\n",
    "    ents_gt = get_ground_truth(file_name)\n",
    "\n",
    "    list_gt = []\n",
    "    for ent in ents_gt['entities']:\n",
    "        list_gt.append((dataset_p.loc[i, 'tokens'][ent[0]:ent[1]], ent[2]))\n",
    "    \n",
    "    print('NER Entities: ', ents_ne)\n",
    "    print('Ground Truth: ', list_gt)\n",
    "    precision, recall, f1_score = compute_ner_metric(list_gt, ents_ne)\n",
    "\n",
    "    print()\n",
    "\n",
    "    precission_all.append(precision)\n",
    "    recall_all.append(recall)\n",
    "    f1_score_all.append(f1_score)\n",
    "\n",
    "print(\"Precision:\", np.round(np.mean(precission_all),2), '(+/-)', np.round(np.std(precission_all),2))\n",
    "print(\"Recall:\", np.round(np.mean(recall_all),2), '(+/-)', np.round(np.std(recall_all),2))\n",
    "print(\"F1-Score:\", np.round(np.mean(f1_score_all),2), '(+/-)', np.round(np.std(f1_score_all),2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For evaluation we focus on two approaches:\n",
    "- Manual evaluation: comparation of NER entities list and GT entities list.\n",
    "- Automatic evaluation: use Precission, recall and F1-Score."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For what can see here the automatic metrics give us very low performance. And double checking with manual evaluation we can see that automatic NER classification would need more fine-tunning of the model by retraining it with large annotated corpus. Althouh the NER types we obtain for the first text is quite similar to the one we thought about. There are some entities that our model finds as ('mm', 'PERSON') that is clearly not correct. Or ('single', 'DOSAGE') in the last text where looking to the context we can say it is more a frequency as it is not given a medicament. We can also state that it generally works correclty for drugs recognition. And it would be interesting to have a model that can detect body parts or illnesses. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another limitations that we should have into account to understand the results is that the GT data could be not as accurate as it should. We faced some complications with what exaclty entities refer to and we found some entities are worth to annotate that the model fails to find."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABJAAAAJ+CAYAAADsTvrhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABQAElEQVR4nO3de7htZVk3/u8t5CFPgKIVYGBipSYeSC07WL4phoWd1Uoqkw5ampWi9WqvpVFapqaWJqnva5qZqaWmZppZnjCPeEgSVPhpoGhamobevz/GWDD32muNDfuwxpibz+e61sUcz5hr73tP1pqH73ie567uDgAAAABs5ypzFwAAAADAsgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmHTo3AXsretf//p97LHHzl0GAAAAwEHjrW9968e7+8jN42sbIB177LE566yz5i4DAAAA4KBRVR/aatwSNgAAAAAm7TFAqqozq+rCqnr3pvFfqKr3VdXZVfW7K+MPq6pzqur9VXXXlfGTxrFzqur0lfHjqupN4/ifV9VV99c/DgAAAIB9d3lmID0zyUmrA1X1HUlOSXJCd988yePG8ZsluWeSm4/f85SqOqSqDkny5CR3S3KzJPca75skv5Pk8d19kySfTHLfff1HAQAAALD/7DFA6u7XJbl40/DPJTmjuz8/3ufCcfyUJM/r7s9397lJzklyu/HrnO7+YHd/IcnzkpxSVZXkO5O8YPz+ZyW5x779kwAAAADYn/Z2D6SbJvnWcenZP1TVN47jRyX5yMr9zh/Hthu/XpJPdfclm8YBAAAAWIi97cJ2aJIjktwhyTcmeX5V3Xi/VbWNqjotyWlJcqMb3ehA/3UAAAAAZO9nIJ2f5IU9eHOSLyW5fpILkhyzcr+jx7Htxj+R5LCqOnTT+Ja6+2ndfWJ3n3jkkUfuZekAAAAAXBF7GyC9KMl3JElV3TTJVZN8PMlLktyzqq5WVcclOT7Jm5O8JcnxY8e1q2bYaPsl3d1JXpPkB8c/99QkL97LmgAAAAA4APa4hK2qnpvkTkmuX1XnJ3lkkjOTnFlV707yhSSnjmHQ2VX1/CTvSXJJkvt39xfHP+cBSV6R5JAkZ3b32eNf8dAkz6uq30rytiTP2I//PgAAAAD2UQ25z/o58cQT+6yzzpq7DAAAAICDRlW9tbtP3Dy+t0vYAAAAALiSECABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMOnQuQu4sjv29JfOXcKlzjvj5LlLAAAAABbIDCQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASXsMkKrqzKq6sKrevcW5X66qrqrrj8dVVU+sqnOq6p1VdZuV+55aVR8Yv05dGb9tVb1r/J4nVlXtr38cAAAAAPvu8sxAemaSkzYPVtUxSe6S5MMrw3dLcvz4dVqSp473PSLJI5PcPsntkjyyqg4fv+epSe638n27/V0AAAAAzGePAVJ3vy7JxVucenyShyTplbFTkjy7B29MclhVfWWSuyZ5VXdf3N2fTPKqJCeN567T3W/s7k7y7CT32Kd/EQAAAAD71V7tgVRVpyS5oLvfsenUUUk+snJ8/jg2NX7+FuPb/b2nVdVZVXXWRRddtDelAwAAAHAFXeEAqaq+PMnDkzxi/5czrbuf1t0ndveJRx555E7/9QAAAABXSnszA+lrkhyX5B1VdV6So5P8S1V9RZILkhyzct+jx7Gp8aO3GAcAAABgIa5wgNTd7+ruG3T3sd19bIZlZ7fp7o8leUmS+4zd2O6Q5D+6+6NJXpHkLlV1+Lh59l2SvGI89+mqusPYfe0+SV68n/5tAAAAAOwHewyQquq5Sd6Q5Gur6vyquu/E3V+W5INJzkny9CQ/nyTdfXGS30zylvHrUeNYxvv8yfg9/5bk5Xv3TwEAAADgQDh0T3fo7nvt4fyxK7c7yf23ud+ZSc7cYvysJLfYUx0AAAAAzGOvurABAAAAcOUhQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYNIeA6SqOrOqLqyqd6+MPbaq3ldV76yqv6qqw1bOPayqzqmq91fVXVfGTxrHzqmq01fGj6uqN43jf15VV92P/z4AAAAA9tHlmYH0zCQnbRp7VZJbdPctk/xrkoclSVXdLMk9k9x8/J6nVNUhVXVIkicnuVuSmyW513jfJPmdJI/v7psk+WSS++7TvwgAAACA/WqPAVJ3vy7JxZvGXtndl4yHb0xy9Hj7lCTP6+7Pd/e5Sc5Jcrvx65zu/mB3fyHJ85KcUlWV5DuTvGD8/mcluce+/ZMAAAAA2J/2xx5IP5Xk5ePto5J8ZOXc+ePYduPXS/KplTBqY3xLVXVaVZ1VVWdddNFF+6F0AAAAAPZknwKkqvq1JJckec7+KWdadz+tu0/s7hOPPPLInfgrAQAAAK70Dt3bb6yqn0hy9yR37u4ehy9IcszK3Y4ex7LN+CeSHFZVh46zkFbvDwAAAMAC7NUMpKo6KclDknxvd3925dRLktyzqq5WVcclOT7Jm5O8JcnxY8e1q2bYaPslY/D0miQ/OH7/qUlevHf/FAAAAAAOhD0GSFX13CRvSPK1VXV+Vd03yR8muXaSV1XV26vqj5Kku89O8vwk70nyt0nu391fHGcXPSDJK5K8N8nzx/smyUOTPLiqzsmwJ9Iz9uu/EAAAAIB9ssclbN19ry2Gtw15uvvRSR69xfjLkrxsi/EPZujSBgAAAMAC7Y8ubAAAAAAcxARIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABM2mOAVFVnVtWFVfXulbEjqupVVfWB8b+Hj+NVVU+sqnOq6p1VdZuV7zl1vP8HqurUlfHbVtW7xu95YlXV/v5HAgAAALD3Ls8MpGcmOWnT2OlJXt3dxyd59XicJHdLcvz4dVqSpyZD4JTkkUlun+R2SR65ETqN97nfyvdt/rsAAAAAmNEeA6Tufl2SizcNn5LkWePtZyW5x8r4s3vwxiSHVdVXJrlrkld198Xd/ckkr0py0njuOt39xu7uJM9e+bMAAAAAWIC93QPpht390fH2x5LccLx9VJKPrNzv/HFsavz8Lca3VFWnVdVZVXXWRRddtJelAwAAAHBF7PMm2uPMod4PtVyev+tp3X1id5945JFH7sRfCQAAAHClt7cB0r+Py88y/vfCcfyCJMes3O/ocWxq/OgtxgEAAABYiL0NkF6SZKOT2qlJXrwyfp+xG9sdkvzHuNTtFUnuUlWHj5tn3yXJK8Zzn66qO4zd1+6z8mcBAAAAsACH7ukOVfXcJHdKcv2qOj9DN7Uzkjy/qu6b5ENJfni8+8uSfHeSc5J8NslPJkl3X1xVv5nkLeP9HtXdGxtz/3yGTm/XSPLy8QsAAACAhdhjgNTd99rm1J23uG8nuf82f86ZSc7cYvysJLfYUx0AAAAAzGOfN9EGAAAA4OAmQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGDSoXMXwPo49vSXzl1CkuS8M06euwQAAAC4UjEDCQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJi0TwFSVf1SVZ1dVe+uqudW1dWr6riqelNVnVNVf15VVx3ve7Xx+Jzx/LErf87DxvH3V9Vd9/HfBAAAAMB+tNcBUlUdleQXk5zY3bdIckiSeyb5nSSP7+6bJPlkkvuO33LfJJ8cxx8/3i9VdbPx+26e5KQkT6mqQ/a2LgAAAAD2r31dwnZokmtU1aFJvjzJR5N8Z5IXjOefleQe4+1TxuOM5+9cVTWOP6+7P9/d5yY5J8nt9rEuAAAAAPaTvQ6QuvuCJI9L8uEMwdF/JHlrkk919yXj3c5PctR4+6gkHxm/95Lx/tdbHd/ie3ZRVadV1VlVddZFF120t6UDAAAAcAXsyxK2wzPMHjouyVcluWaGJWgHTHc/rbtP7O4TjzzyyAP5VwEAAAAw2pclbP8rybndfVF3/0+SFya5Y5LDxiVtSXJ0kgvG2xckOSZJxvPXTfKJ1fEtvgcAAACAme1LgPThJHeoqi8f9zK6c5L3JHlNkh8c73NqkhePt18yHmc8//fd3eP4PccubcclOT7Jm/ehLgAAAAD2o0P3fJetdfebquoFSf4lySVJ3pbkaUlemuR5VfVb49gzxm95RpL/W1XnJLk4Q+e1dPfZVfX8DOHTJUnu391f3Nu6AAAAANi/9jpASpLufmSSR24a/mC26KLW3f+d5Ie2+XMeneTR+1ILAAAAAAfGvixhAwAAAOBKQIAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMCkfQqQquqwqnpBVb2vqt5bVd9UVUdU1auq6gPjfw8f71tV9cSqOqeq3llVt1n5c04d7/+Bqjp1X/9RAAAAAOw/+zoD6QlJ/ra7vy7JCUnem+T0JK/u7uOTvHo8TpK7JTl+/DotyVOTpKqOSPLIJLdPcrskj9wInQAAAACY314HSFV13STfluQZSdLdX+juTyU5Jcmzxrs9K8k9xtunJHl2D96Y5LCq+sokd03yqu6+uLs/meRVSU7a27oAAAAA2L/2ZQbScUkuSvKnVfW2qvqTqrpmkht290fH+3wsyQ3H20cl+cjK958/jm03vpuqOq2qzqqqsy666KJ9KB0AAACAy2tfAqRDk9wmyVO7+9ZJ/iuXLVdLknR3J+l9+Dt20d1P6+4Tu/vEI488cn/9sQAAAABM2JcA6fwk53f3m8bjF2QIlP59XJqW8b8XjucvSHLMyvcfPY5tNw4AAADAAux1gNTdH0vykar62nHozknek+QlSTY6qZ2a5MXj7Zckuc/Yje0OSf5jXOr2iiR3qarDx82z7zKOAQAAALAAh+7j9/9CkudU1VWTfDDJT2YIpZ5fVfdN8qEkPzze92VJvjvJOUk+O9433X1xVf1mkreM93tUd1+8j3UBAAAAsJ/sU4DU3W9PcuIWp+68xX07yf23+XPOTHLmvtQCAAAAwIGxL3sgAQAAAHAlIEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmHTo3AXAgXDs6S+du4QkyXlnnDx3CQAAALDPzEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJhkE22Ykc2+AQAAWAdmIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADDp0LkLANbDsae/dO4SkiTnnXHy3CUAAABc6ZiBBAAAAMCkfQ6QquqQqnpbVf3NeHxcVb2pqs6pqj+vqquO41cbj88Zzx+78mc8bBx/f1XddV9rAgAAAGD/2R8zkB6Y5L0rx7+T5PHdfZMkn0xy33H8vkk+OY4/frxfqupmSe6Z5OZJTkrylKo6ZD/UBQAAAMB+sE8BUlUdneTkJH8yHleS70zygvEuz0pyj/H2KeNxxvN3Hu9/SpLndffnu/vcJOckud2+1AUAAADA/rOvM5D+IMlDknxpPL5ekk919yXj8flJjhpvH5XkI0kynv+P8f6Xjm/xPbuoqtOq6qyqOuuiiy7ax9IBAAAAuDz2OkCqqrsnubC737of65nU3U/r7hO7+8Qjjzxyp/5aAAAAgCu1Q/fhe++Y5Hur6ruTXD3JdZI8IclhVXXoOMvo6CQXjPe/IMkxSc6vqkOTXDfJJ1bGN6x+DwAAAAAz2+sZSN39sO4+uruPzbAJ9t93948meU2SHxzvdmqSF4+3XzIeZzz/993d4/g9xy5txyU5Psmb97YuAAAAAPavfZmBtJ2HJnleVf1WkrclecY4/owk/7eqzklycYbQKd19dlU9P8l7klyS5P7d/cUDUBcAAAAAe2G/BEjd/dokrx1vfzBbdFHr7v9O8kPbfP+jkzx6f9QCAAAAwP61r13YAAAAADjICZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEmHzl0AwP527OkvnbuEJMl5Z5w8dwkAAAD7hRlIAAAAAEwSIAEAAAAwSYAEAAAAwCR7IAHMxF5NAADAujADCQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEm6sAGwRzrGAQDAlZsZSAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADDp0LkLAID95djTXzp3CZc674yT5y4BAAD2GzOQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYJkAAAAACYJEACAAAAYJIACQAAAIBJAiQAAAAAJu11gFRVx1TVa6rqPVV1dlU9cBw/oqpeVVUfGP97+DheVfXEqjqnqt5ZVbdZ+bNOHe//gao6dd//WQAAAADsL/syA+mSJL/c3TdLcock96+qmyU5Pcmru/v4JK8ej5PkbkmOH79OS/LUZAickjwyye2T3C7JIzdCJwAAAADmt9cBUnd/tLv/Zbz9mSTvTXJUklOSPGu827OS3GO8fUqSZ/fgjUkOq6qvTHLXJK/q7ou7+5NJXpXkpL2tCwAAAID9a7/sgVRVxya5dZI3Jblhd390PPWxJDccbx+V5CMr33b+OLbd+FZ/z2lVdVZVnXXRRRftj9IBAAAA2IN9DpCq6lpJ/jLJg7r706vnuruT9L7+HSt/3tO6+8TuPvHII4/cX38sAAAAABP2KUCqqi/LEB49p7tfOA7/+7g0LeN/LxzHL0hyzMq3Hz2ObTcOAAAAwAIcurffWFWV5BlJ3tvdv79y6iVJTk1yxvjfF6+MP6Cqnpdhw+z/6O6PVtUrkjxmZePsuyR52N7WBQDr4NjTXzp3CUmS8844ee4SAABYA3sdICW5Y5IfT/Kuqnr7OPbwDMHR86vqvkk+lOSHx3MvS/LdSc5J8tkkP5kk3X1xVf1mkreM93tUd1+8D3UBAAAAsB/tdYDU3a9PUtucvvMW9+8k99/mzzozyZl7WwsAAAAAB85+6cIGAAAAwMFLgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAw6dC5CwAAlu3Y0186dwlJkvPOOHnuEgAArrTMQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmCRAAgAAAGCSAAkAAACASQIkAAAAACYdOncBAAD7w7Gnv3TuEpIk551x8twlAADsd2YgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAw6dC5CwAAuLI59vSXzl1CkuS8M06euwQAYE2YgQQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMEmABAAAAMAkARIAAAAAkwRIAAAAAEwSIAEAAAAwSYAEAAAAwCQBEgAAAACTBEgAAAAATBIgAQAAADBJgAQAAADAJAESAAAAAJMESAAAAABMEiABAAAAMOnQuQsAAGC5jj39pXOXkCQ574yT5y4BAK7UzEACAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABgkgAJAAAAgEkCJAAAAAAmCZAAAAAAmHTo3AUAAMC+Ovb0l85dQpLkvDNOnrsEADggzEACAAAAYJIACQAAAIBJlrABAMAOstwOgHUkQAIAAHazlKArEXYBLMFilrBV1UlV9f6qOqeqTp+7HgAAAAAGi5iBVFWHJHlyku9Kcn6St1TVS7r7PfNWBgAALN1SZktdnplS61QrwKqlzEC6XZJzuvuD3f2FJM9LcsrMNQEAAACQpLp77hpSVT+Y5KTu/unx+MeT3L67H7DpfqclOW08/Nok79/RQpfr+kk+PncRl5Na9791qTNR64GwLnUmaj0Q1qXOZH1qXZc6E7UeCOtSZ6LWA2Fd6kzWp9Z1qTNR64GwLnUm61XrTvjq7j5y8+AilrBdXt39tCRPm7uOpamqs7r7xLnruDzUuv+tS52JWg+EdakzUeuBsC51JutT67rUmaj1QFiXOhO1HgjrUmeyPrWuS52JWg+EdakzWa9a57SUJWwXJDlm5fjocQwAAACAmS0lQHpLkuOr6riqumqSeyZ5ycw1AQAAAJCFLGHr7kuq6gFJXpHkkCRndvfZM5e1TtZpWZ9a9791qTNR64GwLnUmaj0Q1qXOZH1qXZc6E7UeCOtSZ6LWA2Fd6kzWp9Z1qTNR64GwLnUm61XrbBaxiTYAAAAAy7WUJWwAAAAALJQACQAAAIBJAiQAAAAAJgmQ4CBRVV81dw0HG4/p3quqO8xdw8Gkqq4zce5GO1nLlUVVLaLRCAdOVR07dw2XV1U9Zu4aYE/W6bV/bODEflRVh8xdAweeAIkDpqq+c+X2cZvOff/OV7S9qvrTqjpzm69nzF3f5fTGuQvYUFVHV9W3rBw/uKoeMX7dZM7arqDFPKbJev1OJXlKVf1xVR02dyF7UlUPWbn9Q5vOLeVD22s3blTVqzede9GOVnI5VNURU19z17ehql6/cvv/bjr95h0uZ1JV3aCq/qCq/qaqfnsqVORy+7uqOn1NwsKT5i7gYFNVj62qn9li/Geq6ow5atpOVR1ZVTfbYvxmVXXkHDVt4ylzF3AF/NTcBVxeVfXKuWu4nN5aVd80dxGXx5q9p14UAdKaqapTqur+K8dvqqoPjl8/OGdtW3jcyu2/3HTu13eykMvhb5K8dNPXO5PcOcldZ6zriqi5C1jx2CSHrRz/TJL/StJJ/s8cBe2lJT2myXr9Tp2Y5L1J3lxVPz53MXtwz5XbD9t0bikf2lZ/FjcHMEv7OU2Sjyd5e5Kzxq+3rnydNV9Zu7nmyu2bbzq3tMf12RmeR5+U5FpJnjhvOVurqndV1Tu3+HpXVb1z7vo2uXWSG2b40POtcxezB4dU1eFLD2WTpKrOXXlv+sFNx/82d30rvjNbt+1+epK773Ate/KkJNffYvx6SZ6ww7Ww85YUEk75mSRPqKqnV9XhcxezB+v0nnpR1uGKC7t6SHb9sHO1JN+Y4U3wnyZ5wRxFbaO2ub3V8ay6+9Injqq6cZKHJ/m2JGckWZcZSD13ASu+trv/ZuX4s939e0lSVf84U017Y0mPabJev1NfSvIH41WzN1TVUzI8njWc7iXNnliHx7W3ub3V8RI8Mcl3JPmnJM9N8vruXmKdUzUtrd6v7O5fG2+/oqr+ZdZqtrfxwbsyXIz57hlrmdTdn0nyS1V12ySvrqrzk3wplz1P3XLWAnf1dRkC2K2ekzrJjXe2nEknbjq+SpIfTvIrSd628+Vs62pbPS9195eqainP/Rtu0t2v2zzY3f9YVU+do6Bt3LiqXrLdye7+3p0sZg9uWVWf3mJ8ie9Trjs1K6a7X7iTxWynu99UVbdP8rNJzqqql2d4Tt04/4uzFbe7dXjvt0gCpPVz1e7+yMrx67v7E0k+UVXX3O6bZrJWH3iq6usyJM63zjCD5me7+5J5q9pVVT0pWz92lV1n/Mzt6puO77xye6sraLNZo8c0Wb/fqfsmOT3JryV58kIDhGQ9HtcbVNWDM/xcbtzOeLy4K5Pd/aDxA9idkvx4kieNYeJTu/vcWYvb1WFV9X0ZPuAetvIGvZJcd76ytjZe0d14Y3vI6nF3XzxbYSu6+0Mbt6vq86vHSzQuY3hCkj9J8uSsfNhZmPd0963nLuLyGN+XpqqukuH3/1czzEg8ubvfM2Npm32uqo7v7g+sDlbV8Uk+N1NN27n2xLkv27Eq9uyiJL83dxGX07vW5Xcqw+vR3bN9gLyIAGl0RIbJDRdlCL2X+py6Du/9FkmAtH52mQ7Y3asbwC3tQ8TGVYjKrlckKslx23/bzquqv0hy2wwver+U5ItJrrNxAWopb8wzvfRjSctCPlNVN+3uf00ue/zGkO4zs1a2u3V5TJP1+p365yTnJfnW7v7YzOXsyQnjVchKco2VK5KV3cPQuTw9l32AWL2dDB98F2cMDF9TVW/LMHP2N5N8IEP9S/EPSb535fb3rJzb7Wr/zK6b3WegbMxCWtoMlLVQVc9LcnSSe3f3u+au52BRVV+WYX+ZX0ry+iT36O5z5q1qS49I8vKq+q0Mv1vJMHvqYUkeNFdR2zinqr67u1+2OlhVd0vywZlq2sp/dvc/zF3EQehD3b34PZuq6mczBMaPTXLfBV84TNboPfXS1LL/v7JZVT0nyWu7++mbxn8myZ26+17zVLa7qvr2qfNLeoGpqvNyWdq8scxmQ3f3ot+YV9XVk3xPd//F3LUkSVWdlGEJy6Nz2Qec22ZYGvjA7n75XLVdXkt7TJO1+5363iRv6O6LNo0fmeQz3f3f81S2u6r6su7+n7nrOJiMM2JPSfIjGS5uvDDJ87v7w7MWxgFXVbdZOXxOkntn5TW1uxez9K6qfqW7H7fne86vqn4+yV+syXPq+UkuSfIHSXb7nV/KcpskqapbZPjAe4tx6Owkj11aoDjOinppkn/OrmHXNyW5+8YFu7lV1Qu7ey02IK6qh3f3bo0yquouSX61u79rhrK2VFVvW4fZUlX1/5I8uLsvnLuWPVmn99RLI0BaM1V1gwwddz6fXT+YXy3DFZ5/n6m03VTVM7v7J+au42BVQ6vMuya5V5K7JPnH7l7MRurjm7KH5LKNad+d4U3Zu+eratrSH9MkqapbJblJkrO7+70zl7Otqnpakr/d/EFhXC50l+7+uXkq211V/Ut332bP95xPVU1umLywfQVSVf+VYbbR88b/7vJmYykfIKvq6CTHdvfrx+MHZ9igOkn+bKGzJnZRVTfN8GHnfnPXkiRV9ZqJ093d3zlxfketw+/+hjV7Tn1mtl8C0kufSVFVxyS5Z3c/du5aVlXV1TIEsqth158tLDz89kws/9lqH6e5jMtX/yjJV2X4bPU7GfaTrSSPXsrrVJJU1c27++y569iTqvqx7v5/4+07dvc/rZx7QHf/4XzVsb8IkNbU+KS38cH87O7++znr2co6vTFLkqq6apIfzcrjmuGF+fPzVbW78cX53hk2Jn1zkjsmuXF3f3bWwi6nqrrR0mYhrMtjWlWPSPJjGa4+3j7Jb2+ejbgUVfXW7r7tNufO7u7NHa9msw5X9qrqCxlC2Ocn+f+yaR+E7n7WHHVtZ10+QFbVc5M8Z2PT/6p6f4auTF+e5Ou6+0fnrG9VVd0yQ9eYjQ87T07yhxmeC36vux8/X3WXqapv6u43zF3H5bFO71PW6Tl1HY0zuX4owwWkr0ryV939K/NWtbsa2o1v/L9+T3cvaflaquqvtxjuJLdMckx3H7LDJW1rXF79S0nekORuSf5fktOXGHJU1bnZ9TW1Vo67u79m56va3epz6ubn16U931bVKUmO7u4nj8dvymXbwTyku5fUmGpR7IG0ZuqyVq1vH786yadmKmdPvryqbp1tdrJf2DT2myV5SYaOQRtTg++U5Neq6pSlpP7jtPAPJ3lqkl/p7s9U1blLCzqS4UNEkqOSvK67Lxw//Jye5FuTHDNrcSvW6THNsBzoVt392aq6XpK/zbL2k1n15RPnrrJjVVw+R65sSr2b7v79nSxmG1+Z4cPNj2RYGvLnSV7Q3Z+as6jtrNHs03XqGPn0DM9Tb0hyUob3AM9K8qNLmoGQIdhazIeEPfi6qnrnFuNL7MK2Ts+pq0vDVi/KPW5JS8Oq6tpJvj/DBaSbZlhqe1x3Hz1rYVuoqutk2O/uthl+9yvJrarqrRn2mtmqm9iO6+7VfeRSVXfM0KDmY0l+YZaiJnT3a8ebL6qqC5YYHo3WpbPhOnU2W6fO5osiQFo/b81KK+zxv9eqqnck+enuPm/G2jY7KsOm1Nt1DFjMNPYkT0ryc939qtXBqvpfGa7wfscsVe3uBUnukeFD5Ber6sVZYKeAqnpshm4Rb0/y0Kp6RZKfTvLbGTbWXJK1eExHn98Itrr7EzV0uFmqC6vqdt395tXBqtrozLEkh2RYtrS0NzeX6qGr0R8l+aNx2dU9k7ynqh7a3f933uq2Ni4JPby7Pz4eXzXJTyT5pe7++jlrW7E2HSMztBx/5nj7/VX1wO5+yJwFbWOxv0dbODe7bpy+ZGvznDpe2X9chtf8jY5cJyZ54bjv1ItnK25XF2aYdfzrGboa97gkcImemOQ9GZbWfSlJqqqS/O8M71PvM2Ntu6mqO2eorZM8ZvP764W4bl3WeTNJDl09XtIStl6fzobr1NlsnTqbL4olbAeJ8QnvtO4+ae5aNqzDspANVfW+7v66bc69d0EfdjbeMNwpwzTr787Qmee+SV7W3f85Y2mXqqr3JLlNd/93DW2mP5LkFgsLOC+1Do9pklTVp3JZZ6jKMJvrdbnsavn3bvOtO66qbpdhudUzs+uGn/fJ8Ab4TTOVtpulTaueMm5QfK8k35Xhcf29hb15TJJU1T2T/HGSjb2QHp3kzCRvSfKbS5mBOk5Z//HetAFtDR0jn93dt5unst1V1fsy/L/fCGh22aB6QY/ppzLRwW5hz1Pr9D5lnZ5T35HklM2v+VV1bJIXd/cJc9S1WVU9KEMYf80kz80ws/NVvcDGKVX1ge4+/oqe22lVdXKSX0vyHxn2EXr9zCVtq6r+dOL0YpZaJ0nt3tnwjF7gHn1V9dkk52R4Xfqa8XbG4xt392KCmao6p7tvss25f1vKssAlEiAdRJb2IWjqjVlVfWN3v2Wna9pOVf1rkm/YvN9RDZ243rWUF+bNxheUkzK8Abprdy/iivkW657X6U36Ih/T5NK9mjbbeBKvXljHiKq6YZKfz64bfv5hL6w7xzr8fFbVo5KcnOS9GTam/tvuvmTeqrZXVe/O2Lp7DL3ekOQHu3ur/TFmU2vUMbLWZHPqqvpAhhmnW1rS81RV/WF3P2DuOi6vNXpO3XZPpqp6T3ffbKdrmlJVN87wmn+vJMcneWSGPZAW0dks2WOAtO0H4Z1WVV9Kcn6Sd2SLGSdLCpCnVNUNe1mNidais2FVffXU+e7+0E7Vsie1Rp3Nl0aAdJCoqmtlmHp3q7lr2VBVd+nuV64c3yzDi/O9knyquzev551NVf16kjskuf/Gk9t4peyJSc7q7kfNWN7lUlXX6O7PzV1HsuUV6G/LQmfKTFnSY5psueHfmzNs+NdJHtrdfzFnfeuqhs1Tv7gy1BmeoxbzAjm+KT83ycbeXJcGh1neXi1bhcjv7u5bTH3PXGr3jpFnJ/ndXnDHyCVbh0B2Q1X9cqY7Ri1h/7O1M85A+p7e1DBj/HD510t5vqotmnqMzwf3SvIjSwllkqSqnpXk3zLM4OyV8f+d5Kbd/eOzFbdimwtdl1pSgLxZVR2W5AcyzOz8+u7+qnkrukytSWOKdVJr1Nl8aQRIa2abjV4PT/K9Ga5CLWpD3TGE2QiN/ifJVyc5cYlLmarqARk+RHx5hg9l/5lhw8cnzVrYivEK9NQLyJ23ObejVt5AXCPD1bzOMI31c8my3kCsy2OaJFX1TxmWKnxkPH57hj1brpnkTxdW6zo9rhvdTVb3brlWhiuoi9hbbp2u6iWXXi1d/fD94NXjpX8wr+W28b5Bkvtn17DryUuagVJVL+zu79/zPedXVY+cOt/d/2enatmTqnpXdt9f5ONJXpPhvcpiNlKvqnsk+d0kj8muy+1Oz3Cx40XzVLar2rVj1F929w/MXdN2athE+xkZNqh/+zh8qwwbKN+3u/9jnsp2VVXX6W029N4qsJtbVV0jySkZQqNbJ7l2hn0xX9fjXlNcfrUm3eJW1Rp0Nl8aAdKa2eLNTif5RIYnusV0tkiSqnpDkutkWG7xvO7+QA3drY6bubRJNXTlSHd/Zu5aNquqrVr43iFD8HVhd3/jDpe0pXEZ2KMzrNfeeLNwTIa9Gx7e3f8zU2m7WZfHNEmq6i2r9awuv6iqN3b3Hearblfr9LhuZ4l7y62LdfpgvqEW3sa7hm5Gf5Zd98C5bZJTM3Ri+6eZSttFVX1rhk5Wzx6PX5Bko4Psby3pzXlVPaCX23VpF9uEyEdk+P9/ze6+3w6XNKmqTkjyy1lpOZ8h6HrHfFXtanW23LrMnKuqr0mysQTwPd39b1X1oO7+gxnLutSmUO7VqxeLFrjVxp9l2EvylRk+q/x9knOW+Dmlqv6gux803n5gdz9h5dwzeyGdT2voELxqtVvcvyw5pE0u/f26d4YLSFsuw0WAdFBZWrJfVS/KcKXkJUn+rLv/uao+2MvcnPB7krxzZfnaIzJMY/1Qhn0wzp2zvq2Ms3z+d4YuQo/uZe3V8fgMMzgevBHEjVfPHpehTfaDZixvW0t+TJPpfQ6WvOHf0h/XKUt5w1tV/5Vdl9ldeirDVb3r7HBJB4Xauo33j/Qy23i/MUO30LdtGr9Vkj/u7tvPUtgmVfXqJL/Q4+bu48yZn8gwU/LhSwpkl/L7va/WJfxYmk1hx9r+LFTVh7v7RnPXkUyHckv7OR1ncV8lybMzXOg+f8GfU7b9WV3iz27t3i3uMb3Ahh9JUlVflaET872TfEOG7pEvXNrEjCU5dO4CuOKq6puSHJVh1tGFVXXLDNOCvzXDLI9F6O57VNV1M7w5/42qOj7JYbVFG9oFeHSG2RGpqrsn+bEMV6FvnaF19l3nK21XVXXXDC1nP5/hw/jUxqpzuXuGNfmXJtTd/emq+rkk70vyoLkK28qaPKZJ8qaqut/mpao1bPi3tN+pdXpct1TD3nJXmbuO0b8u6Y335VFVd0vysFx2tfzsJL/T3S+br6rdrFMb7+tsDo+SpLvfvjFzdiGus+mDwge6+61JUlW/PVNNB7ulPE8lSarqrzO9t9RS9kE8oao+nSGIv8Z4O1m/YL72fJcdszZt3Lv7VjV03LxXkr+rqo8nuXYtbAPtUW1ze1Fq925x9+gFdotLkqo6LcP/+6MydLi8b4YukYubIb00AqQ1U1WPzfDh/O1JHlpVr8jQ7eS3M/zCLsq4JvtPk/zpuHfDjyR5/DhbajFhV4Y3Chub035/kmeMb3jfWlU/P2Ndu6iqt2TYNPmxGboapYYOR0mW08Y5w+O5VfeNL1bVot5ArNFjmgwvyC+qqntniw3/5ipqK+v0uNYe9pbb4XK2s6jfmz2pqvsl+ZkMSxbPGodPTHJGVR3d3U+brbhdPSxD96WnJHluVf35zPVMqao6vLs/uWnwiCwrQDhs9aB33Q/phjtbyh7dciU0WLW4AGH1+XPF4RkueL1ui3NzetzcBVwe3X3I3DXsJ0t6fbjB+JpaK7czHh85X1lb6+73Zei698hx6f29krylqs7v7m+et7pdXKWqDs/wXL9xeyNIWtLP8bnZtVvcLceJDkmW0y1u9IcZ3p/eu7vPSpKlfUZZKkvY1kxVvSfJbbr7v8cnj48kuUUvYJPXKePeEunui8bjr+4FbfpaVe9M8s0ZOhydm+QHVp5MFtNytqpem+mNiZfSxvlFGaZ/PnvT+I8l+eEFXX1cm8d0Va3Bhn/r9LhusV/P4vaWq903pd5FL2xT6vG16lu6++JN49fLMNPn6+epbGu1Hm28T0tyv4x7SYzDt03yO0nO7O4/nqu2VePskz/q7pduGr97hiV4J89T2e6WtqRmSg2NCVZtPE+9NsnTell7Cy5mT5aDRVV9Jlu/plaSa3T3IiYFbPF6uot1mN1RVZXkW7v7dePxw7p71tmTVXVeki9l69lHvZRld7VG3eLG9yMb+x5+RYZZSD+xsAkOiyRAWjNbrHtd7Juf8Qn4kUkekMvS8UuSPKm7HzVbYVuoqp9K8vAkn86wwe9J4/itM2z6uJiOUeugqo7KsJfI57JrB5ZrJPm+7r5grtpgyrhsLd39n3PXsqqqPprkqdlm6vrS3pRX1Xu3C4mmzi1BLbSNd3JpCPOQ7NqF7bHd/dfzVbWrqrpJkpcm+efsGnR9c5K7LyyUW+x7qHW2xD1ZYG/5eT74VdXRGVbJ3CvDfn1/1d0Pn7eq5RIgrZmq+lQum6pcGfY9unTq8sJmdjw4yd0ydDE6dxy7cYYPQX/b3Y+fs77NxtDjBkne0WPrzqr6yiSH9tg2fW41dIXa1sKmhm6eKfOe7n71nPVsZd0e03VRVQ/p7t8db/9Qd//FyrnHLO2Fedyf62EZ3jgkyX9m2K/nKfNVdZl1ewNbVW/K8Nz/jk3jJyR5enffbp7Kpo37N9wiyQXdfeHc9ayrqrpakh/NrkHXn/WCWs0nSVU9vLsfM3cdl1dVfW2S05J83Tj03gyzjxYTyiVJVb0vwwex7QLvxSxhZv8at9o4Z/OMyHGvxuO6+/R5Ktt7Swiax1m9z0ny3O7+4Jy17Ml4EeZXs+vz/+OWMqN7T6rqphm6sC1qssOSCJDWTA3djLbV3f+wU7XsSVW9Lcl3dffHN40fmeSVcz8ZXx7jk8iv9kLa41bVn06cXtTU0HXhMT0wao06hlTVr2eYHfGAjTdmY9j9hCRv6u7fmrO+sZ7Z38BeEVX1LRne7P5pdp2FeGqSH+vu189V26qq+qMMs2LPrqHpwxsydLs7IsmvdPdzZy1wRQ3dQbfT3f2bO1bMQWJcbjO13GIxj2kNDVRemORpGWZ2VYZGH/dL8v3d/cYZy9vFuNzqLdl+uc1iljCzf1XVW5Oc2Js+YNbQleud3X2LeSrbe0t4zzJefLlnkh/OsHT1uUn+vLv/vznr2qyqTsmwB9pvZ9f9Dx+W4TX1xXPVttkWF5A7yceTvL3HDtJsTYC0xjbvK7Q0VfXu7V4ops7NYdzg7XFJvirJi5I8OcPmardP8ntLmy0FS1fr1cr3/UlO2Dw7oqqukWFG4k3nqWyXWo7YvJ/Q0lXVDZPcPyuzEJM8ubs/Nl9Vu6qqs7v75uPtByW5Uw8dRL8iycsX9nP6y1sMXzND55jrdfe1drikLVXVuZkOZb5mJ+uZss1j+uUZmpMs5jFNkqp6eYZZka/dNP7tSU7v7rvNUtgWlvYcz87Zw3v/S59v18nSfp6r6g4Zllv9QJJ/yzC78+nT37UzquodSU7pTXvzVtWxGTqcnTBHXVvZ5gLyEUlumeS+vcD9RZdiERuucfmN+wo9IskvZNiJv6pqkfsKJfnCXp6bw9MzLK17Q5KTMnS5e1aSH13glPu1mMK+TjymB8TatPLN8KF2t9/z7v5cVX1pjoI2W8Pw6Ebd/eEMr1dLtvpa9F1J/iJJuvtjw8vtcnT3723crqprJ3lgkp9M8rwkv7fd983gxE3HV8lw1fxXkrxt58vZ3jaP6U9leY9pknzN5vAoGWaeV9VSuhruUVV9Y3e/Ze46OGA+V1XHd/cHVger6vgM+2Kuo7/Y8112zjjb8I1V9eIkj89wwXsRAVKGbT/O2zzY3eeNS8QXo7t/cqvxqvrqDBtq335nK1ofS2r7yuXzS0m+Jck3dvcR3X14hh/wO1bVL81b2m5OqKpPb/H1mSTfMHdxm1ytu5/Z3e/v7ick+a/ufsgCw6NvytBx5T8zTGN/epL/SvLa8YoEV5DH9IA5YeX3/ZYL//2/oKp22yh/3MProzPUczB40caNqvrLGevYk09V1d1raJhwxyR/myRVdWiGTf8XpaqOqKrfSvLODBcBb9PdD13Sfk3d/Ynu/kSSTya5e5LXJPmmJCd39w/MWtwW1uExHU0tqfivHavi8nno6kFV3ayqfrOqzslwsY6D1yOSvLyqfqKqvmH8+skMG+sv6oJCVb1y5fbDtrvfkvZJq6pvrKrfr6oPJfmNJH+cYfXEUlxSVTfaPDiGMpfMUM8V1kOX8EWFXUtjBtL6+fFs2leouz9YQ3v0V2ZIohehuw/Z870W4+rjB4iNS86fXz1e0IaPj0hyr01XIV9UVX+foePdYqawrxGP6QGwZr//v5jkxVX1+uy6X88dk5wyW1XrbXX6ziLaC2/jZ5I8MclXJnnQyvK6O2f4wLMY4+a0358h6P6GXlinwA3jVeafynDB6/VJ7tHd58xb1dbW5TEdHVNVT9xivJIctdPFTOnuV45LVu41fv1Pkq/OsDfOeTOWxgHW3S+vqntk2ET5F8bhs5P8wAI3UT5y5fYPZdi3Z5Gq6jEZlq1dnGGG5B27+/x5q9rSI5P83Vjv6vup07MpWF6qcVXC5+euY8nsgbRm1mlfoXVSVa/N9J4Ni9jwsar+dbv9WKrq/d39tTtd07rzmB4YVXXEpqFO8qnNG2suRVVdPcm9s+t+Pc9Z2izEdVETm6izd8bllJ/PcBV39feoMrxOXWeWwjapqvMz1PgHST68+XwvqLPlujymSVJVp06d7+5n7VQte1JVb0hynQwfdJ/X3R+oqnO7+7iZS2MmtbCmNMl6vU7V0EThuZuXBi7RuOH3L2fX91OP601dWedWVX+d3T/7HZHhgtKPdfcbdr6q9WAG0vpZp32F1kZ332nuGi6ndZrCvi48pgfGWzO8MK/ORLl2Vb09yU8v7Sr0GBSduTpWVVepqh/t7ufMVNY6O6GqPp3h//81xtvJMj+Y3y3D1dHVlsO/090vm6+q3XX3umw78HcZfvdPGL9WdYZOYouwRo/pLgFRVV1rHFvqjKl/zzAr6oYZZnl8IMvb+44DoHZtSvNXSZ6SlaY0M5a2lRtX1UsyvC5t3L5Ud3/vPGVt6b83wqOq+qHuvnRfpqp6THc/fL7SdjUGRfeZu47L4XGbjjtDh7sPdLfP1BPMQFozVfXFbP2htpJcvbut2dwLVfWQ7v7d8fZin5ir6sIMV/R2O5Xkh7v7hjtc0trzmO6sGtqmntbdJ81dy4aquk6GbmFHJXlxhg/A98+w6e87utsytoNUVd0vwzK2h2TXlsNnJPmT7l6bzYnZP6rqmkm+L8PS5pPnrmdVVf1chnbY1xyH/jND2PmU+araWlVdN8PywHslOT7JYUnu2t1vnrMuDqyqelMua0pztww/r89K8oilzeitoYPhtrr7H3aqlj2Zmi21pNlTm0O4zRYWyrGXBEiQtXpiXpsp7OvCY7rzlvQ7lSRjJ5NPZnjDe+ckN8gQID6wu98+Y2kcYFX1niTfsrnLXVVdL8nru/vr56lsfVXVH3T3g8bbDxwbU2yce2Z3/8RctW2nqq6a5OQMy1jvmuQvk7ywu/961sJWVNWvJ/nmJA/o7g+OYzdO8oQkb+ru35qzvilVdYMM+7fcM8mNuvuYmUviAKmqt3f3rVaOP9jdS94Hb2MJ+03Gw3OWFnQlSVW9rbtvvfn2VsdzqqqLknwkyXOTvCm7zkJfWih33yRHdPdjx+PzMyy9rQzLLf9ozvqWzBI2GNQ2t7c6ns1UmLFV1wP2zGO6s8alF0tbNnLj7v6GJKmqP8nQee1GS3wTyX5Xm8OjZOgkVrWYp/51820rt0/NEHBsuOUO1zKpqu6SYYbMXTJ0i3t2hi63W7Z3ntmPJzlh9XlpbKLyw0nekWSxAdLY0e5JVfX/Mszu5OC1Lk1pNrptPibDpv8fylDjMVX1p0l+rbv/Z876Nultbm91PKevSPJdGZ5X752hGcVzu/vsWava2s8mWZ0Nf1F3Hz0Giq9IIkDahgAJBuvyxLzRdv6oJK/r7gvH9eanJ/nWJK7q7QWP6f5XVQ/eYvjwJN+bYT+EJbn0TWJ3f7GqzhceXWl8uqpO2Ly557gJ6NT+aGxv6oLM0vxtkn/MMAvt3CSpqidMf8tseqvnpe7+3LgZ+GJU1TFJ/neGfXBelGE2wqMyhGDPna8ydsDHkvz+NsedZBFNaUaPTXLtJMd192eSS5e0P278euCMtW02ta/g1ecra1fd/cUMz6t/W1VXyxAkvbaq/k93L+29X3X3J1aO/yIZ9sSsqmvMVNNaECDBYC2emMeWw3dP8vYkD62qVyT56QytR39qxtLWlsf0gLn2puPO8EbyxxbYyveETb/z11h5PljUhs/sd7+c5CXjFefVlsOnJvmx2apab1epqsMzzDTcuL0RJB0yX1lbuk2GZVV/V1UfzLAf3tJq3HBBVd25u1+9OlhV35lh1uSSPDvJP2RYCnhShv3F3p7klt39sRnr4gBbo6Y0yfDe76ar3WG7+9PjXmPvy4ICpO5e6vPSbsbg6OQM4dGxSZ6YYUP1pTls9aC7H5MMDVSSXH+OgtaFPZBgjYz7ddxmTMcPz7DO+BZL62i1TjymB954RS/d/ek93Rd2WlV9RZKfz64th5/sg+7eqarzknwp28w+Wmor96r65gwfeH4gw5Kwv1rSJupVdfMMm/y/PruGnXdMcsqSlohU1Tu6+4SV4/MzLAte1Ewp9r91aUqTJFX1r9190yt6ju1V1bOT3CLJy5I8r7vfPXNJ26qqpyS5uLt/fdP4byW5fnf/7DyVLZ8ACXLpBno/m2ETvXcmObO7L5m3qt1tscH3YjbOW1ce0wOnqh6YobvVxiy+T2ToxPK8qjqmuz8yX3Wwq6o6Mkm6+6K5a2E+49Xn/5Xknt29qFmo43uVe2fXsPM5S1tyW1XvSHKnXBYivmb1eKt9xzg4rEtTmiSpqhdl2Cz/2ZvGfyxDF14dw66gcTntRrfw1ZBhcTO6x46bf5LkGzNcNEiSEzLMmPzp7v7PuWpbOgESJKmqP8+wD8o/Zmg7+qHuXszU1Q1V9akkr1sZ+rbVYy92V9zEY7rxYucx3QtV9RtJbpetOwa9Psn9uvsm2/8JcODVsFP2IzNs7LuxROCLSZ7U3Y+arbCDUFXdNENnm/vNXcuGqtr8YbaTfHydwu0x8LpXdz9n7lo27GEWWi+9Kxd7b126hSVJVR2V5IVJPpddZ/VdI8n3dfcFc9XGzhnfm14aynf3v81ZzzoQIEGSqnrXShemQ5O8eUlXSTZU1bdPnV9Se8x1sfKYXiPJ8Rk+QJyT4Q2Fx3QvVdUHknzD5ivj48aEFyW5d3e/ZJbiYDRu9n63JKetbKJ84yRPTfK33f34OetbR2MTgsflsg2Un5xh4/zbJ/m9JT2mVfWaLYaPSHLVDDOQ3rHF+VmMS4Hvn6Hhw4uT/N14/CtJ3tHdp8xYHiRZrxlIG8Z9xFYDhM37jB3e3Z/c+co4kPbUabm7P7xTtawbARJkfV7ktjN2PLlndz927lrWTVV9WZJHZ9gwe+PF4pgkz0zy8IW1cV0bVfW+7v66bc69v7u/dqdrgs2q6m1Jvqu7P75p/Mgkr1zS1fJ1UVVvyhDAvSHDBsoPT/KsDMtXF7XUajtVdWKS3+/ub5u7lg1V9eIkn8zwuN45yQ0yzPB5YHe/fcbSdnMwzOxi71TVFzMsYaoMF+Y+u3EqydW7+8vmqm1vrdtnAi6fqnpXhuem1ZmSneTIJDdYp43Ld5oubDBYuy5M4wecH8qw6edXZZkdDtbB7ya5VrZu4/rYJA+ar7S1NtUxyLRwluLLNodHybAP0hguc8VdrbufOd5+f1U9sLsfMmdBV1R3n1VV15q7jk1uvDJT+k8ydF670UJDud/bYuyIqrpqhuV2b9/hetghB+mH7i0bArDeNp5PN1TVsUkemmEPvMfMUdO6ECBB1ucFr6quneT7M2yiedMMa7eP6+6jZy1sve2pjeuD5ipszf1ikhdX1VYdg+wrxVJ8YS/Psb2rV9Wtc9mHrs+vHnf3v8xW2eVUVTfMrhvALsGls2G7+4tVdf5Cw6N093dsNT7O7Hpihr0GOQitS1OaK2hpzwXsR1V1fJJfy7jMOskvWn0wzRI2WCNV9bkkb07y60le391dVR+0IeXe08b1wFmXjkFcea0st9jtVNZ0ucXcttlXaEN393fuWDF7UFVPyu4fDo9I8s0Zlob99c5XtbVNS4OSy5YHLXam9FYsBzq4rUtTmivCz+zBqapukSE4unmG1QjP7e4vzlvVehAgwRqpqgcluWeSayZ5bpI/T/IqAdLe08b1wKiqr+vu9423r9bdn185d4fufuN81QEHyjr9flfVqePNa2aYlX/dJP+S5C3dfeFshR2kxpldL+vu285dCwfGujSl2ZOqump3f2G8vajucewfYyj/kSQvzdB9dRfd/Ys7XtSaECDBGhq7BN0zw/5Hxyd5RJIXdfe/zlrYGtLG9cBYx04swL5bp9/vcU+e301ynyTnjcM3TPKk7j6jqm61lP161mlp0DrN7GL/WqfX+6p6RHc/aovx6yZ5cXffaTw+orsv3un6OLBWLiBsqbuftVO1rBsBEqyRqrpJkht29z+tjH1Dkick+fZ12ctpifbUxpUrZvWK3eard67mwcFrnX6/q+qJGS4WPHiLJgpfTHJSdx83Y4mXWqelQVt8MOskn4iZXQe9TcuCVzuxLW6pZVW9MsPP5K+tjN0wySsyzEzfLVzi4DeG9d/T3X8xdy1LJUCCNVJVf5PkYd39rk3jt0zymO6++zyVwa7MQIIrp6r6VJLXbXd+ScuCq+qcJMf3pjfDVXVIko8nudtSluOt69KgsWNsuvuiuWuBVWNQ8IIk/9rdDx43U355ksd19x/NWx07aXzOv2uGlR13SfKP3f2D81a1XLqwwXq54ebwKEm6+51V9dVzFATbOHq8ul8rtzMeHzVfWcABdlG2buO+RF/aHB4ll3Y5u2gp4dFotQvbJVXL7ixeVY9M8gtJrjIc1iUZlgaa1cEidPd/V9X3JfnzqnpuhiWWD+ruv5q5NHZIVX17hmYv352hSdEdM3S3/uyshS2cAAnWy2ET566xU0XA5fCrK7fP2nRu8zFw8PjP7v6HuYu4nN5TVffZponCe2eqaTsnVNWnx9uV5Brj8RKXBj04ybck+cbuPnccu3GSp1bVL3X342ctEHLpz2mSvCnJQzIsDz1uY7y7f3+u2jjwqur8JB9O8tQkv9Ldn6mqc4VHeyZAgvVyVlXdr7ufvjpYVT+dyzaAhtlNbT5YVTfayVqAHfXJqvqK7v5YklTVfZL8QJIPJfmNhW1Ge/8kL6yqn8oWTRRmq2oLa7bH4Y8n+a7u/vjGQHd/cAzmXplEgMQSXHvl9hO3GOPg9oIk90jyI0m+WFUvzu6b/7MFeyDBGhk39/urJF/Irm92r5qhY9jH5qoNNquqb8qwXO113X3huFfX6Um+tbuPmbc64ECoqn9J8r+6++Kq+rYkz8uwlOlWSb5+iftKaKKwf1XVu7v7Flf0HCxFVV2zu/9rz/dkndWwFvhOGfY++u4k101y3yQv6+7/nLG0RRMgwRqqqu9IsvEG7Ozu/vs564HNquqxSe6e5O0Z2k6/IslPJ/ntJH/c3f89X3XAgVJVb+/uW423n5zkou7+jc3nOHhNNUrQRIElqaqjknxlknd29xeq6gZJHpTkJ7r7q2Ytjh1VVV+WyzbSvmt3X3/mkhbLEjZYQ939miSvmbsOmHBykluPm1QenuQjSW7R3efNWxZwgB1aVYd29yVJ7pzktNVzM9XEzlrdr2lVJbn6ThcDW6mqByX5tSTnJLlaVT0lye8keXaS285YGjPo7v9J8jdJ/qaqHjZ3PUvmhRyAA+G/N2YZdfcnq+oDwiO4Unhukn+oqo8n+VyGjWlTVTdJ8h9zFsbOWLP9mrjyOi3J147LbW+U5F+T3LG77SnKz2WYMc8WLGEDYL+rqk8led3K0LetHnf39+50TcDOqKo7ZFgW8sqNfUSq6qZJrtXd/zJrcQDZfTllVb2ju0+YsyaWoao+Yq/O7QmQANjvqurbp86vUZtvAOAgU1UXZtjkf8M9V4+7+xd3vCgWoao+3N06Bm/DEjYADoRzu/vDcxcBALCFX910bOnalUhVfSbJVjNpKsk1drictWIGEgD73erU8Kr6y+7+gblrAgDYrKqulSRat8OeXWXuAgA4KNXK7RvPVgUAwBaq6ueq6sNJPpTkQ1X1oar6+bnrgiUTIAFwIPQ2twEAZlVVv57ke5Lcqbuv193XS/IdSe42ngO2YAkbAPtdVX0xyX/lsrXkn904laS7+zpz1QYAXLlV1fuTnNDd/71p/BpJ3tHdN52nMlg2m2gDsN919yFz1wAAsI3eHB6Ng5+rqi/NURCsA0vYAAAAuDK5oKruvHmwqr4zyUdnqAfWgiVsAAAAXGlU1c2SvCTJ65O8dRw+Mckdk5zS3WfPVRssmQAJAACAK42qelCSNyQ5IcnGfkfvTfKcrZa2AQN7IAEAAHBlcnSSxyf5+iTvTPJPSf49yZcnESDBNsxAAgAA4Eqnqq6aYenaNyf5pvHrU919s1kLg4UyAwkAAIAro2skuU6S645f/1+Sd81aESyYGUgAAABcaVTV05LcPMlnkrwpyRuTvLG7PzlrYbBwV5m7AAAAANhBN0pytSQfS3JBkvOTfGrOgmAdmIEEAADAlUpVVYZZSN88ft0iycVJ3tDdj5yzNlgqARIAAABXSlV1dJI7ZgiR7p7ket192KxFwUIJkAAAALjSqKpfzGUzj/4nyT+vfL2ru780Y3mwWLqwAQAAcGVybJK/SPJL3f3RmWuBtWEGEgAAAACTdGEDAAAAYJIACQAAAIBJAiQAAAAAJgmQAAAAAJgkQAIAAABg0v8PGekETxkGzS4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Count the entities\n",
    "entities_count = {}\n",
    "entities_dic = {}\n",
    "\n",
    "for i in range(len(dataset_p)):\n",
    "    # Get the entities from the text\n",
    "    text = dataset_p.loc[i, 'tokens']\n",
    "    doc_web_sm = nlp(text)\n",
    "\n",
    "    doc_med7 = med7(text)\n",
    "    docs = [doc_web_sm, doc_med7]\n",
    "\n",
    "    #  Get the entities from the text\n",
    "    combined_entities = []\n",
    "    for doc in docs:         \n",
    "        combined_entities.extend([(ent.text, ent.label_) for ent in doc.ents])\n",
    "\n",
    "    # Count the entities\n",
    "    for ent in combined_entities:\n",
    "        if ent[1] in entities_count:\n",
    "            entities_count[ent[1]] += 1\n",
    "            entities_dic[ent[1]].append(ent[0])\n",
    "        else:\n",
    "            entities_count[ent[1]] = 1\n",
    "            entities_dic[ent[1]] = [ent[0]]\n",
    "\n",
    "# Sort the entities by count\n",
    "entities_count = dict(sorted(entities_count.items(), key=lambda item: item[1], reverse=True))\n",
    "\n",
    "# Plot the entities\n",
    "plt.figure(figsize=(20,10))\n",
    "plt.bar(entities_count.keys(), entities_count.values())\n",
    "plt.xticks(rotation=90)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "med7 entities\n",
      "DRUG :  ['lidocaine', 'colace', 'proscar']\n",
      "ROUTE :  ['intravenous', 'iv', 'intravenous']\n",
      "FREQUENCY :  ['daily', 'daily', 'daily']\n",
      "FORM :  ['patch', 'norco', 'tablet']\n",
      "STRENGTH :  ['pamelor mg', 'g', 'iv mg']\n",
      "DOSAGE :  ['one', 'cc', 'total cc']\n",
      "DURATION :  ['six week', 'four hour', 'taken twice day']\n"
     ]
    }
   ],
   "source": [
    "print('med7 entities')\n",
    "for ent in entities_dic:\n",
    "    if ent in med7.pipe_labels['ner']:\n",
    "        print(ent, ': ', entities_dic[ent][:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spacy entities\n",
      "PERSON :  ['mynx', 'xyz abc', 'tobrex']\n",
      "GPE :  ['anesthesia', 'anesthesia', 'scan']\n",
      "FAC :  ['mynx closure', 'macrodantin hydralazine physical', 'hour ampicillin']\n",
      "NORP :  ['scan', 'allis', 'meatus']\n",
      "ORG :  ['anesthesia department digital rectal', 'abc abc', 'abc']\n",
      "TIME :  ['daytime', 'three hour', 'morning']\n",
      "DATE :  ['five year', 'today', 'reported week']\n",
      "ORDINAL :  ['first', 'second', 'third']\n",
      "CARDINAL :  ['two', 'transcription', 'one']\n",
      "PRODUCT :  ['tympanic', 'tympanic', 'tympanic']\n",
      "QUANTITY :  ['foot inch', 'approximately ml nonionic', 'two x mm']\n",
      "LOC :  ['zuba', 'hagl lesion', 'po digoxin']\n",
      "PERCENT :  ['ejection fraction percent', 'minus ten percent percent', 'ii percent']\n",
      "WORK_OF_ART :  ['gram fluid surface', 'gram fluid surface', 'anesthesia tympanic membrane']\n",
      "MONEY :  ['e penny', 'e penny', 'several bolus fluid tachycardia']\n",
      "LANGUAGE :  ['english', 'spanish', 'spanish']\n",
      "EVENT :  ['vietnam', 'vietnam', 'provera advair']\n"
     ]
    }
   ],
   "source": [
    "print('spacy entities')\n",
    "for ent in entities_dic:\n",
    "    if ent in nlp.pipe_labels['ner']:\n",
    "        print(ent, ': ', entities_dic[ent][:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After the midterm presentation, we received feedback regarding ambiguities in our Named Entity Recognition (NER) entities and suggestions on how to enhance the model by avoiding lemmatization for NER recognition. Firstly, our ambiguities arose from the model sometimes assigning 'CARDINAL' and other times 'DOSAGE'. We couldn't merge them or eliminate one, as 'DOSAGE' also includes terms that may not be captured by 'CARDINAL', such as single. Furthermore, not every instance of a cardinal number implies a dosage of a medication.\n",
    "\n",
    "Secondly, as demonstrated at the end of our notebook, not normalizing the data does not yield better results.\n",
    "\n",
    "Additionally, in text 4671, our model successfully recognizes some routes of medication administration (such as endotracheal or ear) but fails to identify others that we have found (such as oral or nasal). It's important to note that the model is not failing; rather, the texts are challenging, containing numerous medical terms that, as non-experts, we may not have considered. Upon visually inspecting various texts, we can conclude that we have developed a robust model for recognizing drugs in medical texts.\n",
    "\n",
    "As seen in the printed examples from the med8 and english models, the recognized entities make sense and are correct."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Describe how NER type information could help in other NLP use cases."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NER can improve the understanding of texts and enable more advanced processing of text. The following are some examples where NER type information can benefit NLP applications:\n",
    "\n",
    "1. **Information Extraction**: NER can identify and classify entities, such as people, organizations, locations, dates, and more. This information is crucial for extracting structured data from unstructured text (legal documents, forms, news).\n",
    "\n",
    "2. **Sentiment Analysis**: identifying that a positive sentiment is expressed towards a particular brand or product in customer reviews can be valuable for businesses.\n",
    "\n",
    "3. **Question Answering**: NER can assist in locating entities within a text that are relevant to user queries.\n",
    "\n",
    "4. **Machine Translation**: NER can help identify and preserve the entities during the translation process, ensuring that names, places, and other important entities remain consistent across languages.\n",
    " NER can help identify sensitive information, such as personal names, addresses, or credit card numbers, and facilitate data protection measures.\n",
    "\n",
    "In summary, NER type information serves as a foundational element in many NLP applications, enhancing their ability to understand, extract, and act on information in text, making them more powerful and context-aware."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### NER without text processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also applied the standard NER classifier from spaCy without text preprocessing. Because we can't use the same annotations from preprocessed data, we need to annotate it again. It seems that the scores are worse."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found NE for Doc 1:\n",
      "\"58-year-old\"->DATE, \"African-American\"->NORP, \"16 years\"->DATE, \"X.\"->PERSON, \"second\"->ORDINAL, \"Hospital Center\"->FAC, \"four years\"->DATE, \"September 2009\"->DATE, \"three\"->CARDINAL, \"90 days\"->DATE, \"State Services\"->ORG, \"these 90 days\"->DATE, \"one\"->CARDINAL, \"September 2008\"->DATE, \"two sick days\"->DATE, \",In July of this year\"->DATE, \"Ph.D.\"->WORK_OF_ART, \"Comprehensive Independent Medical Evaluation\"->ORG, \"08/14/2009\"->DATE, \"08/27/2009\"->DATE, \"Henry Fein\"->PERSON, \"M.D.\"->GPE, \"09/23/2009\"->DATE, \"20/30\"->CARDINAL, \"daily\"->DATE, \"1991\"->DATE, \"approximately five\"->CARDINAL, \"the past year\"->DATE, \",OTHER MEDICAL\"->PERSON, \"hemangioma\"->ORG, \"C7\"->CARDINAL, \"C6-C7\"->PRODUCT, \"GERD\"->ORG, \"1976\"->DATE, \"1974\"->DATE, \"2007\"->DATE, \"approximately seven to eight hours\"->DATE, \"NovoLog\"->DATE, \"Topamax\"->PERSON, \"Naprosyn\"->PERSON, \"two\"->CARDINAL, \"four\"->CARDINAL, \"North Carolina\"->GPE, \"sixth\"->ORDINAL, \"nine\"->CARDINAL, \"third\"->ORDINAL, \"60\"->CARDINAL, \"93 years old\"->DATE, \"fourth\"->ORDINAL, \"Bachelor of Science in Nursing through\"->ORG, \"State University\"->ORG, \"1979\"->DATE, \"two years\"->DATE, \"1980\"->DATE, \"ages 43 and\"->DATE, \"30\"->CARDINAL, \"New York\"->GPE, \"2-1/2 years\"->DATE, \"Walter P. Carter Center\"->PERSON, \"21 years\"->DATE, \"OB\"->GPE, \"between 1991 and 1997\"->DATE, \"Prozac\"->PERSON, \"ADMINISTERED:,Clinical Interview\"->ORG, \"MMSE),Cognistat Neurobehavioral Cognitive Status Examination\"->ORG, \"Variables of Attention\"->ORG, \"Symbol Digit Modalities Test\"->ORG, \"Aphasia Examination\"->ORG, \"Token Test\"->PERSON, \"Sentence Repetition\"->ORG, \"Spelling Test\"->ORG, \"Aural Comprehension\"->ORG, \"Boston Naming Test\"->ORG, \n",
      "Found NE for Doc 2:\n",
      "\"TAH\"->ORG, \"250\"->CARDINAL, \"1500\"->CARDINAL, \"125\"->CARDINAL, \"ovarian\"->NORP, \"44-year-old\"->DATE, \"May\"->DATE, \"three weeks ago\"->DATE, \"anesthesia\"->GPE, \"Pfannenstiel\"->GPE, \"first\"->ORDINAL, \"second\"->ORDINAL, \"Mayo\"->ORG, \"Ochsner\"->ORG, \"Lahey\"->ORG, \"Allis\"->PERSON, \"#0\"->MONEY, \"Heaney\"->GPE, \"Russian\"->NORP, \"Metzenbaum\"->ORG, \"Ray-Tec\"->PERSON, \"Jorgenson\"->PERSON, \"Betadine\"->ORG, \"0\"->MONEY, \"Vicryl\"->ORG, \"#3-0\"->MONEY, \n"
     ]
    }
   ],
   "source": [
    "#Retrieve NE \n",
    "def retrieve_NE(text):\n",
    "    doc = nlp(text) \n",
    "    ents = dict((e.text, e.label_) for e in doc.ents)\n",
    "    for key, value in ents.items():\n",
    "        print(f\"\\\"{key}\\\"->{value}, \", end=\"\")\n",
    "    print(\"\")\n",
    "    return doc, ents\n",
    "\n",
    "print(\"Found NE for Doc 1:\")\n",
    "doc1, ents_1 = retrieve_NE(dataset[\"train\"][951][\"text\"])\n",
    "print(\"Found NE for Doc 2:\")\n",
    "doc2, ents_2 = retrieve_NE(dataset[\"train\"][4794][\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get ground truth from annotations which were manually labelled with https://tecoholic.github.io/ner-annotator/\n",
    "# Get ground truth from annotations which were manually labelled with https://tecoholic.github.io/ner-annotator/\n",
    "ground_truth_1 = get_ground_truth(\"gt_annotations/annotations_1.json\")\n",
    "ground_truth_2 = get_ground_truth(\"gt_annotations/annotations_2.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores:\n",
      "Precision: 0.25396825396825395\n",
      "Recall: 0.2882882882882883\n",
      "F1-Score: 0.270042194092827\n"
     ]
    }
   ],
   "source": [
    "#Calculcate scores\n",
    "from spacy.training import Example\n",
    "def print_scores(docs_with_ground_truth):\n",
    "    scorer = Scorer()\n",
    "    scores = scorer.score(docs_with_ground_truth) \n",
    "    entities_precision=scores[\"ents_p\"]\n",
    "    entities_recall=scores[\"ents_r\"]\n",
    "    entities_f1=scores[\"ents_f\"]\n",
    "    entities_per_type=scores[\"ents_per_type\"]\n",
    "    \n",
    "    #print scores\n",
    "    print(f\"Precision: {entities_precision}\")\n",
    "    print(f\"Recall: {entities_recall}\")\n",
    "    print(f\"F1-Score: {entities_f1}\")\n",
    "\n",
    "\n",
    "docs_with_ground_truth = [Example.from_dict(doc1, ground_truth_1), Example.from_dict(doc2, ground_truth_2)]\n",
    "print(\"Scores:\")\n",
    "print_scores(docs_with_ground_truth) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}