csg_datafusion_extract_fields_from_reports.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reports Fields Extractor and stats displayer\n",
    "----------------\n",
    "## DESCRIPTION\n",
    "From a folder of pdf and doc/docx files, this notebook will extract all pertinent informations and build a CSV database. The information is mainly extracted from the conclusion, but some information like sedation can be extracted from the rest of the document body.\n",
    "\n",
    "This notebook is specifically tailored to work on Coma Science Group's patients reports, but it can provide a good basis for a custom tailored reports extractor by adapting some of the code (in particular the \"Conclusion\" section detection) or by adapting your report to fit the template.\n",
    "\n",
    "Part of this notebook was reused to make the python module easy_textract (available on pypi).\n",
    "\n",
    "By Stephen Larroque @ Coma Science Group, GIGA Research, University of Liege\n",
    "2017-2019\n",
    "Creation date: 2017-01-29\n",
    "License: MIT\n",
    "v1.7.1\n",
    "\n",
    "## INSTALL NOTE\n",
    "You need to pip install textract and pandas and install tesseract for your platform before launching this script.\n",
    "Tested on Python 2.7.11\n",
    "\n",
    "## TODO\n",
    "* Provide a reports template.\n",
    "* check that the exclude ref rejects all false detection of coma (because of coma science group or nociception/nociceptive coma scale)\n",
    "* fix sedation NA if possible\n",
    "* clinical and other modalities: use finditer and accept multiple positions (assign diagnosis to the closest modality on the left, not on the right)\n",
    "* accident_date use in priority the \"xx years xx months xx days post...\" instead of date because more reliable (or we can test, if it is > acquisition_date_end then the accident_date is wrong! ex DeVaal)\n",
    "* accident_etiology use finditer and append with '/' separator all etiologies found\n",
    "* non deterministic script, some rows in csv change after running the same code! try to find why -> Should be fixed, was probably because of unidecode not supporting relative import (so sometimes it used the fallback lib unicodedata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forcefully autoreload all python modules\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_folder_of_reports = r'G:\\Topreproc\\Reports\\reports_all'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reports Fields Extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "cur_path = os.path.realpath('.')\n",
    "sys.path.append(os.path.join(cur_path, 'csg_fileutil_libs'))  # for unidecode, because it does not support relative paths (yet? they need to use __import__(path, globals(), level=2))\n",
    "\n",
    "import re\n",
    "import six\n",
    "import shutil\n",
    "import textract\n",
    "\n",
    "import csg_fileutil_libs.dateutil.parser as dateutil_parser\n",
    "import csg_fileutil_libs.langdetect as langdetect\n",
    "\n",
    "from collections import OrderedDict\n",
    "from tempfile import mkdtemp, mkstemp\n",
    "from textract.parsers.utils import ShellParser\n",
    "\n",
    "from csg_fileutil_libs.aux_funcs import disambiguate_names, _unidecode, _tqdm, recwalk\n",
    "\n",
    "\n",
    "##### Auxiliary functions #####\n",
    "\n",
    "def replace_buggy_accents(s, encoding=None):\n",
    "    \"\"\"Fix weird encodings that even ftfy cannot fix\"\"\"\n",
    "    # todo enhance speed? or is it the new regex on name?\n",
    "    dic_replace = {\n",
    "        '\\xc4\\x82\\xc2\\xa8': 'e',\n",
    "        'ĂŠ': 'e',\n",
    "        'Ăť': 'u',\n",
    "        'â': 'a',\n",
    "        'Ă´': 'o',\n",
    "        'Â°': '°',\n",
    "        'â': \"'\",\n",
    "        'ĂŞ': 'e',\n",
    "        'ÂŤ': '«',\n",
    "        'Âť': '»',\n",
    "        'Ă': 'a',\n",
    "        'AŠ': 'e',\n",
    "        'AŞ': 'e',\n",
    "        'A¨': 'e',\n",
    "        'A¨': 'e',\n",
    "        'Ă': 'E',\n",
    "        'â˘': '*',\n",
    "        'č': 'e',\n",
    "        '’': '\\'',\n",
    "    }\n",
    "    for pat, rep in dic_replace.items():\n",
    "        if encoding:\n",
    "            pat = pat.decode(encoding)\n",
    "            rep = rep.decode(encoding)\n",
    "        s = s.replace(pat, rep)\n",
    "    return s\n",
    "\n",
    "def date_fr2en(s):\n",
    "    \"\"\"Convert french month names into english so that dateutil.parse works\"\"\"\n",
    "    s = s.lower()\n",
    "    rep = {\n",
    "        'jan\\w+': 'jan',\n",
    "        'fe\\w+': 'feb',\n",
    "        'mar\\w+': 'march',\n",
    "        'av\\w+': 'april',\n",
    "        'mai\\w+': 'may',\n",
    "        'juin\\w+': 'june',\n",
    "        'juil\\w+': 'july',\n",
    "        'ao\\w+': 'august',\n",
    "        'se\\w+': 'september',\n",
    "        'oc\\w+': 'october',\n",
    "        'no\\w+': 'november',\n",
    "        'de\\w+': 'december',\n",
    "    }\n",
    "    for m, r in rep.items():\n",
    "        s = re.sub(m, r, s)\n",
    "    return s\n",
    "\n",
    "def calculate_age(birthdate, acquisitiondate):\n",
    "    # The correct way to do this is to calculate the age directly from the dates, by subtracting the two years, and then subtracting one if the current month/day precedes the birth month/day.\n",
    "    # from http://stackoverflow.com/a/765862/1121352\n",
    "    return acquisitiondate.year - birthdate.year - ((acquisitiondate.month, acquisitiondate.day) < (birthdate.month, birthdate.day))\n",
    "\n",
    "\n",
    "##### Custom text filtering classes #####\n",
    "\n",
    "class TextDiagnosticFilter(object):\n",
    "    '''Test if a diagnosis should be rejected or excluded given text patterns'''\n",
    "    def __init__(self, text, exclude_patterns, reject_patterns, confirm_patterns):\n",
    "        #self.text = text\n",
    "        self.exclude_patterns = exclude_patterns\n",
    "        self.reject_patterns = reject_patterns\n",
    "        self.confirm_patterns = confirm_patterns\n",
    "        # Construct list of exclusion sentences positions\n",
    "        self.pos_excludes = []\n",
    "        for excl_pat in exclude_patterns:\n",
    "            for m in re.finditer(excl_pat.replace(' ', '\\s+'), text):\n",
    "                self.pos_excludes.append(m)\n",
    "        # Find all reject and confirmation terms (text patterns that reject or accept the diagnosis)\n",
    "        self.pos_rejects = [match.span(0)[0] for match in re.finditer('('+'|'.join(reject_patterns).replace(' ', '\\s+')+')', text)]\n",
    "        self.pos_confirms = [match.span(0)[0] for match in re.finditer('('+'|'.join(confirm_patterns).replace(' ', '\\s+')+')', text)]\n",
    "        # Find positions of all dots (to stop reject terms)\n",
    "        self.pos_dots = [match.span(0)[0] for match in re.finditer('(\\.+)', text)]\n",
    "\n",
    "    def reject_filter(self, start_pos, pos_end):\n",
    "        '''Limit reject terms to only a specified start and end boundary'''\n",
    "        # Filter reject/confirm terms that are outside of boundaries\n",
    "        self.pos_rejects_filt = filter(lambda x: start_pos <= x <= pos_end, self.pos_rejects)\n",
    "        self.pos_confirms_filt = filter(lambda x: start_pos <= x <= pos_end, self.pos_confirms)\n",
    "        self.pos_dots_filt = filter(lambda x: start_pos <= x <= pos_end, self.pos_dots)\n",
    "\n",
    "    def reject_test(self, state_pos, wholetext=False, bidirectional=False):\n",
    "        '''Test if the given position is AFTER one of the reject terms (and that there is no confirm term in-between)'''\n",
    "        rejected = False\n",
    "        if self.pos_rejects:\n",
    "            if wholetext:\n",
    "                pos_confirms = self.pos_confirms\n",
    "                pos_rejects = self.pos_rejects\n",
    "                pos_dots = self.pos_dots\n",
    "            else:  # account only for reject and confirm terms in a specified start and end position\n",
    "                pos_confirms = self.pos_confirms_filt\n",
    "                pos_rejects = self.pos_rejects_filt\n",
    "                pos_dots = self.pos_dots_filt\n",
    "\n",
    "            for pos_reject in pos_rejects:\n",
    "                # Reject term found before the diagnosis and no confirm term in-between: skip this diagnosis\n",
    "                if pos_reject < state_pos and not any(filter(lambda pos_confirm: pos_reject < pos_confirm < state_pos, pos_confirms)) and not any(filter(lambda pos_dot: pos_reject < pos_dot < state_pos, pos_dots)):\n",
    "                # One-liner to do the same: if not any(filter(lambda pos_reject: pos_reject < pos_diag and not any(filter(lambda pos_confirm: pos_reject < pos_confirm < pos_diag, pos_confirms)), pos_rejects)):  # skip if there was a reject term anywhere before the diagnosis\n",
    "                    rejected = True\n",
    "                    break\n",
    "                # Same if reject term found after diagnosis and no confirm term in-between\n",
    "                if bidirectional and pos_reject > state_pos and not any(filter(lambda pos_confirm: state_pos < pos_confirm < pos_reject, pos_confirms)) and not any(filter(lambda pos_dot: state_pos < pos_dot < pos_reject, pos_dots)):\n",
    "                    rejected = True\n",
    "                    break\n",
    "        return rejected\n",
    "\n",
    "    def exclude_test(self, pos_diag):\n",
    "        '''Test if the given position is INSIDE one of the matched excludes'''\n",
    "        if (not any(self.pos_excludes) or not any([excl.start() <= pos_diag <= excl.end() for excl in self.pos_excludes if excl])):\n",
    "            return False\n",
    "        else:\n",
    "            return True\n",
    "\n",
    "\n",
    "##### Custom extraction classes for textract #####\n",
    "\n",
    "class MyDocParser(ShellParser):\n",
    "    \"\"\"Extract text from doc files using antiword (need to be placed in C:\\antiword\\antiword.exe or ~/antiword/antiword).\"\"\"\n",
    "\n",
    "    def extract(self, filename, procpath=None, **kwargs):\n",
    "        if procpath is None:\n",
    "            if os.name == 'nt':\n",
    "                procpath = 'C:/antiword/antiword.exe'\n",
    "            else:\n",
    "                procpath = '~/antiword/antiword'\n",
    "        stdout, stderr = self.run([procpath, filename])\n",
    "        return stdout\n",
    "\n",
    "class MyOCRParser(ShellParser):\n",
    "    \"\"\"Extract text from various image file formats or pdf containing scan images using tesseract-ocr (compatible with tesseract v3.02.02, only version currently available on Windows)\"\"\"\n",
    "    \n",
    "    def extract(self, filename, **kwargs):\n",
    "        if filename.endswith('.pdf'):\n",
    "            return self.extract_pdf(filename, **kwargs)\n",
    "        else:\n",
    "            return self.extract_image(filename, **kwargs)\n",
    "\n",
    "    def extract_image(self, filename, **kwargs):\n",
    "        \"\"\"Extract text from various image file formats using tesseract-ocr (compatible with tesseract v3.02.02, only version currently available on Windows)\"\"\"\n",
    "        # TODO: if proportion of image wrong, resize automatically to fit A4 proportions using PILLOW! if width > percentage_threshold, downsize width, else if width <, then downsize height.\n",
    "        filename = os.path.abspath(filename)  # tesseract need absolute paths!\n",
    "        dirpath = os.path.dirname(filename)\n",
    "        # Create a temporary output txt file for tesseract\n",
    "        tempfilefh, tempfilepath = mkstemp(suffix='.txt')  # tesseract < 3.03 do not support \"stdout\" argument, so need to save into a file\n",
    "        os.close(tempfilefh)  # close to allow writing to tesseract\n",
    "        tempfile = tempfilepath[:-4]  # remove suffix to supply as argument to tesseract, because tesseract always append '.txt'\n",
    "        # if language given as argument, specify language for tesseract to use\n",
    "        if 'language' in kwargs:\n",
    "            args = ['tesseract', filename, tempfile, '-l', kwargs['language']]\n",
    "        else:\n",
    "            args = ['tesseract', filename, tempfile]\n",
    "\n",
    "        stdout, _ = self.run(args)\n",
    "        # Read the results of extraction\n",
    "        with open(tempfilepath, 'rb') as f:\n",
    "            res = f.read()\n",
    "        # Remove temporary output file\n",
    "        os.remove(tempfilepath)\n",
    "        return res\n",
    "\n",
    "    def extract_pdf(self, filename, **kwargs):\n",
    "        \"\"\"Extract text from pdfs using tesseract (per-page OCR).\"\"\"\n",
    "        temp_dir = mkdtemp()\n",
    "        base = os.path.join(temp_dir, 'conv')\n",
    "        contents = []\n",
    "        try:\n",
    "            stdout, _ = self.run(['pdftoppm', filename, base])  # from poppler, http://poppler.freedesktop.org\n",
    "\n",
    "            for page in sorted(os.listdir(temp_dir)):\n",
    "                page_path = os.path.join(temp_dir, page)\n",
    "                page_content = self.extract_image(page_path, **kwargs)\n",
    "                contents.append(page_content)\n",
    "            return six.b('').join(contents)\n",
    "        finally:\n",
    "            shutil.rmtree(temp_dir)\n",
    "\n",
    "\n",
    "##### Main reports fields extraction functions #####\n",
    "\n",
    "def extract_report_fields(report_path, root_path=None, ocr=False, return_text=False):\n",
    "    \"\"\"Extract patient's fields from a PDF report\"\"\"\n",
    "    patient_fields = {}\n",
    "    # List of accepted languages (to exclude gibberish pdf)\n",
    "    langs_ok = ['fr', 'en', 'nl']\n",
    "    langs_ok_prob = 0.9  # probability of confidence necessary to not reject the lang\n",
    "    # Extract text from document and remove accentuated characters and strip blank spaces\n",
    "    if report_path.endswith('.doc'):  # .doc filetype needs antiword (not docx, use textract directly!)\n",
    "        docparser = MyDocParser()\n",
    "        report_text = _unidecode(replace_buggy_accents(docparser.process(report_path, 'utf8').decode('utf8'), 'utf8')).lower()\n",
    "    else:  # other filetypes should be supported as-is\n",
    "        try:\n",
    "            report_text = _unidecode(replace_buggy_accents(textract.process(report_path).decode('utf8'), 'utf8')).lower().strip()\n",
    "            # Failed to decode anything from document (maybe pdf contains only image and no text? Can try to use tesseract with textract but lot of work for not much...)\n",
    "            if not report_text:\n",
    "                raise ValueError('No text extractable from the specified file.')\n",
    "            else:\n",
    "                lang_check = langdetect.detect_langs(report_text)[0]\n",
    "                if lang_check.lang not in langs_ok or lang_check.prob < langs_ok_prob:\n",
    "                    raise ValueError('No text extractable or language unrecognized from the specified file.')\n",
    "        except Exception as exc:\n",
    "            # Try to decode using OCR\n",
    "            if ocr:\n",
    "                ocrparser = MyOCRParser()\n",
    "                #report_text = _unidecode(replace_buggy_accents(textract.process(report_path, method='tesseract', language='fra').decode('utf8'), 'utf8')).lower().strip()  # Should work, but does not on Windows because you need tesseract v3.03 with support for \"stdout\", which is currently unavailable on Windows...\n",
    "                report_text = _unidecode(replace_buggy_accents(ocrparser.process(report_path, 'utf8').decode('utf8'), 'utf8')).lower().strip()\n",
    "            if not ocr or not report_text:  # Failed again, raise the exception!\n",
    "                raise\n",
    "    report_text = re.sub('[ \\t\\f\\v]+', ' ', report_text)  # replace abusive spaces\n",
    "    report_text = re.sub('[\\n\\r]+', '\\n', report_text)  # replace abusive line breaks\n",
    "    report_text = re.sub('(\\r?\\s?\\n\\r?\\s?)+', '\\n', report_text)  # replace abusive line breaks\n",
    "    # Failed to decode anything from document, raise exception\n",
    "    if not report_text:\n",
    "        raise ValueError('No text extractable from the specified file.')\n",
    "    # Detect language\n",
    "    try:\n",
    "        lang = langdetect.detect(report_text)\n",
    "    except Exception as exc:\n",
    "        lang = 'fr'\n",
    "\n",
    "    # Mandatory fields, if missing, we need to fail loudly\n",
    "    patient_fields['report_lang'] = lang\n",
    "    patient_fields['report_path'] = report_path if not root_path else os.path.relpath(report_path, root_path)\n",
    "    # Get name, the most important field\n",
    "    try:\n",
    "        pts_name = re.search('(concern(e|ing|s)?|betreft|patient\\s*:\\s*)\\s*(:\\s+)?(rapport.+?|etude.+?de\\s+)?((mrs?\\.|monsieur|madame|mademoiselle)\\s*)?((\\s*[a-zA-Z\\-]+(,?|\\s+|\\-+|/)){2,6}?)([,.\\r\\n]|\\s+nee?\\s+|\\s*patient|\\s+agee?\\s+|\\s*[\\(]|\\s*\\d)', report_text).group(7)\n",
    "        if len(pts_name) < 4:\n",
    "            raise AttributeError\n",
    "    except AttributeError as exc:\n",
    "        #pts_name = re.search('\\s+(([a-zA-Z\\-]+(,?\\s|\\-+|/)){2,4}?)\\W{,5}(\\s+nee?\\s+|patient|agee?\\s+|\\s*[\\(]).{,15}\\d+', report_text, re.S).group(1)\n",
    "        pts_name = re.search('\\s+(([a-zA-Z\\-]+(,?\\s|\\-+|/)){2,4}?)(\\W{,5}|[0-9\\.\\-/]+)(\\s+(nee?|born)\\s+|patient|agee?\\s+|\\s*[\\(])(.{,15}\\d+)?', report_text, re.S).group(1)\n",
    "    # Clean up the name (cannot perfectly detect, but if at least we match the name, we can delete the rest more easily)\n",
    "    pts_name = re.sub('(^|\\s+)(geboren(\\s+op|\\W+)?|born(\\s+on|\\W+|$)|(nas|ny|nee?)\\s+le|(chef\\s+)?de\\s+service|pet\\s*\\-+\\s*scan\\s+\\-+\\s+|(coma\\s+)?science\\s+group|patiente?s?)', '', pts_name)  # remove \"born on\" and other wrongly matched sentences\n",
    "    patient_fields['name'] = re.sub('\\-+', '-', re.sub('\\s+', ' ', pts_name)).strip().replace('\\r', '').replace('\\n', '').replace('\\t', '').replace(',', ' ').replace('  ', ' ').strip()  # clean up spaces, punctuation and double dashes in name\n",
    "\n",
    "    if lang == 'fr':\n",
    "        try:\n",
    "            patient_fields['birthdate'] = re.search(pts_name+'.+?(((nee?\\s*(le|en|12)?|ans\\W+?)\\s*).*?)?[\\(\\s]?((?!<[^\\d])\\d{1,2}[-/.]\\d{1,2}[-/.]\\d{2,4}|\\d{4})', report_text, re.S).group(5)\n",
    "            # if the birthdate is only a single number (eg, 100) and not a year (eg, 1974), then we matched the wrong number because this is not a date format. Retry without \"ne le\"\n",
    "            if re.match('^\\d+$', patient_fields['birthdate']) and not re.match('^(19|20)\\d{2}$', patient_fields['birthdate']):\n",
    "                raise AttributeError()\n",
    "        except AttributeError as exc:\n",
    "            # Optional middle part (\"nee le ...\")\n",
    "            patient_fields['birthdate'] = re.search(pts_name+'.+?((nee?\\s*(le|en|12)?|ans\\W+?|[\\(])\\s*)?[\\(\\s]?((\\d+[-/.]\\d+[-/.])?\\d{2,4})', report_text, re.S).group(4)\n",
    "    elif lang == 'nl':\n",
    "        patient_fields['birthdate'] = re.search('betreft.+?(geboren(\\s+op)\\s*)(\\d+([-/.]\\d+[-/.]\\d+)?)', report_text, re.S).group(3)\n",
    "    else:  # lang == 'en'\n",
    "        bd1 = re.search('(years?(\\s+|-)?old|born).+?(\\d+[-/.]\\d+[-/.]\\d+)', report_text, re.S)\n",
    "        bd2 = re.search('(\\d+[-/.]\\d+[-/.]\\d+)\\s+(born|years)', report_text, re.S)\n",
    "        if bd2 and bd2.span(1)[0] < bd1.span(3)[0]:\n",
    "            patient_fields['birthdate'] = bd2.group(1)\n",
    "        else:\n",
    "            patient_fields['birthdate'] = bd1.group(3)\n",
    "\n",
    "    # Optional fields, can be fetched from DICOM for most of them\n",
    "    # Find gender\n",
    "    gender = 'NA'\n",
    "    if lang == 'fr':\n",
    "        if 'patiente' in report_text: # and report_text.find('patiente') <= report_text.find('patient'):  # sometimes there are mistakes in subsequent pages, but first page is usually safe\n",
    "            gender = 'F'\n",
    "        elif 'patient' in report_text:\n",
    "            gender = 'M'\n",
    "    elif lang == 'en':\n",
    "        if 'female' in report_text or 'woman' in report_text or ' she ' in report_text or ' her ' in report_text:\n",
    "            gender = 'F'\n",
    "        elif ' male ' in report_text or ' man ' in report_text or ' his ' in report_text or ' he ' in report_text:\n",
    "            gender = 'M'\n",
    "    elif lang == 'nl':\n",
    "        if 'vrouw' in report_text:\n",
    "            gender = 'F'\n",
    "        elif ' man ' in report_text:\n",
    "            gender = 'M'\n",
    "    patient_fields['gender'] = gender\n",
    "\n",
    "    # Age\n",
    "    try:\n",
    "        patient_fields['age'] = re.search('(\\d+)(\\s+|-)(ans|years?(\\s+|-)old)', report_text).group(1)\n",
    "    except AttributeError as exc:\n",
    "        try:\n",
    "            # try without any space but with parentheses\n",
    "            patient_fields['age'] = re.search('[\\(]\\s*(\\d+)(\\s|-)?(ans|years?(\\s+|-)old)\\s*[\\)]', report_text).group(1)\n",
    "        except AttributeError as exc:\n",
    "            patient_fields['age'] = 'NA'\n",
    "    # Accident's date and etiology (\"post\" something)\n",
    "    try:\n",
    "        #match_etiology = re.search(r'((accident|trauma|arrest|arret\\s+cardiaque|(months|years)\\s+(?!old)|post[\\-\\s]+(?!hospital))([\\(\\)a-zA-Z0-9\\<\\>\\-]+\\s+){1,11}?).*?(le|on)?(\\d+(/|-)\\s*\\d+(/|-)\\s*\\d+|\\d+\\s*\\w+\\s*\\d+)', report_text, re.S)\n",
    "        match_etiology = re.search(r'(((accident|trauma|cardiac\\s+arrest|arrest|arret\\s+cardiaque|(months|years)\\s+(?!old)(?=[\\(\\)a-zA-Z0-9\\<\\>\\-])|post[\\-\\s]+(?!hospital))([\\(\\)a-zA-Z0-9\\<\\>\\-]+\\s+){,11}?)[^\\.]{,100}?)(\\s+(le|on)\\s+)?(\\d+(/|-)\\s*\\d+(/|-)\\s*\\d+|\\d+\\s*\\w+\\s*\\d+)', report_text, re.S)\n",
    "        patient_fields['accident_date'] = match_etiology.group(8)\n",
    "        patient_fields['accident_etiology'] = match_etiology.group(1).replace('\\n', ' ').strip()\n",
    "    except AttributeError as exc:\n",
    "        patient_fields['accident_date'] = 'NA'\n",
    "        patient_fields['accident_etiology'] = 'NA'\n",
    "\n",
    "    # Get report date if available (in filename)\n",
    "    report_filename = os.path.basename(os.path.normpath(report_path))\n",
    "    try:\n",
    "        patient_fields['report_date'] = re.findall('(\\d{0,4}(19|20)\\d{2})', report_filename)[-1][0]\n",
    "    except IndexError as exc:\n",
    "        # Get report date in header \"liege, le xx month xxxx\"\n",
    "        try:\n",
    "            patient_fields['report_date'] = re.search('liege\\s*,?\\s*le\\s+(\\d+\\s+[a-zA-Z0-9]+\\s+\\d+)', report_text).group(1)\n",
    "        except AttributeError as exc:\n",
    "            patient_fields['report_date'] = 'NA'\n",
    "\n",
    "    # Get acquisition date\n",
    "    try:\n",
    "        # TODO: Get from DICOM! Problem: there are multiple sessions for some patients, so it's not bulletproof...\n",
    "        patient_acquisition_dates = re.search(r'((du|from)\\s+(\\w+\\s+){,2})?((\\d+[\\-/\\.\\s]+)?(\\d+[\\-/\\.\\s]+)?\\d+)\\s+(au|to)\\s+((\\d+[\\-/\\.\\s]+)?(\\d+[\\-/\\.\\s]+|\\w+\\s+)\\d+)', report_text)\n",
    "        patient_acquisition_start = patient_acquisition_dates.group(4)\n",
    "        patient_acquisition_end = patient_acquisition_dates.group(8)\n",
    "        patient_fields['acquisition_date'] = '%s - %s' % (patient_acquisition_start, patient_acquisition_end)\n",
    "        patient_fields['acquisition_date_end'] = '%s' % (patient_acquisition_end)\n",
    "        # TODO: compute age from acquisition if missing age: acquisition_date - birth_date\n",
    "    except AttributeError as exc:\n",
    "        try:\n",
    "            patient_acquisition_dates = re.search(r'((evalu\\w+)(\\s+\\w+){,10}\\s+le\\s+|(perform\\w+)(\\s+\\w+){,10}\\s+on\\s+(the\\s+)?|rmn.{,30}?)((\\d+[th\\-/\\.\\s]+)?(\\d+[\\-/\\.\\s]+|\\w+\\s+)\\d+)', report_text)\n",
    "            patient_acquisition_end = patient_acquisition_dates.group(7)\n",
    "            patient_fields['acquisition_date'] = '%s - %s' % (patient_acquisition_end, patient_acquisition_end)\n",
    "            patient_fields['acquisition_date_end'] = '%s' % (patient_acquisition_end)\n",
    "        except AttributeError as exc:\n",
    "            patient_fields['acquisition_date'] = 'NA'\n",
    "            patient_fields['acquisition_date_end'] = 'NA'\n",
    "\n",
    "    # Get sedation usage\n",
    "    mrisedation = 'NA'\n",
    "    if lang == 'fr':\n",
    "        if re.search('(irm|imagerie)\\s+fonctionnelle.+?(avec|sous)\\s+sedation', report_text):\n",
    "            mrisedation = 'yes'\n",
    "        elif re.search('(irm|imagerie)\\s+fonctionnelle.+?(sans|pas\\s+de|aucune)\\s+sedation', report_text):\n",
    "            mrisedation = 'no'\n",
    "        elif re.search('(sous|avec)\\s+(sedation|anesthesi)', report_text):\n",
    "            mrisedation = 'yes'\n",
    "        elif re.search('(sans|pas\\s+de|aucune)\\s+(sedation|anesthesi)', report_text):\n",
    "            mrisedation = 'no'\n",
    "        elif re.search('patiente?\\s+(sedatee?|anesthesie)', report_text):\n",
    "            mrisedation = 'yes'\n",
    "        elif re.search('patiente?\\s+non\\s+(sedatee?|anesthesie)', report_text):\n",
    "            mrisedation = 'no'\n",
    "    elif lang == 'en':\n",
    "        if re.search('functional\\s+(mri|imagery).+?(with|under)\\s+sedation', report_text):\n",
    "            mrisedation = 'yes'\n",
    "        elif re.search('functional\\s+(mri|imagery).+?(without|no)\\s+sedation', report_text):\n",
    "            mrisedation = 'yes'\n",
    "        elif re.search('((with|under)\\s+sedation|\\s+sedated)', report_text):\n",
    "            mrisedation = 'yes'\n",
    "        elif re.search('((without|no)\\s+sedation|unsedated)', report_text):\n",
    "            mrisedation = 'no'\n",
    "        elif re.search('\\s+(sedated|anesthetized)\\s+patient', report_text):\n",
    "            mrisedation = 'yes'\n",
    "    elif lang == 'nl':\n",
    "        if re.search('(niet|zonder(\\s+dat)?|geen)\\s+(verdoving|sedatie)', report_text):\n",
    "            mrisedation = 'no'\n",
    "        elif re.search('\\s+(verdoving|sedatie)', report_text):\n",
    "            mrisedation = 'yes'\n",
    "    patient_fields['mri_sedation'] = mrisedation\n",
    "\n",
    "    # Get final diagnosis, should be mandatory but might fail, in this case check manually the reports\n",
    "    # Extract all \"conclusion parts\" positions (start and end)\n",
    "    if lang == 'nl':\n",
    "        idx_s_all = [m.start() for m in re.finditer('(^|\\s+)conclusie[^,.]', report_text)]\n",
    "        end_of_report = report_text.find('met collegiale groeten')\n",
    "    else:\n",
    "        idx_s_all = [m.start() for m in re.finditer('(^|\\s+)(conclusion|conlcusion)[^,.]', report_text)]  # avoid \"en conclusion, ...\"\n",
    "        end_of_report = report_text.find('confraternellement')\n",
    "    # If can't find confraternellement, try to find Laureys's signature\n",
    "    if not end_of_report or end_of_report == -1:\n",
    "        end_of_report = report_text.find('(sincere|confidentially,)')  # sincerely/sincerement\n",
    "        if not end_of_report or end_of_report == -1:\n",
    "            #end_of_report = report_text.rfind('laureys')\n",
    "            end_of_report = len(report_text)\n",
    "\n",
    "    # Find the \"conclusion\" that is the closest to \"confraternellement\" (because there can be several conclusion parts for each modality, but we are looking for the final conclusion part that is doing a summary of all the results)\n",
    "    #idx_s = 0 # in case there is no \"conclusion\" part\n",
    "    idx_s_id = -1\n",
    "    idx_s_all.insert(0, 0) # in case there is no \"conclusion\" part, then search through the whole document\n",
    "    for i in range(len(idx_s_all)):\n",
    "        if idx_s_all[i] > end_of_report:\n",
    "            break\n",
    "        else:\n",
    "            #idx_s = idx_s_all[i]\n",
    "            idx_s_id = i\n",
    "\n",
    "    # Define blacklist sentences (ie, any diagnosis found in these sentences will be excluded, such as bibliographic citations)\n",
    "    # Note: spaces will be automatically replaced by '\\s+'\n",
    "    # Note2: must be all lowercase! Else no detection!\n",
    "    exclude_patterns = ['from\\s+unresponsive\\s+wakefulness\\s+to\\s+minimally\\s+conscious\\s+plus\\s+and\\s+functional\\s+locked-in\\s+syndromes',\n",
    "                'indique\\s+l\\'emergence\\s+de\\s+l\\'etat\\s+de\\s+conscience\\s+minimale',\n",
    "                'indique\\s+un\\s+etat\\s+de\\s+conscience\\s+minimale',\n",
    "                'ecm+\\s*:\\s*etat\\s+de\\s+conscience\\s+minimale\\s+plus\\s+\\(presence\\s+d’une\\s+reponse\\s+a\\s+la\\s+commande\\)',\n",
    "                'ecm-\\s*:\\s*etat\\s+de\\s+conscience\\s+minimale\\s+moins\\s+\\(pas\\s+de\\s+reponse\\s+a\\s+la\\s+commande\\)',\n",
    "                'comportements\\s+associes\\s+au\\s+diagnostic\\s+d’etat\\s+de\\s+conscience\\s+minimale',\n",
    "                'ev\\s*:\\s*etat\\s+vegetatif\\s+/\\s+syndrome\\s+d’eveil\\s+non\\s+repondant',\n",
    "                'who\\s+emerge\\s+from\\s+the\\s+(mcs|minimal(ly)?\\s+conscious\\s+state)\\.',\n",
    "                'indicates?\\s+(the\\s+)?emergence\\s+(of|from)\\s+minimally\\s+conscious\\s+state',\n",
    "                'indicates?\\s+(a|the)\\s+minimally\\s+conscious\\s+state(\\s+\\(mcs\\))?',\n",
    "                '(nocicepti(on|ve)\\s+coma\\s+scale|coma\\s+science\\s+group)',\n",
    "                'mcs\\s*-: patient shows non-reflexive behavior',\n",
    "                'mcs+: patient shows command following',\n",
    "                'mcs+: minimally conscious state plus',\n",
    "                'uws/vs: unresponsive wakefulness syndrome\\s*/\\s*vegetative state',\n",
    "                'denotes emergence (from|of) (mcs|minimally conscious state)',\n",
    "                'denotes (mcs|minimally conscious state)',\n",
    "                'vs/uws\\s*:\\s*vegetative state/unresponsive wakefulness syndrome',\n",
    "                'functional connectivity in the default network during resting state is preserved in a vegetative but not in a brain dead patient',\n",
    "                'magnetic resonance spectroscopy and diffusion tensor imaging in coma survivors: promises and pitfalls',\n",
    "                'centre d\\'etude du coma',\n",
    "                '(\\s+|\\n|^)coma@chu.ulg.ac.be',\n",
    "                'www.comascience.org',\n",
    "                '(consultations?[\\s\\<\"]+post-coma|post-coma[\\s\\<\"]+consultation)',\n",
    "                'echelle\\s+([\\-a-zA-Z0-9]+\\s+){1,9}coma',\n",
    "                '\\s+coma recovery scale[\\-\\s]+revised',\n",
    "                'differences in neuroanatomy of the vegetative state\\s*: insights from diffusion tensor imaging and functional implications',\n",
    "                'visual fixation in the vegetative state\\s*: an observational case series pet study',\n",
    "                'et al [^\\n]+',  # try to skip any bibliographic reference\n",
    "                'protocole? .{1,30} res?ponse .{1,20} (command|demand)',\n",
    "                'traduisant une reponse a la commande via la modulation de l\\'activite neuronale',\n",
    "                'aucune reponse a la commande',\n",
    "                '3\\s*-\\s*poursuite visuelle',\n",
    "                'diagnostic de sortie\\s*:',\n",
    "                'afin de tenter d\\'obtenir une reponse a la commande',\n",
    "                ]\n",
    "    # Find all reject and confirmation terms (text patterns that reject or accept the diagnosis)\n",
    "    # Note: spaces will be automatically replaced by '\\s+'\n",
    "    reject_patterns = ['incompatible', '(not|pas)\\s+(compatible|possible)', '(?<!cannot) reject', 'inconsistent', 'inconsistant', 'refut(e|ant)', 'exclu(t|e)?\\s', 'atypi(cal|que)', 'que de celui typiquement observe', 'is not', 'in contrast to', 'absence (d|of)', 'pas observe', 'aucun', 'n\\'a pas ', 'pas .{,20} evidence', 'failed']\n",
    "    confirm_patterns = ['(is|est)\\s+compatible', 'confirm', 'classiquement\\s+observe', 'observed', 'conclude', 'conclue', 'plus proche de', 'but is diagnosed as', 'oriente vers', 'presence (d|of)', 'preserv', 'avons pu observe', 'cannot reject']\n",
    "    #reject_pattern_regex = '|'.join(['(?<!'+r+')' for r in reject_pattern])\n",
    "    # Match with one of possible states\n",
    "    # Note: order matters here: we want to test the most specific first (eg, mcs+) to the least specific (mcs)\n",
    "    # Note2: must be all lowercase! Else no detection!\n",
    "    diag_states = OrderedDict((  # be careful with dashes, place them first or escape, eg \"[\\s-]\" is bad, prefer \"[-\\s]\" or \"[\\-\\s]\"\n",
    "            ('(partial\\s+locked[-\\s]+in|locked[-\\s]+in\\s+(partiel|.+?incomplet)|partiele\\s+locked[-\\s]+in|\\s+plis(\\s+|\\.))', 'partial LIS'),\n",
    "            ('(functional\\s+locked[-\\s]+in|locked[-\\s]+in\\s+fonctionnel|\\s+flis(\\s+|\\.))', 'functional LIS'),\n",
    "            ('(complete\\s+locked[-\\s]+in|locked[-\\s]+in\\s+complet|\\s+clis(\\s+|\\.))', 'CLIS'),\n",
    "            ('(locked[-\\s]+in|\\s+lis(\\s+|\\.))', 'LIS'),\n",
    "            ('(emergence|sortie.{2,9}etat\\s+de\\s+conscience\\s+minim|emerged|\\s+emcs?(\\s+|\\.))', 'EMCS'),\n",
    "            ('(consci(en|ou)\\w+\\s+)?(minimale?\\s+plus|minimal(ly)?\\s+conscious\\s+state\\s+plus|\\s+(mcs|ecm)(\\+|\\s+plus))', 'MCS+'),  # Note: mcs+ and mcs- pattern must start at the same place as mcs pattern, else mcs might get precedence (because the position of the pattern mcs in the text will precede the pattern mcs+/-)\n",
    "            ('(res?pon(ses?|d\\w+)\\s+(\\w+\\s+){1,7}(command|demand))', 'MCS+'),\n",
    "            ('(consci(en|ou)\\w+\\s+)?(minimale?\\s+moins|minimal(ly)?\\s+conscious\\s+state\\s+minus|\\s+(mcs|ecm)(\\-|\\s+(minus|moins)))', 'MCS-'),\n",
    "            ('(visual\\s+(\\w+\\s+){,4}pursuit|poursuite\\s+(\\w+\\s+){,4}visuelle)', 'MCS-'),\n",
    "            ('(conscience\\s+minimale?|minimal(ly)?\\s+conscious|\\s+mcs(\\s+|\\.)|ecm(\\s+|\\.))', 'MCS'),\n",
    "            ('(non[-\\s]+repondant|unresponsive\\s+wakefulness|\\s+uws(\\s+|\\.))', 'UWS'),\n",
    "            ('(not\\s+observe\\s+any\\s+sign\\s+of\\s+consciousness|aucun\\s+signe\\s+de\\s+conscience)', 'UWS'),\n",
    "            ('(vegetatif|vegetative|\\s+vs(\\s+|\\.)|aucun\\s+signe\\s+de\\s+conscience)', 'UWS'),\n",
    "            ('(\\s+enr\\s+)', 'UWS'),\n",
    "            ('(?<!nociception)\\s+coma(?!\\s+science\\s+group)', 'coma'),  # match coma only if not followed by science group (using negative lookahead)\n",
    "            ))\n",
    "\n",
    "    # Get the best clinical or paraclinical diagnosis (keep the highest consciousness level found, except if it's inside an exclude group/blacklist)\n",
    "    # This part is critical to do first, because it will allow to scan where we can find at least one diagnosis, and thus define the right \"conclusion part\" to scan.\n",
    "    # Backtrack to previous conclusions parts if the current one does not contain any diagnosis (eg, \"conclusions internistiques\")\n",
    "    patient_fields['best_clinical_paraclinical_diagnosis'] = 'NA'\n",
    "    for idx_s in idx_s_all[:idx_s_id+1][::-1]:  # we reverse the list to backtrack from the lowest conclusion to the first in document (cannot be below \"confraternellement\")\n",
    "        # Extract the conclusion part (we will retry with other potential conclusion parts - including the whole document - until we find at least one diagnosis - so if the best diagnosis is NA it means there is nothing to find in the whole document)\n",
    "        report_conclusion = report_text[idx_s:end_of_report]\n",
    "        #print(repr(report_conclusion))  # debugline\n",
    "        # Load the diagnostic filter object (and construct the list of exclusion/rejection/confirmation sentences positions)\n",
    "        diag_filter_conclusion = TextDiagnosticFilter(report_conclusion, exclude_patterns, reject_patterns, confirm_patterns)\n",
    "        # Match with one of possible states\n",
    "        patient_diag = 'NA'\n",
    "        for state_name, state_abbrv in diag_states.items():\n",
    "            match = re.search(state_name, report_conclusion)\n",
    "            # Check if there is a match for this diagnosis anywhere in the conclusion\n",
    "            if match and match.start() >= 0:\n",
    "                pos_diag = match.start()\n",
    "                # Check if the match is not in an excluded excerpt (references, CRS-R scale comments, etc.) nor after a rejection term ('absence of', 'is not', etc.)\n",
    "                if not diag_filter_conclusion.exclude_test(pos_diag) and not diag_filter_conclusion.reject_test(pos_diag, wholetext=True):\n",
    "                    #print(match.group(0))  # debugline\n",
    "                    #print(match.start())  # debugline\n",
    "                    patient_diag = state_abbrv\n",
    "                    break\n",
    "        # Found a diagnosis? Store it and cut the loop, the conclusion part is good enough!\n",
    "        if patient_diag != 'NA':\n",
    "            # Since we loop from the best diagnosis to the worst one (because we use an OrderedDict), the first diagnosis found is the best (most optimistic) one\n",
    "            patient_fields['best_clinical_paraclinical_diagnosis'] = patient_diag\n",
    "            break\n",
    "        # else we continue to other conclusion parts\n",
    "\n",
    "    # Get conflicting diagnoses (between clinical and a modality) - search only in the conclusion (not in previous sections)\n",
    "    patient_diag_clin_pet = []\n",
    "    patient_diag_clin_mri = []\n",
    "    try:\n",
    "        pos_clinical = re.search('clinical|clinique|behaviou?r|comportement|diagnostic\\s+evaluation|compte\\s+tenu\\s+de\\s+la\\s+presence', report_conclusion).start()  # TODO: use re.finditer and allow for a list of clinical pos, eg for Amakrane\n",
    "        # TODO: also scan full sections, eg: \"evaluations comportementales\"\n",
    "    except AttributeError as exc:\n",
    "        pos_clinical = -1\n",
    "        patient_diag_clin_pet.append('NA')\n",
    "        patient_diag_clin_mri.append('NA')\n",
    "\n",
    "    try:\n",
    "        pos_pet = re.search('(\\s+|/|-)(pet|tep|fdg)(\\s+|/|-)', report_conclusion).start()\n",
    "    except AttributeError as exc:\n",
    "        pos_pet = -1\n",
    "\n",
    "    try:\n",
    "        pos_mri = re.search('\\W+(magnetic|irmf?|f?mri|rmn)\\W+', report_conclusion).start()\n",
    "    except AttributeError as exc:\n",
    "        pos_mri = -1\n",
    "\n",
    "    # Construct list of all diagnoses found in the conclusion (with their text position)\n",
    "    patient_alldiags_conclusion = {}\n",
    "    for state_name, state_abbrv in diag_states.items():\n",
    "        #all_matchs = re.finditer('.*'+state_name, report_conclusion)\n",
    "        all_matchs = re.finditer(state_name, report_conclusion)\n",
    "        if all_matchs:\n",
    "            for match in all_matchs:\n",
    "                if match:\n",
    "                    try:\n",
    "                        start_idx = match.span(0)[0]\n",
    "                    except IndexError as exc:\n",
    "                        continue\n",
    "                    # Keep highest diagnosis for this text position (eg, MCS+ and reject MCS) and exclude references\n",
    "                    if not start_idx in patient_alldiags_conclusion and not diag_filter_conclusion.exclude_test(match.start()):\n",
    "                        patient_alldiags_conclusion[start_idx] = state_abbrv\n",
    "\n",
    "    # Assign diagnoses by modality (and reject if there are reject terms and no confirmation term in-between)\n",
    "    # Init\n",
    "    pos_modalities = {'clinical': pos_clinical,\n",
    "                      'pet': pos_pet,\n",
    "                      'mri': pos_mri\n",
    "                     }\n",
    "    patient_diags_per_modality = {}\n",
    "    for modality in pos_modalities.keys():\n",
    "        patient_diags_per_modality[modality] = set()\n",
    "    # Assign (and reject)\n",
    "    if len(patient_alldiags_conclusion) > 1:\n",
    "        pos_starts = pos_modalities.values()\n",
    "        for modality, start_pos in pos_modalities.items():\n",
    "            if start_pos >= 0:\n",
    "                # Define boundaries: from the modality marker to the next modality marker (or end of text)\n",
    "                # here we construct the end of conclusion text about this modality part\n",
    "                pos_starts_filt = list(pos_starts)\n",
    "                pos_starts_filt.remove(start_pos)\n",
    "                try:\n",
    "                    pos_end = min(filter(lambda x: x >= 0 and x > start_pos, pos_starts_filt))\n",
    "                # Last item, last position, greatest one, so we just set the total length of the report's conclusion\n",
    "                except ValueError:\n",
    "                    pos_end = len(report_conclusion)\n",
    "                # Filter reject/confirm terms that are outside of boundaries\n",
    "                diag_filter_conclusion.reject_filter(start_pos, pos_end)\n",
    "                # Loop through all diagnoses found in the conclusion, and assign to the closest modality (if not rejected)\n",
    "                for state_pos, state_abbrv in patient_alldiags_conclusion.items():\n",
    "                    # Assign to this modality if it is between the boundaries of this part (clinical part, pet part, etc.) and not rejected\n",
    "                    if start_pos <= state_pos <= pos_end and not diag_filter_conclusion.reject_test(state_pos):\n",
    "                        # TODO: Add only if not a subterm of something we already detected (eg, MCS should not be added if we know MCS+)\n",
    "                        patient_diags_per_modality[modality].add(state_abbrv)\n",
    "\n",
    "    patient_fields['diagnoses_clinical'] = '/'.join(patient_diags_per_modality['clinical'])\n",
    "    patient_fields['diagnoses_pet'] = '/'.join(patient_diags_per_modality['pet'])\n",
    "    patient_fields['diagnoses_fmri'] = '/'.join(patient_diags_per_modality['mri'])\n",
    "\n",
    "    # Get the final (clinical) diagnosis = best CRS-R result = the gold standard\n",
    "    # Note: we cannot scan the CRS-R table, we only scan the conclusion part\n",
    "    # Algorithm: use the best clinical diagnosis if any found, else extract the first diagnosis found in the conclusion part\n",
    "    patient_clin_diag = 'NA'\n",
    "    if patient_fields['diagnoses_clinical']:\n",
    "        # Get ordered set of possible diagnoses, first being the best (most optimistic)\n",
    "        diag_states_vals = diag_states.values()\n",
    "        # Order clinical diagnoses from best to worst\n",
    "        clinical_diag_sorted = sorted(patient_fields['diagnoses_clinical'].split('/'), key=lambda x: diag_states_vals.index(x))\n",
    "        # Extract the first = best diagnosis\n",
    "        patient_clin_diag = clinical_diag_sorted[0]\n",
    "    # Else, extract the first diagnosis found in the conclusion that is not rejected\n",
    "    elif patient_alldiags_conclusion:\n",
    "        for pos_diag in sorted(patient_alldiags_conclusion):\n",
    "            if not diag_filter_conclusion.reject_test(pos_diag, wholetext=True):\n",
    "                patient_clin_diag = patient_alldiags_conclusion[pos_diag]\n",
    "                break\n",
    "        #patient_clin_diag = patient_alldiags_conclusion[sorted(patient_alldiags_conclusion)[0]]\n",
    "    patient_fields['final_diagnosis'] = patient_clin_diag\n",
    "\n",
    "    # OLD Algorithm: extract the first diagnosis found in the conclusion part, and check if it is consistent with what was found for the clinical modality diagnosis (if not NAN)\n",
    "    #if patient_alldiags_conclusion:\n",
    "    #    patient_clin_diag = patient_alldiags_conclusion[sorted(patient_alldiags_conclusion)[0]]\n",
    "    #if patient_fields['diagnoses_clinical'] and patient_clin_diag not in patient_fields['diagnoses_clinical']:\n",
    "    #    patient_clin_diag = patient_fields['diagnoses_clinical'].split('/')[0]\n",
    "    #patient_fields['final_diagnosis'] = patient_clin_diag\n",
    "\n",
    "    # Diagnosis at admission (before our team diagnosed the patient)\n",
    "    patterns_admission_diag = [\n",
    "                                'diagnostic du corps medical a l\\'admission',\n",
    "                                'diagnostic (à|a) l\\'admission',\n",
    "                                'diagnostic actuel',\n",
    "                                'diagnosis (on|at) admission',\n",
    "                                'diagnose bij aankomst'\n",
    "                                'diagnos(is|ic|e).{,30}(admission|aankomst)',\n",
    "                              ]\n",
    "    admission_diag = 'NA'\n",
    "    # Load the diagnostic filter object for the whole text\n",
    "    diag_filter = TextDiagnosticFilter(report_text, exclude_patterns, reject_patterns, confirm_patterns)\n",
    "    # Find the \"admission diagnosis\" part\n",
    "    admission_diag_match = re.search('(' + '|'.join(patterns_admission_diag).replace(' ', '\\s+') + ')', report_text)\n",
    "    if admission_diag_match:\n",
    "        # Construct list of all diagnoses found in the whole text (with their text position)\n",
    "        patient_alldiags = {}\n",
    "        for state_name, state_abbrv in diag_states.items():\n",
    "            #all_matchs = re.finditer('.*'+state_name, report_conclusion)\n",
    "            all_matchs = re.finditer(state_name, report_text)\n",
    "            if all_matchs:\n",
    "                for match in all_matchs:\n",
    "                    if match:\n",
    "                        try:\n",
    "                            start_idx = match.span(1)[0]\n",
    "                        except IndexError as exc:\n",
    "                            continue\n",
    "                        # Keep highest diagnosis for this text position (eg, MCS+ and reject MCS) and exclude references\n",
    "                        if not start_idx in patient_alldiags and not diag_filter.exclude_test(match.start()):\n",
    "                            patient_alldiags[start_idx] = state_abbrv\n",
    "        # Get the first diagnosis found just after the admission diagnosis marker\n",
    "        admission_diag_match_start = admission_diag_match.start()\n",
    "        admission_diag_pos = filter(lambda x: x > admission_diag_match_start, sorted(patient_alldiags))\n",
    "        if admission_diag_pos:\n",
    "            admission_diag = patient_alldiags[admission_diag_pos[0]]\n",
    "    patient_fields['admission_diagnosis'] = admission_diag\n",
    "\n",
    "    # Atypical pattern?\n",
    "    if re.search('((pattern|metaboli).{1,50}atypique|atypical.{1,50}(pattern|metaboli)|surprenant|surprising|(pas|non|not)\\s+(typique|typical))', report_text):\n",
    "        patient_fields['atypical_pattern'] = 'True'\n",
    "    else:\n",
    "        patient_fields['atypical_pattern'] = 'False'\n",
    "\n",
    "    # Epileptic?\n",
    "    if re.search('(epilepsia|epileptic|epileptique|epilepsie|epilepsy)', report_conclusion):\n",
    "        patient_fields['epileptic'] = 'True'\n",
    "    else:\n",
    "        patient_fields['epileptic'] = 'False'\n",
    "\n",
    "    # Mental disorders?\n",
    "    mental_disorders = []\n",
    "    mental_disorders_match = re.finditer('(suicide|depressi(on|ve)|hallucination|schizo|bipolar|aphas[\\w\\-]+|alzheimer|parkinson|akineto[\\s\\-]*mutique|anosognos[\\w\\-]+)', report_text)\n",
    "    if mental_disorders_match:\n",
    "        for m in mental_disorders_match:\n",
    "            mental_disorders.append(m.group(0))\n",
    "    patient_fields['mental_disorders'] = '/'.join(set(mental_disorders))\n",
    "\n",
    "    # Zolpidem tested?\n",
    "    if re.search('zolpidem', report_text):\n",
    "        patient_fields['zolpidem_mention'] = 'True'\n",
    "        if report_text.count('zolpidem') > 2:\n",
    "            # If Zolpidem mentioned only once or twice, probably just anamnese or suggestion of treatment and bibliographic ref, but no test\n",
    "            patient_fields['zolpidem_tested'] = 'True'\n",
    "            # Zolpidem respondent? (proceed by elimination, to reduce likelihood of false negatives, since real positives are rare)\n",
    "            if re.search('zolpidem.{,400}?(aucune\\s+amelioration|aucun\\s+changement|aucune\\s+evolution|pas\\s+.{1,15}amelioration|disparition|disappear|no\\s+beneficial|not\\s+observe|any\\s+amelioration|no\\s+.{,30}(therapeutic|effect|change|improvement))', report_text, re.S):\n",
    "                if re.search('zolpidem.{,400}?montre\\s+capable', report_text, re.S):\n",
    "                    # Special case (no change in diagnostic because already EMCS, but still better performances after zolpidem)\n",
    "                    patient_fields['zolpidem_respondent'] = 'True'\n",
    "                else:    \n",
    "                    patient_fields['zolpidem_respondent'] = 'False'\n",
    "            else:\n",
    "                patient_fields['zolpidem_respondent'] = 'True'\n",
    "            # Zolpidem anti-respondent?\n",
    "            if re.search('zolpidem.{,400}?(disparition|disappear|diminution|decrease|dimin)', report_text, re.S):\n",
    "                patient_fields['zolpidem_antirespondent'] = 'True'\n",
    "            else:\n",
    "                patient_fields['zolpidem_antirespondent'] = 'False'\n",
    "        else:\n",
    "            patient_fields['zolpidem_tested'] = 'False'\n",
    "            patient_fields['zolpidem_respondent'] = 'False'\n",
    "        mzolp = re.search('zolpidem', report_text)\n",
    "        patient_fields['zolpidem_context'] = report_text[mzolp.start()-50:mzolp.end()+1200]\n",
    "    else:\n",
    "        patient_fields['zolpidem_mention'] = 'False'\n",
    "        patient_fields['zolpidem_tested'] = 'False'\n",
    "        patient_fields['zolpidem_context'] = ''\n",
    "        patient_fields['zolpidem_respondent'] = 'False'\n",
    "        patient_fields['zolpidem_antirespondent'] = 'False'\n",
    "\n",
    "    # Had a pet?\n",
    "    if re.search('(\\s+|/|-)(pet|tep|fdg)(\\s+|/|-)', report_text):\n",
    "        patient_fields['had_pet'] = 'True'\n",
    "    else:\n",
    "        patient_fields['had_pet'] = 'False'\n",
    "\n",
    "    # Had a mri?\n",
    "    if re.search('\\W+(magnetic|irmf?|f?mri|rmn)\\W+', report_text):\n",
    "        patient_fields['had_mri'] = 'True'\n",
    "    else:\n",
    "        patient_fields['had_mri'] = 'False'\n",
    "\n",
    "    # Get number of days since accident (ie, between accident and acquisition)\n",
    "    try:\n",
    "        # First way: try to get the days directly from the text content if specified\n",
    "        m_acc_delay = re.search(r'[^\\d]((\\d+)\\s*(ans?|years?))?(\\s*(et|and|,)?\\s*)((\\d+)\\s*(months?|mois))?(\\s*(et|and|,)?\\s*)((\\d+)\\s*(days?|jours?))?.{,10}?post', report_text, re.S)\n",
    "        acc_years = m_acc_delay.group(2)\n",
    "        acc_months = m_acc_delay.group(7)\n",
    "        acc_days = m_acc_delay.group(12)\n",
    "        # Calculate number of days from years, months and days\n",
    "        if acc_days is None:\n",
    "            acc_days = 0\n",
    "        if acc_months is None:\n",
    "            acc_months = 0\n",
    "        if acc_years is None:\n",
    "            acc_years = 0\n",
    "        diff_acq_acc = int(acc_years) * 365 + int(acc_months) * 30 + int(acc_days)\n",
    "        # If number is 0 then we misdetected probably\n",
    "        if diff_acq_acc == 0:\n",
    "            raise AttributeError()\n",
    "        else:\n",
    "            patient_fields['acquisition_minus_accident_days'] = '%i' % diff_acq_acc\n",
    "    except AttributeError as exc:\n",
    "        # Else, try to compute time between acquisition and accident dates (to detect acute\n",
    "        try:\n",
    "            if patient_fields['acquisition_date_end'] == 'NA' or patient_fields['accident_date'] == 'NA':\n",
    "                raise ValueError()\n",
    "            # Parse strings into date objects (always consider first integer to be the day if 3 integers date, and also allow fuzzy matching)\n",
    "            acq_date = dateutil_parser.parse(date_fr2en(patient_fields['acquisition_date_end']), dayfirst=True, fuzzy=True)\n",
    "            acc_date = dateutil_parser.parse(date_fr2en(patient_fields['accident_date']), dayfirst=True, fuzzy=True)\n",
    "            # Save the difference number of days\n",
    "            patient_fields['acquisition_minus_accident_days'] = '%i' % (acq_date - acc_date).days\n",
    "        except ValueError as exc:\n",
    "            patient_fields['acquisition_minus_accident_days'] = 'NA'\n",
    "\n",
    "    # Acute? (less than one month between acquisition and injury)\n",
    "    if patient_fields['acquisition_minus_accident_days'] != 'NA':\n",
    "        diff_acq_acc_days = int(patient_fields['acquisition_minus_accident_days'])\n",
    "        if 0 <= diff_acq_acc_days < 32:\n",
    "            patient_fields['acute'] = 'True'\n",
    "        elif diff_acq_acc_days >= 32:\n",
    "            patient_fields['acute'] = 'False'\n",
    "        else:  # misdetection bug, number of days is negative\n",
    "            patient_fields['acute'] = 'NA'\n",
    "    else:\n",
    "        patient_fields['acute'] = 'NA'\n",
    "\n",
    "    # If age is missing, try to calculate it from other fields (acquisition_date - birthdate)\n",
    "    try:\n",
    "        if patient_fields['age'] == 'NA' and patient_fields['acquisition_date_end'] != 'NA' and patient_fields['birthdate'] != 'NA':\n",
    "            # Parse strings into date objects\n",
    "            acq_date = dateutil_parser.parse(date_fr2en(patient_fields['acquisition_date_end']), dayfirst=True, fuzzy=True)\n",
    "            bir_date = dateutil_parser.parse(date_fr2en(patient_fields['birthdate']), dayfirst=True, fuzzy=True)\n",
    "            # Save the difference number of years\n",
    "            patient_fields['age'] = '%i' % calculate_age(bir_date, acq_date)\n",
    "    except ValueError as exc:\n",
    "        pass\n",
    "\n",
    "    # Ophtalmologic report?\n",
    "    if re.search('ophtalmo', report_text) or re.search('oogheelk', report_text):\n",
    "        patient_fields['ophtalmologic_report'] = 'True'\n",
    "    else:\n",
    "        patient_fields['ophtalmologic_report'] = 'False'\n",
    "\n",
    "    # Nociception report?\n",
    "    if re.search('nociception.{1,40}coma', report_text):\n",
    "        patient_fields['nociception_report'] = 'True'\n",
    "    else:\n",
    "        patient_fields['nociception_report'] = 'False'\n",
    "\n",
    "    # Debug\n",
    "    #print(pos_modalities)\n",
    "    #print('Confirms', diag_filter_conclusion.pos_confirms)\n",
    "    #print('Rejects', diag_filter_conclusion.pos_rejects)\n",
    "    #print(patient_alldiags_conclusion)\n",
    "    #print(patient_diags_per_modality)\n",
    "    #print(report_conclusion)\n",
    "    #for pos in patient_alldiags_conclusion.keys():\n",
    "        #print('----')\n",
    "        #print(report_conclusion[pos:pos+200])\n",
    "    #for pos in patient_alldiags.keys():\n",
    "        #print('----')\n",
    "        #print(report_text[pos:pos+200])\n",
    "\n",
    "    # Remove any line break in any field (easier to save as a csv)\n",
    "    for key in patient_fields.keys():\n",
    "        patient_fields[key] = patient_fields[key].replace('\\n', ' ')\n",
    "\n",
    "    # All done, return the extracted fields\n",
    "    if not return_text:\n",
    "        return patient_fields\n",
    "    else:\n",
    "        return patient_fields, report_text, report_conclusion\n",
    "\n",
    "def extract_report_fields_all(reports_root_dir, filetype=None, ocr=False, tolerant=False, verbose=False):\n",
    "    results = {}\n",
    "    conflicts = []\n",
    "    errors = []\n",
    "    total = 0\n",
    "    for report_dir, report_filename in recwalk(reports_root_dir, folders=False, filetype=filetype):\n",
    "        total += 1\n",
    "    for report_dir, report_filename in _tqdm(recwalk(reports_root_dir, folders=False, filetype=filetype), total=total, leave=True, unit='reports'):\n",
    "        report_path = os.path.join(report_dir, report_filename)\n",
    "        pts = None\n",
    "        if verbose:\n",
    "            print('* Processing file: %s' % report_path)\n",
    "        try:\n",
    "            pts = extract_report_fields(report_path, reports_root_dir, ocr=ocr)\n",
    "        except Exception as exc:\n",
    "            if 'Syntax Warning: May not be a PDF file' in str(exc) or 'File is not a zip file' in str(exc) or 'No text extractable' in str(exc) or 'Unsupported image type' in str(exc):\n",
    "                pts = None\n",
    "                if verbose:\n",
    "                    print(str(exc))\n",
    "                pass\n",
    "            else:\n",
    "                if tolerant:\n",
    "                    print(str(exc))\n",
    "                    pass\n",
    "                else:\n",
    "                    raise\n",
    "        if pts is None:\n",
    "            errors.append(report_path)\n",
    "            if verbose:\n",
    "                print('* Warning: error reading file %s, might not contain any text or unrecognized format, skipping file.' % report_path)\n",
    "        else:\n",
    "            pts_id = '%s_%s' % (pts['name'], pts['age']) # TODO: patient id should be name + scandate (from DICOM!)\n",
    "            if pts_id not in results.keys():\n",
    "                results[pts_id] = pts\n",
    "            else:\n",
    "                conflicts.append( (pts['report_path'], results[pts_id]['report_path']) )\n",
    "                if verbose:\n",
    "                    print('* Warning: conflict detected, two reports have the same patient name and date: %s and %s. Both will be saved anyway.' % (results[pts_id]['report_path'], pts['report_path']))\n",
    "                # Save under another id\n",
    "                pts_id = '%s_%s_%s' % (pts['name'], pts['age'], pts['report_date']) # first just try to append report date\n",
    "                if pts_id in results.keys(): # if it does not work, append the report's path (should be unique)\n",
    "                    pts_id = pts_id + pts['report_path']\n",
    "                results[pts_id] = pts\n",
    "    return (results, conflicts, errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Extract fields of only one report (good for debug)\n",
    "report_path = '../reports_all/some-patients-report.pdf'\n",
    "patients_fields, report_text, report_conclusion = extract_report_fields(report_path, return_text=True)\n",
    "patients_fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Debug by accessing the preprocessed report_text directly\n",
    "#report_path = '../reports_all/specific-file.pdf'\n",
    "#patient_fields, report_text, report_conclusion = extract_report_fields(report_path, ocr=True, return_text=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Main program\n",
    "verbose = False\n",
    "ocr = True # takes a lot more time with OCR\n",
    "tolerant = True # skip errors\n",
    "print('== Patients PDF reports fields extractor ==')\n",
    "print('Extracting patients fields from reports, please wait...')\n",
    "all_patients_fields, conflicts, errors = extract_report_fields_all(path_to_folder_of_reports, filetype=['.pdf', '.doc', '.docx'], ocr=ocr, tolerant=tolerant, verbose=verbose)\n",
    "print('Total reports processed: %i' % len(all_patients_fields))\n",
    "#print(all_patients_fields)\n",
    "# Fake unit test for missing diagnoses\n",
    "#for pts_key in all_patients_fields.keys():\n",
    "#    all_patients_fields[pts_key]['best_clinical_paraclinical_diagnosis'] = 'NA'\n",
    "\n",
    "# Display missing diagnoses\n",
    "missing_diagnoses = [patient_fields['name'] for patient_fields in all_patients_fields.values() if patient_fields['best_clinical_paraclinical_diagnosis'] == 'NA']\n",
    "if not missing_diagnoses:\n",
    "    print('No missing diagnosis, congratulations!')\n",
    "else:\n",
    "    print('Missing diagnoses: %i reports: %s' % (len(missing_diagnoses), ', '.join(sorted(missing_diagnoses))))\n",
    "\n",
    "# Display unreadable (error) reports\n",
    "if errors:\n",
    "    print('Total number of unreadable reports: %i. Here is the detailed list:' % len(errors))\n",
    "    for err in errors:\n",
    "        print('* %s' % err)\n",
    "# Display conflicts (ie, two reports with similar fields)\n",
    "if conflicts:\n",
    "    print('Total number of conflicting reports: %i. Here is the detailed list:' % len(conflicts))\n",
    "    for conf in conflicts:\n",
    "        print('* %s with %s' % (conf[1], conf[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save all extracted fields to a csv file!\n",
    "from csg_fileutil_libs.aux_funcs import save_dict_as_csv\n",
    "\n",
    "output_file = 'all_patients_fields.csv'\n",
    "fields_order = ['name', 'gender', 'age', 'final_diagnosis', 'mri_sedation']\n",
    "save_dict_as_csv(all_patients_fields, output_file, fields_order, csv_order_by='name', verbose=True)\n",
    "print('All results saved to csv file: %s' % output_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------------------------\n",
    "## Disambiguate names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "cf = pd.read_csv('all_patients_fields.csv', sep=';').fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Disambiguate names\n",
    "dist_threshold = 0.2\n",
    "cf = disambiguate_names(cf, dist_threshold=dist_threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save cleaned up disambiguated names back to the csv\n",
    "cf.sort_values(['name'], inplace=True)  # reorder by name\n",
    "cf.to_csv('all_patients_fields.csv', sep=';', na_rep='NA', index=False)\n",
    "print('Disambiguated csv correctly saved.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------\n",
    "## Reading the database and display interesting stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('List of potential hypometabolic MCS (UWS*):')\n",
    "count = 0\n",
    "for rowid, c in cf.iterrows():\n",
    "    # If clinical diagnosis = MCS+/- or EMCS, but paraclinical diagnoses are UWS\n",
    "    if ('MCS' in c['diagnoses_clinical'] or 'MCS' in c['final_diagnosis']) and ('UWS' in c['diagnoses_pet'] or 'UWS' in c['diagnoses_fmri']):\n",
    "        print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "        count += 1\n",
    "print('Total: %i' % count)\n",
    "print('\\n')\n",
    "\n",
    "print('Patients with an atypical pattern:')\n",
    "atypical = cf[cf['atypical_pattern'] == True]\n",
    "for rowid, c in atypical.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total atypical: %i' % len(atypical))\n",
    "print('\\n')\n",
    "\n",
    "print('Patients with epilepsia:')\n",
    "epileptic = cf[cf['epileptic'] == True]\n",
    "for rowid, c in epileptic.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total epileptic: %i' % len(epileptic))\n",
    "print('\\n')\n",
    "\n",
    "print('List of surely conflicting diagnoses between clinical and paraclinical:')\n",
    "countconflicts1 = 0\n",
    "for rowid, c in cf.iterrows():\n",
    "    if c['final_diagnosis'] != c['best_clinical_paraclinical_diagnosis']:\n",
    "        print('* ' + c['name'] + ': clinical: ' + c['final_diagnosis'] + ' ; paraclinical: ' + c['best_clinical_paraclinical_diagnosis'] + ' ; path: ' + c['report_path'])\n",
    "        countconflicts1 += 1\n",
    "print('Total: %i' % countconflicts1)\n",
    "print('\\n')\n",
    "\n",
    "print('List of potential conflicting diagnoses between clinical and paraclinical:')\n",
    "countconflicts2 = 0\n",
    "for rowid, c in cf.iterrows():\n",
    "    clin_diags = c['diagnoses_clinical'].strip().split('/')\n",
    "    # Print only patients with a clinical diagnosis\n",
    "    if clin_diags and clin_diags[0]:\n",
    "        # Construct list of paraclinical diagnoses\n",
    "        para_diags = []\n",
    "        para_diags.extend(c['diagnoses_pet'].split('/'))\n",
    "        para_diags.extend(c['diagnoses_fmri'].split('/'))\n",
    "        if para_diags and para_diags[0]:\n",
    "            # If there is any paraclinical diagnosis not in the clinical diagnoses, we print!\n",
    "            if any([para_diag for para_diag in para_diags if para_diag not in clin_diags]):\n",
    "                print('* ' + c['name'] + ': clinical: ' + c['diagnoses_clinical'] + ' ; pet: ' + c['diagnoses_pet'] + ' ; fmri: ' + c['diagnoses_fmri'] + ' ; path: ' + c['report_path'])\n",
    "                countconflicts2 += 1\n",
    "print('Total: %i' % countconflicts2)\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print('Patients with NAN sedation:')\n",
    "sedated = cf[cf['mri_sedation'] == '']\n",
    "for rowid, c in sedated.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total NAN sedation: %i' % len(sedated))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Patients without sedation:')\n",
    "notsedated = cf[cf['mri_sedation'] == 'no']\n",
    "for rowid, c in notsedated.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total without sedation: %i' % len(notsedated))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Patients with missing final diagnosis:')\n",
    "missing = cf[cf['final_diagnosis'] == '']\n",
    "missing_best = cf[cf['best_clinical_paraclinical_diagnosis'] == '']\n",
    "for rowid, c in missing.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total missing final diagnosis: %i (but for best diagnosis: only %i missing)' % (len(missing), len(missing_best)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('List of potential MCS* (clinical UWS but paraclinical MCS):')\n",
    "count = 0\n",
    "for rowid, c in cf.iterrows():\n",
    "    # If clinical diagnosis = MCS+/- or EMCS, but paraclinical diagnoses are UWS\n",
    "    if 'UWS' in c['diagnoses_clinical'] and ('MCS' in c['diagnoses_pet'] or 'UWS' in c['diagnoses_fmri']):\n",
    "        print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "        count += 1\n",
    "print('Total: %i' % count)\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('List of EMCS:')\n",
    "emcs = cf[cf['best_clinical_paraclinical_diagnosis'] == 'EMCS']\n",
    "print(len(emcs.ix[:,['name']]))\n",
    "emcs.ix[:,['name', 'report_path']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('List of clinically non-EMCS but paraclinically EMCS (these are bugs to correct!):')\n",
    "emcs_clin = cf[cf['final_diagnosis'] == 'EMCS']\n",
    "emcs_bugs = emcs[~emcs.isin(emcs_clin)].dropna(how='all')\n",
    "print(len(emcs_bugs.ix[:,['name']]))\n",
    "emcs_bugs.ix[:,['name', 'report_path']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Patients potentially Zolpidem-respondents:')\n",
    "zolpidem_resp = cf[cf['zolpidem_respondent'] == True]\n",
    "for rowid, c in zolpidem_resp.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total potential zolpidem-respondents: %i' % len(zolpidem_resp))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Patients potentially Zolpidem-antirespondents:')\n",
    "zolpidem_antiresp = cf[cf['zolpidem_antirespondent'] == True]\n",
    "for rowid, c in zolpidem_antiresp.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total potential zolpidem-antirespondents: %i' % len(zolpidem_antiresp))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print('Patients tested with Zolpidem but could not be detected as respondent nor anti-respondents (probably just non-responders):')\n",
    "zolpidem_test = cf[(cf['zolpidem_tested'] == True) & (cf['zolpidem_respondent'] == False) & (cf['zolpidem_antirespondent'] == False)]\n",
    "for rowid, c in zolpidem_test.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total patients tested with zolpidem but neither respondent nor antirespondent (probably just non-responders): %i' % len(zolpidem_test))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Patients with an ophtalmologic report:')\n",
    "ophtalmo = cf[cf['ophtalmologic_report'] == True]\n",
    "for rowid, c in ophtalmo.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total patients with an ophtalmologic report: %i' % len(ophtalmo))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Patients with a nociception coma scale report:')\n",
    "noci = cf[cf['nociception_report'] == True]\n",
    "for rowid, c in noci.iterrows():\n",
    "    print('* ' + c['name'] + ': ' + c['report_path'])\n",
    "print('Total patients with a nociception coma scale report: %i' % len(noci))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "------------------------------------\n",
    "## Code tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import Image\n",
    "except ImportError:\n",
    "    from PIL import Image\n",
    "import pytesseract\n",
    "print(pytesseract.image_to_string(Image.open(report_path), lang='fra'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_path = 'reports_bad/some-file.pdf'\n",
    "#report_path = 'reports_all/test.png'\n",
    "ocrparser = MyOCRParser()\n",
    "report_text = _unidecode(ocrparser.process(report_path, 'utf8').decode('utf8')).lower()\n",
    "print(report_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, report_text, report_conclusion = extract_report_fields(report_path, return_text=True, ocr=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(report_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "match_etiology = re.search(r'((accident|trauma|arrest|arret\\s+cardiaque|(months|years)\\s+|post[\\-\\s]+(?!hospital))([\\(\\)a-zA-Z0-9\\<\\>\\-]+\\s+){1,9}).+?(le|on)?(\\d+(/|-)\\s*\\d+(/|-)\\s*\\d+|\\d+\\s*\\w+\\s*\\d+)', report_text, re.S)\n",
    "match_etiology.group(1)\n",
    "match_etiology.group(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_acquisition_dates = re.search(r'(du|from)\\s+((\\d+[\\-/\\.\\s]+)?(\\d+[\\-/\\.\\s]+)?\\d+)\\s+(au|to)\\s+((\\d+[\\-/\\.\\s]+)?(\\d+[\\-/\\.\\s]+|\\w+\\s+)\\d+)', report_text)\n",
    "patient_acquisition_start = patient_acquisition_dates.group(2)\n",
    "patient_acquisition_end = patient_acquisition_dates.group(6)\n",
    "patient_fields['acquisition_date'] = '%s - %s' % (patient_acquisition_start, patient_acquisition_end)\n",
    "patient_fields['acquisition_date']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}