csg_datafusion_dicoms_anonymizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dicoms anonymizer\n",
    "# By Stephen Larroque @ Coma Science Group, GIGA Research, University of Liege\n",
    "# Creation date: 2017-02-07\n",
    "# License: MIT\n",
    "# v1.2.1\n",
    "#\n",
    "# INSTALL NOTE:\n",
    "# You need to pip install pandas before launching this script.\n",
    "# Tested on Python 2.7.11\n",
    "#\n",
    "# TODO:\n",
    "# * unify dicom names (if not already done)\n",
    "# * unify demographics names (if not already done)\n",
    "# * check if recursion ok (to anonymize MRI & PET at the same time for example).\n",
    "# * convert cells to functions\n",
    "# * put in a python script and use gooey (except if --cmd passed as argument)\n",
    "# * freeze using pyinstaller\n",
    "# * make a nice progress bar in gooey? add support in tqdm?\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forcefully autoreload all python modules\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AUX FUNCS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auxiliary libraries and necessary functions\n",
    "\n",
    "import os\n",
    "import re\n",
    "import shutil\n",
    "import sys\n",
    "\n",
    "cur_path = os.path.realpath('.')\n",
    "sys.path.append(os.path.join(cur_path, 'csg_fileutil_libs'))  # for pydicom, because it does not support relative paths (yet?)\n",
    "\n",
    "import csg_fileutil_libs.pydicom as dicom\n",
    "from csg_fileutil_libs.pydicom.filereader import InvalidDicomError\n",
    "from csg_fileutil_libs.distance import distance\n",
    "\n",
    "from csg_fileutil_libs.aux_funcs import recwalk, replace_buggy_accents, cleanup_name_df, disambiguate_names, compute_names_distance_matrix, _unidecode, _tqdm, cleanup_name, save_dict_as_csv\n",
    "\n",
    "def get_list_of_folders(rootpath):\n",
    "    return [item for item in os.listdir(rootpath) if os.path.isdir(os.path.join(rootpath, item))]\n",
    "\n",
    "def get_list_of_zip(rootpath):\n",
    "    return [item for item in os.listdir(rootpath) if os.path.isfile(os.path.join(rootpath, item)) and item.endswith('.zip')]\n",
    "\n",
    "def get_dcm_names_from_dir(rootpath, dcm_subj_list=None, folder_to_name=None, verbose=False):\n",
    "    if dcm_subj_list is None:\n",
    "        dcm_subj_list = []  # store list of subjects names from dicom files (useful for csv filtering)\n",
    "    if folder_to_name is None:\n",
    "        folder_to_name = {}  # store the name of the patient stored in each root folder (useful for anonymization later on)\n",
    "    for subject in get_list_of_folders(rootpath):\n",
    "        if verbose:\n",
    "            print('- Processing subject %s' % unicode(subject, 'latin1'))\n",
    "        fullpath = os.path.join(rootpath, subject)\n",
    "        if not isinstance(fullpath, unicode):\n",
    "            fullpath = unicode(fullpath, 'latin1')\n",
    "        pts_name = None\n",
    "        for dirpath, filename in recwalk(fullpath, filetype=['.dcm', '']):\n",
    "            try:\n",
    "                #print('* Try to read fields from dicom file: %s' % os.path.join(dirpath, filename))\n",
    "                dcmdata = dicom.read_file(os.path.join(dirpath, filename), stop_before_pixels=True)  # stop_before_pixels allow for faster processing since we do not read the full dicom data, and here we can use it because we do not modify the dicom, we only read it to extract the dicom patient name\n",
    "                #print(dcmdata.PatientName)\n",
    "                pts_name = cleanup_name(dcmdata.PatientName)\n",
    "                dcm_subj_list.append( pts_name )\n",
    "                break\n",
    "            except (InvalidDicomError, AttributeError) as exc:\n",
    "                pass\n",
    "        folder_to_name[subject] = pts_name\n",
    "    return dcm_subj_list, folder_to_name\n",
    "\n",
    "from tempfile import mkdtemp, mkstemp\n",
    "def get_dcm_names_from_zip(rootpath, dcm_subj_list=None, folder_to_name=None, verbose=False):\n",
    "    if dcm_subj_list is None:\n",
    "        dcm_subj_list = []  # store list of subjects names from dicom files (useful for csv filtering)\n",
    "    if folder_to_name is None:\n",
    "        folder_to_name = {}  # store the name of the patient stored in each root folder (useful for anonymization later on)\n",
    "    # Create a temporary file (to extract one dicom from zip)\n",
    "    dcmfilefh, dcmfilepath = mkstemp(suffix='.dcm')  # tesseract < 3.03 do not support \"stdout\" argument, so need to save into a file\n",
    "    os.close(dcmfilefh)  # close file to allow writing after\n",
    "    #dcmfilepath = 'tempdicomextract/tempdicom.dcm'\n",
    "    #try:\n",
    "    #    os.makedirs(os.path.dirname(dcmfilepath))\n",
    "    #except OSError as exc:\n",
    "    #    pass\n",
    "\n",
    "    # Extract names from zipped dicom files (extract the first dicom file we can read and use its fields)\n",
    "    for zipfilename in get_list_of_zip(rootpath):\n",
    "        zfilepath = os.path.join(rootpath, zipfilename)\n",
    "        if verbose:\n",
    "            print('- Processing file %s' % zipfilename)\n",
    "        with zipfile.ZipFile(zfilepath, 'r') as zipfh:\n",
    "            # Extract only files, not directories (end with '/', this is standard detection in zipfile)\n",
    "            zfolder = (item for item in zipfh.namelist() if item.endswith('/'))\n",
    "            zfiles = (item for item in zipfh.namelist() if not item.endswith('/'))\n",
    "            # Get first top folder inside zip to extract folder name (because when we will extract the zip, we need the folder name)\n",
    "            try:\n",
    "                folder_name = zfolder.next().strip('/')\n",
    "            except StopIteration:\n",
    "                folder_name = re.search('^([^\\\\/]+)[\\\\/]', zipfh.namelist()[0]).group(1)\n",
    "            # Get first dicom file we can find\n",
    "            pts_name = None\n",
    "            for zf in zfiles:\n",
    "                # Need to extract because pydicom does not support not having seek() (and zipfile in-memory does not provide seek())\n",
    "                z = zipfh.read(zf) # do not use .extract(), the path can be anything and it does not support unicode (so it can easily extract to the root instead of target folder!)\n",
    "                with open(dcmfilepath, 'wb') as dcmf:\n",
    "                    dcmf.write(z)\n",
    "                # Try to open the extracted dicom\n",
    "                try:\n",
    "                    if verbose:\n",
    "                        print('Try to decode dicom fields with file %s' % zf)\n",
    "                    dcmdata = dicom.read_file(dcmfilepath, stop_before_pixels=True)\n",
    "                    pts_name = cleanup_name(dcmdata.PatientName)\n",
    "                    dcm_subj_list.append( pts_name )\n",
    "                    os.remove(dcmfilepath)\n",
    "                    break\n",
    "                except (InvalidDicomError, AttributeError) as exc:\n",
    "                    continue\n",
    "                except IOError as exc:\n",
    "                    if 'no tag to read' in str(exc).lower():\n",
    "                        continue\n",
    "                    else:\n",
    "                        raise\n",
    "            # Add to the folder name -> dicom patient name mapping\n",
    "            folder_to_name[folder_name] = pts_name\n",
    "    return dcm_subj_list, folder_to_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "import os\n",
    "import cStringIO\n",
    "\n",
    "def zipwalk(zfilename):\n",
    "    \"\"\"Zip file tree generator.\n",
    "\n",
    "    For each file entry in a zip archive, this yields\n",
    "    a two tuple of the zip information and the data\n",
    "    of the file as a StringIO object.\n",
    "\n",
    "    zipinfo, filedata\n",
    "\n",
    "    zipinfo is an instance of zipfile.ZipInfo class\n",
    "    which gives information of the file contained\n",
    "    in the zip archive. filedata is a StringIO instance\n",
    "    representing the actual file data.\n",
    "\n",
    "    If the file again a zip file, the generator extracts\n",
    "    the contents of the zip file and walks them.\n",
    "\n",
    "    Inspired by os.walk .\n",
    "    Source: by Anand http://code.activestate.com/recipes/425840-zip-walker-zip-file-tree-generator/\n",
    "    \"\"\"\n",
    "\n",
    "    tempdir=os.environ.get('TEMP',os.environ.get('TMP',os.environ.get('TMPDIR','/tmp')))\n",
    "    \n",
    "    try:\n",
    "        z=zipfile.ZipFile(zfilename,'r')\n",
    "        for info in z.infolist():\n",
    "            fname = info.filename\n",
    "            data = z.read(fname)\n",
    "            extn = (os.path.splitext(fname)[1]).lower()\n",
    "\n",
    "            if extn=='.zip':\n",
    "                checkz=False\n",
    "                \n",
    "                tmpfpath = os.path.join(tempdir,os.path.basename(fname))\n",
    "                try:\n",
    "                    open(tmpfpath,'w+b').write(data)\n",
    "                except (IOError, OSError),e:\n",
    "                    print e\n",
    "\n",
    "                if zipfile.is_zipfile(tmpfpath):\n",
    "                    checkz=True\n",
    "\n",
    "                if checkz:\n",
    "                    try:\n",
    "                        for x in zipwalk(tmpfpath):\n",
    "                            yield x\n",
    "                    except Exception, e:\n",
    "                        raise\n",
    "                    \n",
    "                try:\n",
    "                    os.remove(tmpfpath)\n",
    "                except:\n",
    "                    pass\n",
    "            else:\n",
    "                yield (info, cStringIO.StringIO(data))\n",
    "    except RuntimeError, e:\n",
    "        print 'Runtime Error'\n",
    "    except zipfile.error, e:\n",
    "        raise\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----------------------------------------\n",
    "# Part 1\n",
    "## Extract dicom names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Params\n",
    "rootpath = 'dicoms'  # path to dicom folders and zipfiles\n",
    "demo_csv = 'db_reports_plus_fmp.csv'  # path to the demographics csv file\n",
    "verbose = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Get the list of dicoms (they must all be at the first level, one folder per subject)\n",
    "\n",
    "# Get unzipped dicom folders list\n",
    "print('Constructing list of DICOM subjects through folders names, please wait...')\n",
    "subjects_list = get_list_of_folders(rootpath)\n",
    "\n",
    "# Extract name from first readable dicom\n",
    "if verbose:\n",
    "    print('Found subjects dicom folders: %s' % ', '.join(subjects_list))\n",
    "\n",
    "dcm_subj_list, folder_to_name = get_dcm_names_from_dir(rootpath, verbose=verbose)\n",
    "print('Total dicom subjects: %i. Detailed list: %s' % (len(dcm_subj_list), ', '.join(dcm_subj_list)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# -- Extracting subjects names from zip files\n",
    "# NOTE: anonymization is NOT supported on zip files, only on unzipped dicom folders!\n",
    "\n",
    "# Extract list of zip files\n",
    "subjects_zip_list = get_list_of_zip(rootpath)\n",
    "print(subjects_zip_list)\n",
    "\n",
    "dcm_subj_list, folder_to_name = get_dcm_names_from_zip(rootpath, dcm_subj_list, folder_to_name, verbose=verbose)\n",
    "print('Total dicom subjects: %i. Detailed list: %s' % (len(dcm_subj_list), ', '.join(dcm_subj_list)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_to_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save all extracted fields to a csv file!\n",
    "from csg_fileutil_libs.aux_funcs import save_dict_as_csv\n",
    "\n",
    "output_file = 'dicom_names.csv'\n",
    "save_dict_as_csv([{'name': name, 'path': path} for name, path in zip(dcm_subj_list, subjects_list + subjects_zip_list)], output_file, csv_order_by='name', verbose=True)\n",
    "print('Dicom patients names saved to csv file: %s' % output_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------------------------------\n",
    "# Part 2: Generate anonymization mapping\n",
    "## Anonymization initialization (generate anonymized ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Params\n",
    "anon_prefix = 'subj_'  # prefix of the generated anonymized ids\n",
    "\n",
    "anon_salt = 'some random string'  # set this to a string of your choice to generate unique hashes, this adds protection against decrypting the ids (but keep the same if you want to be able to update anonymized data)\n",
    "anon_permanent_ids = True  # if you want the anonymized id to always be the same (useful if you want to add new subjects and keep the same ids for old ones, eg, if you have multiple datasets with overlapping subjects but with different ones as well), but at the expense of security (the anonymized id can potentially be decrypted). If set to False, you will get near impossible decryption and a nice simple id scheme (from 1 to the number of subjects)\n",
    "anon_hash_algo = 'md5'\n",
    "anon_length = 8 # length of the id. Set to None to disable. Shortening might be an added security if anon_permanent_ids == True, but raises the risks of collisions (two names having same id). You can try, there will be an exception anyway if there is a collision.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "dcm_names = pd.read_csv('dicom_names.csv', sep=';').fillna('')\n",
    "dcm_subj_list = dcm_names.to_dict()['name'].values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate disambiguated list of dicom names\n",
    "# Disambiguate dicom names\n",
    "cd_unique = disambiguate_names(dcm_names, verbose=True)\n",
    "# Extract list of unique dicom names\n",
    "dcm_unique = set(cd_unique['name'].to_dict().values())\n",
    "\n",
    "print(dcm_unique)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unique id extractor from name, insensitive to accentuated charaters nor non-alphabetical characters nor firstname/lastname position switching\n",
    "import hashlib\n",
    "import re\n",
    "\n",
    "from csg_fileutil_libs.aux_funcs import sort_list_a_given_list_b, replace_buggy_accents, _unidecode\n",
    "\n",
    "def clean_name(name):\n",
    "    '''Clean name from accents and non-alphabetical characters'''\n",
    "    return re.sub(r'\\W', r'', _unidecode(replace_buggy_accents(name.decode('utf8'), 'utf8')).lower())\n",
    "def extract_ordered_letters(name):\n",
    "    '''Order letters composing a name to alphabetical order'''\n",
    "    alphabet = list('abcdefghijklmnopqrstuvwxyz1234567890-')\n",
    "    return ''.join(sort_list_a_given_list_b(list(clean_name(name)), alphabet))\n",
    "def get_hash(string, algo=None):\n",
    "    if algo is None or algo == 'md5':\n",
    "        return hashlib.md5(string).hexdigest()\n",
    "    elif algo == 'sha1':\n",
    "        return hashlib.md5(string).hexdigest()\n",
    "    else:\n",
    "        raise NameError('Hash algorithm not recognized: %s' % algo)\n",
    "def get_ordered_hash(hash_func, string, salt=None, algo=None):\n",
    "    '''Get a unique hash insensitive to accents, non-alphabetical characters nor words position switching'''\n",
    "    return hash_func(extract_ordered_letters(string+(salt if salt else '')), algo=algo)\n",
    "def get_duplicates(d):\n",
    "    seen = set()\n",
    "    for k, v in d.items():\n",
    "        if v in seen:\n",
    "            yield k, v\n",
    "        else:\n",
    "            seen.add(v)\n",
    "\n",
    "# Unit test\n",
    "name1 = 'rajaé chatila'\n",
    "name2 = 'chatila  rajaé|'\n",
    "assert(get_ordered_hash(get_hash, name1) == get_ordered_hash(get_hash, name2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "##### Generate anonymization scheme from dicoms patients names #####\n",
    "\n",
    "# Generate unique hashes from each dicom's patient name\n",
    "# Generate an anonymized id resilient to spaces and non letters characters and words switching\n",
    "# to do that, we take the name, and reorder all letters (and remove any non-letter symbol) by alphabetical order, which gives us simply the ordered sequence of letters composing each name\n",
    "anon_hashes = {name: get_ordered_hash(get_hash, name, anon_salt, algo=anon_hash_algo) for name in dcm_unique}\n",
    "# Shortening is an added security, so that if someone tries to bruteforce, there will be missing info to reconstitute the original name that gave this hash (because we are missing parts of the hash, so lots of dissimilar names will have the same shortened hash)\n",
    "if anon_length:\n",
    "    for name, h in anon_hashes.items():\n",
    "        anon_hashes[name] = anon_hashes[name][:anon_length]\n",
    "# There can be collisions in hashes, then check that there is none\n",
    "anon_dups = dict(get_duplicates(anon_hashes))\n",
    "if anon_dups:\n",
    "    anon_dups_print = {h: [name for name, h2 in anon_hashes.items() if h2 == h] for h in anon_dups.values()}\n",
    "    raise ValueError('Two names have the same id! Please use another hashing algorithm or raise hash length or another salt or turn off anon_permanent_ids. Here is the list of names with same ids: %s' % anon_dups_print)\n",
    "del anon_dups\n",
    "\n",
    "# Generate the final id (second step)\n",
    "if anon_permanent_ids:\n",
    "    # generate a straightforward id from a shortened hash\n",
    "    names_and_ids = anon_hashes\n",
    "else:\n",
    "    # generate a unique id based on order (simply the order number when ordered by the hash - since we use the \"ordered hash\", we get the same properties: the same set of patients names will always generate the same order, and the order cannot be traced back, since it depends both on the hash AND the exact set of patients names to get the exact same order)\n",
    "    # the big advantage of this approach is that it is nearly impossible to decrypt the original name, since the id gives strictly no information at all\n",
    "    # the disadvantage is that it is dependent on the subjects names list, so if you add a subject, nearly all ids will change\n",
    "    names_and_ids = {name: str(id+1).zfill(anon_length) for id, name in enumerate(sorted(anon_hashes, key=anon_hashes.get))}\n",
    "# Prepend prefix and save the anonymized ids\n",
    "anon_ids = {(\"%s%s\" % (anon_prefix, id)): name for name, id in names_and_ids.items()}\n",
    "anon_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write anonymization csv\n",
    "ids_to_name = [{'id': pts_id, 'name': anon_ids[pts_id]} for pts_id in sorted(anon_ids)]\n",
    "save_dict_as_csv(ids_to_name, 'idtoname.csv', fields_order=['id', 'name'], csv_order_by='id', verbose=False)\n",
    "print('Conversion list (id -> name) saved to idtoname.csv.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------------------------------------\n",
    "# Part 3: applying anonymization\n",
    "TODO: remake to adapt the dcm_names to the ones in the specified rootpath, not the ones in dicom_names.csv!\n",
    "## Merging dicom names and demographics csv names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Params\n",
    "rootpath = 'dicoms'  # path to dicom folders and zipfiles, this can be another folder than the one you used to generate the anonymization mapping\n",
    "# TODO: auto add new dicom names to anonymization mapping (we have the list of names, we can generate a new mapping!)\n",
    "demo_csv = 'db_reports_plus_fmp.csv'  # path to the demographics csv file\n",
    "verbose = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "cf = pd.read_csv(demo_csv, sep=';').fillna('')\n",
    "dcm_names = pd.read_csv('dicom_names.csv', sep=';').fillna('')\n",
    "dcm_subj_list = dcm_names.to_dict()['name'].values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Disambiguate dicom names\n",
    "cd_unique = disambiguate_names(dcm_names, verbose=True)\n",
    "# Extract list of unique dicom names\n",
    "dcm_unique = set(cd_unique['name'].to_dict().values())\n",
    "# Generate dicom name to unique name mapping\n",
    "dcmname_to_uniquename = {(c['alt_names'] if c['alt_names'] else c['name']): c['name'] for c in cd_unique.iterrows()}\n",
    "\n",
    "print(dcm_unique)\n",
    "dcmname_to_uniquename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Disambiguate and clean up csv names\n",
    "dist_threshold = 0.2\n",
    "\n",
    "# Cleanup names\n",
    "cf = cleanup_name_df(cf)\n",
    "\n",
    "# Disambiguate (ie, same name with typos or inversed firstname/lastname)\n",
    "cf = disambiguate_names(cf, dist_threshold=dist_threshold)\n",
    "\n",
    "# Save cleaned up disambiguated names back to the csv\n",
    "#cf.to_csv(demo_csv, sep=';', na_rep='NA', index=False)\n",
    "#print('Disambiguated csv correctly saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Computing distance matrix (ie, finding similar names between dicoms and demographics csv)\n",
    "from csg_fileutil_libs.aux_funcs import distance_jaccard_words_split\n",
    "dist_threshold = 0.2 # normalized distance threshold to match similar names. 0.0 is no difference, 1.0 is everything different.\n",
    "dist_matches = {}\n",
    "name_to_anon_ids = {v: k for k, v in anon_ids.items()}\n",
    "for subj in dcm_unique:\n",
    "    found = False\n",
    "    for rowid, c in cf.iterrows():\n",
    "        if distance.nlevenshtein(subj, c['name'], method=1) <= dist_threshold or distance_jaccard_words_split(subj, c['name'], partial=True, norm=True, dist=dist_threshold) <= dist_threshold: # use shortest distance with normalized levenshtein\n",
    "            if subj not in dist_matches:\n",
    "                dist_matches[subj] = []\n",
    "            dist_matches[subj].append(c['name'])\n",
    "            found = True\n",
    "    if not found:\n",
    "        dist_matches[subj] = None\n",
    "\n",
    "# Remove duplicate values (ie, csv names)\n",
    "dist_matches = {k: (list(set(v)) if v else v) for k, v in dist_matches.items()}\n",
    "# Find missing subjects (ie, dicom name present but missing in csv database)\n",
    "missing_subj = {k: v for k, v in dist_matches.items() if not v}\n",
    "# Print results\n",
    "if not missing_subj:\n",
    "    print('No missing subject, congratulations!')\n",
    "else:\n",
    "    print('Missing subjects from csv database (saved in missing_demo.csv): %i, names: %s' % (len(missing_subj), ', '.join(sorted(missing_subj.keys()))))\n",
    "    save_dict_as_csv([{'name': msubj, 'id': name_to_anon_ids[msubj]} for msubj in missing_subj.keys()], 'missing_demo.csv', fields_order=['name', 'id'], csv_order_by='name', verbose=False)\n",
    "    save_dict_as_csv([{'id': name_to_anon_ids[msubj]} for msubj in missing_subj.keys()], 'missing_demo_anonymized.csv', fields_order=['id'], csv_order_by='id', verbose=False)\n",
    "\n",
    "print('\\nList of all matches (dicom : csv):')\n",
    "print(dist_matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Compute the csv name to dicom unique name mapping\n",
    "csvname_to_uniquename = {}\n",
    "for uniquename in dist_matches.keys():\n",
    "    if dist_matches[uniquename]:\n",
    "        for csv_name in dist_matches[uniquename]:\n",
    "            csvname_to_uniquename[csv_name] = uniquename\n",
    "csvname_to_uniquename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten_gen(L):\n",
    "    for item in L:\n",
    "        if isinstance(item, list):\n",
    "            for i in flatten(item):\n",
    "                yield i\n",
    "        else:\n",
    "            yield item\n",
    "\n",
    "def flatten(L):\n",
    "    return list(flatten_gen(L))\n",
    "\n",
    "def get_unique_names(L):\n",
    "    return filter(None, set(L))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "demo_short_csv = 'demographics_shortened.csv'\n",
    "# Get unique demo csv names (which matched with dicom in the distance matrix)\n",
    "demo_names = get_unique_names(flatten(dist_matches.values()))\n",
    "# Shorten demographics to only names present in dicoms\n",
    "cf_short = cf[cf['name'].isin(demo_names)]\n",
    "# Save shortened demographics\n",
    "cf_short.to_csv(demo_short_csv, sep=';', na_rep='NA', index=False)\n",
    "print('Shortened demographics (to only the dicoms available) were saved to %s.' % demo_short_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cf_short"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------------------------\n",
    "## Anonymizing demographics csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "with open('idtoname.csv', mode='r') as f:\n",
    "    reader = csv.DictReader(f, delimiter=';')\n",
    "    anon_ids = {row['id']: row['name'] for row in reader}\n",
    "anon_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_to_anon_ids = {v: k for k, v in anon_ids.items()}\n",
    "name_to_anon_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Anonymize demographics csv\n",
    "\n",
    "demo_anon_csv = 'demographics_anonymized.csv'\n",
    "\n",
    "# Make a new dataframe from shortened demographics csv\n",
    "cf_anon = cf_short.copy()\n",
    "# Anonymize names by using the csv name -> unique name -> anonymized id mapping\n",
    "# TODO: might be better to use pandas join?\n",
    "for rowid, c in cf_anon.iterrows():\n",
    "    cf_anon.ix[rowid, 'name'] = name_to_anon_ids[csvname_to_uniquename[c['name']]]\n",
    "# Drop column report_path and alt_names\n",
    "if 'report_path' in cf_anon.columns:\n",
    "    cf_anon.drop('report_path', axis=1, inplace=True)\n",
    "if 'alt_names' in cf_anon.columns:\n",
    "    cf_anon.drop('alt_names', axis=1, inplace=True)\n",
    "# Save anonymized demographics\n",
    "cf_anon.to_csv(demo_anon_csv, sep=';', na_rep='NA', index=False)\n",
    "print('Anonymized demographics successfully saved to %s.' % demo_anon_csv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------------\n",
    "## Anonymizing dicoms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unzip all into folders, because we cannot anonymize zip files (ie, can't modify files inside a zip)\n",
    "# NOTE: ZIP FILES WILL BE DELETED!\n",
    "import zipfile\n",
    "\n",
    "def unzip(zipfilepath, outputpath):\n",
    "    zip_ref = zipfile.ZipFile(zipfilepath, 'r')\n",
    "    zip_ref.extractall(outputpath)\n",
    "    zip_ref.close()\n",
    "\n",
    "# First we need to delete all .DS_Store files (else we get permission denied IOError)\n",
    "count_dstore = 0\n",
    "for dirpath, filename in _tqdm(recwalk(rootpath, topdown=False, folders=True), unit='files', desc='DSTOREDEL'):\n",
    "    if filename.lower() == '.ds_store':\n",
    "        fullfilepath = os.path.join(dirpath, filename)\n",
    "        os.remove(fullfilepath)\n",
    "        count_dstore += 1\n",
    "print('Total .DS_Store files deleted: %i.' % count_dstore)\n",
    "\n",
    "# Unzip files and delete zip\n",
    "subjects_list_zip = get_list_of_zip(rootpath)\n",
    "count_zip = 0\n",
    "for z in _tqdm(subjects_list_zip, unit='files', desc='UNZIP'):\n",
    "    zipfilepath = os.path.join(rootpath, z)\n",
    "    if verbose:\n",
    "        print('- Unzipping file: %s' % z)\n",
    "    # Unzip file into the same root directory as other dicom folders\n",
    "    unzip(zipfilepath, rootpath)\n",
    "    # Delete the zip (since we cannot anonymize it)\n",
    "    os.remove(zipfilepath)\n",
    "    count_zip += 1\n",
    "print('Total zipfiles unzipped: %i.' % count_zip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# -- Anonymization of dicom files\n",
    "# Note: this will anonymize only the dicoms fields (name of patient, whatever field it is found in).\n",
    "# If there are any other file containing the patient's name (such as .txt, .csv, .xls, etc), the files might be deleted if you want (add the extension in the list) or they will stay.\n",
    "#from dicom.filebase import DicomFileLike  # fix for IOError access denied, see https://github.com/darcymason/pydicom/issues/69\n",
    "import re\n",
    "\n",
    "def find_hidden_name_fields(dcmdata, dcm_pts_names, hidden_name_fields=None):\n",
    "    '''From a pydicom object, return all fields where one of the dcm_pts_name (a list) is present.\n",
    "    This ease the detection of additional fields where patient name was stored.'''\n",
    "    if hidden_name_fields is None:\n",
    "        hidden_name_fields = set()\n",
    "    # Convert name to regex friendly (because dicoms often replace spaces by ^)\n",
    "    dcm_pts_names = [pts_name.replace(' ', '[\\W]+') for pts_name in dcm_pts_names]\n",
    "    # Walk through each dicom field\n",
    "    for dcmfield in dcmdata.keys(): # different from dir()?\n",
    "        if dcmdata[dcmfield]:\n",
    "            try:\n",
    "                #dcmfieldval = dcmdata.data_element(dcmfield).value\n",
    "                dcmfieldval = dcmdata[dcmfield].value\n",
    "                check = False\n",
    "                if isinstance(dcmfieldval, list):\n",
    "                    dcmfieldval_lower = [s.lower() if isinstance(s, str) else s for s in dcmfieldval]\n",
    "                    check = any(pts_name in dcmfieldval_lower for pts_name in dcm_pts_names)\n",
    "                elif isinstance(dcmfieldval, (int, float)):\n",
    "                    check = False\n",
    "                else:\n",
    "                    check = any(re.search(pts_name, dcmfieldval.lower()) for pts_name in dcm_pts_names)\n",
    "                if check:\n",
    "                    hidden_name_fields.add(dcmfield)\n",
    "            except AttributeError:\n",
    "                print('Error with field: %s' % str(dcmfield))\n",
    "                raise\n",
    "    return hidden_name_fields\n",
    "\n",
    "\n",
    "reports_delete = True  # delete pdf/doc/docx/txt files automatically?\n",
    "skip_already_processed = True\n",
    "remove_private_tags = False\n",
    "fields_to_del = ['PatientAddress', 'PatientBirthTime', 'PatientTelephoneNumbers', 'OtherPatientNames']\n",
    "print('-- Anonymization started, please wait, this might take a while (also make sure you unzipped all dicoms into folders)...')\n",
    "print('Note: if you get an IOError permission denied error, make sure you close any file explorer or application using any of the subjects folder (including Windows Explorer, else folders cannot be renamed).')\n",
    "print('Note2: JPEG2000 compressed dicom files are unsupported, please uncompress them beforehand (eg, using dcmdjpeg).')\n",
    "print('Note3: in case of an Access Error, you can continue the anonymization, it will restart from the start but it will skip already processed dicom files.')\n",
    "count_anon = 0\n",
    "count_files = 0\n",
    "count_delete = 0\n",
    "count_files_skipped = 0\n",
    "# Init path and 1st level folders list\n",
    "uni_rootpath = unicode(rootpath, 'latin1')  # convert rootpath to unicode before walking with os.listdir and recwalk, so we get back unicode strings too (else we won't be able to enter folders with accentuated characters)\n",
    "subjects_list = get_list_of_folders(uni_rootpath)\n",
    "# Precompute total number of files (for progressbar)\n",
    "print('Precomputing total number of files, please wait...')\n",
    "for subject in _tqdm(subjects_list, unit='folders', desc='PRECOMP'):\n",
    "    fullpath = os.path.join(uni_rootpath, subject)\n",
    "    for dirpath, filename in recwalk(fullpath, topdown=False, folders=True):\n",
    "        count_files += 1\n",
    "# Get folder_to_name mapping\n",
    "_, folder_to_name = get_dcm_names_from_dir(uni_rootpath)\n",
    "_, folder_to_name = get_dcm_names_from_zip(uni_rootpath, folder_to_name=folder_to_name)\n",
    "# Loop through each subject root directory to rewrite dicoms\n",
    "print('Launching anonymization of dicoms fields, please wait...')\n",
    "tbar = _tqdm(total=count_files, unit='files', desc='ANON')\n",
    "hfields = set()\n",
    "for folder in subjects_list:\n",
    "    # Already processed folder and there are several sessions, extract the id from folder name\n",
    "    subject = folder\n",
    "    try:\n",
    "        subject = re.match('(^%s.+)_s\\d+$' % anon_prefix, subject).group(1)\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    # Already processed folder, then retrieve back the patient's name from the anonymized id\n",
    "    if subject in anon_ids:\n",
    "        pts_name = anon_ids[subject]\n",
    "        if skip_already_processed:\n",
    "            fullpath = os.path.join(uni_rootpath, folder)\n",
    "            c = 0\n",
    "            for _, _ in recwalk(fullpath, topdown=False, folders=True):\n",
    "                c += 1\n",
    "            tbar.update(c)\n",
    "            count_files_skipped += c\n",
    "            continue\n",
    "    # Partially processed folder, get the original name and continue\n",
    "    elif folder_to_name[subject] in anon_ids:\n",
    "        pts_name = anon_ids[folder_to_name[subject]]\n",
    "    else:\n",
    "        pts_name = folder_to_name[subject]\n",
    "    # Special case: no dicom with a patient name can be found inside the folder (might be nifti files instead?), so we just skip\n",
    "    if pts_name is None:\n",
    "        continue\n",
    "    # Fetch the anonymized id from folder name (because we already looked inside to get the first dicom's patientname)\n",
    "    anon_id = name_to_anon_ids[dcmname_to_uniquename[pts_name]]\n",
    "    if verbose:\n",
    "        print('- Processing subject %s -> %s in folder %s' % (pts_name, anon_id, folder))\n",
    "    #fullpath = unicode(os.path.join(rootpath, folder), 'latin1')\n",
    "    fullpath = os.path.join(uni_rootpath, folder)  # no need to use unicode(str, 'latin1') here because rootpath and folder were both converted to unicode before\n",
    "    # Loop through each subfiles and subfolders for this subject (we assume all dicoms are for one subject, so we rename them all to this subject)\n",
    "    for dirpath, filename in recwalk(fullpath, topdown=False, folders=True):\n",
    "        fullfilepath = os.path.join(dirpath, filename)\n",
    "        # Report file: delete if option enabled\n",
    "        if reports_delete and filename.endswith( ('pdf', 'doc', 'docx', 'txt', 'csv', 'xls', 'xlsx') ):\n",
    "            os.remove(fullfilepath)\n",
    "            count_delete += 1\n",
    "            continue\n",
    "        elif os.path.isdir(fullfilepath):  # else we get an IOError...\n",
    "            continue\n",
    "        else:\n",
    "            # Dicom file: change PatientName field\n",
    "            try:\n",
    "                #TODO: autodetect if name is in filename and change!\n",
    "                #print('* Try to read fields from dicom file: %s' % os.path.join(dirpath, filename))\n",
    "                # Read dicom's file data\n",
    "                #os_id = os.open(str(fullfilepath), os.O_BINARY | os.O_RDONLY)\n",
    "                #fd = os.fdopen(os_id)\n",
    "                #dcmdata = dicom.read_file(DicomFileLike(fd), stop_before_pixels=False)\n",
    "                dcmdata = dicom.read_file(fullfilepath, stop_before_pixels=False)  # need to read the full dicom here since we will modify it, so stop_before_pixels must be False\n",
    "                # Store current name (to check at the end if we correctly cleaned up the name)\n",
    "                try:\n",
    "                    dcm_pts_name = _unidecode(dcmdata.PatientName.decode('latin1').replace('^', ' ')).lower().strip()\n",
    "                    # Already anonymized dicom? Get the original patient's name from the anonymized id\n",
    "                    if dcm_pts_name in anon_ids:\n",
    "                        dcm_pts_name = anon_ids[dcm_pts_name]\n",
    "                        if skip_already_processed:\n",
    "                            tbar.update()\n",
    "                            continue\n",
    "                except AttributeError as exc:\n",
    "                    if filename.upper() == 'DCMDIR' or filename.upper() == 'DICOMDIR':\n",
    "                        os.remove(fullfilepath)  # DICOMDIR files are useless, they are only descriptive files for CD/DVD of dicoms\n",
    "                        # DOES NOT WORK: pydicom can read and edit dicomdir files but cannot save them yet!\n",
    "                        #dcmdata = dicom.read_dicomdir(r'dicoms\\ANTOINE_el\\EPI_T1\\DICOMDIR')\n",
    "                        #for record in dcmdata.patient_records:\n",
    "                            #record.PatientName = anon_id\n",
    "                        continue\n",
    "                    else:\n",
    "                        raise\n",
    "                # Anonymize\n",
    "                dcmdata.PatientName = anon_id\n",
    "                dcmdata.PatientID = anon_id\n",
    "                if [0x33,0x1013] in dcmdata:  # custom patientname field...\n",
    "                    dcmdata[0x33,0x1013].value = anon_id\n",
    "                # Delete private fields\n",
    "                for field in fields_to_del:\n",
    "                    if field in dcmdata:\n",
    "                        if isinstance(field, str):\n",
    "                            del dcmdata[dcmdata.data_element(field).tag]\n",
    "                        else:\n",
    "                            del dcmdata[field]\n",
    "                if remove_private_tags:\n",
    "                    dcmdata.remove_private_tags()\n",
    "                # Try to anonymize hidden name fields\n",
    "                hfields = find_hidden_name_fields(dcmdata, [dcm_pts_name, pts_name], hfields)\n",
    "                for dcmfield in hfields:\n",
    "                    if dcmfield in dcmdata:\n",
    "                        dcmdata[dcmfield].value = re.sub(dcm_pts_name.replace(' ', '[\\W]+'), anon_id, dcmdata[dcmfield].value, flags=re.I)\n",
    "                        dcmdata[dcmfield].value = re.sub(pts_name.replace(' ', '[\\W]+'), anon_id, dcmdata[dcmfield].value, flags=re.I)\n",
    "                # Last check just in case we could not remove the name everywhere!\n",
    "                dcm_data_str = _unidecode(str(dcmdata).decode('latin1').replace('^', ' ')).lower().strip()  # read all fields at once\n",
    "                if dcm_pts_name in str(dcm_data_str) or pts_name in str(dcm_data_str):  # if patient's name is still in the file, that's bad!\n",
    "                    print('Hidden name fields found: %s' % hfields)\n",
    "                    print('names: %s - %s' %(dcm_pts_name, pts_name))  # debugline\n",
    "                    print(str(dcm_data_str))\n",
    "                    raise ValueError('Error: could not remove name totally (there must be an additional non-standard PatientName field) from file: %s' % fullfilepath)\n",
    "                # Save anonymized dicom file\n",
    "                dcmdata.save_as(fullfilepath)\n",
    "                # Close the dicom file\n",
    "                #os.close(os_id)\n",
    "                del dcmdata\n",
    "                count_anon += 1\n",
    "            except (InvalidDicomError) as exc:\n",
    "                pass\n",
    "            except AttributeError as exc:\n",
    "                print(fullfilepath)\n",
    "                raise\n",
    "        tbar.update()  # update progressbar\n",
    "tbar.close()\n",
    "\n",
    "print('Hidden name fields found (and automagically anonymized): %s' % hfields)\n",
    "print('Total dicom anonymized: %i over %i total. Total dicom files skipped: %i. Total reports/non-dicom files deleted: %i.' % (count_anon, count_files, count_files_skipped, count_delete))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Rename files if filename include a patient's name\n",
    "\n",
    "# Compile regex to find any patient name (of any patient!) in a string. Non-alphabetical characters are ignored.\n",
    "filename_patterns = re.compile('(' + '|'.join(re.sub('[^a-zA-Z]+', '[^a-zA-Z]*', s) for s in dcm_unique) + ')', flags=re.I)\n",
    "\n",
    "uni_rootpath = unicode(rootpath, 'latin1')  # convert rootpath to unicode before walking with os.listdir and recwalk, so we get back unicode strings too (else we won't be able to enter folders with accentuated characters)\n",
    "subjects_list = get_list_of_folders(uni_rootpath)\n",
    "\n",
    "count_files = 0\n",
    "# Precompute total number of files (for progressbar)\n",
    "print('Precomputing total number of files, please wait...')\n",
    "for subject in _tqdm(subjects_list, unit='folders', desc='PRECOMP'):\n",
    "    fullpath = os.path.join(uni_rootpath, subject)\n",
    "    for dirpath, filename in recwalk(fullpath, topdown=False, folders=True):\n",
    "        count_files += 1\n",
    "\n",
    "# Rename files if they have a patient's name\n",
    "count_moved = 0\n",
    "tbar = _tqdm(total=count_files, unit='files', desc='ANONFN')\n",
    "print('Anonymizing of file/folder names, please wait...')\n",
    "for folder in subjects_list:  # do not rename the top directories, this will be done separately\n",
    "    if verbose:\n",
    "        print('- Processing top folder %s' % (folder))\n",
    "    fullpath = os.path.join(uni_rootpath, folder)\n",
    "    for dirpath, filename in recwalk(fullpath, topdown=False, folders=True):\n",
    "        # Find any name (of any patient) in the filename\n",
    "        # TODO: construct re all permutations of all names, and re.compile, it will be fast\n",
    "        # TODO: try to do levenshtein distance on names? (but just with current patient name, else it will take too much time with all patients...) it will considerably slow down the anonymization... Is there a faster way?\n",
    "        matchs = filename_patterns.finditer(filename)\n",
    "        # If found, we find the anonymized id for each match to replace\n",
    "        to_replace = []\n",
    "        for m in matchs:\n",
    "            # Clean up the name\n",
    "            pts_name_in_filename = re.sub('[^a-zA-Z]+', ' ', m.group(1).lower())\n",
    "            # Find the closest unique name\n",
    "            dst_mat = compute_names_distance_matrix([pts_name_in_filename], dcm_unique)\n",
    "            # Get the anonymized id from unique name\n",
    "            if dst_mat[pts_name_in_filename]:\n",
    "                anon_id = name_to_anon_ids[dst_mat[pts_name_in_filename][0]]\n",
    "            else:  # could not find an id, just anonymize with a random name\n",
    "                anon_id = 'anon'\n",
    "            # Add slide index and anonymized id to replace all at once later\n",
    "            to_replace.append( ( anon_id, slice(m.start(1), m.end(1)) ) )\n",
    "        # Replace all matchs at once\n",
    "        if to_replace:\n",
    "            # Can't modify strings, need to convert to a list\n",
    "            filename_anon = list(filename)\n",
    "            # For each match, replace with anonymized id\n",
    "            for anon_id, slidx in to_replace[::-1]:  # reverse list because else the subsequent items won't be aligned anymore when we will replace the first items in the list\n",
    "                filename_anon[slidx] = anon_id\n",
    "            # Convert back to a string\n",
    "            filename_anon = ''.join(filename_anon)\n",
    "            # Rename the file/folder\n",
    "            shutil.move(os.path.join(dirpath, filename), os.path.join(dirpath, filename_anon))\n",
    "            count_moved += 1\n",
    "        tbar.update()\n",
    "tbar.close()\n",
    "print('Total dicom files/folders moved: %i over %i total.' % (count_moved, count_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename folders if enabled\n",
    "rename_folders = True  # rename root folder # TODO: rename any subfolder with patient's name\n",
    "delete_empty = True  # delete empty folders (or not containing any DICOM, such as nifti folders)?\n",
    "\n",
    "count_folder = 0\n",
    "count_skipped = 0\n",
    "count_empty = 0\n",
    "uni_rootpath = unicode(rootpath, 'latin1')  # convert rootpath to unicode before walking with os.listdir and recwalk, so we get back unicode strings too (else we won't be able to enter folders with accentuated characters)\n",
    "if rename_folders:\n",
    "    print('Launching anonymization of dicom folders, please wait...')\n",
    "    # Get folder_to_name mapping\n",
    "    _, folder_to_name = get_dcm_names_from_dir(uni_rootpath)\n",
    "    _, folder_to_name = get_dcm_names_from_zip(uni_rootpath, folder_to_name=folder_to_name)\n",
    "    # Get list of folders\n",
    "    subjects_list = get_list_of_folders(uni_rootpath)\n",
    "    for subject in _tqdm(subjects_list, unit='folder', desc='RENAME'):\n",
    "        # Already anonymized folder, just skip\n",
    "        if subject in anon_ids or re.match('(^%s.+)_s\\d+$' % anon_prefix, subject):\n",
    "            count_skipped += 1\n",
    "            continue\n",
    "        pts_name = folder_to_name[subject]\n",
    "        # Already partially anonymized, we get the original name from the anonymization mapping\n",
    "        if pts_name in anon_ids:\n",
    "            pts_name = anon_ids[pts_name]\n",
    "        # Special case: no dicom with a patient name can be found inside the folder (might be nifti files instead?), so we just remove this folder!\n",
    "        if pts_name is None and delete_empty:\n",
    "            filepath = os.path.join(uni_rootpath, subject)\n",
    "            if os.path.isdir(filepath):\n",
    "                shutil.rmtree(filepath)\n",
    "            else:\n",
    "                os.remove(filepath)\n",
    "            count_empty += 1\n",
    "            continue\n",
    "        # Fetch the anonymized id from folder name (because we already looked inside to get the first dicom's patientname)\n",
    "        anon_id = name_to_anon_ids[dcmname_to_uniquename[pts_name]]\n",
    "        if verbose:\n",
    "            print('- Processing subject %s -> %s in folder %s' % (pts_name, anon_id, subject))\n",
    "        fullpath = os.path.join(uni_rootpath, subject)\n",
    "        # Rename subject directory\n",
    "        new_folder_name = os.path.join(uni_rootpath, anon_id)\n",
    "        if not os.path.exists(new_folder_name):\n",
    "            os.rename(fullpath, new_folder_name)\n",
    "        else:\n",
    "            # if new folder already exists, find a new name (append \"_sx\" where x is a number)\n",
    "            for i in range(2, 1000):\n",
    "                alt_folder_name = \"%s_s%i\" % (new_folder_name, i)\n",
    "                if not os.path.exists(alt_folder_name):\n",
    "                    os.rename(fullpath, alt_folder_name)\n",
    "                    break\n",
    "        count_folder += 1\n",
    "\n",
    "print('Total dicom folders renamed (anonymized): %i over %i total. Skipped: %i. Empty folders (or containing non-dicom files) and thus deleted: %i.' % (count_folder, len(subjects_list), count_skipped, count_empty))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shorten (again) anonymized demographics to only the subjects we have dicom folders for\n",
    "demo_anon_csv = 'demographics_anonymized.csv'\n",
    "cf_anon = pd.read_csv(demo_anon_csv, sep=';').fillna('')\n",
    "# Get list of anonymized dicom names\n",
    "dcm_ids, _ = get_dcm_names_from_dir(rootpath)\n",
    "# Shorten anonymized demographics to only the ids present in dicoms\n",
    "cf_anon = cf_anon[cf_anon['name'].isin(dcm_ids)]\n",
    "# Save shortened anonymized demographics\n",
    "cf_anon.to_csv(demo_anon_csv, sep=';', na_rep='NA', index=False)\n",
    "print('Shortened anonymized demographics (to only the dicoms available) were saved to %s.' % demo_anon_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: add new ids (new dicom folders) even if not in demographics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------------------------------------------------\n",
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcmdata = dicom.read_file(r'123456789.dcm', stop_before_pixels=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hfields = find_hidden_name_fields(dcmdata, ['smith'])\n",
    "for dcmfield in hfields:\n",
    "    print(dcmfield)\n",
    "    print(dcmdata[dcmfield].value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcm_pts_name = pts_name = 'smith'\n",
    "for dcmfield in hfields:\n",
    "    if dcmfield in dcmdata:\n",
    "        dcmdata[dcmfield].value = re.sub(dcm_pts_name, anon_id, dcmdata[dcmfield].value, flags=re.I)\n",
    "        dcmdata[dcmfield].value = re.sub(pts_name, anon_id, dcmdata[dcmfield].value, flags=re.I)\n",
    "find_hidden_name_fields(dcmdata, ['smith'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "str(_unidecode(str(dcmdata).decode('latin1').replace('^', ' ')).lower().strip()).find('smith')  # read all fields at once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "str(dcmdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcmdata[0x0029, 0x1010].value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = dcmdata.keys()[0]\n",
    "dcmdata[a].value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcm_pts_name = 'smith'\n",
    "anon_id = 'subj003'\n",
    "\n",
    "def find_hidden_name_fields(dcmdata, dcm_pts_name, hidden_name_fields=None):\n",
    "    if hidden_name_fields is None:\n",
    "        hidden_name_fields = set()\n",
    "    for dcmfield in dcmdata.keys(): # different from dir()?\n",
    "        if dcmdata[dcmfield]:\n",
    "            try:\n",
    "                #dcmfieldval = dcmdata.data_element(dcmfield).value\n",
    "                dcmfieldval = dcmdata[dcmfield].value\n",
    "                check = False\n",
    "                if isinstance(dcmfieldval, list):\n",
    "                    check = dcm_pts_name in [s.lower() if isinstance(s, str) else s for s in dcmfieldval]\n",
    "                elif isinstance(dcmfieldval, (int, float)):\n",
    "                    check = False\n",
    "                else:\n",
    "                    check = (dcmfieldval.lower().find(dcm_pts_name) >= 0)\n",
    "                if check:\n",
    "                    hidden_name_fields.add(dcmfield)\n",
    "            except AttributeError:\n",
    "                print(dcmfield)\n",
    "                raise\n",
    "    return hidden_name_fields\n",
    "\n",
    "hfields = find_hidden_name_fields(dcmdata, dcm_pts_name)\n",
    "print(hfields)\n",
    "for dcmfield in hfields:\n",
    "    dcmdata[dcmfield].value = re.sub(dcm_pts_name, anon_id, dcmdata[dcmfield].value, flags=re.I)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dcmdata.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for s in dcmdata.formatted_lines():\n",
    "    print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dcmdata.dir() # dcmdata[0x0029, 0x1010] or CSA Image Header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcmdata[0x33,0x1013].value = 'haha'\n",
    "del dcmdata[0x33,0x1013]\n",
    "[0x33,0x1013] in dcmdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcmdata.PatientWeight\n",
    "dcmdata.dir('OtherPatientNames')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del dcmdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pts_name = _unidecode(dcmdata.PatientName.decode('latin1').replace('^', ' ')).lower().strip()\n",
    "dcm_data_str = _unidecode(str(dcmdata).decode('latin1').replace('^', ' ')).lower().strip()\n",
    "#pts_name = dcmdata.PatientName.decode('latin1')\n",
    "pts_name\n",
    "pts_name in dcm_data_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcmdata[0x0033, 0x1013].value.decode('latin1').lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pts_name in dcmdata[0x0033, 0x1013].value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(pts_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcmdata = dicom.read_dicomdir(r'DICOMDIR')\n",
    "dcmdata\n",
    "# if [0x0004, 0x1130] in dcmdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcmdata = dicom.read_dicomdir(r'DICOMDIR')\n",
    "for record in dcmdata.patient_records:\n",
    "    record.PatientName = 'hehe'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for record in dcmdata.patient_records:\n",
    "    print(record.PatientName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcmdata.save_as(r'DICOMDIR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}