Merge pull request #119 from Emory-HITI/ramon349-patch-1

Fixed issues with make dirs and error with multiprocessing pooling

modules/png-extraction/ImageExtractor.py

@@ -15,6 +15,7 @@
 import numpy as np
 import pandas as pd
 import pydicom as dicom 
+import png 
 #pydicom imports needed to handle data errrors
 from pydicom import config
 from pydicom import datadict
@@ -152,8 +153,7 @@ def extract_images(i):
             ID=filedata.iloc[i].loc['PatientID']  # Unique identifier for the Patient.
             folderName = hashlib.sha224(ID.encode('utf-8')).hexdigest()
             #check for existence of patient folder. Create if it does not exist.
-            if not (os.path.exists(png_destination + folderName)): # it is completely possible for multiple proceses to run this check at same time.
-                os.mkdir(png_destination + folderName)
+            os.makedirs(png_destination + folderName,exist_ok=True)
         elif flattened_to_level == 'study':
             ID1=filedata.iloc[i].loc['PatientID']  # Unique identifier for the Patient.
             try:
@@ -163,8 +163,7 @@ def extract_images(i):
             folderName = hashlib.sha224(ID1.encode('utf-8')).hexdigest() + "/" + \
                          hashlib.sha224(ID2.encode('utf-8')).hexdigest()
             #check for existence of the folder tree patient/study/series. Create if it does not exist.
-            if not (os.path.exists(png_destination + folderName)): # it is completely possible for multiple proceses to run this check at same time.
-                os.makedirs(png_destination + folderName)
+            os.makedirs(png_destination + folderName,exist_ok=True)
         else:
             ID1=filedata.iloc[i].loc['PatientID']  # Unique identifier for the Patient.
             try:
@@ -176,8 +175,7 @@ def extract_images(i):
             folderName = hashlib.sha224(ID1.encode('utf-8')).hexdigest() + "/" + \
                          hashlib.sha224(ID2.encode('utf-8')).hexdigest() + "/" + hashlib.sha224(ID3.encode('utf-8')).hexdigest()
             #check for existence of the folder tree patient/study/series. Create if it does not exist.
-            if not (os.path.exists(png_destination + folderName)): # it is completely possible for multiple proceses to run this check at same time.
-                os.makedirs(png_destination + folderName)
+            os.makedirs(png_destination + folderName,exist_ok=True)
 
 
         pngfile = png_destination+folderName+'/' + hashlib.sha224(imName.encode('utf-8')).hexdigest() + '.png'
@@ -344,19 +342,17 @@ def fix_mismatch(with_VRs=['PN', 'DS', 'IS']):
         filedata=data
         total = len(chunk)
         stamp = time.time()
-        p = Pool(core_count) 
-        res = p.imap_unordered(extract_images,range(len(filedata)))
-        for out in res:
-            (fmap,fail_path,err) = out
-            if err:
-                count +=1
-                copyfile(fail_path[0],fail_path[1])
-                err_msg = str(count) + ' out of ' + str(len(chunk)) + ' dicom images have failed extraction'
-                logging.error(err_msg)
-            else:
-                fm.write(fmap)
-        p.join()
-        p.close()
+        with Pool(core_count) as p:
+            res = p.imap_unordered(extract_images,range(len(filedata)))
+            for out in res:
+                (fmap,fail_path,err) = out
+                if err:
+                    count +=1
+                    copyfile(fail_path[0],fail_path[1])
+                    err_msg = str(count) + ' out of ' + str(len(chunk)) + ' dicom images have failed extraction'
+                    logging.error(err_msg)
+                else:
+                    fm.write(fmap)
     fm.close()
     logging.info('Chunk run time: %s %s', time.time() - t_start, ' seconds!')