Merge pull request #331 from Emory-HITI/dev

Make anonymizations configurable in Suvpar

modules/suvpar/README.md

@@ -22,6 +22,10 @@ Find the config.json file in the folder and modify accordingly.
 
 * *IsStatistics*: If you want the statistics then set it as true otherwise false(default).
 
+* *IsFinalCSV*: Do you want to drop the intermediate fields and produce the final csv. By default, true. If false, only pre-processing of data to anonymize the data and prepare an intermediate file that is ready for Suvpar processing.
+
+* *IsAnonymized*: Do you want to anonymize certain sensitive PHI headers. By default, true.
+
 # Running Niffler SUVPaR
 
 First, to run the script to trim the file.

modules/suvpar/Suvpar.py

@@ -9,11 +9,10 @@
 sta = {}
 statistics_csv = {}
 output_csv = {}
-final_csv = True
 
 
 def initialize():
-    global output_csv, df, device_SN, scanner_filter, statistics_csv, isStatistics
+    global output_csv, df, device_SN, scanner_filter, statistics_csv, isStatistics, final_csv, isAnonymized
     with open('config.json', 'r') as f:
         config = json.load(f)
 
@@ -24,6 +23,8 @@ def initialize():
     scanner_filter = bool(config['ScannerFilter'])
     statistics_csv = config['Statistics_File']
     isStatistics = bool(config['IsStatistics'])
+    final_csv = bool(config['IsFinalCSV'])
+    isAnonymized = bool(config['IsAnonymized'])
     text_file = open(feature_file, "r")
     feature_list = text_file.read().split('\n')
     # Consider some Device Serial Number and remove other.
@@ -61,26 +62,27 @@ def suvpar():
     # Check for the AcquisitionTime > SeriesTime case, currently observed in Philips and FONAR scanners.
     df['AltCase'] = numpy.where(df['Manufacturer'].str.contains('Philips|FONAR'), True, False)
 
-    # Apply hashing function to the column.
-    df['AccessionNumber'] = df['AccessionNumber'].astype(str).apply(
-        lambda x:
-        hashlib.sha256(x.encode()).hexdigest()
-    )
-
-    df['InstitutionAddress'] = df['InstitutionAddress'].astype(str).apply(
-        lambda x:
-        hashlib.sha256(x.encode()).hexdigest()
-    )
-
-    df['PatientID'] = df['PatientID'].astype(str).apply(
-        lambda x:
-        hashlib.sha256(x.encode()).hexdigest()
-    )
-
-    df['SeriesInstanceUID'] = df['SeriesInstanceUID'].astype(str).apply(
-        lambda x:
-        hashlib.sha256(x.encode()).hexdigest()
-    )
+    if isAnonymized:
+        # Apply hashing function to the column.
+        df['AccessionNumber'] = df['AccessionNumber'].astype(str).apply(
+            lambda x:
+            hashlib.sha256(x.encode()).hexdigest()
+        )
+
+        df['InstitutionAddress'] = df['InstitutionAddress'].astype(str).apply(
+            lambda x:
+            hashlib.sha256(x.encode()).hexdigest()
+        )
+
+        df['PatientID'] = df['PatientID'].astype(str).apply(
+            lambda x:
+            hashlib.sha256(x.encode()).hexdigest()
+        )
+
+        df['SeriesInstanceUID'] = df['SeriesInstanceUID'].astype(str).apply(
+            lambda x:
+            hashlib.sha256(x.encode()).hexdigest()
+        )
 
     # Add computed non-DICOM fields and drop a few attributes, if we are producing a final_csv and not an intermediate.
     if final_csv:

modules/suvpar/config.json

@@ -3,7 +3,9 @@
   "OutputFile": "output.csv",
   "ScannerDetails": "scanner.txt",
   "ScannerFilter": false,
-  "FeaturesetFile": "featureset.txt",
+  "FeaturesetFile": "featureset1.txt",
   "IsStatistics": false,
-  "Statistics_File": "statistic.csv"
+  "Statistics_File": "statistic.csv",
+  "IsFinalCSV": true,
+  "IsAnonymized": true
 }

modules/suvpar/featureset1.txt

@@ -21,4 +21,5 @@ AcquisitionDuration
 Modality
 Manufacturer
 ManufacturerModelName
-SeriesInstanceUID
+SeriesInstanceUID
+[SliceMeasurementDuration]