Save reports in a reports folder.

devops/nciseer/provision.py

@@ -12,72 +12,79 @@
 from girder.utility.server import configureServer
 
 from girder_large_image.constants import PluginSettings as liSettings
-import nci_seer
-from nci_seer.constants import PluginSettings
 
-# This loads plugins, allowing setting validation
-configureServer()
 
-# If there is are no users, create an admin user
-if User().findOne() is None:
-    User().createUser('admin', 'password', 'Admin', 'Admin', '[email protected]')
-adminUser = User().findOne({'admin': True})
+def provision():
+    import nci_seer
+    from nci_seer.constants import PluginSettings
 
-# Set branding
-Setting().set('core.brand_name', 'SEER DSA')
-Setting().set('homepage.markdown', """
-# SEER Digital Slide Archive
----
-## NCI SEER Pediatic WSI Pilot
+    # If there is are no users, create an admin user
+    if User().findOne() is None:
+        User().createUser('admin', 'password', 'Admin', 'Admin', '[email protected]')
+    adminUser = User().findOne({'admin': True})
 
-Welcome to the **SEER Digital Slide Archive**.
+    # Set branding
+    Setting().set('core.brand_name', 'SEER DSA')
+    Setting().set('homepage.markdown', """
+    # SEER Digital Slide Archive
+    ---
+    ## NCI SEER Pediatic WSI Pilot
 
-Developers who want to use the Girder REST API should check out the
-[interactive web API docs](api/v1).
+    Welcome to the **SEER Digital Slide Archive**.
 
-NCI SEER Version: %s
-""" % (nci_seer.__version__))
+    Developers who want to use the Girder REST API should check out the
+    [interactive web API docs](api/v1).
 
-# Make sure we have an assetstore
-if Assetstore().findOne() is None:
-    Assetstore().createFilesystemAssetstore('Assetstore', '/assetstore')
+    NCI SEER Version: %s
+    """ % (nci_seer.__version__))
 
-# Make sure we have the SEER collection
-if Collection().findOne({'name': 'SEER'}) is None:
-    Collection().createCollection('SEER', adminUser)
-seerCollection = Collection().findOne({'name': 'SEER'})
-# Create default folders.  Set the settings to those folders
-folders = {
-    PluginSettings.HUI_INGEST_FOLDER: ('AvailableToProcess', True),
-    PluginSettings.HUI_QUARANTINE_FOLDER: ('Quarantined', True),
-    PluginSettings.HUI_PROCESSED_FOLDER: ('Redacted', True),
-    PluginSettings.HUI_REJECTED_FOLDER: ('Rejected', True),
-    PluginSettings.HUI_ORIGINAL_FOLDER: ('Original', True),
-    PluginSettings.HUI_FINISHED_FOLDER: ('Approved', True),
-}
-for settingKey, (folderName, public) in folders.items():
-    folder = None
-    folderId = Setting().get(settingKey)
-    if folderId:
-        folder = Folder().load(folderId, force=True)
-    if not folder:
-        folder = Folder().createFolder(
-            seerCollection, folderName, parentType='collection',
-            public=public, creator=adminUser, reuseExisting=True)
-        Setting().set(settingKey, str(folder['_id']))
-    elif folder['name'] != folderName or folder['public'] != public:
-        folder['name'] = folderName
-        folder['public'] = public
-        Folder().save(folder)
-# Set default import/export paths
-if not Setting().get(PluginSettings.NCISEER_IMPORT_PATH):
-    Setting().set(PluginSettings.NCISEER_IMPORT_PATH, '/import')
-if not Setting().get(PluginSettings.NCISEER_EXPORT_PATH):
-    Setting().set(PluginSettings.NCISEER_EXPORT_PATH, '/export')
-# Show label and macro images, plus tile and internal metadata for all users
-Setting().set(liSettings.LARGE_IMAGE_SHOW_EXTRA_ADMIN, '{"images": ["label", "macro"]}')
-Setting().set(liSettings.LARGE_IMAGE_SHOW_EXTRA, '{"images": ["label", "macro"]}')
-Setting().set(liSettings.LARGE_IMAGE_SHOW_ITEM_EXTRA_ADMIN,
-              '{"metadata": ["tile", "internal"], "images": ["label", "macro", "*"]}')
-Setting().set(liSettings.LARGE_IMAGE_SHOW_ITEM_EXTRA,
-              '{"metadata": ["tile", "internal"], "images": ["label", "macro", "*"]}')
+    # Make sure we have an assetstore
+    if Assetstore().findOne() is None:
+        Assetstore().createFilesystemAssetstore('Assetstore', '/assetstore')
+
+    # Make sure we have the SEER collection
+    if Collection().findOne({'name': 'SEER'}) is None:
+        Collection().createCollection('SEER', adminUser)
+    seerCollection = Collection().findOne({'name': 'SEER'})
+    # Create default folders.  Set the settings to those folders
+    folders = {
+        PluginSettings.HUI_INGEST_FOLDER: ('AvailableToProcess', True),
+        PluginSettings.HUI_QUARANTINE_FOLDER: ('Quarantined', True),
+        PluginSettings.HUI_PROCESSED_FOLDER: ('Redacted', True),
+        PluginSettings.HUI_REJECTED_FOLDER: ('Rejected', True),
+        PluginSettings.HUI_ORIGINAL_FOLDER: ('Original', True),
+        PluginSettings.HUI_FINISHED_FOLDER: ('Approved', True),
+        PluginSettings.HUI_REPORTS_FOLDER: ('Reports', True),
+    }
+    for settingKey, (folderName, public) in folders.items():
+        folder = None
+        folderId = Setting().get(settingKey)
+        if folderId:
+            folder = Folder().load(folderId, force=True)
+        if not folder:
+            folder = Folder().createFolder(
+                seerCollection, folderName, parentType='collection',
+                public=public, creator=adminUser, reuseExisting=True)
+            Setting().set(settingKey, str(folder['_id']))
+        elif folder['name'] != folderName or folder['public'] != public:
+            folder['name'] = folderName
+            folder['public'] = public
+            Folder().save(folder)
+    # Set default import/export paths
+    if not Setting().get(PluginSettings.NCISEER_IMPORT_PATH):
+        Setting().set(PluginSettings.NCISEER_IMPORT_PATH, '/import')
+    if not Setting().get(PluginSettings.NCISEER_EXPORT_PATH):
+        Setting().set(PluginSettings.NCISEER_EXPORT_PATH, '/export')
+    # Show label and macro images, plus tile and internal metadata for all users
+    Setting().set(liSettings.LARGE_IMAGE_SHOW_EXTRA_ADMIN, '{"images": ["label", "macro"]}')
+    Setting().set(liSettings.LARGE_IMAGE_SHOW_EXTRA, '{"images": ["label", "macro"]}')
+    Setting().set(liSettings.LARGE_IMAGE_SHOW_ITEM_EXTRA_ADMIN,
+                  '{"metadata": ["tile", "internal"], "images": ["label", "macro", "*"]}')
+    Setting().set(liSettings.LARGE_IMAGE_SHOW_ITEM_EXTRA,
+                  '{"metadata": ["tile", "internal"], "images": ["label", "macro", "*"]}')
+
+
+if __name__ == '__main__':
+    # This loads plugins, allowing setting validation
+    configureServer()
+    provision()

nci_seer/__init__.py

@@ -21,6 +21,7 @@
     PluginSettings.HUI_REJECTED_FOLDER,
     PluginSettings.HUI_ORIGINAL_FOLDER,
     PluginSettings.HUI_FINISHED_FOLDER,
+    PluginSettings.HUI_REPORTS_FOLDER,
 })
 def validateSettingsFolder(doc):
     if not doc.get('value', None):

nci_seer/constants.py

@@ -5,5 +5,6 @@ class PluginSettings(object):
     HUI_REJECTED_FOLDER = 'histomicsui.rejected_folder'
     HUI_ORIGINAL_FOLDER = 'histomicsui.original_folder'
     HUI_FINISHED_FOLDER = 'histomicsui.finished_folder'
+    HUI_REPORTS_FOLDER = 'histomicsui.reports_folder'
     NCISEER_IMPORT_PATH = 'nciseer.import_path'
     NCISEER_EXPORT_PATH = 'nciseer.export_path'

nci_seer/import_export.py

@@ -3,20 +3,25 @@
 import os
 import pandas as pd
 import shutil
+import tempfile
 
 from girder import logger
 from girder.models.assetstore import Assetstore
 from girder.models.file import File
 from girder.models.folder import Folder
 from girder.models.item import Item
 from girder.models.setting import Setting
+from girder.models.upload import Upload
 
 from girder_large_image.models.image_item import ImageItem
 
 from . import process
 from .constants import PluginSettings
 
 
+XLSX_MIMETYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+
+
 def readExcelData(filepath):
     """
     Read in the data from excel, while attempting to be forgiving about
@@ -62,16 +67,19 @@ def readExcelFiles(filelist, ctx):
         try:
             df = readExcelData(filepath)
         except Exception as exc:
-            report.append({
-                'path': filepath,
-                'status': 'notexcel',
-            })
             if isinstance(exc, ValueError):
                 message = 'Cannot read %s; it is not formatted correctly' % (
                     os.path.basename(filepath), )
+                status = 'badformat'
             else:
                 message = 'Cannot read %s; it is not an Excel file' % (
                     os.path.basename(filepath), )
+                status = 'notexcel'
+            report.append({
+                'path': filepath,
+                'status': status,
+                'reason': message,
+            })
             ctx.update(message=message)
             logger.info(message)
             continue
@@ -206,10 +214,68 @@ def ingestData(ctx, user=None):  # noqa
     for image in imageFiles:
         status = 'unlisted'
         report.append({'record': None, 'status': status, 'path': image})
-    # TODO: emit a report
+    importReport(ctx, report, excelReport, user)
     return reportSummary(report, excelReport)
 
 
+def importReport(ctx, report, excelReport, user):
+    """
+    Create an import report.
+
+    :param ctx: a progress context.
+    :param report: a list of files that were exported.
+    :param excelReport: a list of excel files that were processed.
+    :param user: the user triggering this.
+    """
+    ctx.update(message='Generating report')
+    excelStatusDict = {
+        'parsed': 'Parsed',
+        'notexcel': 'Not Excel',
+        'badformat': 'Bad Format',
+    }
+    statusDict = {
+        'added': 'Imported',
+        'present': 'Already imported',
+        'replaced': 'Updated',
+        'missing': 'File missing',
+        'unlisted': 'Not in DeID Upload file',
+    }
+    keyToColumns = {
+        'excel': 'ExcelFilePath',
+    }
+    dataList = []
+    for row in excelReport:
+        data = {
+            'ExcelFilePath': row['path'],
+            'Status': excelStatusDict.get(row['status'], row['status']),
+            'FailureReason': row.get('reason'),
+        }
+        dataList.append(data)
+    for row in report:
+        data = {
+            'FilePath': row['path'],
+            'Status': statusDict.get(row['status'], row['status']),
+        }
+        if row.get('record'):
+            fields = row['record'].get('fields')
+            data.update(fields)
+            for k, v in row['record'].items():
+                if k != 'fields':
+                    data[keyToColumns.get(k, k)] = v
+        dataList.append(data)
+    df = pd.DataFrame(dataList, columns=[
+        'ExcelFilePath', 'FilePath', 'Status',
+        'TokenID', 'Proc_Seq', 'Proc_Type', 'Spec_Site', 'Slide_ID', 'ImageID',
+        'FailureReason',
+    ])
+    reportName = 'DeID Import %s.xlsx' % datetime.datetime.now().strftime('%Y%m%d %H%M%S')
+    with tempfile.TemporaryDirectory(prefix='nciseer') as tempdir:
+        path = os.path.join(tempdir, reportName)
+        ctx.update(message='Saving report')
+        df.to_excel(path, index=False)
+        saveToReports(path, XLSX_MIMETYPE, user)
+
+
 def reportSummary(*args):
     result = {}
     for report in args:
@@ -238,7 +304,7 @@ def exportItems(ctx, user=None, all=False):
     exportFolder = Folder().load(exportFolderId, force=True, exc=True)
     report = []
     for filepath, file in Folder().fileList(exportFolder, user, data=False):
-        item = Item().load(file['itemId'], force=True, exc=True)
+        item = Item().load(file['itemId'], force=True, exc=False)
         try:
             tileSource = ImageItem().tileSource(item)
         except Exception:
@@ -271,7 +337,7 @@ def exportItems(ctx, user=None, all=False):
                 'time': exportedRecord[-1]['time'],
             })
     exportNoteRejected(report, user, all)
-    exportReport(ctx, exportPath, report)
+    exportReport(ctx, exportPath, report, user)
     return reportSummary(report)
 
 
@@ -314,13 +380,14 @@ def exportNoteRejected(report, user, all):
             })
 
 
-def exportReport(ctx, exportPath, report):
+def exportReport(ctx, exportPath, report, user):
     """
     Create an export report.
 
     :param ctx: a progress context.
     :param exportPath: directory for exports
     :param report: a list of files that were exported.
+    :param user: the user triggering this.
     """
     ctx.update(message='Generating report')
     statusDict = {
@@ -332,7 +399,6 @@ def exportReport(ctx, exportPath, report):
     }
     dataList = []
     for row in report:
-        print(row)
         row['item']['meta'].setdefault('deidUpload', {})
         data = {}
         data.update(row['item']['meta']['deidUpload'])
@@ -365,3 +431,23 @@ def exportReport(ctx, exportPath, report):
     path = os.path.join(exportPath, exportName)
     ctx.update(message='Saving report')
     df.to_excel(path, index=False)
+    saveToReports(path, XLSX_MIMETYPE, user)
+
+
+def saveToReports(path, mimetype=None, user=None):
+    """
+    Save a file to the reports folder.
+
+    :param path: path of the file to save.
+    :param mimetype: the mimetype of the file.
+    :param user: the user triggering this.
+    """
+    reportsFolderId = Setting().get(PluginSettings.HUI_REPORTS_FOLDER)
+    reportsFolder = Folder().load(reportsFolderId, force=True, exc=False)
+    if not reportsFolder:
+        raise Exception('Reports folder not specified.')
+    with open(path, 'rb') as f:
+        Upload().uploadFromFile(
+            f, size=os.path.getsize(path), name=os.path.basename(path),
+            parentType='folder', parent=reportsFolder, user=user,
+            mimeType=mimetype)

nci_seer/rest.py

@@ -205,6 +205,30 @@ def get_first_item(folder, user):
             return item
 
 
+def ingestData(user=None, progress=True):
+    """
+    Ingest data from the import folder.
+
+    :param user: the user that started this.
+    """
+    with ProgressContext(progress, user=user, title='Importing data') as ctx:
+        result = import_export.ingestData(ctx, user)
+    result['action'] = 'ingest'
+    return result
+
+
+def exportData(user=None, progress=True):
+    """
+    Export data to the export folder.
+
+    :param user: the user that started this.
+    """
+    with ProgressContext(progress, user=user, title='Exporting recent finished items') as ctx:
+        result = import_export.exportItems(ctx, user)
+    result['action'] = 'export'
+    return result
+
+
 class NCISeerResource(Resource):
     def __init__(self):
         super(NCISeerResource, self).__init__()
@@ -280,10 +304,7 @@ def setRedactList(self, item, redactList):
     def ingest(self):
         setResponseTimeLimit(86400)
         user = self.getCurrentUser()
-        with ProgressContext(True, user=user, title='Importing data') as ctx:
-            result = import_export.ingestData(ctx, user)
-        result['action'] = 'ingest'
-        return result
+        return ingestData(user)
 
     @autoDescribeRoute(
         Description('Export recently finished items to the export folder asynchronously.')
@@ -293,10 +314,7 @@ def ingest(self):
     def export(self):
         setResponseTimeLimit(86400)
         user = self.getCurrentUser()
-        with ProgressContext(True, user=user, title='Exporting recent finished items') as ctx:
-            result = import_export.exportItems(ctx, user)
-        result['action'] = 'export'
-        return result
+        return exportData(user)
 
     @autoDescribeRoute(
         Description('Export all finished items to the export folder asynchronously.')