Add tests for unfiled images using default schema

tests/data/default_schema_deidUpload.csv

@@ -0,0 +1,5 @@
+TokenID,SurgPathNum,Registry,First_Name,Last_Name,Date_of_Birth_mmddyyyy,Tumor_Rec_Number,Histology_Code,Behavior_Code
+0590XY112001,S16-1234,66,Natalina,Eltun,12012008,01,9440,3
+0590XY112002,C17-3456,66,Moria,Sterman,12012009,01,9440,3
+0590XY112003,C16-9876,66,Aila,Neles,11012009,01,9530,3
+0590XY112004,S17-3421,66,Shaylah,Hardacre,12012009,01,9440,3

tests/datastore.py

@@ -12,6 +12,12 @@
     # Philips file
     # Source: sample_image.ptif
     'philips.ptif': 'sha512:ec0ec688537080e4ec2abb3978c14577df87250a2c0af42beaadc8f00f0baba210997d5d2fe7cfeeceb841885b6adad0c9f607e35eddcc479eb487bd3c1e28ac',  # noqa
+    # Sample mouse tissue files to use with default schema tests
+    'SEER_Mouse_1_17158539.svs': 'sha512:1ace33abb4bf96382c8823706c43deb1115b08d46bf96e2b03f425697a85e767631f0dc5568cce68c9110f1df66d264a97671fd488466f3cf59295513d6c7792', # noqa
+    'SEER_Mouse_1_17158540.svs': 'sha512:74a0d97e9f8e3f42409c10be37eca9fb5d3be06f0a4f258d4b58efb5adc67f3f787a868bce6aa75649d25b221f4a0b32109bcc1ca1c696d4060981cb7d6b65f1', # noqa
+    'SEER_Mouse_1_17158541.svs': 'sha512:f5fcbc5f31201ff7a28886d2b386a34fe9c0eb84cf9ce4e30601e6abe43c120ed242e23b1af555e69abab7823aef59a7220e6159b12f2cf565a2de69eb2cf1cb', # noqa
+    'SEER_Mouse_1_17158542.svs': 'sha512:e763630a72b307932c64e442253fbf3c1877e7c9db77abb93864d37418fe7f94b9ace5286dff44ff0242ca1acfc58e3282b71959a864abdd925dc409452e40b7', # noqa
+    'SEER_Mouse_1_17158543.svs': 'sha512:60ef06d11da310b713327adcab122e128a8a9506a68047dcff3ac5a327766e168f97ec032ba92b997b0d83e455864f8f61998c8c5f24767471b8b3c951838de1'  # noqa
 }
 
 

tests/test_web_client.py

@@ -3,7 +3,7 @@
 import pytest
 from pytest_girder.web_client import runWebClientTest
 
-from .utilities import provisionBoundServer, resetConfig  # noqa
+from .utilities import provisionBoundServer, provisionDefaultSchemaBoundServer, resetConfig  # noqa
 
 
 @pytest.mark.plugin('wsi_deid')
@@ -17,3 +17,22 @@ def testWebClient(boundServer, db, spec, provisionBoundServer, resetConfig):  #
         os.path.dirname(wsi_deid.import_export.SCHEMA_FILE_PATH), 'importManifestSchema.test.json')
     spec = os.path.join(os.path.dirname(__file__), '..', 'wsi_deid', 'web_client', 'tests', spec)
     runWebClientTest(boundServer, spec, 15000)
+
+
+@pytest.mark.plugin('wsi_deid')
+@pytest.mark.parametrize('spec', (
+    'wsi_deidDefaultSchemaSpec.js',
+))
+def testDefaultSchemaWebClient(boundServer, db, spec, provisionDefaultSchemaBoundServer, mocker): # noqa
+    def mockStartOcrForUnfiled(*args):
+        return None
+
+    mocker.patch('wsi_deid.import_export.startOcrJobForUnfiled', mockStartOcrForUnfiled)
+    specPath = os.path.join(
+        os.path.dirname(__file__),
+        '..', 'wsi_deid',
+        'web_client',
+        'tests',
+        spec
+    )
+    runWebClientTest(boundServer, specPath, 15000)

tests/test_workflow.py

@@ -6,7 +6,7 @@
 from girder.models.item import Item
 from girder.models.setting import Setting
 
-from .utilities import provisionServer  # noqa
+from .utilities import provisionDefaultSchemaServer, provisionServer  # noqa
 
 
 @pytest.mark.plugin('wsi_deid')
@@ -72,3 +72,32 @@ def test_workflow_with_options(server, provisionServer, user):  # noqa
     assert len(list(Folder().fileList(finishedFolder))) == 3
     rest.exportData(user, False)
     assert len(os.listdir(exportPath)) == 3
+
+
+@pytest.mark.plugin('wsi_deid')
+@pytest.mark.plugin('large_image')
+def test_workflow_with_default_schema(server, provisionDefaultSchemaServer, user, mocker): # noqa
+    import wsi_deid.import_export
+    from wsi_deid import rest
+    from wsi_deid.constants import PluginSettings
+
+    def mockStartOcrForUnfiled(*args):
+        return None
+
+    mocker.patch('wsi_deid.import_export.startOcrJobForUnfiled', mockStartOcrForUnfiled)
+
+    config = girder.utility.config.getConfig()
+    config[wsi_deid.config.CONFIG_SECTION] = {
+        'import_text_association_columns': [
+            'SurgPathNum',
+            'First_Name',
+            'Last_Name',
+            'Date_of_Birth_mmddyyyy'
+        ]
+    }
+
+    unfiledFolderId = Setting().get(PluginSettings.WSI_DEID_UNFILED_FOLDER)
+    unfiledFolder = Folder().load(unfiledFolderId, force=True, exc=True)
+    assert len(list(Folder().fileList(unfiledFolder))) == 0
+    rest.ingestData(user, False)
+    assert len(list(Folder().fileList(unfiledFolder, user, data=False))) == 5

tests/utilities.py

@@ -20,6 +20,16 @@ def provisionBoundServer(boundServer, admin, fsAssetstore, tmp_path):
     yield _provisionServer(tmp_path)
 
 
+@pytest.fixture
+def provisionDefaultSchemaServer(server, admin, fsAssetstore, tmp_path):
+    yield _provisionDefaultSchemaServer(tmp_path)
+
+
+@pytest.fixture
+def provisionDefaultSchemaBoundServer(boundServer, admin, fsAssetstore, tmp_path):
+    yield _provisionDefaultSchemaServer(tmp_path)
+
+
 def _provisionServer(tmp_path):
     sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'devops', 'wsi_deid'))
     import provision  # noqa
@@ -51,3 +61,46 @@ def resetConfig():
     # Use default wsi_deid config for tests
     config = girder.utility.config.getConfig()
     config[wsi_deid.config.CONFIG_SECTION] = {}
+
+
+def _provisionDefaultSchemaServer(tmp_path):
+    import girder.utility.config  # noqa
+
+    import wsi_deid.config  # noqa
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'devops', 'wsi_deid'))
+    config = girder.utility.config.getConfig()
+    config[wsi_deid.config.CONFIG_SECTION] = {
+        'import_text_association_columns': [
+            'SurgPathNum',
+            'First_Name',
+            'Last_Name',
+            'Date_of_Birth_mmddyyyy'
+        ]
+    }
+    import provision  # noqa
+    provision.provision()
+    del sys.path[-1]
+
+    importPath = tmp_path / 'import'
+    os.makedirs(importPath, exist_ok=True)
+    exportPath = tmp_path / 'export'
+    os.makedirs(exportPath, exist_ok=True)
+    # unfiledPath = tmp_path / 'unfiled'
+    # os.makedirs(unfiledPath, exist_ok=True)
+    Setting().set(PluginSettings.WSI_DEID_IMPORT_PATH, str(importPath))
+    Setting().set(PluginSettings.WSI_DEID_EXPORT_PATH, str(exportPath))
+    # Setting().set(PluginSettings.WSI_DEID_UNFILED_FOLDER, str(unfiledPath))
+    for filename in {
+        'SEER_Mouse_1_17158539.svs',
+        'SEER_Mouse_1_17158540.svs',
+        'SEER_Mouse_1_17158541.svs',
+        'SEER_Mouse_1_17158542.svs',
+        'SEER_Mouse_1_17158543.svs'
+    }:
+        path = datastore.fetch(filename)
+        shutil.copy(path, str(importPath / filename))
+    dataPath = os.path.join(os.path.dirname(__file__), 'data')
+    for filename in {'default_schema_deidUpload.csv'}:
+        path = os.path.join(dataPath, filename)
+        shutil.copy(path, importPath / filename)
+    return importPath, exportPath

tox.ini

@@ -10,6 +10,7 @@ deps =
   coverage
   pooch
   pytest
+  pytest-mock
   pytest-cov
   pytest-girder
   pytest-xdist