Merge pull request #438 from girder/internal-metadata

Report internal metadata as a separate method and endpoint.

girder/girder_large_image/models/image_item.py

@@ -173,6 +173,12 @@ def getMetadata(self, item, **kwargs):
         tileSource = self._loadTileSource(item, **kwargs)
         return tileSource.getMetadata()
 
+    def getInternalMetadata(self, item, **kwargs):
+        tileSource = self._loadTileSource(item, **kwargs)
+        result = tileSource.getInternalMetadata() or {}
+        result['tilesource'] = tileSource.name
+        return result
+
     def getTile(self, item, x, y, z, mayRedirect=False, **kwargs):
         tileSource = self._loadTileSource(item, **kwargs)
         imageParams = {}

girder/girder_large_image/rest/tiles.py

@@ -103,6 +103,8 @@ def __init__(self, apiRoot):
                            self.getDZIInfo)
         apiRoot.item.route('GET', (':itemId', 'tiles', 'dzi_files', ':level', ':xandy'),
                            self.getDZITile)
+        apiRoot.item.route('GET', (':itemId', 'tiles', 'internal_metadata'),
+                           self.getInternalMetadata)
         filter_logging.addLoggingFilter(
             'GET (/[^/ ?#]+)*/item/[^/ ?#]+/tiles/zxy(/[^/ ?#]+){3}',
             frequency=250)
@@ -248,9 +250,22 @@ def _setContentDisposition(self, item, contentDisposition, mime, subname):
     @access.public
     @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
     def getTilesInfo(self, item, params):
-        # TODO: parse params?
         return self._getTilesInfo(item, params)
 
+    @describeRoute(
+        Description('Get large image internal metadata.')
+        .param('itemId', 'The ID of the item.', paramType='path')
+        .errorResponse('ID was invalid.')
+        .errorResponse('Read access was denied for the item.', 403)
+    )
+    @access.public
+    @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
+    def getInternalMetadata(self, item, params):
+        try:
+            return self.imageItemModel.getInternalMetadata(item, **params)
+        except TileGeneralException as e:
+            raise RestException(e.args[0], code=400)
+
     @describeRoute(
         Description('Get test large image metadata.')
     )

girder/test_girder/test_tiles_rest.py

@@ -1119,3 +1119,13 @@ def testTilesHistogram(server, admin, fsAssetstore):
     assert len(resp.json[0]['hist']) == 256
     assert resp.json[1]['samples'] == 2801664
     assert resp.json[1]['hist'][128] == 176
+
+
+@pytest.mark.usefixtures('unbindLargeImage')
+@pytest.mark.plugin('large_image')
+def testTilesInternalMetadata(server, admin, fsAssetstore):
+    file = utilities.uploadExternalFile(
+        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+    itemId = str(file['itemId'])
+    resp = server.request(path='/item/%s/tiles/internal_metadata' % itemId)
+    assert resp.json['tilesource'] == 'tiff'

large_image/tilesource/base.py

@@ -1494,6 +1494,16 @@ def getMetadata(self):
             'mm_y': mag['mm_y'],
         }
 
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        return None
+
     @methodcache()
     def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False,
                 sparseFallback=False, frame=None):

sources/gdal/large_image_source_gdal/__init__.py

@@ -546,6 +546,31 @@ def getMetadata(self):
             metadata['netcdf'] = self._netcdf
         return metadata
 
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        print('HERE')
+        result = {}
+        with self._getDatasetLock:
+            result['driverShortName'] = self.dataset.GetDriver().ShortName
+            result['driverLongName'] = self.dataset.GetDriver().LongName
+            result['fileList'] = self.dataset.GetFileList()
+            result['RasterXSize'] = self.dataset.RasterXSize
+            result['RasterYSize'] = self.dataset.RasterYSize
+            result['GeoTransform'] = self.dataset.GetGeoTransform()
+            result['GCPProjection'] = self.dataset.GetGCPProjection()
+            result['Metadata'] = self.dataset.GetMetadata_List()
+            for key in ['IMAGE_STRUCTURE', 'SUBDATASETS', 'GEOLOCATION', 'RPC']:
+                metadatalist = self.dataset.GetMetadata_List(key)
+                if metadatalist:
+                    result['Metadata_' + key] = metadatalist
+        return result
+
     def getTileCorners(self, z, x, y):
         """
         Returns bounds of a tile for a given x,y,z index.

sources/nd2/large_image_source_nd2/__init__.py

@@ -237,15 +237,7 @@ def getMetadata(self):  # noqa
         """
         result = super(ND2FileTileSource, self).getMetadata()
 
-        # If two imgeas haven't panned by this factor of their size, treat them
-        # as the same IndexXY
-        result['nd2'] = self._metadata
-        result['nd2'].pop('custom_data', None)
-        result['nd2'].pop('image_metadata', None)
-        result['nd2'].pop('image_metadata_sequence', None)
-        result['nd2_sizes'] = sizes = self._nd2.sizes
-        result['nd2_axes'] = self._nd2.axes
-        result['nd2_iter_axes'] = self._nd2.iter_axes
+        sizes = self._nd2.sizes
         # We may want to reformat the frames to standardize this across sources
         # An example of frames from OMETiff: {
         #   "DeltaT": "3532.529541",
@@ -312,6 +304,24 @@ def getMetadata(self):  # noqa
         }
         return result
 
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        result = {}
+        result['nd2'] = self._metadata
+        # result['nd2'].pop('custom_data', None)
+        # result['nd2'].pop('image_metadata', None)
+        # result['nd2'].pop('image_metadata_sequence', None)
+        result['nd2_sizes'] = self._nd2.sizes
+        result['nd2_axes'] = self._nd2.axes
+        result['nd2_iter_axes'] = self._nd2.iter_axes
+        return result
+
     @methodcache()
     def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs):
         if z < 0 or z >= self.levels:

sources/ometiff/large_image_source_ometiff/__init__.py

@@ -259,6 +259,17 @@ def getMetadata(self):
                 cname: c for c, cname in enumerate(channels[:maxref.get('IndexC', 1)])}
             for frame in result['frames']:
                 frame['Channel'] = channels[frame.get('IndexC', 0)]
+        return result
+
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        result = {}
         result['omeinfo'] = self._omeinfo
         return result
 

sources/openjpeg/large_image_source_openjpeg/__init__.py

@@ -213,6 +213,19 @@ def _readbox(self, box):
         except Exception:
             pass
 
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        results = {}
+        if hasattr(self, '_description_xml'):
+            results['xml'] = self._description_xml
+        return results
+
     @methodcache()
     def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs):
         if z < 0 or z >= self.levels:

sources/openslide/large_image_source_openslide/__init__.py

@@ -244,6 +244,19 @@ def getNativeMagnification(self):
             'mm_y': mm_y,
         }
 
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        results = {'openslide': {}}
+        for key in self._openslide.properties:
+            results['openslide'][key] = self._openslide.properties[key]
+        return results
+
     @methodcache()
     def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs):
         if z < 0:

sources/pil/large_image_source_pil/__init__.py

@@ -147,6 +147,22 @@ def getState(self):
         return super(PILFileTileSource, self).getState() + ',' + str(
             self._maxSize)
 
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        results = {'pil': {}}
+        for key in ('filename', 'format', 'mode', 'size', 'width', 'height', 'palette', 'info'):
+            try:
+                results['pil'][key] = getattr(self._pilImage, key)
+            except Exception:
+                pass
+        return results
+
     @methodcache()
     def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False,
                 mayRedirect=False, **kwargs):

sources/test/large_image_source_test/__init__.py

@@ -109,6 +109,16 @@ def fractalTile(self, image, x, y, widthCount, color=(0, 0, 0)):
                             ], color, None)
             sq //= 2
 
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        return {'fractal': self.fractal}
+
     @methodcache()
     def getTile(self, x, y, z, *args, **kwargs):
         widthCount = 2 ** z

sources/tiff/large_image_source_tiff/__init__.py

@@ -249,6 +249,21 @@ def getNativeMagnification(self):
             'mm_y': mm_y,
         }
 
+    def getInternalMetadata(self, **kwargs):
+        """
+        Return additional known metadata about the tile source.  Data returned
+        from this method is not guaranteed to be in any particular format or
+        have specific values.
+
+        :returns: a dictionary of data or None.
+        """
+        results = {}
+        for idx, dir in enumerate(self._tiffDirectories[::-1]):
+            if dir and hasattr(dir, '_description_xml'):
+                results['xml' + (
+                    '' if not results.get('xml') else '_' + str(idx))] = dir._description_xml
+        return results
+
     @methodcache()
     def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False,
                 sparseFallback=False, **kwargs):

test/test_source_gdal.py

@@ -332,3 +332,11 @@ def testRetileProjection():
     assert ti['tile'].size == 3000000
     tile = ts.getTile(1178, 1507, 12)
     assert len(tile) > 1000
+
+
+def testInternalMetadata():
+    testDir = os.path.dirname(os.path.realpath(__file__))
+    imagePath = os.path.join(testDir, 'test_files', 'rgb_geotiff.tiff')
+    source = large_image_source_gdal.GDALFileTileSource(imagePath)
+    metadata = source.getInternalMetadata()
+    assert metadata['driverShortName'] == 'GTiff'

test/test_source_nd2.py

@@ -27,3 +27,10 @@ def testTilesFromND2():
     assert tileMetadata['channels'] == ['Brightfield', 'YFP', 'A594', 'DAPI']
     assert tileMetadata['IndexRange'] == {'IndexC': 4, 'IndexXY': 2, 'IndexZ': 29}
     utilities.checkTilesZXY(source, tileMetadata)
+
+
+def testInternalMetadata():
+    imagePath = utilities.externaldata('data/ITGA3Hi_export_crop2.nd2.sha512')
+    source = large_image_source_nd2.ND2FileTileSource(imagePath)
+    metadata = source.getInternalMetadata()
+    assert 'nd2' in metadata

test/test_source_ometiff.py

@@ -74,3 +74,10 @@ def testStyleAutoMinMax():
     assert image[128][128][0] < imageB[128][128][0]
     assert image[0][128][0] < imageB[0][128][0]
     assert image[240][128][0] < imageB[240][128][0]
+
+
+def testInternalMetadata():
+    imagePath = utilities.externaldata('data/sample.ome.tif.sha512')
+    source = large_image_source_ometiff.OMETiffFileTileSource(imagePath)
+    metadata = source.getInternalMetadata()
+    assert 'omeinfo' in metadata

test/test_source_openjpeg.py

@@ -54,3 +54,10 @@ def testBelowLevelTilesFromOpenJPEG():
     large_image_source_openjpeg.OpenjpegFileTileSource._minTileSize = origMin
     large_image_source_openjpeg.OpenjpegFileTileSource._maxTileSize = origMax
     cachesClear()
+
+
+def testInternalMetadata():
+    imagePath = utilities.externaldata('data/sample_image.jp2.sha512')
+    source = large_image_source_openjpeg.OpenjpegFileTileSource(imagePath)
+    metadata = source.getInternalMetadata()
+    assert 'ScanInfo' in metadata['xml']

test/test_source_openslide.py

@@ -442,3 +442,12 @@ def testEdgeOptions():
     assert width == 240
     assert height == 240
     assert imageB != image
+
+
+def testInternalMetadata():
+    imagePath = utilities.externaldata(
+        'data/sample_jp2k_33003_TCGA-CV-7242-11A-01-TS1.1838afb1-9eee-'
+        '4a70-9ae3-50e3ab45e242.svs.sha512')
+    source = large_image_source_openslide.OpenslideFileTileSource(imagePath)
+    metadata = source.getInternalMetadata()
+    assert 'openslide' in metadata

test/test_source_pil.py

@@ -60,3 +60,10 @@ def testReadingVariousColorFormats():
     for name in files:
         imagePath = os.path.join(testDir, 'test_files', name)
         assert large_image_source_pil.PILFileTileSource.canRead(imagePath) is True
+
+
+def testInternalMetadata():
+    imagePath = utilities.externaldata('data/sample_Easy1.png.sha512')
+    source = large_image_source_pil.PILFileTileSource(imagePath)
+    metadata = source.getInternalMetadata()
+    assert 'pil' in metadata

test/test_source_tiff.py

@@ -646,3 +646,10 @@ def testSingleTileIteratorResample():
     assert tile['width'] == 255
     assert tile['tile_mm_x'] == 0.0005
     assert tile['tile_magnification'] == 20.0
+
+
+def testInternalMetadata():
+    imagePath = utilities.externaldata('data/sample_image.ptif.sha512')
+    source = large_image_source_tiff.TiffFileTileSource(imagePath)
+    metadata = source.getInternalMetadata()
+    assert 'xml' in metadata