Allow scheduling histogram computation

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## 1.23.6
+
+### Improvements
+- Allow scheduling histogram computation in Girder ([#1282](../../pull/1282))
+
 ## 1.23.5
 
 ### Improvements

girder/girder_large_image/models/image_item.py

@@ -550,9 +550,9 @@ def tileFrames(self, item, checkAndCreate='nosave', **kwargs):
 
         :param item: the item with the tile source.
         :param checkAndCreate: if False, use the cache.  If True and the result
-            is already cached, just return True.  If is does not, create,
-            cache, and return it.  If 'nosave', return values from the cache,
-            but do not store new results in the cache.
+            is already cached, just return True.  If is not, create, cache, and
+            return it.  If 'nosave', return values from the cache, but do not
+            store new results in the cache.
         :param kwargs: optional arguments.  Some options are left, top,
             right, bottom, regionWidth, regionHeight, units, width, height,
             encoding, jpegQuality, jpegSubsampling, and tiffCompression.  This
@@ -577,7 +577,7 @@ def getPixel(self, item, **kwargs):
         tileSource = self._loadTileSource(item, **kwargs)
         return tileSource.getPixel(**kwargs)
 
-    def histogram(self, item, **kwargs):
+    def histogram(self, item, checkAndCreate=False, **kwargs):
         """
         Using a tile source, get a histogram of the image.
 
@@ -592,8 +592,10 @@ def histogram(self, item, **kwargs):
         else:
             imageKey = 'histogram'
             result = self._getAndCacheImageOrData(
-                item, 'histogram', False, dict(kwargs, imageKey=imageKey),
-                pickleCache=True, **kwargs)[0]
+                item, 'histogram', checkAndCreate,
+                dict(kwargs, imageKey=imageKey), pickleCache=True, **kwargs)
+            if not isinstance(result, bool):
+                result = result[0]
         return result
 
     def getBandInformation(self, item, statistics=True, **kwargs):
@@ -667,3 +669,28 @@ def _scheduleTileFrames(self, item, tileFramesList, user):
         )
         Job().scheduleJob(job)
         return job
+
+    def _scheduleHistograms(self, item, histogramList, user):
+        """
+        Schedule generating histograms in a local job.
+
+        :param item: the item.
+        :param histogramList: a list of dictionary of parameters to pass to
+            the histogram method.
+        :param user: the user owning the job.
+        """
+        job = Job().createLocalJob(
+            module='large_image_tasks.tasks',
+            function='cache_histograms_job',
+            kwargs={
+                'itemId': str(item['_id']),
+                'histogramList': histogramList,
+            },
+            title='Cache Histograms',
+            type='large_image_cache_histograms',
+            user=user,
+            public=True,
+            asynchronous=True,
+        )
+        Job().scheduleJob(job)
+        return job

girder/girder_large_image/rest/tiles.py

@@ -973,6 +973,28 @@ def getTilesPixel(self, item, params):
             raise RestException('Value Error: %s' % e.args[0])
         return pixel
 
+    def _cacheHistograms(self, item, histRange, cache, params):
+        needed = []
+        result = {'cached': []}
+        tilesource = self.imageItemModel._loadTileSource(item, **params)
+        for frame in range(tilesource.frames):
+            if histRange is not None and histRange != 'round':
+                continue
+            checkParams = params.copy()
+            checkParams['range'] = histRange
+            if tilesource.frames > 1:
+                checkParams['frame'] = frame
+            else:
+                checkParams.pop('frame', None)
+            result['cached'].append(self.imageItemModel.histogram(
+                item, checkAndCreate='check', **checkParams))
+            if not result['cached'][-1]:
+                needed.append(checkParams)
+        if cache == 'schedule' and not all(result['cached']):
+            result['scheduledJob'] = str(self.imageItemModel._scheduleHistograms(
+                item, needed, self.getCurrentUser())['_id'])
+        return result
+
     @describeRoute(
         Description('Get a histogram for any region of a large image item.')
         .notes('This can take all of the parameters as the region endpoint, '
@@ -1010,6 +1032,10 @@ def getTilesPixel(self, item, params):
                required=False, dataType='boolean', default=False)
         .param('density', 'If true, scale the results by the number of '
                'samples.', required=False, dataType='boolean', default=False)
+        .param('cache', 'Report on or request caching the specified histogram '
+               'for all frames.  Scheduling creates a local job.',
+               required=False,
+               enum=['none', 'report', 'schedule'])
         .errorResponse('ID was invalid.')
         .errorResponse('Read access was denied for the item.', 403),
     )
@@ -1053,9 +1079,13 @@ def getHistogram(self, item, params):
         if params.get('roundRange'):
             if params.pop('roundRange', False) and histRange is None:
                 histRange = 'round'
+
+        cache = params.pop('cache', None)
+        if cache in {'report', 'schedule'}:
+            return self._cacheHistograms(item, histRange, cache, params)
         result = self.imageItemModel.histogram(item, range=histRange, **params)
         result = result['histogram']
-        # Cast everything to lists and floats so json with encode properly
+        # Cast everything to lists and floats so json will encode properly
         for entry in result:
             for key in {'bin_edges', 'hist', 'range'}:
                 if key in entry:

girder/test_girder/test_tiles_rest.py

@@ -1251,6 +1251,31 @@ def testTilesHistogramWithRange(server, admin, fsAssetstore):
     assert resp.json[1]['samples'] < 1000000
 
 
+@pytest.mark.usefixtures('unbindLargeImage')
+@pytest.mark.plugin('large_image')
+def testTilesHistogramCachingJob(server, admin, fsAssetstore):
+    file = utilities.uploadExternalFile(
+        'sample_image.ptif', admin, fsAssetstore)
+    itemId = str(file['itemId'])
+    resp = server.request(
+        path='/item/%s/tiles/histogram' % itemId,
+        params={'width': 2048, 'height': 2048, 'resample': False, 'cache': 'report'})
+    assert resp.json['cached'] == [False]
+    resp = server.request(
+        path='/item/%s/tiles/histogram' % itemId,
+        params={'width': 2048, 'height': 2048, 'resample': False, 'cache': 'schedule'})
+    assert 'scheduledJob' in resp.json
+
+    job = Job().load(resp.json['scheduledJob'], force=True)
+    while job['status'] not in (JobStatus.SUCCESS, JobStatus.ERROR, JobStatus.CANCELED):
+        time.sleep(0.1)
+        job = Job().load(job['_id'], force=True)
+    resp = server.request(
+        path='/item/%s/tiles/histogram' % itemId,
+        params={'width': 2048, 'height': 2048, 'resample': False, 'cache': 'report'})
+    assert resp.json['cached'] == [True]
+
+
 @pytest.mark.usefixtures('unbindLargeImage')
 @pytest.mark.plugin('large_image')
 def testTilesInternalMetadata(server, admin, fsAssetstore):

utilities/tasks/large_image_tasks/tasks.py

@@ -191,3 +191,29 @@ def cache_tile_frames_job(job):
         logger.exception('Failed caching tile frames')
         job = Job().updateJob(
             job, log='Failed caching tile frames (%s)\n' % exc, status=JobStatus.ERROR)
+
+
+def cache_histograms_job(job):
+    from girder_jobs.constants import JobStatus
+    from girder_jobs.models.job import Job
+    from girder_large_image.models.image_item import ImageItem
+
+    from girder import logger
+
+    kwargs = job['kwargs']
+    item = ImageItem().load(kwargs.pop('itemId'), force=True)
+    job = Job().updateJob(
+        job, log='Started caching histograms\n',
+        status=JobStatus.RUNNING)
+    try:
+        for entry in kwargs.get('histogramList'):
+            job = Job().load(job['_id'], force=True)
+            if job['status'] == JobStatus.CANCELED:
+                return
+            job = Job().updateJob(job, log='Caching %r\n' % entry)
+            ImageItem().histogram(item, checkAndCreate=True, **entry)
+        job = Job().updateJob(job, log='Finished caching histograms\n', status=JobStatus.SUCCESS)
+    except Exception as exc:
+        logger.exception('Failed caching histograms')
+        job = Job().updateJob(
+            job, log='Failed caching histograms (%s)\n' % exc, status=JobStatus.ERROR)