diff --git a/CHANGELOG.md b/CHANGELOG.md index 40c661dad..23784fa76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## 1.23.6 + +### Improvements +- Allow scheduling histogram computation in Girder ([#1282](../../pull/1282)) + ## 1.23.5 ### Improvements diff --git a/girder/girder_large_image/models/image_item.py b/girder/girder_large_image/models/image_item.py index f17eb27b4..ef3f93182 100644 --- a/girder/girder_large_image/models/image_item.py +++ b/girder/girder_large_image/models/image_item.py @@ -550,9 +550,9 @@ def tileFrames(self, item, checkAndCreate='nosave', **kwargs): :param item: the item with the tile source. :param checkAndCreate: if False, use the cache. If True and the result - is already cached, just return True. If is does not, create, - cache, and return it. If 'nosave', return values from the cache, - but do not store new results in the cache. + is already cached, just return True. If is not, create, cache, and + return it. If 'nosave', return values from the cache, but do not + store new results in the cache. :param kwargs: optional arguments. Some options are left, top, right, bottom, regionWidth, regionHeight, units, width, height, encoding, jpegQuality, jpegSubsampling, and tiffCompression. This @@ -577,7 +577,7 @@ def getPixel(self, item, **kwargs): tileSource = self._loadTileSource(item, **kwargs) return tileSource.getPixel(**kwargs) - def histogram(self, item, **kwargs): + def histogram(self, item, checkAndCreate=False, **kwargs): """ Using a tile source, get a histogram of the image. @@ -592,8 +592,10 @@ def histogram(self, item, **kwargs): else: imageKey = 'histogram' result = self._getAndCacheImageOrData( - item, 'histogram', False, dict(kwargs, imageKey=imageKey), - pickleCache=True, **kwargs)[0] + item, 'histogram', checkAndCreate, + dict(kwargs, imageKey=imageKey), pickleCache=True, **kwargs) + if not isinstance(result, bool): + result = result[0] return result def getBandInformation(self, item, statistics=True, **kwargs): @@ -667,3 +669,28 @@ def _scheduleTileFrames(self, item, tileFramesList, user): ) Job().scheduleJob(job) return job + + def _scheduleHistograms(self, item, histogramList, user): + """ + Schedule generating histograms in a local job. + + :param item: the item. + :param histogramList: a list of dictionary of parameters to pass to + the histogram method. + :param user: the user owning the job. + """ + job = Job().createLocalJob( + module='large_image_tasks.tasks', + function='cache_histograms_job', + kwargs={ + 'itemId': str(item['_id']), + 'histogramList': histogramList, + }, + title='Cache Histograms', + type='large_image_cache_histograms', + user=user, + public=True, + asynchronous=True, + ) + Job().scheduleJob(job) + return job diff --git a/girder/girder_large_image/rest/tiles.py b/girder/girder_large_image/rest/tiles.py index 15191afe0..45eeb6e97 100644 --- a/girder/girder_large_image/rest/tiles.py +++ b/girder/girder_large_image/rest/tiles.py @@ -973,6 +973,28 @@ def getTilesPixel(self, item, params): raise RestException('Value Error: %s' % e.args[0]) return pixel + def _cacheHistograms(self, item, histRange, cache, params): + needed = [] + result = {'cached': []} + tilesource = self.imageItemModel._loadTileSource(item, **params) + for frame in range(tilesource.frames): + if histRange is not None and histRange != 'round': + continue + checkParams = params.copy() + checkParams['range'] = histRange + if tilesource.frames > 1: + checkParams['frame'] = frame + else: + checkParams.pop('frame', None) + result['cached'].append(self.imageItemModel.histogram( + item, checkAndCreate='check', **checkParams)) + if not result['cached'][-1]: + needed.append(checkParams) + if cache == 'schedule' and not all(result['cached']): + result['scheduledJob'] = str(self.imageItemModel._scheduleHistograms( + item, needed, self.getCurrentUser())['_id']) + return result + @describeRoute( Description('Get a histogram for any region of a large image item.') .notes('This can take all of the parameters as the region endpoint, ' @@ -1010,6 +1032,10 @@ def getTilesPixel(self, item, params): required=False, dataType='boolean', default=False) .param('density', 'If true, scale the results by the number of ' 'samples.', required=False, dataType='boolean', default=False) + .param('cache', 'Report on or request caching the specified histogram ' + 'for all frames. Scheduling creates a local job.', + required=False, + enum=['none', 'report', 'schedule']) .errorResponse('ID was invalid.') .errorResponse('Read access was denied for the item.', 403), ) @@ -1053,9 +1079,13 @@ def getHistogram(self, item, params): if params.get('roundRange'): if params.pop('roundRange', False) and histRange is None: histRange = 'round' + + cache = params.pop('cache', None) + if cache in {'report', 'schedule'}: + return self._cacheHistograms(item, histRange, cache, params) result = self.imageItemModel.histogram(item, range=histRange, **params) result = result['histogram'] - # Cast everything to lists and floats so json with encode properly + # Cast everything to lists and floats so json will encode properly for entry in result: for key in {'bin_edges', 'hist', 'range'}: if key in entry: diff --git a/girder/test_girder/test_tiles_rest.py b/girder/test_girder/test_tiles_rest.py index 8946edd03..6f414c125 100644 --- a/girder/test_girder/test_tiles_rest.py +++ b/girder/test_girder/test_tiles_rest.py @@ -1251,6 +1251,31 @@ def testTilesHistogramWithRange(server, admin, fsAssetstore): assert resp.json[1]['samples'] < 1000000 +@pytest.mark.usefixtures('unbindLargeImage') +@pytest.mark.plugin('large_image') +def testTilesHistogramCachingJob(server, admin, fsAssetstore): + file = utilities.uploadExternalFile( + 'sample_image.ptif', admin, fsAssetstore) + itemId = str(file['itemId']) + resp = server.request( + path='/item/%s/tiles/histogram' % itemId, + params={'width': 2048, 'height': 2048, 'resample': False, 'cache': 'report'}) + assert resp.json['cached'] == [False] + resp = server.request( + path='/item/%s/tiles/histogram' % itemId, + params={'width': 2048, 'height': 2048, 'resample': False, 'cache': 'schedule'}) + assert 'scheduledJob' in resp.json + + job = Job().load(resp.json['scheduledJob'], force=True) + while job['status'] not in (JobStatus.SUCCESS, JobStatus.ERROR, JobStatus.CANCELED): + time.sleep(0.1) + job = Job().load(job['_id'], force=True) + resp = server.request( + path='/item/%s/tiles/histogram' % itemId, + params={'width': 2048, 'height': 2048, 'resample': False, 'cache': 'report'}) + assert resp.json['cached'] == [True] + + @pytest.mark.usefixtures('unbindLargeImage') @pytest.mark.plugin('large_image') def testTilesInternalMetadata(server, admin, fsAssetstore): diff --git a/utilities/tasks/large_image_tasks/tasks.py b/utilities/tasks/large_image_tasks/tasks.py index 2c7414608..6a1eb5a3c 100644 --- a/utilities/tasks/large_image_tasks/tasks.py +++ b/utilities/tasks/large_image_tasks/tasks.py @@ -191,3 +191,29 @@ def cache_tile_frames_job(job): logger.exception('Failed caching tile frames') job = Job().updateJob( job, log='Failed caching tile frames (%s)\n' % exc, status=JobStatus.ERROR) + + +def cache_histograms_job(job): + from girder_jobs.constants import JobStatus + from girder_jobs.models.job import Job + from girder_large_image.models.image_item import ImageItem + + from girder import logger + + kwargs = job['kwargs'] + item = ImageItem().load(kwargs.pop('itemId'), force=True) + job = Job().updateJob( + job, log='Started caching histograms\n', + status=JobStatus.RUNNING) + try: + for entry in kwargs.get('histogramList'): + job = Job().load(job['_id'], force=True) + if job['status'] == JobStatus.CANCELED: + return + job = Job().updateJob(job, log='Caching %r\n' % entry) + ImageItem().histogram(item, checkAndCreate=True, **entry) + job = Job().updateJob(job, log='Finished caching histograms\n', status=JobStatus.SUCCESS) + except Exception as exc: + logger.exception('Failed caching histograms') + job = Job().updateJob( + job, log='Failed caching histograms (%s)\n' % exc, status=JobStatus.ERROR)