From 0ddf05a9aaf8542f364c623644db59a54a4ba431 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Tue, 16 Jul 2024 13:14:31 -0400 Subject: [PATCH 1/6] Add option to reattempt unreadable large images --- .../rest/large_image_resource.py | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index cb39c054e..59f14588f 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -451,7 +451,7 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): @autoDescribeRoute( Description('Create new large images for all items within a folder.') .notes('Does not work for items with multiple files and skips over items with ' - 'existing or unfinished large images.') + 'existing large images.') .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) .param('force', 'Whether creation job(s) should be forced for each large image.', @@ -460,6 +460,11 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): default=False, dataType='boolean') .param('recurse', 'Whether child folders should be recursed.', required=False, default=False, dataType='boolean') + .param('reattempt', 'Whether unreadable large images should be removed and ' + 'reattempted. This will cancel unfinished large image creation jobs ' + 'related to items in the folder. If false, items with unreadable ' + 'large images will be skipped.', required=False, default=False, + dataType='boolean') .errorResponse('ID was invalid.') .errorResponse('Write access was denied for the folder.', 403), ) @@ -468,39 +473,59 @@ def createLargeImages(self, folder, params): createJobs = 'always' if self.boolParam('force', params, default=False) else True return self.createImagesRecurseOption(folder=folder, createJobs=createJobs, user=user, recurse=params.get('recurse'), - localJobs=params.get('localJobs')) + localJobs=params.get('localJobs'), + reattempt=params.get('reattempt')) - def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs): + def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs, reattempt): result = {'childFoldersRecursed': 0, 'itemsSkipped': 0, + 'jobsCancelled': 0, 'largeImagesCreated': 0, - 'largeImagesRemovedAndRecreated': 0, + 'largeImagesRemovedAndReattempted': 0, 'totalItems': 0} if recurse: for childFolder in Folder().childFolders(parent=folder, parentType='folder'): result['childFoldersRecursed'] += 1 childResult = self.createImagesRecurseOption(folder=childFolder, createJobs=createJobs, user=user, - recurse=recurse, localJobs=localJobs) + recurse=recurse, localJobs=localJobs, + reattempt=reattempt) for key in childResult: result[key] += childResult[key] for item in Folder().childItems(folder=folder): result['totalItems'] += 1 if item.get('largeImage'): if item['largeImage'].get('expected'): - result['itemsSkipped'] += 1 + if reattempt: + job = Job().load(item['largeImage']['jobId'], force=True) + if job and job.get('status') in ( + JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.INACTIVE): + job = Job().cancelJob(job) + result['jobsCancelled'] += 1 + previousFileId = item['largeImage'].get('originalId') + ImageItem().delete(item) + ImageItem().createImageItem(item, + File().load(user=user, id=previousFileId), + createJob=createJobs, localJob=localJobs) + result['largeImagesRemovedAndReattempted'] += 1 + else: + result['itemsSkipped'] += 1 else: try: ImageItem().getMetadata(item) result['itemsSkipped'] += 1 continue except (TileSourceError, KeyError): - previousFileId = item['largeImage'].get('originalId', - item['largeImage']['fileId']) - ImageItem().delete(item) - ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), - createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndRecreated'] += 1 + if reattempt: + previousFileId = item['largeImage'].get('originalId', + item['largeImage']['fileId']) + ImageItem().delete(item) + ImageItem().createImageItem(item, + File().load(user=user, id=previousFileId), + createJob=createJobs, localJob=localJobs) + result['largeImagesRemovedAndReattempted'] += 1 + else: + result['itemsSkipped'] += 1 else: files = list(Item().childFiles(item=item, limit=2)) if len(files) == 1: From 07c2adbf6e331385a1b2229b591109633f1ca414 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Fri, 19 Jul 2024 11:04:07 -0400 Subject: [PATCH 2/6] Remove reattempt option & add option to cancel pending creation jobs --- .../rest/large_image_resource.py | 80 ++++++++++--------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index 59f14588f..de4961733 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -450,64 +450,69 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): @access.user(scope=TokenScope.DATA_WRITE) @autoDescribeRoute( Description('Create new large images for all items within a folder.') - .notes('Does not work for items with multiple files and skips over items with ' - 'existing large images.') + .notes('Does not work for items with multiple files.') .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) .param('force', 'Whether creation job(s) should be forced for each large image.', required=False, default=False, dataType='boolean') - .param('localJobs', 'Whether the job(s) created should be local.', required=False, - default=False, dataType='boolean') + .param('localJobs', 'Whether the job(s) created should be local.', + required=False, default=False, dataType='boolean') .param('recurse', 'Whether child folders should be recursed.', required=False, default=False, dataType='boolean') - .param('reattempt', 'Whether unreadable large images should be removed and ' - 'reattempted. This will cancel unfinished large image creation jobs ' - 'related to items in the folder. If false, items with unreadable ' - 'large images will be skipped.', required=False, default=False, - dataType='boolean') + .param('cancelJobs', 'Whether unfinished large image job(s) associated with ' + 'items in the folder should be canceled, then each large image ' + 'reattempted. If false, items with an unfinished large image will ' + 'be skipped.', required=False, default=False, dataType='boolean') .errorResponse('ID was invalid.') .errorResponse('Write access was denied for the folder.', 403), ) def createLargeImages(self, folder, params): user = self.getCurrentUser() createJobs = 'always' if self.boolParam('force', params, default=False) else True - return self.createImagesRecurseOption(folder=folder, createJobs=createJobs, user=user, - recurse=params.get('recurse'), - localJobs=params.get('localJobs'), - reattempt=params.get('reattempt')) + return self.createLargeImagesRecurse(folder=folder, createJobs=createJobs, user=user, + recurse=params.get('recurse'), + localJobs=params.get('localJobs'), + cancelJobs=params.get('cancelJobs')) - def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs, reattempt): + def createLargeImagesRecurse(self, folder, createJobs, user, recurse, localJobs, cancelJobs): result = {'childFoldersRecursed': 0, 'itemsSkipped': 0, - 'jobsCancelled': 0, + 'jobsCanceled': 0, + 'jobsFailedToCancel': 0, 'largeImagesCreated': 0, 'largeImagesRemovedAndReattempted': 0, 'totalItems': 0} if recurse: for childFolder in Folder().childFolders(parent=folder, parentType='folder'): result['childFoldersRecursed'] += 1 - childResult = self.createImagesRecurseOption(folder=childFolder, - createJobs=createJobs, user=user, - recurse=recurse, localJobs=localJobs, - reattempt=reattempt) + childResult = self.createLargeImagesRecurse(folder=childFolder, + createJobs=createJobs, user=user, + recurse=recurse, localJobs=localJobs, + cancelJobs=cancelJobs) for key in childResult: result[key] += childResult[key] for item in Folder().childItems(folder=folder): result['totalItems'] += 1 if item.get('largeImage'): if item['largeImage'].get('expected'): - if reattempt: + if cancelJobs: job = Job().load(item['largeImage']['jobId'], force=True) - if job and job.get('status') in ( - JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.INACTIVE): + if job and job.get('status') in (JobStatus.QUEUED, JobStatus.RUNNING, + JobStatus.INACTIVE): job = Job().cancelJob(job) - result['jobsCancelled'] += 1 - previousFileId = item['largeImage'].get('originalId') - ImageItem().delete(item) - ImageItem().createImageItem(item, - File().load(user=user, id=previousFileId), - createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndReattempted'] += 1 + if job and job.get('status') in (JobStatus.QUEUED, JobStatus.RUNNING, + JobStatus.INACTIVE): + result['jobsFailedToCancel'] += 1 + result['itemsSkipped'] += 1 + continue + else: + result['jobsCanceled'] += 1 + previousFileId = item['largeImage'].get('originalId') + ImageItem().delete(item) + ImageItem().createImageItem(item, + File().load(user=user, id=previousFileId), + createJob=createJobs, localJob=localJobs) + result['largeImagesRemovedAndReattempted'] += 1 else: result['itemsSkipped'] += 1 else: @@ -516,16 +521,13 @@ def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs result['itemsSkipped'] += 1 continue except (TileSourceError, KeyError): - if reattempt: - previousFileId = item['largeImage'].get('originalId', - item['largeImage']['fileId']) - ImageItem().delete(item) - ImageItem().createImageItem(item, - File().load(user=user, id=previousFileId), - createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndReattempted'] += 1 - else: - result['itemsSkipped'] += 1 + previousFileId = item['largeImage'].get('originalId', + item['largeImage']['fileId']) + ImageItem().delete(item) + ImageItem().createImageItem(item, + File().load(user=user, id=previousFileId), + createJob=createJobs, localJob=localJobs) + result['largeImagesRemovedAndReattempted'] += 1 else: files = list(Item().childFiles(item=item, limit=2)) if len(files) == 1: From d6f7af7994a15b52ece9bd86796b4193b31a9805 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Mon, 22 Jul 2024 11:44:41 -0400 Subject: [PATCH 3/6] Allow using job(s) without forcing --- .../rest/large_image_resource.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index de4961733..f17979e60 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -453,8 +453,10 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): .notes('Does not work for items with multiple files.') .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) - .param('force', 'Whether creation job(s) should be forced for each large image.', - required=False, default=False, dataType='boolean') + .param('createJobs', 'Whether job(s) should be used to create each large image. ' + 'Select true to use a job if needed, false to never use a job, ' + 'and always to always use a job.', required=False, default='false', + dataType='string', enum=['true', 'false', 'always']) .param('localJobs', 'Whether the job(s) created should be local.', required=False, default=False, dataType='boolean') .param('recurse', 'Whether child folders should be recursed.', required=False, @@ -468,13 +470,15 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): ) def createLargeImages(self, folder, params): user = self.getCurrentUser() - createJobs = 'always' if self.boolParam('force', params, default=False) else True - return self.createLargeImagesRecurse(folder=folder, createJobs=createJobs, user=user, - recurse=params.get('recurse'), + createJobs = params.get('createJobs') + if createJobs in ['true', 'false']: + createJobs = self.boolParam('createJobs', params, default=False) + return self.createLargeImagesRecurse(folder=folder, user=user, + recurse=params.get('recurse'), createJobs=createJobs, localJobs=params.get('localJobs'), cancelJobs=params.get('cancelJobs')) - def createLargeImagesRecurse(self, folder, createJobs, user, recurse, localJobs, cancelJobs): + def createLargeImagesRecurse(self, folder, user, recurse, createJobs, localJobs, cancelJobs): result = {'childFoldersRecursed': 0, 'itemsSkipped': 0, 'jobsCanceled': 0, @@ -485,9 +489,9 @@ def createLargeImagesRecurse(self, folder, createJobs, user, recurse, localJobs, if recurse: for childFolder in Folder().childFolders(parent=folder, parentType='folder'): result['childFoldersRecursed'] += 1 - childResult = self.createLargeImagesRecurse(folder=childFolder, - createJobs=createJobs, user=user, - recurse=recurse, localJobs=localJobs, + childResult = self.createLargeImagesRecurse(folder=childFolder, user=user, + recurse=recurse, createJobs=createJobs, + localJobs=localJobs, cancelJobs=cancelJobs) for key in childResult: result[key] += childResult[key] From 139359ccc956f79b2bd7b6abfe64621a9fb6feff Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Tue, 23 Jul 2024 16:25:41 -0400 Subject: [PATCH 4/6] Add option to redo existing large images --- .../rest/large_image_resource.py | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index f17979e60..03e25d7a2 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -449,22 +449,26 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): @access.user(scope=TokenScope.DATA_WRITE) @autoDescribeRoute( - Description('Create new large images for all items within a folder.') + Description('Create large images for all items within a folder.') .notes('Does not work for items with multiple files.') - .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, - required=True) - .param('createJobs', 'Whether job(s) should be used to create each large image. ' - 'Select true to use a job if needed, false to never use a job, ' - 'and always to always use a job.', required=False, default='false', - dataType='string', enum=['true', 'false', 'always']) + .modelParam('id', 'The ID of the folder.', model=Folder, + level=AccessType.WRITE, required=True) + .param('createJobs', 'Whether job(s) should be used to create each ' + 'large image. Select true to use a job if needed, false to ' + 'never use a job, and always to always use a job.', + required=False, default='false', dataType='string', + enum=['true', 'false', 'always']) .param('localJobs', 'Whether the job(s) created should be local.', required=False, default=False, dataType='boolean') - .param('recurse', 'Whether child folders should be recursed.', required=False, - default=False, dataType='boolean') - .param('cancelJobs', 'Whether unfinished large image job(s) associated with ' - 'items in the folder should be canceled, then each large image ' - 'reattempted. If false, items with an unfinished large image will ' - 'be skipped.', required=False, default=False, dataType='boolean') + .param('recurse', 'Whether child folders should be recursed.', + required=False, default=False, dataType='boolean') + .param('cancelJobs', 'Whether unfinished large image job(s) associated ' + 'with items in the folder should be canceled, then a new large ' + 'image created. If false, items with an unfinished large image ' + 'will be skipped.', required=False, default=False, + dataType='boolean') + .param('redoExisting', 'Whether existing large images should be removed and ' + 'recreated.', required=False, default=False, dataType='boolean') .errorResponse('ID was invalid.') .errorResponse('Write access was denied for the folder.', 403), ) @@ -476,15 +480,17 @@ def createLargeImages(self, folder, params): return self.createLargeImagesRecurse(folder=folder, user=user, recurse=params.get('recurse'), createJobs=createJobs, localJobs=params.get('localJobs'), - cancelJobs=params.get('cancelJobs')) + cancelJobs=params.get('cancelJobs'), + redo=params.get('redoExisting')) - def createLargeImagesRecurse(self, folder, user, recurse, createJobs, localJobs, cancelJobs): + def createLargeImagesRecurse( + self, folder, user, recurse, createJobs, localJobs, cancelJobs, redo): result = {'childFoldersRecursed': 0, 'itemsSkipped': 0, 'jobsCanceled': 0, 'jobsFailedToCancel': 0, 'largeImagesCreated': 0, - 'largeImagesRemovedAndReattempted': 0, + 'largeImagesRemovedAndRecreated': 0, 'totalItems': 0} if recurse: for childFolder in Folder().childFolders(parent=folder, parentType='folder'): @@ -492,7 +498,7 @@ def createLargeImagesRecurse(self, folder, user, recurse, createJobs, localJobs, childResult = self.createLargeImagesRecurse(folder=childFolder, user=user, recurse=recurse, createJobs=createJobs, localJobs=localJobs, - cancelJobs=cancelJobs) + cancelJobs=cancelJobs, redo=redo) for key in childResult: result[key] += childResult[key] for item in Folder().childItems(folder=folder): @@ -508,7 +514,6 @@ def createLargeImagesRecurse(self, folder, user, recurse, createJobs, localJobs, JobStatus.INACTIVE): result['jobsFailedToCancel'] += 1 result['itemsSkipped'] += 1 - continue else: result['jobsCanceled'] += 1 previousFileId = item['largeImage'].get('originalId') @@ -516,14 +521,22 @@ def createLargeImagesRecurse(self, folder, user, recurse, createJobs, localJobs, ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndReattempted'] += 1 + result['largeImagesRemovedAndRecreated'] += 1 else: result['itemsSkipped'] += 1 else: try: ImageItem().getMetadata(item) - result['itemsSkipped'] += 1 - continue + if redo: + previousFileId = item['largeImage'].get('originalId', + item['largeImage']['fileId']) + ImageItem().delete(item) + ImageItem().createImageItem(item, + File().load(user=user, id=previousFileId), + createJob=createJobs, localJob=localJobs) + result['largeImagesRemovedAndRecreated'] += 1 + else: + result['itemsSkipped'] += 1 except (TileSourceError, KeyError): previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) @@ -531,7 +544,7 @@ def createLargeImagesRecurse(self, folder, user, recurse, createJobs, localJobs, ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndReattempted'] += 1 + result['largeImagesRemovedAndRecreated'] += 1 else: files = list(Item().childFiles(item=item, limit=2)) if len(files) == 1: From e22df9496ed745dc055801bb6328d64c75afcc15 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Thu, 25 Jul 2024 10:23:51 -0400 Subject: [PATCH 5/6] Tweak parameter descriptions --- .../rest/large_image_resource.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index 03e25d7a2..f7020fefd 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -453,22 +453,22 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): .notes('Does not work for items with multiple files.') .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) - .param('createJobs', 'Whether job(s) should be used to create each ' - 'large image. Select true to use a job if needed, false to ' - 'never use a job, and always to always use a job.', - required=False, default='false', dataType='string', - enum=['true', 'false', 'always']) - .param('localJobs', 'Whether the job(s) created should be local.', - required=False, default=False, dataType='boolean') - .param('recurse', 'Whether child folders should be recursed.', - required=False, default=False, dataType='boolean') - .param('cancelJobs', 'Whether unfinished large image job(s) associated ' - 'with items in the folder should be canceled, then a new large ' - 'image created. If false, items with an unfinished large image ' - 'will be skipped.', required=False, default=False, - dataType='boolean') - .param('redoExisting', 'Whether existing large images should be removed and ' - 'recreated.', required=False, default=False, dataType='boolean') + .param('createJobs', 'If true, a job will be used to create the image ' + 'when needed; if always, a job will always be used; if false, a ' + 'job will never be used.', dataType='string', default='false', + required=False, enum=['true', 'false', 'always']) + .param('localJobs', 'If true, run each creation job locally; if false, ' + 'run via the remote worker.', dataType='boolean', default='false', + required=False) + .param('recurse', 'If true, items in child folders will also be checked.', + dataType='boolean', default=False, required=False) + .param('cancelJobs', 'If true, unfinished large image job(s) associated ' + 'with items in the folder will be canceled, then a new large ' + 'image created; if false, items with an unfinished large image ' + 'will be skipped.', dataType='boolean', default=False, required=False) + .param('redoExisting', 'If true, existing large images should be removed and ' + 'recreated. Otherwise they will be skipped.', dataType='boolean', + default=False, required=False) .errorResponse('ID was invalid.') .errorResponse('Write access was denied for the folder.', 403), ) From 9e3861668ca9217d067d113133f12a8597a49de0 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Tue, 6 Aug 2024 15:18:24 -0400 Subject: [PATCH 6/6] Refactor for clarity and to reduce code duplication --- .../rest/large_image_resource.py | 152 +++++++++--------- 1 file changed, 75 insertions(+), 77 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index 7292d415c..d600499c5 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -41,6 +41,7 @@ from girder.models.folder import Folder from girder.models.item import Item from girder.models.setting import Setting +from girder.utility import toBool from large_image import cache_util from large_image.exceptions import TileGeneralError, TileSourceError @@ -450,12 +451,13 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): @access.user(scope=TokenScope.DATA_WRITE) @autoDescribeRoute( Description('Create large images for all items within a folder.') - .notes('Does not work for items with multiple files.') + .notes('Does not work for new items with multiple files.') .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) .param('createJobs', 'If true, a job will be used to create the image ' - 'when needed; if always, a job will always be used; if false, a ' - 'job will never be used.', dataType='string', default='false', + 'when needed; if always, a job will always be used; if false, ' + 'a job will never be used, creating a version of the image in ' + 'a preferred format.', dataType='string', default='false', required=False, enum=['true', 'false', 'always']) .param('localJobs', 'If true, run each creation job locally; if false, ' 'run via the remote worker.', dataType='boolean', default='false', @@ -472,88 +474,84 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): .errorResponse('ID was invalid.') .errorResponse('Write access was denied for the folder.', 403), ) - def createLargeImages(self, folder, params): + def createLargeImages(self, folder, createJobs, localJobs, recurse, cancelJobs, redoExisting): user = self.getCurrentUser() - createJobs = params.get('createJobs') - if createJobs in ['true', 'false']: - createJobs = self.boolParam('createJobs', params, default=False) - return self.createLargeImagesRecurse(folder=folder, user=user, - recurse=params.get('recurse'), createJobs=createJobs, - localJobs=params.get('localJobs'), - cancelJobs=params.get('cancelJobs'), - redo=params.get('redoExisting')) - - def createLargeImagesRecurse( - self, folder, user, recurse, createJobs, localJobs, cancelJobs, redo): - result = {'childFoldersRecursed': 0, - 'itemsSkipped': 0, - 'jobsCanceled': 0, - 'jobsFailedToCancel': 0, - 'largeImagesCreated': 0, - 'largeImagesRemovedAndRecreated': 0, - 'totalItems': 0} + if createJobs != 'always': + createJobs = toBool(createJobs) + return self._createLargeImagesRecurse( + folder=folder, user=user, recurse=recurse, createJobs=createJobs, + localJobs=localJobs, cancelJobs=cancelJobs, redo=redoExisting) + + def _createLargeImagesRecurse( + self, folder, user, recurse, createJobs, localJobs, cancelJobs, + redo, result=None): + if result is None: + result = {'childFoldersRecursed': 0, + 'itemsSkipped': 0, + 'jobsCanceled': 0, + 'jobsFailedToCancel': 0, + 'largeImagesCreated': 0, + 'largeImagesNotCreated': 0, + 'largeImagesRemovedAndRecreated': 0, + 'largeImagesRemovedAndNotRecreated': 0, + 'totalItems': 0} if recurse: for childFolder in Folder().childFolders(parent=folder, parentType='folder'): result['childFoldersRecursed'] += 1 - childResult = self.createLargeImagesRecurse(folder=childFolder, user=user, - recurse=recurse, createJobs=createJobs, - localJobs=localJobs, - cancelJobs=cancelJobs, redo=redo) - for key in childResult: - result[key] += childResult[key] + self.createLargeImagesRecurse( + childFolder, user, recurse, createJobs, localJobs, + cancelJobs, redo, result) for item in Folder().childItems(folder=folder): result['totalItems'] += 1 - if item.get('largeImage'): - if item['largeImage'].get('expected'): - if cancelJobs: - job = Job().load(item['largeImage']['jobId'], force=True) - if job and job.get('status') in (JobStatus.QUEUED, JobStatus.RUNNING, - JobStatus.INACTIVE): - job = Job().cancelJob(job) - if job and job.get('status') in (JobStatus.QUEUED, JobStatus.RUNNING, - JobStatus.INACTIVE): - result['jobsFailedToCancel'] += 1 - result['itemsSkipped'] += 1 - else: - result['jobsCanceled'] += 1 - previousFileId = item['largeImage'].get('originalId') - ImageItem().delete(item) - ImageItem().createImageItem(item, - File().load(user=user, id=previousFileId), - createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndRecreated'] += 1 - else: - result['itemsSkipped'] += 1 - else: - try: - ImageItem().getMetadata(item) - if redo: - previousFileId = item['largeImage'].get('originalId', - item['largeImage']['fileId']) - ImageItem().delete(item) - ImageItem().createImageItem(item, - File().load(user=user, id=previousFileId), - createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndRecreated'] += 1 - else: - result['itemsSkipped'] += 1 - except (TileSourceError, KeyError): - previousFileId = item['largeImage'].get('originalId', - item['largeImage']['fileId']) - ImageItem().delete(item) - ImageItem().createImageItem(item, - File().load(user=user, id=previousFileId), - createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndRecreated'] += 1 + self._createLargeImagesItem( + item, user, createJobs, localJobs, cancelJobs, redo, result) + return result + + def _createLargeImagesItem( + self, item, user, createJobs, localJobs, cancelJobs, redo, result): + if item.get('largeImage'): + previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) + if item['largeImage'].get('expected'): + if not cancelJobs: + result['itemsSkipped'] += 1 + return + job = Job().load(item['largeImage']['jobId'], force=True) + if job and job.get('status') in { + JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.INACTIVE}: + job = Job().cancelJob(job) + if job and job.get('status') in { + JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.INACTIVE}: + result['jobsFailedToCancel'] += 1 + result['itemsSkipped'] += 1 + return + result['jobsCanceled'] += 1 else: - files = list(Item().childFiles(item=item, limit=2)) - if len(files) == 1: - ImageItem().createImageItem(item, files[0], createJob=createJobs, - localJob=localJobs) + try: + ImageItem().getMetadata(item) + if not redo: + result['itemsSkipped'] += 1 + return + except (TileSourceError, KeyError): + pass + ImageItem().delete(item) + try: + ImageItem().createImageItem( + item, File().load(user=user, id=previousFileId), + createJob=createJobs, localJob=localJobs) + result['largeImagesRemovedAndRecreated'] += 1 + except Exception: + result['largeImagesRemovedAndNotRecreated'] += 1 + else: + files = list(Item().childFiles(item=item, limit=2)) + if len(files) == 1: + try: + ImageItem().createImageItem( + item, files[0], createJob=createJobs, localJob=localJobs) result['largeImagesCreated'] += 1 - else: - result['itemsSkipped'] += 1 - return result + except Exception: + result['largeImagesNotCreated'] += 1 + else: + result['itemsSkipped'] += 1 @describeRoute( Description('Remove large images from items where the large image job '