Merge pull request #1500 from girder/vips-new-from-file-lock

Guard a race condition in vips new_from_file

CHANGELOG.md

@@ -5,6 +5,9 @@
 ### Changes
 - Improve algorithm sweep example options ([#1498](../../pull/1498))
 
+### Bug Fixes
+- Guard a race condition in vips new_from_file ([#1500](../../pull/1500))
+
 ## 1.28.0
 
 ### Features

large_image/tilesource/utilities.py

@@ -1,6 +1,7 @@
 import io
 import math
 import os
+import threading
 import types
 import xml.etree.ElementTree
 from collections import defaultdict
@@ -27,6 +28,11 @@
 # Turn off decompression warning check
 PIL.Image.MAX_IMAGE_PIXELS = None
 
+# This is used by any submodule that uses vips to avoid a race condition in
+# new_from_file.  Since vips is technically optional and the various modules
+# might pull it in independently, it is located here to make is shareable.
+_newFromFileLock = threading.RLock()
+
 # Extend colors so G and GREEN map to expected values.  CSS green is #0080ff,
 # which is unfortunate.
 colormap = {

sources/vips/large_image_source_vips/__init__.py

@@ -16,7 +16,7 @@
                                    dtypeToGValue)
 from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError
 from large_image.tilesource import FileTileSource
-from large_image.tilesource.utilities import _imageToNumpy
+from large_image.tilesource.utilities import _imageToNumpy, _newFromFileLock
 
 logging.getLogger('pyvips').setLevel(logging.ERROR)
 
@@ -72,7 +72,8 @@ def __init__(self, path, **kwargs):
 
         config._ignoreSourceNames('vips', self._largeImagePath)
         try:
-            self._image = pyvips.Image.new_from_file(self._largeImagePath)
+            with _newFromFileLock:
+                self._image = pyvips.Image.new_from_file(self._largeImagePath)
         except pyvips.error.Error:
             if not os.path.isfile(self._largeImagePath):
                 raise TileSourceFileNotFoundError(self._largeImagePath) from None
@@ -87,7 +88,8 @@ def __init__(self, path, **kwargs):
         self._frames = [0]
         for page in range(1, pages):
             subInputPath = self._largeImagePath + '[page=%d]' % page
-            subImage = pyvips.Image.new_from_file(subInputPath)
+            with _newFromFileLock:
+                subImage = pyvips.Image.new_from_file(subInputPath)
             if subImage.width == self.sizeX and subImage.height == self.sizeY:
                 self._frames.append(page)
                 continue
@@ -187,7 +189,8 @@ def _getFrameImage(self, frame=0):
             with self._frameLock:
                 if frame not in self._recentFrames:
                     subpath = self._largeImagePath + '[page=%d]' % self._frames[frame]
-                    img = pyvips.Image.new_from_file(subpath)
+                    with _newFromFileLock:
+                        img = pyvips.Image.new_from_file(subpath)
                     self._recentFrames[frame] = img
                 else:
                     img = self._recentFrames[frame]

utilities/converter/large_image_converter/__init__.py

@@ -18,7 +18,9 @@
 import tifftools
 
 import large_image
-from large_image.tilesource.utilities import _gdalParameters, _vipsCast, _vipsParameters
+from large_image.tilesource.utilities import (_gdalParameters,
+                                              _newFromFileLock, _vipsCast,
+                                              _vipsParameters)
 
 from . import format_aperio
 
@@ -77,7 +79,7 @@ def _data_from_large_image(path, outputPath, **kwargs):
     _import_pyvips()
     if not path.startswith('large_image://test'):
         try:
-            ts = large_image.open(path)
+            ts = large_image.open(path, noCache=True)
         except Exception:
             return
     else:
@@ -164,7 +166,8 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
     """
     _import_pyvips()
 
-    image = pyvips.Image.new_from_file(inputPath)
+    with _newFromFileLock:
+        image = pyvips.Image.new_from_file(inputPath)
     width = image.width
     height = image.height
     pages = 1
@@ -180,7 +183,8 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
     # Process each image separately to pyramidize it
     for page in range(pages):
         subInputPath = inputPath + '[page=%d]' % page
-        subImage = pyvips.Image.new_from_file(subInputPath)
+        with _newFromFileLock:
+            subImage = pyvips.Image.new_from_file(subInputPath)
         imageSizes.append((subImage.width, subImage.height, subInputPath, page))
         if subImage.width != width or subImage.height != height:
             if subImage.width * subImage.height <= width * height:
@@ -274,7 +278,8 @@ def _convert_via_vips(inputPathOrBuffer, outputPath, tempPath, forTiled=True,
         image = pyvips.Image.new_from_buffer(inputPathOrBuffer, '')
     else:
         source = inputPathOrBuffer
-        image = pyvips.Image.new_from_file(inputPathOrBuffer)
+        with _newFromFileLock:
+            image = pyvips.Image.new_from_file(inputPathOrBuffer)
     logger.info('Input: %s, Output: %s, Options: %r%s',
                 source, outputPath, convertParams, status)
     image = image.autorot()
@@ -736,7 +741,8 @@ def _is_multiframe(path):
     """
     _import_pyvips()
     try:
-        image = pyvips.Image.new_from_file(path)
+        with _newFromFileLock:
+            image = pyvips.Image.new_from_file(path)
     except Exception:
         try:
             open(path, 'rb').read(1)
@@ -971,7 +977,8 @@ def is_vips(path):
     """
     _import_pyvips()
     try:
-        image = pyvips.Image.new_from_file(path)
+        with _newFromFileLock:
+            image = pyvips.Image.new_from_file(path)
         # image(0, 0) will throw if vips can't decode the image
         if not image.width or not image.height or image(0, 0) is None:
             return False