Fix parallel buffered slice writer bug

webknossos/Changelog.md

@@ -19,6 +19,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 ### Changed
 
 ### Fixed
+- Fixed a bug where parallel access to the properties json leads to an JsonDecodeError in the webknossos CLI [#919](https://github.com/scalableminds/webknossos-libs/issues/919)
 
 
 ## [0.13.1](https://github.com/scalableminds/webknossos-libs/releases/tag/v0.13.1) - 2023-07-17

webknossos/webknossos/cli/convert.py

@@ -96,7 +96,7 @@ def main(
             source,
             target,
             voxel_size,
-            name=name,
+            name,
             data_format=data_format,
             executor=executor,
             compress=compress,

webknossos/webknossos/cli/main.py

@@ -17,7 +17,7 @@
     upsample,
 )
 
-app = typer.Typer(no_args_is_help=True)
+app = typer.Typer(no_args_is_help=True, pretty_exceptions_short=False)
 
 app.command("check-equality")(check_equality.main)
 app.command("compress")(compress.main)

webknossos/webknossos/dataset/_utils/buffered_slice_writer.py

@@ -9,6 +9,7 @@
 import numpy as np
 import psutil
 
+from webknossos.dataset.mag_view import MagView
 from webknossos.geometry import Vec3Int, Vec3IntLike
 
 if TYPE_CHECKING:
@@ -34,6 +35,9 @@ def __init__(
         self,
         view: "View",
         offset: Optional[Vec3IntLike] = None,
+        # update_bbox enables the update of the bounding box and rewriting of the properties json.
+        # It should be False when parallel access is intended.
+        update_bbox: bool = True,
         # buffer_size specifies, how many slices should be aggregated until they are flushed.
         buffer_size: int = 32,
         dimension: int = 2,  # z
@@ -48,6 +52,7 @@ def __init__(
         self.buffer_size = buffer_size
         self.dtype = self.view.get_dtype()
         self.use_logging = use_logging
+        self.update_bbox = update_bbox
         if offset is None and relative_offset is None and absolute_offset is None:
             relative_offset = Vec3Int.zeros()
         if offset is not None:
@@ -124,12 +129,22 @@ def _write_buffer(self) -> None:
             buffer_start_list[self.dimension] = self.buffer_start_slice
             buffer_start = Vec3Int(buffer_start_list)
             buffer_start_mag1 = buffer_start * self.view.mag.to_vec3_int()
-            self.view.write(
-                data,
-                offset=buffer_start.add_or_none(self.offset),
-                relative_offset=buffer_start_mag1.add_or_none(self.relative_offset),
-                absolute_offset=buffer_start_mag1.add_or_none(self.absolute_offset),
-            )
+            if isinstance(self.view, MagView):
+                self.view.write(
+                    data,
+                    offset=buffer_start.add_or_none(self.offset),
+                    relative_offset=buffer_start_mag1.add_or_none(self.relative_offset),
+                    absolute_offset=buffer_start_mag1.add_or_none(self.absolute_offset),
+                    update_bbox=self.update_bbox,
+                )
+            else:
+                self.view.write(
+                    data,
+                    offset=buffer_start.add_or_none(self.offset),
+                    relative_offset=buffer_start_mag1.add_or_none(self.relative_offset),
+                    absolute_offset=buffer_start_mag1.add_or_none(self.absolute_offset),
+                    allow_write_outside_bbox=not self.update_bbox,
+                )
 
         except Exception as exc:
             error(

webknossos/webknossos/dataset/_utils/pims_images.py

@@ -477,6 +477,7 @@ def copy_to_view(
         args: Tuple[int, int],
         mag_view: MagView,
         is_segmentation: bool,
+        update_bbox: bool = False,
         dtype: Optional[DTypeLike] = None,
     ) -> Tuple[Tuple[int, int], Optional[int]]:
         """Copies the images according to the passed arguments to the given mag_view.
@@ -496,6 +497,7 @@ def copy_to_view(
             with mag_view.get_buffered_slice_writer(
                 relative_offset=(0, 0, z_start * mag_view.mag.z),
                 buffer_size=mag_view.info.chunk_shape.z,
+                update_bbox=update_bbox,
             ) as writer:
                 for image_slice in images[z_start:z_end]:
                     image_slice = np.array(image_slice)

webknossos/webknossos/dataset/dataset.py

@@ -1205,6 +1205,7 @@ def add_layer_from_images(
                 mag_view=mag_view,
                 is_segmentation=category == "segmentation",
                 dtype=current_dtype,
+                update_bbox=False,
             )
 
             args = []
@@ -1248,9 +1249,11 @@ def add_layer_from_images(
                     .offset(topleft)
                 )
             if pims_images.expected_shape != actual_size:
-                warnings.warn(
+                logger.info(
                     "[WARNING] Some images are larger than expected, smaller slices are padded with zeros now. "
-                    + f"New size is {actual_size}, expected {pims_images.expected_shape}.",
+                    + "New size is %s, expected %s.",
+                    actual_size,
+                    pims_images.expected_shape,
                 )
             if first_layer is None:
                 first_layer = layer

webknossos/webknossos/dataset/mag_view.py

@@ -145,6 +145,7 @@ def write(
         self,
         data: np.ndarray,
         offset: Optional[Vec3IntLike] = None,  # deprecated, relative, in current mag
+        update_bbox: bool = True,
         *,
         relative_offset: Optional[Vec3IntLike] = None,  # in mag1
         absolute_offset: Optional[Vec3IntLike] = None,  # in mag1
@@ -177,10 +178,14 @@ def write(
 
         # Only update the layer's bbox if we are actually larger
         # than the mag-aligned, rounded up bbox (self.bounding_box):
-        if not self.bounding_box.contains_bbox(mag1_bbox):
+        if update_bbox and not self.bounding_box.contains_bbox(mag1_bbox):
             self.layer.bounding_box = self.layer.bounding_box.extended_by(mag1_bbox)
 
-        super().write(data, absolute_offset=mag1_bbox.topleft)
+        super().write(
+            data,
+            absolute_offset=mag1_bbox.topleft,
+            allow_write_outside_bbox=not update_bbox,
+        )
 
     def read(
         self,

webknossos/webknossos/dataset/view.py

@@ -192,6 +192,7 @@ def write(
         self,
         data: np.ndarray,
         offset: Optional[Vec3IntLike] = None,  # deprecated, relative, in current mag
+        allow_write_outside_bbox: bool = False,
         *,
         relative_offset: Optional[Vec3IntLike] = None,  # in mag1
         absolute_offset: Optional[Vec3IntLike] = None,  # in mag1
@@ -243,9 +244,10 @@ def write(
             abs_mag1_offset=absolute_offset,
             current_mag_size=Vec3Int(data.shape[-3:]),
         )
-        assert self.bounding_box.contains_bbox(
-            mag1_bbox
-        ), f"The bounding box to write {mag1_bbox} is larger than the view's bounding box {self.bounding_box}"
+        if not allow_write_outside_bbox:
+            assert self.bounding_box.contains_bbox(
+                mag1_bbox
+            ), f"The bounding box to write {mag1_bbox} is larger than the view's bounding box {self.bounding_box}"
 
         if len(data.shape) == 4 and data.shape[0] == 1:
             data = data[0]  # remove channel dimension for single-channel data
@@ -633,6 +635,9 @@ def get_buffered_slice_writer(
         offset: Optional[Vec3IntLike] = None,
         buffer_size: int = 32,
         dimension: int = 2,  # z
+        # update_bbox enables the update of the bounding box and rewriting of the properties json.
+        # It should be False when parallel access is intended.
+        update_bbox: bool = True,
         *,
         relative_offset: Optional[Vec3IntLike] = None,  # in mag1
         absolute_offset: Optional[Vec3IntLike] = None,  # in mag1
@@ -674,6 +679,7 @@ def get_buffered_slice_writer(
         return BufferedSliceWriter(
             view=self,
             offset=offset,
+            update_bbox=update_bbox,
             buffer_size=buffer_size,
             dimension=dimension,
             relative_offset=relative_offset,