Memory optimization for image chunk preparation (#8778)

<!-- Raise an issue to propose your change
(https://github.com/cvat-ai/cvat/issues).
It helps to avoid duplication of efforts from multiple independent
contributors.
Discuss your ideas with maintainers to be sure that changes will be
approved and merged.
Read the [Contribution guide](https://docs.cvat.ai/docs/contributing/).
-->

<!-- Provide a general summary of your changes in the Title above -->

### Motivation and context
<!-- Why is this change required? What problem does it solve? If it
fixes an open
issue, please link to the issue here. Describe your changes in detail,
add
screenshots. -->

### How has this been tested?
<!-- Please describe in detail how you tested your changes.
Include details of your testing environment, and the tests you ran to
see how your change affects other areas of the code, etc. -->

### Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply.
If an item isn't applicable for some reason, then ~~explicitly
strikethrough~~ the whole
line. If you don't do that, GitHub will show incorrect progress for the
pull request.
If you're unsure about any of these, don't hesitate to ask. We're here
to help! -->
- [x] I submit my changes into the `develop` branch
- [x] I have created a changelog fragment <!-- see top comment in
CHANGELOG.md -->
- [ ] I have updated the documentation accordingly
- [ ] I have added tests to cover my changes
- [ ] I have linked related issues (see [GitHub docs](

https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword))
- [ ] I have increased versions of npm packages if it is necessary

([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning),

[cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning),

[cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning)
and

[cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning))

### License

- [x] I submit _my code changes_ under the same [MIT License](
https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the
project.
  Feel free to contact the maintainers if that's a concern.


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

- **New Features**
- Enhanced image loading functionality with a new `load_image` feature,
improving handling of various image formats and EXIF rotation.
  - Improved segment preview generation, specifically for 3D segments.

- **Bug Fixes**
- Resolved issues with TIFF image handling, ensuring correct image
rotation and processing.

- **Improvements**
- Refined task creation process with better validation checks and
progress tracking for long-running tasks.
  - Updated utility functions for better type safety and error handling.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

changelog.d/20241205_165408_andrey_memory.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Memory consumption during preparation of image chunks
+  (<https://github.com/cvat-ai/cvat/pull/8778>)

cvat/apps/engine/cache.py

@@ -62,14 +62,10 @@
     VideoReaderWithManifest,
     ZipChunkWriter,
     ZipCompressedChunkWriter,
-)
-from cvat.apps.engine.rq_job_handler import RQJobMetaField
-from cvat.apps.engine.utils import (
-    CvatChunkTimestampMismatchError,
-    get_rq_lock_for_job,
     load_image,
-    md5_hash,
 )
+from cvat.apps.engine.rq_job_handler import RQJobMetaField
+from cvat.apps.engine.utils import CvatChunkTimestampMismatchError, get_rq_lock_for_job, md5_hash
 from utils.dataset_manifest import ImageManifestManager
 
 slogger = ServerLogManager(__name__)

cvat/apps/engine/media_extractors.py

@@ -101,6 +101,12 @@ def image_size_within_orientation(img: Image.Image):
 def has_exif_rotation(img: Image.Image):
     return img.getexif().get(ORIENTATION_EXIF_TAG, ORIENTATION.NORMAL_HORIZONTAL) != ORIENTATION.NORMAL_HORIZONTAL
 
+
+def load_image(image: tuple[str, str, str])-> tuple[Image.Image, str, str]:
+    with Image.open(image[0]) as pil_img:
+        pil_img.load()
+        return pil_img, image[1], image[2]
+
 _T = TypeVar("_T")
 
 
@@ -837,13 +843,15 @@ def _compress_image(source_image: av.VideoFrame | io.IOBase | Image.Image, quali
         if isinstance(source_image, av.VideoFrame):
             image = source_image.to_image()
         elif isinstance(source_image, io.IOBase):
-            with Image.open(source_image) as _img:
-                image = ImageOps.exif_transpose(_img)
+            image, _, _ = load_image((source_image, None, None))
         elif isinstance(source_image, Image.Image):
-            image = ImageOps.exif_transpose(source_image)
+            image = source_image
 
         assert image is not None
 
+        if has_exif_rotation(image):
+            image = ImageOps.exif_transpose(image)
+
         # Ensure image data fits into 8bit per pixel before RGB conversion as PIL clips values on conversion
         if image.mode == "I":
             # Image mode is 32bit integer pixels.
@@ -868,16 +876,14 @@ def _compress_image(source_image: av.VideoFrame | io.IOBase | Image.Image, quali
             image = Image.fromarray(image, mode="L") # 'L' := Unsigned Integer 8, Grayscale
             image = ImageOps.equalize(image)         # The Images need equalization. High resolution with 16-bit but only small range that actually contains information
 
-        converted_image = image.convert('RGB')
+        if image.mode != 'RGB' and image.mode != 'L':
+            image = image.convert('RGB')
 
-        try:
-            buf = io.BytesIO()
-            converted_image.save(buf, format='JPEG', quality=quality, optimize=True)
-            buf.seek(0)
-            width, height = converted_image.size
-            return width, height, buf
-        finally:
-            converted_image.close()
+        buf = io.BytesIO()
+        image.save(buf, format='JPEG', quality=quality, optimize=True)
+        buf.seek(0)
+
+        return image.width, image.height, buf
 
     @abstractmethod
     def save_as_chunk(self, images, chunk_path):

cvat/apps/engine/task.py

@@ -34,12 +34,13 @@
 from cvat.apps.engine.media_extractors import (
     MEDIA_TYPES, CachingMediaIterator, IMediaReader, ImageListReader,
     Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, RandomAccessIterator,
-    ValidateDimension, ZipChunkWriter, ZipCompressedChunkWriter, get_mime, sort
+    ValidateDimension, ZipChunkWriter, ZipCompressedChunkWriter, get_mime, sort,
+    load_image,
 )
 from cvat.apps.engine.models import RequestAction, RequestTarget
 from cvat.apps.engine.utils import (
     av_scan_paths, format_list,get_rq_job_meta,
-    define_dependent_job, get_rq_lock_by_user, load_image
+    define_dependent_job, get_rq_lock_by_user
 )
 from cvat.apps.engine.rq_job_handler import RQId
 from cvat.utils.http import make_requests_session, PROXIES_FOR_UNTRUSTED_URLS

cvat/apps/engine/utils.py

@@ -377,10 +377,6 @@ def sendfile(
 
     return _sendfile(request, filename, attachment, attachment_filename, mimetype, encoding)
 
-def load_image(image: tuple[str, str, str])-> tuple[Image.Image, str, str]:
-    pil_img = Image.open(image[0])
-    pil_img.load()
-    return pil_img, image[1], image[2]
 
 def build_backup_file_name(
     *,