Core: Check MPU threshold (#84)

* Core: Check MPU threshold

* Core: Check MPU threshold

tosfs/consts.py

@@ -23,7 +23,7 @@
 RETRY_NUM = 5
 PART_MIN_SIZE = 5 * 2**20
 PART_MAX_SIZE = 5 * 2**30
-
+MPU_PART_SIZE_THRESHOLD = 4 * 2**20  # 4MB
 FILE_OPERATION_READ_WRITE_BUFFER_SIZE = 5 * 2**20  # 5MB
 PUT_OBJECT_OPERATION_SMALL_FILE_THRESHOLD = 5 * 2**30  # 5GB
 GET_OBJECT_OPERATION_DEFAULT_READ_CHUNK_SIZE = 2**16  # 64KB

tosfs/core.py

@@ -46,6 +46,7 @@
     LS_OPERATION_DEFAULT_MAX_ITEMS,
     MANAGED_COPY_MAX_THRESHOLD,
     MANAGED_COPY_MIN_THRESHOLD,
+    MPU_PART_SIZE_THRESHOLD,
     PART_MAX_SIZE,
     PUT_OBJECT_OPERATION_SMALL_FILE_THRESHOLD,
     RETRY_NUM,
@@ -1991,6 +1992,12 @@ def __init__(
         bucket, key, path_version_id = fs._split_path(path)
         if not key:
             raise ValueError("Attempt to open non key-like path: %s" % path)
+
+        if "r" not in mode and int(block_size) < MPU_PART_SIZE_THRESHOLD:
+            raise ValueError(
+                f"Block size must be >= {MPU_PART_SIZE_THRESHOLD // (2**20)}MB."
+            )
+
         super().__init__(
             fs,
             path,

tosfs/tests/test_tosfs.py

@@ -747,6 +747,18 @@ def test_file_write_mpu(
     )
 
 
+def test_file_write_mpu_threshold_check(
+    tosfs: TosFileSystem, bucket: str, temporary_workspace: str
+):
+    file_name = random_str()
+    content = "a" * 1 * 1024
+    block_size = 4 * 1024
+    with pytest.raises(ValueError, match="Block size must be >= 4MB."), tosfs.open(
+        f"{bucket}/{temporary_workspace}/{file_name}", "w", block_size=block_size
+    ) as f:
+        f.write(content)
+
+
 def test_file_write_append(
     tosfs: TosFileSystem, bucket: str, temporary_workspace: str
 ) -> None: