save progress

apache · Nov 17, 2024 · b9410c4 · b9410c4
1 parent 75d9a50
commit b9410c4
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/src/shuffle/reader.rs b/src/shuffle/reader.rs
@@ -149,8 +149,14 @@ impl ExecutionPlan for ShuffleReaderExec {
                             )))
                         }
                         Err(e) => {
-                            // TODO need to distinguish between file not existing and other errors
-                            println!("Download failed: {}", e);
+                            let error_message = e.to_string();
+                            if error_message.contains("NotFound")
+                                || error_message.contains("NoSuchKey")
+                            {
+                                // this is fine
+                            } else {
+                                println!("Download failed: {}", e);
+                            }
                             Ok(None)
                         }
                     }

diff --git a/src/shuffle/writer.rs b/src/shuffle/writer.rs
@@ -337,7 +337,7 @@ impl IPCWriter {
 
             let object_store_path = ObjectStorePath::from_filesystem_path(&self.path)?;
 
-            // disabled due to error "Your proposed upload is smaller than the minimum allowed object size"
+            // TODO with multipart, we need to upload in chunks of at least 5 MB
             let multi_part_enabled = false;
 
             // TODO make threshold configurable