From dd124fa49215fc94c1578dbf633fe2cbd05beafe Mon Sep 17 00:00:00 2001
From: Github Action <deltars@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:28:16 +0000
Subject: [PATCH] doc update for tag `python-v0.21.0`

---
 404.html                                      |   38 +-
 api/catalog/index.html                        |   38 +-
 .../delta_table_alterer/index.html            |   38 +-
 api/delta_table/delta_table_merger/index.html |   38 +-
 .../delta_table_optimizer/index.html          |   38 +-
 api/delta_table/index.html                    |   38 +-
 api/delta_table/metadata/index.html           |   38 +-
 api/delta_writer/index.html                   |   38 +-
 api/exceptions/index.html                     |   38 +-
 api/schema/index.html                         |   38 +-
 api/storage/index.html                        |   38 +-
 delta-lake-best-practices/index.html          |   38 +-
 delta-lake-big-data-small-data/index.html     |   38 +-
 .../architecture-of-delta-table/index.html    |   46 +-
 .../delta-lake-acid-transactions/index.html   |   38 +-
 .../delta-lake-file-skipping/index.html       |   38 +-
 index.html                                    |   38 +-
 integrations/delta-lake-arrow/index.html      |   38 +-
 integrations/delta-lake-daft/index.html       |   38 +-
 integrations/delta-lake-dagster/index.html    |   38 +-
 integrations/delta-lake-dask/index.html       |   38 +-
 integrations/delta-lake-datafusion/index.html |   38 +-
 integrations/delta-lake-pandas/index.html     |   38 +-
 integrations/delta-lake-polars/index.html     |   38 +-
 integrations/object-storage/adls/index.html   |   38 +-
 integrations/object-storage/gcs/index.html    |   38 +-
 integrations/object-storage/hdfs/index.html   |   38 +-
 .../object-storage/s3-like/index.html         |   38 +-
 integrations/object-storage/s3/index.html     |   38 +-
 search/search_index.json                      |    2 +-
 sitemap.xml                                   |   93 +-
 sitemap.xml.gz                                |  Bin 639 -> 643 bytes
 .../index.html                                |   38 +-
 usage/constraints/index.html                  |   38 +-
 usage/create-delta-lake-table/index.html      |   38 +-
 .../index.html                                |   38 +-
 usage/examining-table/index.html              |   38 +-
 usage/installation/index.html                 |   38 +-
 usage/loading-table/index.html                |   38 +-
 usage/managing-tables/index.html              |   44 +-
 usage/merging-tables/index.html               | 2388 +++++++++++++++++
 usage/optimize/delta-lake-z-order/index.html  |   38 +-
 .../index.html                                |   38 +-
 usage/overview/index.html                     |   38 +-
 usage/querying-delta-tables/index.html        |   44 +-
 usage/read-cdf/index.html                     |   38 +-
 usage/writing/index.html                      |   38 +-
 .../index.html                                |   38 +-
 why-use-delta-lake/index.html                 |   38 +-
 49 files changed, 3798 insertions(+), 415 deletions(-)
 create mode 100644 usage/merging-tables/index.html

diff --git a/404.html b/404.html
index a2af71d0d1..0d630ca03a 100644
--- a/404.html
+++ b/404.html
@@ -528,6 +528,8 @@
         
       
         
+      
+        
       
         
       
@@ -751,6 +753,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="/delta-io/delta-rs/usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="/delta-io/delta-rs/usage/managing-tables/" class="md-nav__link">
         
@@ -791,7 +813,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -807,14 +829,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -893,10 +915,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -907,8 +929,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/catalog/index.html b/api/catalog/index.html
index 8227078fd6..c5e51abbc4 100644
--- a/api/catalog/index.html
+++ b/api/catalog/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/delta_table/delta_table_alterer/index.html b/api/delta_table/delta_table_alterer/index.html
index 33236c048a..89d2c16d16 100644
--- a/api/delta_table/delta_table_alterer/index.html
+++ b/api/delta_table/delta_table_alterer/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/delta_table/delta_table_merger/index.html b/api/delta_table/delta_table_merger/index.html
index 41dcc91fef..feeadb70fa 100644
--- a/api/delta_table/delta_table_merger/index.html
+++ b/api/delta_table/delta_table_merger/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/delta_table/delta_table_optimizer/index.html b/api/delta_table/delta_table_optimizer/index.html
index 561ba8d0e8..ec441fb214 100644
--- a/api/delta_table/delta_table_optimizer/index.html
+++ b/api/delta_table/delta_table_optimizer/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/delta_table/index.html b/api/delta_table/index.html
index ae5f0d64d5..6de4ae3fa6 100644
--- a/api/delta_table/index.html
+++ b/api/delta_table/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/delta_table/metadata/index.html b/api/delta_table/metadata/index.html
index 16f9d750d4..25b7895e72 100644
--- a/api/delta_table/metadata/index.html
+++ b/api/delta_table/metadata/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/delta_writer/index.html b/api/delta_writer/index.html
index bf708451b5..c4fbbdd65c 100644
--- a/api/delta_writer/index.html
+++ b/api/delta_writer/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/exceptions/index.html b/api/exceptions/index.html
index 353538e4c7..037c535b58 100644
--- a/api/exceptions/index.html
+++ b/api/exceptions/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/schema/index.html b/api/schema/index.html
index 9b0ae5ee20..9f6b605095 100644
--- a/api/schema/index.html
+++ b/api/schema/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/api/storage/index.html b/api/storage/index.html
index aed39a2044..9039c1d9f5 100644
--- a/api/storage/index.html
+++ b/api/storage/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/delta-lake-best-practices/index.html b/delta-lake-best-practices/index.html
index 7704b80de3..6b25a83bee 100644
--- a/delta-lake-best-practices/index.html
+++ b/delta-lake-best-practices/index.html
@@ -658,6 +658,8 @@
         
       
         
+      
+        
       
         
       
@@ -881,6 +883,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../usage/managing-tables/" class="md-nav__link">
         
@@ -921,7 +943,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -937,14 +959,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -1023,10 +1045,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1037,8 +1059,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/delta-lake-big-data-small-data/index.html b/delta-lake-big-data-small-data/index.html
index a54542c1d2..b98998d4ec 100644
--- a/delta-lake-big-data-small-data/index.html
+++ b/delta-lake-big-data-small-data/index.html
@@ -616,6 +616,8 @@
         
       
         
+      
+        
       
         
       
@@ -839,6 +841,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../usage/managing-tables/" class="md-nav__link">
         
@@ -879,7 +901,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -895,14 +917,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -981,10 +1003,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -995,8 +1017,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/how-delta-lake-works/architecture-of-delta-table/index.html b/how-delta-lake-works/architecture-of-delta-table/index.html
index 9d8d71be93..26c955925e 100644
--- a/how-delta-lake-works/architecture-of-delta-table/index.html
+++ b/how-delta-lake-works/architecture-of-delta-table/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
@@ -1893,7 +1915,7 @@ <h1 id="architecture-of-a-delta-lake-table">Architecture of a Delta Lake table</
 <h2 id="delta-lake-transaction-examples">Delta Lake transaction examples</h2>
 <p>Start by creating a pandas DataFrame and writing it out to a Delta table.</p>
 <div class="language-python highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
-</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="kn">from</span> <span class="nn">deltalake</span> <span class="kn">import</span> <span class="n">DeltaTable</span><span class="p">,</span> <span class="n">write_deltalake</span>
+</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="kn">from</span> <span class="nn">deltalake</span> <span class="kn">import</span> <span class="n">write_deltalake</span>
 </span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a>
 </span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;num&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;letter&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">,</span> <span class="s2">&quot;c&quot;</span><span class="p">]})</span>
 </span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp/some-table&quot;</span><span class="p">,</span> <span class="n">df</span><span class="p">)</span>
@@ -1958,7 +1980,7 @@ <h2 id="delta-lake-transaction-examples">Delta Lake transaction examples</h2>
 </ul>
 <p>Create another pandas DataFrame and append it to the Delta table to see how this transaction is recorded.</p>
 <div class="language-python highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;num&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="mi">9</span><span class="p">],</span> <span class="s2">&quot;letter&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;dd&quot;</span><span class="p">,</span> <span class="s2">&quot;ee&quot;</span><span class="p">]})</span>
-</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">cwd</span><span class="si">}</span><span class="s2">/tmp/delta-table&quot;</span><span class="p">,</span> <span class="n">df</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;append&quot;</span><span class="p">)</span>
+</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp/some-table&quot;</span><span class="p">,</span> <span class="n">df</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;append&quot;</span><span class="p">)</span>
 </span></code></pre></div>
 <p>Here are the files in storage:</p>
 <div class="language-text highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a>tmp/some-table
@@ -1994,7 +2016,7 @@ <h2 id="delta-lake-transaction-examples">Delta Lake transaction examples</h2>
 <p>The transaction log records that the second file has been persisted in the Delta table.</p>
 <p>Now create a third pandas DataFrame and overwrite the Delta table with the new data.</p>
 <div class="language-python highlight"><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;num&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">11</span><span class="p">,</span> <span class="mi">22</span><span class="p">],</span> <span class="s2">&quot;letter&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;aa&quot;</span><span class="p">,</span> <span class="s2">&quot;bb&quot;</span><span class="p">]})</span>
-</span><span id="__span-6-2"><a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">cwd</span><span class="si">}</span><span class="s2">/tmp/delta-table&quot;</span><span class="p">,</span> <span class="n">df</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;append&quot;</span><span class="p">)</span>
+</span><span id="__span-6-2"><a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp/some-table&quot;</span><span class="p">,</span> <span class="n">df</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;overwrite&quot;</span><span class="p">)</span>
 </span></code></pre></div>
 <p>Here are the files in storage:</p>
 <div class="language-text highlight"><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>tmp/some-table
@@ -2053,7 +2075,7 @@ <h2 id="delta-lake-transaction-examples">Delta Lake transaction examples</h2>
 <h2 id="how-delta-table-operations-differ-from-data-lakes">How Delta table operations differ from data lakes</h2>
 <p>Data lakes consist of data files persisted in storage. They don't have a transaction log that retain metadata about the transactions.</p>
 <p>Data lakes perform transactions differently than Delta tables.</p>
-<p>When you perform an overwrite tranasction with a Delta table, you logically delete the exiting data without physically removing it.</p>
+<p>When you perform an overwrite transaction with a Delta table, you logically delete the exiting data without physically removing it.</p>
 <p>Data lakes don't support logical deletes, so you have to physically delete the data from storage.</p>
 <p>Logical data operations are safer because they can be rolled back if they don't complete successfully. Physically removing data from storage can be dangerous, especially if it's before a transaction is complete.</p>
 <p>We're now ready to look into <a href="../delta-lake-acid-transactions/">Delta Lake ACID transactions</a> in more detail.</p>
diff --git a/how-delta-lake-works/delta-lake-acid-transactions/index.html b/how-delta-lake-works/delta-lake-acid-transactions/index.html
index f3a08d4164..7a8453f769 100644
--- a/how-delta-lake-works/delta-lake-acid-transactions/index.html
+++ b/how-delta-lake-works/delta-lake-acid-transactions/index.html
@@ -541,6 +541,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/how-delta-lake-works/delta-lake-file-skipping/index.html b/how-delta-lake-works/delta-lake-file-skipping/index.html
index a3e3d8d0ae..c0361d0946 100644
--- a/how-delta-lake-works/delta-lake-file-skipping/index.html
+++ b/how-delta-lake-works/delta-lake-file-skipping/index.html
@@ -539,6 +539,8 @@
         
       
         
+      
+        
       
         
       
@@ -762,6 +764,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -802,7 +824,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -818,14 +840,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -904,10 +926,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -918,8 +940,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/index.html b/index.html
index e15bf718ee..ad595a7a2a 100644
--- a/index.html
+++ b/index.html
@@ -539,6 +539,8 @@
         
       
         
+      
+        
       
         
       
@@ -762,6 +764,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="usage/managing-tables/" class="md-nav__link">
         
@@ -802,7 +824,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -818,14 +840,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -904,10 +926,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -918,8 +940,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/delta-lake-arrow/index.html b/integrations/delta-lake-arrow/index.html
index f0458f0570..6242591cbe 100644
--- a/integrations/delta-lake-arrow/index.html
+++ b/integrations/delta-lake-arrow/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/delta-lake-daft/index.html b/integrations/delta-lake-daft/index.html
index cf73fed6d7..e226dbcb50 100644
--- a/integrations/delta-lake-daft/index.html
+++ b/integrations/delta-lake-daft/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/delta-lake-dagster/index.html b/integrations/delta-lake-dagster/index.html
index 8af4b0e690..27a63b90af 100644
--- a/integrations/delta-lake-dagster/index.html
+++ b/integrations/delta-lake-dagster/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/delta-lake-dask/index.html b/integrations/delta-lake-dask/index.html
index 2cab685efa..e4617c877b 100644
--- a/integrations/delta-lake-dask/index.html
+++ b/integrations/delta-lake-dask/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/delta-lake-datafusion/index.html b/integrations/delta-lake-datafusion/index.html
index a8bf0971ce..3fe6c0bf34 100644
--- a/integrations/delta-lake-datafusion/index.html
+++ b/integrations/delta-lake-datafusion/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/delta-lake-pandas/index.html b/integrations/delta-lake-pandas/index.html
index af17a9d2e4..bbfa081b8d 100644
--- a/integrations/delta-lake-pandas/index.html
+++ b/integrations/delta-lake-pandas/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/delta-lake-polars/index.html b/integrations/delta-lake-polars/index.html
index 24622f8214..c73866503e 100644
--- a/integrations/delta-lake-polars/index.html
+++ b/integrations/delta-lake-polars/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/object-storage/adls/index.html b/integrations/object-storage/adls/index.html
index d7010308fa..bae829c2f9 100644
--- a/integrations/object-storage/adls/index.html
+++ b/integrations/object-storage/adls/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/object-storage/gcs/index.html b/integrations/object-storage/gcs/index.html
index f0d755328d..098a1d65c7 100644
--- a/integrations/object-storage/gcs/index.html
+++ b/integrations/object-storage/gcs/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/object-storage/hdfs/index.html b/integrations/object-storage/hdfs/index.html
index df31344b39..d1f4dbb550 100644
--- a/integrations/object-storage/hdfs/index.html
+++ b/integrations/object-storage/hdfs/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/object-storage/s3-like/index.html b/integrations/object-storage/s3-like/index.html
index bb28f1435e..b444c28898 100644
--- a/integrations/object-storage/s3-like/index.html
+++ b/integrations/object-storage/s3-like/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/integrations/object-storage/s3/index.html b/integrations/object-storage/s3/index.html
index d8ea17dd48..8b33ee9132 100644
--- a/integrations/object-storage/s3/index.html
+++ b/integrations/object-storage/s3/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -766,6 +768,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../../usage/managing-tables/" class="md-nav__link">
         
@@ -806,7 +828,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -822,14 +844,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -908,10 +930,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -922,8 +944,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/search/search_index.json b/search/search_index.json
index 6f53bd515d..363888dc8f 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"The deltalake package","text":"<p>This is the documentation for the native Rust/Python implementation of Delta Lake. It is based on the delta-rs Rust library and requires no Spark or JVM dependencies. For the PySpark implementation, see delta-spark instead.</p> <p>This module provides the capability to read, write, and manage Delta Lake tables with Python or Rust without Spark or Java. It uses Apache Arrow under the hood, so is compatible with other Arrow-native or integrated libraries such as pandas, DuckDB, and Polars.</p>"},{"location":"#important-terminology","title":"Important terminology","text":"<ul> <li>\"Rust deltalake\" refers to the Rust API of delta-rs (no Spark dependency)</li> <li>\"Python deltalake\" refers to the Python API of delta-rs (no Spark dependency)</li> <li>\"Delta Spark\" refers to the Scala implementation of the Delta Lake transaction log protocol.  This depends on Spark and Java.</li> </ul>"},{"location":"#why-implement-the-delta-lake-transaction-log-protocol-in-rust-and-scala","title":"Why implement the Delta Lake transaction log protocol in Rust and Scala?","text":"<p>Delta Spark depends on Java and Spark, which is fine for many use cases, but not all Delta Lake users want to depend on these libraries.  delta-rs allows using Delta Lake in Rust or other native projects when using a JVM is often not an option.</p> <p>Python deltalake lets you query Delta tables without depending on Java/Scala.</p> <p>Suppose you want to query a Delta table with pandas on your local machine.  Python deltalake makes it easy to query the table with a simple <code>pip install</code> command - no need to install Java.</p>"},{"location":"#contributing","title":"Contributing","text":"<p>The Delta Lake community welcomes contributors from all developers, regardless of your experience or programming background.</p> <p>You can write Rust code, Python code, documentation, submit bugs, or give talks to the community.  We welcome all of these contributions.</p> <p>Feel free to join our Slack and message us in the #delta-rs channel any time!</p> <p>We value kind communication and building a productive, friendly environment for maximum collaboration and fun.</p>"},{"location":"#project-history","title":"Project history","text":"<p>Check out this video by Denny Lee &amp; QP Hou to learn about the genesis of the delta-rs project:</p>"},{"location":"delta-lake-best-practices/","title":"Delta Lake Best Practices","text":"<p>This page outlines Delta Lake best practices.</p> <p>You should consider several factors to optimize the performance of your Delta tables and minimize costs.</p> <p>The Delta Lake best practices depend on your data ingestion into the Delta table and query patterns.  You must understand your data and how users run queries to best leverage Delta Lake.</p>"},{"location":"delta-lake-best-practices/#compacting-small-files","title":"Compacting small files","text":"<p>Delta tables work best when the files are \u201cright-sized\u201d.  Files that are too small create I/O overhead.  Files that are too large limit the impact of file skipping (a critical query optimization).</p> <p>Delta tables can accumulate a lot of small files, especially if you\u2019re frequently writing small amounts of data.  If your table has many small files, you should run a small compaction operation to consolidate all the tiny files into \u201cright-sized\u201d files.</p> <p>It\u2019s generally best for files in a Delta table to be between 100MB and 1GB, but that can vary based on the overall size of the table and the query patterns.</p> <p>Delta Lake makes it easy to compact the small files.</p>"},{"location":"delta-lake-best-practices/#optimizing-table-layout","title":"Optimizing table layout","text":"<p>You can colocate similar data in the same files to make file skipping more effective.  Delta Lake supports Z Ordering, which can colocate similar data in the same files.</p> <p>Z Ordering can yield impressive performance gains for low-cardinality columns but also works well for high-cardinality columns.  This is an advantage compared to Hive-style partitioning, which is only suitable for low-cardinality columns.</p> <p>You must analyze the most common query patterns and Z Order your dataset based on the columns allowing the most file skipping.  The ability to colocate data in the same files decreases when you add more Z Order columns.</p> <p>Let\u2019s look at Hive-style partitioning, another way to colocate data in the same files.  You can also use Hive-style partitioning in conjunction with Z Ordering.</p>"},{"location":"delta-lake-best-practices/#partitioning-datasets","title":"Partitioning datasets","text":"<p>You can partition your Delta tables, which separates the data by one or more partition keys into separate folders.  Partitioning can be an excellent performance optimization (when you filter on the partition key) and is a good way to sidestep concurrency conflict issues.</p> <p>Hive-style partitioning also has some significant downsides.</p> <ul> <li>It\u2019s only suitable for low-cardinality columns.</li> <li>It can create many small files, especially if you use the wrong partition key or frequently update the Delta table.</li> <li>It can cause some queries that don\u2019t rely on the partition key to run slower (because of the excessive number of small files).  A large number of small files is problematic for I/O throughput.</li> </ul> <p>Hive-style partitioning can be a great data management tactic and a fantastic option for many Delta tables.  Beware of the downsides before partitioning your tables.</p> <p>You can use Hive-style partitioning in conjunction with Z Ordering.  You can partition a table by one column and Z Order by another.  They\u2019re different tactics that aim to help you skip more files and run queries faster.</p> <p>Let\u2019s look at some of the built-in Delta features that help maintain the integrity of your tables.</p>"},{"location":"delta-lake-best-practices/#use-appropriate-quality-controls","title":"Use appropriate quality controls","text":"<p>Delta Lake supports schema enforcement and column constraints to protect the integrity of your data.</p> <p>Delta Lake enabled schema enforcement by default, so you can only append data to an existing table with the same exact schema.  You can bypass schema enforcement by enabling schema evolution, which allows you to append mismatched schemas to a table.</p> <p>You should only enable schema evolution when you want to allow the schema of your table to change.  You should not enable schema evolution if you don\u2019t want this flexibility.  Schema enforcement is a good default setting.</p> <p>Column-level constraints prevent you from appending data that fail SQL predicates.  For example, you may add a constraint that requires all the values in the <code>age</code> column of a table to be positive.</p> <p>You should add column constraints to your table whenever you want a column only to include values that satisfy a predicate.</p> <p>No data is appended when you apply a constraint and a row check fails.  For example, if you try to append 100 rows of data to a table and one row has a failing check, then no data is added.</p> <p>When you have column constraints, it\u2019s often a good idea to append the failing data to a \u201cquarantine table\u201d and the passing data to the main Delta table.  Or you can filter out the failing rows and just append the passing rows.  Keeping a history of the failing rows in a quarantine table is helpful for debugging.</p> <p>See here to learn more about Delta Lake constraints.</p>"},{"location":"delta-lake-best-practices/#best-practices-for-dml-operations","title":"Best practices for DML operations","text":"<p>DML operations like deleting, updating, and merging write existing data in new files and mark existing files for deletion in the transaction log.  Rewriting data files is expensive, so you want to minimize the number of rewritten files when you run DML operations.</p> <p>Delta Lake supports a table feature called deletion vectors that implements DML transactions more efficiently under the hood.  Enabling deletion vectors is usually the best way to make DML operations run faster.  Note: delta-rs doesn\u2019t support deletion vectors yet.</p> <p>You should periodically purge deletion vectors because they can accumulate and slow subsequent read operations.  Once you enable the feature, you must purge the deletion vectors in your table with an appropriate cadence.</p>"},{"location":"delta-lake-best-practices/#use-vacuum-to-save-storage-costs","title":"Use vacuum to save storage costs","text":"<p>Delta Lake supports transactions, which necessitates keeping old versions of data in storage, even the files marked for removal in the transactions log.</p> <p>Keeping old versions of Delta tables in storage is often desirable because it allows for versioned data, time travel, and rolling back tables to a previous state.</p> <p>If you don\u2019t want to leverage older versions of a table, then you should remove the legacy files from storage with the vacuum command.  Vacuum will remove all files older than the table retention period and marked for removal in the transaction log.</p> <p>You only need to vacuum when you perform operations that mark files for removal in the transaction log.  An append-only table doesn\u2019t create legacy files that need to be vacuumed.</p> <p>Create a good vacuum strategy for your tables to minimize your storage costs.</p>"},{"location":"delta-lake-best-practices/#delta-lake-best-practices-to-minimize-costs","title":"Delta Lake best practices to minimize costs","text":"<p>Delta Lake helps you minimize costs in many ways:</p> <ul> <li>It's a free, open source format (based on Parquet). It's not a proprietary format that you need to pay for.</li> <li>Delta tables store column-level min/max values in the transaction log, allowing file skipping.</li> <li>Delta tables can be optimized (small file compaction, Z Ordering, etc.), so your queries run faster. When your queries run faster, then you pay less on compute.</li> <li>Deletion vectors let you perform DML operations (delete, update, merge) much faster. If your delete operation runs 100x faster, then you pay 100x less compute.</li> <li>It's easy to remove legacy files from storage with VACUUM, which minimizes storage costs.</li> </ul> <p>You should understand your organization\u2019s query patterns and use these features to minimize the overall cost.  You need to assess tradeoffs.  For example, Z Ordering is a computation that costs money, but it can save you lots of money in the long run if all your subsequent queries run a lot faster and use less compute.</p>"},{"location":"delta-lake-best-practices/#collect-metadata-stats-on-columns-used-for-file-skipping","title":"Collect metadata stats on columns used for file skipping","text":"<p>Delta tables don\u2019t always store each column's min/max values.  Some Delta Lake implementations only store min/max values for the first 32 columns in the table, for example.</p> <p>Delta Lake can only apply file-skipping when it has min/max values for the relevant columns stored in the transaction log.  Suppose you\u2019re running a filtering operation on <code>col_a,</code> for example.  Delta Lake can only apply file skipping when the transaction log stores <code>col_a</code> min/max metadata.</p> <p>Ensure the transaction log stores metadata stats for all the columns that benefit from file skipping.</p>"},{"location":"delta-lake-best-practices/#dont-collect-column-metadata-when-its-unnecessary","title":"Don\u2019t collect column metadata when it\u2019s unnecessary","text":"<p>It takes some time to compute column statistics when writing files, and it isn\u2019t worth the effort if you cannot use the column for file skipping.</p> <p>Suppose you have a table column containing a long string of arbitrary text.  It\u2019s unlikely that this column would ever provide any data-skipping benefits.  So, you can just avoid the overhead of collecting the statistics for this particular column.</p>"},{"location":"delta-lake-best-practices/#additional-reading","title":"Additional reading","text":"<p>Delta Lake relies on transactions, and you should check out this page to learn more.</p> <p>Many Delta Lake performance benefits rely on file skipping, which you should understand well to get the most out of Delta.</p>"},{"location":"delta-lake-best-practices/#conclusion","title":"Conclusion","text":"<p>Delta Lake is a powerful technology that makes your data pipelines more reliable, saves money, and makes everyday data processing tasks easy.</p> <p>You need to learn how Delta Lake works at a high level to leverage Delta's power fully.  You will not be able to leverage Delta Lake\u2019s full performance potential if your table has improperly sized files or if you\u2019re not colocating data in the same files to maximize data skipping, for example.</p> <p>Luckily, there are only a few details that are important to learn.  You don\u2019t need to know the implementation details - just the essential high-level concepts.</p>"},{"location":"delta-lake-big-data-small-data/","title":"Delta Lake for big data and small data","text":"<p>Delta Lake is an excellent storage format for big data and small data.</p> <p>This post explains why Delta Lake is suitable for massive datasets and why many of these features that are great, even for tiny tables.  Delta Lake is fine for a table with less than 1 GB of data or hundreds of petabytes of data.</p> <p>Let\u2019s start by discussing the features that are great for small data.</p>"},{"location":"delta-lake-big-data-small-data/#delta-lake-for-small-data-tables","title":"Delta Lake for small data tables","text":"<p>Delta Lake has many features that are useful for small datasets:</p> <ul> <li>Reliable transactions</li> <li>Better performance via file skipping</li> <li>DML operations to make deletes, updates, and merges easy and performant</li> <li>Features like schema enforcement and constraints to enforce data quality</li> <li>Versioned data &amp; time travel</li> </ul> <p>All of these features are great for large and small tables.</p> <p>Delta Lake DML operations are ACID transactions, so they either finish entirely or don\u2019t finish at all.  Delta tables don\u2019t require any downtime while DML operations are running.  The Delta Lake user experience is better than a data lake that doesn\u2019t support transactions and has downtime while running DML operations.</p> <p>The Delta Lake API also makes it easy to run DML operations.  You can delete a line of code from a Delta table with a single line of code.  Writing code to delete rows from CSV files is more challenging, especially if you want to implement this operation efficiently.</p> <p>Delta Lake has built-in checks to retain the integrity of your tables.  For example, Delta tables have schema enforcement and prevent you from appending DataFrames with mismatched schema from the existing table.  Delta Lake also lets you add constraints that only allow appending specific values to a column.  Data quality is also essential for small tables!</p> <p>Delta Lake splits data into multiple files with file-level metadata in the transaction log, so query engines can sometimes skip data.  Data skipping can be a huge performance benefit, depending on how much data can be ignored by the query engine.  </p> <p>As previously mentioned, Delta tables record all DML operations as transactions.  Recording operations as transactions means that existing data isn\u2019t mutated.  So Delta Lake provides versioned data and time travel out of the box.  Versioning data is better because it allows you to roll back mistakes and compare the state of the table at different points in time.</p> <p>Delta Lake has many useful features for small data tables.  Let\u2019s look at how Delta Lake is scalable for massive datasets.</p>"},{"location":"delta-lake-big-data-small-data/#delta-lake-for-large-data-tables","title":"Delta Lake for large data tables","text":"<p>Delta Lake is designed to be scalable and can handle tables with terabytes or petabytes of data.</p> <p>See here for an example of an organization ingesting 220 TB of data into a Delta table daily.</p> <p>Delta tables store data in Parquet files, and cloud object stores allow engines to write any number of files.  Delta tables store metadata information in the transaction log as JSON files, which are periodically compacted into Parquet files, so an arbitrarily large amount of Delta table metadata can also be stored.</p> <p>Delta Lake transactions and concurrency protection maintain the integrity of tables, even for large write operations or long-running computations.</p> <p>It\u2019s well known that Delta tables are scalable, even for the most enormous tables.</p>"},{"location":"delta-lake-big-data-small-data/#small-data-operations-on-large-tables","title":"Small data operations on large tables","text":"<p>Delta Lake is flexible and allows you to use \u201csmall data engines,\u201d even for large tables, depending on the computation.</p> <p>Suppose you have a Delta table containing 10 TB of data and a pipeline that appends 0.5 GB of data to the table every hour.  You don\u2019t need a big data query engine to append a small amount of data.  You can set up this job to run the Delta table append with a small data engine like pandas or Polars.</p> <p>Delta tables are flexible and interoperable with many technologies so that you can use the right tool for each data processing job.  This allows you to design pipelines how you\u2019d like and minimize costs.</p>"},{"location":"delta-lake-big-data-small-data/#when-delta-lake-isnt-needed","title":"When Delta Lake isn\u2019t needed","text":"<p>You don\u2019t need Delta Lake for a small dataset that never changes and can be stored in a single Parquet file.</p> <p>Suppose you have a 0.5 GB dataset in a Parquet file that never needs to be updated.  You can just keep that data in a Parquet table.  Reading the metadata from the Parquet footer of a single file isn\u2019t expensive.  You won\u2019t be taking advantage of Delta Lake's features like transactions, convenient DML operations, or versioned data.</p> <p>But in most cases, it\u2019s best to use Delta Lake because its features protect the integrity of your tables and make your life easier.</p>"},{"location":"delta-lake-big-data-small-data/#conclusion","title":"Conclusion","text":"<p>Delta Lake is well known for being scalable to huge tables but is also an excellent technology for small tables.</p> <p>Delta Lake is a lightweight technology, so there is little overhead.  Writing the metadata file after performing a transaction is fast.  It\u2019s a minuscule cost, considering the benefits you receive.</p> <p>Many reasons that make Delta Lake better than data lakes for large tables also apply to small tables!</p>"},{"location":"why-use-delta-lake/","title":"Why use Delta Lake","text":"<p>This page explains why Delta Lake is a better storage format for most tabular data analyses than data lake alternatives.</p> <p>Delta Lake provides developer-friendly features, reliable transactions, and fast performance compared with alternatives like Parquet or CSV.</p>"},{"location":"why-use-delta-lake/#fast-performance","title":"Fast performance","text":"<p>Delta tables store data in Parquet files and persist file-level metadata in the transaction log.</p> <p>This offers two main performance advantages:</p> <ul> <li>File skipping based on metadata that\u2019s quickly accessible</li> <li>Easy identification of all file paths for the table, compared to file listing operations that can be slow, especially on cloud object stores</li> </ul> <p>Delta Lake stores min/max values for each column of each file in the table.  Certain queries can skip entire files based on the metadata.  File skipping can be a massive performance optimization.</p> <p>Delta Lake also makes it easy to rearrange data in the table, so more file skipping is possible.  For example, the table can be partitioned or Z Ordered, so that similar data is colocated in the same files and data skipping is optimal for your query patterns.</p> <p>For data lakes, you need to run file listing operations to get the file paths before you can actually read the data.  Listing all the files in a data lake can take a long time, especially if there are a lot of files and they are stored in Hive-style partitions.</p> <p>Delta Lake stores all the file paths in the transaction log.  So you can quickly get the file paths directly from the log and then run your query.  Delta Lake also stores the file-level metadata in the transaction log which is quicker than opening all the files in the data lake and grabbing the metadata from the file footer.</p>"},{"location":"why-use-delta-lake/#developer-friendly-features","title":"Developer friendly features","text":"<p>Many basic data operations are hard in data lakes but quite easy with Delta Lake.  The only data operation that\u2019s easy with a data lake is appending data.  Delta Lake makes all data operations easy including the following:</p> <ul> <li>Appends</li> <li>Upserts</li> <li>Deletes</li> <li>Replace where</li> </ul> <p>Even deleting a few rows of data from a data lake is hard.  It\u2019s even harder if you want to run the operation in a performant manner.</p> <p>Delta Lake makes it easy to run common data operations and executes them performantly under the hood.</p> <p>Delta Lake also executes write operations as transactions, which makes data operations safer and prevents downtime.  Write operations will cause data lakes to be in an unstable state while the computations is running.  For example, if you read a data lake while a delete operation is running, then you may get the wrong data.</p> <p>Let\u2019s explore the benefits of reliable transactions in more detail.</p>"},{"location":"why-use-delta-lake/#reliable-transactions","title":"Reliable transactions","text":"<p>Delta Lake supports transactions which means that write operations have the following characteristics:</p> <ul> <li>They either finish completely or don\u2019t run at all</li> <li>They are executed in a serial manner and don\u2019t conflict with other transactions</li> <li>They don\u2019t corrupt a table or violate table constraints</li> </ul> <p>Data lakes don\u2019t support transactions, so the write operations can cause the following errors:</p> <ul> <li>There is no schema enforcement, so you can append data to a data lake with a mismatching schema</li> <li>Reading the data lake often yields incorrect results while write transactions are performed</li> <li>Data lakes can be corrupted for invalid write operations or computations that error-out</li> <li>Concurrent transactions that conflict can cause data loss</li> </ul> <p>Production data systems should rely on storage systems like Delta Lake that support transactions.</p>"},{"location":"why-use-delta-lake/#interoperability","title":"Interoperability","text":"<p>Delta Lake tables are interoperable and can be read/written by multiple different query engines.</p> <p>For example, you can create a Delta table with Spark, append to it with pandas, and then read it with Polars.</p> <p></p> <p>Delta tables are powerful because they are interoperable with various query engines and computation runtimes.</p> <p>Suppose you have a Delta table that\u2019s updated with an AWS Lambda function every 5 minutes.  There is only a small amount of data collected every 5 minutes, so a lightweight runtime like AWS Lambda is sufficient.</p> <p>Further suppose that the overall table is quite large.  So when you want to perform DML operations or query the whole table, your team uses a Spark cluster.</p> <p>Delta Lake is flexible to allow these types of operations from multiple readers and writers.  This provides teams with the flexibility to choose the right tool for the job.</p>"},{"location":"why-use-delta-lake/#support-for-many-languages","title":"Support for many languages","text":"<p>Delta tables can be queried with a variety of different languages.  This project provides APIs for Rust and Python users and does not depend on Java or Scala.  This project is a great alternative for pandas, Polars, DuckDB, or DataFusion.</p> <p>Delta Lake supports many languages and even more language support is coming soon!</p>"},{"location":"why-use-delta-lake/#support-on-multiple-clouds","title":"Support on multiple clouds","text":"<p>Delta Lake supports multiple clouds including GCP, AWS, and Azure.</p> <p>You can also use Delta Lake on your local machine or in an on-prem environment.</p> <p>Delta Lake is quite portable.</p>"},{"location":"why-use-delta-lake/#conclusion","title":"Conclusion","text":"<p>Delta Lake is a mature table format that offers users tons of advantages over a data lake with virtually no downsides.</p> <p>Once you start using Delta Lake, you will never want to go back to data lakes that expose you to a variety of dangerous bugs, poor performance, and reliability issues.</p> <p>The Delta Lake community is also welcome and open.  We gladly accept new contributors and help users with their questions.</p>"},{"location":"api/catalog/","title":"Catalog","text":"","boost":2},{"location":"api/catalog/#deltalake.data_catalog.DataCatalog","title":"deltalake.data_catalog.DataCatalog","text":"<p>             Bases: <code>Enum</code></p> <p>List of the Data Catalogs</p>","boost":2},{"location":"api/catalog/#deltalake.data_catalog.DataCatalog.AWS","title":"AWS  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>AWS = 'glue'\n</code></pre> <p>Refers to the <code>AWS Glue Data Catalog &lt;https://docs.aws.amazon.com/glue/latest/dg/catalog-and-crawler.html&gt;</code>_</p>","boost":2},{"location":"api/catalog/#deltalake.data_catalog.DataCatalog.UNITY","title":"UNITY  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>UNITY = 'unity'\n</code></pre> <p>Refers to the <code>Databricks Unity Catalog &lt;https://docs.databricks.com/data-governance/unity-catalog/index.html&gt;</code>_</p>","boost":2},{"location":"api/delta_writer/","title":"Writer","text":"","boost":10},{"location":"api/delta_writer/#write-to-delta-tables","title":"Write to Delta Tables","text":"","boost":10},{"location":"api/delta_writer/#deltalake.write_deltalake","title":"deltalake.write_deltalake","text":"<pre><code>write_deltalake(table_or_uri: Union[str, Path, DeltaTable], data: Union[pd.DataFrame, ds.Dataset, pa.Table, pa.RecordBatch, Iterable[pa.RecordBatch], RecordBatchReader, ArrowStreamExportable], *, schema: Optional[Union[pa.Schema, DeltaSchema]] = None, partition_by: Optional[Union[List[str], str]] = None, mode: Literal['error', 'append', 'overwrite', 'ignore'] = 'error', file_options: Optional[ds.ParquetFileWriteOptions] = None, max_partitions: Optional[int] = None, max_open_files: int = 1024, max_rows_per_file: int = 10 * 1024 * 1024, min_rows_per_group: int = 64 * 1024, max_rows_per_group: int = 128 * 1024, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, schema_mode: Optional[Literal['merge', 'overwrite']] = None, storage_options: Optional[Dict[str, str]] = None, partition_filters: Optional[List[Tuple[str, str, Any]]] = None, predicate: Optional[str] = None, target_file_size: Optional[int] = None, large_dtypes: bool = False, engine: Literal['pyarrow', 'rust'] = 'rust', writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; None\n</code></pre> <p>Write to a Delta Lake table</p> <p>If the table does not already exist, it will be created.</p> <p>The pyarrow writer supports protocol version 2 currently and won't be updated. For higher protocol support use engine='rust', this will become the default eventually.</p> <p>To enable safe concurrent writes when writing to S3, an additional locking mechanism must be supplied. For more information on enabling concurrent writing to S3, follow this guide</p> <p>Parameters:</p> Name Type Description Default <code>table_or_uri</code> <code>Union[str, Path, DeltaTable]</code> <p>URI of a table or a DeltaTable object.</p> required <code>data</code> <code>Union[DataFrame, Dataset, Table, RecordBatch, Iterable[RecordBatch], RecordBatchReader, ArrowStreamExportable]</code> <p>Data to write. If passing iterable, the schema must also be given.</p> required <code>schema</code> <code>Optional[Union[Schema, Schema]]</code> <p>Optional schema to write.</p> <code>None</code> <code>partition_by</code> <code>Optional[Union[List[str], str]]</code> <p>List of columns to partition the table by. Only required when creating a new table.</p> <code>None</code> <code>mode</code> <code>Literal['error', 'append', 'overwrite', 'ignore']</code> <p>How to handle existing data. Default is to error if table already exists. If 'append', will add new data. If 'overwrite', will replace table with new data. If 'ignore', will not write anything if table already exists.</p> <code>'error'</code> <code>file_options</code> <code>Optional[ParquetFileWriteOptions]</code> <p>Optional write options for Parquet (ParquetFileWriteOptions). Can be provided with defaults using ParquetFileWriteOptions().make_write_options(). Please refer to https://github.com/apache/arrow/blob/master/python/pyarrow/_dataset_parquet.pyx#L492-L533 for the list of available options. Only used in pyarrow engine.</p> <code>None</code> <code>max_partitions</code> <code>Optional[int]</code> <p>the maximum number of partitions that will be used. Only used in pyarrow engine.</p> <code>None</code> <code>max_open_files</code> <code>int</code> <p>Limits the maximum number of files that can be left open while writing. If an attempt is made to open too many files then the least recently used file will be closed. If this setting is set too low you may end up fragmenting your data into many small files. Only used in pyarrow engine.</p> <code>1024</code> <code>max_rows_per_file</code> <code>int</code> <p>Maximum number of rows per file. If greater than 0 then this will limit how many rows are placed in any single file. Otherwise there will be no limit and one file will be created in each output directory unless files need to be closed to respect max_open_files min_rows_per_group: Minimum number of rows per group. When the value is set, the dataset writer will batch incoming data and only write the row groups to the disk when sufficient rows have accumulated. Only used in pyarrow engine.</p> <code>10 * 1024 * 1024</code> <code>max_rows_per_group</code> <code>int</code> <p>Maximum number of rows per group. If the value is set, then the dataset writer may split up large incoming batches into multiple row groups. If this value is set, then min_rows_per_group should also be set.</p> <code>128 * 1024</code> <code>name</code> <code>Optional[str]</code> <p>User-provided identifier for this table.</p> <code>None</code> <code>description</code> <code>Optional[str]</code> <p>User-provided description for this table.</p> <code>None</code> <code>configuration</code> <code>Optional[Mapping[str, Optional[str]]]</code> <p>A map containing configuration options for the metadata action.</p> <code>None</code> <code>schema_mode</code> <code>Optional[Literal['merge', 'overwrite']]</code> <p>If set to \"overwrite\", allows replacing the schema of the table. Set to \"merge\" to merge with existing schema.</p> <code>None</code> <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>options passed to the native delta filesystem.</p> <code>None</code> <code>predicate</code> <code>Optional[str]</code> <p>When using <code>Overwrite</code> mode, replace data that matches a predicate. Only used in rust engine.'</p> <code>None</code> <code>target_file_size</code> <code>Optional[int]</code> <p>Override for target file size for data files written to the delta table. If not passed, it's taken from <code>delta.targetFileSize</code>.</p> <code>None</code> <code>partition_filters</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>the partition filters that will be used for partition overwrite. Only used in pyarrow engine.</p> <code>None</code> <code>large_dtypes</code> <code>bool</code> <p>Only used for pyarrow engine</p> <code>False</code> <code>engine</code> <code>Literal['pyarrow', 'rust']</code> <p>writer engine to write the delta table. PyArrow engine is deprecated, and will be removed in v1.0.</p> <code>'rust'</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code>","boost":10},{"location":"api/delta_writer/#deltalake.BloomFilterProperties","title":"deltalake.BloomFilterProperties  <code>dataclass</code>","text":"<pre><code>BloomFilterProperties(set_bloom_filter_enabled: Optional[bool], fpp: Optional[float] = None, ndv: Optional[int] = None)\n</code></pre> <p>The Bloom Filter Properties instance for the Rust parquet writer.</p> <p>Create a Bloom Filter Properties instance for the Rust parquet writer:</p> <p>Parameters:</p> Name Type Description Default <code>set_bloom_filter_enabled</code> <code>Optional[bool]</code> <p>If True and no fpp or ndv are provided, the default values will be used.</p> required <code>fpp</code> <code>Optional[float]</code> <p>The false positive probability for the bloom filter. Must be between 0 and 1 exclusive.</p> <code>None</code> <code>ndv</code> <code>Optional[int]</code> <p>The number of distinct values for the bloom filter.</p> <code>None</code>","boost":10},{"location":"api/delta_writer/#deltalake.ColumnProperties","title":"deltalake.ColumnProperties  <code>dataclass</code>","text":"<pre><code>ColumnProperties(dictionary_enabled: Optional[bool] = None, max_statistics_size: Optional[int] = None, bloom_filter_properties: Optional[BloomFilterProperties] = None)\n</code></pre> <p>The Column Properties instance for the Rust parquet writer.</p> <p>Create a Column Properties instance for the Rust parquet writer:</p> <p>Parameters:</p> Name Type Description Default <code>dictionary_enabled</code> <code>Optional[bool]</code> <p>Enable dictionary encoding for the column.</p> <code>None</code> <code>max_statistics_size</code> <code>Optional[int]</code> <p>Maximum size of statistics for the column.</p> <code>None</code> <code>bloom_filter_properties</code> <code>Optional[BloomFilterProperties]</code> <p>Bloom Filter Properties for the column.</p> <code>None</code>","boost":10},{"location":"api/delta_writer/#deltalake.WriterProperties","title":"deltalake.WriterProperties  <code>dataclass</code>","text":"<pre><code>WriterProperties(data_page_size_limit: Optional[int] = None, dictionary_page_size_limit: Optional[int] = None, data_page_row_count_limit: Optional[int] = None, write_batch_size: Optional[int] = None, max_row_group_size: Optional[int] = None, compression: Optional[Literal['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'ZSTD', 'LZ4_RAW']] = None, compression_level: Optional[int] = None, statistics_truncate_length: Optional[int] = None, default_column_properties: Optional[ColumnProperties] = None, column_properties: Optional[Dict[str, ColumnProperties]] = None)\n</code></pre> <p>A Writer Properties instance for the Rust parquet writer.</p> <p>Create a Writer Properties instance for the Rust parquet writer:</p> <p>Parameters:</p> Name Type Description Default <code>data_page_size_limit</code> <code>Optional[int]</code> <p>Limit DataPage size to this in bytes.</p> <code>None</code> <code>dictionary_page_size_limit</code> <code>Optional[int]</code> <p>Limit the size of each DataPage to store dicts to this amount in bytes.</p> <code>None</code> <code>data_page_row_count_limit</code> <code>Optional[int]</code> <p>Limit the number of rows in each DataPage.</p> <code>None</code> <code>write_batch_size</code> <code>Optional[int]</code> <p>Splits internally to smaller batch size.</p> <code>None</code> <code>max_row_group_size</code> <code>Optional[int]</code> <p>Max number of rows in row group.</p> <code>None</code> <code>compression</code> <code>Optional[Literal['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'ZSTD', 'LZ4_RAW']]</code> <p>compression type.</p> <code>None</code> <code>compression_level</code> <code>Optional[int]</code> <p>If none and compression has a level, the default level will be used, only relevant for GZIP: levels (1-9), BROTLI: levels (1-11), ZSTD: levels (1-22),</p> <code>None</code> <code>statistics_truncate_length</code> <code>Optional[int]</code> <p>maximum length of truncated min/max values in statistics.</p> <code>None</code> <code>default_column_properties</code> <code>Optional[ColumnProperties]</code> <p>Default Column Properties for the Rust parquet writer.</p> <code>None</code> <code>column_properties</code> <code>Optional[Dict[str, ColumnProperties]]</code> <p>Column Properties for the Rust parquet writer.</p> <code>None</code>","boost":10},{"location":"api/delta_writer/#convert-to-delta-tables","title":"Convert to Delta Tables","text":"","boost":10},{"location":"api/delta_writer/#deltalake.convert_to_deltalake","title":"deltalake.convert_to_deltalake","text":"<pre><code>convert_to_deltalake(uri: Union[str, Path], mode: Literal['error', 'ignore'] = 'error', partition_by: Optional[pa.Schema] = None, partition_strategy: Optional[Literal['hive']] = None, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, storage_options: Optional[Dict[str, str]] = None, custom_metadata: Optional[Dict[str, str]] = None) -&gt; None\n</code></pre> <p><code>Convert</code> parquet tables <code>to delta</code> tables.</p> <p>Currently only HIVE partitioned tables are supported. <code>Convert to delta</code> creates a transaction log commit with add actions, and additional properties provided such as configuration, name, and description.</p> <p>Parameters:</p> Name Type Description Default <code>uri</code> <code>Union[str, Path]</code> <p>URI of a table.</p> required <code>partition_by</code> <code>Optional[Schema]</code> <p>Optional partitioning schema if table is partitioned.</p> <code>None</code> <code>partition_strategy</code> <code>Optional[Literal['hive']]</code> <p>Optional partition strategy to read and convert</p> <code>None</code> <code>mode</code> <code>Literal['error', 'ignore']</code> <p>How to handle existing data. Default is to error if table already exists. If 'ignore', will not convert anything if table already exists.</p> <code>'error'</code> <code>name</code> <code>Optional[str]</code> <p>User-provided identifier for this table.</p> <code>None</code> <code>description</code> <code>Optional[str]</code> <p>User-provided description for this table.</p> <code>None</code> <code>configuration</code> <code>Optional[Mapping[str, Optional[str]]]</code> <p>A map containing configuration options for the metadata action.</p> <code>None</code> <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>options passed to the native delta filesystem. Unused if 'filesystem' is defined.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>custom metadata that will be added to the transaction commit</p> <code>None</code>","boost":10},{"location":"api/exceptions/","title":"Exceptions","text":"","boost":2},{"location":"api/exceptions/#deltalake.exceptions.DeltaError","title":"deltalake.exceptions.DeltaError","text":"<p>             Bases: <code>builtins.Exception</code></p> <p>The base class for Delta-specific errors.</p>","boost":2},{"location":"api/exceptions/#deltalake.exceptions.DeltaProtocolError","title":"deltalake.exceptions.DeltaProtocolError","text":"<p>             Bases: <code>_internal.DeltaError</code></p> <p>Raised when a violation with the Delta protocol specs ocurred.</p>","boost":2},{"location":"api/exceptions/#deltalake.exceptions.TableNotFoundError","title":"deltalake.exceptions.TableNotFoundError","text":"<p>             Bases: <code>_internal.DeltaError</code></p> <p>Raised when a Delta table cannot be loaded from a location.</p>","boost":2},{"location":"api/exceptions/#deltalake.exceptions.CommitFailedError","title":"deltalake.exceptions.CommitFailedError","text":"<p>             Bases: <code>_internal.DeltaError</code></p> <p>Raised when a commit to a Delta table fails.</p>","boost":2},{"location":"api/schema/","title":"Schema","text":"","boost":2},{"location":"api/schema/#schema-and-field","title":"Schema and field","text":"<p>Schemas, fields, and data types are provided in the <code>deltalake.schema</code> submodule.</p>","boost":2},{"location":"api/schema/#deltalake.Schema","title":"deltalake.Schema","text":"<pre><code>Schema(fields: List[Field])\n</code></pre> <p>             Bases: <code>deltalake._internal.StructType</code></p> <p>A Delta Lake schema</p> <p>Create using a list of :class:<code>Field</code>:</p> <p>Schema([Field(\"x\", \"integer\"), Field(\"y\", \"string\")]) Schema([Field(x, PrimitiveType(\"integer\"), nullable=True), Field(y, PrimitiveType(\"string\"), nullable=True)])</p> <p>Or create from a PyArrow schema:</p> <p>import pyarrow as pa Schema.from_pyarrow(pa.schema({\"x\": pa.int32(), \"y\": pa.string()})) Schema([Field(x, PrimitiveType(\"integer\"), nullable=True), Field(y, PrimitiveType(\"string\"), nullable=True)])</p>","boost":2},{"location":"api/schema/#deltalake.Schema.invariants","title":"invariants","text":"<pre><code>invariants: List[Tuple[str, str]] = &lt;attribute 'invariants' of 'deltalake._internal.Schema' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Schema.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(schema_json) -&gt; Schema\n</code></pre> <p>Create a new Schema from a JSON string.</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required Example <p>A schema has the same JSON format as a StructType. <pre><code>Schema.from_json('''{\n    \"type\": \"struct\",\n    \"fields\": [{\"name\": \"x\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]\n    }\n)'''\n# Returns Schema([Field(x, PrimitiveType(\"integer\"), nullable=True)])\n</code></pre></p>","boost":2},{"location":"api/schema/#deltalake.Schema.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; Schema\n</code></pre> <p>Create a Schema from a PyArrow Schema type</p> <p>Will raise <code>TypeError</code> if the PyArrow type is not a primitive type.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>Schema</code> <p>A PyArrow Schema</p> required <p>Returns:</p> Type Description <code>Schema</code> <p>a Schema</p>","boost":2},{"location":"api/schema/#deltalake.Schema.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get the JSON string representation of the Schema.</p> <p>Returns:</p> Type Description <code>str</code> <p>a JSON string</p> Example <p>A schema has the same JSON format as a StructType. <pre><code>Schema([Field(\"x\", \"integer\")]).to_json()\n# Returns '{\"type\":\"struct\",\"fields\":[{\"name\":\"x\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}'\n</code></pre></p>","boost":2},{"location":"api/schema/#deltalake.Schema.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow(as_large_types: bool = False) -&gt; pyarrow.Schema\n</code></pre> <p>Return equivalent PyArrow schema</p> <p>Parameters:</p> Name Type Description Default <code>as_large_types</code> <code>bool</code> <p>get schema with all variable size types (list, binary, string) as large variants (with int64 indices). This is for compatibility with systems like Polars that only support the large versions of Arrow types.</p> <code>False</code> <p>Returns:</p> Type Description <code>Schema</code> <p>a PyArrow Schema</p>","boost":2},{"location":"api/schema/#deltalake.Field","title":"deltalake.Field","text":"<pre><code>Field(name: str, type: DataType, *, nullable: bool = True, metadata: Optional[Dict[str, Any]] = None)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.metadata","title":"metadata","text":"<pre><code>metadata: Dict[str, Any] = &lt;attribute 'metadata' of 'deltalake._internal.Field' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.name","title":"name","text":"<pre><code>name: str = &lt;attribute 'name' of 'deltalake._internal.Field' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.nullable","title":"nullable","text":"<pre><code>nullable: bool = &lt;attribute 'nullable' of 'deltalake._internal.Field' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.type","title":"type","text":"<pre><code>type: DataType = &lt;attribute 'type' of 'deltalake._internal.Field' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(field_json) -&gt; Field\n</code></pre> <p>Create a Field from a JSON string.</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>the JSON string.</p> required <p>Returns:</p> Type Description <code>Field</code> <p>Field</p> Example <pre><code>Field.from_json('''{\n        \"name\": \"col\",\n        \"type\": \"integer\",\n        \"nullable\": true,\n        \"metadata\": {}\n    }'''\n)\n# Returns Field(col, PrimitiveType(\"integer\"), nullable=True)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(field: pyarrow.Field) -&gt; Field\n</code></pre> <p>Create a Field from a PyArrow field Note: This currently doesn't preserve field metadata.</p> <p>Parameters:</p> Name Type Description Default <code>field</code> <code>Field</code> <p>a PyArrow Field</p> required <p>Returns:</p> Type Description <code>Field</code> <p>a Field</p>","boost":2},{"location":"api/schema/#deltalake.Field.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get the field as JSON string.</p> <p>Returns:</p> Type Description <code>str</code> <p>a JSON string</p> Example <pre><code>Field(\"col\", \"integer\").to_json()\n# Returns '{\"name\":\"col\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}'\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.Field\n</code></pre> <p>Convert to an equivalent PyArrow field Note: This currently doesn't preserve field metadata.</p> <p>Returns:</p> Type Description <code>Field</code> <p>a pyarrow Field</p>","boost":2},{"location":"api/schema/#data-types","title":"Data types","text":"","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType","title":"deltalake.schema.PrimitiveType","text":"<pre><code>PrimitiveType(data_type: str)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType.type","title":"type","text":"<pre><code>type: str = &lt;attribute 'type' of 'deltalake._internal.PrimitiveType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(type_json) -&gt; PrimitiveType\n</code></pre> <p>Create a PrimitiveType from a JSON string</p> <p>The JSON representation for a primitive type is just a quoted string: <code>PrimitiveType.from_json('\"integer\"')</code></p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required <p>Returns:</p> Type Description <code>PrimitiveType</code> <p>a PrimitiveType type</p>","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; PrimitiveType\n</code></pre> <p>Create a PrimitiveType from a PyArrow datatype</p> <p>Will raise <code>TypeError</code> if the PyArrow type is not a primitive type.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>DataType</code> <p>A PyArrow DataType</p> required <p>Returns:</p> Type Description <code>PrimitiveType</code> <p>a PrimitiveType</p>","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.DataType\n</code></pre> <p>Get the equivalent PyArrow type (pyarrow.DataType)</p>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType","title":"deltalake.schema.ArrayType","text":"<pre><code>ArrayType(element_type: DataType, *, contains_null: bool = True)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.contains_null","title":"contains_null","text":"<pre><code>contains_null: bool = &lt;attribute 'contains_null' of 'deltalake._internal.ArrayType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.element_type","title":"element_type","text":"<pre><code>element_type: DataType = &lt;attribute 'element_type' of 'deltalake._internal.ArrayType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.type","title":"type","text":"<pre><code>type: Literal['array'] = &lt;attribute 'type' of 'deltalake._internal.ArrayType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(type_json) -&gt; ArrayType\n</code></pre> <p>Create an ArrayType from a JSON string</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required <p>Returns:</p> Type Description <code>ArrayType</code> <p>an ArrayType</p> Example <p>The JSON representation for an array type is an object with <code>type</code> (set to <code>\"array\"</code>), <code>elementType</code>, and <code>containsNull</code>. <pre><code>ArrayType.from_json(\n    '''{\n        \"type\": \"array\",\n        \"elementType\": \"integer\",\n        \"containsNull\": false\n    }'''\n)\n# Returns ArrayType(PrimitiveType(\"integer\"), contains_null=False)\n</code></pre></p>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; ArrayType\n</code></pre> <p>Create an ArrayType from a pyarrow.ListType.</p> <p>Will raise <code>TypeError</code> if a different PyArrow DataType is provided.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>ListType</code> <p>The PyArrow ListType</p> required <p>Returns:</p> Type Description <code>ArrayType</code> <p>an ArrayType</p>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get the JSON string representation of the type.</p>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.ListType\n</code></pre> <p>Get the equivalent PyArrow type.</p>","boost":2},{"location":"api/schema/#deltalake.schema.MapType","title":"deltalake.schema.MapType","text":"<pre><code>MapType(key_type: DataType, value_type: DataType, *, value_contains_null: bool = True)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.key_type","title":"key_type","text":"<pre><code>key_type: DataType = &lt;attribute 'key_type' of 'deltalake._internal.MapType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.type","title":"type","text":"<pre><code>type: Literal['map'] = &lt;attribute 'type' of 'deltalake._internal.MapType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.value_contains_null","title":"value_contains_null","text":"<pre><code>value_contains_null: bool = &lt;attribute 'value_contains_null' of 'deltalake._internal.MapType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.value_type","title":"value_type","text":"<pre><code>value_type: DataType = &lt;attribute 'value_type' of 'deltalake._internal.MapType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(type_json) -&gt; MapType\n</code></pre> <p>Create a MapType from a JSON string</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required <p>Returns:</p> Type Description <code>MapType</code> <p>an ArrayType</p> Example <p>The JSON representation for a map type is an object with <code>type</code> (set to <code>map</code>), <code>keyType</code>, <code>valueType</code>, and <code>valueContainsNull</code>:</p> <pre><code>MapType.from_json(\n    '''{\n        \"type\": \"map\",\n        \"keyType\": \"integer\",\n        \"valueType\": \"string\",\n        \"valueContainsNull\": true\n    }'''\n)\n# Returns MapType(PrimitiveType(\"integer\"), PrimitiveType(\"string\"), value_contains_null=True)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; MapType\n</code></pre> <p>Create a MapType from a PyArrow MapType.</p> <p>Will raise <code>TypeError</code> if passed a different type.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>MapType</code> <p>the PyArrow MapType</p> required <p>Returns:</p> Type Description <code>MapType</code> <p>a MapType</p>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get JSON string representation of map type.</p> <p>Returns:</p> Type Description <code>str</code> <p>a JSON string</p>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.MapType\n</code></pre> <p>Get the equivalent PyArrow data type.</p>","boost":2},{"location":"api/schema/#deltalake.schema.StructType","title":"deltalake.schema.StructType","text":"<pre><code>StructType(fields: List[Field])\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.fields","title":"fields","text":"<pre><code>fields: List[Field] = &lt;attribute 'fields' of 'deltalake._internal.StructType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.type","title":"type","text":"<pre><code>type: Literal['struct'] = &lt;attribute 'type' of 'deltalake._internal.StructType' objects&gt;\n</code></pre> <p>The string \"struct\"</p>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(type_json) -&gt; StructType\n</code></pre> <p>Create a new StructType from a JSON string.</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required <p>Returns:</p> Type Description <code>StructType</code> <p>a StructType</p> Example <pre><code>StructType.from_json(\n    '''{\n        \"type\": \"struct\",\n        \"fields\": [{\"name\": \"x\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]\n    }'''\n)\n# Returns StructType([Field(x, PrimitiveType(\"integer\"), nullable=True)])\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; StructType\n</code></pre> <p>Create a new StructType from a PyArrow struct type.</p> <p>Will raise <code>TypeError</code> if a different data type is provided.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>StructType</code> <p>a PyArrow struct type.</p> required <p>Returns:</p> Type Description <code>StructType</code> <p>a StructType</p>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get the JSON representation of the type.</p> <p>Returns:</p> Type Description <code>str</code> <p>a JSON string</p> Example <pre><code>StructType([Field(\"x\", \"integer\")]).to_json()\n# Returns '{\"type\":\"struct\",\"fields\":[{\"name\":\"x\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}'\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.StructType\n</code></pre> <p>Get the equivalent PyArrow StructType</p> <p>Returns:</p> Type Description <code>StructType</code> <p>a PyArrow StructType</p>","boost":2},{"location":"api/storage/","title":"Storage","text":"<p>The delta filesystem handler for the pyarrow engine writer.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler","title":"deltalake.fs.DeltaStorageHandler","text":"<pre><code>DeltaStorageHandler(table_uri: str, options: Optional[Dict[str, str]] = None, known_sizes: Optional[Dict[str, int]] = None)\n</code></pre> <p>             Bases: <code>FileSystemHandler</code></p> <p>DeltaStorageHandler is a concrete implementations of a PyArrow FileSystemHandler.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.copy_file","title":"copy_file","text":"<pre><code>copy_file(src: str, dst: str) -&gt; None\n</code></pre> <p>Copy a file.</p> <p>If the destination exists and is a directory, an error is returned. Otherwise, it is replaced.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.create_dir","title":"create_dir","text":"<pre><code>create_dir(path: str, recursive: bool = True) -&gt; None\n</code></pre> <p>Create a directory and subdirectories.</p> <p>This function succeeds if the directory already exists.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.delete_dir","title":"delete_dir","text":"<pre><code>delete_dir(path: str) -&gt; None\n</code></pre> <p>Delete a directory and its contents, recursively.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.delete_dir_contents","title":"delete_dir_contents","text":"<pre><code>delete_dir_contents(path: str, *, accept_root_dir: bool = False, missing_dir_ok: bool = False) -&gt; None\n</code></pre> <p>Delete a directory's contents, recursively.</p> <p>Like delete_dir, but doesn't delete the directory itself.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.delete_file","title":"delete_file","text":"<pre><code>delete_file(path: str) -&gt; None\n</code></pre> <p>Delete a file.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.delete_root_dir_contents","title":"delete_root_dir_contents","text":"<pre><code>delete_root_dir_contents() -&gt; None\n</code></pre> <p>Delete the root directory contents, recursively.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.get_file_info","title":"get_file_info","text":"<pre><code>get_file_info(paths: List[str]) -&gt; List[FileInfo]\n</code></pre> <p>Get info for the given files.</p> <p>A non-existing or unreachable file returns a FileStat object and has a FileType of value NotFound. An exception indicates a truly exceptional condition (low-level I/O error, etc.).</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.get_file_info_selector","title":"get_file_info_selector","text":"<pre><code>get_file_info_selector(selector: FileSelector) -&gt; List[FileInfo]\n</code></pre> <p>Get info for the files defined by FileSelector.</p> <p>Parameters:</p> Name Type Description Default <code>selector</code> <code>FileSelector</code> <p>FileSelector object</p> required <p>Returns:</p> Type Description <code>List[FileInfo]</code> <p>list of file info objects</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.move","title":"move","text":"<pre><code>move(src: str, dest: str) -&gt; None\n</code></pre> <p>Move / rename a file or directory.</p> <p>If the destination exists: - if it is a non-empty directory, an error is returned - otherwise, if it has the same type as the source, it is replaced - otherwise, behavior is unspecified (implementation-dependent).</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.normalize_path","title":"normalize_path","text":"<pre><code>normalize_path(path: str) -&gt; str\n</code></pre> <p>Normalize filesystem path.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.open_input_file","title":"open_input_file","text":"<pre><code>open_input_file(path: str) -&gt; pa.PythonFile\n</code></pre> <p>Open an input file for random access reading.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>The source to open for reading.</p> required <p>Returns:</p> Type Description <code>PythonFile</code> <p>NativeFile</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.open_input_stream","title":"open_input_stream","text":"<pre><code>open_input_stream(path: str) -&gt; pa.PythonFile\n</code></pre> <p>Open an input stream for sequential reading.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>The source to open for reading.</p> required <p>Returns:</p> Type Description <code>PythonFile</code> <p>NativeFile</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.open_output_stream","title":"open_output_stream","text":"<pre><code>open_output_stream(path: str, metadata: Optional[Dict[str, str]] = None) -&gt; pa.PythonFile\n</code></pre> <p>Open an output stream for sequential writing.</p> <p>If the target already exists, existing data is truncated.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>The source to open for writing.</p> required <code>metadata</code> <code>Optional[Dict[str, str]]</code> <p>If not None, a mapping of string keys to string values.</p> <code>None</code> <p>Returns:</p> Type Description <code>PythonFile</code> <p>NativeFile</p>","boost":2},{"location":"api/delta_table/","title":"DeltaTable","text":"","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable","title":"deltalake.DeltaTable  <code>dataclass</code>","text":"<pre><code>DeltaTable(table_uri: Union[str, Path, os.PathLike[str]], version: Optional[int] = None, storage_options: Optional[Dict[str, str]] = None, without_files: bool = False, log_buffer_size: Optional[int] = None)\n</code></pre> <p>Represents a Delta Table</p> <p>Create the Delta Table from a path with an optional version. Multiple StorageBackends are currently supported: AWS S3, Azure Data Lake Storage Gen2, Google Cloud Storage (GCS) and local URI. Depending on the storage backend used, you could provide options values using the <code>storage_options</code> parameter.</p> <p>Parameters:</p> Name Type Description Default <code>table_uri</code> <code>Union[str, Path, PathLike[str]]</code> <p>the path of the DeltaTable</p> required <code>version</code> <code>Optional[int]</code> <p>version of the DeltaTable</p> <code>None</code> <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>a dictionary of the options to use for the storage backend</p> <code>None</code> <code>without_files</code> <code>bool</code> <p>If True, will load table without tracking files.                 Some append-only applications might have no need of tracking any files. So, the                 DeltaTable will be loaded with a significant memory reduction.</p> <code>False</code> <code>log_buffer_size</code> <code>Optional[int]</code> <p>Number of files to buffer when reading the commit log. A positive integer.                 Setting a value greater than 1 results in concurrent calls to the storage api.                 This can decrease latency if there are many files in the log since the last checkpoint,                 but will also increase memory usage. Possible rate limits of the storage backend should                 also be considered for optimal performance. Defaults to 4 * number of cpus.</p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.alter","title":"alter  <code>property</code>","text":"<pre><code>alter: TableAlterer\n</code></pre> <p>Namespace for all table alter related methods.</p> <p>Returns:</p> Name Type Description <code>TableAlterer</code> <code>TableAlterer</code> <p>TableAlterer Object</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.optimize","title":"optimize  <code>property</code>","text":"<pre><code>optimize: TableOptimizer\n</code></pre> <p>Namespace for all table optimize related methods.</p> <p>Returns:</p> Name Type Description <code>TableOptimizer</code> <code>TableOptimizer</code> <p>TableOptimizer Object</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.cleanup_metadata","title":"cleanup_metadata","text":"<pre><code>cleanup_metadata() -&gt; None\n</code></pre> <p>Delete expired log files before current version from table. The table log retention is based on the <code>delta.logRetentionDuration</code> value, 30 days by default.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.create","title":"create  <code>classmethod</code>","text":"<pre><code>create(table_uri: Union[str, Path], schema: Union[pyarrow.Schema, DeltaSchema], mode: Literal['error', 'append', 'overwrite', 'ignore'] = 'error', partition_by: Optional[Union[List[str], str]] = None, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, storage_options: Optional[Dict[str, str]] = None, custom_metadata: Optional[Dict[str, str]] = None, raise_if_key_not_exists: bool = True) -&gt; DeltaTable\n</code></pre> <p><code>CREATE</code> or <code>CREATE_OR_REPLACE</code> a delta table given a table_uri.</p> <p>Parameters:</p> Name Type Description Default <code>table_uri</code> <code>Union[str, Path]</code> <p>URI of a table</p> required <code>schema</code> <code>Union[Schema, Schema]</code> <p>Table schema</p> required <code>mode</code> <code>Literal['error', 'append', 'overwrite', 'ignore']</code> <p>How to handle existing data. Default is to error if table already exists. If 'append', returns not support error if table exists. If 'overwrite', will <code>CREATE_OR_REPLACE</code> table. If 'ignore', will not do anything if table already exists. Defaults to \"error\".</p> <code>'error'</code> <code>partition_by</code> <code>Optional[Union[List[str], str]]</code> <p>List of columns to partition the table by.</p> <code>None</code> <code>name</code> <code>Optional[str]</code> <p>User-provided identifier for this table.</p> <code>None</code> <code>description</code> <code>Optional[str]</code> <p>User-provided description for this table.</p> <code>None</code> <code>configuration</code> <code>Optional[Mapping[str, Optional[str]]]</code> <p>A map containing configuration options for the metadata action.</p> <code>None</code> <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>Options passed to the object store crate.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Custom metadata that will be added to the transaction commit.</p> <code>None</code> <code>raise_if_key_not_exists</code> <code>bool</code> <p>Whether to raise an error if the configuration uses keys that are not Delta keys</p> <code>True</code> <p>Returns:</p> Name Type Description <code>DeltaTable</code> <code>DeltaTable</code> <p>created delta table</p> Example <pre><code>import pyarrow as pa\n\nfrom deltalake import DeltaTable\n\ndt = DeltaTable.create(\n    table_uri=\"my_local_table\",\n    schema=pa.schema(\n        [pa.field(\"foo\", pa.string()), pa.field(\"bar\", pa.string())]\n    ),\n    mode=\"error\",\n    partition_by=\"bar\",\n)\n</code></pre>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.delete","title":"delete","text":"<pre><code>delete(predicate: Optional[str] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Delete records from a Delta Table that statisfy a predicate.</p> <p>When a predicate is not provided then all records are deleted from the Delta Table. Otherwise a scan of the Delta table is performed to mark any files that contain records that satisfy the predicate. Once files are determined they are rewritten without the records.</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>Optional[str]</code> <p>a SQL where clause. If not passed, will delete all rows.</p> <code>None</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>the metrics from delete.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.file_uris","title":"file_uris","text":"<pre><code>file_uris(partition_filters: Optional[FilterConjunctionType] = None) -&gt; List[str]\n</code></pre> <p>Get the list of files as absolute URIs, including the scheme (e.g. \"s3://\").</p> <p>Local files will be just plain absolute paths, without a scheme. (That is, no 'file://' prefix.)</p> <p>Use the partition_filters parameter to retrieve a subset of files that match the given filters.</p> <p>Parameters:</p> Name Type Description Default <code>partition_filters</code> <code>Optional[FilterConjunctionType]</code> <p>the partition filters that will be used for getting the matched files</p> <code>None</code> <p>Returns:</p> Type Description <code>List[str]</code> <p>list of the .parquet files with an absolute URI referenced for the current version of the DeltaTable</p> <p>Predicates are expressed in disjunctive normal form (DNF), like [(\"x\", \"=\", \"a\"), ...]. DNF allows arbitrary boolean logical combinations of single partition predicates. The innermost tuples each describe a single partition predicate. The list of inner predicates is interpreted as a conjunction (AND), forming a more selective and multiple partition predicates. Each tuple has format: (key, op, value) and compares the key with the value. The supported op are: <code>=</code>, <code>!=</code>, <code>in</code>, and <code>not in</code>. If the op is in or not in, the value must be a collection such as a list, a set or a tuple. The supported type for value is str. Use empty string <code>''</code> for Null partition value.</p> Example <pre><code>(\"x\", \"=\", \"a\")\n(\"x\", \"!=\", \"a\")\n(\"y\", \"in\", [\"a\", \"b\", \"c\"])\n(\"z\", \"not in\", [\"a\",\"b\"])\n</code></pre>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.files","title":"files","text":"<pre><code>files(partition_filters: Optional[List[Tuple[str, str, Any]]] = None) -&gt; List[str]\n</code></pre> <p>Get the .parquet files of the DeltaTable.</p> <p>The paths are as they are saved in the delta log, which may either be relative to the table root or absolute URIs.</p> <p>Parameters:</p> Name Type Description Default <code>partition_filters</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>the partition filters that will be used for                 getting the matched files</p> <code>None</code> <p>Returns:</p> Type Description <code>List[str]</code> <p>list of the .parquet files referenced for the current version of the DeltaTable</p> <p>Predicates are expressed in disjunctive normal form (DNF), like [(\"x\", \"=\", \"a\"), ...]. DNF allows arbitrary boolean logical combinations of single partition predicates. The innermost tuples each describe a single partition predicate. The list of inner predicates is interpreted as a conjunction (AND), forming a more selective and multiple partition predicates. Each tuple has format: (key, op, value) and compares the key with the value. The supported op are: <code>=</code>, <code>!=</code>, <code>in</code>, and <code>not in</code>. If the op is in or not in, the value must be a collection such as a list, a set or a tuple. The supported type for value is str. Use empty string <code>''</code> for Null partition value.</p> Example <pre><code>(\"x\", \"=\", \"a\")\n(\"x\", \"!=\", \"a\")\n(\"y\", \"in\", [\"a\", \"b\", \"c\"])\n(\"z\", \"not in\", [\"a\",\"b\"])\n</code></pre>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.files_by_partitions","title":"files_by_partitions","text":"<pre><code>files_by_partitions(partition_filters: PartitionFilterType) -&gt; List[str]\n</code></pre> <p>Get the files for each partition</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.from_data_catalog","title":"from_data_catalog  <code>classmethod</code>","text":"<pre><code>from_data_catalog(data_catalog: DataCatalog, database_name: str, table_name: str, data_catalog_id: Optional[str] = None, version: Optional[int] = None, log_buffer_size: Optional[int] = None) -&gt; DeltaTable\n</code></pre> <p>Create the Delta Table from a Data Catalog.</p> <p>Parameters:</p> Name Type Description Default <code>data_catalog</code> <code>DataCatalog</code> <p>the Catalog to use for getting the storage location of the Delta Table</p> required <code>database_name</code> <code>str</code> <p>the database name inside the Data Catalog</p> required <code>table_name</code> <code>str</code> <p>the table name inside the Data Catalog</p> required <code>data_catalog_id</code> <code>Optional[str]</code> <p>the identifier of the Data Catalog</p> <code>None</code> <code>version</code> <code>Optional[int]</code> <p>version of the DeltaTable</p> <code>None</code> <code>log_buffer_size</code> <code>Optional[int]</code> <p>Number of files to buffer when reading the commit log. A positive integer.                 Setting a value greater than 1 results in concurrent calls to the storage api.                 This can decrease latency if there are many files in the log since the last checkpoint,                 but will also increase memory usage. Possible rate limits of the storage backend should                 also be considered for optimal performance. Defaults to 4 * number of cpus.</p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.get_add_actions","title":"get_add_actions","text":"<pre><code>get_add_actions(flatten: bool = False) -&gt; pyarrow.RecordBatch\n</code></pre> <p>Return a dataframe with all current add actions.</p> <p>Add actions represent the files that currently make up the table. This data is a low-level representation parsed from the transaction log.</p> <p>Parameters:</p> Name Type Description Default <code>flatten</code> <code>bool</code> <p>whether to flatten the schema. Partition values columns are         given the prefix <code>partition.</code>, statistics (null_count, min, and max) are         given the prefix <code>null_count.</code>, <code>min.</code>, and <code>max.</code>, and tags the         prefix <code>tags.</code>. Nested field names are concatenated with <code>.</code>.</p> <code>False</code> <p>Returns:</p> Type Description <code>RecordBatch</code> <p>a PyArrow RecordBatch containing the add action data.</p> Example <pre><code>from pprint import pprint\nfrom deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data, partition_by=[\"x\"])\ndt = DeltaTable(\"tmp\")\ndf = dt.get_add_actions().to_pandas()\ndf[\"path\"].sort_values(ignore_index=True)\n0    x=1/0\n1    x=2/0\n2    x=3/0\n</code></pre> <pre><code>df = dt.get_add_actions(flatten=True).to_pandas()\ndf[\"partition.x\"].sort_values(ignore_index=True)\n0    1\n1    2\n2    3\n</code></pre>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.history","title":"history","text":"<pre><code>history(limit: Optional[int] = None) -&gt; List[Dict[str, Any]]\n</code></pre> <p>Run the history command on the DeltaTable. The operations are returned in reverse chronological order.</p> <p>Parameters:</p> Name Type Description Default <code>limit</code> <code>Optional[int]</code> <p>the commit info limit to return</p> <code>None</code> <p>Returns:</p> Type Description <code>List[Dict[str, Any]]</code> <p>list of the commit infos registered in the transaction log</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.is_deltatable","title":"is_deltatable  <code>staticmethod</code>","text":"<pre><code>is_deltatable(table_uri: str, storage_options: Optional[Dict[str, str]] = None) -&gt; bool\n</code></pre> <p>Returns True if a Delta Table exists at specified path. Returns False otherwise.</p> <p>Parameters:</p> Name Type Description Default <code>table_uri</code> <code>str</code> <p>the path of the DeltaTable</p> required <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>a dictionary of the options to use for the storage backend</p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.load_as_version","title":"load_as_version","text":"<pre><code>load_as_version(version: Union[int, str, datetime]) -&gt; None\n</code></pre> <p>Load/time travel a DeltaTable to a specified version number, or a timestamp version of the table. If a string is passed then the argument should be an RFC 3339 and ISO 8601 date and time string format. If a datetime object without a timezone is passed, the UTC timezone will be assumed.</p> <p>Parameters:</p> Name Type Description Default <code>version</code> <code>Union[int, str, datetime]</code> <p>the identifier of the version of the DeltaTable to load</p> required Example <p>Use a version number <pre><code>dt = DeltaTable(\"test_table\")\ndt.load_as_version(1)\n</code></pre></p> <p>Use a datetime object <pre><code>dt.load_as_version(datetime(2023, 1, 1))\ndt.load_as_version(datetime(2023, 1, 1, tzinfo=timezone.utc))\n</code></pre></p> <p>Use a datetime in string format <pre><code>dt.load_as_version(\"2018-01-26T18:30:09Z\")\ndt.load_as_version(\"2018-12-19T16:39:57-08:00\")\ndt.load_as_version(\"2018-01-26T18:30:09.453+00:00\")\n</code></pre></p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.merge","title":"merge","text":"<pre><code>merge(source: Union[pyarrow.Table, pyarrow.RecordBatch, pyarrow.RecordBatchReader, ds.Dataset, pd.DataFrame], predicate: str, source_alias: Optional[str] = None, target_alias: Optional[str] = None, error_on_type_mismatch: bool = True, writer_properties: Optional[WriterProperties] = None, large_dtypes: Optional[bool] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; TableMerger\n</code></pre> <p>Pass the source data which you want to merge on the target delta table, providing a predicate in SQL query like format. You can also specify on what to do when the underlying data types do not match the underlying table.</p> <p>Parameters:</p> Name Type Description Default <code>source</code> <code>Union[Table, RecordBatch, RecordBatchReader, Dataset, DataFrame]</code> <p>source data</p> required <code>predicate</code> <code>str</code> <p>SQL like predicate on how to merge</p> required <code>source_alias</code> <code>Optional[str]</code> <p>Alias for the source table</p> <code>None</code> <code>target_alias</code> <code>Optional[str]</code> <p>Alias for the target table</p> <code>None</code> <code>error_on_type_mismatch</code> <code>bool</code> <p>specify if merge will return error if data types are mismatching :default = True</p> <code>True</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer</p> <code>None</code> <code>large_dtypes</code> <code>Optional[bool]</code> <p>Deprecated, will be removed in 1.0</p> <code>None</code> <code>arrow_schema_conversion_mode</code> <p>Large converts all types of data schema into Large Arrow types, passthrough keeps string/binary/list types untouched</p> required <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties for the commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.metadata","title":"metadata","text":"<pre><code>metadata() -&gt; Metadata\n</code></pre> <p>Get the current metadata of the DeltaTable.</p> <p>Returns:</p> Type Description <code>Metadata</code> <p>the current Metadata registered in the transaction log</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.partitions","title":"partitions","text":"<pre><code>partitions(partition_filters: Optional[List[Tuple[str, str, Any]]] = None) -&gt; List[Dict[str, str]]\n</code></pre> <p>Returns the partitions as a list of dicts. Example: <code>[{'month': '1', 'year': '2020', 'day': '1'}, ...]</code></p> <p>Parameters:</p> Name Type Description Default <code>partition_filters</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>The partition filters that will be used for getting the matched partitions, defaults to <code>None</code> (no filtering).</p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.protocol","title":"protocol","text":"<pre><code>protocol() -&gt; ProtocolVersions\n</code></pre> <p>Get the reader and writer protocol versions of the DeltaTable.</p> <p>Returns:</p> Type Description <code>ProtocolVersions</code> <p>the current ProtocolVersions registered in the transaction log</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.repair","title":"repair","text":"<pre><code>repair(dry_run: bool = False, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Repair the Delta Table by auditing active files that do not exist in the underlying filesystem and removes them. This can be useful when there are accidental deletions or corrupted files.</p> <p>Active files are ones that have an add action in the log, but no corresponding remove action. This operation creates a new FSCK transaction containing a remove action for each of the missing or corrupted files.</p> <p>Parameters:</p> Name Type Description Default <code>dry_run</code> <code>bool</code> <p>when activated, list only the files, otherwise add remove actions to transaction log. Defaults to False.</p> <code>False</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>The metrics from repair (FSCK) action.</p> Example <p><pre><code>from deltalake import DeltaTable\ndt = DeltaTable('TEST')\ndt.repair(dry_run=False)\n</code></pre> Results in <pre><code>{'dry_run': False, 'files_removed': ['6-0d084325-6885-4847-b008-82c1cf30674c-0.parquet', 5-4fba1d3e-3e20-4de1-933d-a8e13ac59f53-0.parquet']}\n</code></pre></p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.restore","title":"restore","text":"<pre><code>restore(target: Union[int, datetime, str], *, ignore_missing_files: bool = False, protocol_downgrade_allowed: bool = False, custom_metadata: Optional[Dict[str, str]] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Run the Restore command on the Delta Table: restore table to a given version or datetime.</p> <p>Parameters:</p> Name Type Description Default <code>target</code> <code>Union[int, datetime, str]</code> <p>the expected version will restore, which represented by int, date str or datetime.</p> required <code>ignore_missing_files</code> <code>bool</code> <p>whether the operation carry on when some data files missing.</p> <code>False</code> <code>protocol_downgrade_allowed</code> <code>bool</code> <p>whether the operation when protocol version upgraded.</p> <code>False</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>the metrics from restore.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.schema","title":"schema","text":"<pre><code>schema() -&gt; DeltaSchema\n</code></pre> <p>Get the current schema of the DeltaTable.</p> <p>Returns:</p> Type Description <code>Schema</code> <p>the current Schema registered in the transaction log</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.to_pandas","title":"to_pandas","text":"<pre><code>to_pandas(partitions: Optional[List[Tuple[str, str, Any]]] = None, columns: Optional[List[str]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, filters: Optional[Union[FilterType, Expression]] = None) -&gt; pd.DataFrame\n</code></pre> <p>Build a pandas dataframe using data from the DeltaTable.</p> <p>Parameters:</p> Name Type Description Default <code>partitions</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>A list of partition filters, see help(DeltaTable.files_by_partitions) for filter syntax</p> <code>None</code> <code>columns</code> <code>Optional[List[str]]</code> <p>The columns to project. This can be a list of column names to include (order and duplicates will be preserved)</p> <code>None</code> <code>filesystem</code> <code>Optional[Union[str, FileSystem]]</code> <p>A concrete implementation of the Pyarrow FileSystem or a fsspec-compatible interface. If None, the first file path will be used to determine the right FileSystem</p> <code>None</code> <code>filters</code> <code>Optional[Union[FilterType, Expression]]</code> <p>A disjunctive normal form (DNF) predicate for filtering rows, or directly a pyarrow.dataset.Expression. If you pass a filter you do not need to pass <code>partitions</code></p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.to_pyarrow_dataset","title":"to_pyarrow_dataset","text":"<pre><code>to_pyarrow_dataset(partitions: Optional[FilterConjunctionType] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, parquet_read_options: Optional[ParquetReadOptions] = None, schema: Optional[pyarrow.Schema] = None, as_large_types: bool = False) -&gt; pyarrow.dataset.Dataset\n</code></pre> <p>Build a PyArrow Dataset using data from the DeltaTable.</p> <p>Parameters:</p> Name Type Description Default <code>partitions</code> <code>Optional[FilterConjunctionType]</code> <p>A list of partition filters, see help(DeltaTable.files_by_partitions) for filter syntax</p> <code>None</code> <code>filesystem</code> <code>Optional[Union[str, FileSystem]]</code> <p>A concrete implementation of the Pyarrow FileSystem or a fsspec-compatible interface. If None, the first file path will be used to determine the right FileSystem</p> <code>None</code> <code>parquet_read_options</code> <code>Optional[ParquetReadOptions]</code> <p>Optional read options for Parquet. Use this to handle INT96 to timestamp conversion for edge cases like 0001-01-01 or 9999-12-31</p> <code>None</code> <code>schema</code> <code>Optional[Schema]</code> <p>The schema to use for the dataset. If None, the schema of the DeltaTable will be used. This can be used to force reading of Parquet/Arrow datatypes that DeltaLake can't represent in it's schema (e.g. LargeString). If you only need to read the schema with large types (e.g. for compatibility with Polars) you may want to use the <code>as_large_types</code> parameter instead.</p> <code>None</code> <code>as_large_types</code> <code>bool</code> <p>get schema with all variable size types (list, binary, string) as large variants (with int64 indices). This is for compatibility with systems like Polars that only support the large versions of Arrow types. If <code>schema</code> is passed it takes precedence over this option.</p> <code>False</code> <p>More info: https://arrow.apache.org/docs/python/generated/pyarrow.dataset.ParquetReadOptions.html</p> Example <p><code>deltalake</code> will work with any storage compliant with :class:<code>pyarrow.fs.FileSystem</code>, however the root of the filesystem has to be adjusted to point at the root of the Delta table. We can achieve this by wrapping the custom filesystem into a :class:<code>pyarrow.fs.SubTreeFileSystem</code>. <pre><code>import pyarrow.fs as fs\nfrom deltalake import DeltaTable\n\ntable_uri = \"s3://&lt;bucket&gt;/&lt;path&gt;\"\nraw_fs, normalized_path = fs.FileSystem.from_uri(table_uri)\nfilesystem = fs.SubTreeFileSystem(normalized_path, raw_fs)\n\ndt = DeltaTable(table_uri)\nds = dt.to_pyarrow_dataset(filesystem=filesystem)\n</code></pre></p> <p>Returns:</p> Type Description <code>Dataset</code> <p>the PyArrow dataset in PyArrow</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.to_pyarrow_table","title":"to_pyarrow_table","text":"<pre><code>to_pyarrow_table(partitions: Optional[List[Tuple[str, str, Any]]] = None, columns: Optional[List[str]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, filters: Optional[Union[FilterType, Expression]] = None) -&gt; pyarrow.Table\n</code></pre> <p>Build a PyArrow Table using data from the DeltaTable.</p> <p>Parameters:</p> Name Type Description Default <code>partitions</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>A list of partition filters, see help(DeltaTable.files_by_partitions) for filter syntax</p> <code>None</code> <code>columns</code> <code>Optional[List[str]]</code> <p>The columns to project. This can be a list of column names to include (order and duplicates will be preserved)</p> <code>None</code> <code>filesystem</code> <code>Optional[Union[str, FileSystem]]</code> <p>A concrete implementation of the Pyarrow FileSystem or a fsspec-compatible interface. If None, the first file path will be used to determine the right FileSystem</p> <code>None</code> <code>filters</code> <code>Optional[Union[FilterType, Expression]]</code> <p>A disjunctive normal form (DNF) predicate for filtering rows, or directly a pyarrow.dataset.Expression. If you pass a filter you do not need to pass <code>partitions</code></p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.update","title":"update","text":"<pre><code>update(updates: Optional[Dict[str, str]] = None, new_values: Optional[Dict[str, Union[int, float, str, datetime, bool, List[Any]]]] = None, predicate: Optional[str] = None, writer_properties: Optional[WriterProperties] = None, error_on_type_mismatch: bool = True, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p><code>UPDATE</code> records in the Delta Table that matches an optional predicate. Either updates or new_values needs to be passed for it to execute.</p> <p>Parameters:</p> Name Type Description Default <code>updates</code> <code>Optional[Dict[str, str]]</code> <p>a mapping of column name to update SQL expression.</p> <code>None</code> <code>new_values</code> <code>Optional[Dict[str, Union[int, float, str, datetime, bool, List[Any]]]]</code> <p>a mapping of column name to python datatype.</p> <code>None</code> <code>predicate</code> <code>Optional[str]</code> <p>a logical expression.</p> <code>None</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>error_on_type_mismatch</code> <code>bool</code> <p>specify if update will return error if data types are mismatching :default = True</p> <code>True</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:     the metrics from update</p> Example <p>Update some row values with SQL predicate</p> <p>This is equivalent to <code>UPDATE table SET deleted = true WHERE id = '3'</code> <pre><code>from deltalake import write_deltalake, DeltaTable\nimport pandas as pd\ndf = pd.DataFrame(\n    {\"id\": [\"1\", \"2\", \"3\"],\n    \"deleted\": [False, False, False],\n    \"price\": [10., 15., 20.]\n    })\nwrite_deltalake(\"tmp\", df)\ndt = DeltaTable(\"tmp\")\ndt.update(predicate=\"id = '3'\", updates = {\"deleted\": 'True'})\n\n{'num_added_files': 1, 'num_removed_files': 1, 'num_updated_rows': 1, 'num_copied_rows': 2, 'execution_time_ms': ..., 'scan_time_ms': ...}\n</code></pre></p> <p>Update all row values</p> <p>This is equivalent to <code>UPDATE table SET deleted = true, id = concat(id, '_old')</code>. <pre><code>dt.update(updates = {\"deleted\": 'True', \"id\": \"concat(id, '_old')\"})\n\n{'num_added_files': 1, 'num_removed_files': 1, 'num_updated_rows': 3, 'num_copied_rows': 0, 'execution_time_ms': ..., 'scan_time_ms': ...}\n</code></pre></p> <p>Use Python objects instead of SQL strings</p> <p>Use the <code>new_values</code> parameter instead of the <code>updates</code> parameter. For example, this is equivalent to <code>UPDATE table SET price = 150.10 WHERE id = '1'</code> <pre><code>dt.update(predicate=\"id = '1_old'\", new_values = {\"price\": 150.10})\n\n{'num_added_files': 1, 'num_removed_files': 1, 'num_updated_rows': 1, 'num_copied_rows': 2, 'execution_time_ms': ..., 'scan_time_ms': ...}\n</code></pre></p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.update_incremental","title":"update_incremental","text":"<pre><code>update_incremental() -&gt; None\n</code></pre> <p>Updates the DeltaTable to the latest version by incrementally applying newer versions.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.vacuum","title":"vacuum","text":"<pre><code>vacuum(retention_hours: Optional[int] = None, dry_run: bool = True, enforce_retention_duration: bool = True, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; List[str]\n</code></pre> <p>Run the Vacuum command on the Delta Table: list and delete files no longer referenced by the Delta table and are older than the retention threshold.</p> <p>Parameters:</p> Name Type Description Default <code>retention_hours</code> <code>Optional[int]</code> <p>the retention threshold in hours, if none then the value from <code>delta.deletedFileRetentionDuration</code> is used or default of 1 week otherwise.</p> <code>None</code> <code>dry_run</code> <code>bool</code> <p>when activated, list only the files, delete otherwise</p> <code>True</code> <code>enforce_retention_duration</code> <code>bool</code> <p>when disabled, accepts retention hours smaller than the value from <code>delta.deletedFileRetentionDuration</code>.</p> <code>True</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:     the list of files no longer referenced by the Delta Table and are older than the retention threshold.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.version","title":"version","text":"<pre><code>version() -&gt; int\n</code></pre> <p>Get the version of the DeltaTable.</p> <p>Returns:</p> Type Description <code>int</code> <p>The current version of the DeltaTable</p>","boost":2},{"location":"api/delta_table/delta_table_alterer/","title":"TableAlterer","text":"","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer","title":"deltalake.table.TableAlterer","text":"<pre><code>TableAlterer(table: DeltaTable)\n</code></pre> <p>API for various table alteration commands.</p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.add_columns","title":"add_columns","text":"<pre><code>add_columns(fields: Union[DeltaField, List[DeltaField]], custom_metadata: Optional[Dict[str, str]] = None, commit_properties: Optional[CommitProperties] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None) -&gt; None\n</code></pre> <p>Add new columns and/or update the fields of a stuctcolumn</p> <p>Parameters:</p> Name Type Description Default <code>fields</code> <code>Union[Field, List[Field]]</code> <p>fields to merge into schema</p> required <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> Example <p>from deltalake.schema import Field, PrimitiveType, StructType dt = DeltaTable(\"test_table\") new_fields = [     Field(\"baz\", StructType([Field(\"bar\", PrimitiveType(\"integer\"))])),     Field(\"bar\", PrimitiveType(\"integer\")) ] dt.alter.add_columns(     new_fields ) ```</p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.add_constraint","title":"add_constraint","text":"<pre><code>add_constraint(constraints: Dict[str, str], custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; None\n</code></pre> <p>Add constraints to the table. Limited to <code>single constraint</code> at once.</p> <p>Parameters:</p> Name Type Description Default <code>constraints</code> <code>Dict[str, str]</code> <p>mapping of constraint name to SQL-expression to evaluate on write</p> required <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> Example <pre><code>from deltalake import DeltaTable\ndt = DeltaTable(\"test_table_constraints\")\ndt.alter.add_constraint({\n    \"value_gt_5\": \"value &gt; 5\",\n})\n</code></pre> <p>Check configuration <pre><code>dt.metadata().configuration\n{'delta.constraints.value_gt_5': 'value &gt; 5'}\n</code></pre></p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.add_feature","title":"add_feature","text":"<pre><code>add_feature(feature: Union[TableFeatures, List[TableFeatures]], allow_protocol_versions_increase: bool = False, commit_properties: Optional[CommitProperties] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None) -&gt; None\n</code></pre> <p>Enable a table feature.</p> <p>Parameters:</p> Name Type Description Default <code>feature</code> <code>Union[TableFeatures, List[TableFeatures]]</code> <p>Table Feature e.g. Deletion Vectors, Change Data Feed</p> required <code>allow_protocol_versions_increase</code> <code>bool</code> <p>Allow the protocol to be implicitily bumped to reader 3 or writer 7</p> <code>False</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> Example <pre><code>from deltalake import DeltaTable\ndt = DeltaTable(\"test_table\")\ndt.alter.add_feature(TableFeatures.AppendOnly)\n</code></pre> <p>Check protocol <pre><code>dt.protocol()\nProtocolVersions(min_reader_version=1, min_writer_version=7, writer_features=['appendOnly'], reader_features=None)\n</code></pre></p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.drop_constraint","title":"drop_constraint","text":"<pre><code>drop_constraint(name: str, raise_if_not_exists: bool = True, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; None\n</code></pre> <p>Drop constraints from a table. Limited to <code>single constraint</code> at once.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>constraint name which to drop.</p> required <code>raise_if_not_exists</code> <code>bool</code> <p>set if should raise if not exists.</p> <code>True</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> Example <pre><code>from deltalake import DeltaTable\ndt = DeltaTable(\"test_table_constraints\")\ndt.metadata().configuration\n{'delta.constraints.value_gt_5': 'value &gt; 5'}\n</code></pre> <p>Drop the constraint <pre><code>dt.alter.drop_constraint(name = \"value_gt_5\")\n</code></pre></p> <p>Configuration after dropping <pre><code>dt.metadata().configuration\n{}\n</code></pre></p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.set_table_properties","title":"set_table_properties","text":"<pre><code>set_table_properties(properties: Dict[str, str], raise_if_not_exists: bool = True, custom_metadata: Optional[Dict[str, str]] = None, commit_properties: Optional[CommitProperties] = None) -&gt; None\n</code></pre> <p>Set properties from the table.</p> <p>Parameters:</p> Name Type Description Default <code>properties</code> <code>Dict[str, str]</code> <p>properties which set</p> required <code>raise_if_not_exists</code> <code>bool</code> <p>set if should raise if not exists.</p> <code>True</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> Example <pre><code>from deltalake import write_deltalake, DeltaTable\nimport pandas as pd\ndf = pd.DataFrame(\n    {\"id\": [\"1\", \"2\", \"3\"],\n    \"deleted\": [False, False, False],\n    \"price\": [10., 15., 20.]\n    })\nwrite_deltalake(\"tmp\", df)\n\ndt = DeltaTable(\"tmp\")\ndt.alter.set_table_properties({\"delta.enableChangeDataFeed\": \"true\"})\n</code></pre>","boost":10},{"location":"api/delta_table/delta_table_merger/","title":"TableMerger","text":"","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger","title":"deltalake.table.TableMerger","text":"<pre><code>TableMerger(builder: PyMergeBuilder, table: RawDeltaTable)\n</code></pre> <p>API for various table <code>MERGE</code> commands.</p>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.execute","title":"execute","text":"<pre><code>execute() -&gt; Dict[str, Any]\n</code></pre> <p>Executes <code>MERGE</code> with the previously provided settings in Rust with Apache Datafusion query engine.</p> <p>Returns:</p> Name Type Description <code>Dict</code> <code>Dict[str, Any]</code> <p>metrics</p>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_delete","title":"when_matched_delete","text":"<pre><code>when_matched_delete(predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Delete a matched row from the table only if the given <code>predicate</code> (if specified) is true for the matched row. If not specified it deletes all matches.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>(str | None, Optional)</code> <p>SQL like predicate on when to delete.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <p>Delete on a predicate</p> <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [2, 3], \"deleted\": [False, True]})\n\n(\n    dt.merge(\n        source=new_data,\n        predicate='target.x = source.x',\n        source_alias='source',\n        target_alias='target')\n    .when_matched_delete(\n        predicate=\"source.deleted = true\")\n    .execute()\n)\n{'num_source_rows': 2, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 0, 'num_target_rows_deleted': 1, 'num_target_rows_copied': 2, 'num_output_rows': 2, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  4\n1  2  5\n</code></pre> <p>Delete all records that were matched <pre><code>dt = DeltaTable(\"tmp\")\n(\n    dt.merge(\n        source=new_data,\n        predicate='target.x = source.x',\n        source_alias='source',\n        target_alias='target')\n    .when_matched_delete()\n    .execute()\n)\n{'num_source_rows': 2, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 0, 'num_target_rows_deleted': 1, 'num_target_rows_copied': 1, 'num_output_rows': 1, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas()\n   x  y\n0  1  4\n</code></pre></p>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_update","title":"when_matched_update","text":"<pre><code>when_matched_update(updates: Dict[str, str], predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Update a matched table row based on the rules defined by <code>updates</code>. If a <code>predicate</code> is specified, then it must evaluate to true for the row to be updated.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>updates</code> <code>Dict[str, str]</code> <p>a mapping of column name to update SQL expression.</p> required <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to update.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"1y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [1], \"1y\": [7]})\n\n(\n     dt.merge(\n         source=new_data,\n         predicate=\"target.x = source.x\",\n         source_alias=\"source\",\n         target_alias=\"target\")\n     .when_matched_update(updates={\"x\": \"source.x\", \"`1y`\": \"source.`1y`\"})\n     .execute()\n)\n{'num_source_rows': 1, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 1, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 2, 'num_output_rows': 3, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas()\n   x  y\n0  1  7\n1  2  5\n2  3  6\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_update_all","title":"when_matched_update_all","text":"<pre><code>when_matched_update_all(predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Updating all source fields to target fields, source and target are required to have the same field names. If a <code>predicate</code> is specified, then it must evaluate to true for the row to be updated.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to update all columns.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [1], \"y\": [7]})\n\n(\n    dt.merge(\n        source=new_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\")\n    .when_matched_update_all()\n    .execute()\n)\n{'num_source_rows': 1, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 1, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 2, 'num_output_rows': 3, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas()\n   x  y\n0  1  7\n1  2  5\n2  3  6\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_by_source_delete","title":"when_not_matched_by_source_delete","text":"<pre><code>when_not_matched_by_source_delete(predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Delete a target row that has no matches in the source from the table only if the given <code>predicate</code> (if specified) is true for the target row.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to delete when not matched by source.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_by_source_update","title":"when_not_matched_by_source_update","text":"<pre><code>when_not_matched_by_source_update(updates: Dict[str, str], predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Update a target row that has no matches in the source based on the rules defined by <code>updates</code>. If a <code>predicate</code> is specified, then it must evaluate to true for the row to be updated.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>updates</code> <code>Dict[str, str]</code> <p>a mapping of column name to update SQL expression.</p> required <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to update.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [2, 3, 4]})\n\n(\n   dt.merge(\n       source=new_data,\n       predicate='target.x = source.x',\n       source_alias='source',\n       target_alias='target')\n   .when_not_matched_by_source_update(\n       predicate = \"y &gt; 3\",\n       updates = {\"y\": \"0\"})\n   .execute()\n)\n{'num_source_rows': 3, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 1, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 2, 'num_output_rows': 3, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  0\n1  2  5\n2  3  6\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_insert","title":"when_not_matched_insert","text":"<pre><code>when_not_matched_insert(updates: Dict[str, str], predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Insert a new row to the target table based on the rules defined by <code>updates</code>. If a <code>predicate</code> is specified, then it must evaluate to true for the new row to be inserted.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>updates</code> <code>dict</code> <p>a mapping of column name to insert SQL expression.</p> required <code>predicate</code> <code>(str | None, Optional)</code> <p>SQL like predicate on when to insert.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [4], \"y\": [7]})\n\n(\n    dt.merge(\n        source=new_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\",)\n    .when_not_matched_insert(\n        updates={\n            \"x\": \"source.x\",\n            \"y\": \"source.y\",\n        })\n    .execute()\n)\n{'num_source_rows': 1, 'num_target_rows_inserted': 1, 'num_target_rows_updated': 0, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 3, 'num_output_rows': 4, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  4\n1  2  5\n2  3  6\n3  4  7\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_insert_all","title":"when_not_matched_insert_all","text":"<pre><code>when_not_matched_insert_all(predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Insert a new row to the target table, updating all source fields to target fields. Source and target are required to have the same field names. If a <code>predicate</code> is specified, then it must evaluate to true for the new row to be inserted.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to insert.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [4], \"y\": [7]})\n\n(\n   dt.merge(\n       source=new_data,\n       predicate='target.x = source.x',\n       source_alias='source',\n       target_alias='target')\n   .when_not_matched_insert_all()\n   .execute()\n)\n{'num_source_rows': 1, 'num_target_rows_inserted': 1, 'num_target_rows_updated': 0, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 3, 'num_output_rows': 4, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  4\n1  2  5\n2  3  6\n3  4  7\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_optimizer/","title":"TableOptimizer","text":"","boost":10},{"location":"api/delta_table/delta_table_optimizer/#deltalake.table.TableOptimizer","title":"deltalake.table.TableOptimizer","text":"<pre><code>TableOptimizer(table: DeltaTable)\n</code></pre> <p>API for various table optimization commands.</p>","boost":10},{"location":"api/delta_table/delta_table_optimizer/#deltalake.table.TableOptimizer.compact","title":"compact","text":"<pre><code>compact(partition_filters: Optional[FilterConjunctionType] = None, target_size: Optional[int] = None, max_concurrent_tasks: Optional[int] = None, min_commit_interval: Optional[Union[int, timedelta]] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Compacts small files to reduce the total number of files in the table.</p> <p>This operation is idempotent; if run twice on the same table (assuming it has not been updated) it will do nothing the second time.</p> <p>If this operation happens concurrently with any operations other than append, it will fail.</p> <p>Parameters:</p> Name Type Description Default <code>partition_filters</code> <code>Optional[FilterConjunctionType]</code> <p>the partition filters that will be used for getting the matched files</p> <code>None</code> <code>target_size</code> <code>Optional[int]</code> <p>desired file size after bin-packing files, in bytes. If not             provided, will attempt to read the table configuration value <code>delta.targetFileSize</code>.             If that value isn't set, will use default value of 256MB.</p> <code>None</code> <code>max_concurrent_tasks</code> <code>Optional[int]</code> <p>the maximum number of concurrent tasks to use for                     file compaction. Defaults to number of CPUs. More concurrent tasks can make compaction                     faster, but will also use more memory.</p> <code>None</code> <code>min_commit_interval</code> <code>Optional[Union[int, timedelta]]</code> <p>minimum interval in seconds or as timedeltas before a new commit is                     created. Interval is useful for long running executions. Set to 0 or timedelta(0), if you                     want a commit per partition.</p> <code>None</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>the metrics from optimize</p> Example <p>Use a timedelta object to specify the seconds, minutes or hours of the interval. <pre><code>from deltalake import DeltaTable, write_deltalake\nfrom datetime import timedelta\nimport pyarrow as pa\n\nwrite_deltalake(\"tmp\", pa.table({\"x\": [1], \"y\": [4]}))\nwrite_deltalake(\"tmp\", pa.table({\"x\": [2], \"y\": [5]}), mode=\"append\")\n\ndt = DeltaTable(\"tmp\")\ntime_delta = timedelta(minutes=10)\ndt.optimize.compact(min_commit_interval=time_delta)\n{'numFilesAdded': 1, 'numFilesRemoved': 2, 'filesAdded': ..., 'filesRemoved': ..., 'partitionsOptimized': 1, 'numBatches': 2, 'totalConsideredFiles': 2, 'totalFilesSkipped': 0, 'preserveInsertionOrder': True}\n</code></pre></p>","boost":10},{"location":"api/delta_table/delta_table_optimizer/#deltalake.table.TableOptimizer.z_order","title":"z_order","text":"<pre><code>z_order(columns: Iterable[str], partition_filters: Optional[FilterConjunctionType] = None, target_size: Optional[int] = None, max_concurrent_tasks: Optional[int] = None, max_spill_size: int = 20 * 1024 * 1024 * 1024, min_commit_interval: Optional[Union[int, timedelta]] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Reorders the data using a Z-order curve to improve data skipping.</p> <p>This also performs compaction, so the same parameters as compact() apply.</p> <p>Parameters:</p> Name Type Description Default <code>columns</code> <code>Iterable[str]</code> <p>the columns to use for Z-ordering. There must be at least one column.</p> required <code>partition_filters</code> <code>Optional[FilterConjunctionType]</code> <p>the partition filters that will be used for getting the matched files</p> <code>None</code> <code>target_size</code> <code>Optional[int]</code> <p>desired file size after bin-packing files, in bytes. If not             provided, will attempt to read the table configuration value <code>delta.targetFileSize</code>.             If that value isn't set, will use default value of 256MB.</p> <code>None</code> <code>max_concurrent_tasks</code> <code>Optional[int]</code> <p>the maximum number of concurrent tasks to use for                     file compaction. Defaults to number of CPUs. More concurrent tasks can make compaction                     faster, but will also use more memory.</p> <code>None</code> <code>max_spill_size</code> <code>int</code> <p>the maximum number of bytes allowed in memory before spilling to disk. Defaults to 20GB.</p> <code>20 * 1024 * 1024 * 1024</code> <code>min_commit_interval</code> <code>Optional[Union[int, timedelta]]</code> <p>minimum interval in seconds or as timedeltas before a new commit is                     created. Interval is useful for long running executions. Set to 0 or timedelta(0), if you                     want a commit per partition.</p> <code>None</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>the metrics from optimize</p> Example <p>Use a timedelta object to specify the seconds, minutes or hours of the interval. <pre><code>from deltalake import DeltaTable, write_deltalake\nfrom datetime import timedelta\nimport pyarrow as pa\n\nwrite_deltalake(\"tmp\", pa.table({\"x\": [1], \"y\": [4]}))\nwrite_deltalake(\"tmp\", pa.table({\"x\": [2], \"y\": [5]}), mode=\"append\")\n\ndt = DeltaTable(\"tmp\")\ntime_delta = timedelta(minutes=10)\ndt.optimize.z_order([\"x\"], min_commit_interval=time_delta)\n{'numFilesAdded': 1, 'numFilesRemoved': 2, 'filesAdded': ..., 'filesRemoved': ..., 'partitionsOptimized': 0, 'numBatches': 1, 'totalConsideredFiles': 2, 'totalFilesSkipped': 0, 'preserveInsertionOrder': True}\n</code></pre></p>","boost":10},{"location":"api/delta_table/metadata/","title":"Metadata","text":"","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata","title":"deltalake.Metadata  <code>dataclass</code>","text":"<pre><code>Metadata(table: RawDeltaTable)\n</code></pre> <p>Create a Metadata instance.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.configuration","title":"configuration  <code>property</code>","text":"<pre><code>configuration: Dict[str, str]\n</code></pre> <p>Return the DeltaTable properties.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.created_time","title":"created_time  <code>property</code>","text":"<pre><code>created_time: int\n</code></pre> <p>Return The time when this metadata action is created, in milliseconds since the Unix epoch of the DeltaTable.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.description","title":"description  <code>property</code>","text":"<pre><code>description: str\n</code></pre> <p>Return the user-provided description of the DeltaTable.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.id","title":"id  <code>property</code>","text":"<pre><code>id: int\n</code></pre> <p>Return the unique identifier of the DeltaTable.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.name","title":"name  <code>property</code>","text":"<pre><code>name: str\n</code></pre> <p>Return the user-provided identifier of the DeltaTable.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.partition_columns","title":"partition_columns  <code>property</code>","text":"<pre><code>partition_columns: List[str]\n</code></pre> <p>Return an array containing the names of the partitioned columns of the DeltaTable.</p>","boost":2},{"location":"how-delta-lake-works/architecture-of-delta-table/","title":"Architecture of a Delta Lake table","text":"<p>A Delta table consists of Parquet files that contain data and a transaction log that stores metadata about the transactions.</p> <p></p> <p>Let's create a Delta table, perform some operations, and inspect the files that are created.</p>"},{"location":"how-delta-lake-works/architecture-of-delta-table/#delta-lake-transaction-examples","title":"Delta Lake transaction examples","text":"<p>Start by creating a pandas DataFrame and writing it out to a Delta table.</p> <pre><code>import pandas as pd\nfrom deltalake import DeltaTable, write_deltalake\n\ndf = pd.DataFrame({\"num\": [1, 2, 3], \"letter\": [\"a\", \"b\", \"c\"]})\nwrite_deltalake(\"tmp/some-table\", df)\n</code></pre> <p>Now inspect the files created in storage:</p> <pre><code>tmp/some-table\n\u251c\u2500\u2500 0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u2514\u2500\u2500 00000000000000000000.json\n</code></pre> <p>The Parquet file stores the data that was written. The <code>_delta_log</code> directory stores metadata about the transactions. Let's inspect the <code>_delta_log/00000000000000000000.json</code> file.</p> <pre><code>{\n  \"protocol\": {\n    \"minReaderVersion\": 1,\n    \"minWriterVersion\": 1\n  }\n}\n{\n  \"metaData\": {\n    \"id\": \"b96ea1a2-1830-4da2-8827-5334cc6104ed\",\n    \"name\": null,\n    \"description\": null,\n    \"format\": {\n      \"provider\": \"parquet\",\n      \"options\": {}\n    },\n    \"schemaString\": \"{\\\"type\\\":\\\"struct\\\",\\\"fields\\\":[{\\\"name\\\":\\\"num\\\",\\\"type\\\":\\\"long\\\",\\\"nullable\\\":true,\\\"metadata\\\":{}},{\\\"name\\\":\\\"letter\\\",\\\"type\\\":\\\"string\\\",\\\"nullable\\\":true,\\\"metadata\\\":{}}]}\",\n    \"partitionColumns\": [],\n    \"createdTime\": 1701740315599,\n    \"configuration\": {}\n  }\n}\n{\n  \"add\": {\n    \"path\": \"0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\",\n    \"size\": 2208,\n    \"partitionValues\": {},\n    \"modificationTime\": 1701740315597,\n    \"dataChange\": true,\n    \"stats\": \"{\\\"numRecords\\\": 3, \\\"minValues\\\": {\\\"num\\\": 1, \\\"letter\\\": \\\"a\\\"}, \\\"maxValues\\\": {\\\"num\\\": 3, \\\"letter\\\": \\\"c\\\"}, \\\"nullCount\\\": {\\\"num\\\": 0, \\\"letter\\\": 0}}\"\n  }\n}\n{\n  \"commitInfo\": {\n    \"timestamp\": 1701740315602,\n    \"operation\": \"CREATE TABLE\",\n    \"operationParameters\": {\n      \"location\": \"file:///Users/matthew.powers/Documents/code/delta/delta-examples/notebooks/python-deltalake/tmp/some-table\",\n      \"metadata\": \"{\\\"configuration\\\":{},\\\"created_time\\\":1701740315599,\\\"description\\\":null,\\\"format\\\":{\\\"options\\\":{},\\\"provider\\\":\\\"parquet\\\"},\\\"id\\\":\\\"b96ea1a2-1830-4da2-8827-5334cc6104ed\\\",\\\"name\\\":null,\\\"partition_columns\\\":[],\\\"schema\\\":{\\\"fields\\\":[{\\\"metadata\\\":{},\\\"name\\\":\\\"num\\\",\\\"nullable\\\":true,\\\"type\\\":\\\"long\\\"},{\\\"metadata\\\":{},\\\"name\\\":\\\"letter\\\",\\\"nullable\\\":true,\\\"type\\\":\\\"string\\\"}],\\\"type\\\":\\\"struct\\\"}}\",\n      \"protocol\": \"{\\\"minReaderVersion\\\":1,\\\"minWriterVersion\\\":1}\",\n      \"mode\": \"ErrorIfExists\"\n    },\n    \"clientVersion\": \"delta-rs.0.17.0\"\n  }\n}\n</code></pre> <p>The transaction log file contains the following information:</p> <ul> <li>the files added to the Delta table</li> <li>schema of the files</li> <li>column level metadata including the min/max value for each file</li> </ul> <p>Create another pandas DataFrame and append it to the Delta table to see how this transaction is recorded.</p> <pre><code>df = pd.DataFrame({\"num\": [8, 9], \"letter\": [\"dd\", \"ee\"]})\nwrite_deltalake(f\"{cwd}/tmp/delta-table\", df, mode=\"append\")\n</code></pre> <p>Here are the files in storage:</p> <pre><code>tmp/some-table\n\u251c\u2500\u2500 0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\n\u251c\u2500\u2500 1-57abb6fb-2249-43ba-a7be-cf09bcc230de-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u251c\u2500\u2500 00000000000000000000.json\n    \u2514\u2500\u2500 00000000000000000001.json\n</code></pre> <p>Here are the contents of the <code>_delta_log/00000000000000000001.json</code> file:</p> <pre><code>{\n  \"add\": {\n    \"path\": \"1-57abb6fb-2249-43ba-a7be-cf09bcc230de-0.parquet\",\n    \"size\": 2204,\n    \"partitionValues\": {},\n    \"modificationTime\": 1701740386169,\n    \"dataChange\": true,\n    \"stats\": \"{\\\"numRecords\\\": 2, \\\"minValues\\\": {\\\"num\\\": 8, \\\"letter\\\": \\\"dd\\\"}, \\\"maxValues\\\": {\\\"num\\\": 9, \\\"letter\\\": \\\"ee\\\"}, \\\"nullCount\\\": {\\\"num\\\": 0, \\\"letter\\\": 0}}\"\n  }\n}\n{\n  \"commitInfo\": {\n    \"timestamp\": 1701740386169,\n    \"operation\": \"WRITE\",\n    \"operationParameters\": {\n      \"partitionBy\": \"[]\",\n      \"mode\": \"Append\"\n    },\n    \"clientVersion\": \"delta-rs.0.17.0\"\n  }\n}\n</code></pre> <p>The transaction log records that the second file has been persisted in the Delta table.</p> <p>Now create a third pandas DataFrame and overwrite the Delta table with the new data.</p> <pre><code>df = pd.DataFrame({\"num\": [11, 22], \"letter\": [\"aa\", \"bb\"]})\nwrite_deltalake(f\"{cwd}/tmp/delta-table\", df, mode=\"append\")\n</code></pre> <p>Here are the files in storage:</p> <pre><code>tmp/some-table\n\u251c\u2500\u2500 0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\n\u251c\u2500\u2500 1-57abb6fb-2249-43ba-a7be-cf09bcc230de-0.parquet\n\u251c\u2500\u2500 2-95ef2108-480c-4b89-96f0-ff9185dab9ad-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u251c\u2500\u2500 00000000000000000000.json\n    \u251c\u2500\u2500 00000000000000000001.json\n    \u2514\u2500\u2500 00000000000000000002.json\n</code></pre> <p>Here are the contents of the <code>_delta_log/0002.json</code> file:</p> <pre><code>{\n  \"add\": {\n    \"path\": \"2-95ef2108-480c-4b89-96f0-ff9185dab9ad-0.parquet\",\n    \"size\": 2204,\n    \"partitionValues\": {},\n    \"modificationTime\": 1701740465102,\n    \"dataChange\": true,\n    \"stats\": \"{\\\"numRecords\\\": 2, \\\"minValues\\\": {\\\"num\\\": 11, \\\"letter\\\": \\\"aa\\\"}, \\\"maxValues\\\": {\\\"num\\\": 22, \\\"letter\\\": \\\"bb\\\"}, \\\"nullCount\\\": {\\\"num\\\": 0, \\\"letter\\\": 0}}\"\n  }\n}\n{\n  \"remove\": {\n    \"path\": \"0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\",\n    \"deletionTimestamp\": 1701740465102,\n    \"dataChange\": true,\n    \"extendedFileMetadata\": false,\n    \"partitionValues\": {},\n    \"size\": 2208\n  }\n}\n{\n  \"remove\": {\n    \"path\": \"1-57abb6fb-2249-43ba-a7be-cf09bcc230de-0.parquet\",\n    \"deletionTimestamp\": 1701740465102,\n    \"dataChange\": true,\n    \"extendedFileMetadata\": false,\n    \"partitionValues\": {},\n    \"size\": 2204\n  }\n}\n{\n  \"commitInfo\": {\n    \"timestamp\": 1701740465102,\n    \"operation\": \"WRITE\",\n    \"operationParameters\": {\n      \"mode\": \"Overwrite\",\n      \"partitionBy\": \"[]\"\n    },\n    \"clientVersion\": \"delta-rs.0.17.0\"\n  }\n}\n</code></pre> <p>This transaction adds a data file and marks the two exising data files for removal. Marking a file for removal in the transaction log is known as \"tombstoning the file\" or a \"logical delete\". This is different from a \"physical delete\" which actually removes the data file from storage.</p>"},{"location":"how-delta-lake-works/architecture-of-delta-table/#how-delta-table-operations-differ-from-data-lakes","title":"How Delta table operations differ from data lakes","text":"<p>Data lakes consist of data files persisted in storage. They don't have a transaction log that retain metadata about the transactions.</p> <p>Data lakes perform transactions differently than Delta tables.</p> <p>When you perform an overwrite tranasction with a Delta table, you logically delete the exiting data without physically removing it.</p> <p>Data lakes don't support logical deletes, so you have to physically delete the data from storage.</p> <p>Logical data operations are safer because they can be rolled back if they don't complete successfully. Physically removing data from storage can be dangerous, especially if it's before a transaction is complete.</p> <p>We're now ready to look into Delta Lake ACID transactions in more detail.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/","title":"Delta Lake Transactions","text":"<p>This page teaches you about Delta Lake transactions and why transactions are important in production data settings. Data lakes don\u2019t support transactions and this is a huge downside because they offer a poor user experience, lack functionality, and can easily be corrupted.</p> <p>Transactions on Delta Lake tables are operations that change the state of table and record descriptive entries (metadata) of those changes to the Delta Lake transaction log. Here are some examples of transactions:</p> <ul> <li>Deleting rows</li> <li>Appending to the table</li> <li>Compacting small files</li> <li>Upserting</li> <li>Overwriting rows</li> </ul> <p>All Delta Lake write operations are transactions in Delta tables. Reads actually aren\u2019t technically transactions because they don\u2019t result in new entries being appended to the transaction log.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#what-are-transactions","title":"What are transactions?","text":"<p>Transactions are any Delta operation that change the underlying files of a Delta table and result in new entries metadata entries in the transaction log. Some Delta operations rearrange data in the existing table (like Z Ordering the table or compacting the small files) and these are also transactions. Let\u2019s look at a simple example.</p> <p>Suppose you have a Delta table with the following data:</p> <pre><code>num animal\n1   cat\n2   dog\n3   snake\n</code></pre> <p>Here\u2019s how to create this Delta table:</p> <pre><code>import pandas as pd\nfrom deltalake import write_deltalake, DeltaTable\n\ndf = pd.DataFrame({\"num\": [1, 2, 3], \"animal\": [\"cat\", \"dog\", \"snake\"]})\nwrite_deltalake(\"tmp/my-delta-table\", df)\n</code></pre> <p>Here are the files created in storage.</p> <pre><code>tmp/my-delta-table\n\u251c\u2500\u2500 0-fea2de92-861a-423e-9708-a9e91dafb27b-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u2514\u2500\u2500 00000000000000000000.json\n</code></pre> <p>Let\u2019s perform an operation to delete every animal from the Delta table that is a cat.</p> <pre><code>dt = DeltaTable(\"tmp/my-delta-table\")\ndt.delete(\"animal = 'cat'\")\n</code></pre> <p>Let\u2019s take a look at the contents of the Delta table now that the transaction is complete:</p> <pre><code>tmp/my-delta-table\n\u251c\u2500\u2500 0-fea2de92-861a-423e-9708-a9e91dafb27b-0.parquet\n\u251c\u2500\u2500 _delta_log\n\u2502   \u251c\u2500\u2500 00000000000000000000.json\n\u2502   \u2514\u2500\u2500 00000000000000000001.json\n\u2514\u2500\u2500 part-00001-90312b96-b487-4a8f-9edc-1b9b3963f136-c000.snappy.parquet\n</code></pre> <p>Notice the <code>00000000000000000001.json</code> file that was added to the transaction log to record this transaction. Let\u2019s inspect the content of the file.</p> <pre><code>{\n  \"add\": {\n    \"path\": \"part-00001-90312b96-b487-4a8f-9edc-1b9b3963f136-c000.snappy.parquet\",\n    \"partitionValues\": {},\n    \"size\": 858,\n    \"modificationTime\": 1705070631953,\n    \"dataChange\": true,\n    \"stats\": \"{\\\"numRecords\\\":2,\\\"minValues\\\":{\\\"num\\\":2,\\\"animal\\\":\\\"dog\\\"},\\\"maxValues\\\":{\\\"num\\\":3,\\\"animal\\\":\\\"snake\\\"},\\\"nullCount\\\":{\\\"num\\\":0,\\\"animal\\\":0}}\",\n    \"tags\": null,\n    \"deletionVector\": null,\n    \"baseRowId\": null,\n    \"defaultRowCommitVersion\": null,\n    \"clusteringProvider\": null\n  }\n}\n{\n  \"remove\": {\n    \"path\": \"0-fea2de92-861a-423e-9708-a9e91dafb27b-0.parquet\",\n    \"dataChange\": true,\n    \"deletionTimestamp\": 1705070631953,\n    \"extendedFileMetadata\": true,\n    \"partitionValues\": {},\n    \"size\": 895\n  }\n}\n{\n  \"commitInfo\": {\n    \"timestamp\": 1705070631953,\n    \"operation\": \"DELETE\",\n    \"operationParameters\": {\n      \"predicate\": \"animal = 'cat'\"\n    },\n    \"readVersion\": 0,\n    \"operationMetrics\": {\n      \"execution_time_ms\": 8013,\n      \"num_added_files\": 1,\n      \"num_copied_rows\": 2,\n      \"num_deleted_rows\": 1,\n      \"num_removed_files\": 1,\n      \"rewrite_time_ms\": 2,\n      \"scan_time_ms\": 5601\n    },\n    \"clientVersion\": \"delta-rs.0.17.0\"\n  }\n}\n</code></pre> <p>We can see that this transaction includes two components:</p> <ul> <li>Remove file <code>0-fea2de92-861a-423e-9708-a9e91dafb27b-0.parquet</code></li> <li>Add file <code>part-00001-90312b96-b487-4a8f-9edc-1b9b3963f136-c000.snappy.parquet</code></li> </ul> <p>Transactions are recorded in the transaction log. The transaction log is also referred to as the table metadata and is the <code>_delta_log</code> directory in storage.</p> <p>Let\u2019s see how Delta Lake implements transactions.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#how-delta-lake-implements-transactions","title":"How Delta Lake implements transactions","text":"<p>Here is how Delta Lake implements transactions:</p> <ol> <li>Read the existing metadata</li> <li>Read the existing Parquet data files</li> <li>Write the Parquet files for the current transaction</li> <li>Record the new transaction in the transaction log (if there are no conflicts)</li> </ol> <p>Let\u2019s recall our delete operation from the prior section and see how it fits into this transaction model:</p> <ol> <li>We read the existing metadata to find the file paths for the existing Parquet files</li> <li>We read the existing Parquet files and identify the files that contains data that should be removed</li> <li>We write new Parquet files with the deleted data filtered out</li> <li>Once the new Parquet files are written, we check for conflicts and then make an entry in the transaction log. The next section will discuss transaction conflicts in more detail.</li> </ol> <p>Blind append operations can skip a few steps and are executed as follows:</p> <ol> <li>Write the Parquet files for the current transaction</li> <li>Record the new transaction in the metadata</li> </ol> <p>Delta implements a non locking MVCC (multi version concurrency control) so writers optimistically write new data and simply abandon the transaction if it conflicts at the end. The alternative would be getting a lock at the start thereby guaranteeing the transaction immediately.</p> <p>Let\u2019s look at the case when a Delta Lake transaction conflicts.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#how-delta-lake-transactions-can-conflict","title":"How Delta Lake transactions can conflict","text":"<p>Suppose you have a transaction that deletes a row of data that\u2019s stored in FileA (Transaction 1). While this job is running, there is another transaction that deletes some other rows in FileA (Transaction 2). Transaction 1 finishes running first and is recorded in the metadata.</p> <p>Before Transaction 2 is recorded as a transaction, it will check the metadata, find that Transaction 2 conflicts with a transaction that was already recorded (from Transaction 1), and error without recording a new transaction.</p> <p>Transactions 2 will write Parquet data files, but will not be recorded as a transaction, so the data files will be ignored. The zombie Parquet files can be easily cleaned up via subsequent vacuum operations.</p> <p>Transaction 2 must fail otherwise it would cause the data to be incorrect.</p> <p>Delta Lake transactions prevent users from making changes that would corrupt the table. Transaction conflict behavior can differ based on isolation level, which controls the degree to which a transaction must be isolated from modifications made by other concurrent transactions. More about this in the concurrency section.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#transactions-rely-on-atomic-primitives-storage-guarantees","title":"Transactions rely on atomic primitives storage guarantees","text":"<p>Suppose you have two transactions that are finishishing at the same exact time. Both of these transactions look at the existing Delta Lake transaction log, see that the latest transaction was <code>003.json</code> and determine that the next entry should be <code>004.json</code>.</p> <p>If both transactions are recorded in the <code>004.json</code> file, then one of them will be clobbered, and the transaction log entry for the clobbered metadata entry will be lost.</p> <p>Delta tables rely on storage systems that provide atomic primitives for safe concurrency. The storage system must allow Delta Lake to write the file, only if it does not exist already, and error out otherwise. The storage system must NOT permit concurrent writers to overwrite existing metadata entries.</p> <p>Some clouds have filesystems that don\u2019t explicitly support these atomic primitives, and therefore must be coupled with other services to provide the necessary guarantees.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#delta-lake-transactions-are-only-for-a-single-table","title":"Delta Lake transactions are only for a single table","text":"<p>Delta Lake transactions are only valid for a single table.</p> <p>Some databases offer transaction support for operations that impact multiple tables. Delta Lake does not support multi-table transactions.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#data-lakes-dont-support-transactions","title":"Data lakes don\u2019t support transactions","text":"<p>Data lakes consist of many files in a storage system (e.g. a cloud storage system) and don\u2019t support transactions.</p> <p>Data lakes don\u2019t have a metadata layer, conflict resolution, or any way to store information about transactions.</p> <p>Data lakes are prone to multiple types of errors because they don\u2019t support transactions:</p> <ul> <li>Easy to corrupt</li> <li>Downtime/unstable state while jobs are running</li> <li>Operations can conflict</li> </ul> <p>Data lakes have many downsides and it\u2019s almost always better to use a lakehouse storage system like Delta Lake compared to a data lake.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#acid-transactions","title":"ACID Transactions","text":"<p>We\u2019ve already explored how Delta Lake supports transactions. This section explains how Delta Lake transactions have the Atomic, Consistent, Isolated and Durable (ACID transaction) properties. Reading this section is optional.</p> <p>ACID transactions are commonplace in databases but notably absent for data lakes.</p> <p>Delta Lake\u2019s ACID transaction support is one of the major reasons it is almost always a better option than a data lake.</p> <p>Let\u2019s explore how Delta Lake allows for ACID transactions.</p> <p>Atomic transactions</p> <p>An atomic transaction either fully completes or fully fails, with nothing in between.</p> <p>Delta Lake transactions are atomic, unlike data lake transactions that are not atomic.</p> <p>Suppose you have a job that\u2019s writing 100 files to a table. Further suppose that the job errors out and the cluster dies after writing 40 files:</p> <ul> <li>For a Delta table, no additional data will be added to the table. Parquet files were written to the table, but the job errored, so no transaction log entry was added and no data was added to the table.</li> <li>For a data lake, the 40 files are added and the transaction \u201cpartially succeeds\u201d.</li> </ul> <p>For data tables, it\u2019s almost always preferable to have a transaction that \u201cfully fails\u201d instead of one that \u201cpartially succeeds\u201d because partial writes are hard to unwind and debug.</p> <p>Delta Lake implements atomic transactions by writing data files first before making a new entry in the Delta transaction log.</p> <p>These guarantees are provided at the protocol level through the \"transaction\" abstraction. We\u2019ve already discussed what constitutes a transaction for Delta Lake.</p> <p>If there is an error with the transaction and some files don\u2019t get written, then no metadata entry is made and the partial data write is ignored. The zombie Parquet files can be easily cleaned up via subsequent vacuum operations.</p> <p>Now let\u2019s look at how Delta Lake also provides consistent transactions.</p> <p>Consistent transactions</p> <p>Consistency means that transactions won\u2019t violate integrity constraints on the Delta table.</p> <p>Delta Lake has two types of consistency checks:</p> <ul> <li>Schema enforcement checks</li> <li>Column constraints</li> </ul> <p>Schema enforcement checks verify that new data appended to a Delta table matches the schema of the existing table. You cannot append data with a different schema, unless you enable schema evolution.</p> <p>Delta Lake column constraints allow users to specify the requirements of data that\u2019s added to a Delta table. For example, if you have an age column with a constraint that requires the value to be positive, then Delta Lake will reject appends of any data that doesn\u2019t meet the constraint.</p> <p>Data lakes don\u2019t support schema enforcement or column constraints. That\u2019s another reason why data lakes are not ACID-compliant.</p> <p>Isolated transactions</p> <p>Isolation means that transactions are applied to a Delta table sequentially.</p> <p>Delta Lake transactions are persisted in monotonically increasing transaction files, as we saw in the previous example. First <code>00000000000000000000.json</code>, then <code>00000000000000000001.json</code>, then <code>00000000000000000002.json</code>, and so on.</p> <p>Delta Lake uses concurrency control to ensure that transactions are executed sequentially, even when user operations are performed concurrently. The next page of this guide explains concurrency in Delta Lake in detail.</p> <p>Durable transactions</p> <p>Delta tables are generally persisted in cloud object stores which provide durability guarantees.</p> <p>Durability means that all transactions that are successfully completed will always remain persisted, even if there are service outages or program crashes.</p> <p>Suppose you have a Delta table that\u2019s persisted in Azure blob storage. The Delta table transactions that are committed will always remain available, even in these circumstances:</p> <ul> <li>When there are Azure service outages</li> <li>If a computation cluster that\u2019s writing the Delta table crashes for some reason</li> <li>Two operations are running concurrently and one of them fails</li> </ul> <p>Successful transactions are always registered in the Delta table and persisted no matter what.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#conclusion","title":"Conclusion","text":"<p>Delta Lake supports transactions which provide necessary reliability guarantees for production data systems.</p> <p>Vanilla data lakes don\u2019t provide transactions and this can cause nasty bugs and a bad user experience. Let\u2019s look at a couple of scenarios when the lack of transactions cause a poor user experience:</p> <ul> <li>While running a compaction operation on a data lake, newly compacted \u201cright sized\u201d files are added before the small files are deleted. If you read the data lake while this operation is running, you will see duplicate data.</li> <li>While writing to a data lake, a job might fail, which leaves behind partially written files. These files are corrupt, which means that the data lake cannot be read until the corrupt files are manually removed.</li> <li>Users want to run a simple DML operation like deleting a few rows of data which require a few files to be rewritten. This operation renders the data lake unusable until it\u2019s done running.</li> </ul> <p>Transactions are a key advantage of Delta Lake vs. data lakes. There are many other advantages, but proper transactions are necessary in production data environments.</p> <p>Let's take a look at File Skipping in the next section.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/","title":"Delta Lake File Skipping","text":"<p>Delta tables store file-level metadata information, which allows for a powerful optimization called file skipping.</p> <p>This page explains how Delta Lake implements file skipping, how to optimize your tables to maximize file skipping, and the benefits of file skipping.</p> <p>Let\u2019s start by looking at the file-level metadata in Delta tables.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#delta-lake-file-metadata","title":"Delta Lake file metadata","text":"<p>Delta Lake stores metadata about each file's min/max values in the table.  Query engines can skip entire files when they don\u2019t contain data that\u2019s relevant to the query.</p> <p>Suppose you have a Delta table with data stored in two files and has the following metadata.</p> <pre><code>filename    min_name    max_name    min_age max_age\nfileA       alice       joy         12      46  \nfileB       allan       linda       34      78\n</code></pre> <p>Suppose you want to run the following query: <code>select * from the_table where age &amp;lt; 20</code>.</p> <p>The engine only needs to read <code>fileA</code> to execute this query.  <code>fileB</code> has a <code>min_age</code> of 34, so we know there aren\u2019t any rows of data with an <code>age</code> less than 20.</p> <p>The benefit of file skipping depends on the query and the data layout of the Delta table.  Some queries cannot take advantage of any file skipping.  Here\u2019s an example query that does not benefit from file skipping: <code>select * from the_table group by age</code>.</p> <p>Let\u2019s recreate this example with Polars to drive the point home.</p> <p>Start by writing out one file of data:</p> <pre><code>import polars as pl\nfrom deltalake import DeltaTable\n\ndf = pl.DataFrame({\"name\": [\"alice\", \"cat\", \"joy\"], \"age\": [12, 35, 46]})\ndf.write_delta(\"tmp/a_table\")\n</code></pre> <p>Now, write out another file of data:</p> <pre><code>df = pl.DataFrame({\"name\": [\"allan\", \"brian\", \"linda\"], \"age\": [34, 35, 78]})\ndf.write_delta(\"tmp/a_table\", mode=\"append\")\n</code></pre> <p>Here are the contents of the Delta table:</p> <pre><code>tmp/a_table\n\u251c\u2500\u2500 0-7d414a88-a634-4c2f-9c5b-c29b6ee5f524-0.parquet\n\u251c\u2500\u2500 1-0617ef60-b17b-46a5-9b0f-c7dda1b73eee-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u251c\u2500\u2500 00000000000000000000.json\n    \u2514\u2500\u2500 00000000000000000001.json\n</code></pre> <p>Now run a query to fetch all the records where the age is less than 20:</p> <pre><code>pl.scan_delta(\"tmp/a_table\").filter(pl.col(\"age\") &lt; 20).collect()\n</code></pre> <pre><code>+-------+-----+\n| name  | age |\n| ---   | --- |\n| str   | i64 |\n+=============+\n| alice | 12  |\n+-------+-----+\n</code></pre> <p>Polars can use the Delta table metadata to skip the file that does not contain data relevant to the query.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#how-delta-lake-implements-file-skipping","title":"How Delta Lake implements file skipping","text":"<p>Here\u2019s how engines execute queries on Delta tables:</p> <ul> <li>Start by reading the transaction log to get the file paths, file sizes, and min/max value for each column</li> <li>Parse the query and push down the predicates to skip files</li> <li>Read the minimal subset of the files needed for the query</li> </ul> <p>Some file formats don\u2019t allow for file skipping.  For example, CSV files don\u2019t have file-level metadata, so query engines can\u2019t read a minimal subset of the data.  The query engine has to check all the files, even if they don\u2019t contain any relevant data.</p> <p>When data is in Parquet files, the query engine can open up all the files, read the footers, build the file-level metadata, and perform file skipping.  Fetching metadata in each file is slower than grabbing the pre-built file-level metadata from the transaction log.</p> <p>Now, let\u2019s see how to structure your tables to allow for more file skipping.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#file-skipping-for-different-file-sizes","title":"File skipping for different file sizes","text":"<p>Delta tables store data in files.  Smaller files allow for more file skipping compared to bigger files.</p> <p>However, an excessive number of small files isn\u2019t good because it creates I/O overhead and slows down queries.</p> <p>Your Delta tables should have files that are \u201cright-sized\u201d.  For a table with 150 GB of data, 5 GB files would probably be too large, and 10 KB files would be too small.  It\u2019s generally best to store data in files that are between 100 MB and 1 GB.</p> <p>Delta Lake has an optimize function that performs small file compaction, so you don\u2019t need to program this logic yourself.</p> <p>Now, let's investigate how to store data in files to maximize the file skipping opportunities.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#how-to-maximize-file-skipping","title":"How to maximize file skipping","text":"<p>You can maximize file-skipping by colocating similar data in the same files.</p> <p>Suppose you have a table with test scores and frequently run queries that filter based on the <code>test_score</code> column.</p> <pre><code>filename    min_test_score  max_test_score\nfileA       45              100\nfileB       65              98\nfileC       50              96\n</code></pre> <p>Suppose you want to run the following query: <code>select * from exams where test_score &gt; 90</code>.</p> <p>This query cannot skip files, given the current organization of the data.  You can rearrange the data to colocate similar test scores in the same files to allow for file skipping.  Here\u2019s the new layout: </p> <pre><code>filename    min_test_score  max_test_score\nfileD       45              70\nfileE       55              80\nfileF       78              100\n</code></pre> <p>The query (<code>select * from exams where test_score &gt; 90</code>) can skip two of the three files with the new Delta table layout.  The query engine only has to read <code>fileF</code> for this query.</p> <p>Now, let\u2019s look at how file skipping works with string values.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#how-file-skipping-works-with-strings","title":"How file skipping works with strings","text":"<p>File skipping is also effective when filtering on string values.</p> <p>Suppose you have a table with <code>person_name</code> and <code>country</code> columns.  There are millions of rows of data.  Here are the first three rows of data:</p> <pre><code>person_name country\nperson1     angola\nperson2     china\nperson3     mexico\n</code></pre> <p>The Delta table contains three files with the following metadata:</p> <pre><code>filename    min_country max_country\nfileA       albania     mali\nfileB       libia       paraguay\nfileC       oman        zimbabwe\n</code></pre> <p>Suppose you want to run the following query: <code>select * from some_people where country = 'austria'</code>.</p> <p>You only need to read the data in <code>fileA</code> to run this query.  The <code>min_country</code> value for <code>fileB</code> and <code>fileC</code> are greater than \u201caustria\u201d, so we know those files don\u2019t contain any data relevant to the query.</p> <p>File skipping can also be a robust optimization for string values.  Now, let\u2019s see how file skipping works for partitioned tables.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#file-skipping-for-partitioned-tables","title":"File skipping for partitioned tables","text":"<p>You can partition Delta tables for file skipping as well.  Suppose we have the same data as in the previous section, but the table is partitioned by <code>country</code>.</p> <p>Here\u2019s the Delta table:</p> <pre><code>filename    partition\nfileA       albania\nfileB       libia\nfileC       oman\nfileD       jamaica\nfileE       albania\nfileF       oman\n</code></pre> <p>Suppose you want to run the following query on this partitioned table: <code>select * from some_partitioned_table where country = 'albania'</code>.</p> <p>You only need to read <code>fileA</code> and <code>fileE</code> to execute this query.  Delta Lake provides the file-level partition metadata in the transaction log so that this query will run quickly.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#conclusion","title":"Conclusion","text":"<p>Delta Lake allows for file skipping, which is a powerful performance optimization.</p> <p>Delta Lake also provides built-in utilities to colocate data in the same files like partitioning, Z Ordering, and compaction to improve file skipping.</p> <p>Delta Lake users need to know how to assess the tradeoffs of these techniques to optimize file skipping.  Users also need to understand the most frequent query patterns of their tables to best allow for file maximal file skipping.</p>"},{"location":"integrations/delta-lake-arrow/","title":"Delta Lake Arrow Integrations","text":"<p>Delta Lake tables can be exposed as Arrow tables and Arrow datasets, which allows for interoperability with a variety of query engines.</p> <p>This page shows you how to convert Delta tables to Arrow data structures and teaches you the difference between Arrow tables and Arrow datasets.  Tables are \"eager\" and datasets are \"lazy\", which has important performance implications, keep reading to learn more!</p>"},{"location":"integrations/delta-lake-arrow/#delta-lake-to-arrow-dataset","title":"Delta Lake to Arrow Dataset","text":"<p>Delta tables can easily be exposed as Arrow datasets.  This makes it easy for any query engine that can read Arrow datasets to read a Delta table.</p> <p>Let's take a look at the h2o groupby dataset that contains 9 columns of data.  Here are three representative rows of data:</p> <pre><code>+-------+-------+--------------+-------+-------+--------+------+------+---------+\n| id1   | id2   | id3          |   id4 |   id5 |    id6 |   v1 |   v2 |      v3 |\n|-------+-------+--------------+-------+-------+--------+------+------+---------|\n| id016 | id046 | id0000109363 |    88 |    13 | 146094 |    4 |    6 | 18.8377 |\n| id039 | id087 | id0000466766 |    14 |    30 | 111330 |    4 |   14 | 46.7973 |\n| id047 | id098 | id0000307804 |    85 |    23 | 187639 |    3 |    5 | 47.5773 |\n+-------+-------+--------------+-------+-------+--------+------+------+---------+\n</code></pre> <p>Here's how to expose the Delta table as a PyArrow dataset and run a query with DuckDB:</p> <pre><code>import duckdb\nfrom deltalake import DeltaTable\n\ntable = DeltaTable(\"delta/G1_1e9_1e2_0_0\")\ndataset = table.to_pyarrow_dataset()\nquack = duckdb.arrow(dataset)\nquack.filter(\"id1 = 'id016' and v2 &gt; 10\")\n</code></pre> <p>Here's the result:</p> <pre><code>\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502   id1   \u2502   id2   \u2502     id3      \u2502  id4  \u2502  id5  \u2502   id6   \u2502  v1   \u2502  v2   \u2502    v3     \u2502\n\u2502 varchar \u2502 varchar \u2502   varchar    \u2502 int32 \u2502 int32 \u2502  int32  \u2502 int32 \u2502 int32 \u2502  double   \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 id016   \u2502 id054   \u2502 id0002309114 \u2502    62 \u2502    95 \u2502 7180859 \u2502     4 \u2502    13 \u2502  7.750173 \u2502\n\u2502 id016   \u2502 id044   \u2502 id0003968533 \u2502    63 \u2502    98 \u2502 2356363 \u2502     4 \u2502    14 \u2502  3.942417 \u2502\n\u2502 id016   \u2502 id034   \u2502 id0001082839 \u2502    58 \u2502    73 \u2502 8039808 \u2502     5 \u2502    12 \u2502 76.820135 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 ? rows (&gt;9999 rows, 3 shown)                                                 9 columns \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n</code></pre> <p>Arrow datasets allow for the predicates to get pushed down to the query engine, so the query is executed quickly.</p>"},{"location":"integrations/delta-lake-arrow/#delta-lake-to-arrow-table","title":"Delta Lake to Arrow Table","text":"<p>You can also run the same query with DuckDB on an Arrow table:</p> <pre><code>quack = duckdb.arrow(table.to_pyarrow_table())\nquack.filter(\"id1 = 'id016' and v2 &gt; 10\")\n</code></pre> <p>This returns the same result, but it runs slower.</p>"},{"location":"integrations/delta-lake-arrow/#difference-between-arrow-dataset-and-arrow-table","title":"Difference between Arrow Dataset and Arrow Table","text":"<p>Arrow Datasets are lazy and allow for full predicate pushdown unlike Arrow tables which are eagerly loaded into memory.</p> <p>The previous DuckDB queries were run on a 1 billion row dataset that's roughly 50 GB when stored as an uncompressed CSV file.  Here are the runtimes when the data is stored in a Delta table and the queries are executed on a 2021 Macbook M1 with 64 GB of RAM:</p> <ul> <li>Arrow table: 17.1 seconds</li> <li>Arrow dataset: 0.01 seconds</li> </ul> <p>The query runs much faster on an Arrow dataset because the predicates can be pushed down to the query engine and lots of data can be skipped.</p> <p>Arrow tables are eagerly materialized in memory and don't allow for the same amount of data skipping.</p>"},{"location":"integrations/delta-lake-arrow/#multiple-query-engines-can-query-arrow-datasets","title":"Multiple query engines can query Arrow Datasets","text":"<p>Other query engines like DataFusion can also query Arrow datasets, see the following example:</p> <pre><code>from datafusion import SessionContext\n\nctx = SessionContext()\nctx.register_dataset(\"my_dataset\", table.to_pyarrow_dataset())\nctx.sql(\"select * from my_dataset where v2 &gt; 5\")\n</code></pre> <p>Here's the result:</p> <pre><code>+-------+-------+--------------+-----+-----+--------+----+----+-----------+\n| id1   | id2   | id3          | id4 | id5 | id6    | v1 | v2 | v3        |\n+-------+-------+--------------+-----+-----+--------+----+----+-----------+\n| id082 | id049 | id0000022715 | 97  | 55  | 756924 | 2  | 11 | 74.161136 |\n| id053 | id052 | id0000113549 | 19  | 56  | 139048 | 1  | 10 | 95.178444 |\n| id090 | id043 | id0000637409 | 94  | 50  | 12448  | 3  | 12 | 60.21896  |\n+-------+-------+--------------+-----+-----+--------+----+----+-----------+\n</code></pre> <p>Any query engine that's capable of reading an Arrow table/dataset can read a Delta table.</p>"},{"location":"integrations/delta-lake-arrow/#conclusion","title":"Conclusion","text":"<p>Delta tables can easily be exposed as Arrow tables/datasets.</p> <p>Therefore any query engine that can read an Arrow table/dataset can also read a Delta table.</p> <p>Arrow datasets allow for more predicates to be pushed down to the query engine, so they can perform better performance than Arrow tables.</p>"},{"location":"integrations/delta-lake-daft/","title":"Using Delta Lake with Daft","text":"<p>Daft is a framework for ETL, analytics, and ML/AI at scale with a familiar Python dataframe API, implemented in Rust.</p> <p>Daft and Delta Lake work really well together. Daft provides unified compute for Delta Lake\u2019s unified storage. Together, Delta Lake and Daft give you high-performance query optimization and distributed compute on massive datasets.</p> <p>Delta Lake is a great storage format for Daft workloads. Delta gives Daft users:</p> <ul> <li>Query optimization via file-skipping and column pruning</li> <li>Versioning for easy time travel functionality</li> <li>Faster reads via Z-ordering</li> <li>ACID transactions and schema enforcement for more reliable reads and writes</li> </ul> <p>For Delta Lake users, Daft is a great data processing tool because it gives you the following features:</p> <ul> <li>Multimodal Dataframes: read, write and transform multimodal data incl. images, JSON, PDF, audio, etc.</li> <li>Parallel + Distributed Reads: Daft parallelizes Delta Lake table reads over all cores of your machine, if using the default multithreading runner, or all cores + machines of your Ray cluster, if using the distributed Ray runner.</li> <li>Skipping Filtered Data: Daft implements automatic partition pruning and stats-based file pruning for filter predicates, skipping data that doesn\u2019t need to be read.</li> </ul> <p>Let's look at how to use Delta Lake with Daft.</p>"},{"location":"integrations/delta-lake-daft/#installing-daft-for-delta-lake","title":"Installing Daft for Delta Lake","text":"<p>The easiest way to use the Delta Lake table format with Daft DataFrames is to install Daft with the <code>[deltalake]</code> extras using <code>pip</code>:</p> <pre><code>!pip install -U \"getdaft[deltalake]\"\n</code></pre> <p>This adds the <code>deltalake</code> Python package to your install. This package is used to fetch metadata about the Delta Lake table, such as paths to the underlying Parquet files and table statistics. You can of course also install the <code>deltalake</code> manually.</p>"},{"location":"integrations/delta-lake-daft/#read-delta-lake-into-a-daft-dataframe","title":"Read Delta Lake into a Daft DataFrame","text":"<p>You can easily read Delta Lake tables into a Daft DataFrame using the <code>read_delta_lake</code> method. Let's use it to read some data stored in a Delta Lake on disk. You can access the data stored as a Delta Lake on Github</p> <pre><code>import daft\n\n# read delta table into Daft DataFrame\ndf = daft.read_delta_lake(\"path/to/delta_table\")\n</code></pre> <p>You can also read in Delta Lake data from remote sources like S3:</p> <pre><code># table_uri = (\n#     \"s3://daft-public-datasets/red-pajamas/\"\n#     \"stackexchange-sample-north-germanic-deltalake\"\n# )\n# df = daft.read_delta_lake(table_uri)\n</code></pre> <pre><code>df\n</code></pre> first_nameUtf8last_nameUtf8countryUtf8continentUtf8 (No data to display: Dataframe not materialized) <p>Daft DataFrames are lazy by default. This means that the contents will not be computed (\"materialized\") unless you explicitly tell Daft to do so. This is best practice for working with larger-than-memory datasets and parallel/distributed architectures.</p> <p>The Delta table we have just loaded only has 5 rows. You can materialize it in memory using the <code>.collect</code> method:</p> <pre><code>&gt; df.collect()\n\n|    | first_name   | last_name   | country   | continent   |\n|---:|:-------------|:------------|:----------|:------------|\n|  0 | Ernesto      | Guevara     | Argentina | NaN         |\n|  1 | Bruce        | Lee         | China     | Asia        |\n|  2 | Jack         | Ma          | China     | Asia        |\n|  3 | Wolfgang     | Manche      | Germany   | NaN         |\n|  4 | Soraya       | Jala        | Germany   | NaN         |\n</code></pre>"},{"location":"integrations/delta-lake-daft/#write-to-delta-lake","title":"Write to Delta Lake","text":"<p>You can use <code>write_deltalake</code> to write a Daft DataFrame to a Delta table:</p> <pre><code>df.write_deltalake(\"tmp/daft-table\", mode=\"overwrite\")\n</code></pre> <p>Daft supports multiple write modes. See the Daft documentation for more information.</p>"},{"location":"integrations/delta-lake-daft/#what-can-i-do-with-a-daft-dataframe","title":"What can I do with a Daft DataFrame?","text":"<p>Daft gives you full-featured DataFrame functionality, similar to what you might be used to from pandas, Dask or PySpark.</p> <p>On top of this, Daft also gives you:</p> <ul> <li>Multimodal data type support to work with Images, URLs, Tensors and more</li> <li>Expressions API for easy column transformations</li> <li>UDFs for multi-column transformation, incl. ML applications</li> </ul> <p>Check out the Daft User Guide for a complete list of DataFrame operations.</p>"},{"location":"integrations/delta-lake-daft/#data-skipping-optimizations","title":"Data Skipping Optimizations","text":"<p>Delta Lake and Daft work together to give you highly-optimized query performance.</p> <p>Delta Lake stores your data in Parquet files. Parquet is a columnar row format that natively supports column pruning. If your query only needs to read data from a specific column or set of columns, you don't need to read in the entire file. This can save you lots of time and compute.</p> <p>Delta Lake goes beyond the basic Parquet features by also giving you:</p> <ul> <li>partitioned reads</li> <li>file skipping via z-ordering.</li> </ul> <p>This is great for Daft users who want to run efficient queries on large-scale data.</p> <p>Let's look at how this works.</p>"},{"location":"integrations/delta-lake-daft/#partitioned-reads","title":"Partitioned Reads","text":"<p>You may have noticed the Delta Lake warning at the top when we first called <code>collect()</code> on our DataFrame:</p> <p><code>WARNING: has partitioning keys = [PartitionField(country#Utf8)], but no partition filter was specified. This will result in a full table scan.</code></p> <p>Delta Lake is informing us that the data is partitioned on the <code>country</code> column.</p> <p>Daft does some nice magic here to help you out. The Daft query optimizer has access to all of the Delta Lake metadata. This means it can optimize your query by skipping the partitions that are not relevant for this query. Instead of having to read all 3 partitions, we can read only 1 and get the same result, just faster!</p> <pre><code># Filter on partition columns will result in efficient partition pruning; non-matching partitions will be skipped.\n&gt; df.where(df[\"country\"] == \"Germany\").show()\n\n|    | first_name   | last_name   | country   |   continent |\n|---:|:-------------|:------------|:----------|------------:|\n|  0 | Wolfgang     | Manche      | Germany   |         nan |\n|  1 | Soraya       | Jala        | Germany   |         nan |\n</code></pre> <p>You can use the <code>explain()</code> method to see how Daft is optimizing your query.</p> <p>Since we've already called <code>collect</code> on our DataFrame, it is already in memory. So below we copy the output of <code>explain(show_all=True)</code> before calling <code>collect</code>:</p> <p>Running <code>df.where(df[\"continent\"] == \"Asia\").explain(True)</code> returns:</p> <pre><code>(...)\n\n== Optimized Logical Plan ==\n\n* PythonScanOperator: DeltaLakeScanOperator(None)\n|   File schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8\n|   Partitioning keys = [PartitionField(country#Utf8)]\n|   Filter pushdown = col(continent) == lit(\"Asia\")\n|   Output schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8\n\n\n== Physical Plan ==\n\n* TabularScan:\n|   Num Scan Tasks = 3\n|   Estimated Scan Bytes = 3045\n|   Clustering spec = { Num partitions = 3 }\n</code></pre> <p>Whereas running <code>df.where(df[\"country\"] == \"Germany\").explain(True)</code> returns:</p> <pre><code>(...)\n\n== Optimized Logical Plan ==\n\n* PythonScanOperator: DeltaLakeScanOperator(None)\n|   File schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8\n|   Partitioning keys = [PartitionField(country#Utf8)]\n|   Partition Filter = col(country) == lit(\"Germany\")\n|   Output schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8\n\n\n== Physical Plan ==\n\n* TabularScan:\n|   Num Scan Tasks = 1\n|   Estimated Scan Bytes = 1025\n|   Clustering spec = { Num partitions = 1 }\n</code></pre> <p>Running a query on a non-partitioned column like <code>continent</code> will require reading in all partitions, totalling 3045 bytes in the case of this toy example.</p> <p>Instead, running a query on a partitioned column (<code>country</code> in this case) means Daft only has to read only the relevant partition, saving us a ~60% of the compute. This has huge impacts when you're working at scale.</p>"},{"location":"integrations/delta-lake-daft/#z-ordering-for-enhanced-file-skipping","title":"Z-Ordering for enhanced file skipping","text":"<p>Z-ordering stores similar data close together to optimize query performance. This is especially useful when you're querying on one or multiple columns.</p> <p>Using Z-Ordered Delta tables instead of regular Parquet can give Daft users significant speed-ups.</p> <p>Read High-Performance Querying on Massive Delta Lake Tables with Daft for an in-depth benchmarking of query optimization with Delta Lake and Daft using partitioning and Z-ordering.</p>"},{"location":"integrations/delta-lake-daft/#daft-gives-you-multimodal-data-type-support","title":"Daft gives you Multimodal Data Type Support","text":"<p>Daft has a rich multimodal type-system with support for Python objects, Images, URLs, Tensors and more.</p> <p>The Expressions API provides useful tools to work with these data types. By combining multimodal data support with the User-Defined Functions API you can run ML workloads right within your DataFrame.</p> <p>Take a look at the notebook in the <code>delta-examples</code> Github repository for a closer look at how Daft handles URLs, images and ML applications.</p>"},{"location":"integrations/delta-lake-daft/#contribute-to-daft","title":"Contribute to <code>daft</code>","text":"<p>Excited about Daft and want to contribute? Join them on Github \ud83d\ude80</p> <p>Like many technologies, Daft collects some non-identifiable telemetry to improve the product. This is stricly non-identifiable metadata. You can disable telemetry by setting the following environment variable: <code>DAFT_ANALYTICS_ENABLED=0</code>. Read more in the Daft documentation.</p>"},{"location":"integrations/delta-lake-dagster/","title":"Using Delta Lake with Dagster\u00b6","text":"<p>Delta Lake is a great storage format for Dagster workflows. This page will explain why and how to use Delta Lake with Dagster.</p> <p>You will learn how to use the Delta Lake I/O Manager to read and write your Dagster Software-Defined Assets (SDAs). You will also learn about the unique advantages Delta Lake offers the Dagster community.</p> <p>Here are some of the benefits that Delta Lake provides Dagster users: - native PyArrow integration for lazy computation of large datasets, - more efficient querying with file skipping via Z Ordering and liquid clustering - built-in vacuuming to remove unnecessary files and versions - ACID transactions for reliable writes - smooth versioning integration so that versions can be use to trigger downstream updates. - surfacing table stats based on the file statistics </p>"},{"location":"integrations/delta-lake-dagster/#dagster-io-managers","title":"Dagster I/O Managers","text":"<p>Dagster uses I/O Managers to simplify data reads and writes. I/O Managers help you reduce boilerplate code by storing Dagster Asset and Op outputs and loading them as inputs to downstream objects. They make it easy to change where and how your data is stored.</p> <p>You only need to define your I/O Manager and its settings (such as storage location and schema) once and the I/O Manager will take care of correctly reading and writing all your Dagster Assets automatically.</p> <p>If you need lower-level access than the Dagster I/O Managers provide, take a look at the Delta Table Resource.</p>"},{"location":"integrations/delta-lake-dagster/#the-delta-lake-io-manager","title":"The Delta Lake I/O Manager","text":"<p>You can easily read and write Delta Lake Tables from Dagster by using the <code>DeltaLakeIOManager()</code>. </p> <p>Install the DeltaLakeIOManager:</p> <pre><code>pip install dagster-deltalake\n</code></pre> <p>Next, configure the following settings in your project\u2019s <code>__init__.py</code> file: - <code>io_manager</code>: set this to <code>DeltaLakeIOManager()</code>, this sets the default I/O Manager for all your Assets</p> <p>Within the DeltaLakeIOManager, define: - <code>root_uri</code>: the root path where your Delta Tables will be created - <code>storage_options</code>: configuration for accessing storage location - <code>schema</code>: name of schema to use (optional, defaults to public)</p> <pre><code>defs = Definitions(\n   assets=all_assets,\n   resources={\n        \"io_manager\": DeltaLakePyarrowIOManager(\n            root_uri=\"path/to/deltalake\",\n            storage_options=LocalConfig(),\n            schema=\"dagster_deltalake\",\n        ),\n   },\n)\n</code></pre> <p>Now, when you materialize an Asset, it will be saved as a Delta Lake in a folder <code>dagster_deltalake/asset_name</code> under the root directory <code>path/to/deltalake</code>.</p> <p>The default Delta Lake I/O Manager supports Arrow reads and writes. You can also use the Delta Lake I/O Manager with pandas or polars. </p>"},{"location":"integrations/delta-lake-dagster/#creating-delta-lake-tables-with-dagster","title":"Creating Delta Lake Tables with Dagster","text":"<p>You don\u2019t need to do anything else to store your Dagster Assets as Delta Lake tables. The I/O Manager will handle storing and loading your Assets as Delta Lake tables from now on.</p> <p>You can proceed to write Dagster code as you normally would. For example, you can create an Asset that reads in some toy data about animals and writes it out to an Arrow Table:</p> <pre><code>import pyarrow as pa\nfrom pyarrow import csv\n\nfrom dagster import asset\n\n@asset\ndef raw_dataset() -&gt; pa.Table:\n   n_legs = pa.array([2, 4, None, 100])\n   animals = pa.array([\"Flamingo\", \"Horse\", \"Brittle stars\", \"Centipede\"])\n   data = {'n_legs': n_legs, 'animals': animals}\n\n   return pa.Table.from_pydict(data)\n</code></pre> <p>When you materialize the Asset defined above (using the config settings defined earlier), the Delta Lake I/O Manager will create the table <code>dagster_deltalake/iris_dataset</code> if it doesn\u2019t exist yet.</p>"},{"location":"integrations/delta-lake-dagster/#overwrites-when-rematerializing-assets","title":"Overwrites when Rematerializing Assets","text":"<p>If the table does already exist at the specified location, the Delta Lake I/O Manager will perform an overwrite. Delta Lake\u2019s transaction log maintains a record of all changes to your Delta Lake tables. You can inspect the record of changes to your Delta Lake tables by taking a look at these transaction logs.</p>"},{"location":"integrations/delta-lake-dagster/#loading-delta-lake-tables-in-downstream-assets","title":"Loading Delta Lake Tables in Downstream Assets","text":"<p>You can use Assets stored as Delta Lake tables as input to downstream Assets. Dagster and the Delta Lake I/O Manager make this easy for you.</p> <p>You can write Dagster code as you normally would. Pass the upstream Asset as an argument to the downstream object to set up the dependency. Make sure to define the correct data type.</p> <p>The Delta Lake I/O Manager will handle reading and writing the data from your Delta Lake.</p> <pre><code>import pyarrow as pa\nfrom dagster import asset\n\n# ... raw_dataset asset is defined here ...\n\n@asset\ndef clean_dataset(raw_dataset: pa.Table) -&gt; pa.Table:\n   return raw_dataset.drop_null()\n</code></pre>"},{"location":"integrations/delta-lake-dagster/#reading-existing-delta-lake-tables-into-dagster","title":"Reading Existing Delta Lake Tables into Dagster","text":"<p>You can make existing Delta Lake tables (that were not created in Dagster) available to your Dagster assets. Use the <code>SourceAsset</code> object and pass the table name as the key argument: </p> <pre><code>from dagster import SourceAsset\n\niris_harvest_data = SourceAsset(key=\"more_animal_data\")\n</code></pre> <p>This will load a table <code>more_animal_data</code> located at <code>&lt;root_uri&gt;/&lt;schema&gt;</code> as configured in the Definitions object above (see Delta Lake I/O Manager section).</p>"},{"location":"integrations/delta-lake-dagster/#column-pruning","title":"Column Pruning","text":"<p>You can often improve the efficiency of your computations by only loading specific columns of your Delta table. This is called column pruning. </p> <p>With the Delta Lake I/O manager, you can select specific columns to load defining the <code>columns</code> in the <code>metadata</code> parameter of the <code>AssetIn</code> that loads the upstream Asset:</p> <pre><code>import pyarrow as pa\nfrom dagster import AssetIn, asset\n\n# this example uses the clean_dataset Asset defined earlier\n\n@asset(\n       ins={\n           \"mammal_bool\": AssetIn(\n               key=\"clean_dataset\",\n               metadata={\"columns\": [\"is_mammal\", \"animals\"]},\n           )\n       }\n)\ndef mammal_data(mammal_bool: pa.Table) -&gt; pa.Table:\n   mammals = mammal_bool[\"is_mammal\"].cast(\"bool\")\n   animals = mammal_bool[\"animals\"]\n   data = {\"mammal_bool\": mammals, \"animals\": animals}\n   return pa.Table.from_pydict(data)\n</code></pre> <p>Here, we select only the <code>sepal_length_cm</code> and <code>sepal_width_cm</code> columns from the <code>iris_dataset</code> table and load them into an <code>AssetIn</code> object called <code>iris_sepal</code>. This AssetIn object is used to create a new Asset <code>sepal_data</code>, containing only the selected columns.</p>"},{"location":"integrations/delta-lake-dagster/#working-with-partitioned-assets","title":"Working with Partitioned Assets","text":"<p>Partitioning is an important feature of Delta Lake that can make your computations more efficient. The Delta Lake I/O manager helps you read and write partitioned data easily. You can work with static partitions, time-based partitions, multi-partitions, and dynamic partitions.</p> <p>For example, you can partition the Iris dataset on the <code>species</code> column as follows:</p> <pre><code>import pyarrow as pa\n\nfrom dagster import StaticPartitionsDefinition, asset\n\n@asset(\n  partitions_def=StaticPartitionsDefinition(\n      [\"Human\", \"Horse\",]\n  ),\n  metadata={\"partition_expr\": \"n_legs\"},\n)\ndef dataset_partitioned(\n   context,\n   clean_dataset: pa.Table,\n   ) -&gt; pa.Table:\n   animals = context.asset_partition_key_for_output()\n   table = clean_dataset\n\n   return table.filter(pc.field(\"animals\") == animals)\n</code></pre> <p>To partition your data, make sure to include the relevant <code>partitions_def</code> and <code>metadata</code> arguments to the <code>@asset</code> decorator. Refer to the Dagster documentation on partitioning assets for more information.</p>"},{"location":"integrations/delta-lake-dagster/#using-delta-lake-and-dagster-with-pandas","title":"Using Delta Lake and Dagster with Pandas","text":"<p>To read and write data to Delta Lake using pandas, use the <code>DeltaLakePandasIOManager()</code>. </p> <p>You will need to install it using:</p> <pre><code>pip install dagster-deltalake-pandas\n</code></pre> <p>In your <code>Definitions</code> object, change the <code>io_manager</code> to <code>DeltaLakePandasIOManager()</code>:</p> <pre><code>from dagster_deltalake_pandas import DeltaLakePandasIOManager\n\n\ndefs = Definitions(\n   assets=all_assets,\n   resources={\n        \"io_manager\": DeltaLakePandasIOManager(\n            root_uri=\"path/to/deltalake\",\n            storage_options=LocalConfig(),\n            schema=\"dagster_deltalake\",\n        ),\n   },\n)\n</code></pre> <p>Now you can read and write Dagster Assets defined as pandas DataFrames in Delta Lake format. For example:</p> <pre><code>import pandas as pd\nfrom dagster import asset\n\n@asset\ndef iris_dataset() -&gt; pd.DataFrame:\n   return pd.read_csv(\n       \"https://docs.dagster.io/assets/iris.csv\",\n       names=[\n           \"sepal_length_cm\",\n           \"sepal_width_cm\",\n           \"petal_length_cm\",\n           \"petal_width_cm\",\n           \"species\",\n       ],\n   )\n</code></pre>"},{"location":"integrations/delta-lake-dagster/#using-delta-lake-and-dagster-with-polars","title":"Using Delta Lake and Dagster with Polars","text":"<p>To read and write data to Delta Lake using polars, use the <code>DeltaLakePolarsIOManager()</code>. </p> <p>You will need to install it using:</p> <pre><code>pip install dagster-deltalake-polars\n</code></pre> <p>In your <code>Definitions</code> object, change the <code>io_manager</code> to <code>DeltaLakePolarsIOManager()</code>:</p> <pre><code>from dagster_deltalake_polars import DeltaLakePolarsIOManager\n\ndefs = Definitions(\n   assets=all_assets,\n   resources={\n        \"io_manager\": DeltaLakePolarsIOManager(\n            root_uri=\"path/to/deltalake\",\n            storage_options=LocalConfig(),\n            schema=\"dagster_deltalake\",\n        ),\n   },\n)\n</code></pre> <p>Now you can read and write Dagster Assets defined as Polars DataFrames in Delta Lake format. For example:</p> <pre><code>import polars as pl\nfrom dagster import asset\n\n\n@asset\ndef iris_dataset() -&gt; pl.DataFrame:\n   return pl.read_csv(\n       \"https://docs.dagster.io/assets/iris.csv\",\n       new_columns=[\n          \"sepal_length_cm\",\n          \"sepal_width_cm\",\n          \"petal_length_cm\",\n          \"petal_width_cm\",\n          \"species\",\n      ],\n   has_header=False\n)\n</code></pre>"},{"location":"integrations/delta-lake-dagster/#delta-lake-table-resource","title":"Delta Lake Table Resource","text":"<p>I/O managers are a helpful tool in many common usage situations. But when you need lower-level access, the I/O Manager might not be the right tool to use. In these cases you may want to use the Delta Lake Table Resource.</p> <p>The Delta Lake Table Resource is a low-level access method to the table object. It gives you more fine-grained control and allows for modeling of more complex data. You can also use the Table Resource to run optimization and vacuuming jobs.</p>"},{"location":"integrations/delta-lake-dagster/#schema-and-constraint-enforcement","title":"Schema and Constraint Enforcement","text":"<p>Delta Lake provides built-in checks to ensure schema consistency when appending data to a table, as well as the ability to evolve the schema. This is a great feature for the Dagster community as it prevents bad data from being appended to tables, ensuring data consistency and accuracy. </p> <p>Read more about how to add constraints to a table in the Delta Lake documentation.</p>"},{"location":"integrations/delta-lake-dagster/#z-ordering","title":"Z-Ordering","text":"<p>Delta Lake offers Z-ordering functionality to colocate similar data in the same files. This can make your Delta Table queries much more efficient via file skipping. Dagster users can now benefit from this great feature through the Delta Lake I/O Manager.</p> <p>Read more about Z-Ordering on the Delta Lake blog.</p>"},{"location":"integrations/delta-lake-dagster/#contribute","title":"Contribute","text":"<p>To contribute to the Delta Lake and Dagster integration, go to [link]</p>"},{"location":"integrations/delta-lake-dask/","title":"Using Delta Lake with Dask","text":"<p>Delta Lake is a great storage format for Dask analyses. This page explains why and how to use Delta Lake with Dask.</p> <p>You will learn how to read Delta Lakes into Dask DataFrames, how to query Delta tables with Dask, and the unique advantages Delta Lake offers the Dask community.</p> <p>Here are some of the benefits that Delta Lake provides Dask users:</p> <ul> <li>better performance with file skipping</li> <li>enhanced file skipping via Z Ordering</li> <li>ACID transactions for reliable writes</li> <li>easy time-travel functionality</li> </ul> <p>\u2757\ufe0f dask-deltatable currently works with deltalake&lt;=13.0. See https://github.com/dask-contrib/dask-deltatable/issues/65</p>"},{"location":"integrations/delta-lake-dask/#install-dask-deltatable","title":"Install Dask-Deltatable","text":"<p>To use Delta Lake with Dask, first install the library using</p> <pre><code>pip install dask-deltatable\n</code></pre>"},{"location":"integrations/delta-lake-dask/#reading-delta-tables-into-a-dask-dataframe","title":"Reading Delta Tables into a Dask DataFrame","text":"<p>You can read data stored in a Delta Lake into a Dask DataFrame using <code>dask-deltatable.read_deltalake</code>.</p> <p>Let's read in a toy dataset to see what we can do with Delta Lake and Dask. You can access the data stored as a Delta Lake on Github</p> <pre><code>import dask_deltatable as ddt\n\n# read delta table into Dask DataFrame\ndelta_path = \"path/to/data/people_countries_delta_dask\"\nddf = ddt.read_deltalake(delta_path)\n</code></pre> <p>Dask is a library for efficient distributed computing and works with lazy evaluation. Function calls to <code>dask.dataframe</code> build a task graph in the background. To trigger computation, call <code>.compute()</code>:</p> <pre><code>&gt; ddf.compute()\n\n|    | first_name   | last_name   | country   | continent   |\n|---:|:-------------|:------------|:----------|:------------|\n|  0 | Ernesto      | Guevara     | Argentina | NaN         |\n|  0 | Bruce        | Lee         | China     | Asia        |\n|  1 | Jack         | Ma          | China     | Asia        |\n|  0 | Wolfgang     | Manche      | Germany   | NaN         |\n|  1 | Soraya       | Jala        | Germany   | NaN         |\n</code></pre> <p>You can read in specific versions of Delta tables by specifying a <code>version</code> number or a timestamp:</p> <pre><code># with specific version\nddf = ddt.read_deltalake(delta_path, version=3)\n\n# with specific datetime\nddt.read_deltalake(delta_path, datetime=\"2018-12-19T16:39:57-08:00\")\n</code></pre> <p><code>dask-deltatable</code> also supports reading from remote sources like S3 with:</p> <pre><code>ddt.read_deltalake(\"s3://bucket_name/delta_path\", version=3)\n</code></pre> <p>To read data from remote sources you'll need to make sure the credentials are properly configured in environment variables or config files. Refer to your cloud provider documentation to configure these.</p>"},{"location":"integrations/delta-lake-dask/#what-can-i-do-with-a-dask-deltatable","title":"What can I do with a Dask Deltatable?","text":"<p>Reading a Delta Lake in with <code>dask-deltatable</code> returns a regular Dask DataFrame. You can perform all the regular Dask operations on this DataFrame.</p> <p>Let's take a look at the first few rows:</p> <pre><code>&gt; ddf.head(n=3)\n\n|    | first_name   | last_name   | country   |   continent |\n|---:|:-------------|:------------|:----------|------------:|\n|  0 | Ernesto      | Guevara     | Argentina |         nan |\n</code></pre> <p><code>dask.dataframe.head()</code> shows you the first rows of the first partition in the dataframe. In this case, the first partition only has 1 row.</p> <p>This is because the Delta Lake has been partitioned by country:</p> <pre><code>&gt; !ls ../../data/people_countries_delta_dask\n_delta_log        country=Argentina country=China     country=Germany\n</code></pre> <p><code>dask-deltatable</code> neatly reads in the partitioned Delta Lake into corresponding Dask DataFrame partitions:</p> <pre><code>&gt; # see number of partitions\n&gt; ddf.npartitions\n3\n</code></pre> <p>You can inspect a single partition using <code>dask.dataframe.get_partition()</code>:</p> <pre><code>&gt; ddf.get_partition(n=1).compute()\n\n|    | first_name   | last_name   | country   | continent   |\n|---:|:-------------|:------------|:----------|:------------|\n|  0 | Bruce        | Lee         | China     | Asia        |\n|  1 | Jack         | Ma          | China     | Asia        |\n</code></pre>"},{"location":"integrations/delta-lake-dask/#perform-dask-operations","title":"Perform Dask Operations","text":"<p>Let's perform some basic computations over the Delta Lake data that's now stored in our Dask DataFrame.</p> <p>Suppose you want to group the dataset by the <code>country</code> column:</p> <pre><code>&gt; ddf.groupby(['country']).count().compute()\n\n| country   |   first_name |   last_name |   continent |\n|:----------|-------------:|------------:|------------:|\n| Argentina |            1 |           1 |           1 |\n| China     |            2 |           2 |           2 |\n| Germany   |            2 |           2 |           2 |\n</code></pre> <p>Dask executes this <code>groupby</code> operation in parallel across all available cores.</p>"},{"location":"integrations/delta-lake-dask/#map-functions-over-partitions","title":"Map Functions over Partitions","text":"<p>You can also use Dask's <code>map_partitions</code> method to map a custom Python function over all the partitions.</p> <p>Let's write a function that will replace the missing <code>continent</code> values with the right continent names.</p> <pre><code># define custom python function\n\n# get na_string\ndf = ddf.get_partition(0).compute()\nna_string = df.iloc[0].continent\nna_string\n\n# define function\ndef replace_proper(partition, na_string):\n    if [partition.country == \"Argentina\"]:\n        partition.loc[partition.country==\"Argentina\"] = partition.loc[partition.country==\"Argentina\"].replace(na_string, \"South America\")\n    if [partition.country == \"Germany\"]:\n        partition.loc[partition.country==\"Germany\"] = partition.loc[partition.country==\"Germany\"].replace(na_string, \"Europe\")\n    else:\n        pass\n    return partition\n</code></pre> <p>Now map this over all partitions in the Dask DataFrame:</p> <pre><code># define metadata and map function over partitions\n&gt; meta = dict(ddf.dtypes)\n&gt; ddf3 = ddf.map_partitions(replace_proper, na_string, meta=meta)\n&gt; ddf3.compute()\n\n|    | first_name   | last_name   | country   | continent     |\n|---:|:-------------|:------------|:----------|:--------------|\n|  0 | Ernesto      | Guevara     | Argentina | South America |\n|  0 | Bruce        | Lee         | China     | Asia          |\n|  1 | Jack         | Ma          | China     | Asia          |\n|  0 | Wolfgang     | Manche      | Germany   | Europe        |\n|  1 | Soraya       | Jala        | Germany   | Europe        |\n</code></pre>"},{"location":"integrations/delta-lake-dask/#write-to-delta-lake","title":"Write to Delta Lake","text":"<p>After doing your data processing in Dask, you can write the data back out to Delta Lake using <code>to_deltalake</code>:</p> <pre><code>ddt.to_deltalake(\"tmp/test_write\", ddf)\n</code></pre>"},{"location":"integrations/delta-lake-dask/#contribute-to-dask-deltalake","title":"Contribute to <code>dask-deltalake</code>","text":"<p>To contribute, go to the <code>dask-deltalake</code> Github repository.</p>"},{"location":"integrations/delta-lake-datafusion/","title":"Using Delta Lake with DataFusion","text":"<p>This page explains how to use Delta Lake with DataFusion.</p> <p>Delta Lake offers DataFusion users better performance and more features compared to other formats like CSV or Parquet.</p> <p>Delta Lake works well with the DataFusion Rust API and the DataFusion Python API.  It's a great option for all DataFusion users.</p> <p>Delta Lake also depends on DataFusion to implement SQL-related functionality under the hood.  We will also discuss this dependency at the end of this guide in case you're interested in learning more about the symbiotic relationship between the two libraries.</p>"},{"location":"integrations/delta-lake-datafusion/#delta-lake-performance-benefits-for-datafusion-users","title":"Delta Lake performance benefits for DataFusion users","text":"<p>Let's run some DataFusion queries on a Parquet file and a Delta table with the same data to learn more about the performance benefits of Delta Lake.</p> <p>Suppose you have the following dataset with 1 billion rows and 9 columns.  Here are the first three rows of data:</p> <pre><code>+-------+-------+--------------+-------+-------+--------+------+------+---------+\n| id1   | id2   | id3          |   id4 |   id5 |    id6 |   v1 |   v2 |      v3 |\n|-------+-------+--------------+-------+-------+--------+------+------+---------|\n| id016 | id046 | id0000109363 |    88 |    13 | 146094 |    4 |    6 | 18.8377 |\n| id039 | id087 | id0000466766 |    14 |    30 | 111330 |    4 |   14 | 46.7973 |\n| id047 | id098 | id0000307804 |    85 |    23 | 187639 |    3 |    5 | 47.5773 |\n+-------+-------+--------------+-------+-------+--------+------+------+---------+\n</code></pre> <p>Here's how to register a Delta Lake table as a PyArrow dataset:</p> <pre><code>from datafusion import SessionContext\nfrom deltalake import DeltaTable\n\nctx = SessionContext()\ntable = DeltaTable(\"G1_1e9_1e2_0_0\")\nctx.register_dataset(\"my_delta_table\", table.to_pyarrow_dataset())\n</code></pre> <p>Now query the table:</p> <pre><code>ctx.sql(\"select id1, sum(v1) as v1 from my_delta_table where id1='id096' group by id1\")\n</code></pre> <p>That query takes 2.8 seconds to execute.</p> <p>Let's register the same dataset as a Parquet table, run the same query, and compare the runtime difference.</p> <p>Register the Parquet table and run the query:</p> <pre><code>path = \"G1_1e9_1e2_0_0.parquet\"\nctx.register_parquet(\"my_parquet_table\", path)\nctx.sql(\"select id1, sum(v1) as v1 from my_parquet_table where id1='id096' group by id1\")\n</code></pre> <p>This query takes 5.3 seconds to run.</p> <p>Parquet stores data in row groups and DataFusion can intelligently skip row groups that don't contain relevant data, so the query is faster than a file format like CSV which doesn't support row group skipping.</p> <p>Delta Lake stores file-level metadata information in the transaction log, so it can skip entire files when queries are executed.  Delta Lake can skip entire files and then skip row groups within the individual files.  This makes Delta Lake even faster than Parquet files, especially for larger datasets spread across many files.</p>"},{"location":"integrations/delta-lake-datafusion/#delta-lake-features-for-datafusion-users","title":"Delta Lake features for DataFusion users","text":"<p>Delta Lake also provides other features that are useful for DataFusion users like ACID transactions, concurrency protection, time travel, versioned data, and more.</p>"},{"location":"integrations/delta-lake-datafusion/#why-delta-lake-depends-on-datafusion","title":"Why Delta Lake depends on DataFusion","text":"<p>Delta Lake depends on DataFusion to provide some end-user features.</p> <p>DataFusion is useful in providing SQL-related Delta Lake features. Some examples:</p> <ul> <li>Update and merge are written in terms of SQL expressions.</li> <li>Invariants and constraints are written in terms of SQL expressions.</li> </ul> <p>Anytime we have to evaluate SQL, we need some sort of SQL engine.  We use DataFusion for that.</p>"},{"location":"integrations/delta-lake-datafusion/#conclusion","title":"Conclusion","text":"<p>Delta Lake is a great file format for DataFusion users.</p> <p>Delta Lake also uses DataFusion to provide some end-user features.</p> <p>DataFusion and Delta Lake have a wonderful symbiotic relationship and play very nicely with each other.</p> <p>See this guide for more information on Delta Lake and PyArrow and why PyArrow Datasets are often a better option than PyArrow tables.</p>"},{"location":"integrations/delta-lake-pandas/","title":"Using Delta Lake with pandas","text":"<p>Delta Lake is a great storage system for pandas analyses.  This page shows how it's easy to use Delta Lake with pandas, the unique features Delta Lake offers pandas users, and how Delta Lake can make your pandas analyses run faster.</p> <p>Delta Lake is very easy to install for pandas analyses, just run <code>pip install deltalake</code>.</p> <p>Delta Lake allows for performance optimizations, so pandas queries can run much faster than the query run on data stored in CSV or Parquet.  See the following chart for the query runtime for the a Delta tables compared with CSV/Parquet.</p> <p></p> <p>Z Ordered Delta tables run this query much faster than when the data is stored in Parquet or CSV.  Let's dive in deeper and see how Delta Lake makes pandas faster.</p>"},{"location":"integrations/delta-lake-pandas/#delta-lake-makes-pandas-queries-run-faster","title":"Delta Lake makes pandas queries run faster","text":"<p>There are a few reasons Delta Lake can make pandas queries run faster:</p> <ol> <li>column pruning: only grabbing the columns relevant for a query</li> <li>file skipping: only reading files with data for the query</li> <li>row group skipping: only reading row groups with data for the query</li> <li>Z ordering data: colocating similar data in the same files, so file skipping is more effective</li> </ol> <p>Reading less data (fewer columns and/or fewer rows) is how Delta Lake makes pandas queries run faster.</p> <p>Parquet allows for column pruning and row group skipping, but doesn't support file-level skipping or Z Ordering.  CSV doesn't support any of these performance optimizations.</p> <p>Let's take a look at a sample dataset and run a query to see the performance enhancements offered by Delta Lake.</p> <p>Suppose you have a 1 billion row dataset with 9 columns, here are the first three rows of the dataset:</p> <pre><code>+-------+-------+--------------+-------+-------+--------+------+------+---------+\n| id1   | id2   | id3          |   id4 |   id5 |    id6 |   v1 |   v2 |      v3 |\n|-------+-------+--------------+-------+-------+--------+------+------+---------|\n| id016 | id046 | id0000109363 |    88 |    13 | 146094 |    4 |    6 | 18.8377 |\n| id039 | id087 | id0000466766 |    14 |    30 | 111330 |    4 |   14 | 46.7973 |\n| id047 | id098 | id0000307804 |    85 |    23 | 187639 |    3 |    5 | 47.5773 |\n+-------+-------+--------------+-------+-------+--------+------+------+---------+\n</code></pre> <p>The dataset is roughly 50 GB when stored as an uncompressed CSV files.  Let's run some queries on a 2021 Macbook M1 with 64 GB of RAM.</p> <p>Start by running the query on an uncompressed CSV file:</p> <pre><code>(\n    pd.read_csv(f\"{Path.home()}/data/G1_1e9_1e2_0_0.csv\", usecols=[\"id1\", \"id2\", \"v1\"])\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>This query takes 234 seconds to execute.  It runs out of memory if the <code>usecols</code> parameter is not set.</p> <p>Now let's convert the CSV dataset to Parquet and run the same query on the data stored in a Parquet file.</p> <pre><code>(\n    pd.read_parquet(\n        f\"{Path.home()}/data/G1_1e9_1e2_0_0.parquet\", columns=[\"id1\", \"id2\", \"v1\"]\n    )\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>This query takes 118 seconds to execute.</p> <p>Parquet stores data in row groups and allows for skipping when the <code>filters</code> predicates are set.  Run the Parquet query again with row group skipping enabled:</p> <pre><code>(\n    pd.read_parquet(\n        f\"{Path.home()}/data/G1_1e9_1e2_0_0.parquet\",\n        columns=[\"id1\", \"id2\", \"v1\"],\n        filters=[(\"id1\", \"==\", \"id016\")],\n    )\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>This query runs in 19 seconds.  Lots of row groups can be skipped for this particular query.</p> <p>Now let's run the same query on a Delta table to see the out-of-the box performance:</p> <pre><code>(\n    DeltaTable(f\"{Path.home()}/data/deltalake_baseline_G1_1e9_1e2_0_0\", version=0)\n    .to_pandas(filters=[(\"id1\", \"==\", \"id016\")], columns=[\"id1\", \"id2\", \"v1\"])\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>This query runs in 8 seconds, which is a significant performance enhancement.</p> <p>Now let's Z Order the Delta table by <code>id1</code> which will make the data skipping even better.  Run the query again on the Z Ordered Delta table:</p> <pre><code>(\n    DeltaTable(f\"{Path.home()}/data/deltalake_baseline_G1_1e9_1e2_0_0\", version=1)\n    .to_pandas(filters=[(\"id1\", \"==\", \"id016\")], columns=[\"id1\", \"id2\", \"v1\"])\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>The query now executes in 2.4 seconds.</p> <p>Delta tables can make certain pandas queries run much faster.</p>"},{"location":"integrations/delta-lake-pandas/#delta-lake-lets-pandas-users-time-travel","title":"Delta Lake lets pandas users time travel","text":"<p>Start by creating a Delta table:</p> <pre><code>from deltalake import write_deltalake, DeltaTable\n\ndf = pd.DataFrame({\"num\": [1, 2, 3], \"letter\": [\"a\", \"b\", \"c\"]})\nwrite_deltalake(\"tmp/some-table\", df)\n</code></pre> <p>Here are the contents of the Delta table (version 0 of the Delta table):</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n+-------+----------+\n</code></pre> <p>Now append two rows to the Delta table:</p> <pre><code>df = pd.DataFrame({\"num\": [8, 9], \"letter\": [\"dd\", \"ee\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"append\")\n</code></pre> <p>Here are the contents after the append operation (version 1 of the Delta table):</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre> <p>Now perform an overwrite transaction:</p> <pre><code>df = pd.DataFrame({\"num\": [11, 22], \"letter\": [\"aa\", \"bb\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"overwrite\")\n</code></pre> <p>Here are the contents after the overwrite operation (version 2 of the Delta table):</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre> <p>Read in the Delta table and it will grab the latest version by default:</p> <pre><code>DeltaTable(\"tmp/some-table\").to_pandas()\n\n+-------+----------+\n|   num | letter   |\n|-------+----------|\n|    11 | aa       |\n|    22 | bb       |\n+-------+----------+\n</code></pre> <p>You can easily time travel back to version 0 of the Delta table:</p> <pre><code>DeltaTable(\"tmp/some-table\", version=0).to_pandas()\n\n+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n+-------+----------+\n</code></pre> <p>You can also time travel to version 1 of the Delta table:</p> <pre><code>DeltaTable(\"tmp/some-table\", version=1).to_pandas()\n\n+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre> <p>Time travel is a powerful feature that pandas users cannot access with CSV or Parquet.</p>"},{"location":"integrations/delta-lake-pandas/#schema-enforcement","title":"Schema enforcement","text":"<p>Delta tables only allow you to append DataFrame with matching schema by default.  Suppose you have a DataFrame with <code>num</code> and <code>animal</code> columns, which is different from the Delta table that has columns with <code>num</code> and <code>letter</code> columns.</p> <p>Try to append this DataFrame with a mismatched schema to the existing table:</p> <pre><code>df = pd.DataFrame({\"num\": [5, 6], \"animal\": [\"cat\", \"dog\"]})\nwrite_deltalake(\"tmp/some-table\", df)\n</code></pre> <p>This transaction will be rejected and will return the following error message:</p> <pre><code>ValueError: Schema of data does not match table schema\nData schema:\nnum: int64\nanimal: string\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 474\nTable Schema:\nnum: int64\nletter: string\n</code></pre> <p>Schema enforcement protects your table from getting corrupted by appending data with mismatched schema.  Parquet and CSV don't offer schema enforcement for pandas users.</p>"},{"location":"integrations/delta-lake-pandas/#overwriting-schema-of-table","title":"Overwriting schema of table","text":"<p>You can overwrite the table contents and schema by setting the <code>schema_mode</code> option.  Here's how to overwrite the table contents:</p> <pre><code>write_deltalake(\"tmp/some-table\", df, mode=\"overwrite\", schema_mode=\"overwrite\")\n</code></pre> <p>Here are the contents of the table after the values and schema have been overwritten:</p> <pre><code>+-------+----------+\n|   num | animal   |\n|-------+----------|\n|     5 | cat      |\n|     6 | dog      |\n+-------+----------+\n</code></pre> <p>If you want the schema to be merged instead, specify schema_mode=\"merge\".</p>"},{"location":"integrations/delta-lake-pandas/#in-memory-vs-in-storage-data-changes","title":"In-memory vs. in-storage data changes","text":"<p>It's important to distinguish between data stored in-memory and data stored on disk when understanding the functionality offered by Delta Lake.</p> <p>pandas loads data from storage (CSV, Parquet, or Delta Lake) into in-memory DataFrames.</p> <p>pandas makes it easy to modify the data in memory, say update a column value.  It's not easy to update a column value in storage systems like CSV or Parquet using pandas.</p> <p>Delta Lake makes it easy for pandas users to update data in storage.</p>"},{"location":"integrations/delta-lake-pandas/#why-delta-lake-allows-for-faster-queries","title":"Why Delta Lake allows for faster queries","text":"<p>Delta tables store data in many files and metadata about the files in the transaction log.  Delta Lake allows for certain queries to skip entire files, which makes pandas queries run much faster.</p>"},{"location":"integrations/delta-lake-pandas/#more-resources","title":"More resources","text":"<p>See this talk on why Delta Lake is the best file format for pandas analyses to learn more: </p>"},{"location":"integrations/delta-lake-pandas/#conclusion","title":"Conclusion","text":"<p>Delta Lake provides many features that make it an excellent format for pandas analyses:</p> <ul> <li>performance optimizations make pandas queries run faster</li> <li>data management features make pandas analyses more reliable</li> <li>advanced features allow you to perform more complex pandas analyses</li> </ul> <p>Python deltalake offers pandas users a better experience compared with CSV/Parquet.</p>"},{"location":"integrations/delta-lake-polars/","title":"Using Delta Lake with polars","text":"<p>This page explains why Delta Lake is a great storage system for Polars analyses.</p> <p>You will learn how to create Delta tables with Polars, how to query Delta tables with Polars, and the unique advantages Delta Lake offers the Polars community.</p> <p>Here are some amazing benefits that Delta Lake provides Polars users:</p> <ul> <li>time travel</li> <li>ACID transactions for reliable writes</li> <li>better performance with file skipping</li> <li>enhanced file skipping via Z Ordering</li> <li>ability to rollback mistakes</li> <li>and many, many more</li> </ul> <p>Let's start by showing how to use Polars with Delta Lake, explore how Delta Lake can make Polars queries run faster, and then look at all the cool features Delta Lake offers Polars users.</p>"},{"location":"integrations/delta-lake-polars/#creating-a-delta-lake-table-with-polars","title":"Creating a Delta Lake table with Polars","text":"<p>Create a Polars DataFrame and write it out to a Delta table:</p> <pre><code>import polars as pl\n\ndf = pl.DataFrame({\"x\": [1, 2, 3]})\ndf.write_delta(\"tmp/bear_delta_lake\")\n</code></pre> <p>Inspect the contents of the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\"))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 1   |\n| 2   |\n| 3   |\n+-----+\n</code></pre> <p>Now create another Polars DataFrame and append it to the existing Delta table:</p> <pre><code>df2 = pl.DataFrame({\"x\": [8, 9, 10]})\ndf2.write_delta(\"tmp/bear_delta_lake\", mode=\"append\")\n</code></pre> <p>Re-inspect the contents of the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\"))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 1   |\n| 2   |\n| 3   |\n| 8   |\n| 9   |\n| 10  |\n+-----+\n</code></pre> <p>Now overwrite the existing Delta table:</p> <pre><code>df3 = pl.DataFrame({\"x\": [55, 66, 77]})\ndf3.write_delta(\"tmp/bear_delta_lake\", mode=\"overwrite\")\n</code></pre> <p>Inspect the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\"))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 55  |\n| 66  |\n| 77  |\n+-----+\n</code></pre> <p>The Delta table now has three versions, as shown in the following diagram:</p> <p></p>"},{"location":"integrations/delta-lake-polars/#time-travel-with-delta-lake-for-polars","title":"Time travel with Delta Lake for Polars","text":"<p>Time travel back to version 0 of the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\", version=0))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 1   |\n| 2   |\n| 3   |\n+-----+\n</code></pre> <p>Time travel back to version 1 of the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\", version=1))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 1   |\n| 2   |\n| 3   |\n| 9   |\n| 8   |\n| 10  |\n+-----+\n</code></pre> <p>Read the Delta table without specifying a version and see how it reads the latest version by default:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\"))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 55  |\n| 66  |\n| 77  |\n+-----+\n</code></pre> <p>Let's dive into how to read Delta tables with Polars in more detail and compare the query runtime performance on larger datasets.</p>"},{"location":"integrations/delta-lake-polars/#reading-a-delta-lake-table-with-polars","title":"Reading a Delta Lake table with Polars","text":"<p>Let's look at the h2o groupby dataset that has 1 billion rows and 9 columns.  Here are the first three rows of the dataset:</p> <pre><code>+-------+-------+--------------+-------+-------+--------+------+------+---------+\n| id1   | id2   | id3          |   id4 |   id5 |    id6 |   v1 |   v2 |      v3 |\n|-------+-------+--------------+-------+-------+--------+------+------+---------|\n| id016 | id046 | id0000109363 |    88 |    13 | 146094 |    4 |    6 | 18.8377 |\n| id039 | id087 | id0000466766 |    14 |    30 | 111330 |    4 |   14 | 46.7973 |\n| id047 | id098 | id0000307804 |    85 |    23 | 187639 |    3 |    5 | 47.5773 |\n+-------+-------+--------------+-------+-------+--------+------+------+---------+\n</code></pre> <p>This dataset is 50GB when stored in an uncompressed CSV file.  Let's run some queries on this dataset when it's stored in different file formats with Polars.</p> <p>This section will show the runtime for a query when the data is stored in CSV, Parquet, and Delta Lake and explain why Delta tables are the fastest.</p> <p></p> <p>Start by running a query on an uncompressed CSV file with <code>read_csv</code>:</p> <pre><code>pl.read_csv(\"~/data/G1_1e9_1e2_0_0.csv\").filter(pl.col(\"id1\") &lt; \"id016\").group_by(\n    [\"id1\", \"id2\"]\n).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query errors out after running for several minutes.  The machine runs out of memory.  Let's try it again with <code>scan_csv</code>.</p> <pre><code>pl.scan_csv(\"~/data/G1_1e9_1e2_0_0.csv\").filter(pl.col(\"id1\") &lt; \"id016\").group_by(\n    [\"id1\", \"id2\"]\n).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query runs in 56.2 seconds.</p> <p>Now let's run the same query when the data is stored in a Parquet file:</p> <pre><code>pl.scan_parquet(\"~/data/G1_1e9_1e2_0_0.parquet\").filter(\n    pl.col(\"id1\") &lt; \"id016\"\n).group_by([\"id1\", \"id2\"]).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query runs in 8.3 seconds.  It's much faster because Polars is optimized to skip row groups in Parquet files that don't contain data that's relevant for the query.</p> <p>Then run the query on newly created Delta table:</p> <pre><code>pl.scan_delta(\"~/data/deltalake/G1_1e9_1e2_0_0\", version=1).filter(\n    pl.col(\"id1\") &lt; \"id016\"\n).group_by([\"id1\", \"id2\"]).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query runs in 7.2 seconds.  Polars can run this query faster because it can inspect the Delta transaction log and skip entire files that don't contain relevant data before performing the ordinary Parquet row group skipping.</p> <p>Finally run the query on the Delta table after it has been Z Ordered by <code>id1</code>:</p> <pre><code>pl.scan_delta(\"~/data/deltalake/G1_1e9_1e2_0_0\", version=2).filter(\n    pl.col(\"id1\") &lt; \"id016\"\n).group_by([\"id1\", \"id2\"]).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query runs in 3.5 seconds.  The query on the Z Ordered Delta table is even faster because similar data has been co-located in the same files.  This allows for even greater data skipping.</p> <p>Polars can leverage file skipping to query Delta tables very quickly.</p>"},{"location":"integrations/delta-lake-polars/#why-polars-is-fast-with-delta-lake","title":"Why Polars is fast with Delta Lake","text":"<p>Delta tables consist of metadata in a transaction log and data stored in Parquet files.</p> <p>When Polars queries a Delta table, it starts by consulting the transaction log to understand the metadata of each file in the Delta table.  This allows for Polars to quickly identify which files should be skipped by the query.</p> <p>CSV files don't contain any such metadata, so file skipping isn't an option.  Polars can skip Parquet files based on metadata, but it needs to open up each file and read the metadata, which is slower that grabbing the file-level metadata directly from the transaction log.</p> <p>Parquet doesn't allow users to easily Z Order the data and colocate similar data in the same row groups.  The Z Order optimizations are only supported in Delta tables.</p> <p>Delta Lake offers Polars users with unique performance optimizations.</p>"},{"location":"integrations/delta-lake-polars/#other-delta-lake-features-relevant-for-polars-users","title":"Other Delta Lake features relevant for Polars users","text":"<ul> <li>ACID transactions for reliable writes</li> <li>better performance with file skipping</li> <li>enhanced file skipping via Z Ordering</li> <li>ability to rollback mistakes</li> </ul>"},{"location":"integrations/delta-lake-polars/#conclusion","title":"Conclusion","text":"<p>This guide shows how Delta Lake is a great storage format for Polars analyses.</p> <p>Delta Lake is easy to use, fast, and full of features that are great for Polars users.</p>"},{"location":"integrations/object-storage/adls/","title":"Azure ADLS Storage Backend","text":"<p><code>delta-rs</code> offers native support for using Microsoft Azure Data Lake Storage (ADSL) as an object storage backend.</p> <p>You don\u2019t need to install any extra dependencies to read/write Delta tables to S3 with engines that use <code>delta-rs</code>. You do need to configure your ADLS access credentials correctly.</p>"},{"location":"integrations/object-storage/adls/#passing-credentials-explicitly","title":"Passing Credentials Explicitly","text":"<p>You can also pass ADLS credentials to your query engine explicitly.</p> <p>For Polars, you would do this using the <code>storage_options</code> keyword as demonstrated above. This will forward your credentials to the <code>object store</code> library that Polars uses for cloud storage access under the hood. Read the <code>object store</code> documentation for more information defining specific credentials.</p>"},{"location":"integrations/object-storage/adls/#example-write-delta-table-to-adls-with-polars","title":"Example: Write Delta table to ADLS with Polars","text":"<p>Using Polars, you can write a Delta table to ADLS directly like this:</p> <pre><code>import polars as pl\n\ndf = pl.DataFrame({\"foo\": [1, 2, 3, 4, 5]})\n\n# define container name\ncontainer = &lt;container_name&gt;\n\n# define credentials\nstorage_options = {\n    \"ACCOUNT_NAME\": &lt;account_name&gt;,\n    \"ACCESS_KEY\": &lt;access_key&gt;,\n}\n\n# write Delta to ADLS\ndf_pl.write_delta(\n    f\"abfs://{container}/delta_table\",\n    storage_options = storage_options\n)\n</code></pre>"},{"location":"integrations/object-storage/adls/#example-with-pandas","title":"Example with pandas","text":"<p>For libraries without direct <code>write_delta</code> methods (like Pandas), you can use the <code>write_deltalake</code> function from the <code>deltalake</code> library:</p> <pre><code>import pandas as pd\nfrom deltalake import write_deltalake\n\ndf = pd.DataFrame({\"foo\": [1, 2, 3, 4, 5]})\n\nwrite_deltalake(\n    f\"abfs://{container}/delta_table_pandas\",\n    df,\n    storage_options=storage_options\n)\n</code></pre>"},{"location":"integrations/object-storage/adls/#using-local-authentication","title":"Using Local Authentication","text":"<p>If your local session is authenticated using the Azure CLI then you can write Delta tables directly to ADLS. Read more about this in the Azure CLI documentation.</p>"},{"location":"integrations/object-storage/gcs/","title":"GCS Storage Backend","text":"<p><code>delta-rs</code> offers native support for using Google Cloud Storage (GCS) as an object storage backend.</p> <p>You don\u2019t need to install any extra dependencies to read/write Delta tables to GCS with engines that use <code>delta-rs</code>. You do need to configure your GCS access credentials correctly.</p>"},{"location":"integrations/object-storage/gcs/#using-application-default-credentials","title":"Using Application Default Credentials","text":"<p>Application Default Credentials (ADC) is a strategy used by GCS to automatically find credentials based on the application environment.</p> <p>If you are working from your local machine and have ADC set up then you can read/write Delta tables from GCS directly, without having to pass your credentials explicitly.</p>"},{"location":"integrations/object-storage/gcs/#example-write-delta-tables-to-gcs-with-polars","title":"Example: Write Delta tables to GCS with Polars","text":"<p>Using Polars, you can write a Delta table to GCS like this:</p> <pre><code># create a toy dataframe\nimport polars as pl\ndf = pl.DataFrame({\"foo\": [1, 2, 3, 4, 5]})\n\n# define path\ntable_path = \"gs://bucket/delta-table\"\n\n# write Delta to GCS\ndf.write_delta(table_path)\n</code></pre>"},{"location":"integrations/object-storage/gcs/#passing-gcs-credentials-explicitly","title":"Passing GCS Credentials explicitly","text":"<p>Alternatively, you can pass GCS credentials to your query engine explicitly.</p> <p>For Polars, you would do this using the <code>storage_options</code> keyword. This will forward your credentials to the <code>object store</code> library that Polars uses under the hood. Read the Polars documentation and the <code>object store</code> documentation for more information.</p>"},{"location":"integrations/object-storage/gcs/#delta-lake-on-gcs-required-permissions","title":"Delta Lake on GCS: Required permissions","text":"<p>You will need the following permissions in your GCS account:</p> <ul> <li><code>storage.objects.create</code></li> <li><code>storage.objects.delete</code> (only required for uploads that overwrite an existing object)</li> <li><code>storage.objects.get</code> (only required if you plan on using the Google Cloud CLI)</li> <li><code>storage.objects.list</code> (only required if you plan on using the Google Cloud CLI)</li> </ul> <p>For more information, see the GCP documentation</p>"},{"location":"integrations/object-storage/hdfs/","title":"HDFS Storage Backend","text":"<p>HDFS support is provided via the hdfs-native-object-store package, which sits on top of hdfs-native. This is an HDFS client written from scratch in Rust, with no bindings to libhdfs or any use of Java. While it supports most common cluster configurations, it does not support every possible client configuration that could exist.</p>"},{"location":"integrations/object-storage/hdfs/#supported-configurations","title":"Supported Configurations","text":"<p>By default, the client looks for existing Hadoop configs in following manner:</p> <ul> <li>If the <code>HADOOP_CONF_DIR</code> environment variable is defined, load configs from <code>$HADOOP_CONF_DIR/core-site.xml</code> and <code>$HADOOP_CONF_DIR/hdfs-site.xml</code></li> <li>Otherwise, if the <code>HADOOP_HOME</code> environment variable is set, load configs from <code>$HADOOP_HOME/etc/hadoop/core-site.xml</code> and <code>$HADOOP_HOME/etc/hadoop/hdfs-site.xml</code></li> </ul> <p>Additionally, you can pass Hadoop configs as <code>storage_options</code> and these will take precedence over the above configs.</p> <p>Currently the supported client configuration parameters are:</p> <ul> <li><code>dfs.ha.namenodes.*</code> - name service support</li> <li><code>dfs.namenode.rpc-address.*</code> - name service support</li> <li><code>fs.viewfs.mounttable.*.link.*</code> - ViewFS links</li> <li><code>fs.viewfs.mounttable.*.linkFallback</code> - ViewFS link fallback</li> </ul> <p>If you find your setup is not supported, please file an issue in the hdfs-native repository.</p>"},{"location":"integrations/object-storage/hdfs/#secure-clusters","title":"Secure Clusters","text":"<p>The client supports connecting to secure clusters through both Kerberos authentication as well as token authentication, and all SASL protection types are supported. The highest supported protection mechanism advertised by the server will be used.</p>"},{"location":"integrations/object-storage/hdfs/#kerberos-support","title":"Kerberos Support","text":"<p>Kerberos is supported through dynamically loading the <code>libgssapi_krb5</code> library. This must be installed separately through your package manager, and currently only works on Linux and Mac.</p> <p>Debian-based systems: <pre><code>apt-get install libgssapi-krb5-2\n</code></pre></p> <p>RHEL-based systems: <pre><code>yum install krb5-libs\n</code></pre></p> <p>MacOS: <pre><code>brew install krb5\n</code></pre></p> <p>Then simply <code>kinit</code> to get your TGT and authentication to HDFS should just work.</p>"},{"location":"integrations/object-storage/hdfs/#token-support","title":"Token Support","text":"<p>Token authentication is supported by looking for a token file located at the environment variable <code>HADOOP_TOKEN_FILE_LOCATION</code>. This is the location systems like YARN will automatically place a delegation token, so things will just work inside of YARN jobs.</p>"},{"location":"integrations/object-storage/hdfs/#issues","title":"Issues","text":"<p>If you face any HDFS-specific issues, please report to the hdfs-native-object-store repository.</p>"},{"location":"integrations/object-storage/s3-like/","title":"CloudFlare R2 &amp; Minio","text":"<p><code>delta-rs</code> offers native support for using Cloudflare R2 or Minio as an S3-compatible storage backend. R2 and Minio support conditional puts, which removes the need for DynamoDB for safe concurrent writes.  However, we have to pass the <code>aws_conditional_put</code> flag into <code>storage_options</code>. See the example below.</p> <p>You don\u2019t need to install any extra dependencies to read/write Delta tables to S3 with engines that use <code>delta-rs</code>. You do need to configure your AWS access credentials correctly.</p>"},{"location":"integrations/object-storage/s3-like/#passing-s3-credentials","title":"Passing S3 Credentials","text":"<p>You can pass your AWS credentials explicitly by using:</p> <ul> <li>the <code>storage_options</code>kwarg</li> <li>Environment variables</li> </ul>"},{"location":"integrations/object-storage/s3-like/#example","title":"Example","text":"<p>Let's work through an example with Polars. The same logic applies to other Python engines like Pandas, Daft, Dask, etc.</p> <p>Follow the steps below to use Delta Lake on S3 (R2/Minio) with Polars:</p> <ol> <li>Install Polars and deltalake. For example, using:</li> </ol> <p><code>pip install polars deltalake</code></p> <ol> <li>Create a dataframe with some toy data.</li> </ol> <p><code>df = pl.DataFrame({'x': [1, 2, 3]})</code></p> <ol> <li>Set your <code>storage_options</code> correctly.</li> </ol> <pre><code>storage_options = {\n    'AWS_SECRET_ACCESS_KEY': &lt;access_key&gt;,\n    'aws_conditional_put': 'etag', # Here we say to use conditional put, this provides safe concurrency.\n}\n</code></pre> <ol> <li>Write data to Delta table using the <code>storage_options</code> kwarg.</li> </ol> <pre><code>df.write_delta(\n    \"s3://bucket/delta_table\",\n    storage_options=storage_options,\n)\n</code></pre>"},{"location":"integrations/object-storage/s3-like/#minio-and-docker","title":"Minio and Docker","text":"<p>Minio is straightforward to host locally with Docker and docker-compose, via the following <code>docker-compose.yml</code> file - just run <code>docker-compose up</code>:</p> <pre><code>version: '3.8'\n\nservices:\n  minio:\n    image: minio/minio\n    ports:\n      - \"9000:9000\"\n      - \"9001:9001\"\n    environment:\n      MINIO_ROOT_USER: ...\n      MINIO_ROOT_PASSWORD: ...\n    command: server /data --console-address \":9001\"\n</code></pre> <p>With this configuration, Minio will host its S3-compatible API over HTTP, not HTTPS, on port 9000.  This requires an additional flag in <code>storage_options</code>, <code>AWS_ALLOW_HTTP</code>, to be set to <code>true</code>:</p> <pre><code>storage_options = {\n    'AWS_ACCESS_KEY_ID': ...,\n    'AWS_SECRET_ACCESS_KEY': ...,\n    'AWS_ENDPOINT_URL': 'http://localhost:9000',\n    'AWS_ALLOW_HTTP': 'true',\n    'aws_conditional_put': 'etag'\n}\n</code></pre>"},{"location":"integrations/object-storage/s3/","title":"AWS S3 Storage Backend","text":"<p><code>delta-rs</code> offers native support for using AWS S3 as an object storage backend.</p> <p>You don\u2019t need to install any extra dependencies to read/write Delta tables to S3 with engines that use <code>delta-rs</code>. You do need to configure your AWS access credentials correctly.</p>"},{"location":"integrations/object-storage/s3/#note-for-boto3-users","title":"Note for boto3 users","text":"<p>Many Python engines use boto3 to connect to AWS. This library supports reading credentials automatically from your local <code>.aws/config</code> or <code>.aws/creds</code> file.</p> <p>For example, if you\u2019re running locally with the proper credentials in your local <code>.aws/config</code> or <code>.aws/creds</code> file then you can write a Parquet file to S3 like this with pandas:</p> <pre><code>    import pandas as pd\n    df = pd.DataFrame({'x': [1, 2, 3]})\n    df.to_parquet(\"s3://avriiil/parquet-test-pandas\")\n</code></pre> <p>The <code>delta-rs</code> writer does not use <code>boto3</code> and therefore does not support taking credentials from your <code>.aws/config</code> or <code>.aws/creds</code> file. If you\u2019re used to working with writers from Python engines like Polars, pandas or Dask, this may mean a small change to your workflow.</p>"},{"location":"integrations/object-storage/s3/#passing-aws-credentials","title":"Passing AWS Credentials","text":"<p>You can pass your AWS credentials explicitly by using:</p> <ul> <li>the <code>storage_options</code>kwarg</li> <li>Environment variables</li> <li>EC2 metadata if using EC2 instances</li> <li>AWS Profiles</li> </ul>"},{"location":"integrations/object-storage/s3/#example","title":"Example","text":"<p>Let's work through an example with Polars. The same logic applies to other Python engines like Pandas, Daft, Dask, etc.</p> <p>Follow the steps below to use Delta Lake on S3 with Polars:</p> <ol> <li>Install Polars and deltalake. For example, using:</li> </ol> <p><code>pip install polars deltalake</code></p> <ol> <li>Create a dataframe with some toy data.</li> </ol> <p><code>df = pl.DataFrame({'x': [1, 2, 3]})</code></p> <ol> <li>Set your <code>storage_options</code> correctly.</li> </ol> <pre><code>storage_options = {\n    \"AWS_REGION\":&lt;region_name&gt;,\n    'AWS_ACCESS_KEY_ID': &lt;key_id&gt;,\n    'AWS_SECRET_ACCESS_KEY': &lt;access_key&gt;,\n    'AWS_S3_LOCKING_PROVIDER': 'dynamodb',\n    'DELTA_DYNAMO_TABLE_NAME': 'delta_log',\n}\n</code></pre> <ol> <li>Write data to Delta table using the <code>storage_options</code> kwarg.</li> </ol> <pre><code>df.write_delta(\n    \"s3://bucket/delta_table\",\n    storage_options=storage_options,\n)\n</code></pre>"},{"location":"integrations/object-storage/s3/#delta-lake-on-aws-s3-safe-concurrent-writes","title":"Delta Lake on AWS S3: Safe Concurrent Writes","text":"<p>You need a locking provider to ensure safe concurrent writes when writing Delta tables to AWS S3. This is because AWS S3 does not guarantee mutual exclusion.</p> <p>A locking provider guarantees that only one writer is able to create the same file. This prevents corrupted or conflicting data.</p> <p><code>delta-rs</code> uses DynamoDB to guarantee safe concurrent writes.</p> <p>Run the code below in your terminal to create a DynamoDB table that will act as your locking provider.</p> <pre><code>    aws dynamodb create-table \\\n    --table-name delta_log \\\n    --attribute-definitions AttributeName=tablePath,AttributeType=S AttributeName=fileName,AttributeType=S \\\n    --key-schema AttributeName=tablePath,KeyType=HASH AttributeName=fileName,KeyType=RANGE \\\n    --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5\n</code></pre> <p>If for some reason you don't want to use DynamoDB as your locking mechanism you can choose to set the <code>AWS_S3_ALLOW_UNSAFE_RENAME</code> variable to <code>true</code> in order to enable S3 unsafe writes.</p> <p>Read more in the Usage section.</p>"},{"location":"integrations/object-storage/s3/#delta-lake-on-aws-s3-required-permissions","title":"Delta Lake on AWS S3: Required permissions","text":"<p>You need to have permissions to get, put and delete objects in the S3 bucket you're storing your data in. Please note that you must be allowed to delete objects even if you're just appending to the Delta Lake, because there are temporary files into the log folder that are deleted after usage.</p> <p>In AWS S3, you will need the following permissions:</p> <ul> <li>s3:GetObject</li> <li>s3:PutObject</li> <li>s3:DeleteObject</li> </ul> <p>In DynamoDB, you will need the following permissions:</p> <ul> <li>dynamodb:GetItem</li> <li>dynamodb:Query</li> <li>dynamodb:PutItem</li> <li>dynamodb:UpdateItem</li> </ul>"},{"location":"usage/appending-overwriting-delta-lake-table/","title":"Appending to and overwriting a Delta Lake table","text":"<p>This section explains how to append to an exising Delta table and how to overwrite a Delta table.</p>"},{"location":"usage/appending-overwriting-delta-lake-table/#delta-lake-append-transactions","title":"Delta Lake append transactions","text":"<p>Suppose you have a Delta table with the following contents:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n+-------+----------+\n</code></pre> <p>Append two additional rows of data to the table:</p> <pre><code>from deltalake import write_deltalake, DeltaTable\n\ndf = pd.DataFrame({\"num\": [8, 9], \"letter\": [\"dd\", \"ee\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"append\")\n</code></pre> <p>Here are the updated contents of the Delta table:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre> <p>Now let's see how to perform an overwrite transaction.</p>"},{"location":"usage/appending-overwriting-delta-lake-table/#delta-lake-overwrite-transactions","title":"Delta Lake overwrite transactions","text":"<p>Now let's see how to overwrite the exisitng Delta table.</p> <pre><code>df = pd.DataFrame({\"num\": [11, 22], \"letter\": [\"aa\", \"bb\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"overwrite\")\n</code></pre> <p>Here are the contents of the Delta table after the overwrite operation:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|    11 | aa       |\n|    22 | bb       |\n+-------+----------+\n</code></pre> <p>Overwriting just performs a logical delete.  It doesn't physically remove the previous data from storage.  Time travel back to the previous version to confirm that the old version of the table is still accessable.</p> <pre><code>dt = DeltaTable(\"tmp/some-table\", version=1)\n\n+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre>"},{"location":"usage/constraints/","title":"Adding a Constraint to a table","text":"<p>Check constraints are a way to enforce that only data that meets the constraint is allowed to be added to the table.</p>"},{"location":"usage/constraints/#add-the-constraint","title":"Add the Constraint","text":"Python Rust <p> <code>DeltaTable</code> <pre><code>from deltalake import DeltaTable\n\ndt = DeltaTable(\"../rust/tests/data/simple_table\")\n\n# Check the schema before hand\nprint(dt.schema())\n# Add the constraint to the table.\ndt.alter.add_constraint({\"id_gt_0\": \"id &gt; 0\"})\n</code></pre></p> <p> <code>DeltaTable</code> <pre><code>let table = deltalake::open_table(\"../rust/tests/data/simple_table\").await?;\nlet ops = DeltaOps(table);\nops.with_constraint(\"id_gt_0\", \"id &gt; 0\").await?;\n</code></pre></p> <p>After you have added the constraint to the table attempting to append data to the table that violates the constraint will instead throw an error.</p>"},{"location":"usage/constraints/#verify-the-constraint-by-trying-to-add-some-data","title":"Verify the constraint by trying to add some data","text":"Python Rust <pre><code>from deltalake import write_deltalake, DeltaTable\nimport pandas as pd\n\ndt = DeltaTable(\"../rust/tests/data/simple_table\")\n\ndf = pd.DataFrame({\"id\": [-1]})\nwrite_deltalake(dt, df, mode=\"append\", engine=\"rust\")\n# _internal.DeltaProtocolError: Invariant violations: [\"Check or Invariant (id &gt; 0) violated by value in row: [-1]\"]\n</code></pre> <pre><code>let table = deltalake::open_table(\"../rust/tests/data/simple_table\").await?;\nlet schema = table.get_state().arrow_schema()?;\nlet invalid_values: Vec&lt;Arc&lt;dyn Array&gt;&gt; = vec![\n    Arc::new(Int32Array::from(vec![-10]))\n];\nlet batch = RecordBatch::try_new(schema, invalid_values)?;\ntable.write(vec![batch]).await?;\n</code></pre> <p>Note: ensure you use the <code>engine='rust'</code> parameter when writing to the table as this feature is not supported in the default pyarrow writer. </p>"},{"location":"usage/create-delta-lake-table/","title":"Creating a Delta Lake Table","text":"<p>This section explains how to create a Delta Lake table.</p> <p>You can easily write a DataFrame to a Delta table.</p> pandasPolars <pre><code>from deltalake import write_deltalake\nimport pandas as pd\n\ndf = pd.DataFrame({\"num\": [1, 2, 3], \"letter\": [\"a\", \"b\", \"c\"]})\nwrite_deltalake(\"tmp/some-table\", df)\n</code></pre> <pre><code>import polars as pl\n\ndf = pl.DataFrame({\"num\": [1, 2, 3], \"letter\": [\"a\", \"b\", \"c\"]})\ndf.write_delta(\"tmp/some-table\")\n</code></pre> <p>Here are the contents of the Delta table in storage:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n+-------+----------+\n</code></pre>"},{"location":"usage/deleting-rows-from-delta-lake-table/","title":"Deleting rows from a Delta Lake table","text":"<p>This section explains how to delete rows from a Delta Lake table.</p> <p>Suppose you have the following Delta table with four rows:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     4 | d        |\n+-------+----------+\n</code></pre> <p>Here's how to delete all the rows where the <code>num</code> is greater than 2:</p> <pre><code>dt = DeltaTable(\"tmp/my-table\")\ndt.delete(\"num &gt; 2\")\n</code></pre> <p>Here are the contents of the Delta table after the delete operation has been performed:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n+-------+----------+\n</code></pre> <p><code>dt.delete()</code> accepts any <code>SQL where</code> clause. If no predicate is provided, all rows will be deleted.</p> <p>Read more in the API docs</p>"},{"location":"usage/examining-table/","title":"Examining a Table","text":""},{"location":"usage/examining-table/#metadata","title":"Metadata","text":"<p>The delta log maintains basic metadata about a table, including:</p> <ul> <li>A unique <code>id</code></li> <li>A <code>name</code>, if provided</li> <li>A <code>description</code>, if provided</li> <li>The list of <code>partitionColumns</code>.</li> <li>The <code>created_time</code> of the table</li> <li>A map of table <code>configuration</code>. This includes fields such as     <code>delta.appendOnly</code>, which if <code>true</code> indicates the table is not meant     to have data deleted from it.</li> </ul> <p>Get metadata from a table with the DeltaTable.metadata() method:</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/simple_table\")\n&gt;&gt;&gt; dt.metadata()\nMetadata(id: 5fba94ed-9794-4965-ba6e-6ee3c0d22af9, name: None, description: None, partitionColumns: [], created_time: 1587968585495, configuration={})\n</code></pre>"},{"location":"usage/examining-table/#schema","title":"Schema","text":"<p>The schema for the table is also saved in the transaction log. It can either be retrieved in the Delta Lake form as Schema or as a PyArrow schema. The first allows you to introspect any column-level metadata stored in the schema, while the latter represents the schema the table will be loaded into.</p> <p>Use DeltaTable.schema to retrieve the delta lake schema:</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/simple_table\")\n&gt;&gt;&gt; dt.schema()\nSchema([Field(id, PrimitiveType(\"long\"), nullable=True)])\n</code></pre> <p>These schemas have a JSON representation that can be retrieved. To reconstruct from json, use DeltaTable.schema.to_json().</p> <pre><code>&gt;&gt;&gt; dt.schema().to_json()\n'{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}'\n</code></pre> <p>Use DeltaTable.schema.to_pyarrow() to retrieve the PyArrow schema:</p> <pre><code>&gt;&gt;&gt; dt.schema().to_pyarrow()\nid: int64\n</code></pre>"},{"location":"usage/examining-table/#history","title":"History","text":"<p>Depending on what system wrote the table, the delta table may have provenance information describing what operations were performed on the table, when, and by whom. This information is retained for 30 days by default, unless otherwise specified by the table configuration <code>delta.logRetentionDuration</code>.</p> <p>Note</p> <p>This information is not written by all writers and different writers may use different schemas to encode the actions. For Spark\\'s format, see: https://docs.delta.io/latest/delta-utility.html#history-schema</p> <p>To view the available history, use <code>DeltaTable.history</code>:</p> <pre><code>from deltalake import DeltaTable\n\ndt = DeltaTable(\"../rust/tests/data/simple_table\")\ndt.history()\n</code></pre> <pre><code>[{'timestamp': 1587968626537, 'operation': 'DELETE', 'operationParameters': {'predicate': '[\"((`id` % CAST(2 AS BIGINT)) = CAST(0 AS BIGINT))\"]'}, 'readVersion': 3, 'isBlindAppend': False},\n {'timestamp': 1587968614187, 'operation': 'UPDATE', 'operationParameters': {'predicate': '((id#697L % cast(2 as bigint)) = cast(0 as bigint))'}, 'readVersion': 2, 'isBlindAppend': False},\n {'timestamp': 1587968604143, 'operation': 'WRITE', 'operationParameters': {'mode': 'Overwrite', 'partitionBy': '[]'}, 'readVersion': 1, 'isBlindAppend': False},\n {'timestamp': 1587968596254, 'operation': 'MERGE', 'operationParameters': {'predicate': '(oldData.`id` = newData.`id`)'}, 'readVersion': 0, 'isBlindAppend': False},\n {'timestamp': 1587968586154, 'operation': 'WRITE', 'operationParameters': {'mode': 'ErrorIfExists', 'partitionBy': '[]'}, 'isBlindAppend': True}]\n</code></pre>"},{"location":"usage/examining-table/#current-add-actions","title":"Current Add Actions","text":"<p>The active state for a delta table is determined by the Add actions, which provide the list of files that are part of the table and metadata about them, such as creation time, size, and statistics. You can get a data frame of the add actions data using <code>DeltaTable.get_add_actions</code>:</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/delta-0.8.0\")\n&gt;&gt;&gt; dt.get_add_actions(flatten=True).to_pandas()\n                                                    path  size_bytes   modification_time  data_change  num_records  null_count.value  min.value  max.value\n0  part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a...         440 2021-03-06 15:16:07         True            2                 0          0          2\n1  part-00000-04ec9591-0b73-459e-8d18-ba5711d6cbe...         440 2021-03-06 15:16:16         True            2                 0          2          4\n</code></pre> <p>This works even with past versions of the table:</p> <pre><code>&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/delta-0.8.0\", version=0)\n&gt;&gt;&gt; dt.get_add_actions(flatten=True).to_pandas()\n                                                path  size_bytes   modification_time  data_change  num_records  null_count.value  min.value  max.value\n0  part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a...         440 2021-03-06 15:16:07         True            2                 0          0          2\n1  part-00001-911a94a2-43f6-4acb-8620-5e68c265498...         445 2021-03-06 15:16:07         True            3                 0          2          4\n</code></pre>"},{"location":"usage/installation/","title":"Installation","text":"<p>The <code>deltalake</code> project can be installed via pip for Python or Cargo for Rust.</p>"},{"location":"usage/installation/#install-delta-lake-for-python","title":"Install Delta Lake for Python","text":"<p>With pip:</p> <pre><code>pip install deltalake\n</code></pre> <p>With Conda:</p> <pre><code>conda install -c conda-forge deltalake\n</code></pre>"},{"location":"usage/installation/#install-delta-lake-for-rust","title":"Install Delta Lake for Rust","text":"<p>With Cargo:</p> <pre><code>cargo add deltalake\n</code></pre>"},{"location":"usage/installation/#run-delta-lake-and-pandas-in-a-jupyter-notebook","title":"Run Delta Lake and pandas in a Jupyter Notebook","text":"<p>You can easily run Delta Lake and pandas in a Jupyter notebook.</p> <p>Create an environment file with the dependencies as follows:</p> <pre><code>name: deltalake-minimal\nchannels:\n  - conda-forge\n  - defaults\ndependencies:\n  - python=3.11\n  - ipykernel\n  - pandas\n  - polars\n  - jupyterlab\n  - deltalake\n</code></pre> <p>Create a virtual environment with the dependencies:</p> <pre><code>conda env create -f deltalake-minimal.yml\n</code></pre> <p>Open the Jupyter notebook and run commands as follows:</p> <p></p>"},{"location":"usage/loading-table/","title":"Loading a Delta Table","text":"<p>A DeltaTable represents the state of a delta table at a particular version. This includes which files are currently part of the table, the schema of the table, and other metadata such as creation time.</p>  Python Rust <p> <code>DeltaTable</code> <pre><code>from deltalake import DeltaTable\n\ndt = DeltaTable(\"../rust/tests/data/delta-0.2.0\")\nprint(f\"Version: {dt.version()}\")\nprint(f\"Files: {dt.files()}\")\n</code></pre></p> <p> <code>DeltaTable</code> <pre><code>let table = deltalake::open_table(\"../rust/tests/data/simple_table\").await.unwrap();\nprintln!(\"Version: {}\", table.version());\nprintln!(\"Files: {}\", table.get_files());\n</code></pre></p> <p>Depending on your storage backend, you could use the <code>storage_options</code> parameter to provide some configuration. Configuration is defined for specific backends - s3 options, azure options, gcs options.</p> <pre><code>&gt;&gt;&gt; storage_options = {\"AWS_ACCESS_KEY_ID\": \"THE_AWS_ACCESS_KEY_ID\", \"AWS_SECRET_ACCESS_KEY\":\"THE_AWS_SECRET_ACCESS_KEY\"}\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/delta-0.2.0\", storage_options=storage_options)\n</code></pre> <p>The configuration can also be provided via the environment, and the basic service provider is derived from the URL being used. We try to support many of the well-known formats to identify basic service properties.</p> <p>S3:</p> <ul> <li>s3://\\&lt;bucket&gt;/\\&lt;path&gt;</li> <li>s3a://\\&lt;bucket&gt;/\\&lt;path&gt;</li> </ul> <p>Note that <code>delta-rs</code> does not read credentials from a local <code>.aws/config</code> or <code>.aws/creds</code> file. Credentials can be accessed from environment variables, ec2 metadata, profiles or web identity. You can also pass credentials to <code>storage_options</code> using <code>AWS_ACCESS_KEY_ID</code> and <code>AWS_SECRET_ACCESS_KEY</code>.</p> <p>Azure:</p> <ul> <li>az://\\&lt;container&gt;/\\&lt;path&gt;</li> <li>adl://\\&lt;container&gt;/\\&lt;path&gt;</li> <li>abfs://\\&lt;container&gt;/\\&lt;path&gt;</li> </ul> <p>GCS:</p> <ul> <li>gs://\\&lt;bucket&gt;/\\&lt;path&gt;</li> </ul> <p>Alternatively, if you have a data catalog you can load it by reference to a database and table name. Currently only AWS Glue is supported.</p> <p>For AWS Glue catalog, use AWS environment variables to authenticate.</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; from deltalake import DataCatalog\n&gt;&gt;&gt; database_name = \"simple_database\"\n&gt;&gt;&gt; table_name = \"simple_table\"\n&gt;&gt;&gt; data_catalog = DataCatalog.AWS\n&gt;&gt;&gt; dt = DeltaTable.from_data_catalog(data_catalog=data_catalog, database_name=database_name, table_name=table_name)\n&gt;&gt;&gt; dt.to_pyarrow_table().to_pydict()\n{'id': [5, 7, 9, 5, 6, 7, 8, 9]}\n</code></pre>"},{"location":"usage/loading-table/#verify-table-existence","title":"Verify Table Existence","text":"<p>You can check whether or not a Delta table exists at a particular path by using the <code>DeltaTable.is_deltatable()</code> method.</p> <pre><code>from deltalake import DeltaTable\n\ntable_path = \"&lt;path/to/valid/table&gt;\"\nDeltaTable.is_deltatable(table_path)\n# True\n\ninvalid_table_path = \"&lt;path/to/nonexistent/table&gt;\"\nDeltaTable.is_deltatable(invalid_table_path)\n# False\n\nbucket_table_path = \"&lt;path/to/valid/table/in/bucket&gt;\"\nstorage_options = {\n    \"AWS_ACCESS_KEY_ID\": \"THE_AWS_ACCESS_KEY_ID\",\n    \"AWS_SECRET_ACCESS_KEY\": \"THE_AWS_SECRET_ACCESS_KEY\",\n    ...\n}\nDeltaTable.is_deltatable(bucket_table_path)\n# True\n</code></pre>"},{"location":"usage/loading-table/#custom-storage-backends","title":"Custom Storage Backends","text":"<p>While delta always needs its internal storage backend to work and be properly configured, in order to manage the delta log, it may sometime be advantageous - and is common practice in the arrow world - to customize the storage interface used for reading the bulk data.</p> <p><code>deltalake</code> will work with any storage compliant with <code>pyarrow.fs.FileSystem</code>, however the root of the filesystem has to be adjusted to point at the root of the Delta table. We can achieve this by wrapping the custom filesystem into a <code>pyarrow.fs.SubTreeFileSystem</code>.</p> <pre><code>import pyarrow.fs as fs\nfrom deltalake import DeltaTable\n\npath = \"&lt;path/to/table&gt;\"\nfilesystem = fs.SubTreeFileSystem(path, fs.LocalFileSystem())\n\ndt = DeltaTable(path)\nds = dt.to_pyarrow_dataset(filesystem=filesystem)\n</code></pre> <p>When using the pyarrow factory method for file systems, the normalized path is provided on creation. In case of S3 this would look something like:</p> <pre><code>import pyarrow.fs as fs\nfrom deltalake import DeltaTable\n\ntable_uri = \"s3://&lt;bucket&gt;/&lt;path&gt;\"\nraw_fs, normalized_path = fs.FileSystem.from_uri(table_uri)\nfilesystem = fs.SubTreeFileSystem(normalized_path, raw_fs)\n\ndt = DeltaTable(table_uri)\nds = dt.to_pyarrow_dataset(filesystem=filesystem)\n</code></pre>"},{"location":"usage/loading-table/#time-travel","title":"Time Travel","text":"<p>To load previous table states, you can provide the version number you wish to load:</p> <pre><code>&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/simple_table\", version=2)\n</code></pre> <p>Once you\\'ve loaded a table, you can also change versions using either a version number or datetime string:</p> <pre><code>&gt;&gt;&gt; dt.load_version(1)\n&gt;&gt;&gt; dt.load_with_datetime(\"2021-11-04 00:05:23.283+00:00\")\n</code></pre> <p>Warning</p> <p>Previous table versions may not exist if they have been vacuumed, in which case an exception will be thrown. See Vacuuming tables for more information.</p>"},{"location":"usage/managing-tables/","title":"Managing Delta Tables","text":""},{"location":"usage/managing-tables/#vacuuming-tables","title":"Vacuuming tables","text":"<p>Vacuuming a table will delete any files that have been marked for deletion. This may make some past versions of a table invalid, so this can break time travel. However, it will save storage space. Vacuum will retain files in a certain window, by default one week, so time travel will still work in shorter ranges.</p> <p>Delta tables usually don't delete old files automatically, so vacuuming regularly is considered good practice, unless the table is only appended to.</p> <p>Use <code>DeltaTable.vacuum</code> to perform the vacuum operation. Note that to prevent accidental deletion, the function performs a dry-run by default: it will only list the files to be deleted. Pass <code>dry_run=False</code> to actually delete files.</p> <pre><code>&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/simple_table\")\n&gt;&gt;&gt; dt.vacuum()\n['../rust/tests/data/simple_table/part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet',\n '../rust/tests/data/simple_table/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet',\n '../rust/tests/data/simple_table/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet',\n ...]\n&gt;&gt;&gt; dt.vacuum(dry_run=False) # Don't run this unless you are sure!\n</code></pre>"},{"location":"usage/managing-tables/#optimizing-tables","title":"Optimizing tables","text":"<p>Optimizing a table compacts small files into larger files to avoid the small file problem. This is especially important for tables that get small amounts of data appended to with high frequency. In addition to compacting small files, you can colocate similar data in the same files with Z Ordering, which allows for better file skipping and faster queries.</p> <p>A table <code>dt = DeltaTable(...)</code> has two methods for optimizing it:</p> <ul> <li><code>dt.optimize.compact()</code> for compacting small files,</li> <li><code>dt.optimize.z_order()</code> to compact and apply Z Ordering.</li> </ul> <p>See the section Small file compaction for more information and a detailed example on <code>compact</code>, and the section Z Order for more information on <code>z_order</code>.</p>"},{"location":"usage/overview/","title":"Usage","text":"<p>This guide teaches you how to use Delta Lake.  You will learn how to create Delta tables, run queries, perform DML operations, and optimize your tables.</p> <p>It's easy to use Delta Lake with pandas, Polars, Rust, or any other PyArrow-like DataFrame library.</p> <p>See the Spark Delta Lake documentation if you're using Delta Lake with Spark.</p>"},{"location":"usage/querying-delta-tables/","title":"Querying Delta Tables","text":"<p>Delta tables can be queried in several ways. By loading as Arrow data or an Arrow dataset, they can be used by compatible engines such as Pandas and DuckDB. By passing on the list of files, they can be loaded into other engines such as Dask.</p> <p>Delta tables are often larger than can fit into memory on a single computer, so this module provides ways to read only the parts of the data you need. Partition filters allow you to skip reading files that are part of irrelevant partitions. Only loading the columns required also saves memory. Finally, some methods allow reading tables batch-by-batch, allowing you to process the whole table while only having a portion loaded at any given time.</p> <p>To load into Pandas or a PyArrow table use the <code>DeltaTable.to_pandas</code> and <code>DeltaTable.to_pyarrow_table</code> methods, respectively. Both of these support filtering partitions and selecting particular columns.</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/delta-0.8.0-partitioned\")\n&gt;&gt;&gt; dt.schema().to_pyarrow()\nvalue: string\nyear: string\nmonth: string\nday: string\n&gt;&gt;&gt; dt.to_pandas(partitions=[(\"year\", \"=\", \"2021\")], columns=[\"value\"])\n      value\n0     6\n1     7\n2     5\n3     4\n&gt;&gt;&gt; dt.to_pyarrow_table(partitions=[(\"year\", \"=\", \"2021\")], columns=[\"value\"])\npyarrow.Table\nvalue: string\n</code></pre> <p>Converting to a PyArrow Dataset allows you to filter on columns other than partition columns and load the result as a stream of batches rather than a single table. Convert to a dataset using <code>DeltaTable.to_pyarrow_dataset</code>. Filters applied to datasets will use the partition values and file statistics from the Delta transaction log and push down any other filters to the scanning operation.</p> <pre><code>&gt;&gt;&gt; import pyarrow.dataset as ds\n&gt;&gt;&gt; dataset = dt.to_pyarrow_dataset()\n&gt;&gt;&gt; condition = (ds.field(\"year\") == \"2021\") &amp; (ds.field(\"value\") &gt; \"4\")\n&gt;&gt;&gt; dataset.to_table(filter=condition, columns=[\"value\"]).to_pandas()\n  value\n0     6\n1     7\n2     5\n&gt;&gt;&gt; batch_iter = dataset.to_batches(filter=condition, columns=[\"value\"], batch_size=2)\n&gt;&gt;&gt; for batch in batch_iter: print(batch.to_pandas())\n  value\n0     6\n1     7\n  value\n0     5\n</code></pre> <p>PyArrow datasets may also be passed to compatible query engines, such as DuckDB</p> <pre><code>&gt;&gt;&gt; import duckdb\n&gt;&gt;&gt; ex_data = duckdb.arrow(dataset)\n&gt;&gt;&gt; ex_data.filter(\"year = 2021 and value &gt; 4\").project(\"value\")\n---------------------\n-- Expression Tree --\n---------------------\nProjection [value]\n  Filter [year=2021 AND value&gt;4]\n    arrow_scan(140409099470144, 4828104688, 1000000)\n\n---------------------\n-- Result Columns  --\n---------------------\n- value (VARCHAR)\n\n---------------------\n-- Result Preview  --\n---------------------\nvalue\nVARCHAR\n[ Rows: 3]\n6\n7\n5\n</code></pre> <p>Finally, you can always pass the list of file paths to an engine. For example, you can pass them to <code>dask.dataframe.read_parquet</code>:</p> <pre><code>&gt;&gt;&gt; import dask.dataframe as dd\n&gt;&gt;&gt; df = dd.read_parquet(dt.file_uris())\n&gt;&gt;&gt; df\nDask DataFrame Structure:\n                value             year            month              day\nnpartitions=6\n               object  category[known]  category[known]  category[known]\n                  ...              ...              ...              ...\n...               ...              ...              ...              ...\n                  ...              ...              ...              ...\n                  ...              ...              ...              ...\nDask Name: read-parquet, 6 tasks\n&gt;&gt;&gt; df.compute()\n  value  year month day\n0     1  2020     1   1\n0     2  2020     2   3\n0     3  2020     2   5\n0     4  2021     4   5\n0     5  2021    12   4\n0     6  2021    12  20\n1     7  2021    12  20\n</code></pre>"},{"location":"usage/read-cdf/","title":"Reading the Change Data Feed from a Delta Table","text":"<p>Reading the CDF data from a table with change data is easy.</p>"},{"location":"usage/read-cdf/#reading-cdf-log","title":"Reading CDF Log","text":"Python Rust <pre><code>import polars\nfrom deltalake import DeltaTable\n\ndt = DeltaTable(\"../rust/tests/data/cdf-table\")\ntable = dt.load_cdf(starting_version=0, ending_version=4).read_all()\npt = polars.from_arrow(table)\npt.group_by(\"_commit_version\").len().sort(\"len\", descending=True)\n</code></pre> <pre><code>#[tokio::main]\nasync fn main() -&gt; Result&lt;(), Box&lt;dyn std::error::Error&gt;&gt; {\n\n    let table = deltalake::open_table(\"../rust/tests/data/cdf-table\").await?;\n    let ops = DeltaOps(table);\n    let cdf = ops.load_cdf()\n        .with_starting_version(0)\n        .with_ending_version(4)\n        .build()\n        .await?;\n\n    arrow_cast::pretty::print_batches(&amp;cdf)?;\n\n    Ok(())\n}\n</code></pre> <p>The output can then be used in various execution engines. The python example shows how one might consume the cdf feed inside polars.</p>"},{"location":"usage/optimize/delta-lake-z-order/","title":"Delta Lake Z Order","text":"<p>This section explains how to Z Order a Delta table.</p> <p>Z Ordering colocates similar data in the same files, which allows for better file skipping and faster queries.</p> <p>Suppose you have a table with <code>first_name</code>, <code>age</code>, and <code>country</code> columns.</p> <p>If you Z Order the data by the <code>country</code> column, then individuals from the same country will be stored in the same files.  When you subquently query the data for individuals from a given country, it will execute faster because more data can be skipped.</p> <p>Here's how to Z Order a Delta table:</p> <pre><code>dt = DeltaTable(\"tmp\")\ndt.optimize.z_order([country])\n</code></pre>"},{"location":"usage/optimize/small-file-compaction-with-optimize/","title":"Delta Lake small file compaction with optimize","text":"<p>This post shows you how to perform small file compaction with using the <code>optimize</code> method.  This was added to the <code>DeltaTable</code> class in version 0.9.0.  This command rearranges the small files into larger files which will reduce the number of files and speed up queries.</p> <p>This is very helpful for workloads that append frequently. For example, if you have a table that is appended to every 10 minutes, after a year you will have 52,560 files in the table. If the table is partitioned by another dimension, you will have 52,560 files per partition; with just 100 unique values that's millions of files. By running <code>optimize</code> periodically, you can reduce the number of files in the table to a more manageable number.</p> <p>Typically, you will run optimize less frequently than you append data. If possible, you might run optimize once you know you have finished writing to a particular partition. For example, on a table partitioned by date, you might append data every 10 minutes, but only run optimize once a day at the end of the day. This will ensure you don't need to compact the same data twice.</p> <p>This section will also teach you about how to use <code>vacuum</code> to physically remove files from storage that are no longer needed.  You\u2019ll often want vacuum after running optimize to remove the small files from storage once they\u2019ve been compacted into larger files.</p> <p>Let\u2019s start with an example to explain these key concepts.  All the code covered in this post is stored in this notebook in case you\u2019d like to follow along.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#create-a-delta-table-with-small-files","title":"Create a Delta table with small files","text":"<p>Let\u2019s start by creating a Delta table with a lot of small files so we can demonstrate the usefulness of the <code>optimize</code> command.</p> <p>Start by writing a function that generates on thousand rows of random data given a timestamp.</p> <pre><code>def record_observations(date: datetime) -&gt; pa.Table:\n    \"\"\"Pulls data for a certain datetime\"\"\"\n    nrows = 1000\n    return pa.table(\n        {\n            \"date\": pa.array([date.date()] * nrows),\n            \"timestamp\": pa.array([date] * nrows),\n            \"value\": pc.random(nrows),\n        }\n    )\n</code></pre> <p>Let\u2019s run this function and observe the output:</p> <pre><code>record_observations(datetime(2021, 1, 1, 12)).to_pandas()\n\n    date                timestamp   value\n0   2021-01-01  2021-01-01 12:00:00 0.3186397383362023\n1   2021-01-01  2021-01-01 12:00:00 0.04253766974259088\n2   2021-01-01  2021-01-01 12:00:00 0.9355682965171573\n\u2026\n999 2021-01-01  2021-01-01 12:00:00 0.23207037062879843\n</code></pre> <p>Let\u2019s write 100 hours worth of data to the Delta table.</p> <pre><code># Every hour starting at midnight on 2021-01-01\nhours_iter = (datetime(2021, 1, 1) + timedelta(hours=i) for i in itertools.count())\n\n# Write 100 hours worth of data\nfor timestamp in itertools.islice(hours_iter, 100):\n    write_deltalake(\n        \"observation_data\",\n        record_observations(timestamp),\n        partition_by=[\"date\"],\n        mode=\"append\",\n    )\n</code></pre> <p>This data was appended to the Delta table in 100 separate transactions, so the table will contain 100 transaction log entries and 100 data files.  You can see the number of files with the <code>files()</code> method.</p> <pre><code>dt = DeltaTable(\"observation_data\")\nlen(dt.files()) # 100\n</code></pre> <p>Here\u2019s how the files are persisted in storage.</p> <pre><code>observation_data\n\u251c\u2500\u2500 _delta_log\n\u2502   \u251c\u2500\u2500 00000000000000000000.json\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u2514\u2500\u2500 00000000000000000099.json\n\u251c\u2500\u2500 date=2021-01-01\n\u2502   \u251c\u2500\u2500 0-cfe227c6-edd9-4369-a1b0-db4559a2e693-0.parquet\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u251c\u2500\u2500 23-a4ace29e-e73e-40a1-81d3-0f5dc13093de-0.parquet\n\u251c\u2500\u2500 date=2021-01-02\n\u2502   \u251c\u2500\u2500 24-9698b456-66eb-4075-8732-fe56d81edb60-0.parquet\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u2514\u2500\u2500 47-d3fce527-e018-4c02-8acd-a649f6f523d2-0.parquet\n\u251c\u2500\u2500 date=2021-01-03\n\u2502   \u251c\u2500\u2500 48-fd90a7fa-5a14-42ed-9f59-9fe48d87899d-0.parquet\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u2514\u2500\u2500 71-5f143ade-8ae2-4854-bdc5-61154175665f-0.parquet\n\u251c\u2500\u2500 date=2021-01-04\n\u2502   \u251c\u2500\u2500 72-477c10fe-dc09-4087-80f0-56006e4a7911-0.parquet\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u2514\u2500\u2500 95-1c92cbce-8af4-4fe4-9c11-832245cf4d40-0.parquet\n\u2514\u2500\u2500 date=2021-01-05\n    \u251c\u2500\u2500 96-1b878ee5-25fd-431a-bc3e-6dcacc96b470-0.parquet\n    \u251c\u2500\u2500 \u2026\n    \u2514\u2500\u2500 99-9650ed63-c195-433d-a86b-9469088c14ba-0.parquet\n</code></pre> <p>Each of these Parquet files are tiny - they\u2019re only 10 KB.  Let\u2019s see how to compact these tiny files into larger files, which is more efficient for data queries.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#compact-small-files-in-the-delta-table-with-optimize","title":"Compact small files in the Delta table with optimize","text":"<p>Let\u2019s run the optimize command to compact the existing small files into larger files:</p> <pre><code>dt = DeltaTable(\"observation_data\")\n\ndt.optimize.compact()\n</code></pre> <p>Here\u2019s the output of the command:</p> <pre><code>{'numFilesAdded': 5,\n 'numFilesRemoved': 100,\n 'filesAdded': {'min': 39000,\n  'max': 238282,\n  'avg': 198425.6,\n  'totalFiles': 5,\n  'totalSize': 992128},\n 'filesRemoved': {'min': 10244,\n  'max': 10244,\n  'avg': 10244.0,\n  'totalFiles': 100,\n  'totalSize': 1024400},\n 'partitionsOptimized': 5,\n 'numBatches': 1,\n 'totalConsideredFiles': 100,\n 'totalFilesSkipped': 0,\n 'preserveInsertionOrder': True}\n</code></pre> <p>The optimize operation has added 5 new files and marked 100 exisitng files for removal (this is also known as \u201ctombstoning\u201d files).  It has compacted the 100 tiny files into 5 larger files.</p> <p>Let\u2019s append some more data to the Delta table and see how we can selectively run optimize on the new data that\u2019s added.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#handling-incremental-updates-with-optimize","title":"Handling incremental updates with optimize","text":"<p>Let\u2019s append another 24 hours of data to the Delta table:</p> <pre><code>for timestamp in itertools.islice(hours_iter, 24):\n    write_deltalake(\n        dt,\n        record_observations(timestamp),\n        partition_by=[\"date\"],\n        mode=\"append\",\n    )\n</code></pre> <p>We can use <code>get_add_actions()</code> to introspect the table state. We can see that <code>2021-01-06</code> has only a few hours of data so far, so we don't want to optimize that yet. But <code>2021-01-05</code> has all 24 hours of data, so it's ready to be optimized.</p> <pre><code>dt.get_add_actions(flatten=True).to_pandas()[\n    \"partition.date\"\n].value_counts().sort_index()\n\n2021-01-01     1\n2021-01-02     1\n2021-01-03     1\n2021-01-04     1\n2021-01-05    21\n2021-01-06     4\n</code></pre> <p>To optimize a single partition, you can pass in a <code>partition_filters</code> argument speficying which partitions to optimize.</p> <pre><code>dt.optimize(partition_filters=[(\"date\", \"=\", \"2021-01-05\")])\n\n{'numFilesAdded': 1,\n 'numFilesRemoved': 21,\n 'filesAdded': {'min': 238282,\n  'max': 238282,\n  'avg': 238282.0,\n  'totalFiles': 1,\n  'totalSize': 238282},\n 'filesRemoved': {'min': 10244,\n  'max': 39000,\n  'avg': 11613.333333333334,\n  'totalFiles': 21,\n  'totalSize': 243880},\n 'partitionsOptimized': 1,\n 'numBatches': 1,\n 'totalConsideredFiles': 21,\n 'totalFilesSkipped': 0,\n 'preserveInsertionOrder': True}\n</code></pre> <p>This optimize operation tombstones 21 small data files and adds one file with all the existing data properly condensed.  Let\u2019s take a look a portion of the <code>_delta_log/00000000000000000125.json</code> file, which is the transaction log entry that corresponds with this incremental optimize command.</p> <pre><code>{\n  \"remove\": {\n    \"path\": \"date=2021-01-05/part-00000-41178aab-2491-488f-943d-8f03867295ee-c000.snappy.parquet\",\n    \"deletionTimestamp\": 1683465499480,\n    \"dataChange\": false,\n    \"extendedFileMetadata\": null,\n    \"partitionValues\": {\n      \"date\": \"2021-01-05\"\n    },\n    \"size\": 39000,\n    \"tags\": null\n  }\n}\n\n{\n  \"remove\": {\n    \"path\": \"date=2021-01-05/101-79ae6fc9-c0cc-49ec-bb94-9aba879ac949-0.parquet\",\n    \"deletionTimestamp\": 1683465499481,\n    \"dataChange\": false,\n    \"extendedFileMetadata\": null,\n    \"partitionValues\": {\n      \"date\": \"2021-01-05\"\n    },\n    \"size\": 10244,\n    \"tags\": null\n  }\n}\n\n\u2026\n\n{\n  \"add\": {\n    \"path\": \"date=2021-01-05/part-00000-4b020a40-c836-4a11-851f-4691370c9f3a-c000.snappy.parquet\",\n    \"size\": 238282,\n    \"partitionValues\": {\n      \"date\": \"2021-01-05\"\n    },\n    \"modificationTime\": 1683465499493,\n    \"dataChange\": false,\n    \"stats\": \"{\\\"numRecords\\\":24000,\\\"minValues\\\":{\\\"value\\\":0.00005581532256615507,\\\"timestamp\\\":\\\"2021-01-05T00:00:00.000Z\\\"},\\\"maxValues\\\":{\\\"timestamp\\\":\\\"2021-01-05T23:00:00.000Z\\\",\\\"value\\\":0.9999911402868216},\\\"nullCount\\\":{\\\"timestamp\\\":0,\\\"value\\\":0}}\",\n    \"tags\": null\n  }\n}\n</code></pre> <p>The trasaction log indicates that many files have been tombstoned and one file is added, as expected.</p> <p>The Delta Lake optimize command \u201cremoves\u201d data by marking the data files as removed in the transaction log.  The optimize command doesn\u2019t physically delete the Parquet file from storage.  Optimize performs a \u201clogical remove\u201d not a \u201cphysical remove\u201d.</p> <p>Delta Lake uses logical operations so you can time travel back to earlier versions of your data.  You can vacuum your Delta table to physically remove Parquet files from storage if you don\u2019t need to time travel and don\u2019t want to pay to store the tombstoned files.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#vacuuming-after-optimizing","title":"Vacuuming after optimizing","text":"<p>The vacuum command deletes all files from storage that are marked for removal in the transaction log and older than the retention period which is 7 days by default.</p> <p>It\u2019s normally a good idea to have a retention period of at least 7 days.  For purposes of this example, we will set the retention period to zero, just so you can see how the files get removed from storage.  Adjusting the retention period in this manner isn\u2019t recommended for production use cases.</p> <p>Let\u2019s run the vacuum command:</p> <pre><code>dt.vacuum(retention_hours=0, enforce_retention_duration=False, dry_run=False)\n</code></pre> <p>The command returns a list of all the files that are removed from storage:</p> <pre><code>['date=2021-01-02/39-a98680f2-0e0e-4f26-a491-18b183f9eb05-0.parquet',\n 'date=2021-01-02/41-e96bc8bb-c571-484c-b534-e897424fb7da-0.parquet',\n \u2026\n 'date=2021-01-01/0-cfe227c6-edd9-4369-a1b0-db4559a2e693-0.parquet',\n 'date=2021-01-01/18-ded53418-172b-4e40-bf2e-7c8142e71bd1-0.parquet']\n</code></pre> <p>Let\u2019s look at the content of the Delta table now that all the really small files have been removed from storage:</p> <pre><code>observation_data\n\u251c\u2500\u2500 _delta_log\n\u2502   \u251c\u2500\u2500 00000000000000000000.json\n\u2502   \u251c\u2500\u2500 00000000000000000001.json\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u251c\u2500\u2500 00000000000000000124.json\n\u2502   \u2514\u2500\u2500 00000000000000000125.json\n\u251c\u2500\u2500 date=2021-01-01\n\u2502   \u2514\u2500\u2500 part-00000-31e3df5a-8bbe-425c-b85d-77794f922837-c000.snappy.parquet\n\u251c\u2500\u2500 date=2021-01-02\n\u2502   \u2514\u2500\u2500 part-00000-8af07878-b179-49ce-a900-d58595ffb60a-c000.snappy.parquet\n\u251c\u2500\u2500 date=2021-01-03\n\u2502   \u2514\u2500\u2500 part-00000-5e980864-b32f-4686-a58d-a75fae455c1e-c000.snappy.parquet\n\u251c\u2500\u2500 date=2021-01-04\n\u2502   \u2514\u2500\u2500 part-00000-1e82d23b-084d-47e3-9790-d68289c39837-c000.snappy.parquet\n\u251c\u2500\u2500 date=2021-01-05\n\u2502   \u2514\u2500\u2500 part-00000-4b020a40-c836-4a11-851f-4691370c9f3a-c000.snappy.parquet\n\u2514\u2500\u2500 date=2021-01-06\n    \u251c\u2500\u2500 121-0ecb5d70-4a28-4cd4-b2d2-89ee2285eaaa-0.parquet\n    \u251c\u2500\u2500 122-6b2d2758-9154-4392-b287-fe371ee507ec-0.parquet\n    \u251c\u2500\u2500 123-551d318f-4968-441f-83fc-89f98cd15daf-0.parquet\n    \u2514\u2500\u2500 124-287309d3-662e-449d-b4da-2e67b7cc0557-0.parquet\n</code></pre> <p>All the partitions only contain a single file now, except for the <code>date=2021-01-06</code> partition that has not been compacted yet.</p> <p>An entire partition won\u2019t necessarily get compacted to a single data file when optimize is run.  Each partition has data files that are condensed to the target file size.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#what-causes-the-small-file-problem","title":"What causes the small file problem?","text":"<p>Delta tables can accumulate small files for a variety of reasons:</p> <ul> <li>User error: users can accidentally write files that are too small.  Users should sometimes repartition in memory before writing to disk to avoid appending files that are too small.</li> <li>Frequent appends: systems that append more often tend to append more smaller files.  A pipeline that appends every minute will generally generate ten times as many small files compared to a system that appends every ten minutes.</li> <li>Appending to partitioned data lakes with high cardinality columns can also cause small files.  If you append every hour to a table that\u2019s partitioned on a column with 1,000 distinct values, then every append could create 1,000 new files.  Partitioning by date avoids this problem because the data isn\u2019t split up across partitions in this manner.  </li> </ul>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#conclusion","title":"Conclusion","text":"<p>This page showed you how to create a Delta table with many small files, compact the small files into larger files with optimize, and remove the tombstoned files from storage with vacuum.</p> <p>You also learned about how to incrementally optimize partitioned Delta tables, so you only compact newly added data.</p> <p>An excessive number of small files slows down Delta table queries, so periodic compaction is important.  Make sure to properly maintain your Delta tables, so performance does not degrade over time.</p>"},{"location":"usage/writing/","title":"Writing Delta Tables","text":"<p>For overwrites and appends, use <code>write_deltalake</code>. If the table does not already exist, it will be created. The <code>data</code> parameter will accept a Pandas DataFrame, a PyArrow Table, or an iterator of PyArrow Record Batches.</p> <pre><code>&gt;&gt;&gt; from deltalake import write_deltalake\n&gt;&gt;&gt; df = pd.DataFrame({'x': [1, 2, 3]})\n&gt;&gt;&gt; write_deltalake('path/to/table', df)\n</code></pre> <p>Note: <code>write_deltalake</code> accepts a Pandas DataFrame, but will convert it to a Arrow table before writing. See caveats in <code>pyarrow:python/pandas</code>.</p> <p>By default, writes create a new table and error if it already exists. This is controlled by the <code>mode</code> parameter, which mirrors the behavior of Spark's <code>pyspark.sql.DataFrameWriter.saveAsTable</code> DataFrame method. To overwrite pass in <code>mode='overwrite'</code> and to append pass in <code>mode='append'</code>:</p> <pre><code>&gt;&gt;&gt; write_deltalake('path/to/table', df, mode='overwrite')\n&gt;&gt;&gt; write_deltalake('path/to/table', df, mode='append')\n</code></pre> <p><code>write_deltalake</code> will raise <code>ValueError</code> if the schema of the data passed to it differs from the existing table's schema. If you wish to alter the schema as part of an overwrite pass in <code>schema_mode=\"overwrite\"</code> or <code>schema_mode=\"merge\"</code>. <code>schema_mode=\"overwrite\"</code> will completely overwrite the schema, even if columns are dropped; merge will append the new columns and fill missing columns with <code>null</code>. <code>schema_mode=\"merge\"</code> is also supported on append operations.</p>"},{"location":"usage/writing/#overwriting-a-partition","title":"Overwriting a partition","text":"<p>You can overwrite a specific partition by using <code>mode=\"overwrite\"</code> together with <code>partition_filters</code>. This will remove all files within the matching partition and insert your data as new files. This can only be done on one partition at a time. All of the input data must belong to that partition or else the method will raise an error.</p> <pre><code>&gt;&gt;&gt; from deltalake import write_deltalake\n&gt;&gt;&gt; df = pd.DataFrame({'x': [1, 2, 3], 'y': ['a', 'a', 'b']})\n&gt;&gt;&gt; write_deltalake('path/to/table', df, partition_by=['y'])\n\n&gt;&gt;&gt; table = DeltaTable('path/to/table')\n&gt;&gt;&gt; df2 = pd.DataFrame({'x': [100], 'y': ['b']})\n&gt;&gt;&gt; write_deltalake(table, df2, partition_filters=[('y', '=', 'b')], mode=\"overwrite\")\n\n&gt;&gt;&gt; table.to_pandas()\n     x  y\n0    1  a\n1    2  a\n2  100  b\n</code></pre> <p>This method could also be used to insert a new partition if one doesn't already exist, making this operation idempotent.</p>"},{"location":"usage/writing/#overwriting-part-of-the-table-data-using-a-predicate","title":"Overwriting part of the table data using a predicate","text":"<p>Note</p> <p>This predicate is often called a <code>replaceWhere</code> predicate</p> <p>When you don\u2019t specify the <code>predicate</code>, the overwrite save mode will replace the entire table.  Instead of replacing the entire table (which is costly!), you may want to overwrite only the specific parts of the table that should be changed.  In this case, you can use a <code>predicate</code> to overwrite only the relevant records or partitions.</p> <p>Note</p> <p>Data written must conform to the same predicate, i.e. not contain any records that don't match the <code>predicate</code> condition,  otherwise the operation will fail </p>  Python Rust <p> <code>replaceWhere</code> <pre><code>import pyarrow as pa\nfrom deltalake import write_deltalake\n\n# Assuming there is already a table in this location with some records where `id = '1'` which we want to overwrite\ntable_path = \"/tmp/my_table\"\ndata = pa.table(\n    {\n        \"id\": pa.array([\"1\", \"1\"], pa.string()),\n        \"value\": pa.array([11, 12], pa.int64()),\n    }\n)\nwrite_deltalake(\n    table_path,\n    data,\n    mode=\"overwrite\",\n    predicate=\"id = '1'\",\n    engine=\"rust\",\n)\n</code></pre></p> <p> <code>replaceWhere</code> <pre><code>// Assuming there is already a table in this location with some records where `id = '1'` which we want to overwrite\nuse arrow_array::RecordBatch;\nuse arrow_schema::{DataType, Field, Schema as ArrowSchema};\nuse deltalake::datafusion::logical_expr::{col, lit};\nuse deltalake::protocol::SaveMode;\nuse deltalake::DeltaOps;\n\nlet schema = ArrowSchema::new(vec![\n    Field::new(\"id\", DataType::Utf8, true),\n    Field::new(\"value\", DataType::Int32, true),\n]);\n\nlet data = RecordBatch::try_new(\n    schema.into(),\n    vec![\n        Arc::new(arrow::array::StringArray::from(vec![\"1\", \"1\"])),\n        Arc::new(arrow::array::Int32Array::from(vec![11, 12])),\n    ],\n)\n.unwrap();\n\nlet table = deltalake::open_table(\"/tmp/my_table\").await.unwrap();\nlet _table = DeltaOps(table)\n    .write(vec![data])\n    .with_save_mode(SaveMode::Overwrite)\n    .with_replace_where(col(\"id\").eq(lit(\"1\")))\n    .await\n    .unwrap();\n</code></pre></p>"},{"location":"usage/writing/#using-writer-properites","title":"Using Writer Properites","text":"<p>You can customize the Rust Parquet writer by using the WriterProperties. Additionally, you can apply extra configurations through the BloomFilterProperties and ColumnProperties data classes.</p> <p>Here's how you can do it: <pre><code>from deltalake import BloomFilterProperties, ColumnProperties, WriterProperties, write_deltalake\nimport pyarrow as pa\n\nwp = WriterProperties(\n        statistics_truncate_length=200,\n        default_column_properties=ColumnProperties(\n            bloom_filter_properties=BloomFilterProperties(True, 0.2, 30)\n        ),\n        column_properties={\n            \"value_non_bloom\": ColumnProperties(bloom_filter_properties=None),\n        },\n    )\n\ntable_path = \"/tmp/my_table\"\n\ndata = pa.table(\n        {\n            \"id\": pa.array([\"1\", \"1\"], pa.string()),\n            \"value\": pa.array([11, 12], pa.int64()),\n            \"value_non_bloom\": pa.array([11, 12], pa.int64()),\n        }\n    )\n\nwrite_deltalake(table_path, data, writer_properties=wp)\n</code></pre></p>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/","title":"Writing to S3 with a locking provider","text":"<p>Delta lake guarantees ACID transactions when writing data. This is done by default when writing to all supported object stores except AWS S3. (Some S3 clients like CloudFlare R2 or MinIO may enable concurrent writing without a locking provider, refer to this section for more information).</p> <p>When writing to S3, delta-rs provides a locking mechanism to ensure that concurrent writes are safe. This is done by default when writing to S3, but you can opt-out by setting the <code>AWS_S3_ALLOW_UNSAFE_RENAME</code> variable to <code>true</code>.</p> <p>To enable safe concurrent writes to AWS S3, we must provide an external locking mechanism.</p>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/#dynamodb","title":"DynamoDB","text":"<p>DynamoDB is the only available locking provider at the moment in delta-rs. To enable DynamoDB as the locking provider, you need to set the <code>AWS_S3_LOCKING_PROVIDER</code> to 'dynamodb' as a <code>storage_options</code> or as an environment variable.</p> <p>Additionally, you must create a DynamoDB table with the name <code>delta_log</code> so that it can be automatically recognized by delta-rs. Alternatively, you can use a table name of your choice, but you must set the <code>DELTA_DYNAMO_TABLE_NAME</code> variable to match your chosen table name. The required schema for the DynamoDB table is as follows:</p> <pre><code>\"Table\": {\n    \"AttributeDefinitions\": [\n        {\n            \"AttributeName\": \"fileName\",\n            \"AttributeType\": \"S\"\n        },\n        {\n            \"AttributeName\": \"tablePath\",\n            \"AttributeType\": \"S\"\n        }\n    ],\n    \"TableName\": \"delta_log\",\n    \"KeySchema\": [\n        {\n            \"AttributeName\": \"tablePath\",\n            \"KeyType\": \"HASH\"\n        },\n        {\n            \"AttributeName\": \"fileName\",\n            \"KeyType\": \"RANGE\"\n        }\n    ],\n}\n</code></pre> <p>Here is an example writing to s3 using this mechanism:</p> <pre><code>from deltalake import write_deltalake\ndf = pd.DataFrame({'x': [1, 2, 3]})\nstorage_options = {\n    'AWS_S3_LOCKING_PROVIDER': 'dynamodb',\n    'DELTA_DYNAMO_TABLE_NAME': 'custom_table_name'\n}\nwrite_deltalake(\n    's3a://path/to/table',\n    df,\n    storage_options=storage_options\n)\n</code></pre> <p>This locking mechanism is compatible with the one used by Apache Spark. The <code>tablePath</code> property, denoting the root url of the delta table itself, is part of the primary key, and all writers intending to write to the same table must match this property precisely. In Spark, S3 URLs are prefixed with <code>s3a://</code>, and a table in delta-rs must be configured accordingly.</p> <p>Note that <code>delta-rs</code> does not read credentials from your local <code>.aws/config</code> or <code>.aws/creds</code> file. Credentials can be accessed from environment variables, ec2 metadata, profiles or web identity. You can pass credentials to <code>storage_options</code> using <code>AWS_ACCESS_KEY_ID</code> and <code>AWS_SECRET_ACCESS_KEY</code>.</p> <p>The following code allows creating the necessary DynamoDB table from the AWS cli:</p> <pre><code>aws dynamodb create-table \\\n--table-name delta_log \\\n--attribute-definitions AttributeName=tablePath,AttributeType=S AttributeName=fileName,AttributeType=S \\\n--key-schema AttributeName=tablePath,KeyType=HASH AttributeName=fileName,KeyType=RANGE \\\n--provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5\n</code></pre> <p>You can find additional information in the Delta Lake documentation, which also includes recommendations on configuring a time-to-live (TTL) for the table to avoid growing the table indefinitely.</p>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/#enable-unsafe-writes-in-s3-opt-in","title":"Enable unsafe writes in S3 (opt-in)","text":"<p>If for some reason you don't want to use dynamodb as your locking mechanism you can choose to set the <code>AWS_S3_ALLOW_UNSAFE_RENAME</code> variable to <code>true</code> in order to enable S3 unsafe writes.</p>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/#required-permissions","title":"Required permissions","text":"<p>You need to have permissions to get, put and delete objects in the S3 bucket you're storing your data in. Please note that you must be allowed to delete objects even if you're just appending to the deltalake, because there are temporary files into the log folder that are deleted after usage.</p> <p>In AWS, those would be the required permissions:</p> <ul> <li>s3:GetObject</li> <li>s3:PutObject</li> <li>s3:DeleteObject</li> </ul> <p>In DynamoDB, you need those permissions:</p> <ul> <li>dynamodb:GetItem</li> <li>dynamodb:Query</li> <li>dynamodb:PutItem</li> <li>dynamodb:UpdateItem</li> <li>dynamodb:DeleteItem </li> </ul>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/#enabling-concurrent-writes-for-alternative-clients","title":"Enabling concurrent writes for alternative clients","text":"<p>Unlike AWS S3, some S3 clients support atomic renames by passing some headers in requests.</p> <p>For CloudFlare R2 or Minio passing this in the storage_options will enable concurrent writes:</p> <pre><code>storage_options = {\n    \"conditional_put\": \"etag\",\n}\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"The deltalake package","text":"<p>This is the documentation for the native Rust/Python implementation of Delta Lake. It is based on the delta-rs Rust library and requires no Spark or JVM dependencies. For the PySpark implementation, see delta-spark instead.</p> <p>This module provides the capability to read, write, and manage Delta Lake tables with Python or Rust without Spark or Java. It uses Apache Arrow under the hood, so is compatible with other Arrow-native or integrated libraries such as pandas, DuckDB, and Polars.</p>"},{"location":"#important-terminology","title":"Important terminology","text":"<ul> <li>\"Rust deltalake\" refers to the Rust API of delta-rs (no Spark dependency)</li> <li>\"Python deltalake\" refers to the Python API of delta-rs (no Spark dependency)</li> <li>\"Delta Spark\" refers to the Scala implementation of the Delta Lake transaction log protocol.  This depends on Spark and Java.</li> </ul>"},{"location":"#why-implement-the-delta-lake-transaction-log-protocol-in-rust-and-scala","title":"Why implement the Delta Lake transaction log protocol in Rust and Scala?","text":"<p>Delta Spark depends on Java and Spark, which is fine for many use cases, but not all Delta Lake users want to depend on these libraries.  delta-rs allows using Delta Lake in Rust or other native projects when using a JVM is often not an option.</p> <p>Python deltalake lets you query Delta tables without depending on Java/Scala.</p> <p>Suppose you want to query a Delta table with pandas on your local machine.  Python deltalake makes it easy to query the table with a simple <code>pip install</code> command - no need to install Java.</p>"},{"location":"#contributing","title":"Contributing","text":"<p>The Delta Lake community welcomes contributors from all developers, regardless of your experience or programming background.</p> <p>You can write Rust code, Python code, documentation, submit bugs, or give talks to the community.  We welcome all of these contributions.</p> <p>Feel free to join our Slack and message us in the #delta-rs channel any time!</p> <p>We value kind communication and building a productive, friendly environment for maximum collaboration and fun.</p>"},{"location":"#project-history","title":"Project history","text":"<p>Check out this video by Denny Lee &amp; QP Hou to learn about the genesis of the delta-rs project:</p>"},{"location":"delta-lake-best-practices/","title":"Delta Lake Best Practices","text":"<p>This page outlines Delta Lake best practices.</p> <p>You should consider several factors to optimize the performance of your Delta tables and minimize costs.</p> <p>The Delta Lake best practices depend on your data ingestion into the Delta table and query patterns.  You must understand your data and how users run queries to best leverage Delta Lake.</p>"},{"location":"delta-lake-best-practices/#compacting-small-files","title":"Compacting small files","text":"<p>Delta tables work best when the files are \u201cright-sized\u201d.  Files that are too small create I/O overhead.  Files that are too large limit the impact of file skipping (a critical query optimization).</p> <p>Delta tables can accumulate a lot of small files, especially if you\u2019re frequently writing small amounts of data.  If your table has many small files, you should run a small compaction operation to consolidate all the tiny files into \u201cright-sized\u201d files.</p> <p>It\u2019s generally best for files in a Delta table to be between 100MB and 1GB, but that can vary based on the overall size of the table and the query patterns.</p> <p>Delta Lake makes it easy to compact the small files.</p>"},{"location":"delta-lake-best-practices/#optimizing-table-layout","title":"Optimizing table layout","text":"<p>You can colocate similar data in the same files to make file skipping more effective.  Delta Lake supports Z Ordering, which can colocate similar data in the same files.</p> <p>Z Ordering can yield impressive performance gains for low-cardinality columns but also works well for high-cardinality columns.  This is an advantage compared to Hive-style partitioning, which is only suitable for low-cardinality columns.</p> <p>You must analyze the most common query patterns and Z Order your dataset based on the columns allowing the most file skipping.  The ability to colocate data in the same files decreases when you add more Z Order columns.</p> <p>Let\u2019s look at Hive-style partitioning, another way to colocate data in the same files.  You can also use Hive-style partitioning in conjunction with Z Ordering.</p>"},{"location":"delta-lake-best-practices/#partitioning-datasets","title":"Partitioning datasets","text":"<p>You can partition your Delta tables, which separates the data by one or more partition keys into separate folders.  Partitioning can be an excellent performance optimization (when you filter on the partition key) and is a good way to sidestep concurrency conflict issues.</p> <p>Hive-style partitioning also has some significant downsides.</p> <ul> <li>It\u2019s only suitable for low-cardinality columns.</li> <li>It can create many small files, especially if you use the wrong partition key or frequently update the Delta table.</li> <li>It can cause some queries that don\u2019t rely on the partition key to run slower (because of the excessive number of small files).  A large number of small files is problematic for I/O throughput.</li> </ul> <p>Hive-style partitioning can be a great data management tactic and a fantastic option for many Delta tables.  Beware of the downsides before partitioning your tables.</p> <p>You can use Hive-style partitioning in conjunction with Z Ordering.  You can partition a table by one column and Z Order by another.  They\u2019re different tactics that aim to help you skip more files and run queries faster.</p> <p>Let\u2019s look at some of the built-in Delta features that help maintain the integrity of your tables.</p>"},{"location":"delta-lake-best-practices/#use-appropriate-quality-controls","title":"Use appropriate quality controls","text":"<p>Delta Lake supports schema enforcement and column constraints to protect the integrity of your data.</p> <p>Delta Lake enabled schema enforcement by default, so you can only append data to an existing table with the same exact schema.  You can bypass schema enforcement by enabling schema evolution, which allows you to append mismatched schemas to a table.</p> <p>You should only enable schema evolution when you want to allow the schema of your table to change.  You should not enable schema evolution if you don\u2019t want this flexibility.  Schema enforcement is a good default setting.</p> <p>Column-level constraints prevent you from appending data that fail SQL predicates.  For example, you may add a constraint that requires all the values in the <code>age</code> column of a table to be positive.</p> <p>You should add column constraints to your table whenever you want a column only to include values that satisfy a predicate.</p> <p>No data is appended when you apply a constraint and a row check fails.  For example, if you try to append 100 rows of data to a table and one row has a failing check, then no data is added.</p> <p>When you have column constraints, it\u2019s often a good idea to append the failing data to a \u201cquarantine table\u201d and the passing data to the main Delta table.  Or you can filter out the failing rows and just append the passing rows.  Keeping a history of the failing rows in a quarantine table is helpful for debugging.</p> <p>See here to learn more about Delta Lake constraints.</p>"},{"location":"delta-lake-best-practices/#best-practices-for-dml-operations","title":"Best practices for DML operations","text":"<p>DML operations like deleting, updating, and merging write existing data in new files and mark existing files for deletion in the transaction log.  Rewriting data files is expensive, so you want to minimize the number of rewritten files when you run DML operations.</p> <p>Delta Lake supports a table feature called deletion vectors that implements DML transactions more efficiently under the hood.  Enabling deletion vectors is usually the best way to make DML operations run faster.  Note: delta-rs doesn\u2019t support deletion vectors yet.</p> <p>You should periodically purge deletion vectors because they can accumulate and slow subsequent read operations.  Once you enable the feature, you must purge the deletion vectors in your table with an appropriate cadence.</p>"},{"location":"delta-lake-best-practices/#use-vacuum-to-save-storage-costs","title":"Use vacuum to save storage costs","text":"<p>Delta Lake supports transactions, which necessitates keeping old versions of data in storage, even the files marked for removal in the transactions log.</p> <p>Keeping old versions of Delta tables in storage is often desirable because it allows for versioned data, time travel, and rolling back tables to a previous state.</p> <p>If you don\u2019t want to leverage older versions of a table, then you should remove the legacy files from storage with the vacuum command.  Vacuum will remove all files older than the table retention period and marked for removal in the transaction log.</p> <p>You only need to vacuum when you perform operations that mark files for removal in the transaction log.  An append-only table doesn\u2019t create legacy files that need to be vacuumed.</p> <p>Create a good vacuum strategy for your tables to minimize your storage costs.</p>"},{"location":"delta-lake-best-practices/#delta-lake-best-practices-to-minimize-costs","title":"Delta Lake best practices to minimize costs","text":"<p>Delta Lake helps you minimize costs in many ways:</p> <ul> <li>It's a free, open source format (based on Parquet). It's not a proprietary format that you need to pay for.</li> <li>Delta tables store column-level min/max values in the transaction log, allowing file skipping.</li> <li>Delta tables can be optimized (small file compaction, Z Ordering, etc.), so your queries run faster. When your queries run faster, then you pay less on compute.</li> <li>Deletion vectors let you perform DML operations (delete, update, merge) much faster. If your delete operation runs 100x faster, then you pay 100x less compute.</li> <li>It's easy to remove legacy files from storage with VACUUM, which minimizes storage costs.</li> </ul> <p>You should understand your organization\u2019s query patterns and use these features to minimize the overall cost.  You need to assess tradeoffs.  For example, Z Ordering is a computation that costs money, but it can save you lots of money in the long run if all your subsequent queries run a lot faster and use less compute.</p>"},{"location":"delta-lake-best-practices/#collect-metadata-stats-on-columns-used-for-file-skipping","title":"Collect metadata stats on columns used for file skipping","text":"<p>Delta tables don\u2019t always store each column's min/max values.  Some Delta Lake implementations only store min/max values for the first 32 columns in the table, for example.</p> <p>Delta Lake can only apply file-skipping when it has min/max values for the relevant columns stored in the transaction log.  Suppose you\u2019re running a filtering operation on <code>col_a,</code> for example.  Delta Lake can only apply file skipping when the transaction log stores <code>col_a</code> min/max metadata.</p> <p>Ensure the transaction log stores metadata stats for all the columns that benefit from file skipping.</p>"},{"location":"delta-lake-best-practices/#dont-collect-column-metadata-when-its-unnecessary","title":"Don\u2019t collect column metadata when it\u2019s unnecessary","text":"<p>It takes some time to compute column statistics when writing files, and it isn\u2019t worth the effort if you cannot use the column for file skipping.</p> <p>Suppose you have a table column containing a long string of arbitrary text.  It\u2019s unlikely that this column would ever provide any data-skipping benefits.  So, you can just avoid the overhead of collecting the statistics for this particular column.</p>"},{"location":"delta-lake-best-practices/#additional-reading","title":"Additional reading","text":"<p>Delta Lake relies on transactions, and you should check out this page to learn more.</p> <p>Many Delta Lake performance benefits rely on file skipping, which you should understand well to get the most out of Delta.</p>"},{"location":"delta-lake-best-practices/#conclusion","title":"Conclusion","text":"<p>Delta Lake is a powerful technology that makes your data pipelines more reliable, saves money, and makes everyday data processing tasks easy.</p> <p>You need to learn how Delta Lake works at a high level to leverage Delta's power fully.  You will not be able to leverage Delta Lake\u2019s full performance potential if your table has improperly sized files or if you\u2019re not colocating data in the same files to maximize data skipping, for example.</p> <p>Luckily, there are only a few details that are important to learn.  You don\u2019t need to know the implementation details - just the essential high-level concepts.</p>"},{"location":"delta-lake-big-data-small-data/","title":"Delta Lake for big data and small data","text":"<p>Delta Lake is an excellent storage format for big data and small data.</p> <p>This post explains why Delta Lake is suitable for massive datasets and why many of these features that are great, even for tiny tables.  Delta Lake is fine for a table with less than 1 GB of data or hundreds of petabytes of data.</p> <p>Let\u2019s start by discussing the features that are great for small data.</p>"},{"location":"delta-lake-big-data-small-data/#delta-lake-for-small-data-tables","title":"Delta Lake for small data tables","text":"<p>Delta Lake has many features that are useful for small datasets:</p> <ul> <li>Reliable transactions</li> <li>Better performance via file skipping</li> <li>DML operations to make deletes, updates, and merges easy and performant</li> <li>Features like schema enforcement and constraints to enforce data quality</li> <li>Versioned data &amp; time travel</li> </ul> <p>All of these features are great for large and small tables.</p> <p>Delta Lake DML operations are ACID transactions, so they either finish entirely or don\u2019t finish at all.  Delta tables don\u2019t require any downtime while DML operations are running.  The Delta Lake user experience is better than a data lake that doesn\u2019t support transactions and has downtime while running DML operations.</p> <p>The Delta Lake API also makes it easy to run DML operations.  You can delete a line of code from a Delta table with a single line of code.  Writing code to delete rows from CSV files is more challenging, especially if you want to implement this operation efficiently.</p> <p>Delta Lake has built-in checks to retain the integrity of your tables.  For example, Delta tables have schema enforcement and prevent you from appending DataFrames with mismatched schema from the existing table.  Delta Lake also lets you add constraints that only allow appending specific values to a column.  Data quality is also essential for small tables!</p> <p>Delta Lake splits data into multiple files with file-level metadata in the transaction log, so query engines can sometimes skip data.  Data skipping can be a huge performance benefit, depending on how much data can be ignored by the query engine.  </p> <p>As previously mentioned, Delta tables record all DML operations as transactions.  Recording operations as transactions means that existing data isn\u2019t mutated.  So Delta Lake provides versioned data and time travel out of the box.  Versioning data is better because it allows you to roll back mistakes and compare the state of the table at different points in time.</p> <p>Delta Lake has many useful features for small data tables.  Let\u2019s look at how Delta Lake is scalable for massive datasets.</p>"},{"location":"delta-lake-big-data-small-data/#delta-lake-for-large-data-tables","title":"Delta Lake for large data tables","text":"<p>Delta Lake is designed to be scalable and can handle tables with terabytes or petabytes of data.</p> <p>See here for an example of an organization ingesting 220 TB of data into a Delta table daily.</p> <p>Delta tables store data in Parquet files, and cloud object stores allow engines to write any number of files.  Delta tables store metadata information in the transaction log as JSON files, which are periodically compacted into Parquet files, so an arbitrarily large amount of Delta table metadata can also be stored.</p> <p>Delta Lake transactions and concurrency protection maintain the integrity of tables, even for large write operations or long-running computations.</p> <p>It\u2019s well known that Delta tables are scalable, even for the most enormous tables.</p>"},{"location":"delta-lake-big-data-small-data/#small-data-operations-on-large-tables","title":"Small data operations on large tables","text":"<p>Delta Lake is flexible and allows you to use \u201csmall data engines,\u201d even for large tables, depending on the computation.</p> <p>Suppose you have a Delta table containing 10 TB of data and a pipeline that appends 0.5 GB of data to the table every hour.  You don\u2019t need a big data query engine to append a small amount of data.  You can set up this job to run the Delta table append with a small data engine like pandas or Polars.</p> <p>Delta tables are flexible and interoperable with many technologies so that you can use the right tool for each data processing job.  This allows you to design pipelines how you\u2019d like and minimize costs.</p>"},{"location":"delta-lake-big-data-small-data/#when-delta-lake-isnt-needed","title":"When Delta Lake isn\u2019t needed","text":"<p>You don\u2019t need Delta Lake for a small dataset that never changes and can be stored in a single Parquet file.</p> <p>Suppose you have a 0.5 GB dataset in a Parquet file that never needs to be updated.  You can just keep that data in a Parquet table.  Reading the metadata from the Parquet footer of a single file isn\u2019t expensive.  You won\u2019t be taking advantage of Delta Lake's features like transactions, convenient DML operations, or versioned data.</p> <p>But in most cases, it\u2019s best to use Delta Lake because its features protect the integrity of your tables and make your life easier.</p>"},{"location":"delta-lake-big-data-small-data/#conclusion","title":"Conclusion","text":"<p>Delta Lake is well known for being scalable to huge tables but is also an excellent technology for small tables.</p> <p>Delta Lake is a lightweight technology, so there is little overhead.  Writing the metadata file after performing a transaction is fast.  It\u2019s a minuscule cost, considering the benefits you receive.</p> <p>Many reasons that make Delta Lake better than data lakes for large tables also apply to small tables!</p>"},{"location":"why-use-delta-lake/","title":"Why use Delta Lake","text":"<p>This page explains why Delta Lake is a better storage format for most tabular data analyses than data lake alternatives.</p> <p>Delta Lake provides developer-friendly features, reliable transactions, and fast performance compared with alternatives like Parquet or CSV.</p>"},{"location":"why-use-delta-lake/#fast-performance","title":"Fast performance","text":"<p>Delta tables store data in Parquet files and persist file-level metadata in the transaction log.</p> <p>This offers two main performance advantages:</p> <ul> <li>File skipping based on metadata that\u2019s quickly accessible</li> <li>Easy identification of all file paths for the table, compared to file listing operations that can be slow, especially on cloud object stores</li> </ul> <p>Delta Lake stores min/max values for each column of each file in the table.  Certain queries can skip entire files based on the metadata.  File skipping can be a massive performance optimization.</p> <p>Delta Lake also makes it easy to rearrange data in the table, so more file skipping is possible.  For example, the table can be partitioned or Z Ordered, so that similar data is colocated in the same files and data skipping is optimal for your query patterns.</p> <p>For data lakes, you need to run file listing operations to get the file paths before you can actually read the data.  Listing all the files in a data lake can take a long time, especially if there are a lot of files and they are stored in Hive-style partitions.</p> <p>Delta Lake stores all the file paths in the transaction log.  So you can quickly get the file paths directly from the log and then run your query.  Delta Lake also stores the file-level metadata in the transaction log which is quicker than opening all the files in the data lake and grabbing the metadata from the file footer.</p>"},{"location":"why-use-delta-lake/#developer-friendly-features","title":"Developer friendly features","text":"<p>Many basic data operations are hard in data lakes but quite easy with Delta Lake.  The only data operation that\u2019s easy with a data lake is appending data.  Delta Lake makes all data operations easy including the following:</p> <ul> <li>Appends</li> <li>Upserts</li> <li>Deletes</li> <li>Replace where</li> </ul> <p>Even deleting a few rows of data from a data lake is hard.  It\u2019s even harder if you want to run the operation in a performant manner.</p> <p>Delta Lake makes it easy to run common data operations and executes them performantly under the hood.</p> <p>Delta Lake also executes write operations as transactions, which makes data operations safer and prevents downtime.  Write operations will cause data lakes to be in an unstable state while the computations is running.  For example, if you read a data lake while a delete operation is running, then you may get the wrong data.</p> <p>Let\u2019s explore the benefits of reliable transactions in more detail.</p>"},{"location":"why-use-delta-lake/#reliable-transactions","title":"Reliable transactions","text":"<p>Delta Lake supports transactions which means that write operations have the following characteristics:</p> <ul> <li>They either finish completely or don\u2019t run at all</li> <li>They are executed in a serial manner and don\u2019t conflict with other transactions</li> <li>They don\u2019t corrupt a table or violate table constraints</li> </ul> <p>Data lakes don\u2019t support transactions, so the write operations can cause the following errors:</p> <ul> <li>There is no schema enforcement, so you can append data to a data lake with a mismatching schema</li> <li>Reading the data lake often yields incorrect results while write transactions are performed</li> <li>Data lakes can be corrupted for invalid write operations or computations that error-out</li> <li>Concurrent transactions that conflict can cause data loss</li> </ul> <p>Production data systems should rely on storage systems like Delta Lake that support transactions.</p>"},{"location":"why-use-delta-lake/#interoperability","title":"Interoperability","text":"<p>Delta Lake tables are interoperable and can be read/written by multiple different query engines.</p> <p>For example, you can create a Delta table with Spark, append to it with pandas, and then read it with Polars.</p> <p></p> <p>Delta tables are powerful because they are interoperable with various query engines and computation runtimes.</p> <p>Suppose you have a Delta table that\u2019s updated with an AWS Lambda function every 5 minutes.  There is only a small amount of data collected every 5 minutes, so a lightweight runtime like AWS Lambda is sufficient.</p> <p>Further suppose that the overall table is quite large.  So when you want to perform DML operations or query the whole table, your team uses a Spark cluster.</p> <p>Delta Lake is flexible to allow these types of operations from multiple readers and writers.  This provides teams with the flexibility to choose the right tool for the job.</p>"},{"location":"why-use-delta-lake/#support-for-many-languages","title":"Support for many languages","text":"<p>Delta tables can be queried with a variety of different languages.  This project provides APIs for Rust and Python users and does not depend on Java or Scala.  This project is a great alternative for pandas, Polars, DuckDB, or DataFusion.</p> <p>Delta Lake supports many languages and even more language support is coming soon!</p>"},{"location":"why-use-delta-lake/#support-on-multiple-clouds","title":"Support on multiple clouds","text":"<p>Delta Lake supports multiple clouds including GCP, AWS, and Azure.</p> <p>You can also use Delta Lake on your local machine or in an on-prem environment.</p> <p>Delta Lake is quite portable.</p>"},{"location":"why-use-delta-lake/#conclusion","title":"Conclusion","text":"<p>Delta Lake is a mature table format that offers users tons of advantages over a data lake with virtually no downsides.</p> <p>Once you start using Delta Lake, you will never want to go back to data lakes that expose you to a variety of dangerous bugs, poor performance, and reliability issues.</p> <p>The Delta Lake community is also welcome and open.  We gladly accept new contributors and help users with their questions.</p>"},{"location":"api/catalog/","title":"Catalog","text":"","boost":2},{"location":"api/catalog/#deltalake.data_catalog.DataCatalog","title":"deltalake.data_catalog.DataCatalog","text":"<p>             Bases: <code>Enum</code></p> <p>List of the Data Catalogs</p>","boost":2},{"location":"api/catalog/#deltalake.data_catalog.DataCatalog.AWS","title":"AWS  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>AWS = 'glue'\n</code></pre> <p>Refers to the <code>AWS Glue Data Catalog &lt;https://docs.aws.amazon.com/glue/latest/dg/catalog-and-crawler.html&gt;</code>_</p>","boost":2},{"location":"api/catalog/#deltalake.data_catalog.DataCatalog.UNITY","title":"UNITY  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>UNITY = 'unity'\n</code></pre> <p>Refers to the <code>Databricks Unity Catalog &lt;https://docs.databricks.com/data-governance/unity-catalog/index.html&gt;</code>_</p>","boost":2},{"location":"api/delta_writer/","title":"Writer","text":"","boost":10},{"location":"api/delta_writer/#write-to-delta-tables","title":"Write to Delta Tables","text":"","boost":10},{"location":"api/delta_writer/#deltalake.write_deltalake","title":"deltalake.write_deltalake","text":"<pre><code>write_deltalake(table_or_uri: Union[str, Path, DeltaTable], data: Union[pd.DataFrame, ds.Dataset, pa.Table, pa.RecordBatch, Iterable[pa.RecordBatch], RecordBatchReader, ArrowStreamExportable], *, schema: Optional[Union[pa.Schema, DeltaSchema]] = None, partition_by: Optional[Union[List[str], str]] = None, mode: Literal['error', 'append', 'overwrite', 'ignore'] = 'error', file_options: Optional[ds.ParquetFileWriteOptions] = None, max_partitions: Optional[int] = None, max_open_files: int = 1024, max_rows_per_file: int = 10 * 1024 * 1024, min_rows_per_group: int = 64 * 1024, max_rows_per_group: int = 128 * 1024, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, schema_mode: Optional[Literal['merge', 'overwrite']] = None, storage_options: Optional[Dict[str, str]] = None, partition_filters: Optional[List[Tuple[str, str, Any]]] = None, predicate: Optional[str] = None, target_file_size: Optional[int] = None, large_dtypes: bool = False, engine: Literal['pyarrow', 'rust'] = 'rust', writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; None\n</code></pre> <p>Write to a Delta Lake table</p> <p>If the table does not already exist, it will be created.</p> <p>The pyarrow writer supports protocol version 2 currently and won't be updated. For higher protocol support use engine='rust', this will become the default eventually.</p> <p>To enable safe concurrent writes when writing to S3, an additional locking mechanism must be supplied. For more information on enabling concurrent writing to S3, follow this guide</p> <p>Parameters:</p> Name Type Description Default <code>table_or_uri</code> <code>Union[str, Path, DeltaTable]</code> <p>URI of a table or a DeltaTable object.</p> required <code>data</code> <code>Union[DataFrame, Dataset, Table, RecordBatch, Iterable[RecordBatch], RecordBatchReader, ArrowStreamExportable]</code> <p>Data to write. If passing iterable, the schema must also be given.</p> required <code>schema</code> <code>Optional[Union[Schema, Schema]]</code> <p>Optional schema to write.</p> <code>None</code> <code>partition_by</code> <code>Optional[Union[List[str], str]]</code> <p>List of columns to partition the table by. Only required when creating a new table.</p> <code>None</code> <code>mode</code> <code>Literal['error', 'append', 'overwrite', 'ignore']</code> <p>How to handle existing data. Default is to error if table already exists. If 'append', will add new data. If 'overwrite', will replace table with new data. If 'ignore', will not write anything if table already exists.</p> <code>'error'</code> <code>file_options</code> <code>Optional[ParquetFileWriteOptions]</code> <p>Optional write options for Parquet (ParquetFileWriteOptions). Can be provided with defaults using ParquetFileWriteOptions().make_write_options(). Please refer to https://github.com/apache/arrow/blob/master/python/pyarrow/_dataset_parquet.pyx#L492-L533 for the list of available options. Only used in pyarrow engine.</p> <code>None</code> <code>max_partitions</code> <code>Optional[int]</code> <p>the maximum number of partitions that will be used. Only used in pyarrow engine.</p> <code>None</code> <code>max_open_files</code> <code>int</code> <p>Limits the maximum number of files that can be left open while writing. If an attempt is made to open too many files then the least recently used file will be closed. If this setting is set too low you may end up fragmenting your data into many small files. Only used in pyarrow engine.</p> <code>1024</code> <code>max_rows_per_file</code> <code>int</code> <p>Maximum number of rows per file. If greater than 0 then this will limit how many rows are placed in any single file. Otherwise there will be no limit and one file will be created in each output directory unless files need to be closed to respect max_open_files min_rows_per_group: Minimum number of rows per group. When the value is set, the dataset writer will batch incoming data and only write the row groups to the disk when sufficient rows have accumulated. Only used in pyarrow engine.</p> <code>10 * 1024 * 1024</code> <code>max_rows_per_group</code> <code>int</code> <p>Maximum number of rows per group. If the value is set, then the dataset writer may split up large incoming batches into multiple row groups. If this value is set, then min_rows_per_group should also be set.</p> <code>128 * 1024</code> <code>name</code> <code>Optional[str]</code> <p>User-provided identifier for this table.</p> <code>None</code> <code>description</code> <code>Optional[str]</code> <p>User-provided description for this table.</p> <code>None</code> <code>configuration</code> <code>Optional[Mapping[str, Optional[str]]]</code> <p>A map containing configuration options for the metadata action.</p> <code>None</code> <code>schema_mode</code> <code>Optional[Literal['merge', 'overwrite']]</code> <p>If set to \"overwrite\", allows replacing the schema of the table. Set to \"merge\" to merge with existing schema.</p> <code>None</code> <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>options passed to the native delta filesystem.</p> <code>None</code> <code>predicate</code> <code>Optional[str]</code> <p>When using <code>Overwrite</code> mode, replace data that matches a predicate. Only used in rust engine.'</p> <code>None</code> <code>target_file_size</code> <code>Optional[int]</code> <p>Override for target file size for data files written to the delta table. If not passed, it's taken from <code>delta.targetFileSize</code>.</p> <code>None</code> <code>partition_filters</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>the partition filters that will be used for partition overwrite. Only used in pyarrow engine.</p> <code>None</code> <code>large_dtypes</code> <code>bool</code> <p>Only used for pyarrow engine</p> <code>False</code> <code>engine</code> <code>Literal['pyarrow', 'rust']</code> <p>writer engine to write the delta table. PyArrow engine is deprecated, and will be removed in v1.0.</p> <code>'rust'</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code>","boost":10},{"location":"api/delta_writer/#deltalake.BloomFilterProperties","title":"deltalake.BloomFilterProperties  <code>dataclass</code>","text":"<pre><code>BloomFilterProperties(set_bloom_filter_enabled: Optional[bool], fpp: Optional[float] = None, ndv: Optional[int] = None)\n</code></pre> <p>The Bloom Filter Properties instance for the Rust parquet writer.</p> <p>Create a Bloom Filter Properties instance for the Rust parquet writer:</p> <p>Parameters:</p> Name Type Description Default <code>set_bloom_filter_enabled</code> <code>Optional[bool]</code> <p>If True and no fpp or ndv are provided, the default values will be used.</p> required <code>fpp</code> <code>Optional[float]</code> <p>The false positive probability for the bloom filter. Must be between 0 and 1 exclusive.</p> <code>None</code> <code>ndv</code> <code>Optional[int]</code> <p>The number of distinct values for the bloom filter.</p> <code>None</code>","boost":10},{"location":"api/delta_writer/#deltalake.ColumnProperties","title":"deltalake.ColumnProperties  <code>dataclass</code>","text":"<pre><code>ColumnProperties(dictionary_enabled: Optional[bool] = None, max_statistics_size: Optional[int] = None, bloom_filter_properties: Optional[BloomFilterProperties] = None)\n</code></pre> <p>The Column Properties instance for the Rust parquet writer.</p> <p>Create a Column Properties instance for the Rust parquet writer:</p> <p>Parameters:</p> Name Type Description Default <code>dictionary_enabled</code> <code>Optional[bool]</code> <p>Enable dictionary encoding for the column.</p> <code>None</code> <code>max_statistics_size</code> <code>Optional[int]</code> <p>Maximum size of statistics for the column.</p> <code>None</code> <code>bloom_filter_properties</code> <code>Optional[BloomFilterProperties]</code> <p>Bloom Filter Properties for the column.</p> <code>None</code>","boost":10},{"location":"api/delta_writer/#deltalake.WriterProperties","title":"deltalake.WriterProperties  <code>dataclass</code>","text":"<pre><code>WriterProperties(data_page_size_limit: Optional[int] = None, dictionary_page_size_limit: Optional[int] = None, data_page_row_count_limit: Optional[int] = None, write_batch_size: Optional[int] = None, max_row_group_size: Optional[int] = None, compression: Optional[Literal['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'ZSTD', 'LZ4_RAW']] = None, compression_level: Optional[int] = None, statistics_truncate_length: Optional[int] = None, default_column_properties: Optional[ColumnProperties] = None, column_properties: Optional[Dict[str, ColumnProperties]] = None)\n</code></pre> <p>A Writer Properties instance for the Rust parquet writer.</p> <p>Create a Writer Properties instance for the Rust parquet writer:</p> <p>Parameters:</p> Name Type Description Default <code>data_page_size_limit</code> <code>Optional[int]</code> <p>Limit DataPage size to this in bytes.</p> <code>None</code> <code>dictionary_page_size_limit</code> <code>Optional[int]</code> <p>Limit the size of each DataPage to store dicts to this amount in bytes.</p> <code>None</code> <code>data_page_row_count_limit</code> <code>Optional[int]</code> <p>Limit the number of rows in each DataPage.</p> <code>None</code> <code>write_batch_size</code> <code>Optional[int]</code> <p>Splits internally to smaller batch size.</p> <code>None</code> <code>max_row_group_size</code> <code>Optional[int]</code> <p>Max number of rows in row group.</p> <code>None</code> <code>compression</code> <code>Optional[Literal['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'ZSTD', 'LZ4_RAW']]</code> <p>compression type.</p> <code>None</code> <code>compression_level</code> <code>Optional[int]</code> <p>If none and compression has a level, the default level will be used, only relevant for GZIP: levels (1-9), BROTLI: levels (1-11), ZSTD: levels (1-22),</p> <code>None</code> <code>statistics_truncate_length</code> <code>Optional[int]</code> <p>maximum length of truncated min/max values in statistics.</p> <code>None</code> <code>default_column_properties</code> <code>Optional[ColumnProperties]</code> <p>Default Column Properties for the Rust parquet writer.</p> <code>None</code> <code>column_properties</code> <code>Optional[Dict[str, ColumnProperties]]</code> <p>Column Properties for the Rust parquet writer.</p> <code>None</code>","boost":10},{"location":"api/delta_writer/#convert-to-delta-tables","title":"Convert to Delta Tables","text":"","boost":10},{"location":"api/delta_writer/#deltalake.convert_to_deltalake","title":"deltalake.convert_to_deltalake","text":"<pre><code>convert_to_deltalake(uri: Union[str, Path], mode: Literal['error', 'ignore'] = 'error', partition_by: Optional[pa.Schema] = None, partition_strategy: Optional[Literal['hive']] = None, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, storage_options: Optional[Dict[str, str]] = None, custom_metadata: Optional[Dict[str, str]] = None) -&gt; None\n</code></pre> <p><code>Convert</code> parquet tables <code>to delta</code> tables.</p> <p>Currently only HIVE partitioned tables are supported. <code>Convert to delta</code> creates a transaction log commit with add actions, and additional properties provided such as configuration, name, and description.</p> <p>Parameters:</p> Name Type Description Default <code>uri</code> <code>Union[str, Path]</code> <p>URI of a table.</p> required <code>partition_by</code> <code>Optional[Schema]</code> <p>Optional partitioning schema if table is partitioned.</p> <code>None</code> <code>partition_strategy</code> <code>Optional[Literal['hive']]</code> <p>Optional partition strategy to read and convert</p> <code>None</code> <code>mode</code> <code>Literal['error', 'ignore']</code> <p>How to handle existing data. Default is to error if table already exists. If 'ignore', will not convert anything if table already exists.</p> <code>'error'</code> <code>name</code> <code>Optional[str]</code> <p>User-provided identifier for this table.</p> <code>None</code> <code>description</code> <code>Optional[str]</code> <p>User-provided description for this table.</p> <code>None</code> <code>configuration</code> <code>Optional[Mapping[str, Optional[str]]]</code> <p>A map containing configuration options for the metadata action.</p> <code>None</code> <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>options passed to the native delta filesystem. Unused if 'filesystem' is defined.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>custom metadata that will be added to the transaction commit</p> <code>None</code>","boost":10},{"location":"api/exceptions/","title":"Exceptions","text":"","boost":2},{"location":"api/exceptions/#deltalake.exceptions.DeltaError","title":"deltalake.exceptions.DeltaError","text":"<p>             Bases: <code>builtins.Exception</code></p> <p>The base class for Delta-specific errors.</p>","boost":2},{"location":"api/exceptions/#deltalake.exceptions.DeltaProtocolError","title":"deltalake.exceptions.DeltaProtocolError","text":"<p>             Bases: <code>_internal.DeltaError</code></p> <p>Raised when a violation with the Delta protocol specs ocurred.</p>","boost":2},{"location":"api/exceptions/#deltalake.exceptions.TableNotFoundError","title":"deltalake.exceptions.TableNotFoundError","text":"<p>             Bases: <code>_internal.DeltaError</code></p> <p>Raised when a Delta table cannot be loaded from a location.</p>","boost":2},{"location":"api/exceptions/#deltalake.exceptions.CommitFailedError","title":"deltalake.exceptions.CommitFailedError","text":"<p>             Bases: <code>_internal.DeltaError</code></p> <p>Raised when a commit to a Delta table fails.</p>","boost":2},{"location":"api/schema/","title":"Schema","text":"","boost":2},{"location":"api/schema/#schema-and-field","title":"Schema and field","text":"<p>Schemas, fields, and data types are provided in the <code>deltalake.schema</code> submodule.</p>","boost":2},{"location":"api/schema/#deltalake.Schema","title":"deltalake.Schema","text":"<pre><code>Schema(fields: List[Field])\n</code></pre> <p>             Bases: <code>deltalake._internal.StructType</code></p> <p>A Delta Lake schema</p> <p>Create using a list of :class:<code>Field</code>:</p> <p>Schema([Field(\"x\", \"integer\"), Field(\"y\", \"string\")]) Schema([Field(x, PrimitiveType(\"integer\"), nullable=True), Field(y, PrimitiveType(\"string\"), nullable=True)])</p> <p>Or create from a PyArrow schema:</p> <p>import pyarrow as pa Schema.from_pyarrow(pa.schema({\"x\": pa.int32(), \"y\": pa.string()})) Schema([Field(x, PrimitiveType(\"integer\"), nullable=True), Field(y, PrimitiveType(\"string\"), nullable=True)])</p>","boost":2},{"location":"api/schema/#deltalake.Schema.invariants","title":"invariants","text":"<pre><code>invariants: List[Tuple[str, str]] = &lt;attribute 'invariants' of 'deltalake._internal.Schema' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Schema.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(schema_json) -&gt; Schema\n</code></pre> <p>Create a new Schema from a JSON string.</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required Example <p>A schema has the same JSON format as a StructType. <pre><code>Schema.from_json('''{\n    \"type\": \"struct\",\n    \"fields\": [{\"name\": \"x\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]\n    }\n)'''\n# Returns Schema([Field(x, PrimitiveType(\"integer\"), nullable=True)])\n</code></pre></p>","boost":2},{"location":"api/schema/#deltalake.Schema.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; Schema\n</code></pre> <p>Create a Schema from a PyArrow Schema type</p> <p>Will raise <code>TypeError</code> if the PyArrow type is not a primitive type.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>Schema</code> <p>A PyArrow Schema</p> required <p>Returns:</p> Type Description <code>Schema</code> <p>a Schema</p>","boost":2},{"location":"api/schema/#deltalake.Schema.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get the JSON string representation of the Schema.</p> <p>Returns:</p> Type Description <code>str</code> <p>a JSON string</p> Example <p>A schema has the same JSON format as a StructType. <pre><code>Schema([Field(\"x\", \"integer\")]).to_json()\n# Returns '{\"type\":\"struct\",\"fields\":[{\"name\":\"x\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}'\n</code></pre></p>","boost":2},{"location":"api/schema/#deltalake.Schema.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow(as_large_types: bool = False) -&gt; pyarrow.Schema\n</code></pre> <p>Return equivalent PyArrow schema</p> <p>Parameters:</p> Name Type Description Default <code>as_large_types</code> <code>bool</code> <p>get schema with all variable size types (list, binary, string) as large variants (with int64 indices). This is for compatibility with systems like Polars that only support the large versions of Arrow types.</p> <code>False</code> <p>Returns:</p> Type Description <code>Schema</code> <p>a PyArrow Schema</p>","boost":2},{"location":"api/schema/#deltalake.Field","title":"deltalake.Field","text":"<pre><code>Field(name: str, type: DataType, *, nullable: bool = True, metadata: Optional[Dict[str, Any]] = None)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.metadata","title":"metadata","text":"<pre><code>metadata: Dict[str, Any] = &lt;attribute 'metadata' of 'deltalake._internal.Field' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.name","title":"name","text":"<pre><code>name: str = &lt;attribute 'name' of 'deltalake._internal.Field' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.nullable","title":"nullable","text":"<pre><code>nullable: bool = &lt;attribute 'nullable' of 'deltalake._internal.Field' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.type","title":"type","text":"<pre><code>type: DataType = &lt;attribute 'type' of 'deltalake._internal.Field' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(field_json) -&gt; Field\n</code></pre> <p>Create a Field from a JSON string.</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>the JSON string.</p> required <p>Returns:</p> Type Description <code>Field</code> <p>Field</p> Example <pre><code>Field.from_json('''{\n        \"name\": \"col\",\n        \"type\": \"integer\",\n        \"nullable\": true,\n        \"metadata\": {}\n    }'''\n)\n# Returns Field(col, PrimitiveType(\"integer\"), nullable=True)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(field: pyarrow.Field) -&gt; Field\n</code></pre> <p>Create a Field from a PyArrow field Note: This currently doesn't preserve field metadata.</p> <p>Parameters:</p> Name Type Description Default <code>field</code> <code>Field</code> <p>a PyArrow Field</p> required <p>Returns:</p> Type Description <code>Field</code> <p>a Field</p>","boost":2},{"location":"api/schema/#deltalake.Field.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get the field as JSON string.</p> <p>Returns:</p> Type Description <code>str</code> <p>a JSON string</p> Example <pre><code>Field(\"col\", \"integer\").to_json()\n# Returns '{\"name\":\"col\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}'\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.Field.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.Field\n</code></pre> <p>Convert to an equivalent PyArrow field Note: This currently doesn't preserve field metadata.</p> <p>Returns:</p> Type Description <code>Field</code> <p>a pyarrow Field</p>","boost":2},{"location":"api/schema/#data-types","title":"Data types","text":"","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType","title":"deltalake.schema.PrimitiveType","text":"<pre><code>PrimitiveType(data_type: str)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType.type","title":"type","text":"<pre><code>type: str = &lt;attribute 'type' of 'deltalake._internal.PrimitiveType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(type_json) -&gt; PrimitiveType\n</code></pre> <p>Create a PrimitiveType from a JSON string</p> <p>The JSON representation for a primitive type is just a quoted string: <code>PrimitiveType.from_json('\"integer\"')</code></p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required <p>Returns:</p> Type Description <code>PrimitiveType</code> <p>a PrimitiveType type</p>","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; PrimitiveType\n</code></pre> <p>Create a PrimitiveType from a PyArrow datatype</p> <p>Will raise <code>TypeError</code> if the PyArrow type is not a primitive type.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>DataType</code> <p>A PyArrow DataType</p> required <p>Returns:</p> Type Description <code>PrimitiveType</code> <p>a PrimitiveType</p>","boost":2},{"location":"api/schema/#deltalake.schema.PrimitiveType.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.DataType\n</code></pre> <p>Get the equivalent PyArrow type (pyarrow.DataType)</p>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType","title":"deltalake.schema.ArrayType","text":"<pre><code>ArrayType(element_type: DataType, *, contains_null: bool = True)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.contains_null","title":"contains_null","text":"<pre><code>contains_null: bool = &lt;attribute 'contains_null' of 'deltalake._internal.ArrayType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.element_type","title":"element_type","text":"<pre><code>element_type: DataType = &lt;attribute 'element_type' of 'deltalake._internal.ArrayType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.type","title":"type","text":"<pre><code>type: Literal['array'] = &lt;attribute 'type' of 'deltalake._internal.ArrayType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(type_json) -&gt; ArrayType\n</code></pre> <p>Create an ArrayType from a JSON string</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required <p>Returns:</p> Type Description <code>ArrayType</code> <p>an ArrayType</p> Example <p>The JSON representation for an array type is an object with <code>type</code> (set to <code>\"array\"</code>), <code>elementType</code>, and <code>containsNull</code>. <pre><code>ArrayType.from_json(\n    '''{\n        \"type\": \"array\",\n        \"elementType\": \"integer\",\n        \"containsNull\": false\n    }'''\n)\n# Returns ArrayType(PrimitiveType(\"integer\"), contains_null=False)\n</code></pre></p>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; ArrayType\n</code></pre> <p>Create an ArrayType from a pyarrow.ListType.</p> <p>Will raise <code>TypeError</code> if a different PyArrow DataType is provided.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>ListType</code> <p>The PyArrow ListType</p> required <p>Returns:</p> Type Description <code>ArrayType</code> <p>an ArrayType</p>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get the JSON string representation of the type.</p>","boost":2},{"location":"api/schema/#deltalake.schema.ArrayType.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.ListType\n</code></pre> <p>Get the equivalent PyArrow type.</p>","boost":2},{"location":"api/schema/#deltalake.schema.MapType","title":"deltalake.schema.MapType","text":"<pre><code>MapType(key_type: DataType, value_type: DataType, *, value_contains_null: bool = True)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.key_type","title":"key_type","text":"<pre><code>key_type: DataType = &lt;attribute 'key_type' of 'deltalake._internal.MapType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.type","title":"type","text":"<pre><code>type: Literal['map'] = &lt;attribute 'type' of 'deltalake._internal.MapType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.value_contains_null","title":"value_contains_null","text":"<pre><code>value_contains_null: bool = &lt;attribute 'value_contains_null' of 'deltalake._internal.MapType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.value_type","title":"value_type","text":"<pre><code>value_type: DataType = &lt;attribute 'value_type' of 'deltalake._internal.MapType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(type_json) -&gt; MapType\n</code></pre> <p>Create a MapType from a JSON string</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required <p>Returns:</p> Type Description <code>MapType</code> <p>an ArrayType</p> Example <p>The JSON representation for a map type is an object with <code>type</code> (set to <code>map</code>), <code>keyType</code>, <code>valueType</code>, and <code>valueContainsNull</code>:</p> <pre><code>MapType.from_json(\n    '''{\n        \"type\": \"map\",\n        \"keyType\": \"integer\",\n        \"valueType\": \"string\",\n        \"valueContainsNull\": true\n    }'''\n)\n# Returns MapType(PrimitiveType(\"integer\"), PrimitiveType(\"string\"), value_contains_null=True)\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; MapType\n</code></pre> <p>Create a MapType from a PyArrow MapType.</p> <p>Will raise <code>TypeError</code> if passed a different type.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>MapType</code> <p>the PyArrow MapType</p> required <p>Returns:</p> Type Description <code>MapType</code> <p>a MapType</p>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get JSON string representation of map type.</p> <p>Returns:</p> Type Description <code>str</code> <p>a JSON string</p>","boost":2},{"location":"api/schema/#deltalake.schema.MapType.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.MapType\n</code></pre> <p>Get the equivalent PyArrow data type.</p>","boost":2},{"location":"api/schema/#deltalake.schema.StructType","title":"deltalake.schema.StructType","text":"<pre><code>StructType(fields: List[Field])\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.fields","title":"fields","text":"<pre><code>fields: List[Field] = &lt;attribute 'fields' of 'deltalake._internal.StructType' objects&gt;\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.type","title":"type","text":"<pre><code>type: Literal['struct'] = &lt;attribute 'type' of 'deltalake._internal.StructType' objects&gt;\n</code></pre> <p>The string \"struct\"</p>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.from_json","title":"from_json  <code>staticmethod</code>","text":"<pre><code>from_json(type_json) -&gt; StructType\n</code></pre> <p>Create a new StructType from a JSON string.</p> <p>Parameters:</p> Name Type Description Default <code>json</code> <code>str</code> <p>a JSON string</p> required <p>Returns:</p> Type Description <code>StructType</code> <p>a StructType</p> Example <pre><code>StructType.from_json(\n    '''{\n        \"type\": \"struct\",\n        \"fields\": [{\"name\": \"x\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]\n    }'''\n)\n# Returns StructType([Field(x, PrimitiveType(\"integer\"), nullable=True)])\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.from_pyarrow","title":"from_pyarrow  <code>staticmethod</code>","text":"<pre><code>from_pyarrow(data_type) -&gt; StructType\n</code></pre> <p>Create a new StructType from a PyArrow struct type.</p> <p>Will raise <code>TypeError</code> if a different data type is provided.</p> <p>Parameters:</p> Name Type Description Default <code>type</code> <code>StructType</code> <p>a PyArrow struct type.</p> required <p>Returns:</p> Type Description <code>StructType</code> <p>a StructType</p>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.to_json","title":"to_json  <code>method descriptor</code>","text":"<pre><code>to_json() -&gt; str\n</code></pre> <p>Get the JSON representation of the type.</p> <p>Returns:</p> Type Description <code>str</code> <p>a JSON string</p> Example <pre><code>StructType([Field(\"x\", \"integer\")]).to_json()\n# Returns '{\"type\":\"struct\",\"fields\":[{\"name\":\"x\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}'\n</code></pre>","boost":2},{"location":"api/schema/#deltalake.schema.StructType.to_pyarrow","title":"to_pyarrow  <code>method descriptor</code>","text":"<pre><code>to_pyarrow() -&gt; pyarrow.StructType\n</code></pre> <p>Get the equivalent PyArrow StructType</p> <p>Returns:</p> Type Description <code>StructType</code> <p>a PyArrow StructType</p>","boost":2},{"location":"api/storage/","title":"Storage","text":"<p>The delta filesystem handler for the pyarrow engine writer.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler","title":"deltalake.fs.DeltaStorageHandler","text":"<pre><code>DeltaStorageHandler(table_uri: str, options: Optional[Dict[str, str]] = None, known_sizes: Optional[Dict[str, int]] = None)\n</code></pre> <p>             Bases: <code>FileSystemHandler</code></p> <p>DeltaStorageHandler is a concrete implementations of a PyArrow FileSystemHandler.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.copy_file","title":"copy_file","text":"<pre><code>copy_file(src: str, dst: str) -&gt; None\n</code></pre> <p>Copy a file.</p> <p>If the destination exists and is a directory, an error is returned. Otherwise, it is replaced.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.create_dir","title":"create_dir","text":"<pre><code>create_dir(path: str, recursive: bool = True) -&gt; None\n</code></pre> <p>Create a directory and subdirectories.</p> <p>This function succeeds if the directory already exists.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.delete_dir","title":"delete_dir","text":"<pre><code>delete_dir(path: str) -&gt; None\n</code></pre> <p>Delete a directory and its contents, recursively.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.delete_dir_contents","title":"delete_dir_contents","text":"<pre><code>delete_dir_contents(path: str, *, accept_root_dir: bool = False, missing_dir_ok: bool = False) -&gt; None\n</code></pre> <p>Delete a directory's contents, recursively.</p> <p>Like delete_dir, but doesn't delete the directory itself.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.delete_file","title":"delete_file","text":"<pre><code>delete_file(path: str) -&gt; None\n</code></pre> <p>Delete a file.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.delete_root_dir_contents","title":"delete_root_dir_contents","text":"<pre><code>delete_root_dir_contents() -&gt; None\n</code></pre> <p>Delete the root directory contents, recursively.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.get_file_info","title":"get_file_info","text":"<pre><code>get_file_info(paths: List[str]) -&gt; List[FileInfo]\n</code></pre> <p>Get info for the given files.</p> <p>A non-existing or unreachable file returns a FileStat object and has a FileType of value NotFound. An exception indicates a truly exceptional condition (low-level I/O error, etc.).</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.get_file_info_selector","title":"get_file_info_selector","text":"<pre><code>get_file_info_selector(selector: FileSelector) -&gt; List[FileInfo]\n</code></pre> <p>Get info for the files defined by FileSelector.</p> <p>Parameters:</p> Name Type Description Default <code>selector</code> <code>FileSelector</code> <p>FileSelector object</p> required <p>Returns:</p> Type Description <code>List[FileInfo]</code> <p>list of file info objects</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.move","title":"move","text":"<pre><code>move(src: str, dest: str) -&gt; None\n</code></pre> <p>Move / rename a file or directory.</p> <p>If the destination exists: - if it is a non-empty directory, an error is returned - otherwise, if it has the same type as the source, it is replaced - otherwise, behavior is unspecified (implementation-dependent).</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.normalize_path","title":"normalize_path","text":"<pre><code>normalize_path(path: str) -&gt; str\n</code></pre> <p>Normalize filesystem path.</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.open_input_file","title":"open_input_file","text":"<pre><code>open_input_file(path: str) -&gt; pa.PythonFile\n</code></pre> <p>Open an input file for random access reading.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>The source to open for reading.</p> required <p>Returns:</p> Type Description <code>PythonFile</code> <p>NativeFile</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.open_input_stream","title":"open_input_stream","text":"<pre><code>open_input_stream(path: str) -&gt; pa.PythonFile\n</code></pre> <p>Open an input stream for sequential reading.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>The source to open for reading.</p> required <p>Returns:</p> Type Description <code>PythonFile</code> <p>NativeFile</p>","boost":2},{"location":"api/storage/#deltalake.fs.DeltaStorageHandler.open_output_stream","title":"open_output_stream","text":"<pre><code>open_output_stream(path: str, metadata: Optional[Dict[str, str]] = None) -&gt; pa.PythonFile\n</code></pre> <p>Open an output stream for sequential writing.</p> <p>If the target already exists, existing data is truncated.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>The source to open for writing.</p> required <code>metadata</code> <code>Optional[Dict[str, str]]</code> <p>If not None, a mapping of string keys to string values.</p> <code>None</code> <p>Returns:</p> Type Description <code>PythonFile</code> <p>NativeFile</p>","boost":2},{"location":"api/delta_table/","title":"DeltaTable","text":"","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable","title":"deltalake.DeltaTable  <code>dataclass</code>","text":"<pre><code>DeltaTable(table_uri: Union[str, Path, os.PathLike[str]], version: Optional[int] = None, storage_options: Optional[Dict[str, str]] = None, without_files: bool = False, log_buffer_size: Optional[int] = None)\n</code></pre> <p>Represents a Delta Table</p> <p>Create the Delta Table from a path with an optional version. Multiple StorageBackends are currently supported: AWS S3, Azure Data Lake Storage Gen2, Google Cloud Storage (GCS) and local URI. Depending on the storage backend used, you could provide options values using the <code>storage_options</code> parameter.</p> <p>Parameters:</p> Name Type Description Default <code>table_uri</code> <code>Union[str, Path, PathLike[str]]</code> <p>the path of the DeltaTable</p> required <code>version</code> <code>Optional[int]</code> <p>version of the DeltaTable</p> <code>None</code> <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>a dictionary of the options to use for the storage backend</p> <code>None</code> <code>without_files</code> <code>bool</code> <p>If True, will load table without tracking files.                 Some append-only applications might have no need of tracking any files. So, the                 DeltaTable will be loaded with a significant memory reduction.</p> <code>False</code> <code>log_buffer_size</code> <code>Optional[int]</code> <p>Number of files to buffer when reading the commit log. A positive integer.                 Setting a value greater than 1 results in concurrent calls to the storage api.                 This can decrease latency if there are many files in the log since the last checkpoint,                 but will also increase memory usage. Possible rate limits of the storage backend should                 also be considered for optimal performance. Defaults to 4 * number of cpus.</p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.alter","title":"alter  <code>property</code>","text":"<pre><code>alter: TableAlterer\n</code></pre> <p>Namespace for all table alter related methods.</p> <p>Returns:</p> Name Type Description <code>TableAlterer</code> <code>TableAlterer</code> <p>TableAlterer Object</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.optimize","title":"optimize  <code>property</code>","text":"<pre><code>optimize: TableOptimizer\n</code></pre> <p>Namespace for all table optimize related methods.</p> <p>Returns:</p> Name Type Description <code>TableOptimizer</code> <code>TableOptimizer</code> <p>TableOptimizer Object</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.cleanup_metadata","title":"cleanup_metadata","text":"<pre><code>cleanup_metadata() -&gt; None\n</code></pre> <p>Delete expired log files before current version from table. The table log retention is based on the <code>delta.logRetentionDuration</code> value, 30 days by default.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.create","title":"create  <code>classmethod</code>","text":"<pre><code>create(table_uri: Union[str, Path], schema: Union[pyarrow.Schema, DeltaSchema], mode: Literal['error', 'append', 'overwrite', 'ignore'] = 'error', partition_by: Optional[Union[List[str], str]] = None, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, storage_options: Optional[Dict[str, str]] = None, custom_metadata: Optional[Dict[str, str]] = None, raise_if_key_not_exists: bool = True) -&gt; DeltaTable\n</code></pre> <p><code>CREATE</code> or <code>CREATE_OR_REPLACE</code> a delta table given a table_uri.</p> <p>Parameters:</p> Name Type Description Default <code>table_uri</code> <code>Union[str, Path]</code> <p>URI of a table</p> required <code>schema</code> <code>Union[Schema, Schema]</code> <p>Table schema</p> required <code>mode</code> <code>Literal['error', 'append', 'overwrite', 'ignore']</code> <p>How to handle existing data. Default is to error if table already exists. If 'append', returns not support error if table exists. If 'overwrite', will <code>CREATE_OR_REPLACE</code> table. If 'ignore', will not do anything if table already exists. Defaults to \"error\".</p> <code>'error'</code> <code>partition_by</code> <code>Optional[Union[List[str], str]]</code> <p>List of columns to partition the table by.</p> <code>None</code> <code>name</code> <code>Optional[str]</code> <p>User-provided identifier for this table.</p> <code>None</code> <code>description</code> <code>Optional[str]</code> <p>User-provided description for this table.</p> <code>None</code> <code>configuration</code> <code>Optional[Mapping[str, Optional[str]]]</code> <p>A map containing configuration options for the metadata action.</p> <code>None</code> <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>Options passed to the object store crate.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Custom metadata that will be added to the transaction commit.</p> <code>None</code> <code>raise_if_key_not_exists</code> <code>bool</code> <p>Whether to raise an error if the configuration uses keys that are not Delta keys</p> <code>True</code> <p>Returns:</p> Name Type Description <code>DeltaTable</code> <code>DeltaTable</code> <p>created delta table</p> Example <pre><code>import pyarrow as pa\n\nfrom deltalake import DeltaTable\n\ndt = DeltaTable.create(\n    table_uri=\"my_local_table\",\n    schema=pa.schema(\n        [pa.field(\"foo\", pa.string()), pa.field(\"bar\", pa.string())]\n    ),\n    mode=\"error\",\n    partition_by=\"bar\",\n)\n</code></pre>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.delete","title":"delete","text":"<pre><code>delete(predicate: Optional[str] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Delete records from a Delta Table that statisfy a predicate.</p> <p>When a predicate is not provided then all records are deleted from the Delta Table. Otherwise a scan of the Delta table is performed to mark any files that contain records that satisfy the predicate. Once files are determined they are rewritten without the records.</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>Optional[str]</code> <p>a SQL where clause. If not passed, will delete all rows.</p> <code>None</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>the metrics from delete.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.file_uris","title":"file_uris","text":"<pre><code>file_uris(partition_filters: Optional[FilterConjunctionType] = None) -&gt; List[str]\n</code></pre> <p>Get the list of files as absolute URIs, including the scheme (e.g. \"s3://\").</p> <p>Local files will be just plain absolute paths, without a scheme. (That is, no 'file://' prefix.)</p> <p>Use the partition_filters parameter to retrieve a subset of files that match the given filters.</p> <p>Parameters:</p> Name Type Description Default <code>partition_filters</code> <code>Optional[FilterConjunctionType]</code> <p>the partition filters that will be used for getting the matched files</p> <code>None</code> <p>Returns:</p> Type Description <code>List[str]</code> <p>list of the .parquet files with an absolute URI referenced for the current version of the DeltaTable</p> <p>Predicates are expressed in disjunctive normal form (DNF), like [(\"x\", \"=\", \"a\"), ...]. DNF allows arbitrary boolean logical combinations of single partition predicates. The innermost tuples each describe a single partition predicate. The list of inner predicates is interpreted as a conjunction (AND), forming a more selective and multiple partition predicates. Each tuple has format: (key, op, value) and compares the key with the value. The supported op are: <code>=</code>, <code>!=</code>, <code>in</code>, and <code>not in</code>. If the op is in or not in, the value must be a collection such as a list, a set or a tuple. The supported type for value is str. Use empty string <code>''</code> for Null partition value.</p> Example <pre><code>(\"x\", \"=\", \"a\")\n(\"x\", \"!=\", \"a\")\n(\"y\", \"in\", [\"a\", \"b\", \"c\"])\n(\"z\", \"not in\", [\"a\",\"b\"])\n</code></pre>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.files","title":"files","text":"<pre><code>files(partition_filters: Optional[List[Tuple[str, str, Any]]] = None) -&gt; List[str]\n</code></pre> <p>Get the .parquet files of the DeltaTable.</p> <p>The paths are as they are saved in the delta log, which may either be relative to the table root or absolute URIs.</p> <p>Parameters:</p> Name Type Description Default <code>partition_filters</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>the partition filters that will be used for                 getting the matched files</p> <code>None</code> <p>Returns:</p> Type Description <code>List[str]</code> <p>list of the .parquet files referenced for the current version of the DeltaTable</p> <p>Predicates are expressed in disjunctive normal form (DNF), like [(\"x\", \"=\", \"a\"), ...]. DNF allows arbitrary boolean logical combinations of single partition predicates. The innermost tuples each describe a single partition predicate. The list of inner predicates is interpreted as a conjunction (AND), forming a more selective and multiple partition predicates. Each tuple has format: (key, op, value) and compares the key with the value. The supported op are: <code>=</code>, <code>!=</code>, <code>in</code>, and <code>not in</code>. If the op is in or not in, the value must be a collection such as a list, a set or a tuple. The supported type for value is str. Use empty string <code>''</code> for Null partition value.</p> Example <pre><code>(\"x\", \"=\", \"a\")\n(\"x\", \"!=\", \"a\")\n(\"y\", \"in\", [\"a\", \"b\", \"c\"])\n(\"z\", \"not in\", [\"a\",\"b\"])\n</code></pre>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.files_by_partitions","title":"files_by_partitions","text":"<pre><code>files_by_partitions(partition_filters: PartitionFilterType) -&gt; List[str]\n</code></pre> <p>Get the files for each partition</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.from_data_catalog","title":"from_data_catalog  <code>classmethod</code>","text":"<pre><code>from_data_catalog(data_catalog: DataCatalog, database_name: str, table_name: str, data_catalog_id: Optional[str] = None, version: Optional[int] = None, log_buffer_size: Optional[int] = None) -&gt; DeltaTable\n</code></pre> <p>Create the Delta Table from a Data Catalog.</p> <p>Parameters:</p> Name Type Description Default <code>data_catalog</code> <code>DataCatalog</code> <p>the Catalog to use for getting the storage location of the Delta Table</p> required <code>database_name</code> <code>str</code> <p>the database name inside the Data Catalog</p> required <code>table_name</code> <code>str</code> <p>the table name inside the Data Catalog</p> required <code>data_catalog_id</code> <code>Optional[str]</code> <p>the identifier of the Data Catalog</p> <code>None</code> <code>version</code> <code>Optional[int]</code> <p>version of the DeltaTable</p> <code>None</code> <code>log_buffer_size</code> <code>Optional[int]</code> <p>Number of files to buffer when reading the commit log. A positive integer.                 Setting a value greater than 1 results in concurrent calls to the storage api.                 This can decrease latency if there are many files in the log since the last checkpoint,                 but will also increase memory usage. Possible rate limits of the storage backend should                 also be considered for optimal performance. Defaults to 4 * number of cpus.</p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.get_add_actions","title":"get_add_actions","text":"<pre><code>get_add_actions(flatten: bool = False) -&gt; pyarrow.RecordBatch\n</code></pre> <p>Return a dataframe with all current add actions.</p> <p>Add actions represent the files that currently make up the table. This data is a low-level representation parsed from the transaction log.</p> <p>Parameters:</p> Name Type Description Default <code>flatten</code> <code>bool</code> <p>whether to flatten the schema. Partition values columns are         given the prefix <code>partition.</code>, statistics (null_count, min, and max) are         given the prefix <code>null_count.</code>, <code>min.</code>, and <code>max.</code>, and tags the         prefix <code>tags.</code>. Nested field names are concatenated with <code>.</code>.</p> <code>False</code> <p>Returns:</p> Type Description <code>RecordBatch</code> <p>a PyArrow RecordBatch containing the add action data.</p> Example <pre><code>from pprint import pprint\nfrom deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data, partition_by=[\"x\"])\ndt = DeltaTable(\"tmp\")\ndf = dt.get_add_actions().to_pandas()\ndf[\"path\"].sort_values(ignore_index=True)\n0    x=1/0\n1    x=2/0\n2    x=3/0\n</code></pre> <pre><code>df = dt.get_add_actions(flatten=True).to_pandas()\ndf[\"partition.x\"].sort_values(ignore_index=True)\n0    1\n1    2\n2    3\n</code></pre>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.history","title":"history","text":"<pre><code>history(limit: Optional[int] = None) -&gt; List[Dict[str, Any]]\n</code></pre> <p>Run the history command on the DeltaTable. The operations are returned in reverse chronological order.</p> <p>Parameters:</p> Name Type Description Default <code>limit</code> <code>Optional[int]</code> <p>the commit info limit to return</p> <code>None</code> <p>Returns:</p> Type Description <code>List[Dict[str, Any]]</code> <p>list of the commit infos registered in the transaction log</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.is_deltatable","title":"is_deltatable  <code>staticmethod</code>","text":"<pre><code>is_deltatable(table_uri: str, storage_options: Optional[Dict[str, str]] = None) -&gt; bool\n</code></pre> <p>Returns True if a Delta Table exists at specified path. Returns False otherwise.</p> <p>Parameters:</p> Name Type Description Default <code>table_uri</code> <code>str</code> <p>the path of the DeltaTable</p> required <code>storage_options</code> <code>Optional[Dict[str, str]]</code> <p>a dictionary of the options to use for the storage backend</p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.load_as_version","title":"load_as_version","text":"<pre><code>load_as_version(version: Union[int, str, datetime]) -&gt; None\n</code></pre> <p>Load/time travel a DeltaTable to a specified version number, or a timestamp version of the table. If a string is passed then the argument should be an RFC 3339 and ISO 8601 date and time string format. If a datetime object without a timezone is passed, the UTC timezone will be assumed.</p> <p>Parameters:</p> Name Type Description Default <code>version</code> <code>Union[int, str, datetime]</code> <p>the identifier of the version of the DeltaTable to load</p> required Example <p>Use a version number <pre><code>dt = DeltaTable(\"test_table\")\ndt.load_as_version(1)\n</code></pre></p> <p>Use a datetime object <pre><code>dt.load_as_version(datetime(2023, 1, 1))\ndt.load_as_version(datetime(2023, 1, 1, tzinfo=timezone.utc))\n</code></pre></p> <p>Use a datetime in string format <pre><code>dt.load_as_version(\"2018-01-26T18:30:09Z\")\ndt.load_as_version(\"2018-12-19T16:39:57-08:00\")\ndt.load_as_version(\"2018-01-26T18:30:09.453+00:00\")\n</code></pre></p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.merge","title":"merge","text":"<pre><code>merge(source: Union[pyarrow.Table, pyarrow.RecordBatch, pyarrow.RecordBatchReader, ds.Dataset, pd.DataFrame], predicate: str, source_alias: Optional[str] = None, target_alias: Optional[str] = None, error_on_type_mismatch: bool = True, writer_properties: Optional[WriterProperties] = None, large_dtypes: Optional[bool] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; TableMerger\n</code></pre> <p>Pass the source data which you want to merge on the target delta table, providing a predicate in SQL query like format. You can also specify on what to do when the underlying data types do not match the underlying table.</p> <p>Parameters:</p> Name Type Description Default <code>source</code> <code>Union[Table, RecordBatch, RecordBatchReader, Dataset, DataFrame]</code> <p>source data</p> required <code>predicate</code> <code>str</code> <p>SQL like predicate on how to merge</p> required <code>source_alias</code> <code>Optional[str]</code> <p>Alias for the source table</p> <code>None</code> <code>target_alias</code> <code>Optional[str]</code> <p>Alias for the target table</p> <code>None</code> <code>error_on_type_mismatch</code> <code>bool</code> <p>specify if merge will return error if data types are mismatching :default = True</p> <code>True</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer</p> <code>None</code> <code>large_dtypes</code> <code>Optional[bool]</code> <p>Deprecated, will be removed in 1.0</p> <code>None</code> <code>arrow_schema_conversion_mode</code> <p>Large converts all types of data schema into Large Arrow types, passthrough keeps string/binary/list types untouched</p> required <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties for the commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.metadata","title":"metadata","text":"<pre><code>metadata() -&gt; Metadata\n</code></pre> <p>Get the current metadata of the DeltaTable.</p> <p>Returns:</p> Type Description <code>Metadata</code> <p>the current Metadata registered in the transaction log</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.partitions","title":"partitions","text":"<pre><code>partitions(partition_filters: Optional[List[Tuple[str, str, Any]]] = None) -&gt; List[Dict[str, str]]\n</code></pre> <p>Returns the partitions as a list of dicts. Example: <code>[{'month': '1', 'year': '2020', 'day': '1'}, ...]</code></p> <p>Parameters:</p> Name Type Description Default <code>partition_filters</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>The partition filters that will be used for getting the matched partitions, defaults to <code>None</code> (no filtering).</p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.protocol","title":"protocol","text":"<pre><code>protocol() -&gt; ProtocolVersions\n</code></pre> <p>Get the reader and writer protocol versions of the DeltaTable.</p> <p>Returns:</p> Type Description <code>ProtocolVersions</code> <p>the current ProtocolVersions registered in the transaction log</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.repair","title":"repair","text":"<pre><code>repair(dry_run: bool = False, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Repair the Delta Table by auditing active files that do not exist in the underlying filesystem and removes them. This can be useful when there are accidental deletions or corrupted files.</p> <p>Active files are ones that have an add action in the log, but no corresponding remove action. This operation creates a new FSCK transaction containing a remove action for each of the missing or corrupted files.</p> <p>Parameters:</p> Name Type Description Default <code>dry_run</code> <code>bool</code> <p>when activated, list only the files, otherwise add remove actions to transaction log. Defaults to False.</p> <code>False</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>The metrics from repair (FSCK) action.</p> Example <p><pre><code>from deltalake import DeltaTable\ndt = DeltaTable('TEST')\ndt.repair(dry_run=False)\n</code></pre> Results in <pre><code>{'dry_run': False, 'files_removed': ['6-0d084325-6885-4847-b008-82c1cf30674c-0.parquet', 5-4fba1d3e-3e20-4de1-933d-a8e13ac59f53-0.parquet']}\n</code></pre></p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.restore","title":"restore","text":"<pre><code>restore(target: Union[int, datetime, str], *, ignore_missing_files: bool = False, protocol_downgrade_allowed: bool = False, custom_metadata: Optional[Dict[str, str]] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Run the Restore command on the Delta Table: restore table to a given version or datetime.</p> <p>Parameters:</p> Name Type Description Default <code>target</code> <code>Union[int, datetime, str]</code> <p>the expected version will restore, which represented by int, date str or datetime.</p> required <code>ignore_missing_files</code> <code>bool</code> <p>whether the operation carry on when some data files missing.</p> <code>False</code> <code>protocol_downgrade_allowed</code> <code>bool</code> <p>whether the operation when protocol version upgraded.</p> <code>False</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>the metrics from restore.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.schema","title":"schema","text":"<pre><code>schema() -&gt; DeltaSchema\n</code></pre> <p>Get the current schema of the DeltaTable.</p> <p>Returns:</p> Type Description <code>Schema</code> <p>the current Schema registered in the transaction log</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.to_pandas","title":"to_pandas","text":"<pre><code>to_pandas(partitions: Optional[List[Tuple[str, str, Any]]] = None, columns: Optional[List[str]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, filters: Optional[Union[FilterType, Expression]] = None) -&gt; pd.DataFrame\n</code></pre> <p>Build a pandas dataframe using data from the DeltaTable.</p> <p>Parameters:</p> Name Type Description Default <code>partitions</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>A list of partition filters, see help(DeltaTable.files_by_partitions) for filter syntax</p> <code>None</code> <code>columns</code> <code>Optional[List[str]]</code> <p>The columns to project. This can be a list of column names to include (order and duplicates will be preserved)</p> <code>None</code> <code>filesystem</code> <code>Optional[Union[str, FileSystem]]</code> <p>A concrete implementation of the Pyarrow FileSystem or a fsspec-compatible interface. If None, the first file path will be used to determine the right FileSystem</p> <code>None</code> <code>filters</code> <code>Optional[Union[FilterType, Expression]]</code> <p>A disjunctive normal form (DNF) predicate for filtering rows, or directly a pyarrow.dataset.Expression. If you pass a filter you do not need to pass <code>partitions</code></p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.to_pyarrow_dataset","title":"to_pyarrow_dataset","text":"<pre><code>to_pyarrow_dataset(partitions: Optional[FilterConjunctionType] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, parquet_read_options: Optional[ParquetReadOptions] = None, schema: Optional[pyarrow.Schema] = None, as_large_types: bool = False) -&gt; pyarrow.dataset.Dataset\n</code></pre> <p>Build a PyArrow Dataset using data from the DeltaTable.</p> <p>Parameters:</p> Name Type Description Default <code>partitions</code> <code>Optional[FilterConjunctionType]</code> <p>A list of partition filters, see help(DeltaTable.files_by_partitions) for filter syntax</p> <code>None</code> <code>filesystem</code> <code>Optional[Union[str, FileSystem]]</code> <p>A concrete implementation of the Pyarrow FileSystem or a fsspec-compatible interface. If None, the first file path will be used to determine the right FileSystem</p> <code>None</code> <code>parquet_read_options</code> <code>Optional[ParquetReadOptions]</code> <p>Optional read options for Parquet. Use this to handle INT96 to timestamp conversion for edge cases like 0001-01-01 or 9999-12-31</p> <code>None</code> <code>schema</code> <code>Optional[Schema]</code> <p>The schema to use for the dataset. If None, the schema of the DeltaTable will be used. This can be used to force reading of Parquet/Arrow datatypes that DeltaLake can't represent in it's schema (e.g. LargeString). If you only need to read the schema with large types (e.g. for compatibility with Polars) you may want to use the <code>as_large_types</code> parameter instead.</p> <code>None</code> <code>as_large_types</code> <code>bool</code> <p>get schema with all variable size types (list, binary, string) as large variants (with int64 indices). This is for compatibility with systems like Polars that only support the large versions of Arrow types. If <code>schema</code> is passed it takes precedence over this option.</p> <code>False</code> <p>More info: https://arrow.apache.org/docs/python/generated/pyarrow.dataset.ParquetReadOptions.html</p> Example <p><code>deltalake</code> will work with any storage compliant with :class:<code>pyarrow.fs.FileSystem</code>, however the root of the filesystem has to be adjusted to point at the root of the Delta table. We can achieve this by wrapping the custom filesystem into a :class:<code>pyarrow.fs.SubTreeFileSystem</code>. <pre><code>import pyarrow.fs as fs\nfrom deltalake import DeltaTable\n\ntable_uri = \"s3://&lt;bucket&gt;/&lt;path&gt;\"\nraw_fs, normalized_path = fs.FileSystem.from_uri(table_uri)\nfilesystem = fs.SubTreeFileSystem(normalized_path, raw_fs)\n\ndt = DeltaTable(table_uri)\nds = dt.to_pyarrow_dataset(filesystem=filesystem)\n</code></pre></p> <p>Returns:</p> Type Description <code>Dataset</code> <p>the PyArrow dataset in PyArrow</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.to_pyarrow_table","title":"to_pyarrow_table","text":"<pre><code>to_pyarrow_table(partitions: Optional[List[Tuple[str, str, Any]]] = None, columns: Optional[List[str]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, filters: Optional[Union[FilterType, Expression]] = None) -&gt; pyarrow.Table\n</code></pre> <p>Build a PyArrow Table using data from the DeltaTable.</p> <p>Parameters:</p> Name Type Description Default <code>partitions</code> <code>Optional[List[Tuple[str, str, Any]]]</code> <p>A list of partition filters, see help(DeltaTable.files_by_partitions) for filter syntax</p> <code>None</code> <code>columns</code> <code>Optional[List[str]]</code> <p>The columns to project. This can be a list of column names to include (order and duplicates will be preserved)</p> <code>None</code> <code>filesystem</code> <code>Optional[Union[str, FileSystem]]</code> <p>A concrete implementation of the Pyarrow FileSystem or a fsspec-compatible interface. If None, the first file path will be used to determine the right FileSystem</p> <code>None</code> <code>filters</code> <code>Optional[Union[FilterType, Expression]]</code> <p>A disjunctive normal form (DNF) predicate for filtering rows, or directly a pyarrow.dataset.Expression. If you pass a filter you do not need to pass <code>partitions</code></p> <code>None</code>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.update","title":"update","text":"<pre><code>update(updates: Optional[Dict[str, str]] = None, new_values: Optional[Dict[str, Union[int, float, str, datetime, bool, List[Any]]]] = None, predicate: Optional[str] = None, writer_properties: Optional[WriterProperties] = None, error_on_type_mismatch: bool = True, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p><code>UPDATE</code> records in the Delta Table that matches an optional predicate. Either updates or new_values needs to be passed for it to execute.</p> <p>Parameters:</p> Name Type Description Default <code>updates</code> <code>Optional[Dict[str, str]]</code> <p>a mapping of column name to update SQL expression.</p> <code>None</code> <code>new_values</code> <code>Optional[Dict[str, Union[int, float, str, datetime, bool, List[Any]]]]</code> <p>a mapping of column name to python datatype.</p> <code>None</code> <code>predicate</code> <code>Optional[str]</code> <p>a logical expression.</p> <code>None</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>error_on_type_mismatch</code> <code>bool</code> <p>specify if update will return error if data types are mismatching :default = True</p> <code>True</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:     the metrics from update</p> Example <p>Update some row values with SQL predicate</p> <p>This is equivalent to <code>UPDATE table SET deleted = true WHERE id = '3'</code> <pre><code>from deltalake import write_deltalake, DeltaTable\nimport pandas as pd\ndf = pd.DataFrame(\n    {\"id\": [\"1\", \"2\", \"3\"],\n    \"deleted\": [False, False, False],\n    \"price\": [10., 15., 20.]\n    })\nwrite_deltalake(\"tmp\", df)\ndt = DeltaTable(\"tmp\")\ndt.update(predicate=\"id = '3'\", updates = {\"deleted\": 'True'})\n\n{'num_added_files': 1, 'num_removed_files': 1, 'num_updated_rows': 1, 'num_copied_rows': 2, 'execution_time_ms': ..., 'scan_time_ms': ...}\n</code></pre></p> <p>Update all row values</p> <p>This is equivalent to <code>UPDATE table SET deleted = true, id = concat(id, '_old')</code>. <pre><code>dt.update(updates = {\"deleted\": 'True', \"id\": \"concat(id, '_old')\"})\n\n{'num_added_files': 1, 'num_removed_files': 1, 'num_updated_rows': 3, 'num_copied_rows': 0, 'execution_time_ms': ..., 'scan_time_ms': ...}\n</code></pre></p> <p>Use Python objects instead of SQL strings</p> <p>Use the <code>new_values</code> parameter instead of the <code>updates</code> parameter. For example, this is equivalent to <code>UPDATE table SET price = 150.10 WHERE id = '1'</code> <pre><code>dt.update(predicate=\"id = '1_old'\", new_values = {\"price\": 150.10})\n\n{'num_added_files': 1, 'num_removed_files': 1, 'num_updated_rows': 1, 'num_copied_rows': 2, 'execution_time_ms': ..., 'scan_time_ms': ...}\n</code></pre></p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.update_incremental","title":"update_incremental","text":"<pre><code>update_incremental() -&gt; None\n</code></pre> <p>Updates the DeltaTable to the latest version by incrementally applying newer versions.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.vacuum","title":"vacuum","text":"<pre><code>vacuum(retention_hours: Optional[int] = None, dry_run: bool = True, enforce_retention_duration: bool = True, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; List[str]\n</code></pre> <p>Run the Vacuum command on the Delta Table: list and delete files no longer referenced by the Delta table and are older than the retention threshold.</p> <p>Parameters:</p> Name Type Description Default <code>retention_hours</code> <code>Optional[int]</code> <p>the retention threshold in hours, if none then the value from <code>delta.deletedFileRetentionDuration</code> is used or default of 1 week otherwise.</p> <code>None</code> <code>dry_run</code> <code>bool</code> <p>when activated, list only the files, delete otherwise</p> <code>True</code> <code>enforce_retention_duration</code> <code>bool</code> <p>when disabled, accepts retention hours smaller than the value from <code>delta.deletedFileRetentionDuration</code>.</p> <code>True</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:     the list of files no longer referenced by the Delta Table and are older than the retention threshold.</p>","boost":2},{"location":"api/delta_table/#deltalake.DeltaTable.version","title":"version","text":"<pre><code>version() -&gt; int\n</code></pre> <p>Get the version of the DeltaTable.</p> <p>Returns:</p> Type Description <code>int</code> <p>The current version of the DeltaTable</p>","boost":2},{"location":"api/delta_table/delta_table_alterer/","title":"TableAlterer","text":"","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer","title":"deltalake.table.TableAlterer","text":"<pre><code>TableAlterer(table: DeltaTable)\n</code></pre> <p>API for various table alteration commands.</p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.add_columns","title":"add_columns","text":"<pre><code>add_columns(fields: Union[DeltaField, List[DeltaField]], custom_metadata: Optional[Dict[str, str]] = None, commit_properties: Optional[CommitProperties] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None) -&gt; None\n</code></pre> <p>Add new columns and/or update the fields of a stuctcolumn</p> <p>Parameters:</p> Name Type Description Default <code>fields</code> <code>Union[Field, List[Field]]</code> <p>fields to merge into schema</p> required <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> Example <p>from deltalake.schema import Field, PrimitiveType, StructType dt = DeltaTable(\"test_table\") new_fields = [     Field(\"baz\", StructType([Field(\"bar\", PrimitiveType(\"integer\"))])),     Field(\"bar\", PrimitiveType(\"integer\")) ] dt.alter.add_columns(     new_fields ) ```</p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.add_constraint","title":"add_constraint","text":"<pre><code>add_constraint(constraints: Dict[str, str], custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; None\n</code></pre> <p>Add constraints to the table. Limited to <code>single constraint</code> at once.</p> <p>Parameters:</p> Name Type Description Default <code>constraints</code> <code>Dict[str, str]</code> <p>mapping of constraint name to SQL-expression to evaluate on write</p> required <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> Example <pre><code>from deltalake import DeltaTable\ndt = DeltaTable(\"test_table_constraints\")\ndt.alter.add_constraint({\n    \"value_gt_5\": \"value &gt; 5\",\n})\n</code></pre> <p>Check configuration <pre><code>dt.metadata().configuration\n{'delta.constraints.value_gt_5': 'value &gt; 5'}\n</code></pre></p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.add_feature","title":"add_feature","text":"<pre><code>add_feature(feature: Union[TableFeatures, List[TableFeatures]], allow_protocol_versions_increase: bool = False, commit_properties: Optional[CommitProperties] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None) -&gt; None\n</code></pre> <p>Enable a table feature.</p> <p>Parameters:</p> Name Type Description Default <code>feature</code> <code>Union[TableFeatures, List[TableFeatures]]</code> <p>Table Feature e.g. Deletion Vectors, Change Data Feed</p> required <code>allow_protocol_versions_increase</code> <code>bool</code> <p>Allow the protocol to be implicitily bumped to reader 3 or writer 7</p> <code>False</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> Example <pre><code>from deltalake import DeltaTable\ndt = DeltaTable(\"test_table\")\ndt.alter.add_feature(TableFeatures.AppendOnly)\n</code></pre> <p>Check protocol <pre><code>dt.protocol()\nProtocolVersions(min_reader_version=1, min_writer_version=7, writer_features=['appendOnly'], reader_features=None)\n</code></pre></p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.drop_constraint","title":"drop_constraint","text":"<pre><code>drop_constraint(name: str, raise_if_not_exists: bool = True, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; None\n</code></pre> <p>Drop constraints from a table. Limited to <code>single constraint</code> at once.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>constraint name which to drop.</p> required <code>raise_if_not_exists</code> <code>bool</code> <p>set if should raise if not exists.</p> <code>True</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> Example <pre><code>from deltalake import DeltaTable\ndt = DeltaTable(\"test_table_constraints\")\ndt.metadata().configuration\n{'delta.constraints.value_gt_5': 'value &gt; 5'}\n</code></pre> <p>Drop the constraint <pre><code>dt.alter.drop_constraint(name = \"value_gt_5\")\n</code></pre></p> <p>Configuration after dropping <pre><code>dt.metadata().configuration\n{}\n</code></pre></p>","boost":10},{"location":"api/delta_table/delta_table_alterer/#deltalake.table.TableAlterer.set_table_properties","title":"set_table_properties","text":"<pre><code>set_table_properties(properties: Dict[str, str], raise_if_not_exists: bool = True, custom_metadata: Optional[Dict[str, str]] = None, commit_properties: Optional[CommitProperties] = None) -&gt; None\n</code></pre> <p>Set properties from the table.</p> <p>Parameters:</p> Name Type Description Default <code>properties</code> <code>Dict[str, str]</code> <p>properties which set</p> required <code>raise_if_not_exists</code> <code>bool</code> <p>set if should raise if not exists.</p> <code>True</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> Example <pre><code>from deltalake import write_deltalake, DeltaTable\nimport pandas as pd\ndf = pd.DataFrame(\n    {\"id\": [\"1\", \"2\", \"3\"],\n    \"deleted\": [False, False, False],\n    \"price\": [10., 15., 20.]\n    })\nwrite_deltalake(\"tmp\", df)\n\ndt = DeltaTable(\"tmp\")\ndt.alter.set_table_properties({\"delta.enableChangeDataFeed\": \"true\"})\n</code></pre>","boost":10},{"location":"api/delta_table/delta_table_merger/","title":"TableMerger","text":"","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger","title":"deltalake.table.TableMerger","text":"<pre><code>TableMerger(builder: PyMergeBuilder, table: RawDeltaTable)\n</code></pre> <p>API for various table <code>MERGE</code> commands.</p>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.execute","title":"execute","text":"<pre><code>execute() -&gt; Dict[str, Any]\n</code></pre> <p>Executes <code>MERGE</code> with the previously provided settings in Rust with Apache Datafusion query engine.</p> <p>Returns:</p> Name Type Description <code>Dict</code> <code>Dict[str, Any]</code> <p>metrics</p>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_delete","title":"when_matched_delete","text":"<pre><code>when_matched_delete(predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Delete a matched row from the table only if the given <code>predicate</code> (if specified) is true for the matched row. If not specified it deletes all matches.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>(str | None, Optional)</code> <p>SQL like predicate on when to delete.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <p>Delete on a predicate</p> <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [2, 3], \"deleted\": [False, True]})\n\n(\n    dt.merge(\n        source=new_data,\n        predicate='target.x = source.x',\n        source_alias='source',\n        target_alias='target')\n    .when_matched_delete(\n        predicate=\"source.deleted = true\")\n    .execute()\n)\n{'num_source_rows': 2, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 0, 'num_target_rows_deleted': 1, 'num_target_rows_copied': 2, 'num_output_rows': 2, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  4\n1  2  5\n</code></pre> <p>Delete all records that were matched <pre><code>dt = DeltaTable(\"tmp\")\n(\n    dt.merge(\n        source=new_data,\n        predicate='target.x = source.x',\n        source_alias='source',\n        target_alias='target')\n    .when_matched_delete()\n    .execute()\n)\n{'num_source_rows': 2, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 0, 'num_target_rows_deleted': 1, 'num_target_rows_copied': 1, 'num_output_rows': 1, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas()\n   x  y\n0  1  4\n</code></pre></p>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_update","title":"when_matched_update","text":"<pre><code>when_matched_update(updates: Dict[str, str], predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Update a matched table row based on the rules defined by <code>updates</code>. If a <code>predicate</code> is specified, then it must evaluate to true for the row to be updated.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>updates</code> <code>Dict[str, str]</code> <p>a mapping of column name to update SQL expression.</p> required <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to update.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"1y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [1], \"1y\": [7]})\n\n(\n     dt.merge(\n         source=new_data,\n         predicate=\"target.x = source.x\",\n         source_alias=\"source\",\n         target_alias=\"target\")\n     .when_matched_update(updates={\"x\": \"source.x\", \"`1y`\": \"source.`1y`\"})\n     .execute()\n)\n{'num_source_rows': 1, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 1, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 2, 'num_output_rows': 3, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas()\n   x  y\n0  1  7\n1  2  5\n2  3  6\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_update_all","title":"when_matched_update_all","text":"<pre><code>when_matched_update_all(predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Updating all source fields to target fields, source and target are required to have the same field names. If a <code>predicate</code> is specified, then it must evaluate to true for the row to be updated.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to update all columns.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [1], \"y\": [7]})\n\n(\n    dt.merge(\n        source=new_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\")\n    .when_matched_update_all()\n    .execute()\n)\n{'num_source_rows': 1, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 1, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 2, 'num_output_rows': 3, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas()\n   x  y\n0  1  7\n1  2  5\n2  3  6\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_by_source_delete","title":"when_not_matched_by_source_delete","text":"<pre><code>when_not_matched_by_source_delete(predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Delete a target row that has no matches in the source from the table only if the given <code>predicate</code> (if specified) is true for the target row.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to delete when not matched by source.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_by_source_update","title":"when_not_matched_by_source_update","text":"<pre><code>when_not_matched_by_source_update(updates: Dict[str, str], predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Update a target row that has no matches in the source based on the rules defined by <code>updates</code>. If a <code>predicate</code> is specified, then it must evaluate to true for the row to be updated.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>updates</code> <code>Dict[str, str]</code> <p>a mapping of column name to update SQL expression.</p> required <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to update.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [2, 3, 4]})\n\n(\n   dt.merge(\n       source=new_data,\n       predicate='target.x = source.x',\n       source_alias='source',\n       target_alias='target')\n   .when_not_matched_by_source_update(\n       predicate = \"y &gt; 3\",\n       updates = {\"y\": \"0\"})\n   .execute()\n)\n{'num_source_rows': 3, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 1, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 2, 'num_output_rows': 3, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  0\n1  2  5\n2  3  6\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_insert","title":"when_not_matched_insert","text":"<pre><code>when_not_matched_insert(updates: Dict[str, str], predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Insert a new row to the target table based on the rules defined by <code>updates</code>. If a <code>predicate</code> is specified, then it must evaluate to true for the new row to be inserted.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>updates</code> <code>dict</code> <p>a mapping of column name to insert SQL expression.</p> required <code>predicate</code> <code>(str | None, Optional)</code> <p>SQL like predicate on when to insert.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [4], \"y\": [7]})\n\n(\n    dt.merge(\n        source=new_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\",)\n    .when_not_matched_insert(\n        updates={\n            \"x\": \"source.x\",\n            \"y\": \"source.y\",\n        })\n    .execute()\n)\n{'num_source_rows': 1, 'num_target_rows_inserted': 1, 'num_target_rows_updated': 0, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 3, 'num_output_rows': 4, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  4\n1  2  5\n2  3  6\n3  4  7\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_insert_all","title":"when_not_matched_insert_all","text":"<pre><code>when_not_matched_insert_all(predicate: Optional[str] = None) -&gt; TableMerger\n</code></pre> <p>Insert a new row to the target table, updating all source fields to target fields. Source and target are required to have the same field names. If a <code>predicate</code> is specified, then it must evaluate to true for the new row to be inserted.</p> Note <p>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</p> <p>Parameters:</p> Name Type Description Default <code>predicate</code> <code>Optional[str]</code> <p>SQL like predicate on when to insert.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>TableMerger</code> <code>TableMerger</code> <p>TableMerger Object</p> Example <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\ndata = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", data)\ndt = DeltaTable(\"tmp\")\nnew_data = pa.table({\"x\": [4], \"y\": [7]})\n\n(\n   dt.merge(\n       source=new_data,\n       predicate='target.x = source.x',\n       source_alias='source',\n       target_alias='target')\n   .when_not_matched_insert_all()\n   .execute()\n)\n{'num_source_rows': 1, 'num_target_rows_inserted': 1, 'num_target_rows_updated': 0, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 3, 'num_output_rows': 4, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}\n\ndt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  4\n1  2  5\n2  3  6\n3  4  7\n</code></pre>","boost":2},{"location":"api/delta_table/delta_table_optimizer/","title":"TableOptimizer","text":"","boost":10},{"location":"api/delta_table/delta_table_optimizer/#deltalake.table.TableOptimizer","title":"deltalake.table.TableOptimizer","text":"<pre><code>TableOptimizer(table: DeltaTable)\n</code></pre> <p>API for various table optimization commands.</p>","boost":10},{"location":"api/delta_table/delta_table_optimizer/#deltalake.table.TableOptimizer.compact","title":"compact","text":"<pre><code>compact(partition_filters: Optional[FilterConjunctionType] = None, target_size: Optional[int] = None, max_concurrent_tasks: Optional[int] = None, min_commit_interval: Optional[Union[int, timedelta]] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Compacts small files to reduce the total number of files in the table.</p> <p>This operation is idempotent; if run twice on the same table (assuming it has not been updated) it will do nothing the second time.</p> <p>If this operation happens concurrently with any operations other than append, it will fail.</p> <p>Parameters:</p> Name Type Description Default <code>partition_filters</code> <code>Optional[FilterConjunctionType]</code> <p>the partition filters that will be used for getting the matched files</p> <code>None</code> <code>target_size</code> <code>Optional[int]</code> <p>desired file size after bin-packing files, in bytes. If not             provided, will attempt to read the table configuration value <code>delta.targetFileSize</code>.             If that value isn't set, will use default value of 256MB.</p> <code>None</code> <code>max_concurrent_tasks</code> <code>Optional[int]</code> <p>the maximum number of concurrent tasks to use for                     file compaction. Defaults to number of CPUs. More concurrent tasks can make compaction                     faster, but will also use more memory.</p> <code>None</code> <code>min_commit_interval</code> <code>Optional[Union[int, timedelta]]</code> <p>minimum interval in seconds or as timedeltas before a new commit is                     created. Interval is useful for long running executions. Set to 0 or timedelta(0), if you                     want a commit per partition.</p> <code>None</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>the metrics from optimize</p> Example <p>Use a timedelta object to specify the seconds, minutes or hours of the interval. <pre><code>from deltalake import DeltaTable, write_deltalake\nfrom datetime import timedelta\nimport pyarrow as pa\n\nwrite_deltalake(\"tmp\", pa.table({\"x\": [1], \"y\": [4]}))\nwrite_deltalake(\"tmp\", pa.table({\"x\": [2], \"y\": [5]}), mode=\"append\")\n\ndt = DeltaTable(\"tmp\")\ntime_delta = timedelta(minutes=10)\ndt.optimize.compact(min_commit_interval=time_delta)\n{'numFilesAdded': 1, 'numFilesRemoved': 2, 'filesAdded': ..., 'filesRemoved': ..., 'partitionsOptimized': 1, 'numBatches': 2, 'totalConsideredFiles': 2, 'totalFilesSkipped': 0, 'preserveInsertionOrder': True}\n</code></pre></p>","boost":10},{"location":"api/delta_table/delta_table_optimizer/#deltalake.table.TableOptimizer.z_order","title":"z_order","text":"<pre><code>z_order(columns: Iterable[str], partition_filters: Optional[FilterConjunctionType] = None, target_size: Optional[int] = None, max_concurrent_tasks: Optional[int] = None, max_spill_size: int = 20 * 1024 * 1024 * 1024, min_commit_interval: Optional[Union[int, timedelta]] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None, post_commithook_properties: Optional[PostCommitHookProperties] = None, commit_properties: Optional[CommitProperties] = None) -&gt; Dict[str, Any]\n</code></pre> <p>Reorders the data using a Z-order curve to improve data skipping.</p> <p>This also performs compaction, so the same parameters as compact() apply.</p> <p>Parameters:</p> Name Type Description Default <code>columns</code> <code>Iterable[str]</code> <p>the columns to use for Z-ordering. There must be at least one column.</p> required <code>partition_filters</code> <code>Optional[FilterConjunctionType]</code> <p>the partition filters that will be used for getting the matched files</p> <code>None</code> <code>target_size</code> <code>Optional[int]</code> <p>desired file size after bin-packing files, in bytes. If not             provided, will attempt to read the table configuration value <code>delta.targetFileSize</code>.             If that value isn't set, will use default value of 256MB.</p> <code>None</code> <code>max_concurrent_tasks</code> <code>Optional[int]</code> <p>the maximum number of concurrent tasks to use for                     file compaction. Defaults to number of CPUs. More concurrent tasks can make compaction                     faster, but will also use more memory.</p> <code>None</code> <code>max_spill_size</code> <code>int</code> <p>the maximum number of bytes allowed in memory before spilling to disk. Defaults to 20GB.</p> <code>20 * 1024 * 1024 * 1024</code> <code>min_commit_interval</code> <code>Optional[Union[int, timedelta]]</code> <p>minimum interval in seconds or as timedeltas before a new commit is                     created. Interval is useful for long running executions. Set to 0 or timedelta(0), if you                     want a commit per partition.</p> <code>None</code> <code>writer_properties</code> <code>Optional[WriterProperties]</code> <p>Pass writer properties to the Rust parquet writer.</p> <code>None</code> <code>custom_metadata</code> <code>Optional[Dict[str, str]]</code> <p>Deprecated and will be removed in future versions. Use commit_properties instead.</p> <code>None</code> <code>post_commithook_properties</code> <code>Optional[PostCommitHookProperties]</code> <p>properties for the post commit hook. If None, default values are used.</p> <code>None</code> <code>commit_properties</code> <code>Optional[CommitProperties]</code> <p>properties of the transaction commit. If None, default values are used.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dict[str, Any]</code> <p>the metrics from optimize</p> Example <p>Use a timedelta object to specify the seconds, minutes or hours of the interval. <pre><code>from deltalake import DeltaTable, write_deltalake\nfrom datetime import timedelta\nimport pyarrow as pa\n\nwrite_deltalake(\"tmp\", pa.table({\"x\": [1], \"y\": [4]}))\nwrite_deltalake(\"tmp\", pa.table({\"x\": [2], \"y\": [5]}), mode=\"append\")\n\ndt = DeltaTable(\"tmp\")\ntime_delta = timedelta(minutes=10)\ndt.optimize.z_order([\"x\"], min_commit_interval=time_delta)\n{'numFilesAdded': 1, 'numFilesRemoved': 2, 'filesAdded': ..., 'filesRemoved': ..., 'partitionsOptimized': 0, 'numBatches': 1, 'totalConsideredFiles': 2, 'totalFilesSkipped': 0, 'preserveInsertionOrder': True}\n</code></pre></p>","boost":10},{"location":"api/delta_table/metadata/","title":"Metadata","text":"","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata","title":"deltalake.Metadata  <code>dataclass</code>","text":"<pre><code>Metadata(table: RawDeltaTable)\n</code></pre> <p>Create a Metadata instance.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.configuration","title":"configuration  <code>property</code>","text":"<pre><code>configuration: Dict[str, str]\n</code></pre> <p>Return the DeltaTable properties.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.created_time","title":"created_time  <code>property</code>","text":"<pre><code>created_time: int\n</code></pre> <p>Return The time when this metadata action is created, in milliseconds since the Unix epoch of the DeltaTable.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.description","title":"description  <code>property</code>","text":"<pre><code>description: str\n</code></pre> <p>Return the user-provided description of the DeltaTable.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.id","title":"id  <code>property</code>","text":"<pre><code>id: int\n</code></pre> <p>Return the unique identifier of the DeltaTable.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.name","title":"name  <code>property</code>","text":"<pre><code>name: str\n</code></pre> <p>Return the user-provided identifier of the DeltaTable.</p>","boost":2},{"location":"api/delta_table/metadata/#deltalake.Metadata.partition_columns","title":"partition_columns  <code>property</code>","text":"<pre><code>partition_columns: List[str]\n</code></pre> <p>Return an array containing the names of the partitioned columns of the DeltaTable.</p>","boost":2},{"location":"how-delta-lake-works/architecture-of-delta-table/","title":"Architecture of a Delta Lake table","text":"<p>A Delta table consists of Parquet files that contain data and a transaction log that stores metadata about the transactions.</p> <p></p> <p>Let's create a Delta table, perform some operations, and inspect the files that are created.</p>"},{"location":"how-delta-lake-works/architecture-of-delta-table/#delta-lake-transaction-examples","title":"Delta Lake transaction examples","text":"<p>Start by creating a pandas DataFrame and writing it out to a Delta table.</p> <pre><code>import pandas as pd\nfrom deltalake import write_deltalake\n\ndf = pd.DataFrame({\"num\": [1, 2, 3], \"letter\": [\"a\", \"b\", \"c\"]})\nwrite_deltalake(\"tmp/some-table\", df)\n</code></pre> <p>Now inspect the files created in storage:</p> <pre><code>tmp/some-table\n\u251c\u2500\u2500 0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u2514\u2500\u2500 00000000000000000000.json\n</code></pre> <p>The Parquet file stores the data that was written. The <code>_delta_log</code> directory stores metadata about the transactions. Let's inspect the <code>_delta_log/00000000000000000000.json</code> file.</p> <pre><code>{\n  \"protocol\": {\n    \"minReaderVersion\": 1,\n    \"minWriterVersion\": 1\n  }\n}\n{\n  \"metaData\": {\n    \"id\": \"b96ea1a2-1830-4da2-8827-5334cc6104ed\",\n    \"name\": null,\n    \"description\": null,\n    \"format\": {\n      \"provider\": \"parquet\",\n      \"options\": {}\n    },\n    \"schemaString\": \"{\\\"type\\\":\\\"struct\\\",\\\"fields\\\":[{\\\"name\\\":\\\"num\\\",\\\"type\\\":\\\"long\\\",\\\"nullable\\\":true,\\\"metadata\\\":{}},{\\\"name\\\":\\\"letter\\\",\\\"type\\\":\\\"string\\\",\\\"nullable\\\":true,\\\"metadata\\\":{}}]}\",\n    \"partitionColumns\": [],\n    \"createdTime\": 1701740315599,\n    \"configuration\": {}\n  }\n}\n{\n  \"add\": {\n    \"path\": \"0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\",\n    \"size\": 2208,\n    \"partitionValues\": {},\n    \"modificationTime\": 1701740315597,\n    \"dataChange\": true,\n    \"stats\": \"{\\\"numRecords\\\": 3, \\\"minValues\\\": {\\\"num\\\": 1, \\\"letter\\\": \\\"a\\\"}, \\\"maxValues\\\": {\\\"num\\\": 3, \\\"letter\\\": \\\"c\\\"}, \\\"nullCount\\\": {\\\"num\\\": 0, \\\"letter\\\": 0}}\"\n  }\n}\n{\n  \"commitInfo\": {\n    \"timestamp\": 1701740315602,\n    \"operation\": \"CREATE TABLE\",\n    \"operationParameters\": {\n      \"location\": \"file:///Users/matthew.powers/Documents/code/delta/delta-examples/notebooks/python-deltalake/tmp/some-table\",\n      \"metadata\": \"{\\\"configuration\\\":{},\\\"created_time\\\":1701740315599,\\\"description\\\":null,\\\"format\\\":{\\\"options\\\":{},\\\"provider\\\":\\\"parquet\\\"},\\\"id\\\":\\\"b96ea1a2-1830-4da2-8827-5334cc6104ed\\\",\\\"name\\\":null,\\\"partition_columns\\\":[],\\\"schema\\\":{\\\"fields\\\":[{\\\"metadata\\\":{},\\\"name\\\":\\\"num\\\",\\\"nullable\\\":true,\\\"type\\\":\\\"long\\\"},{\\\"metadata\\\":{},\\\"name\\\":\\\"letter\\\",\\\"nullable\\\":true,\\\"type\\\":\\\"string\\\"}],\\\"type\\\":\\\"struct\\\"}}\",\n      \"protocol\": \"{\\\"minReaderVersion\\\":1,\\\"minWriterVersion\\\":1}\",\n      \"mode\": \"ErrorIfExists\"\n    },\n    \"clientVersion\": \"delta-rs.0.17.0\"\n  }\n}\n</code></pre> <p>The transaction log file contains the following information:</p> <ul> <li>the files added to the Delta table</li> <li>schema of the files</li> <li>column level metadata including the min/max value for each file</li> </ul> <p>Create another pandas DataFrame and append it to the Delta table to see how this transaction is recorded.</p> <pre><code>df = pd.DataFrame({\"num\": [8, 9], \"letter\": [\"dd\", \"ee\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"append\")\n</code></pre> <p>Here are the files in storage:</p> <pre><code>tmp/some-table\n\u251c\u2500\u2500 0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\n\u251c\u2500\u2500 1-57abb6fb-2249-43ba-a7be-cf09bcc230de-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u251c\u2500\u2500 00000000000000000000.json\n    \u2514\u2500\u2500 00000000000000000001.json\n</code></pre> <p>Here are the contents of the <code>_delta_log/00000000000000000001.json</code> file:</p> <pre><code>{\n  \"add\": {\n    \"path\": \"1-57abb6fb-2249-43ba-a7be-cf09bcc230de-0.parquet\",\n    \"size\": 2204,\n    \"partitionValues\": {},\n    \"modificationTime\": 1701740386169,\n    \"dataChange\": true,\n    \"stats\": \"{\\\"numRecords\\\": 2, \\\"minValues\\\": {\\\"num\\\": 8, \\\"letter\\\": \\\"dd\\\"}, \\\"maxValues\\\": {\\\"num\\\": 9, \\\"letter\\\": \\\"ee\\\"}, \\\"nullCount\\\": {\\\"num\\\": 0, \\\"letter\\\": 0}}\"\n  }\n}\n{\n  \"commitInfo\": {\n    \"timestamp\": 1701740386169,\n    \"operation\": \"WRITE\",\n    \"operationParameters\": {\n      \"partitionBy\": \"[]\",\n      \"mode\": \"Append\"\n    },\n    \"clientVersion\": \"delta-rs.0.17.0\"\n  }\n}\n</code></pre> <p>The transaction log records that the second file has been persisted in the Delta table.</p> <p>Now create a third pandas DataFrame and overwrite the Delta table with the new data.</p> <pre><code>df = pd.DataFrame({\"num\": [11, 22], \"letter\": [\"aa\", \"bb\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"overwrite\")\n</code></pre> <p>Here are the files in storage:</p> <pre><code>tmp/some-table\n\u251c\u2500\u2500 0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\n\u251c\u2500\u2500 1-57abb6fb-2249-43ba-a7be-cf09bcc230de-0.parquet\n\u251c\u2500\u2500 2-95ef2108-480c-4b89-96f0-ff9185dab9ad-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u251c\u2500\u2500 00000000000000000000.json\n    \u251c\u2500\u2500 00000000000000000001.json\n    \u2514\u2500\u2500 00000000000000000002.json\n</code></pre> <p>Here are the contents of the <code>_delta_log/0002.json</code> file:</p> <pre><code>{\n  \"add\": {\n    \"path\": \"2-95ef2108-480c-4b89-96f0-ff9185dab9ad-0.parquet\",\n    \"size\": 2204,\n    \"partitionValues\": {},\n    \"modificationTime\": 1701740465102,\n    \"dataChange\": true,\n    \"stats\": \"{\\\"numRecords\\\": 2, \\\"minValues\\\": {\\\"num\\\": 11, \\\"letter\\\": \\\"aa\\\"}, \\\"maxValues\\\": {\\\"num\\\": 22, \\\"letter\\\": \\\"bb\\\"}, \\\"nullCount\\\": {\\\"num\\\": 0, \\\"letter\\\": 0}}\"\n  }\n}\n{\n  \"remove\": {\n    \"path\": \"0-62dffa23-bbe1-4496-8fb5-bff6724dc677-0.parquet\",\n    \"deletionTimestamp\": 1701740465102,\n    \"dataChange\": true,\n    \"extendedFileMetadata\": false,\n    \"partitionValues\": {},\n    \"size\": 2208\n  }\n}\n{\n  \"remove\": {\n    \"path\": \"1-57abb6fb-2249-43ba-a7be-cf09bcc230de-0.parquet\",\n    \"deletionTimestamp\": 1701740465102,\n    \"dataChange\": true,\n    \"extendedFileMetadata\": false,\n    \"partitionValues\": {},\n    \"size\": 2204\n  }\n}\n{\n  \"commitInfo\": {\n    \"timestamp\": 1701740465102,\n    \"operation\": \"WRITE\",\n    \"operationParameters\": {\n      \"mode\": \"Overwrite\",\n      \"partitionBy\": \"[]\"\n    },\n    \"clientVersion\": \"delta-rs.0.17.0\"\n  }\n}\n</code></pre> <p>This transaction adds a data file and marks the two exising data files for removal. Marking a file for removal in the transaction log is known as \"tombstoning the file\" or a \"logical delete\". This is different from a \"physical delete\" which actually removes the data file from storage.</p>"},{"location":"how-delta-lake-works/architecture-of-delta-table/#how-delta-table-operations-differ-from-data-lakes","title":"How Delta table operations differ from data lakes","text":"<p>Data lakes consist of data files persisted in storage. They don't have a transaction log that retain metadata about the transactions.</p> <p>Data lakes perform transactions differently than Delta tables.</p> <p>When you perform an overwrite transaction with a Delta table, you logically delete the exiting data without physically removing it.</p> <p>Data lakes don't support logical deletes, so you have to physically delete the data from storage.</p> <p>Logical data operations are safer because they can be rolled back if they don't complete successfully. Physically removing data from storage can be dangerous, especially if it's before a transaction is complete.</p> <p>We're now ready to look into Delta Lake ACID transactions in more detail.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/","title":"Delta Lake Transactions","text":"<p>This page teaches you about Delta Lake transactions and why transactions are important in production data settings. Data lakes don\u2019t support transactions and this is a huge downside because they offer a poor user experience, lack functionality, and can easily be corrupted.</p> <p>Transactions on Delta Lake tables are operations that change the state of table and record descriptive entries (metadata) of those changes to the Delta Lake transaction log. Here are some examples of transactions:</p> <ul> <li>Deleting rows</li> <li>Appending to the table</li> <li>Compacting small files</li> <li>Upserting</li> <li>Overwriting rows</li> </ul> <p>All Delta Lake write operations are transactions in Delta tables. Reads actually aren\u2019t technically transactions because they don\u2019t result in new entries being appended to the transaction log.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#what-are-transactions","title":"What are transactions?","text":"<p>Transactions are any Delta operation that change the underlying files of a Delta table and result in new entries metadata entries in the transaction log. Some Delta operations rearrange data in the existing table (like Z Ordering the table or compacting the small files) and these are also transactions. Let\u2019s look at a simple example.</p> <p>Suppose you have a Delta table with the following data:</p> <pre><code>num animal\n1   cat\n2   dog\n3   snake\n</code></pre> <p>Here\u2019s how to create this Delta table:</p> <pre><code>import pandas as pd\nfrom deltalake import write_deltalake, DeltaTable\n\ndf = pd.DataFrame({\"num\": [1, 2, 3], \"animal\": [\"cat\", \"dog\", \"snake\"]})\nwrite_deltalake(\"tmp/my-delta-table\", df)\n</code></pre> <p>Here are the files created in storage.</p> <pre><code>tmp/my-delta-table\n\u251c\u2500\u2500 0-fea2de92-861a-423e-9708-a9e91dafb27b-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u2514\u2500\u2500 00000000000000000000.json\n</code></pre> <p>Let\u2019s perform an operation to delete every animal from the Delta table that is a cat.</p> <pre><code>dt = DeltaTable(\"tmp/my-delta-table\")\ndt.delete(\"animal = 'cat'\")\n</code></pre> <p>Let\u2019s take a look at the contents of the Delta table now that the transaction is complete:</p> <pre><code>tmp/my-delta-table\n\u251c\u2500\u2500 0-fea2de92-861a-423e-9708-a9e91dafb27b-0.parquet\n\u251c\u2500\u2500 _delta_log\n\u2502   \u251c\u2500\u2500 00000000000000000000.json\n\u2502   \u2514\u2500\u2500 00000000000000000001.json\n\u2514\u2500\u2500 part-00001-90312b96-b487-4a8f-9edc-1b9b3963f136-c000.snappy.parquet\n</code></pre> <p>Notice the <code>00000000000000000001.json</code> file that was added to the transaction log to record this transaction. Let\u2019s inspect the content of the file.</p> <pre><code>{\n  \"add\": {\n    \"path\": \"part-00001-90312b96-b487-4a8f-9edc-1b9b3963f136-c000.snappy.parquet\",\n    \"partitionValues\": {},\n    \"size\": 858,\n    \"modificationTime\": 1705070631953,\n    \"dataChange\": true,\n    \"stats\": \"{\\\"numRecords\\\":2,\\\"minValues\\\":{\\\"num\\\":2,\\\"animal\\\":\\\"dog\\\"},\\\"maxValues\\\":{\\\"num\\\":3,\\\"animal\\\":\\\"snake\\\"},\\\"nullCount\\\":{\\\"num\\\":0,\\\"animal\\\":0}}\",\n    \"tags\": null,\n    \"deletionVector\": null,\n    \"baseRowId\": null,\n    \"defaultRowCommitVersion\": null,\n    \"clusteringProvider\": null\n  }\n}\n{\n  \"remove\": {\n    \"path\": \"0-fea2de92-861a-423e-9708-a9e91dafb27b-0.parquet\",\n    \"dataChange\": true,\n    \"deletionTimestamp\": 1705070631953,\n    \"extendedFileMetadata\": true,\n    \"partitionValues\": {},\n    \"size\": 895\n  }\n}\n{\n  \"commitInfo\": {\n    \"timestamp\": 1705070631953,\n    \"operation\": \"DELETE\",\n    \"operationParameters\": {\n      \"predicate\": \"animal = 'cat'\"\n    },\n    \"readVersion\": 0,\n    \"operationMetrics\": {\n      \"execution_time_ms\": 8013,\n      \"num_added_files\": 1,\n      \"num_copied_rows\": 2,\n      \"num_deleted_rows\": 1,\n      \"num_removed_files\": 1,\n      \"rewrite_time_ms\": 2,\n      \"scan_time_ms\": 5601\n    },\n    \"clientVersion\": \"delta-rs.0.17.0\"\n  }\n}\n</code></pre> <p>We can see that this transaction includes two components:</p> <ul> <li>Remove file <code>0-fea2de92-861a-423e-9708-a9e91dafb27b-0.parquet</code></li> <li>Add file <code>part-00001-90312b96-b487-4a8f-9edc-1b9b3963f136-c000.snappy.parquet</code></li> </ul> <p>Transactions are recorded in the transaction log. The transaction log is also referred to as the table metadata and is the <code>_delta_log</code> directory in storage.</p> <p>Let\u2019s see how Delta Lake implements transactions.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#how-delta-lake-implements-transactions","title":"How Delta Lake implements transactions","text":"<p>Here is how Delta Lake implements transactions:</p> <ol> <li>Read the existing metadata</li> <li>Read the existing Parquet data files</li> <li>Write the Parquet files for the current transaction</li> <li>Record the new transaction in the transaction log (if there are no conflicts)</li> </ol> <p>Let\u2019s recall our delete operation from the prior section and see how it fits into this transaction model:</p> <ol> <li>We read the existing metadata to find the file paths for the existing Parquet files</li> <li>We read the existing Parquet files and identify the files that contains data that should be removed</li> <li>We write new Parquet files with the deleted data filtered out</li> <li>Once the new Parquet files are written, we check for conflicts and then make an entry in the transaction log. The next section will discuss transaction conflicts in more detail.</li> </ol> <p>Blind append operations can skip a few steps and are executed as follows:</p> <ol> <li>Write the Parquet files for the current transaction</li> <li>Record the new transaction in the metadata</li> </ol> <p>Delta implements a non locking MVCC (multi version concurrency control) so writers optimistically write new data and simply abandon the transaction if it conflicts at the end. The alternative would be getting a lock at the start thereby guaranteeing the transaction immediately.</p> <p>Let\u2019s look at the case when a Delta Lake transaction conflicts.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#how-delta-lake-transactions-can-conflict","title":"How Delta Lake transactions can conflict","text":"<p>Suppose you have a transaction that deletes a row of data that\u2019s stored in FileA (Transaction 1). While this job is running, there is another transaction that deletes some other rows in FileA (Transaction 2). Transaction 1 finishes running first and is recorded in the metadata.</p> <p>Before Transaction 2 is recorded as a transaction, it will check the metadata, find that Transaction 2 conflicts with a transaction that was already recorded (from Transaction 1), and error without recording a new transaction.</p> <p>Transactions 2 will write Parquet data files, but will not be recorded as a transaction, so the data files will be ignored. The zombie Parquet files can be easily cleaned up via subsequent vacuum operations.</p> <p>Transaction 2 must fail otherwise it would cause the data to be incorrect.</p> <p>Delta Lake transactions prevent users from making changes that would corrupt the table. Transaction conflict behavior can differ based on isolation level, which controls the degree to which a transaction must be isolated from modifications made by other concurrent transactions. More about this in the concurrency section.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#transactions-rely-on-atomic-primitives-storage-guarantees","title":"Transactions rely on atomic primitives storage guarantees","text":"<p>Suppose you have two transactions that are finishishing at the same exact time. Both of these transactions look at the existing Delta Lake transaction log, see that the latest transaction was <code>003.json</code> and determine that the next entry should be <code>004.json</code>.</p> <p>If both transactions are recorded in the <code>004.json</code> file, then one of them will be clobbered, and the transaction log entry for the clobbered metadata entry will be lost.</p> <p>Delta tables rely on storage systems that provide atomic primitives for safe concurrency. The storage system must allow Delta Lake to write the file, only if it does not exist already, and error out otherwise. The storage system must NOT permit concurrent writers to overwrite existing metadata entries.</p> <p>Some clouds have filesystems that don\u2019t explicitly support these atomic primitives, and therefore must be coupled with other services to provide the necessary guarantees.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#delta-lake-transactions-are-only-for-a-single-table","title":"Delta Lake transactions are only for a single table","text":"<p>Delta Lake transactions are only valid for a single table.</p> <p>Some databases offer transaction support for operations that impact multiple tables. Delta Lake does not support multi-table transactions.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#data-lakes-dont-support-transactions","title":"Data lakes don\u2019t support transactions","text":"<p>Data lakes consist of many files in a storage system (e.g. a cloud storage system) and don\u2019t support transactions.</p> <p>Data lakes don\u2019t have a metadata layer, conflict resolution, or any way to store information about transactions.</p> <p>Data lakes are prone to multiple types of errors because they don\u2019t support transactions:</p> <ul> <li>Easy to corrupt</li> <li>Downtime/unstable state while jobs are running</li> <li>Operations can conflict</li> </ul> <p>Data lakes have many downsides and it\u2019s almost always better to use a lakehouse storage system like Delta Lake compared to a data lake.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#acid-transactions","title":"ACID Transactions","text":"<p>We\u2019ve already explored how Delta Lake supports transactions. This section explains how Delta Lake transactions have the Atomic, Consistent, Isolated and Durable (ACID transaction) properties. Reading this section is optional.</p> <p>ACID transactions are commonplace in databases but notably absent for data lakes.</p> <p>Delta Lake\u2019s ACID transaction support is one of the major reasons it is almost always a better option than a data lake.</p> <p>Let\u2019s explore how Delta Lake allows for ACID transactions.</p> <p>Atomic transactions</p> <p>An atomic transaction either fully completes or fully fails, with nothing in between.</p> <p>Delta Lake transactions are atomic, unlike data lake transactions that are not atomic.</p> <p>Suppose you have a job that\u2019s writing 100 files to a table. Further suppose that the job errors out and the cluster dies after writing 40 files:</p> <ul> <li>For a Delta table, no additional data will be added to the table. Parquet files were written to the table, but the job errored, so no transaction log entry was added and no data was added to the table.</li> <li>For a data lake, the 40 files are added and the transaction \u201cpartially succeeds\u201d.</li> </ul> <p>For data tables, it\u2019s almost always preferable to have a transaction that \u201cfully fails\u201d instead of one that \u201cpartially succeeds\u201d because partial writes are hard to unwind and debug.</p> <p>Delta Lake implements atomic transactions by writing data files first before making a new entry in the Delta transaction log.</p> <p>These guarantees are provided at the protocol level through the \"transaction\" abstraction. We\u2019ve already discussed what constitutes a transaction for Delta Lake.</p> <p>If there is an error with the transaction and some files don\u2019t get written, then no metadata entry is made and the partial data write is ignored. The zombie Parquet files can be easily cleaned up via subsequent vacuum operations.</p> <p>Now let\u2019s look at how Delta Lake also provides consistent transactions.</p> <p>Consistent transactions</p> <p>Consistency means that transactions won\u2019t violate integrity constraints on the Delta table.</p> <p>Delta Lake has two types of consistency checks:</p> <ul> <li>Schema enforcement checks</li> <li>Column constraints</li> </ul> <p>Schema enforcement checks verify that new data appended to a Delta table matches the schema of the existing table. You cannot append data with a different schema, unless you enable schema evolution.</p> <p>Delta Lake column constraints allow users to specify the requirements of data that\u2019s added to a Delta table. For example, if you have an age column with a constraint that requires the value to be positive, then Delta Lake will reject appends of any data that doesn\u2019t meet the constraint.</p> <p>Data lakes don\u2019t support schema enforcement or column constraints. That\u2019s another reason why data lakes are not ACID-compliant.</p> <p>Isolated transactions</p> <p>Isolation means that transactions are applied to a Delta table sequentially.</p> <p>Delta Lake transactions are persisted in monotonically increasing transaction files, as we saw in the previous example. First <code>00000000000000000000.json</code>, then <code>00000000000000000001.json</code>, then <code>00000000000000000002.json</code>, and so on.</p> <p>Delta Lake uses concurrency control to ensure that transactions are executed sequentially, even when user operations are performed concurrently. The next page of this guide explains concurrency in Delta Lake in detail.</p> <p>Durable transactions</p> <p>Delta tables are generally persisted in cloud object stores which provide durability guarantees.</p> <p>Durability means that all transactions that are successfully completed will always remain persisted, even if there are service outages or program crashes.</p> <p>Suppose you have a Delta table that\u2019s persisted in Azure blob storage. The Delta table transactions that are committed will always remain available, even in these circumstances:</p> <ul> <li>When there are Azure service outages</li> <li>If a computation cluster that\u2019s writing the Delta table crashes for some reason</li> <li>Two operations are running concurrently and one of them fails</li> </ul> <p>Successful transactions are always registered in the Delta table and persisted no matter what.</p>"},{"location":"how-delta-lake-works/delta-lake-acid-transactions/#conclusion","title":"Conclusion","text":"<p>Delta Lake supports transactions which provide necessary reliability guarantees for production data systems.</p> <p>Vanilla data lakes don\u2019t provide transactions and this can cause nasty bugs and a bad user experience. Let\u2019s look at a couple of scenarios when the lack of transactions cause a poor user experience:</p> <ul> <li>While running a compaction operation on a data lake, newly compacted \u201cright sized\u201d files are added before the small files are deleted. If you read the data lake while this operation is running, you will see duplicate data.</li> <li>While writing to a data lake, a job might fail, which leaves behind partially written files. These files are corrupt, which means that the data lake cannot be read until the corrupt files are manually removed.</li> <li>Users want to run a simple DML operation like deleting a few rows of data which require a few files to be rewritten. This operation renders the data lake unusable until it\u2019s done running.</li> </ul> <p>Transactions are a key advantage of Delta Lake vs. data lakes. There are many other advantages, but proper transactions are necessary in production data environments.</p> <p>Let's take a look at File Skipping in the next section.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/","title":"Delta Lake File Skipping","text":"<p>Delta tables store file-level metadata information, which allows for a powerful optimization called file skipping.</p> <p>This page explains how Delta Lake implements file skipping, how to optimize your tables to maximize file skipping, and the benefits of file skipping.</p> <p>Let\u2019s start by looking at the file-level metadata in Delta tables.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#delta-lake-file-metadata","title":"Delta Lake file metadata","text":"<p>Delta Lake stores metadata about each file's min/max values in the table.  Query engines can skip entire files when they don\u2019t contain data that\u2019s relevant to the query.</p> <p>Suppose you have a Delta table with data stored in two files and has the following metadata.</p> <pre><code>filename    min_name    max_name    min_age max_age\nfileA       alice       joy         12      46  \nfileB       allan       linda       34      78\n</code></pre> <p>Suppose you want to run the following query: <code>select * from the_table where age &amp;lt; 20</code>.</p> <p>The engine only needs to read <code>fileA</code> to execute this query.  <code>fileB</code> has a <code>min_age</code> of 34, so we know there aren\u2019t any rows of data with an <code>age</code> less than 20.</p> <p>The benefit of file skipping depends on the query and the data layout of the Delta table.  Some queries cannot take advantage of any file skipping.  Here\u2019s an example query that does not benefit from file skipping: <code>select * from the_table group by age</code>.</p> <p>Let\u2019s recreate this example with Polars to drive the point home.</p> <p>Start by writing out one file of data:</p> <pre><code>import polars as pl\nfrom deltalake import DeltaTable\n\ndf = pl.DataFrame({\"name\": [\"alice\", \"cat\", \"joy\"], \"age\": [12, 35, 46]})\ndf.write_delta(\"tmp/a_table\")\n</code></pre> <p>Now, write out another file of data:</p> <pre><code>df = pl.DataFrame({\"name\": [\"allan\", \"brian\", \"linda\"], \"age\": [34, 35, 78]})\ndf.write_delta(\"tmp/a_table\", mode=\"append\")\n</code></pre> <p>Here are the contents of the Delta table:</p> <pre><code>tmp/a_table\n\u251c\u2500\u2500 0-7d414a88-a634-4c2f-9c5b-c29b6ee5f524-0.parquet\n\u251c\u2500\u2500 1-0617ef60-b17b-46a5-9b0f-c7dda1b73eee-0.parquet\n\u2514\u2500\u2500 _delta_log\n    \u251c\u2500\u2500 00000000000000000000.json\n    \u2514\u2500\u2500 00000000000000000001.json\n</code></pre> <p>Now run a query to fetch all the records where the age is less than 20:</p> <pre><code>pl.scan_delta(\"tmp/a_table\").filter(pl.col(\"age\") &lt; 20).collect()\n</code></pre> <pre><code>+-------+-----+\n| name  | age |\n| ---   | --- |\n| str   | i64 |\n+=============+\n| alice | 12  |\n+-------+-----+\n</code></pre> <p>Polars can use the Delta table metadata to skip the file that does not contain data relevant to the query.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#how-delta-lake-implements-file-skipping","title":"How Delta Lake implements file skipping","text":"<p>Here\u2019s how engines execute queries on Delta tables:</p> <ul> <li>Start by reading the transaction log to get the file paths, file sizes, and min/max value for each column</li> <li>Parse the query and push down the predicates to skip files</li> <li>Read the minimal subset of the files needed for the query</li> </ul> <p>Some file formats don\u2019t allow for file skipping.  For example, CSV files don\u2019t have file-level metadata, so query engines can\u2019t read a minimal subset of the data.  The query engine has to check all the files, even if they don\u2019t contain any relevant data.</p> <p>When data is in Parquet files, the query engine can open up all the files, read the footers, build the file-level metadata, and perform file skipping.  Fetching metadata in each file is slower than grabbing the pre-built file-level metadata from the transaction log.</p> <p>Now, let\u2019s see how to structure your tables to allow for more file skipping.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#file-skipping-for-different-file-sizes","title":"File skipping for different file sizes","text":"<p>Delta tables store data in files.  Smaller files allow for more file skipping compared to bigger files.</p> <p>However, an excessive number of small files isn\u2019t good because it creates I/O overhead and slows down queries.</p> <p>Your Delta tables should have files that are \u201cright-sized\u201d.  For a table with 150 GB of data, 5 GB files would probably be too large, and 10 KB files would be too small.  It\u2019s generally best to store data in files that are between 100 MB and 1 GB.</p> <p>Delta Lake has an optimize function that performs small file compaction, so you don\u2019t need to program this logic yourself.</p> <p>Now, let's investigate how to store data in files to maximize the file skipping opportunities.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#how-to-maximize-file-skipping","title":"How to maximize file skipping","text":"<p>You can maximize file-skipping by colocating similar data in the same files.</p> <p>Suppose you have a table with test scores and frequently run queries that filter based on the <code>test_score</code> column.</p> <pre><code>filename    min_test_score  max_test_score\nfileA       45              100\nfileB       65              98\nfileC       50              96\n</code></pre> <p>Suppose you want to run the following query: <code>select * from exams where test_score &gt; 90</code>.</p> <p>This query cannot skip files, given the current organization of the data.  You can rearrange the data to colocate similar test scores in the same files to allow for file skipping.  Here\u2019s the new layout: </p> <pre><code>filename    min_test_score  max_test_score\nfileD       45              70\nfileE       55              80\nfileF       78              100\n</code></pre> <p>The query (<code>select * from exams where test_score &gt; 90</code>) can skip two of the three files with the new Delta table layout.  The query engine only has to read <code>fileF</code> for this query.</p> <p>Now, let\u2019s look at how file skipping works with string values.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#how-file-skipping-works-with-strings","title":"How file skipping works with strings","text":"<p>File skipping is also effective when filtering on string values.</p> <p>Suppose you have a table with <code>person_name</code> and <code>country</code> columns.  There are millions of rows of data.  Here are the first three rows of data:</p> <pre><code>person_name country\nperson1     angola\nperson2     china\nperson3     mexico\n</code></pre> <p>The Delta table contains three files with the following metadata:</p> <pre><code>filename    min_country max_country\nfileA       albania     mali\nfileB       libia       paraguay\nfileC       oman        zimbabwe\n</code></pre> <p>Suppose you want to run the following query: <code>select * from some_people where country = 'austria'</code>.</p> <p>You only need to read the data in <code>fileA</code> to run this query.  The <code>min_country</code> value for <code>fileB</code> and <code>fileC</code> are greater than \u201caustria\u201d, so we know those files don\u2019t contain any data relevant to the query.</p> <p>File skipping can also be a robust optimization for string values.  Now, let\u2019s see how file skipping works for partitioned tables.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#file-skipping-for-partitioned-tables","title":"File skipping for partitioned tables","text":"<p>You can partition Delta tables for file skipping as well.  Suppose we have the same data as in the previous section, but the table is partitioned by <code>country</code>.</p> <p>Here\u2019s the Delta table:</p> <pre><code>filename    partition\nfileA       albania\nfileB       libia\nfileC       oman\nfileD       jamaica\nfileE       albania\nfileF       oman\n</code></pre> <p>Suppose you want to run the following query on this partitioned table: <code>select * from some_partitioned_table where country = 'albania'</code>.</p> <p>You only need to read <code>fileA</code> and <code>fileE</code> to execute this query.  Delta Lake provides the file-level partition metadata in the transaction log so that this query will run quickly.</p>"},{"location":"how-delta-lake-works/delta-lake-file-skipping/#conclusion","title":"Conclusion","text":"<p>Delta Lake allows for file skipping, which is a powerful performance optimization.</p> <p>Delta Lake also provides built-in utilities to colocate data in the same files like partitioning, Z Ordering, and compaction to improve file skipping.</p> <p>Delta Lake users need to know how to assess the tradeoffs of these techniques to optimize file skipping.  Users also need to understand the most frequent query patterns of their tables to best allow for file maximal file skipping.</p>"},{"location":"integrations/delta-lake-arrow/","title":"Delta Lake Arrow Integrations","text":"<p>Delta Lake tables can be exposed as Arrow tables and Arrow datasets, which allows for interoperability with a variety of query engines.</p> <p>This page shows you how to convert Delta tables to Arrow data structures and teaches you the difference between Arrow tables and Arrow datasets.  Tables are \"eager\" and datasets are \"lazy\", which has important performance implications, keep reading to learn more!</p>"},{"location":"integrations/delta-lake-arrow/#delta-lake-to-arrow-dataset","title":"Delta Lake to Arrow Dataset","text":"<p>Delta tables can easily be exposed as Arrow datasets.  This makes it easy for any query engine that can read Arrow datasets to read a Delta table.</p> <p>Let's take a look at the h2o groupby dataset that contains 9 columns of data.  Here are three representative rows of data:</p> <pre><code>+-------+-------+--------------+-------+-------+--------+------+------+---------+\n| id1   | id2   | id3          |   id4 |   id5 |    id6 |   v1 |   v2 |      v3 |\n|-------+-------+--------------+-------+-------+--------+------+------+---------|\n| id016 | id046 | id0000109363 |    88 |    13 | 146094 |    4 |    6 | 18.8377 |\n| id039 | id087 | id0000466766 |    14 |    30 | 111330 |    4 |   14 | 46.7973 |\n| id047 | id098 | id0000307804 |    85 |    23 | 187639 |    3 |    5 | 47.5773 |\n+-------+-------+--------------+-------+-------+--------+------+------+---------+\n</code></pre> <p>Here's how to expose the Delta table as a PyArrow dataset and run a query with DuckDB:</p> <pre><code>import duckdb\nfrom deltalake import DeltaTable\n\ntable = DeltaTable(\"delta/G1_1e9_1e2_0_0\")\ndataset = table.to_pyarrow_dataset()\nquack = duckdb.arrow(dataset)\nquack.filter(\"id1 = 'id016' and v2 &gt; 10\")\n</code></pre> <p>Here's the result:</p> <pre><code>\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502   id1   \u2502   id2   \u2502     id3      \u2502  id4  \u2502  id5  \u2502   id6   \u2502  v1   \u2502  v2   \u2502    v3     \u2502\n\u2502 varchar \u2502 varchar \u2502   varchar    \u2502 int32 \u2502 int32 \u2502  int32  \u2502 int32 \u2502 int32 \u2502  double   \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 id016   \u2502 id054   \u2502 id0002309114 \u2502    62 \u2502    95 \u2502 7180859 \u2502     4 \u2502    13 \u2502  7.750173 \u2502\n\u2502 id016   \u2502 id044   \u2502 id0003968533 \u2502    63 \u2502    98 \u2502 2356363 \u2502     4 \u2502    14 \u2502  3.942417 \u2502\n\u2502 id016   \u2502 id034   \u2502 id0001082839 \u2502    58 \u2502    73 \u2502 8039808 \u2502     5 \u2502    12 \u2502 76.820135 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 ? rows (&gt;9999 rows, 3 shown)                                                 9 columns \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n</code></pre> <p>Arrow datasets allow for the predicates to get pushed down to the query engine, so the query is executed quickly.</p>"},{"location":"integrations/delta-lake-arrow/#delta-lake-to-arrow-table","title":"Delta Lake to Arrow Table","text":"<p>You can also run the same query with DuckDB on an Arrow table:</p> <pre><code>quack = duckdb.arrow(table.to_pyarrow_table())\nquack.filter(\"id1 = 'id016' and v2 &gt; 10\")\n</code></pre> <p>This returns the same result, but it runs slower.</p>"},{"location":"integrations/delta-lake-arrow/#difference-between-arrow-dataset-and-arrow-table","title":"Difference between Arrow Dataset and Arrow Table","text":"<p>Arrow Datasets are lazy and allow for full predicate pushdown unlike Arrow tables which are eagerly loaded into memory.</p> <p>The previous DuckDB queries were run on a 1 billion row dataset that's roughly 50 GB when stored as an uncompressed CSV file.  Here are the runtimes when the data is stored in a Delta table and the queries are executed on a 2021 Macbook M1 with 64 GB of RAM:</p> <ul> <li>Arrow table: 17.1 seconds</li> <li>Arrow dataset: 0.01 seconds</li> </ul> <p>The query runs much faster on an Arrow dataset because the predicates can be pushed down to the query engine and lots of data can be skipped.</p> <p>Arrow tables are eagerly materialized in memory and don't allow for the same amount of data skipping.</p>"},{"location":"integrations/delta-lake-arrow/#multiple-query-engines-can-query-arrow-datasets","title":"Multiple query engines can query Arrow Datasets","text":"<p>Other query engines like DataFusion can also query Arrow datasets, see the following example:</p> <pre><code>from datafusion import SessionContext\n\nctx = SessionContext()\nctx.register_dataset(\"my_dataset\", table.to_pyarrow_dataset())\nctx.sql(\"select * from my_dataset where v2 &gt; 5\")\n</code></pre> <p>Here's the result:</p> <pre><code>+-------+-------+--------------+-----+-----+--------+----+----+-----------+\n| id1   | id2   | id3          | id4 | id5 | id6    | v1 | v2 | v3        |\n+-------+-------+--------------+-----+-----+--------+----+----+-----------+\n| id082 | id049 | id0000022715 | 97  | 55  | 756924 | 2  | 11 | 74.161136 |\n| id053 | id052 | id0000113549 | 19  | 56  | 139048 | 1  | 10 | 95.178444 |\n| id090 | id043 | id0000637409 | 94  | 50  | 12448  | 3  | 12 | 60.21896  |\n+-------+-------+--------------+-----+-----+--------+----+----+-----------+\n</code></pre> <p>Any query engine that's capable of reading an Arrow table/dataset can read a Delta table.</p>"},{"location":"integrations/delta-lake-arrow/#conclusion","title":"Conclusion","text":"<p>Delta tables can easily be exposed as Arrow tables/datasets.</p> <p>Therefore any query engine that can read an Arrow table/dataset can also read a Delta table.</p> <p>Arrow datasets allow for more predicates to be pushed down to the query engine, so they can perform better performance than Arrow tables.</p>"},{"location":"integrations/delta-lake-daft/","title":"Using Delta Lake with Daft","text":"<p>Daft is a framework for ETL, analytics, and ML/AI at scale with a familiar Python dataframe API, implemented in Rust.</p> <p>Daft and Delta Lake work really well together. Daft provides unified compute for Delta Lake\u2019s unified storage. Together, Delta Lake and Daft give you high-performance query optimization and distributed compute on massive datasets.</p> <p>Delta Lake is a great storage format for Daft workloads. Delta gives Daft users:</p> <ul> <li>Query optimization via file-skipping and column pruning</li> <li>Versioning for easy time travel functionality</li> <li>Faster reads via Z-ordering</li> <li>ACID transactions and schema enforcement for more reliable reads and writes</li> </ul> <p>For Delta Lake users, Daft is a great data processing tool because it gives you the following features:</p> <ul> <li>Multimodal Dataframes: read, write and transform multimodal data incl. images, JSON, PDF, audio, etc.</li> <li>Parallel + Distributed Reads: Daft parallelizes Delta Lake table reads over all cores of your machine, if using the default multithreading runner, or all cores + machines of your Ray cluster, if using the distributed Ray runner.</li> <li>Skipping Filtered Data: Daft implements automatic partition pruning and stats-based file pruning for filter predicates, skipping data that doesn\u2019t need to be read.</li> </ul> <p>Let's look at how to use Delta Lake with Daft.</p>"},{"location":"integrations/delta-lake-daft/#installing-daft-for-delta-lake","title":"Installing Daft for Delta Lake","text":"<p>The easiest way to use the Delta Lake table format with Daft DataFrames is to install Daft with the <code>[deltalake]</code> extras using <code>pip</code>:</p> <pre><code>!pip install -U \"getdaft[deltalake]\"\n</code></pre> <p>This adds the <code>deltalake</code> Python package to your install. This package is used to fetch metadata about the Delta Lake table, such as paths to the underlying Parquet files and table statistics. You can of course also install the <code>deltalake</code> manually.</p>"},{"location":"integrations/delta-lake-daft/#read-delta-lake-into-a-daft-dataframe","title":"Read Delta Lake into a Daft DataFrame","text":"<p>You can easily read Delta Lake tables into a Daft DataFrame using the <code>read_delta_lake</code> method. Let's use it to read some data stored in a Delta Lake on disk. You can access the data stored as a Delta Lake on Github</p> <pre><code>import daft\n\n# read delta table into Daft DataFrame\ndf = daft.read_delta_lake(\"path/to/delta_table\")\n</code></pre> <p>You can also read in Delta Lake data from remote sources like S3:</p> <pre><code># table_uri = (\n#     \"s3://daft-public-datasets/red-pajamas/\"\n#     \"stackexchange-sample-north-germanic-deltalake\"\n# )\n# df = daft.read_delta_lake(table_uri)\n</code></pre> <pre><code>df\n</code></pre> first_nameUtf8last_nameUtf8countryUtf8continentUtf8 (No data to display: Dataframe not materialized) <p>Daft DataFrames are lazy by default. This means that the contents will not be computed (\"materialized\") unless you explicitly tell Daft to do so. This is best practice for working with larger-than-memory datasets and parallel/distributed architectures.</p> <p>The Delta table we have just loaded only has 5 rows. You can materialize it in memory using the <code>.collect</code> method:</p> <pre><code>&gt; df.collect()\n\n|    | first_name   | last_name   | country   | continent   |\n|---:|:-------------|:------------|:----------|:------------|\n|  0 | Ernesto      | Guevara     | Argentina | NaN         |\n|  1 | Bruce        | Lee         | China     | Asia        |\n|  2 | Jack         | Ma          | China     | Asia        |\n|  3 | Wolfgang     | Manche      | Germany   | NaN         |\n|  4 | Soraya       | Jala        | Germany   | NaN         |\n</code></pre>"},{"location":"integrations/delta-lake-daft/#write-to-delta-lake","title":"Write to Delta Lake","text":"<p>You can use <code>write_deltalake</code> to write a Daft DataFrame to a Delta table:</p> <pre><code>df.write_deltalake(\"tmp/daft-table\", mode=\"overwrite\")\n</code></pre> <p>Daft supports multiple write modes. See the Daft documentation for more information.</p>"},{"location":"integrations/delta-lake-daft/#what-can-i-do-with-a-daft-dataframe","title":"What can I do with a Daft DataFrame?","text":"<p>Daft gives you full-featured DataFrame functionality, similar to what you might be used to from pandas, Dask or PySpark.</p> <p>On top of this, Daft also gives you:</p> <ul> <li>Multimodal data type support to work with Images, URLs, Tensors and more</li> <li>Expressions API for easy column transformations</li> <li>UDFs for multi-column transformation, incl. ML applications</li> </ul> <p>Check out the Daft User Guide for a complete list of DataFrame operations.</p>"},{"location":"integrations/delta-lake-daft/#data-skipping-optimizations","title":"Data Skipping Optimizations","text":"<p>Delta Lake and Daft work together to give you highly-optimized query performance.</p> <p>Delta Lake stores your data in Parquet files. Parquet is a columnar row format that natively supports column pruning. If your query only needs to read data from a specific column or set of columns, you don't need to read in the entire file. This can save you lots of time and compute.</p> <p>Delta Lake goes beyond the basic Parquet features by also giving you:</p> <ul> <li>partitioned reads</li> <li>file skipping via z-ordering.</li> </ul> <p>This is great for Daft users who want to run efficient queries on large-scale data.</p> <p>Let's look at how this works.</p>"},{"location":"integrations/delta-lake-daft/#partitioned-reads","title":"Partitioned Reads","text":"<p>You may have noticed the Delta Lake warning at the top when we first called <code>collect()</code> on our DataFrame:</p> <p><code>WARNING: has partitioning keys = [PartitionField(country#Utf8)], but no partition filter was specified. This will result in a full table scan.</code></p> <p>Delta Lake is informing us that the data is partitioned on the <code>country</code> column.</p> <p>Daft does some nice magic here to help you out. The Daft query optimizer has access to all of the Delta Lake metadata. This means it can optimize your query by skipping the partitions that are not relevant for this query. Instead of having to read all 3 partitions, we can read only 1 and get the same result, just faster!</p> <pre><code># Filter on partition columns will result in efficient partition pruning; non-matching partitions will be skipped.\n&gt; df.where(df[\"country\"] == \"Germany\").show()\n\n|    | first_name   | last_name   | country   |   continent |\n|---:|:-------------|:------------|:----------|------------:|\n|  0 | Wolfgang     | Manche      | Germany   |         nan |\n|  1 | Soraya       | Jala        | Germany   |         nan |\n</code></pre> <p>You can use the <code>explain()</code> method to see how Daft is optimizing your query.</p> <p>Since we've already called <code>collect</code> on our DataFrame, it is already in memory. So below we copy the output of <code>explain(show_all=True)</code> before calling <code>collect</code>:</p> <p>Running <code>df.where(df[\"continent\"] == \"Asia\").explain(True)</code> returns:</p> <pre><code>(...)\n\n== Optimized Logical Plan ==\n\n* PythonScanOperator: DeltaLakeScanOperator(None)\n|   File schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8\n|   Partitioning keys = [PartitionField(country#Utf8)]\n|   Filter pushdown = col(continent) == lit(\"Asia\")\n|   Output schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8\n\n\n== Physical Plan ==\n\n* TabularScan:\n|   Num Scan Tasks = 3\n|   Estimated Scan Bytes = 3045\n|   Clustering spec = { Num partitions = 3 }\n</code></pre> <p>Whereas running <code>df.where(df[\"country\"] == \"Germany\").explain(True)</code> returns:</p> <pre><code>(...)\n\n== Optimized Logical Plan ==\n\n* PythonScanOperator: DeltaLakeScanOperator(None)\n|   File schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8\n|   Partitioning keys = [PartitionField(country#Utf8)]\n|   Partition Filter = col(country) == lit(\"Germany\")\n|   Output schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8\n\n\n== Physical Plan ==\n\n* TabularScan:\n|   Num Scan Tasks = 1\n|   Estimated Scan Bytes = 1025\n|   Clustering spec = { Num partitions = 1 }\n</code></pre> <p>Running a query on a non-partitioned column like <code>continent</code> will require reading in all partitions, totalling 3045 bytes in the case of this toy example.</p> <p>Instead, running a query on a partitioned column (<code>country</code> in this case) means Daft only has to read only the relevant partition, saving us a ~60% of the compute. This has huge impacts when you're working at scale.</p>"},{"location":"integrations/delta-lake-daft/#z-ordering-for-enhanced-file-skipping","title":"Z-Ordering for enhanced file skipping","text":"<p>Z-ordering stores similar data close together to optimize query performance. This is especially useful when you're querying on one or multiple columns.</p> <p>Using Z-Ordered Delta tables instead of regular Parquet can give Daft users significant speed-ups.</p> <p>Read High-Performance Querying on Massive Delta Lake Tables with Daft for an in-depth benchmarking of query optimization with Delta Lake and Daft using partitioning and Z-ordering.</p>"},{"location":"integrations/delta-lake-daft/#daft-gives-you-multimodal-data-type-support","title":"Daft gives you Multimodal Data Type Support","text":"<p>Daft has a rich multimodal type-system with support for Python objects, Images, URLs, Tensors and more.</p> <p>The Expressions API provides useful tools to work with these data types. By combining multimodal data support with the User-Defined Functions API you can run ML workloads right within your DataFrame.</p> <p>Take a look at the notebook in the <code>delta-examples</code> Github repository for a closer look at how Daft handles URLs, images and ML applications.</p>"},{"location":"integrations/delta-lake-daft/#contribute-to-daft","title":"Contribute to <code>daft</code>","text":"<p>Excited about Daft and want to contribute? Join them on Github \ud83d\ude80</p> <p>Like many technologies, Daft collects some non-identifiable telemetry to improve the product. This is stricly non-identifiable metadata. You can disable telemetry by setting the following environment variable: <code>DAFT_ANALYTICS_ENABLED=0</code>. Read more in the Daft documentation.</p>"},{"location":"integrations/delta-lake-dagster/","title":"Using Delta Lake with Dagster\u00b6","text":"<p>Delta Lake is a great storage format for Dagster workflows. This page will explain why and how to use Delta Lake with Dagster.</p> <p>You will learn how to use the Delta Lake I/O Manager to read and write your Dagster Software-Defined Assets (SDAs). You will also learn about the unique advantages Delta Lake offers the Dagster community.</p> <p>Here are some of the benefits that Delta Lake provides Dagster users: - native PyArrow integration for lazy computation of large datasets, - more efficient querying with file skipping via Z Ordering and liquid clustering - built-in vacuuming to remove unnecessary files and versions - ACID transactions for reliable writes - smooth versioning integration so that versions can be use to trigger downstream updates. - surfacing table stats based on the file statistics </p>"},{"location":"integrations/delta-lake-dagster/#dagster-io-managers","title":"Dagster I/O Managers","text":"<p>Dagster uses I/O Managers to simplify data reads and writes. I/O Managers help you reduce boilerplate code by storing Dagster Asset and Op outputs and loading them as inputs to downstream objects. They make it easy to change where and how your data is stored.</p> <p>You only need to define your I/O Manager and its settings (such as storage location and schema) once and the I/O Manager will take care of correctly reading and writing all your Dagster Assets automatically.</p> <p>If you need lower-level access than the Dagster I/O Managers provide, take a look at the Delta Table Resource.</p>"},{"location":"integrations/delta-lake-dagster/#the-delta-lake-io-manager","title":"The Delta Lake I/O Manager","text":"<p>You can easily read and write Delta Lake Tables from Dagster by using the <code>DeltaLakeIOManager()</code>. </p> <p>Install the DeltaLakeIOManager:</p> <pre><code>pip install dagster-deltalake\n</code></pre> <p>Next, configure the following settings in your project\u2019s <code>__init__.py</code> file: - <code>io_manager</code>: set this to <code>DeltaLakeIOManager()</code>, this sets the default I/O Manager for all your Assets</p> <p>Within the DeltaLakeIOManager, define: - <code>root_uri</code>: the root path where your Delta Tables will be created - <code>storage_options</code>: configuration for accessing storage location - <code>schema</code>: name of schema to use (optional, defaults to public)</p> <pre><code>defs = Definitions(\n   assets=all_assets,\n   resources={\n        \"io_manager\": DeltaLakePyarrowIOManager(\n            root_uri=\"path/to/deltalake\",\n            storage_options=LocalConfig(),\n            schema=\"dagster_deltalake\",\n        ),\n   },\n)\n</code></pre> <p>Now, when you materialize an Asset, it will be saved as a Delta Lake in a folder <code>dagster_deltalake/asset_name</code> under the root directory <code>path/to/deltalake</code>.</p> <p>The default Delta Lake I/O Manager supports Arrow reads and writes. You can also use the Delta Lake I/O Manager with pandas or polars. </p>"},{"location":"integrations/delta-lake-dagster/#creating-delta-lake-tables-with-dagster","title":"Creating Delta Lake Tables with Dagster","text":"<p>You don\u2019t need to do anything else to store your Dagster Assets as Delta Lake tables. The I/O Manager will handle storing and loading your Assets as Delta Lake tables from now on.</p> <p>You can proceed to write Dagster code as you normally would. For example, you can create an Asset that reads in some toy data about animals and writes it out to an Arrow Table:</p> <pre><code>import pyarrow as pa\nfrom pyarrow import csv\n\nfrom dagster import asset\n\n@asset\ndef raw_dataset() -&gt; pa.Table:\n   n_legs = pa.array([2, 4, None, 100])\n   animals = pa.array([\"Flamingo\", \"Horse\", \"Brittle stars\", \"Centipede\"])\n   data = {'n_legs': n_legs, 'animals': animals}\n\n   return pa.Table.from_pydict(data)\n</code></pre> <p>When you materialize the Asset defined above (using the config settings defined earlier), the Delta Lake I/O Manager will create the table <code>dagster_deltalake/iris_dataset</code> if it doesn\u2019t exist yet.</p>"},{"location":"integrations/delta-lake-dagster/#overwrites-when-rematerializing-assets","title":"Overwrites when Rematerializing Assets","text":"<p>If the table does already exist at the specified location, the Delta Lake I/O Manager will perform an overwrite. Delta Lake\u2019s transaction log maintains a record of all changes to your Delta Lake tables. You can inspect the record of changes to your Delta Lake tables by taking a look at these transaction logs.</p>"},{"location":"integrations/delta-lake-dagster/#loading-delta-lake-tables-in-downstream-assets","title":"Loading Delta Lake Tables in Downstream Assets","text":"<p>You can use Assets stored as Delta Lake tables as input to downstream Assets. Dagster and the Delta Lake I/O Manager make this easy for you.</p> <p>You can write Dagster code as you normally would. Pass the upstream Asset as an argument to the downstream object to set up the dependency. Make sure to define the correct data type.</p> <p>The Delta Lake I/O Manager will handle reading and writing the data from your Delta Lake.</p> <pre><code>import pyarrow as pa\nfrom dagster import asset\n\n# ... raw_dataset asset is defined here ...\n\n@asset\ndef clean_dataset(raw_dataset: pa.Table) -&gt; pa.Table:\n   return raw_dataset.drop_null()\n</code></pre>"},{"location":"integrations/delta-lake-dagster/#reading-existing-delta-lake-tables-into-dagster","title":"Reading Existing Delta Lake Tables into Dagster","text":"<p>You can make existing Delta Lake tables (that were not created in Dagster) available to your Dagster assets. Use the <code>SourceAsset</code> object and pass the table name as the key argument: </p> <pre><code>from dagster import SourceAsset\n\niris_harvest_data = SourceAsset(key=\"more_animal_data\")\n</code></pre> <p>This will load a table <code>more_animal_data</code> located at <code>&lt;root_uri&gt;/&lt;schema&gt;</code> as configured in the Definitions object above (see Delta Lake I/O Manager section).</p>"},{"location":"integrations/delta-lake-dagster/#column-pruning","title":"Column Pruning","text":"<p>You can often improve the efficiency of your computations by only loading specific columns of your Delta table. This is called column pruning. </p> <p>With the Delta Lake I/O manager, you can select specific columns to load defining the <code>columns</code> in the <code>metadata</code> parameter of the <code>AssetIn</code> that loads the upstream Asset:</p> <pre><code>import pyarrow as pa\nfrom dagster import AssetIn, asset\n\n# this example uses the clean_dataset Asset defined earlier\n\n@asset(\n       ins={\n           \"mammal_bool\": AssetIn(\n               key=\"clean_dataset\",\n               metadata={\"columns\": [\"is_mammal\", \"animals\"]},\n           )\n       }\n)\ndef mammal_data(mammal_bool: pa.Table) -&gt; pa.Table:\n   mammals = mammal_bool[\"is_mammal\"].cast(\"bool\")\n   animals = mammal_bool[\"animals\"]\n   data = {\"mammal_bool\": mammals, \"animals\": animals}\n   return pa.Table.from_pydict(data)\n</code></pre> <p>Here, we select only the <code>sepal_length_cm</code> and <code>sepal_width_cm</code> columns from the <code>iris_dataset</code> table and load them into an <code>AssetIn</code> object called <code>iris_sepal</code>. This AssetIn object is used to create a new Asset <code>sepal_data</code>, containing only the selected columns.</p>"},{"location":"integrations/delta-lake-dagster/#working-with-partitioned-assets","title":"Working with Partitioned Assets","text":"<p>Partitioning is an important feature of Delta Lake that can make your computations more efficient. The Delta Lake I/O manager helps you read and write partitioned data easily. You can work with static partitions, time-based partitions, multi-partitions, and dynamic partitions.</p> <p>For example, you can partition the Iris dataset on the <code>species</code> column as follows:</p> <pre><code>import pyarrow as pa\n\nfrom dagster import StaticPartitionsDefinition, asset\n\n@asset(\n  partitions_def=StaticPartitionsDefinition(\n      [\"Human\", \"Horse\",]\n  ),\n  metadata={\"partition_expr\": \"n_legs\"},\n)\ndef dataset_partitioned(\n   context,\n   clean_dataset: pa.Table,\n   ) -&gt; pa.Table:\n   animals = context.asset_partition_key_for_output()\n   table = clean_dataset\n\n   return table.filter(pc.field(\"animals\") == animals)\n</code></pre> <p>To partition your data, make sure to include the relevant <code>partitions_def</code> and <code>metadata</code> arguments to the <code>@asset</code> decorator. Refer to the Dagster documentation on partitioning assets for more information.</p>"},{"location":"integrations/delta-lake-dagster/#using-delta-lake-and-dagster-with-pandas","title":"Using Delta Lake and Dagster with Pandas","text":"<p>To read and write data to Delta Lake using pandas, use the <code>DeltaLakePandasIOManager()</code>. </p> <p>You will need to install it using:</p> <pre><code>pip install dagster-deltalake-pandas\n</code></pre> <p>In your <code>Definitions</code> object, change the <code>io_manager</code> to <code>DeltaLakePandasIOManager()</code>:</p> <pre><code>from dagster_deltalake_pandas import DeltaLakePandasIOManager\n\n\ndefs = Definitions(\n   assets=all_assets,\n   resources={\n        \"io_manager\": DeltaLakePandasIOManager(\n            root_uri=\"path/to/deltalake\",\n            storage_options=LocalConfig(),\n            schema=\"dagster_deltalake\",\n        ),\n   },\n)\n</code></pre> <p>Now you can read and write Dagster Assets defined as pandas DataFrames in Delta Lake format. For example:</p> <pre><code>import pandas as pd\nfrom dagster import asset\n\n@asset\ndef iris_dataset() -&gt; pd.DataFrame:\n   return pd.read_csv(\n       \"https://docs.dagster.io/assets/iris.csv\",\n       names=[\n           \"sepal_length_cm\",\n           \"sepal_width_cm\",\n           \"petal_length_cm\",\n           \"petal_width_cm\",\n           \"species\",\n       ],\n   )\n</code></pre>"},{"location":"integrations/delta-lake-dagster/#using-delta-lake-and-dagster-with-polars","title":"Using Delta Lake and Dagster with Polars","text":"<p>To read and write data to Delta Lake using polars, use the <code>DeltaLakePolarsIOManager()</code>. </p> <p>You will need to install it using:</p> <pre><code>pip install dagster-deltalake-polars\n</code></pre> <p>In your <code>Definitions</code> object, change the <code>io_manager</code> to <code>DeltaLakePolarsIOManager()</code>:</p> <pre><code>from dagster_deltalake_polars import DeltaLakePolarsIOManager\n\ndefs = Definitions(\n   assets=all_assets,\n   resources={\n        \"io_manager\": DeltaLakePolarsIOManager(\n            root_uri=\"path/to/deltalake\",\n            storage_options=LocalConfig(),\n            schema=\"dagster_deltalake\",\n        ),\n   },\n)\n</code></pre> <p>Now you can read and write Dagster Assets defined as Polars DataFrames in Delta Lake format. For example:</p> <pre><code>import polars as pl\nfrom dagster import asset\n\n\n@asset\ndef iris_dataset() -&gt; pl.DataFrame:\n   return pl.read_csv(\n       \"https://docs.dagster.io/assets/iris.csv\",\n       new_columns=[\n          \"sepal_length_cm\",\n          \"sepal_width_cm\",\n          \"petal_length_cm\",\n          \"petal_width_cm\",\n          \"species\",\n      ],\n   has_header=False\n)\n</code></pre>"},{"location":"integrations/delta-lake-dagster/#delta-lake-table-resource","title":"Delta Lake Table Resource","text":"<p>I/O managers are a helpful tool in many common usage situations. But when you need lower-level access, the I/O Manager might not be the right tool to use. In these cases you may want to use the Delta Lake Table Resource.</p> <p>The Delta Lake Table Resource is a low-level access method to the table object. It gives you more fine-grained control and allows for modeling of more complex data. You can also use the Table Resource to run optimization and vacuuming jobs.</p>"},{"location":"integrations/delta-lake-dagster/#schema-and-constraint-enforcement","title":"Schema and Constraint Enforcement","text":"<p>Delta Lake provides built-in checks to ensure schema consistency when appending data to a table, as well as the ability to evolve the schema. This is a great feature for the Dagster community as it prevents bad data from being appended to tables, ensuring data consistency and accuracy. </p> <p>Read more about how to add constraints to a table in the Delta Lake documentation.</p>"},{"location":"integrations/delta-lake-dagster/#z-ordering","title":"Z-Ordering","text":"<p>Delta Lake offers Z-ordering functionality to colocate similar data in the same files. This can make your Delta Table queries much more efficient via file skipping. Dagster users can now benefit from this great feature through the Delta Lake I/O Manager.</p> <p>Read more about Z-Ordering on the Delta Lake blog.</p>"},{"location":"integrations/delta-lake-dagster/#contribute","title":"Contribute","text":"<p>To contribute to the Delta Lake and Dagster integration, go to [link]</p>"},{"location":"integrations/delta-lake-dask/","title":"Using Delta Lake with Dask","text":"<p>Delta Lake is a great storage format for Dask analyses. This page explains why and how to use Delta Lake with Dask.</p> <p>You will learn how to read Delta Lakes into Dask DataFrames, how to query Delta tables with Dask, and the unique advantages Delta Lake offers the Dask community.</p> <p>Here are some of the benefits that Delta Lake provides Dask users:</p> <ul> <li>better performance with file skipping</li> <li>enhanced file skipping via Z Ordering</li> <li>ACID transactions for reliable writes</li> <li>easy time-travel functionality</li> </ul> <p>\u2757\ufe0f dask-deltatable currently works with deltalake&lt;=13.0. See https://github.com/dask-contrib/dask-deltatable/issues/65</p>"},{"location":"integrations/delta-lake-dask/#install-dask-deltatable","title":"Install Dask-Deltatable","text":"<p>To use Delta Lake with Dask, first install the library using</p> <pre><code>pip install dask-deltatable\n</code></pre>"},{"location":"integrations/delta-lake-dask/#reading-delta-tables-into-a-dask-dataframe","title":"Reading Delta Tables into a Dask DataFrame","text":"<p>You can read data stored in a Delta Lake into a Dask DataFrame using <code>dask-deltatable.read_deltalake</code>.</p> <p>Let's read in a toy dataset to see what we can do with Delta Lake and Dask. You can access the data stored as a Delta Lake on Github</p> <pre><code>import dask_deltatable as ddt\n\n# read delta table into Dask DataFrame\ndelta_path = \"path/to/data/people_countries_delta_dask\"\nddf = ddt.read_deltalake(delta_path)\n</code></pre> <p>Dask is a library for efficient distributed computing and works with lazy evaluation. Function calls to <code>dask.dataframe</code> build a task graph in the background. To trigger computation, call <code>.compute()</code>:</p> <pre><code>&gt; ddf.compute()\n\n|    | first_name   | last_name   | country   | continent   |\n|---:|:-------------|:------------|:----------|:------------|\n|  0 | Ernesto      | Guevara     | Argentina | NaN         |\n|  0 | Bruce        | Lee         | China     | Asia        |\n|  1 | Jack         | Ma          | China     | Asia        |\n|  0 | Wolfgang     | Manche      | Germany   | NaN         |\n|  1 | Soraya       | Jala        | Germany   | NaN         |\n</code></pre> <p>You can read in specific versions of Delta tables by specifying a <code>version</code> number or a timestamp:</p> <pre><code># with specific version\nddf = ddt.read_deltalake(delta_path, version=3)\n\n# with specific datetime\nddt.read_deltalake(delta_path, datetime=\"2018-12-19T16:39:57-08:00\")\n</code></pre> <p><code>dask-deltatable</code> also supports reading from remote sources like S3 with:</p> <pre><code>ddt.read_deltalake(\"s3://bucket_name/delta_path\", version=3)\n</code></pre> <p>To read data from remote sources you'll need to make sure the credentials are properly configured in environment variables or config files. Refer to your cloud provider documentation to configure these.</p>"},{"location":"integrations/delta-lake-dask/#what-can-i-do-with-a-dask-deltatable","title":"What can I do with a Dask Deltatable?","text":"<p>Reading a Delta Lake in with <code>dask-deltatable</code> returns a regular Dask DataFrame. You can perform all the regular Dask operations on this DataFrame.</p> <p>Let's take a look at the first few rows:</p> <pre><code>&gt; ddf.head(n=3)\n\n|    | first_name   | last_name   | country   |   continent |\n|---:|:-------------|:------------|:----------|------------:|\n|  0 | Ernesto      | Guevara     | Argentina |         nan |\n</code></pre> <p><code>dask.dataframe.head()</code> shows you the first rows of the first partition in the dataframe. In this case, the first partition only has 1 row.</p> <p>This is because the Delta Lake has been partitioned by country:</p> <pre><code>&gt; !ls ../../data/people_countries_delta_dask\n_delta_log        country=Argentina country=China     country=Germany\n</code></pre> <p><code>dask-deltatable</code> neatly reads in the partitioned Delta Lake into corresponding Dask DataFrame partitions:</p> <pre><code>&gt; # see number of partitions\n&gt; ddf.npartitions\n3\n</code></pre> <p>You can inspect a single partition using <code>dask.dataframe.get_partition()</code>:</p> <pre><code>&gt; ddf.get_partition(n=1).compute()\n\n|    | first_name   | last_name   | country   | continent   |\n|---:|:-------------|:------------|:----------|:------------|\n|  0 | Bruce        | Lee         | China     | Asia        |\n|  1 | Jack         | Ma          | China     | Asia        |\n</code></pre>"},{"location":"integrations/delta-lake-dask/#perform-dask-operations","title":"Perform Dask Operations","text":"<p>Let's perform some basic computations over the Delta Lake data that's now stored in our Dask DataFrame.</p> <p>Suppose you want to group the dataset by the <code>country</code> column:</p> <pre><code>&gt; ddf.groupby(['country']).count().compute()\n\n| country   |   first_name |   last_name |   continent |\n|:----------|-------------:|------------:|------------:|\n| Argentina |            1 |           1 |           1 |\n| China     |            2 |           2 |           2 |\n| Germany   |            2 |           2 |           2 |\n</code></pre> <p>Dask executes this <code>groupby</code> operation in parallel across all available cores.</p>"},{"location":"integrations/delta-lake-dask/#map-functions-over-partitions","title":"Map Functions over Partitions","text":"<p>You can also use Dask's <code>map_partitions</code> method to map a custom Python function over all the partitions.</p> <p>Let's write a function that will replace the missing <code>continent</code> values with the right continent names.</p> <pre><code># define custom python function\n\n# get na_string\ndf = ddf.get_partition(0).compute()\nna_string = df.iloc[0].continent\nna_string\n\n# define function\ndef replace_proper(partition, na_string):\n    if [partition.country == \"Argentina\"]:\n        partition.loc[partition.country==\"Argentina\"] = partition.loc[partition.country==\"Argentina\"].replace(na_string, \"South America\")\n    if [partition.country == \"Germany\"]:\n        partition.loc[partition.country==\"Germany\"] = partition.loc[partition.country==\"Germany\"].replace(na_string, \"Europe\")\n    else:\n        pass\n    return partition\n</code></pre> <p>Now map this over all partitions in the Dask DataFrame:</p> <pre><code># define metadata and map function over partitions\n&gt; meta = dict(ddf.dtypes)\n&gt; ddf3 = ddf.map_partitions(replace_proper, na_string, meta=meta)\n&gt; ddf3.compute()\n\n|    | first_name   | last_name   | country   | continent     |\n|---:|:-------------|:------------|:----------|:--------------|\n|  0 | Ernesto      | Guevara     | Argentina | South America |\n|  0 | Bruce        | Lee         | China     | Asia          |\n|  1 | Jack         | Ma          | China     | Asia          |\n|  0 | Wolfgang     | Manche      | Germany   | Europe        |\n|  1 | Soraya       | Jala        | Germany   | Europe        |\n</code></pre>"},{"location":"integrations/delta-lake-dask/#write-to-delta-lake","title":"Write to Delta Lake","text":"<p>After doing your data processing in Dask, you can write the data back out to Delta Lake using <code>to_deltalake</code>:</p> <pre><code>ddt.to_deltalake(\"tmp/test_write\", ddf)\n</code></pre>"},{"location":"integrations/delta-lake-dask/#contribute-to-dask-deltalake","title":"Contribute to <code>dask-deltalake</code>","text":"<p>To contribute, go to the <code>dask-deltalake</code> Github repository.</p>"},{"location":"integrations/delta-lake-datafusion/","title":"Using Delta Lake with DataFusion","text":"<p>This page explains how to use Delta Lake with DataFusion.</p> <p>Delta Lake offers DataFusion users better performance and more features compared to other formats like CSV or Parquet.</p> <p>Delta Lake works well with the DataFusion Rust API and the DataFusion Python API.  It's a great option for all DataFusion users.</p> <p>Delta Lake also depends on DataFusion to implement SQL-related functionality under the hood.  We will also discuss this dependency at the end of this guide in case you're interested in learning more about the symbiotic relationship between the two libraries.</p>"},{"location":"integrations/delta-lake-datafusion/#delta-lake-performance-benefits-for-datafusion-users","title":"Delta Lake performance benefits for DataFusion users","text":"<p>Let's run some DataFusion queries on a Parquet file and a Delta table with the same data to learn more about the performance benefits of Delta Lake.</p> <p>Suppose you have the following dataset with 1 billion rows and 9 columns.  Here are the first three rows of data:</p> <pre><code>+-------+-------+--------------+-------+-------+--------+------+------+---------+\n| id1   | id2   | id3          |   id4 |   id5 |    id6 |   v1 |   v2 |      v3 |\n|-------+-------+--------------+-------+-------+--------+------+------+---------|\n| id016 | id046 | id0000109363 |    88 |    13 | 146094 |    4 |    6 | 18.8377 |\n| id039 | id087 | id0000466766 |    14 |    30 | 111330 |    4 |   14 | 46.7973 |\n| id047 | id098 | id0000307804 |    85 |    23 | 187639 |    3 |    5 | 47.5773 |\n+-------+-------+--------------+-------+-------+--------+------+------+---------+\n</code></pre> <p>Here's how to register a Delta Lake table as a PyArrow dataset:</p> <pre><code>from datafusion import SessionContext\nfrom deltalake import DeltaTable\n\nctx = SessionContext()\ntable = DeltaTable(\"G1_1e9_1e2_0_0\")\nctx.register_dataset(\"my_delta_table\", table.to_pyarrow_dataset())\n</code></pre> <p>Now query the table:</p> <pre><code>ctx.sql(\"select id1, sum(v1) as v1 from my_delta_table where id1='id096' group by id1\")\n</code></pre> <p>That query takes 2.8 seconds to execute.</p> <p>Let's register the same dataset as a Parquet table, run the same query, and compare the runtime difference.</p> <p>Register the Parquet table and run the query:</p> <pre><code>path = \"G1_1e9_1e2_0_0.parquet\"\nctx.register_parquet(\"my_parquet_table\", path)\nctx.sql(\"select id1, sum(v1) as v1 from my_parquet_table where id1='id096' group by id1\")\n</code></pre> <p>This query takes 5.3 seconds to run.</p> <p>Parquet stores data in row groups and DataFusion can intelligently skip row groups that don't contain relevant data, so the query is faster than a file format like CSV which doesn't support row group skipping.</p> <p>Delta Lake stores file-level metadata information in the transaction log, so it can skip entire files when queries are executed.  Delta Lake can skip entire files and then skip row groups within the individual files.  This makes Delta Lake even faster than Parquet files, especially for larger datasets spread across many files.</p>"},{"location":"integrations/delta-lake-datafusion/#delta-lake-features-for-datafusion-users","title":"Delta Lake features for DataFusion users","text":"<p>Delta Lake also provides other features that are useful for DataFusion users like ACID transactions, concurrency protection, time travel, versioned data, and more.</p>"},{"location":"integrations/delta-lake-datafusion/#why-delta-lake-depends-on-datafusion","title":"Why Delta Lake depends on DataFusion","text":"<p>Delta Lake depends on DataFusion to provide some end-user features.</p> <p>DataFusion is useful in providing SQL-related Delta Lake features. Some examples:</p> <ul> <li>Update and merge are written in terms of SQL expressions.</li> <li>Invariants and constraints are written in terms of SQL expressions.</li> </ul> <p>Anytime we have to evaluate SQL, we need some sort of SQL engine.  We use DataFusion for that.</p>"},{"location":"integrations/delta-lake-datafusion/#conclusion","title":"Conclusion","text":"<p>Delta Lake is a great file format for DataFusion users.</p> <p>Delta Lake also uses DataFusion to provide some end-user features.</p> <p>DataFusion and Delta Lake have a wonderful symbiotic relationship and play very nicely with each other.</p> <p>See this guide for more information on Delta Lake and PyArrow and why PyArrow Datasets are often a better option than PyArrow tables.</p>"},{"location":"integrations/delta-lake-pandas/","title":"Using Delta Lake with pandas","text":"<p>Delta Lake is a great storage system for pandas analyses.  This page shows how it's easy to use Delta Lake with pandas, the unique features Delta Lake offers pandas users, and how Delta Lake can make your pandas analyses run faster.</p> <p>Delta Lake is very easy to install for pandas analyses, just run <code>pip install deltalake</code>.</p> <p>Delta Lake allows for performance optimizations, so pandas queries can run much faster than the query run on data stored in CSV or Parquet.  See the following chart for the query runtime for the a Delta tables compared with CSV/Parquet.</p> <p></p> <p>Z Ordered Delta tables run this query much faster than when the data is stored in Parquet or CSV.  Let's dive in deeper and see how Delta Lake makes pandas faster.</p>"},{"location":"integrations/delta-lake-pandas/#delta-lake-makes-pandas-queries-run-faster","title":"Delta Lake makes pandas queries run faster","text":"<p>There are a few reasons Delta Lake can make pandas queries run faster:</p> <ol> <li>column pruning: only grabbing the columns relevant for a query</li> <li>file skipping: only reading files with data for the query</li> <li>row group skipping: only reading row groups with data for the query</li> <li>Z ordering data: colocating similar data in the same files, so file skipping is more effective</li> </ol> <p>Reading less data (fewer columns and/or fewer rows) is how Delta Lake makes pandas queries run faster.</p> <p>Parquet allows for column pruning and row group skipping, but doesn't support file-level skipping or Z Ordering.  CSV doesn't support any of these performance optimizations.</p> <p>Let's take a look at a sample dataset and run a query to see the performance enhancements offered by Delta Lake.</p> <p>Suppose you have a 1 billion row dataset with 9 columns, here are the first three rows of the dataset:</p> <pre><code>+-------+-------+--------------+-------+-------+--------+------+------+---------+\n| id1   | id2   | id3          |   id4 |   id5 |    id6 |   v1 |   v2 |      v3 |\n|-------+-------+--------------+-------+-------+--------+------+------+---------|\n| id016 | id046 | id0000109363 |    88 |    13 | 146094 |    4 |    6 | 18.8377 |\n| id039 | id087 | id0000466766 |    14 |    30 | 111330 |    4 |   14 | 46.7973 |\n| id047 | id098 | id0000307804 |    85 |    23 | 187639 |    3 |    5 | 47.5773 |\n+-------+-------+--------------+-------+-------+--------+------+------+---------+\n</code></pre> <p>The dataset is roughly 50 GB when stored as an uncompressed CSV files.  Let's run some queries on a 2021 Macbook M1 with 64 GB of RAM.</p> <p>Start by running the query on an uncompressed CSV file:</p> <pre><code>(\n    pd.read_csv(f\"{Path.home()}/data/G1_1e9_1e2_0_0.csv\", usecols=[\"id1\", \"id2\", \"v1\"])\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>This query takes 234 seconds to execute.  It runs out of memory if the <code>usecols</code> parameter is not set.</p> <p>Now let's convert the CSV dataset to Parquet and run the same query on the data stored in a Parquet file.</p> <pre><code>(\n    pd.read_parquet(\n        f\"{Path.home()}/data/G1_1e9_1e2_0_0.parquet\", columns=[\"id1\", \"id2\", \"v1\"]\n    )\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>This query takes 118 seconds to execute.</p> <p>Parquet stores data in row groups and allows for skipping when the <code>filters</code> predicates are set.  Run the Parquet query again with row group skipping enabled:</p> <pre><code>(\n    pd.read_parquet(\n        f\"{Path.home()}/data/G1_1e9_1e2_0_0.parquet\",\n        columns=[\"id1\", \"id2\", \"v1\"],\n        filters=[(\"id1\", \"==\", \"id016\")],\n    )\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>This query runs in 19 seconds.  Lots of row groups can be skipped for this particular query.</p> <p>Now let's run the same query on a Delta table to see the out-of-the box performance:</p> <pre><code>(\n    DeltaTable(f\"{Path.home()}/data/deltalake_baseline_G1_1e9_1e2_0_0\", version=0)\n    .to_pandas(filters=[(\"id1\", \"==\", \"id016\")], columns=[\"id1\", \"id2\", \"v1\"])\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>This query runs in 8 seconds, which is a significant performance enhancement.</p> <p>Now let's Z Order the Delta table by <code>id1</code> which will make the data skipping even better.  Run the query again on the Z Ordered Delta table:</p> <pre><code>(\n    DeltaTable(f\"{Path.home()}/data/deltalake_baseline_G1_1e9_1e2_0_0\", version=1)\n    .to_pandas(filters=[(\"id1\", \"==\", \"id016\")], columns=[\"id1\", \"id2\", \"v1\"])\n    .query(\"id1 == 'id016'\")\n    .groupby(\"id2\")\n    .agg({\"v1\": \"sum\"})\n)\n</code></pre> <p>The query now executes in 2.4 seconds.</p> <p>Delta tables can make certain pandas queries run much faster.</p>"},{"location":"integrations/delta-lake-pandas/#delta-lake-lets-pandas-users-time-travel","title":"Delta Lake lets pandas users time travel","text":"<p>Start by creating a Delta table:</p> <pre><code>from deltalake import write_deltalake, DeltaTable\n\ndf = pd.DataFrame({\"num\": [1, 2, 3], \"letter\": [\"a\", \"b\", \"c\"]})\nwrite_deltalake(\"tmp/some-table\", df)\n</code></pre> <p>Here are the contents of the Delta table (version 0 of the Delta table):</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n+-------+----------+\n</code></pre> <p>Now append two rows to the Delta table:</p> <pre><code>df = pd.DataFrame({\"num\": [8, 9], \"letter\": [\"dd\", \"ee\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"append\")\n</code></pre> <p>Here are the contents after the append operation (version 1 of the Delta table):</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre> <p>Now perform an overwrite transaction:</p> <pre><code>df = pd.DataFrame({\"num\": [11, 22], \"letter\": [\"aa\", \"bb\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"overwrite\")\n</code></pre> <p>Here are the contents after the overwrite operation (version 2 of the Delta table):</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre> <p>Read in the Delta table and it will grab the latest version by default:</p> <pre><code>DeltaTable(\"tmp/some-table\").to_pandas()\n\n+-------+----------+\n|   num | letter   |\n|-------+----------|\n|    11 | aa       |\n|    22 | bb       |\n+-------+----------+\n</code></pre> <p>You can easily time travel back to version 0 of the Delta table:</p> <pre><code>DeltaTable(\"tmp/some-table\", version=0).to_pandas()\n\n+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n+-------+----------+\n</code></pre> <p>You can also time travel to version 1 of the Delta table:</p> <pre><code>DeltaTable(\"tmp/some-table\", version=1).to_pandas()\n\n+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre> <p>Time travel is a powerful feature that pandas users cannot access with CSV or Parquet.</p>"},{"location":"integrations/delta-lake-pandas/#schema-enforcement","title":"Schema enforcement","text":"<p>Delta tables only allow you to append DataFrame with matching schema by default.  Suppose you have a DataFrame with <code>num</code> and <code>animal</code> columns, which is different from the Delta table that has columns with <code>num</code> and <code>letter</code> columns.</p> <p>Try to append this DataFrame with a mismatched schema to the existing table:</p> <pre><code>df = pd.DataFrame({\"num\": [5, 6], \"animal\": [\"cat\", \"dog\"]})\nwrite_deltalake(\"tmp/some-table\", df)\n</code></pre> <p>This transaction will be rejected and will return the following error message:</p> <pre><code>ValueError: Schema of data does not match table schema\nData schema:\nnum: int64\nanimal: string\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 474\nTable Schema:\nnum: int64\nletter: string\n</code></pre> <p>Schema enforcement protects your table from getting corrupted by appending data with mismatched schema.  Parquet and CSV don't offer schema enforcement for pandas users.</p>"},{"location":"integrations/delta-lake-pandas/#overwriting-schema-of-table","title":"Overwriting schema of table","text":"<p>You can overwrite the table contents and schema by setting the <code>schema_mode</code> option.  Here's how to overwrite the table contents:</p> <pre><code>write_deltalake(\"tmp/some-table\", df, mode=\"overwrite\", schema_mode=\"overwrite\")\n</code></pre> <p>Here are the contents of the table after the values and schema have been overwritten:</p> <pre><code>+-------+----------+\n|   num | animal   |\n|-------+----------|\n|     5 | cat      |\n|     6 | dog      |\n+-------+----------+\n</code></pre> <p>If you want the schema to be merged instead, specify schema_mode=\"merge\".</p>"},{"location":"integrations/delta-lake-pandas/#in-memory-vs-in-storage-data-changes","title":"In-memory vs. in-storage data changes","text":"<p>It's important to distinguish between data stored in-memory and data stored on disk when understanding the functionality offered by Delta Lake.</p> <p>pandas loads data from storage (CSV, Parquet, or Delta Lake) into in-memory DataFrames.</p> <p>pandas makes it easy to modify the data in memory, say update a column value.  It's not easy to update a column value in storage systems like CSV or Parquet using pandas.</p> <p>Delta Lake makes it easy for pandas users to update data in storage.</p>"},{"location":"integrations/delta-lake-pandas/#why-delta-lake-allows-for-faster-queries","title":"Why Delta Lake allows for faster queries","text":"<p>Delta tables store data in many files and metadata about the files in the transaction log.  Delta Lake allows for certain queries to skip entire files, which makes pandas queries run much faster.</p>"},{"location":"integrations/delta-lake-pandas/#more-resources","title":"More resources","text":"<p>See this talk on why Delta Lake is the best file format for pandas analyses to learn more: </p>"},{"location":"integrations/delta-lake-pandas/#conclusion","title":"Conclusion","text":"<p>Delta Lake provides many features that make it an excellent format for pandas analyses:</p> <ul> <li>performance optimizations make pandas queries run faster</li> <li>data management features make pandas analyses more reliable</li> <li>advanced features allow you to perform more complex pandas analyses</li> </ul> <p>Python deltalake offers pandas users a better experience compared with CSV/Parquet.</p>"},{"location":"integrations/delta-lake-polars/","title":"Using Delta Lake with polars","text":"<p>This page explains why Delta Lake is a great storage system for Polars analyses.</p> <p>You will learn how to create Delta tables with Polars, how to query Delta tables with Polars, and the unique advantages Delta Lake offers the Polars community.</p> <p>Here are some amazing benefits that Delta Lake provides Polars users:</p> <ul> <li>time travel</li> <li>ACID transactions for reliable writes</li> <li>better performance with file skipping</li> <li>enhanced file skipping via Z Ordering</li> <li>ability to rollback mistakes</li> <li>and many, many more</li> </ul> <p>Let's start by showing how to use Polars with Delta Lake, explore how Delta Lake can make Polars queries run faster, and then look at all the cool features Delta Lake offers Polars users.</p>"},{"location":"integrations/delta-lake-polars/#creating-a-delta-lake-table-with-polars","title":"Creating a Delta Lake table with Polars","text":"<p>Create a Polars DataFrame and write it out to a Delta table:</p> <pre><code>import polars as pl\n\ndf = pl.DataFrame({\"x\": [1, 2, 3]})\ndf.write_delta(\"tmp/bear_delta_lake\")\n</code></pre> <p>Inspect the contents of the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\"))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 1   |\n| 2   |\n| 3   |\n+-----+\n</code></pre> <p>Now create another Polars DataFrame and append it to the existing Delta table:</p> <pre><code>df2 = pl.DataFrame({\"x\": [8, 9, 10]})\ndf2.write_delta(\"tmp/bear_delta_lake\", mode=\"append\")\n</code></pre> <p>Re-inspect the contents of the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\"))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 1   |\n| 2   |\n| 3   |\n| 8   |\n| 9   |\n| 10  |\n+-----+\n</code></pre> <p>Now overwrite the existing Delta table:</p> <pre><code>df3 = pl.DataFrame({\"x\": [55, 66, 77]})\ndf3.write_delta(\"tmp/bear_delta_lake\", mode=\"overwrite\")\n</code></pre> <p>Inspect the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\"))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 55  |\n| 66  |\n| 77  |\n+-----+\n</code></pre> <p>The Delta table now has three versions, as shown in the following diagram:</p> <p></p>"},{"location":"integrations/delta-lake-polars/#time-travel-with-delta-lake-for-polars","title":"Time travel with Delta Lake for Polars","text":"<p>Time travel back to version 0 of the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\", version=0))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 1   |\n| 2   |\n| 3   |\n+-----+\n</code></pre> <p>Time travel back to version 1 of the Delta table:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\", version=1))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 1   |\n| 2   |\n| 3   |\n| 9   |\n| 8   |\n| 10  |\n+-----+\n</code></pre> <p>Read the Delta table without specifying a version and see how it reads the latest version by default:</p> <pre><code>print(pl.read_delta(\"tmp/bear_delta_lake\"))\n\n+-----+\n| x   |\n| --- |\n| i64 |\n+=====+\n| 55  |\n| 66  |\n| 77  |\n+-----+\n</code></pre> <p>Let's dive into how to read Delta tables with Polars in more detail and compare the query runtime performance on larger datasets.</p>"},{"location":"integrations/delta-lake-polars/#reading-a-delta-lake-table-with-polars","title":"Reading a Delta Lake table with Polars","text":"<p>Let's look at the h2o groupby dataset that has 1 billion rows and 9 columns.  Here are the first three rows of the dataset:</p> <pre><code>+-------+-------+--------------+-------+-------+--------+------+------+---------+\n| id1   | id2   | id3          |   id4 |   id5 |    id6 |   v1 |   v2 |      v3 |\n|-------+-------+--------------+-------+-------+--------+------+------+---------|\n| id016 | id046 | id0000109363 |    88 |    13 | 146094 |    4 |    6 | 18.8377 |\n| id039 | id087 | id0000466766 |    14 |    30 | 111330 |    4 |   14 | 46.7973 |\n| id047 | id098 | id0000307804 |    85 |    23 | 187639 |    3 |    5 | 47.5773 |\n+-------+-------+--------------+-------+-------+--------+------+------+---------+\n</code></pre> <p>This dataset is 50GB when stored in an uncompressed CSV file.  Let's run some queries on this dataset when it's stored in different file formats with Polars.</p> <p>This section will show the runtime for a query when the data is stored in CSV, Parquet, and Delta Lake and explain why Delta tables are the fastest.</p> <p></p> <p>Start by running a query on an uncompressed CSV file with <code>read_csv</code>:</p> <pre><code>pl.read_csv(\"~/data/G1_1e9_1e2_0_0.csv\").filter(pl.col(\"id1\") &lt; \"id016\").group_by(\n    [\"id1\", \"id2\"]\n).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query errors out after running for several minutes.  The machine runs out of memory.  Let's try it again with <code>scan_csv</code>.</p> <pre><code>pl.scan_csv(\"~/data/G1_1e9_1e2_0_0.csv\").filter(pl.col(\"id1\") &lt; \"id016\").group_by(\n    [\"id1\", \"id2\"]\n).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query runs in 56.2 seconds.</p> <p>Now let's run the same query when the data is stored in a Parquet file:</p> <pre><code>pl.scan_parquet(\"~/data/G1_1e9_1e2_0_0.parquet\").filter(\n    pl.col(\"id1\") &lt; \"id016\"\n).group_by([\"id1\", \"id2\"]).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query runs in 8.3 seconds.  It's much faster because Polars is optimized to skip row groups in Parquet files that don't contain data that's relevant for the query.</p> <p>Then run the query on newly created Delta table:</p> <pre><code>pl.scan_delta(\"~/data/deltalake/G1_1e9_1e2_0_0\", version=1).filter(\n    pl.col(\"id1\") &lt; \"id016\"\n).group_by([\"id1\", \"id2\"]).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query runs in 7.2 seconds.  Polars can run this query faster because it can inspect the Delta transaction log and skip entire files that don't contain relevant data before performing the ordinary Parquet row group skipping.</p> <p>Finally run the query on the Delta table after it has been Z Ordered by <code>id1</code>:</p> <pre><code>pl.scan_delta(\"~/data/deltalake/G1_1e9_1e2_0_0\", version=2).filter(\n    pl.col(\"id1\") &lt; \"id016\"\n).group_by([\"id1\", \"id2\"]).agg(pl.sum(\"v1\").alias(\"v1_sum\")).collect()\n</code></pre> <p>This query runs in 3.5 seconds.  The query on the Z Ordered Delta table is even faster because similar data has been co-located in the same files.  This allows for even greater data skipping.</p> <p>Polars can leverage file skipping to query Delta tables very quickly.</p>"},{"location":"integrations/delta-lake-polars/#why-polars-is-fast-with-delta-lake","title":"Why Polars is fast with Delta Lake","text":"<p>Delta tables consist of metadata in a transaction log and data stored in Parquet files.</p> <p>When Polars queries a Delta table, it starts by consulting the transaction log to understand the metadata of each file in the Delta table.  This allows for Polars to quickly identify which files should be skipped by the query.</p> <p>CSV files don't contain any such metadata, so file skipping isn't an option.  Polars can skip Parquet files based on metadata, but it needs to open up each file and read the metadata, which is slower that grabbing the file-level metadata directly from the transaction log.</p> <p>Parquet doesn't allow users to easily Z Order the data and colocate similar data in the same row groups.  The Z Order optimizations are only supported in Delta tables.</p> <p>Delta Lake offers Polars users with unique performance optimizations.</p>"},{"location":"integrations/delta-lake-polars/#other-delta-lake-features-relevant-for-polars-users","title":"Other Delta Lake features relevant for Polars users","text":"<ul> <li>ACID transactions for reliable writes</li> <li>better performance with file skipping</li> <li>enhanced file skipping via Z Ordering</li> <li>ability to rollback mistakes</li> </ul>"},{"location":"integrations/delta-lake-polars/#conclusion","title":"Conclusion","text":"<p>This guide shows how Delta Lake is a great storage format for Polars analyses.</p> <p>Delta Lake is easy to use, fast, and full of features that are great for Polars users.</p>"},{"location":"integrations/object-storage/adls/","title":"Azure ADLS Storage Backend","text":"<p><code>delta-rs</code> offers native support for using Microsoft Azure Data Lake Storage (ADSL) as an object storage backend.</p> <p>You don\u2019t need to install any extra dependencies to read/write Delta tables to S3 with engines that use <code>delta-rs</code>. You do need to configure your ADLS access credentials correctly.</p>"},{"location":"integrations/object-storage/adls/#passing-credentials-explicitly","title":"Passing Credentials Explicitly","text":"<p>You can also pass ADLS credentials to your query engine explicitly.</p> <p>For Polars, you would do this using the <code>storage_options</code> keyword as demonstrated above. This will forward your credentials to the <code>object store</code> library that Polars uses for cloud storage access under the hood. Read the <code>object store</code> documentation for more information defining specific credentials.</p>"},{"location":"integrations/object-storage/adls/#example-write-delta-table-to-adls-with-polars","title":"Example: Write Delta table to ADLS with Polars","text":"<p>Using Polars, you can write a Delta table to ADLS directly like this:</p> <pre><code>import polars as pl\n\ndf = pl.DataFrame({\"foo\": [1, 2, 3, 4, 5]})\n\n# define container name\ncontainer = &lt;container_name&gt;\n\n# define credentials\nstorage_options = {\n    \"ACCOUNT_NAME\": &lt;account_name&gt;,\n    \"ACCESS_KEY\": &lt;access_key&gt;,\n}\n\n# write Delta to ADLS\ndf_pl.write_delta(\n    f\"abfs://{container}/delta_table\",\n    storage_options = storage_options\n)\n</code></pre>"},{"location":"integrations/object-storage/adls/#example-with-pandas","title":"Example with pandas","text":"<p>For libraries without direct <code>write_delta</code> methods (like Pandas), you can use the <code>write_deltalake</code> function from the <code>deltalake</code> library:</p> <pre><code>import pandas as pd\nfrom deltalake import write_deltalake\n\ndf = pd.DataFrame({\"foo\": [1, 2, 3, 4, 5]})\n\nwrite_deltalake(\n    f\"abfs://{container}/delta_table_pandas\",\n    df,\n    storage_options=storage_options\n)\n</code></pre>"},{"location":"integrations/object-storage/adls/#using-local-authentication","title":"Using Local Authentication","text":"<p>If your local session is authenticated using the Azure CLI then you can write Delta tables directly to ADLS. Read more about this in the Azure CLI documentation.</p>"},{"location":"integrations/object-storage/gcs/","title":"GCS Storage Backend","text":"<p><code>delta-rs</code> offers native support for using Google Cloud Storage (GCS) as an object storage backend.</p> <p>You don\u2019t need to install any extra dependencies to read/write Delta tables to GCS with engines that use <code>delta-rs</code>. You do need to configure your GCS access credentials correctly.</p>"},{"location":"integrations/object-storage/gcs/#using-application-default-credentials","title":"Using Application Default Credentials","text":"<p>Application Default Credentials (ADC) is a strategy used by GCS to automatically find credentials based on the application environment.</p> <p>If you are working from your local machine and have ADC set up then you can read/write Delta tables from GCS directly, without having to pass your credentials explicitly.</p>"},{"location":"integrations/object-storage/gcs/#example-write-delta-tables-to-gcs-with-polars","title":"Example: Write Delta tables to GCS with Polars","text":"<p>Using Polars, you can write a Delta table to GCS like this:</p> <pre><code># create a toy dataframe\nimport polars as pl\ndf = pl.DataFrame({\"foo\": [1, 2, 3, 4, 5]})\n\n# define path\ntable_path = \"gs://bucket/delta-table\"\n\n# write Delta to GCS\ndf.write_delta(table_path)\n</code></pre>"},{"location":"integrations/object-storage/gcs/#passing-gcs-credentials-explicitly","title":"Passing GCS Credentials explicitly","text":"<p>Alternatively, you can pass GCS credentials to your query engine explicitly.</p> <p>For Polars, you would do this using the <code>storage_options</code> keyword. This will forward your credentials to the <code>object store</code> library that Polars uses under the hood. Read the Polars documentation and the <code>object store</code> documentation for more information.</p>"},{"location":"integrations/object-storage/gcs/#delta-lake-on-gcs-required-permissions","title":"Delta Lake on GCS: Required permissions","text":"<p>You will need the following permissions in your GCS account:</p> <ul> <li><code>storage.objects.create</code></li> <li><code>storage.objects.delete</code> (only required for uploads that overwrite an existing object)</li> <li><code>storage.objects.get</code> (only required if you plan on using the Google Cloud CLI)</li> <li><code>storage.objects.list</code> (only required if you plan on using the Google Cloud CLI)</li> </ul> <p>For more information, see the GCP documentation</p>"},{"location":"integrations/object-storage/hdfs/","title":"HDFS Storage Backend","text":"<p>HDFS support is provided via the hdfs-native-object-store package, which sits on top of hdfs-native. This is an HDFS client written from scratch in Rust, with no bindings to libhdfs or any use of Java. While it supports most common cluster configurations, it does not support every possible client configuration that could exist.</p>"},{"location":"integrations/object-storage/hdfs/#supported-configurations","title":"Supported Configurations","text":"<p>By default, the client looks for existing Hadoop configs in following manner:</p> <ul> <li>If the <code>HADOOP_CONF_DIR</code> environment variable is defined, load configs from <code>$HADOOP_CONF_DIR/core-site.xml</code> and <code>$HADOOP_CONF_DIR/hdfs-site.xml</code></li> <li>Otherwise, if the <code>HADOOP_HOME</code> environment variable is set, load configs from <code>$HADOOP_HOME/etc/hadoop/core-site.xml</code> and <code>$HADOOP_HOME/etc/hadoop/hdfs-site.xml</code></li> </ul> <p>Additionally, you can pass Hadoop configs as <code>storage_options</code> and these will take precedence over the above configs.</p> <p>Currently the supported client configuration parameters are:</p> <ul> <li><code>dfs.ha.namenodes.*</code> - name service support</li> <li><code>dfs.namenode.rpc-address.*</code> - name service support</li> <li><code>fs.viewfs.mounttable.*.link.*</code> - ViewFS links</li> <li><code>fs.viewfs.mounttable.*.linkFallback</code> - ViewFS link fallback</li> </ul> <p>If you find your setup is not supported, please file an issue in the hdfs-native repository.</p>"},{"location":"integrations/object-storage/hdfs/#secure-clusters","title":"Secure Clusters","text":"<p>The client supports connecting to secure clusters through both Kerberos authentication as well as token authentication, and all SASL protection types are supported. The highest supported protection mechanism advertised by the server will be used.</p>"},{"location":"integrations/object-storage/hdfs/#kerberos-support","title":"Kerberos Support","text":"<p>Kerberos is supported through dynamically loading the <code>libgssapi_krb5</code> library. This must be installed separately through your package manager, and currently only works on Linux and Mac.</p> <p>Debian-based systems: <pre><code>apt-get install libgssapi-krb5-2\n</code></pre></p> <p>RHEL-based systems: <pre><code>yum install krb5-libs\n</code></pre></p> <p>MacOS: <pre><code>brew install krb5\n</code></pre></p> <p>Then simply <code>kinit</code> to get your TGT and authentication to HDFS should just work.</p>"},{"location":"integrations/object-storage/hdfs/#token-support","title":"Token Support","text":"<p>Token authentication is supported by looking for a token file located at the environment variable <code>HADOOP_TOKEN_FILE_LOCATION</code>. This is the location systems like YARN will automatically place a delegation token, so things will just work inside of YARN jobs.</p>"},{"location":"integrations/object-storage/hdfs/#issues","title":"Issues","text":"<p>If you face any HDFS-specific issues, please report to the hdfs-native-object-store repository.</p>"},{"location":"integrations/object-storage/s3-like/","title":"CloudFlare R2 &amp; Minio","text":"<p><code>delta-rs</code> offers native support for using Cloudflare R2 or Minio as an S3-compatible storage backend. R2 and Minio support conditional puts, which removes the need for DynamoDB for safe concurrent writes.  However, we have to pass the <code>aws_conditional_put</code> flag into <code>storage_options</code>. See the example below.</p> <p>You don\u2019t need to install any extra dependencies to read/write Delta tables to S3 with engines that use <code>delta-rs</code>. You do need to configure your AWS access credentials correctly.</p>"},{"location":"integrations/object-storage/s3-like/#passing-s3-credentials","title":"Passing S3 Credentials","text":"<p>You can pass your AWS credentials explicitly by using:</p> <ul> <li>the <code>storage_options</code>kwarg</li> <li>Environment variables</li> </ul>"},{"location":"integrations/object-storage/s3-like/#example","title":"Example","text":"<p>Let's work through an example with Polars. The same logic applies to other Python engines like Pandas, Daft, Dask, etc.</p> <p>Follow the steps below to use Delta Lake on S3 (R2/Minio) with Polars:</p> <ol> <li>Install Polars and deltalake. For example, using:</li> </ol> <p><code>pip install polars deltalake</code></p> <ol> <li>Create a dataframe with some toy data.</li> </ol> <p><code>df = pl.DataFrame({'x': [1, 2, 3]})</code></p> <ol> <li>Set your <code>storage_options</code> correctly.</li> </ol> <pre><code>storage_options = {\n    'AWS_SECRET_ACCESS_KEY': &lt;access_key&gt;,\n    'aws_conditional_put': 'etag', # Here we say to use conditional put, this provides safe concurrency.\n}\n</code></pre> <ol> <li>Write data to Delta table using the <code>storage_options</code> kwarg.</li> </ol> <pre><code>df.write_delta(\n    \"s3://bucket/delta_table\",\n    storage_options=storage_options,\n)\n</code></pre>"},{"location":"integrations/object-storage/s3-like/#minio-and-docker","title":"Minio and Docker","text":"<p>Minio is straightforward to host locally with Docker and docker-compose, via the following <code>docker-compose.yml</code> file - just run <code>docker-compose up</code>:</p> <pre><code>version: '3.8'\n\nservices:\n  minio:\n    image: minio/minio\n    ports:\n      - \"9000:9000\"\n      - \"9001:9001\"\n    environment:\n      MINIO_ROOT_USER: ...\n      MINIO_ROOT_PASSWORD: ...\n    command: server /data --console-address \":9001\"\n</code></pre> <p>With this configuration, Minio will host its S3-compatible API over HTTP, not HTTPS, on port 9000.  This requires an additional flag in <code>storage_options</code>, <code>AWS_ALLOW_HTTP</code>, to be set to <code>true</code>:</p> <pre><code>storage_options = {\n    'AWS_ACCESS_KEY_ID': ...,\n    'AWS_SECRET_ACCESS_KEY': ...,\n    'AWS_ENDPOINT_URL': 'http://localhost:9000',\n    'AWS_ALLOW_HTTP': 'true',\n    'aws_conditional_put': 'etag'\n}\n</code></pre>"},{"location":"integrations/object-storage/s3/","title":"AWS S3 Storage Backend","text":"<p><code>delta-rs</code> offers native support for using AWS S3 as an object storage backend.</p> <p>You don\u2019t need to install any extra dependencies to read/write Delta tables to S3 with engines that use <code>delta-rs</code>. You do need to configure your AWS access credentials correctly.</p>"},{"location":"integrations/object-storage/s3/#note-for-boto3-users","title":"Note for boto3 users","text":"<p>Many Python engines use boto3 to connect to AWS. This library supports reading credentials automatically from your local <code>.aws/config</code> or <code>.aws/creds</code> file.</p> <p>For example, if you\u2019re running locally with the proper credentials in your local <code>.aws/config</code> or <code>.aws/creds</code> file then you can write a Parquet file to S3 like this with pandas:</p> <pre><code>    import pandas as pd\n    df = pd.DataFrame({'x': [1, 2, 3]})\n    df.to_parquet(\"s3://avriiil/parquet-test-pandas\")\n</code></pre> <p>The <code>delta-rs</code> writer does not use <code>boto3</code> and therefore does not support taking credentials from your <code>.aws/config</code> or <code>.aws/creds</code> file. If you\u2019re used to working with writers from Python engines like Polars, pandas or Dask, this may mean a small change to your workflow.</p>"},{"location":"integrations/object-storage/s3/#passing-aws-credentials","title":"Passing AWS Credentials","text":"<p>You can pass your AWS credentials explicitly by using:</p> <ul> <li>the <code>storage_options</code>kwarg</li> <li>Environment variables</li> <li>EC2 metadata if using EC2 instances</li> <li>AWS Profiles</li> </ul>"},{"location":"integrations/object-storage/s3/#example","title":"Example","text":"<p>Let's work through an example with Polars. The same logic applies to other Python engines like Pandas, Daft, Dask, etc.</p> <p>Follow the steps below to use Delta Lake on S3 with Polars:</p> <ol> <li>Install Polars and deltalake. For example, using:</li> </ol> <p><code>pip install polars deltalake</code></p> <ol> <li>Create a dataframe with some toy data.</li> </ol> <p><code>df = pl.DataFrame({'x': [1, 2, 3]})</code></p> <ol> <li>Set your <code>storage_options</code> correctly.</li> </ol> <pre><code>storage_options = {\n    \"AWS_REGION\":&lt;region_name&gt;,\n    'AWS_ACCESS_KEY_ID': &lt;key_id&gt;,\n    'AWS_SECRET_ACCESS_KEY': &lt;access_key&gt;,\n    'AWS_S3_LOCKING_PROVIDER': 'dynamodb',\n    'DELTA_DYNAMO_TABLE_NAME': 'delta_log',\n}\n</code></pre> <ol> <li>Write data to Delta table using the <code>storage_options</code> kwarg.</li> </ol> <pre><code>df.write_delta(\n    \"s3://bucket/delta_table\",\n    storage_options=storage_options,\n)\n</code></pre>"},{"location":"integrations/object-storage/s3/#delta-lake-on-aws-s3-safe-concurrent-writes","title":"Delta Lake on AWS S3: Safe Concurrent Writes","text":"<p>You need a locking provider to ensure safe concurrent writes when writing Delta tables to AWS S3. This is because AWS S3 does not guarantee mutual exclusion.</p> <p>A locking provider guarantees that only one writer is able to create the same file. This prevents corrupted or conflicting data.</p> <p><code>delta-rs</code> uses DynamoDB to guarantee safe concurrent writes.</p> <p>Run the code below in your terminal to create a DynamoDB table that will act as your locking provider.</p> <pre><code>    aws dynamodb create-table \\\n    --table-name delta_log \\\n    --attribute-definitions AttributeName=tablePath,AttributeType=S AttributeName=fileName,AttributeType=S \\\n    --key-schema AttributeName=tablePath,KeyType=HASH AttributeName=fileName,KeyType=RANGE \\\n    --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5\n</code></pre> <p>If for some reason you don't want to use DynamoDB as your locking mechanism you can choose to set the <code>AWS_S3_ALLOW_UNSAFE_RENAME</code> variable to <code>true</code> in order to enable S3 unsafe writes.</p> <p>Read more in the Usage section.</p>"},{"location":"integrations/object-storage/s3/#delta-lake-on-aws-s3-required-permissions","title":"Delta Lake on AWS S3: Required permissions","text":"<p>You need to have permissions to get, put and delete objects in the S3 bucket you're storing your data in. Please note that you must be allowed to delete objects even if you're just appending to the Delta Lake, because there are temporary files into the log folder that are deleted after usage.</p> <p>In AWS S3, you will need the following permissions:</p> <ul> <li>s3:GetObject</li> <li>s3:PutObject</li> <li>s3:DeleteObject</li> </ul> <p>In DynamoDB, you will need the following permissions:</p> <ul> <li>dynamodb:GetItem</li> <li>dynamodb:Query</li> <li>dynamodb:PutItem</li> <li>dynamodb:UpdateItem</li> </ul>"},{"location":"usage/appending-overwriting-delta-lake-table/","title":"Appending to and overwriting a Delta Lake table","text":"<p>This section explains how to append to an exising Delta table and how to overwrite a Delta table.</p>"},{"location":"usage/appending-overwriting-delta-lake-table/#delta-lake-append-transactions","title":"Delta Lake append transactions","text":"<p>Suppose you have a Delta table with the following contents:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n+-------+----------+\n</code></pre> <p>Append two additional rows of data to the table:</p> <pre><code>from deltalake import write_deltalake, DeltaTable\n\ndf = pd.DataFrame({\"num\": [8, 9], \"letter\": [\"dd\", \"ee\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"append\")\n</code></pre> <p>Here are the updated contents of the Delta table:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre> <p>Now let's see how to perform an overwrite transaction.</p>"},{"location":"usage/appending-overwriting-delta-lake-table/#delta-lake-overwrite-transactions","title":"Delta Lake overwrite transactions","text":"<p>Now let's see how to overwrite the exisitng Delta table.</p> <pre><code>df = pd.DataFrame({\"num\": [11, 22], \"letter\": [\"aa\", \"bb\"]})\nwrite_deltalake(\"tmp/some-table\", df, mode=\"overwrite\")\n</code></pre> <p>Here are the contents of the Delta table after the overwrite operation:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|    11 | aa       |\n|    22 | bb       |\n+-------+----------+\n</code></pre> <p>Overwriting just performs a logical delete.  It doesn't physically remove the previous data from storage.  Time travel back to the previous version to confirm that the old version of the table is still accessable.</p> <pre><code>dt = DeltaTable(\"tmp/some-table\", version=1)\n\n+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     8 | dd       |\n|     9 | ee       |\n+-------+----------+\n</code></pre>"},{"location":"usage/constraints/","title":"Adding a Constraint to a table","text":"<p>Check constraints are a way to enforce that only data that meets the constraint is allowed to be added to the table.</p>"},{"location":"usage/constraints/#add-the-constraint","title":"Add the Constraint","text":"Python Rust <p> <code>DeltaTable</code> <pre><code>from deltalake import DeltaTable\n\ndt = DeltaTable(\"../rust/tests/data/simple_table\")\n\n# Check the schema before hand\nprint(dt.schema())\n# Add the constraint to the table.\ndt.alter.add_constraint({\"id_gt_0\": \"id &gt; 0\"})\n</code></pre></p> <p> <code>DeltaTable</code> <pre><code>let table = deltalake::open_table(\"../rust/tests/data/simple_table\").await?;\nlet ops = DeltaOps(table);\nops.with_constraint(\"id_gt_0\", \"id &gt; 0\").await?;\n</code></pre></p> <p>After you have added the constraint to the table attempting to append data to the table that violates the constraint will instead throw an error.</p>"},{"location":"usage/constraints/#verify-the-constraint-by-trying-to-add-some-data","title":"Verify the constraint by trying to add some data","text":"Python Rust <pre><code>from deltalake import write_deltalake, DeltaTable\nimport pandas as pd\n\ndt = DeltaTable(\"../rust/tests/data/simple_table\")\n\ndf = pd.DataFrame({\"id\": [-1]})\nwrite_deltalake(dt, df, mode=\"append\", engine=\"rust\")\n# _internal.DeltaProtocolError: Invariant violations: [\"Check or Invariant (id &gt; 0) violated by value in row: [-1]\"]\n</code></pre> <pre><code>let table = deltalake::open_table(\"../rust/tests/data/simple_table\").await?;\nlet schema = table.get_state().arrow_schema()?;\nlet invalid_values: Vec&lt;Arc&lt;dyn Array&gt;&gt; = vec![\n    Arc::new(Int32Array::from(vec![-10]))\n];\nlet batch = RecordBatch::try_new(schema, invalid_values)?;\ntable.write(vec![batch]).await?;\n</code></pre> <p>Note: ensure you use the <code>engine='rust'</code> parameter when writing to the table as this feature is not supported in the default pyarrow writer. </p>"},{"location":"usage/create-delta-lake-table/","title":"Creating a Delta Lake Table","text":"<p>This section explains how to create a Delta Lake table.</p> <p>You can easily write a DataFrame to a Delta table.</p> pandasPolars <pre><code>from deltalake import write_deltalake\nimport pandas as pd\n\ndf = pd.DataFrame({\"num\": [1, 2, 3], \"letter\": [\"a\", \"b\", \"c\"]})\nwrite_deltalake(\"tmp/some-table\", df)\n</code></pre> <pre><code>import polars as pl\n\ndf = pl.DataFrame({\"num\": [1, 2, 3], \"letter\": [\"a\", \"b\", \"c\"]})\ndf.write_delta(\"tmp/some-table\")\n</code></pre> <p>Here are the contents of the Delta table in storage:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n+-------+----------+\n</code></pre>"},{"location":"usage/deleting-rows-from-delta-lake-table/","title":"Deleting rows from a Delta Lake table","text":"<p>This section explains how to delete rows from a Delta Lake table.</p> <p>Suppose you have the following Delta table with four rows:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n|     3 | c        |\n|     4 | d        |\n+-------+----------+\n</code></pre> <p>Here's how to delete all the rows where the <code>num</code> is greater than 2:</p> <pre><code>dt = DeltaTable(\"tmp/my-table\")\ndt.delete(\"num &gt; 2\")\n</code></pre> <p>Here are the contents of the Delta table after the delete operation has been performed:</p> <pre><code>+-------+----------+\n|   num | letter   |\n|-------+----------|\n|     1 | a        |\n|     2 | b        |\n+-------+----------+\n</code></pre> <p><code>dt.delete()</code> accepts any <code>SQL where</code> clause. If no predicate is provided, all rows will be deleted.</p> <p>Read more in the API docs</p>"},{"location":"usage/examining-table/","title":"Examining a Table","text":""},{"location":"usage/examining-table/#metadata","title":"Metadata","text":"<p>The delta log maintains basic metadata about a table, including:</p> <ul> <li>A unique <code>id</code></li> <li>A <code>name</code>, if provided</li> <li>A <code>description</code>, if provided</li> <li>The list of <code>partitionColumns</code>.</li> <li>The <code>created_time</code> of the table</li> <li>A map of table <code>configuration</code>. This includes fields such as     <code>delta.appendOnly</code>, which if <code>true</code> indicates the table is not meant     to have data deleted from it.</li> </ul> <p>Get metadata from a table with the DeltaTable.metadata() method:</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/simple_table\")\n&gt;&gt;&gt; dt.metadata()\nMetadata(id: 5fba94ed-9794-4965-ba6e-6ee3c0d22af9, name: None, description: None, partitionColumns: [], created_time: 1587968585495, configuration={})\n</code></pre>"},{"location":"usage/examining-table/#schema","title":"Schema","text":"<p>The schema for the table is also saved in the transaction log. It can either be retrieved in the Delta Lake form as Schema or as a PyArrow schema. The first allows you to introspect any column-level metadata stored in the schema, while the latter represents the schema the table will be loaded into.</p> <p>Use DeltaTable.schema to retrieve the delta lake schema:</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/simple_table\")\n&gt;&gt;&gt; dt.schema()\nSchema([Field(id, PrimitiveType(\"long\"), nullable=True)])\n</code></pre> <p>These schemas have a JSON representation that can be retrieved. To reconstruct from json, use DeltaTable.schema.to_json().</p> <pre><code>&gt;&gt;&gt; dt.schema().to_json()\n'{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}'\n</code></pre> <p>Use DeltaTable.schema.to_pyarrow() to retrieve the PyArrow schema:</p> <pre><code>&gt;&gt;&gt; dt.schema().to_pyarrow()\nid: int64\n</code></pre>"},{"location":"usage/examining-table/#history","title":"History","text":"<p>Depending on what system wrote the table, the delta table may have provenance information describing what operations were performed on the table, when, and by whom. This information is retained for 30 days by default, unless otherwise specified by the table configuration <code>delta.logRetentionDuration</code>.</p> <p>Note</p> <p>This information is not written by all writers and different writers may use different schemas to encode the actions. For Spark\\'s format, see: https://docs.delta.io/latest/delta-utility.html#history-schema</p> <p>To view the available history, use <code>DeltaTable.history</code>:</p> <pre><code>from deltalake import DeltaTable\n\ndt = DeltaTable(\"../rust/tests/data/simple_table\")\ndt.history()\n</code></pre> <pre><code>[{'timestamp': 1587968626537, 'operation': 'DELETE', 'operationParameters': {'predicate': '[\"((`id` % CAST(2 AS BIGINT)) = CAST(0 AS BIGINT))\"]'}, 'readVersion': 3, 'isBlindAppend': False},\n {'timestamp': 1587968614187, 'operation': 'UPDATE', 'operationParameters': {'predicate': '((id#697L % cast(2 as bigint)) = cast(0 as bigint))'}, 'readVersion': 2, 'isBlindAppend': False},\n {'timestamp': 1587968604143, 'operation': 'WRITE', 'operationParameters': {'mode': 'Overwrite', 'partitionBy': '[]'}, 'readVersion': 1, 'isBlindAppend': False},\n {'timestamp': 1587968596254, 'operation': 'MERGE', 'operationParameters': {'predicate': '(oldData.`id` = newData.`id`)'}, 'readVersion': 0, 'isBlindAppend': False},\n {'timestamp': 1587968586154, 'operation': 'WRITE', 'operationParameters': {'mode': 'ErrorIfExists', 'partitionBy': '[]'}, 'isBlindAppend': True}]\n</code></pre>"},{"location":"usage/examining-table/#current-add-actions","title":"Current Add Actions","text":"<p>The active state for a delta table is determined by the Add actions, which provide the list of files that are part of the table and metadata about them, such as creation time, size, and statistics. You can get a data frame of the add actions data using <code>DeltaTable.get_add_actions</code>:</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/delta-0.8.0\")\n&gt;&gt;&gt; dt.get_add_actions(flatten=True).to_pandas()\n                                                    path  size_bytes   modification_time  data_change  num_records  null_count.value  min.value  max.value\n0  part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a...         440 2021-03-06 15:16:07         True            2                 0          0          2\n1  part-00000-04ec9591-0b73-459e-8d18-ba5711d6cbe...         440 2021-03-06 15:16:16         True            2                 0          2          4\n</code></pre> <p>This works even with past versions of the table:</p> <pre><code>&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/delta-0.8.0\", version=0)\n&gt;&gt;&gt; dt.get_add_actions(flatten=True).to_pandas()\n                                                path  size_bytes   modification_time  data_change  num_records  null_count.value  min.value  max.value\n0  part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a...         440 2021-03-06 15:16:07         True            2                 0          0          2\n1  part-00001-911a94a2-43f6-4acb-8620-5e68c265498...         445 2021-03-06 15:16:07         True            3                 0          2          4\n</code></pre>"},{"location":"usage/installation/","title":"Installation","text":"<p>The <code>deltalake</code> project can be installed via pip for Python or Cargo for Rust.</p>"},{"location":"usage/installation/#install-delta-lake-for-python","title":"Install Delta Lake for Python","text":"<p>With pip:</p> <pre><code>pip install deltalake\n</code></pre> <p>With Conda:</p> <pre><code>conda install -c conda-forge deltalake\n</code></pre>"},{"location":"usage/installation/#install-delta-lake-for-rust","title":"Install Delta Lake for Rust","text":"<p>With Cargo:</p> <pre><code>cargo add deltalake\n</code></pre>"},{"location":"usage/installation/#run-delta-lake-and-pandas-in-a-jupyter-notebook","title":"Run Delta Lake and pandas in a Jupyter Notebook","text":"<p>You can easily run Delta Lake and pandas in a Jupyter notebook.</p> <p>Create an environment file with the dependencies as follows:</p> <pre><code>name: deltalake-minimal\nchannels:\n  - conda-forge\n  - defaults\ndependencies:\n  - python=3.11\n  - ipykernel\n  - pandas\n  - polars\n  - jupyterlab\n  - deltalake\n</code></pre> <p>Create a virtual environment with the dependencies:</p> <pre><code>conda env create -f deltalake-minimal.yml\n</code></pre> <p>Open the Jupyter notebook and run commands as follows:</p> <p></p>"},{"location":"usage/loading-table/","title":"Loading a Delta Table","text":"<p>A DeltaTable represents the state of a delta table at a particular version. This includes which files are currently part of the table, the schema of the table, and other metadata such as creation time.</p>  Python Rust <p> <code>DeltaTable</code> <pre><code>from deltalake import DeltaTable\n\ndt = DeltaTable(\"../rust/tests/data/delta-0.2.0\")\nprint(f\"Version: {dt.version()}\")\nprint(f\"Files: {dt.files()}\")\n</code></pre></p> <p> <code>DeltaTable</code> <pre><code>let table = deltalake::open_table(\"../rust/tests/data/simple_table\").await.unwrap();\nprintln!(\"Version: {}\", table.version());\nprintln!(\"Files: {}\", table.get_files());\n</code></pre></p> <p>Depending on your storage backend, you could use the <code>storage_options</code> parameter to provide some configuration. Configuration is defined for specific backends - s3 options, azure options, gcs options.</p> <pre><code>&gt;&gt;&gt; storage_options = {\"AWS_ACCESS_KEY_ID\": \"THE_AWS_ACCESS_KEY_ID\", \"AWS_SECRET_ACCESS_KEY\":\"THE_AWS_SECRET_ACCESS_KEY\"}\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/delta-0.2.0\", storage_options=storage_options)\n</code></pre> <p>The configuration can also be provided via the environment, and the basic service provider is derived from the URL being used. We try to support many of the well-known formats to identify basic service properties.</p> <p>S3:</p> <ul> <li>s3://\\&lt;bucket&gt;/\\&lt;path&gt;</li> <li>s3a://\\&lt;bucket&gt;/\\&lt;path&gt;</li> </ul> <p>Note that <code>delta-rs</code> does not read credentials from a local <code>.aws/config</code> or <code>.aws/creds</code> file. Credentials can be accessed from environment variables, ec2 metadata, profiles or web identity. You can also pass credentials to <code>storage_options</code> using <code>AWS_ACCESS_KEY_ID</code> and <code>AWS_SECRET_ACCESS_KEY</code>.</p> <p>Azure:</p> <ul> <li>az://\\&lt;container&gt;/\\&lt;path&gt;</li> <li>adl://\\&lt;container&gt;/\\&lt;path&gt;</li> <li>abfs://\\&lt;container&gt;/\\&lt;path&gt;</li> </ul> <p>GCS:</p> <ul> <li>gs://\\&lt;bucket&gt;/\\&lt;path&gt;</li> </ul> <p>Alternatively, if you have a data catalog you can load it by reference to a database and table name. Currently only AWS Glue is supported.</p> <p>For AWS Glue catalog, use AWS environment variables to authenticate.</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; from deltalake import DataCatalog\n&gt;&gt;&gt; database_name = \"simple_database\"\n&gt;&gt;&gt; table_name = \"simple_table\"\n&gt;&gt;&gt; data_catalog = DataCatalog.AWS\n&gt;&gt;&gt; dt = DeltaTable.from_data_catalog(data_catalog=data_catalog, database_name=database_name, table_name=table_name)\n&gt;&gt;&gt; dt.to_pyarrow_table().to_pydict()\n{'id': [5, 7, 9, 5, 6, 7, 8, 9]}\n</code></pre>"},{"location":"usage/loading-table/#verify-table-existence","title":"Verify Table Existence","text":"<p>You can check whether or not a Delta table exists at a particular path by using the <code>DeltaTable.is_deltatable()</code> method.</p> <pre><code>from deltalake import DeltaTable\n\ntable_path = \"&lt;path/to/valid/table&gt;\"\nDeltaTable.is_deltatable(table_path)\n# True\n\ninvalid_table_path = \"&lt;path/to/nonexistent/table&gt;\"\nDeltaTable.is_deltatable(invalid_table_path)\n# False\n\nbucket_table_path = \"&lt;path/to/valid/table/in/bucket&gt;\"\nstorage_options = {\n    \"AWS_ACCESS_KEY_ID\": \"THE_AWS_ACCESS_KEY_ID\",\n    \"AWS_SECRET_ACCESS_KEY\": \"THE_AWS_SECRET_ACCESS_KEY\",\n    ...\n}\nDeltaTable.is_deltatable(bucket_table_path)\n# True\n</code></pre>"},{"location":"usage/loading-table/#custom-storage-backends","title":"Custom Storage Backends","text":"<p>While delta always needs its internal storage backend to work and be properly configured, in order to manage the delta log, it may sometime be advantageous - and is common practice in the arrow world - to customize the storage interface used for reading the bulk data.</p> <p><code>deltalake</code> will work with any storage compliant with <code>pyarrow.fs.FileSystem</code>, however the root of the filesystem has to be adjusted to point at the root of the Delta table. We can achieve this by wrapping the custom filesystem into a <code>pyarrow.fs.SubTreeFileSystem</code>.</p> <pre><code>import pyarrow.fs as fs\nfrom deltalake import DeltaTable\n\npath = \"&lt;path/to/table&gt;\"\nfilesystem = fs.SubTreeFileSystem(path, fs.LocalFileSystem())\n\ndt = DeltaTable(path)\nds = dt.to_pyarrow_dataset(filesystem=filesystem)\n</code></pre> <p>When using the pyarrow factory method for file systems, the normalized path is provided on creation. In case of S3 this would look something like:</p> <pre><code>import pyarrow.fs as fs\nfrom deltalake import DeltaTable\n\ntable_uri = \"s3://&lt;bucket&gt;/&lt;path&gt;\"\nraw_fs, normalized_path = fs.FileSystem.from_uri(table_uri)\nfilesystem = fs.SubTreeFileSystem(normalized_path, raw_fs)\n\ndt = DeltaTable(table_uri)\nds = dt.to_pyarrow_dataset(filesystem=filesystem)\n</code></pre>"},{"location":"usage/loading-table/#time-travel","title":"Time Travel","text":"<p>To load previous table states, you can provide the version number you wish to load:</p> <pre><code>&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/simple_table\", version=2)\n</code></pre> <p>Once you\\'ve loaded a table, you can also change versions using either a version number or datetime string:</p> <pre><code>&gt;&gt;&gt; dt.load_version(1)\n&gt;&gt;&gt; dt.load_with_datetime(\"2021-11-04 00:05:23.283+00:00\")\n</code></pre> <p>Warning</p> <p>Previous table versions may not exist if they have been vacuumed, in which case an exception will be thrown. See Vacuuming tables for more information.</p>"},{"location":"usage/managing-tables/","title":"Managing Delta Tables","text":""},{"location":"usage/managing-tables/#vacuuming-tables","title":"Vacuuming tables","text":"<p>Vacuuming a table will delete any files that have been marked for deletion. This may make some past versions of a table invalid, so this can break time travel. However, it will save storage space. Vacuum will retain files in a certain window, by default one week, so time travel will still work in shorter ranges.</p> <p>Delta tables usually don't delete old files automatically, so vacuuming regularly is considered good practice, unless the table is only appended to.</p> <p>Use <code>DeltaTable.vacuum</code> to perform the vacuum operation. Note that to prevent accidental deletion, the function performs a dry-run by default: it will only list the files to be deleted. Pass <code>dry_run=False</code> to actually delete files.</p> <pre><code>&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/simple_table\")\n&gt;&gt;&gt; dt.vacuum()\n['../rust/tests/data/simple_table/part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet',\n '../rust/tests/data/simple_table/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet',\n '../rust/tests/data/simple_table/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet',\n ...]\n&gt;&gt;&gt; dt.vacuum(dry_run=False) # Don't run this unless you are sure!\n</code></pre>"},{"location":"usage/managing-tables/#optimizing-tables","title":"Optimizing tables","text":"<p>Optimizing a table compacts small files into larger files to avoid the small file problem. This is especially important for tables that get small amounts of data appended to with high frequency. In addition to compacting small files, you can colocate similar data in the same files with Z Ordering, which allows for better file skipping and faster queries.</p> <p>A table <code>dt = DeltaTable(...)</code> has two methods for optimizing it:</p> <ul> <li><code>dt.optimize.compact()</code> for compacting small files,</li> <li><code>dt.optimize.z_order()</code> to compact and apply Z Ordering.</li> </ul> <p>See the section Small file compaction for more information and a detailed example on <code>compact</code>, and the section Z Order for more information on <code>z_order</code>.</p>"},{"location":"usage/merging-tables/","title":"Merging a Table","text":"<p>Delta Lake <code>MERGE</code> operations allow you to merge source data into a target table based on specific conditions. <code>MERGE</code> operations are great for making selective changes to your Delta table without having to rewrite the entire table.</p> <p>Use <code>dt.merge()</code> with a single or multiple conditional statements to perform CRUD operations (Create, Read, Update, Delete) at scale. You can also use Delta Lake merge for efficient Change Data Capture (CDC), Slowly Changing Dimensions (SDC) operations and to ensure GDPR compliance.</p>"},{"location":"usage/merging-tables/#basic-structure-of-merge-command","title":"Basic Structure of <code>MERGE</code> Command","text":"<p>Let\u2019s start by understanding the basic structure of a Delta Lake <code>MERGE</code> command in delta-rs.</p> <p>Use the TableMerger API to construct a <code>MERGE</code> command with one or multiple conditional clauses:</p> <pre><code>    (\n        dt.merge(                                       # target data\n            source=source_data,                         # source data\n            predicate=\"target.x = source.x\",\n            source_alias=\"source\",\n            target_alias=\"target\")\n        .when_matched_update(                           # conditional statement\n            updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n        .execute()\n    )\n</code></pre> <p>In the syntax above:</p> <ul> <li><code>dt</code> is your target Delta table</li> <li><code>source_data</code> is your source data</li> <li><code>source</code> and <code>target</code> are your respective aliases for your source and target datasets</li> <li><code>when_matched_update</code> is one of many possible conditional statements, see the sections below for more</li> <li><code>execute()</code> executes the MERGE operation with the specified settings</li> </ul> <p>Note that executing a <code>MERGE</code> operation automatically writes the changes to your Delta table in a single transaction.</p>"},{"location":"usage/merging-tables/#update","title":"Update","text":"<p>Use <code>when_matched_update</code> to perform an UPDATE operation.</p> <p>You can define the rules for the update using the <code>updates</code> keyword. If a <code>predicate</code> is passed to <code>merge()</code> then only rows which evaluate to true will be updated.</p> <p>For example, let\u2019s update the value of a column in the target table based on a matching row in the source table.</p> <pre><code>from deltalake import DeltaTable, write_deltalake\nimport pyarrow as pa\n\n# define target table\n&gt; target_data = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\n&gt; write_deltalake(\"tmp_table\", target_data)\n&gt; dt = DeltaTable(\"tmp_table\")\n&gt; dt.to_pandas().sort_values(\"x\", ignore_index=True)\n\n   x  y\n0  1  4\n1  2  5\n2  3  6\n\n# define source table\n&gt; source_data = pa.table({\"x\": [2, 3], \"y\": [5,8]})\n&gt; source_data\n\n   x  y\n0  2  5\n1  3  8\n\n# define merge logic\n&gt; (\n&gt;     dt.merge(\n&gt;         source=source_data,\n&gt;         predicate=\"target.x = source.x\",\n&gt;         source_alias=\"source\",\n&gt;         target_alias=\"target\")\n&gt;     .when_matched_update(\n&gt;         updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n&gt;     .execute()\n&gt; )\n</code></pre> <p>First, we match rows for which the <code>x</code> values are the same using <code>predicate=\"target.x = source.x\"</code>. We then update the <code>x</code> and <code>y</code> values of the matched row with the new (source) values using <code>updates={\"x\": \"source.x\", \"y\":\"source.y\"}</code>.</p> <pre><code># inspect result\n&gt; dt.to_pandas().sort_values(\"x\", ignore_index=True)\n\n   x  y\n0  1  4\n1  2  5\n2  3  8\n</code></pre> <p>The value of the <code>y</code> column has been correctly updated for the row that matches our predicate.</p> <p>You can also use <code>when_matched_update_all</code> to update all source fields to target fields. In this case, source and target are required to have the same field names.</p>"},{"location":"usage/merging-tables/#insert","title":"Insert","text":"<p>Use <code>when_not_matched_insert</code> to perform an INSERT operation.</p> <p>For example, let\u2019s say we start with the same target table:</p> <pre><code>&gt; target_data = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\n&gt; write_deltalake(\"tmp_table\", target_data)\n&gt; dt = DeltaTable(\"tmp_table\")\n\n   x  y\n0  1  4\n1  2  5\n2  3  6\n</code></pre> <p>And we want to merge only new records from our source data, without duplication:</p> <pre><code>&gt; source_data = pa.table({\"x\": [2,3,7], \"y\": [4,5,8]})\n\n   x  y\n0  2  5\n1  3  6\n2  7  8\n</code></pre> <p>The <code>MERGE</code> syntax would be as follows:</p> <pre><code>(\n    dt.merge(\n        source=source_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\")\n    .when_not_matched_insert(\n        updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n    .execute()\n)\n\n&gt; # inspect result\n&gt; print(dt.to_pandas().sort_values(\"x\", ignore_index=True))\n\n   x  y\n0  1  4\n1  2  5\n2  3  6\n3  7  8\n</code></pre> <p>The new row has been successfully added to the target dataset.</p> <p>You can also use <code>when_not_matched_insert_all</code> to insert a new row to the target table, updating all source fields to target fields. In this case, source and target are required to have the same field names.</p>"},{"location":"usage/merging-tables/#delete","title":"Delete","text":"<p>Use <code>when_matched_delete</code> to perform a DELETE operation.</p> <p>For example, given the following <code>target_data</code> and <code>source_data</code>:</p> <pre><code>target_data = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp_table\", target_data)\ndt = DeltaTable(\"tmp_table\")\nsource_data = pa.table({\"x\": [2, 3], \"deleted\": [False, True]})\n</code></pre> <p>You can delete the rows that match a predicate (in this case <code>\"deleted\" = True</code>) using:</p> <pre><code>(\n    dt.merge(\n        source=source_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\")\n    .when_matched_delete(\n        predicate=\"source.deleted = true\")\n    .execute()\n)\n</code></pre> <p>This will result in:</p> <pre><code>&gt; dt.to_pandas().sort_values(\"x\", ignore_index=True)\n\n   x  y\n0  1  4\n1  2  5\n</code></pre> <p>The matched row has been successfully deleted.</p>"},{"location":"usage/merging-tables/#upsert","title":"Upsert","text":"<p>You can combine conditional statements to perform more complex operations.</p> <p>To perform an upsert operation, use <code>when_matched_update</code> and <code>when_not_matched_insert</code> in a single <code>merge()</code> clause.</p> <p>For example:</p> <pre><code>target_data = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp_table\", target_data)\ndt = DeltaTable(\"tmp_table\")\nsource_data = pa.table({\"x\": [2, 3, 5], \"y\": [5, 8, 11]})\n\n(\n    dt.merge(\n        source=source_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\")\n    .when_matched_update(\n        updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n    .when_not_matched_insert(\n        updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n    .execute()\n)\n</code></pre> <p>This will give you the following output:</p> <pre><code>&gt; dt.to_pandas().sort_values(\"x\", ignore_index=True)\n\n   x  y\n0  1  4\n1  2  5\n2  3  8\n3  5  11\n</code></pre>"},{"location":"usage/merging-tables/#upsert-with-delete","title":"Upsert with Delete","text":"<p>Use the <code>when_matched_delete</code> or <code>when_not_matched_by_source_delete</code> methods to add a DELETE operation to your upsert. This is helpful if you want to delete stale records from the target data.</p> <p>For example, given the same <code>target_data</code> and <code>source_data</code> used in the section above:</p> <pre><code>(\n    dt.merge(\n        source=source_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\")\n    .when_matched_update(\n        updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n    .when_not_matched_insert(\n        updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n    .when_not_matched_by_source_delete()\n    .execute()\n)\n</code></pre> <p>This will result in:</p> <pre><code>&gt; dt.to_pandas().sort_values(\"x\", ignore_index=True)\n\n   x  y\n0  2  5\n1  3  8\n2  5  11\n</code></pre> <p>The row containing old data no longer present in the source dataset has been successfully deleted.</p>"},{"location":"usage/merging-tables/#multiple-matches","title":"Multiple Matches","text":"<p>Note that when multiple match conditions are met, the first condition that matches is executed.</p> <p>For example, given the following <code>target_data</code> and <code>source_data</code>:</p> <pre><code>target_data = pa.table({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\nwrite_deltalake(\"tmp_table\", target_data)\ndt = DeltaTable(\"tmp_table\")\nsource_data = pa.table({\"x\": [2, 3, 5], \"y\": [5, 8, 11]})\n</code></pre> <p>Let\u2019s perform a merge with <code>when_matched_delete</code> first, followed by <code>when_matched_update</code>:</p> <pre><code>(\n    dt.merge(\n        source=source_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\")\n    .when_matched_delete(\n        predicate=\"source.x = target.x\")\n    .when_matched_update(\n        updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n    .execute()\n)\n</code></pre> <p>This will result in:</p> <pre><code>&gt; dt.to_pandas().sort_values(\"x\", ignore_index=True)\n\n   x  y\n0  1  4\n</code></pre> <p>Let\u2019s now perform the merge with the flipped order: <code>update</code> first, then <code>delete</code>:</p> <pre><code>(\n    dt.merge(\n        source=source_data,\n        predicate=\"target.x = source.x\",\n        source_alias=\"source\",\n        target_alias=\"target\")\n    .when_matched_update(\n        updates={\"x\": \"source.x\", \"y\":\"source.y\"})\n    .when_matched_delete(\n        predicate=\"source.x = target.x\")\n    .execute()\n)\n</code></pre> <p>This will result in:</p> <pre><code>&gt; dt.to_pandas().sort_values(\"x\", ignore_index=True)\n\n   x  y\n0  1  4\n1  2  5\n2  3  8\n</code></pre>"},{"location":"usage/merging-tables/#conditional-updates","title":"Conditional Updates","text":"<p>You can perform conditional updates for rows that have no match in the source data using when_not_matched_by_source_update.</p> <p>For example, given the following target_data and source_data:</p> <pre><code>target_data = pa.table({\"1 x\": [1, 2, 3], \"1y\": [4, 5, 6]})\nwrite_deltalake(\"tmp\", target_data)\ndt = DeltaTable(\"tmp\")\nsource_data = pa.table({\"x\": [2, 3, 4]})\n</code></pre> <p>Set y = 0 for all rows that have no matches in the new source data, provided that the original y value is greater than 3:</p> <pre><code>(\n   dt.merge(\n       source=new_data,\n       predicate='target.x = source.x',\n       source_alias='source',\n       target_alias='target')\n   .when_not_matched_by_source_update(\n       predicate = \"`1y` &gt; 3\",\n       updates = {\"`1y`\": \"0\"})\n   .execute()\n)\n</code></pre> <p>This will result in:</p> <pre><code>&gt; dt.to_pandas().sort_values(\"x\", ignore_index=True)\n   x  y\n0  1  0\n1  2  5\n2  3  6\n</code></pre>"},{"location":"usage/merging-tables/#notes","title":"Notes","text":"<ul> <li>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: \"target.<code>123column</code>\" or \"target.<code>my column</code>\"</li> </ul>"},{"location":"usage/overview/","title":"Usage","text":"<p>This guide teaches you how to use Delta Lake.  You will learn how to create Delta tables, run queries, perform DML operations, and optimize your tables.</p> <p>It's easy to use Delta Lake with pandas, Polars, Rust, or any other PyArrow-like DataFrame library.</p> <p>See the Spark Delta Lake documentation if you're using Delta Lake with Spark.</p>"},{"location":"usage/querying-delta-tables/","title":"Querying Delta Tables","text":"<p>Delta tables can be queried in several ways. By loading as Arrow data or an Arrow dataset, they can be used by compatible engines such as Pandas and DuckDB. By passing on the list of files, they can be loaded into other engines such as Dask.</p> <p>Delta tables are often larger than can fit into memory on a single computer, so this module provides ways to read only the parts of the data you need. Partition filters allow you to skip reading files that are part of irrelevant partitions. Only loading the columns required also saves memory. Finally, some methods allow reading tables batch-by-batch, allowing you to process the whole table while only having a portion loaded at any given time.</p> <p>To load into Pandas or a PyArrow table use the <code>DeltaTable.to_pandas</code> and <code>DeltaTable.to_pyarrow_table</code> methods, respectively. Both of these support filtering partitions and selecting particular columns.</p> <pre><code>&gt;&gt;&gt; from deltalake import DeltaTable\n&gt;&gt;&gt; dt = DeltaTable(\"../rust/tests/data/delta-0.8.0-partitioned\")\n&gt;&gt;&gt; dt.schema().to_pyarrow()\nvalue: string\nyear: string\nmonth: string\nday: string\n&gt;&gt;&gt; dt.to_pandas(partitions=[(\"year\", \"=\", \"2021\")], columns=[\"value\"])\n      value\n0     6\n1     7\n2     5\n3     4\n&gt;&gt;&gt; dt.to_pyarrow_table(partitions=[(\"year\", \"=\", \"2021\")], columns=[\"value\"])\npyarrow.Table\nvalue: string\n</code></pre> <p>Converting to a PyArrow Dataset allows you to filter on columns other than partition columns and load the result as a stream of batches rather than a single table. Convert to a dataset using <code>DeltaTable.to_pyarrow_dataset</code>. Filters applied to datasets will use the partition values and file statistics from the Delta transaction log and push down any other filters to the scanning operation.</p> <pre><code>&gt;&gt;&gt; import pyarrow.dataset as ds\n&gt;&gt;&gt; dataset = dt.to_pyarrow_dataset()\n&gt;&gt;&gt; condition = (ds.field(\"year\") == \"2021\") &amp; (ds.field(\"value\") &gt; \"4\")\n&gt;&gt;&gt; dataset.to_table(filter=condition, columns=[\"value\"]).to_pandas()\n  value\n0     6\n1     7\n2     5\n&gt;&gt;&gt; batch_iter = dataset.to_batches(filter=condition, columns=[\"value\"], batch_size=2)\n&gt;&gt;&gt; for batch in batch_iter: print(batch.to_pandas())\n  value\n0     6\n1     7\n  value\n0     5\n</code></pre> <p>PyArrow datasets may also be passed to compatible query engines, such as DuckDB</p> <pre><code>&gt;&gt;&gt; import duckdb\n&gt;&gt;&gt; ex_data = duckdb.arrow(dataset)\n&gt;&gt;&gt; ex_data.filter(\"year = 2021 and value &gt; 4\").project(\"value\")\n---------------------\n-- Expression Tree --\n---------------------\nProjection [value]\n  Filter [year=2021 AND value&gt;4]\n    arrow_scan(140409099470144, 4828104688, 1000000)\n\n---------------------\n-- Result Columns  --\n---------------------\n- value (VARCHAR)\n\n---------------------\n-- Result Preview  --\n---------------------\nvalue\nVARCHAR\n[ Rows: 3]\n6\n7\n5\n</code></pre> <p>Finally, you can always pass the list of file paths to an engine. For example, you can pass them to <code>dask.dataframe.read_parquet</code>:</p> <pre><code>&gt;&gt;&gt; import dask.dataframe as dd\n&gt;&gt;&gt; df = dd.read_parquet(dt.file_uris())\n&gt;&gt;&gt; df\nDask DataFrame Structure:\n                value             year            month              day\nnpartitions=6\n               object  category[known]  category[known]  category[known]\n                  ...              ...              ...              ...\n...               ...              ...              ...              ...\n                  ...              ...              ...              ...\n                  ...              ...              ...              ...\nDask Name: read-parquet, 6 tasks\n&gt;&gt;&gt; df.compute()\n  value  year month day\n0     1  2020     1   1\n0     2  2020     2   3\n0     3  2020     2   5\n0     4  2021     4   5\n0     5  2021    12   4\n0     6  2021    12  20\n1     7  2021    12  20\n</code></pre>"},{"location":"usage/read-cdf/","title":"Reading the Change Data Feed from a Delta Table","text":"<p>Reading the CDF data from a table with change data is easy.</p>"},{"location":"usage/read-cdf/#reading-cdf-log","title":"Reading CDF Log","text":"Python Rust <pre><code>import polars\nfrom deltalake import DeltaTable\n\ndt = DeltaTable(\"../rust/tests/data/cdf-table\")\ntable = dt.load_cdf(starting_version=0, ending_version=4).read_all()\npt = polars.from_arrow(table)\npt.group_by(\"_commit_version\").len().sort(\"len\", descending=True)\n</code></pre> <pre><code>#[tokio::main]\nasync fn main() -&gt; Result&lt;(), Box&lt;dyn std::error::Error&gt;&gt; {\n\n    let table = deltalake::open_table(\"../rust/tests/data/cdf-table\").await?;\n    let ops = DeltaOps(table);\n    let cdf = ops.load_cdf()\n        .with_starting_version(0)\n        .with_ending_version(4)\n        .build()\n        .await?;\n\n    arrow_cast::pretty::print_batches(&amp;cdf)?;\n\n    Ok(())\n}\n</code></pre> <p>The output can then be used in various execution engines. The python example shows how one might consume the cdf feed inside polars.</p>"},{"location":"usage/optimize/delta-lake-z-order/","title":"Delta Lake Z Order","text":"<p>This section explains how to Z Order a Delta table.</p> <p>Z Ordering colocates similar data in the same files, which allows for better file skipping and faster queries.</p> <p>Suppose you have a table with <code>first_name</code>, <code>age</code>, and <code>country</code> columns.</p> <p>If you Z Order the data by the <code>country</code> column, then individuals from the same country will be stored in the same files.  When you subquently query the data for individuals from a given country, it will execute faster because more data can be skipped.</p> <p>Here's how to Z Order a Delta table:</p> <pre><code>dt = DeltaTable(\"tmp\")\ndt.optimize.z_order([country])\n</code></pre>"},{"location":"usage/optimize/small-file-compaction-with-optimize/","title":"Delta Lake small file compaction with optimize","text":"<p>This post shows you how to perform small file compaction with using the <code>optimize</code> method.  This was added to the <code>DeltaTable</code> class in version 0.9.0.  This command rearranges the small files into larger files which will reduce the number of files and speed up queries.</p> <p>This is very helpful for workloads that append frequently. For example, if you have a table that is appended to every 10 minutes, after a year you will have 52,560 files in the table. If the table is partitioned by another dimension, you will have 52,560 files per partition; with just 100 unique values that's millions of files. By running <code>optimize</code> periodically, you can reduce the number of files in the table to a more manageable number.</p> <p>Typically, you will run optimize less frequently than you append data. If possible, you might run optimize once you know you have finished writing to a particular partition. For example, on a table partitioned by date, you might append data every 10 minutes, but only run optimize once a day at the end of the day. This will ensure you don't need to compact the same data twice.</p> <p>This section will also teach you about how to use <code>vacuum</code> to physically remove files from storage that are no longer needed.  You\u2019ll often want vacuum after running optimize to remove the small files from storage once they\u2019ve been compacted into larger files.</p> <p>Let\u2019s start with an example to explain these key concepts.  All the code covered in this post is stored in this notebook in case you\u2019d like to follow along.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#create-a-delta-table-with-small-files","title":"Create a Delta table with small files","text":"<p>Let\u2019s start by creating a Delta table with a lot of small files so we can demonstrate the usefulness of the <code>optimize</code> command.</p> <p>Start by writing a function that generates on thousand rows of random data given a timestamp.</p> <pre><code>def record_observations(date: datetime) -&gt; pa.Table:\n    \"\"\"Pulls data for a certain datetime\"\"\"\n    nrows = 1000\n    return pa.table(\n        {\n            \"date\": pa.array([date.date()] * nrows),\n            \"timestamp\": pa.array([date] * nrows),\n            \"value\": pc.random(nrows),\n        }\n    )\n</code></pre> <p>Let\u2019s run this function and observe the output:</p> <pre><code>record_observations(datetime(2021, 1, 1, 12)).to_pandas()\n\n    date                timestamp   value\n0   2021-01-01  2021-01-01 12:00:00 0.3186397383362023\n1   2021-01-01  2021-01-01 12:00:00 0.04253766974259088\n2   2021-01-01  2021-01-01 12:00:00 0.9355682965171573\n\u2026\n999 2021-01-01  2021-01-01 12:00:00 0.23207037062879843\n</code></pre> <p>Let\u2019s write 100 hours worth of data to the Delta table.</p> <pre><code># Every hour starting at midnight on 2021-01-01\nhours_iter = (datetime(2021, 1, 1) + timedelta(hours=i) for i in itertools.count())\n\n# Write 100 hours worth of data\nfor timestamp in itertools.islice(hours_iter, 100):\n    write_deltalake(\n        \"observation_data\",\n        record_observations(timestamp),\n        partition_by=[\"date\"],\n        mode=\"append\",\n    )\n</code></pre> <p>This data was appended to the Delta table in 100 separate transactions, so the table will contain 100 transaction log entries and 100 data files.  You can see the number of files with the <code>files()</code> method.</p> <pre><code>dt = DeltaTable(\"observation_data\")\nlen(dt.files()) # 100\n</code></pre> <p>Here\u2019s how the files are persisted in storage.</p> <pre><code>observation_data\n\u251c\u2500\u2500 _delta_log\n\u2502   \u251c\u2500\u2500 00000000000000000000.json\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u2514\u2500\u2500 00000000000000000099.json\n\u251c\u2500\u2500 date=2021-01-01\n\u2502   \u251c\u2500\u2500 0-cfe227c6-edd9-4369-a1b0-db4559a2e693-0.parquet\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u251c\u2500\u2500 23-a4ace29e-e73e-40a1-81d3-0f5dc13093de-0.parquet\n\u251c\u2500\u2500 date=2021-01-02\n\u2502   \u251c\u2500\u2500 24-9698b456-66eb-4075-8732-fe56d81edb60-0.parquet\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u2514\u2500\u2500 47-d3fce527-e018-4c02-8acd-a649f6f523d2-0.parquet\n\u251c\u2500\u2500 date=2021-01-03\n\u2502   \u251c\u2500\u2500 48-fd90a7fa-5a14-42ed-9f59-9fe48d87899d-0.parquet\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u2514\u2500\u2500 71-5f143ade-8ae2-4854-bdc5-61154175665f-0.parquet\n\u251c\u2500\u2500 date=2021-01-04\n\u2502   \u251c\u2500\u2500 72-477c10fe-dc09-4087-80f0-56006e4a7911-0.parquet\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u2514\u2500\u2500 95-1c92cbce-8af4-4fe4-9c11-832245cf4d40-0.parquet\n\u2514\u2500\u2500 date=2021-01-05\n    \u251c\u2500\u2500 96-1b878ee5-25fd-431a-bc3e-6dcacc96b470-0.parquet\n    \u251c\u2500\u2500 \u2026\n    \u2514\u2500\u2500 99-9650ed63-c195-433d-a86b-9469088c14ba-0.parquet\n</code></pre> <p>Each of these Parquet files are tiny - they\u2019re only 10 KB.  Let\u2019s see how to compact these tiny files into larger files, which is more efficient for data queries.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#compact-small-files-in-the-delta-table-with-optimize","title":"Compact small files in the Delta table with optimize","text":"<p>Let\u2019s run the optimize command to compact the existing small files into larger files:</p> <pre><code>dt = DeltaTable(\"observation_data\")\n\ndt.optimize.compact()\n</code></pre> <p>Here\u2019s the output of the command:</p> <pre><code>{'numFilesAdded': 5,\n 'numFilesRemoved': 100,\n 'filesAdded': {'min': 39000,\n  'max': 238282,\n  'avg': 198425.6,\n  'totalFiles': 5,\n  'totalSize': 992128},\n 'filesRemoved': {'min': 10244,\n  'max': 10244,\n  'avg': 10244.0,\n  'totalFiles': 100,\n  'totalSize': 1024400},\n 'partitionsOptimized': 5,\n 'numBatches': 1,\n 'totalConsideredFiles': 100,\n 'totalFilesSkipped': 0,\n 'preserveInsertionOrder': True}\n</code></pre> <p>The optimize operation has added 5 new files and marked 100 exisitng files for removal (this is also known as \u201ctombstoning\u201d files).  It has compacted the 100 tiny files into 5 larger files.</p> <p>Let\u2019s append some more data to the Delta table and see how we can selectively run optimize on the new data that\u2019s added.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#handling-incremental-updates-with-optimize","title":"Handling incremental updates with optimize","text":"<p>Let\u2019s append another 24 hours of data to the Delta table:</p> <pre><code>for timestamp in itertools.islice(hours_iter, 24):\n    write_deltalake(\n        dt,\n        record_observations(timestamp),\n        partition_by=[\"date\"],\n        mode=\"append\",\n    )\n</code></pre> <p>We can use <code>get_add_actions()</code> to introspect the table state. We can see that <code>2021-01-06</code> has only a few hours of data so far, so we don't want to optimize that yet. But <code>2021-01-05</code> has all 24 hours of data, so it's ready to be optimized.</p> <pre><code>dt.get_add_actions(flatten=True).to_pandas()[\n    \"partition.date\"\n].value_counts().sort_index()\n\n2021-01-01     1\n2021-01-02     1\n2021-01-03     1\n2021-01-04     1\n2021-01-05    21\n2021-01-06     4\n</code></pre> <p>To optimize a single partition, you can pass in a <code>partition_filters</code> argument speficying which partitions to optimize.</p> <pre><code>dt.optimize(partition_filters=[(\"date\", \"=\", \"2021-01-05\")])\n\n{'numFilesAdded': 1,\n 'numFilesRemoved': 21,\n 'filesAdded': {'min': 238282,\n  'max': 238282,\n  'avg': 238282.0,\n  'totalFiles': 1,\n  'totalSize': 238282},\n 'filesRemoved': {'min': 10244,\n  'max': 39000,\n  'avg': 11613.333333333334,\n  'totalFiles': 21,\n  'totalSize': 243880},\n 'partitionsOptimized': 1,\n 'numBatches': 1,\n 'totalConsideredFiles': 21,\n 'totalFilesSkipped': 0,\n 'preserveInsertionOrder': True}\n</code></pre> <p>This optimize operation tombstones 21 small data files and adds one file with all the existing data properly condensed.  Let\u2019s take a look a portion of the <code>_delta_log/00000000000000000125.json</code> file, which is the transaction log entry that corresponds with this incremental optimize command.</p> <pre><code>{\n  \"remove\": {\n    \"path\": \"date=2021-01-05/part-00000-41178aab-2491-488f-943d-8f03867295ee-c000.snappy.parquet\",\n    \"deletionTimestamp\": 1683465499480,\n    \"dataChange\": false,\n    \"extendedFileMetadata\": null,\n    \"partitionValues\": {\n      \"date\": \"2021-01-05\"\n    },\n    \"size\": 39000,\n    \"tags\": null\n  }\n}\n\n{\n  \"remove\": {\n    \"path\": \"date=2021-01-05/101-79ae6fc9-c0cc-49ec-bb94-9aba879ac949-0.parquet\",\n    \"deletionTimestamp\": 1683465499481,\n    \"dataChange\": false,\n    \"extendedFileMetadata\": null,\n    \"partitionValues\": {\n      \"date\": \"2021-01-05\"\n    },\n    \"size\": 10244,\n    \"tags\": null\n  }\n}\n\n\u2026\n\n{\n  \"add\": {\n    \"path\": \"date=2021-01-05/part-00000-4b020a40-c836-4a11-851f-4691370c9f3a-c000.snappy.parquet\",\n    \"size\": 238282,\n    \"partitionValues\": {\n      \"date\": \"2021-01-05\"\n    },\n    \"modificationTime\": 1683465499493,\n    \"dataChange\": false,\n    \"stats\": \"{\\\"numRecords\\\":24000,\\\"minValues\\\":{\\\"value\\\":0.00005581532256615507,\\\"timestamp\\\":\\\"2021-01-05T00:00:00.000Z\\\"},\\\"maxValues\\\":{\\\"timestamp\\\":\\\"2021-01-05T23:00:00.000Z\\\",\\\"value\\\":0.9999911402868216},\\\"nullCount\\\":{\\\"timestamp\\\":0,\\\"value\\\":0}}\",\n    \"tags\": null\n  }\n}\n</code></pre> <p>The trasaction log indicates that many files have been tombstoned and one file is added, as expected.</p> <p>The Delta Lake optimize command \u201cremoves\u201d data by marking the data files as removed in the transaction log.  The optimize command doesn\u2019t physically delete the Parquet file from storage.  Optimize performs a \u201clogical remove\u201d not a \u201cphysical remove\u201d.</p> <p>Delta Lake uses logical operations so you can time travel back to earlier versions of your data.  You can vacuum your Delta table to physically remove Parquet files from storage if you don\u2019t need to time travel and don\u2019t want to pay to store the tombstoned files.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#vacuuming-after-optimizing","title":"Vacuuming after optimizing","text":"<p>The vacuum command deletes all files from storage that are marked for removal in the transaction log and older than the retention period which is 7 days by default.</p> <p>It\u2019s normally a good idea to have a retention period of at least 7 days.  For purposes of this example, we will set the retention period to zero, just so you can see how the files get removed from storage.  Adjusting the retention period in this manner isn\u2019t recommended for production use cases.</p> <p>Let\u2019s run the vacuum command:</p> <pre><code>dt.vacuum(retention_hours=0, enforce_retention_duration=False, dry_run=False)\n</code></pre> <p>The command returns a list of all the files that are removed from storage:</p> <pre><code>['date=2021-01-02/39-a98680f2-0e0e-4f26-a491-18b183f9eb05-0.parquet',\n 'date=2021-01-02/41-e96bc8bb-c571-484c-b534-e897424fb7da-0.parquet',\n \u2026\n 'date=2021-01-01/0-cfe227c6-edd9-4369-a1b0-db4559a2e693-0.parquet',\n 'date=2021-01-01/18-ded53418-172b-4e40-bf2e-7c8142e71bd1-0.parquet']\n</code></pre> <p>Let\u2019s look at the content of the Delta table now that all the really small files have been removed from storage:</p> <pre><code>observation_data\n\u251c\u2500\u2500 _delta_log\n\u2502   \u251c\u2500\u2500 00000000000000000000.json\n\u2502   \u251c\u2500\u2500 00000000000000000001.json\n\u2502   \u251c\u2500\u2500 \u2026\n\u2502   \u251c\u2500\u2500 00000000000000000124.json\n\u2502   \u2514\u2500\u2500 00000000000000000125.json\n\u251c\u2500\u2500 date=2021-01-01\n\u2502   \u2514\u2500\u2500 part-00000-31e3df5a-8bbe-425c-b85d-77794f922837-c000.snappy.parquet\n\u251c\u2500\u2500 date=2021-01-02\n\u2502   \u2514\u2500\u2500 part-00000-8af07878-b179-49ce-a900-d58595ffb60a-c000.snappy.parquet\n\u251c\u2500\u2500 date=2021-01-03\n\u2502   \u2514\u2500\u2500 part-00000-5e980864-b32f-4686-a58d-a75fae455c1e-c000.snappy.parquet\n\u251c\u2500\u2500 date=2021-01-04\n\u2502   \u2514\u2500\u2500 part-00000-1e82d23b-084d-47e3-9790-d68289c39837-c000.snappy.parquet\n\u251c\u2500\u2500 date=2021-01-05\n\u2502   \u2514\u2500\u2500 part-00000-4b020a40-c836-4a11-851f-4691370c9f3a-c000.snappy.parquet\n\u2514\u2500\u2500 date=2021-01-06\n    \u251c\u2500\u2500 121-0ecb5d70-4a28-4cd4-b2d2-89ee2285eaaa-0.parquet\n    \u251c\u2500\u2500 122-6b2d2758-9154-4392-b287-fe371ee507ec-0.parquet\n    \u251c\u2500\u2500 123-551d318f-4968-441f-83fc-89f98cd15daf-0.parquet\n    \u2514\u2500\u2500 124-287309d3-662e-449d-b4da-2e67b7cc0557-0.parquet\n</code></pre> <p>All the partitions only contain a single file now, except for the <code>date=2021-01-06</code> partition that has not been compacted yet.</p> <p>An entire partition won\u2019t necessarily get compacted to a single data file when optimize is run.  Each partition has data files that are condensed to the target file size.</p>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#what-causes-the-small-file-problem","title":"What causes the small file problem?","text":"<p>Delta tables can accumulate small files for a variety of reasons:</p> <ul> <li>User error: users can accidentally write files that are too small.  Users should sometimes repartition in memory before writing to disk to avoid appending files that are too small.</li> <li>Frequent appends: systems that append more often tend to append more smaller files.  A pipeline that appends every minute will generally generate ten times as many small files compared to a system that appends every ten minutes.</li> <li>Appending to partitioned data lakes with high cardinality columns can also cause small files.  If you append every hour to a table that\u2019s partitioned on a column with 1,000 distinct values, then every append could create 1,000 new files.  Partitioning by date avoids this problem because the data isn\u2019t split up across partitions in this manner.  </li> </ul>"},{"location":"usage/optimize/small-file-compaction-with-optimize/#conclusion","title":"Conclusion","text":"<p>This page showed you how to create a Delta table with many small files, compact the small files into larger files with optimize, and remove the tombstoned files from storage with vacuum.</p> <p>You also learned about how to incrementally optimize partitioned Delta tables, so you only compact newly added data.</p> <p>An excessive number of small files slows down Delta table queries, so periodic compaction is important.  Make sure to properly maintain your Delta tables, so performance does not degrade over time.</p>"},{"location":"usage/writing/","title":"Writing Delta Tables","text":"<p>For overwrites and appends, use <code>write_deltalake</code>. If the table does not already exist, it will be created. The <code>data</code> parameter will accept a Pandas DataFrame, a PyArrow Table, or an iterator of PyArrow Record Batches.</p> <pre><code>&gt;&gt;&gt; from deltalake import write_deltalake\n&gt;&gt;&gt; df = pd.DataFrame({'x': [1, 2, 3]})\n&gt;&gt;&gt; write_deltalake('path/to/table', df)\n</code></pre> <p>Note: <code>write_deltalake</code> accepts a Pandas DataFrame, but will convert it to a Arrow table before writing. See caveats in <code>pyarrow:python/pandas</code>.</p> <p>By default, writes create a new table and error if it already exists. This is controlled by the <code>mode</code> parameter, which mirrors the behavior of Spark's <code>pyspark.sql.DataFrameWriter.saveAsTable</code> DataFrame method. To overwrite pass in <code>mode='overwrite'</code> and to append pass in <code>mode='append'</code>:</p> <pre><code>&gt;&gt;&gt; write_deltalake('path/to/table', df, mode='overwrite')\n&gt;&gt;&gt; write_deltalake('path/to/table', df, mode='append')\n</code></pre> <p><code>write_deltalake</code> will raise <code>ValueError</code> if the schema of the data passed to it differs from the existing table's schema. If you wish to alter the schema as part of an overwrite pass in <code>schema_mode=\"overwrite\"</code> or <code>schema_mode=\"merge\"</code>. <code>schema_mode=\"overwrite\"</code> will completely overwrite the schema, even if columns are dropped; merge will append the new columns and fill missing columns with <code>null</code>. <code>schema_mode=\"merge\"</code> is also supported on append operations.</p>"},{"location":"usage/writing/#overwriting-a-partition","title":"Overwriting a partition","text":"<p>You can overwrite a specific partition by using <code>mode=\"overwrite\"</code> together with <code>partition_filters</code>. This will remove all files within the matching partition and insert your data as new files. This can only be done on one partition at a time. All of the input data must belong to that partition or else the method will raise an error.</p> <pre><code>&gt;&gt;&gt; from deltalake import write_deltalake\n&gt;&gt;&gt; df = pd.DataFrame({'x': [1, 2, 3], 'y': ['a', 'a', 'b']})\n&gt;&gt;&gt; write_deltalake('path/to/table', df, partition_by=['y'])\n\n&gt;&gt;&gt; table = DeltaTable('path/to/table')\n&gt;&gt;&gt; df2 = pd.DataFrame({'x': [100], 'y': ['b']})\n&gt;&gt;&gt; write_deltalake(table, df2, partition_filters=[('y', '=', 'b')], mode=\"overwrite\")\n\n&gt;&gt;&gt; table.to_pandas()\n     x  y\n0    1  a\n1    2  a\n2  100  b\n</code></pre> <p>This method could also be used to insert a new partition if one doesn't already exist, making this operation idempotent.</p>"},{"location":"usage/writing/#overwriting-part-of-the-table-data-using-a-predicate","title":"Overwriting part of the table data using a predicate","text":"<p>Note</p> <p>This predicate is often called a <code>replaceWhere</code> predicate</p> <p>When you don\u2019t specify the <code>predicate</code>, the overwrite save mode will replace the entire table.  Instead of replacing the entire table (which is costly!), you may want to overwrite only the specific parts of the table that should be changed.  In this case, you can use a <code>predicate</code> to overwrite only the relevant records or partitions.</p> <p>Note</p> <p>Data written must conform to the same predicate, i.e. not contain any records that don't match the <code>predicate</code> condition,  otherwise the operation will fail </p>  Python Rust <p> <code>replaceWhere</code> <pre><code>import pyarrow as pa\nfrom deltalake import write_deltalake\n\n# Assuming there is already a table in this location with some records where `id = '1'` which we want to overwrite\ntable_path = \"/tmp/my_table\"\ndata = pa.table(\n    {\n        \"id\": pa.array([\"1\", \"1\"], pa.string()),\n        \"value\": pa.array([11, 12], pa.int64()),\n    }\n)\nwrite_deltalake(\n    table_path,\n    data,\n    mode=\"overwrite\",\n    predicate=\"id = '1'\",\n    engine=\"rust\",\n)\n</code></pre></p> <p> <code>replaceWhere</code> <pre><code>// Assuming there is already a table in this location with some records where `id = '1'` which we want to overwrite\nuse arrow_array::RecordBatch;\nuse arrow_schema::{DataType, Field, Schema as ArrowSchema};\nuse deltalake::datafusion::logical_expr::{col, lit};\nuse deltalake::protocol::SaveMode;\nuse deltalake::DeltaOps;\n\nlet schema = ArrowSchema::new(vec![\n    Field::new(\"id\", DataType::Utf8, true),\n    Field::new(\"value\", DataType::Int32, true),\n]);\n\nlet data = RecordBatch::try_new(\n    schema.into(),\n    vec![\n        Arc::new(arrow::array::StringArray::from(vec![\"1\", \"1\"])),\n        Arc::new(arrow::array::Int32Array::from(vec![11, 12])),\n    ],\n)\n.unwrap();\n\nlet table = deltalake::open_table(\"/tmp/my_table\").await.unwrap();\nlet _table = DeltaOps(table)\n    .write(vec![data])\n    .with_save_mode(SaveMode::Overwrite)\n    .with_replace_where(col(\"id\").eq(lit(\"1\")))\n    .await\n    .unwrap();\n</code></pre></p>"},{"location":"usage/writing/#using-writer-properites","title":"Using Writer Properites","text":"<p>You can customize the Rust Parquet writer by using the WriterProperties. Additionally, you can apply extra configurations through the BloomFilterProperties and ColumnProperties data classes.</p> <p>Here's how you can do it: <pre><code>from deltalake import BloomFilterProperties, ColumnProperties, WriterProperties, write_deltalake\nimport pyarrow as pa\n\nwp = WriterProperties(\n        statistics_truncate_length=200,\n        default_column_properties=ColumnProperties(\n            bloom_filter_properties=BloomFilterProperties(True, 0.2, 30)\n        ),\n        column_properties={\n            \"value_non_bloom\": ColumnProperties(bloom_filter_properties=None),\n        },\n    )\n\ntable_path = \"/tmp/my_table\"\n\ndata = pa.table(\n        {\n            \"id\": pa.array([\"1\", \"1\"], pa.string()),\n            \"value\": pa.array([11, 12], pa.int64()),\n            \"value_non_bloom\": pa.array([11, 12], pa.int64()),\n        }\n    )\n\nwrite_deltalake(table_path, data, writer_properties=wp)\n</code></pre></p>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/","title":"Writing to S3 with a locking provider","text":"<p>Delta lake guarantees ACID transactions when writing data. This is done by default when writing to all supported object stores except AWS S3. (Some S3 clients like CloudFlare R2 or MinIO may enable concurrent writing without a locking provider, refer to this section for more information).</p> <p>When writing to S3, delta-rs provides a locking mechanism to ensure that concurrent writes are safe. This is done by default when writing to S3, but you can opt-out by setting the <code>AWS_S3_ALLOW_UNSAFE_RENAME</code> variable to <code>true</code>.</p> <p>To enable safe concurrent writes to AWS S3, we must provide an external locking mechanism.</p>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/#dynamodb","title":"DynamoDB","text":"<p>DynamoDB is the only available locking provider at the moment in delta-rs. To enable DynamoDB as the locking provider, you need to set the <code>AWS_S3_LOCKING_PROVIDER</code> to 'dynamodb' as a <code>storage_options</code> or as an environment variable.</p> <p>Additionally, you must create a DynamoDB table with the name <code>delta_log</code> so that it can be automatically recognized by delta-rs. Alternatively, you can use a table name of your choice, but you must set the <code>DELTA_DYNAMO_TABLE_NAME</code> variable to match your chosen table name. The required schema for the DynamoDB table is as follows:</p> <pre><code>\"Table\": {\n    \"AttributeDefinitions\": [\n        {\n            \"AttributeName\": \"fileName\",\n            \"AttributeType\": \"S\"\n        },\n        {\n            \"AttributeName\": \"tablePath\",\n            \"AttributeType\": \"S\"\n        }\n    ],\n    \"TableName\": \"delta_log\",\n    \"KeySchema\": [\n        {\n            \"AttributeName\": \"tablePath\",\n            \"KeyType\": \"HASH\"\n        },\n        {\n            \"AttributeName\": \"fileName\",\n            \"KeyType\": \"RANGE\"\n        }\n    ],\n}\n</code></pre> <p>Here is an example writing to s3 using this mechanism:</p> <pre><code>from deltalake import write_deltalake\ndf = pd.DataFrame({'x': [1, 2, 3]})\nstorage_options = {\n    'AWS_S3_LOCKING_PROVIDER': 'dynamodb',\n    'DELTA_DYNAMO_TABLE_NAME': 'custom_table_name'\n}\nwrite_deltalake(\n    's3a://path/to/table',\n    df,\n    storage_options=storage_options\n)\n</code></pre> <p>This locking mechanism is compatible with the one used by Apache Spark. The <code>tablePath</code> property, denoting the root url of the delta table itself, is part of the primary key, and all writers intending to write to the same table must match this property precisely. In Spark, S3 URLs are prefixed with <code>s3a://</code>, and a table in delta-rs must be configured accordingly.</p> <p>Note that <code>delta-rs</code> does not read credentials from your local <code>.aws/config</code> or <code>.aws/creds</code> file. Credentials can be accessed from environment variables, ec2 metadata, profiles or web identity. You can pass credentials to <code>storage_options</code> using <code>AWS_ACCESS_KEY_ID</code> and <code>AWS_SECRET_ACCESS_KEY</code>.</p> <p>The following code allows creating the necessary DynamoDB table from the AWS cli:</p> <pre><code>aws dynamodb create-table \\\n--table-name delta_log \\\n--attribute-definitions AttributeName=tablePath,AttributeType=S AttributeName=fileName,AttributeType=S \\\n--key-schema AttributeName=tablePath,KeyType=HASH AttributeName=fileName,KeyType=RANGE \\\n--provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5\n</code></pre> <p>You can find additional information in the Delta Lake documentation, which also includes recommendations on configuring a time-to-live (TTL) for the table to avoid growing the table indefinitely.</p>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/#enable-unsafe-writes-in-s3-opt-in","title":"Enable unsafe writes in S3 (opt-in)","text":"<p>If for some reason you don't want to use dynamodb as your locking mechanism you can choose to set the <code>AWS_S3_ALLOW_UNSAFE_RENAME</code> variable to <code>true</code> in order to enable S3 unsafe writes.</p>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/#required-permissions","title":"Required permissions","text":"<p>You need to have permissions to get, put and delete objects in the S3 bucket you're storing your data in. Please note that you must be allowed to delete objects even if you're just appending to the deltalake, because there are temporary files into the log folder that are deleted after usage.</p> <p>In AWS, those would be the required permissions:</p> <ul> <li>s3:GetObject</li> <li>s3:PutObject</li> <li>s3:DeleteObject</li> </ul> <p>In DynamoDB, you need those permissions:</p> <ul> <li>dynamodb:GetItem</li> <li>dynamodb:Query</li> <li>dynamodb:PutItem</li> <li>dynamodb:UpdateItem</li> <li>dynamodb:DeleteItem </li> </ul>"},{"location":"usage/writing/writing-to-s3-with-locking-provider/#enabling-concurrent-writes-for-alternative-clients","title":"Enabling concurrent writes for alternative clients","text":"<p>Unlike AWS S3, some S3 clients support atomic renames by passing some headers in requests.</p> <p>For CloudFlare R2 or Minio passing this in the storage_options will enable concurrent writes:</p> <pre><code>storage_options = {\n    \"conditional_put\": \"etag\",\n}\n</code></pre>"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 40ff2839cf..e2fbb04486 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,222 +2,227 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://github.com/delta-io/delta-rs/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/delta-lake-best-practices/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/delta-lake-big-data-small-data/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/why-use-delta-lake/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/catalog/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/delta_writer/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/exceptions/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/schema/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/storage/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/delta_table/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/delta_table/delta_table_alterer/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/delta_table/delta_table_merger/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/delta_table/delta_table_optimizer/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/api/delta_table/metadata/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/how-delta-lake-works/architecture-of-delta-table/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/how-delta-lake-works/delta-lake-acid-transactions/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/how-delta-lake-works/delta-lake-file-skipping/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/delta-lake-arrow/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/delta-lake-daft/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/delta-lake-dagster/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/delta-lake-dask/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/delta-lake-datafusion/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/delta-lake-pandas/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/delta-lake-polars/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/object-storage/adls/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/object-storage/gcs/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/object-storage/hdfs/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/object-storage/s3-like/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/integrations/object-storage/s3/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/appending-overwriting-delta-lake-table/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/constraints/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/create-delta-lake-table/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/deleting-rows-from-delta-lake-table/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/examining-table/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/installation/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/loading-table/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/managing-tables/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
+    <url>
+         <loc>https://github.com/delta-io/delta-rs/usage/merging-tables/</loc>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/overview/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/querying-delta-tables/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/read-cdf/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/optimize/delta-lake-z-order/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/optimize/small-file-compaction-with-optimize/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/writing/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://github.com/delta-io/delta-rs/usage/writing/writing-to-s3-with-locking-provider/</loc>
-         <lastmod>2024-10-08</lastmod>
+         <lastmod>2024-10-30</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 2cac5a06f742e3adcfda8ce32edad70125e36482..b21bb7dff5e8169a31477b35691aa9ef3a21a416 100644
GIT binary patch
literal 643
zcmV-}0(|`+iwFpgP$Fjn|8r?{Wo=<_E_iKh0M(hna@!yfhVOX_kM3Ab(nH&^m0O>n
zJ!jUiLexkQK$e}S?;<IQrqi@DIdrtT7??Hl{VQOX)&Akq+RYh5G_E@ox7%H@!A?45
zI(;a9eE%js7Z26Vz7IBH+K_P<4@Hxb|5}#AFl?hq*n*E+7jzkAL!DU8gXM1frKoN;
z^yCSq*7)qLlU1H2(lpwnrax|_YfFVTfiP}vgjnv&Y`Xj>#MC-f-R<r^iQApH-;v2Y
zwlI?obQ<e`Pn7~=ulsT_G9i_(OdqD38Wv6{ju?~h0c0`~IdK~;6i_d=VC~pkIsDLE
zMITYTiC8%>cvDL9t96<I>DYN30`1|z02eP3eWKGVGJu^9NDiRM1<=T5xh6PTZnTT`
zBUsvPEcV-b3R(=BwE#z_NNb+iX=pJ4gQ(rOVG*&#;KGTt1ldrWk*N<TTs<3b*vx;F
zvlswrlt=-(m@~rlN2h=3sf|SuPsV$?8LV83=@RMyliQc?Bm_4wWmixqw&*&pKbL%;
zKAza3C#d^;vu7&6Lr3oC3hyigR`BlllR~k0$^AiD&iGn#rf*cu89&|&YgS*F`MZLZ
z!}oFAq~2qftF^+NG32UEZZD1@4$1T+MbcZKjG10a<iQ&d67uvlb#YuNlo3Z!2iNk{
zffr~^mxb_QHnhya+VR+70c;&8UM_B-1O1B3<2cC$G-k*-<eUu-EMT8{4A-~d$jHqU
zXOxht=CLN`GV1aw^ekLZtFHk3#y@=z9qV3X@24`J7-$CZYQsU$oD;HQeaT~!6Lgap
deNHo-@{kYiZ2rq)8h;|D>K9KBOwT1D006nLH;DiM

literal 639
zcmV-_0)YJ=iwFqKfdyv*|8r?{Wo=<_E_iKh0M(hzZrd;n$M1cLz<0%Iw?j81?rl%7
zoeM^yZ7s5;QgWR1=|_LGHHLKqdKgS!A_<J}`_mFh$@_;(<2GlcXnlLgZnwK^gRS<)
zw$&l~@%@{+&mM}KeMc^0+OTmO4_Td3_?qW^-*2N$*g%L|PgNdu&7GLfgZXazB`a<=
z{0tLJt?}7AuZv-p$kS9d)!lKceUlq>36%A7gJQnV2h-(0A*RNg;%;~MS>5i`?w(EN
zv4xqgp{=kad@2lBcircUkqIe(W%@AP)NpV@b;OudAkfKbl*DZ-Wq^CJ0q4f%%HjL^
zs=A2kO~lH9A=q5AU!AW6kdB?lo_G(F09?Fi42h#x6aYH~NC}|H6I3W>xh6O|X|#*>
zBRJk|BKF&R3L2zJv;fahq_NN9G&Gn%K-R8(zsT6C_jF<{q-)MII(3A~m$QL{&HP6>
ziviHqs6^1lp(0#=cKVl|(mGV}WJBPS!OFGRHep3DxqbOgh<q=U-9VYdqE~VKwdDKs
z@gx>KLD>x_d!YgXwCsMV@ZJH5g7?RtoQl;;?GMID#;>$w`r4F|@#81u?CLLO{;pu<
z@Le1?X$aU3x3$Wjk%n8Fp}h!05{l_bisrY#6*Iq-D1(;}63X;7cX3=PToFfAl5b?{
zzza0C9fZhXw!F;2xiRdB0CpaXEEhM>LiLI)Bb*HJ`)sin0sGV;UEeZ|AUAW+HcFdP
z#+sP#B$wxkXXVMP{y6;`|MYQjyhLLAFx`o%o@Y?6HWCEQ1(qn*m!dFv#bx>EbDrrm
Z3<dIM`(GB*_!BV|zW@{HHO6uv003iMNp%1K

diff --git a/usage/appending-overwriting-delta-lake-table/index.html b/usage/appending-overwriting-delta-lake-table/index.html
index 9266a75bb3..e13a92a767 100644
--- a/usage/appending-overwriting-delta-lake-table/index.html
+++ b/usage/appending-overwriting-delta-lake-table/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -818,6 +820,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -858,7 +880,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -874,14 +896,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -960,10 +982,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -974,8 +996,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/constraints/index.html b/usage/constraints/index.html
index 74be4a743b..cf61db1e68 100644
--- a/usage/constraints/index.html
+++ b/usage/constraints/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -818,6 +820,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -858,7 +880,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -874,14 +896,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -960,10 +982,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -974,8 +996,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/create-delta-lake-table/index.html b/usage/create-delta-lake-table/index.html
index 35c3bbea1d..af201ceeaa 100644
--- a/usage/create-delta-lake-table/index.html
+++ b/usage/create-delta-lake-table/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -774,6 +776,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -814,7 +836,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -830,14 +852,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -916,10 +938,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -930,8 +952,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/deleting-rows-from-delta-lake-table/index.html b/usage/deleting-rows-from-delta-lake-table/index.html
index e6cc1b65ce..182d2fd762 100644
--- a/usage/deleting-rows-from-delta-lake-table/index.html
+++ b/usage/deleting-rows-from-delta-lake-table/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -916,10 +938,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -930,8 +952,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/examining-table/index.html b/usage/examining-table/index.html
index ed2fd67c04..778fd09a3d 100644
--- a/usage/examining-table/index.html
+++ b/usage/examining-table/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -832,6 +834,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -872,7 +894,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -888,14 +910,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -974,10 +996,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -988,8 +1010,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/installation/index.html b/usage/installation/index.html
index 4ac1bfbcfa..cd756572cf 100644
--- a/usage/installation/index.html
+++ b/usage/installation/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -825,6 +827,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -865,7 +887,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -881,14 +903,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -967,10 +989,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -981,8 +1003,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/loading-table/index.html b/usage/loading-table/index.html
index f78a1d520f..5a95c43bee 100644
--- a/usage/loading-table/index.html
+++ b/usage/loading-table/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -825,6 +827,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -865,7 +887,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -881,14 +903,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -967,10 +989,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -981,8 +1003,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/managing-tables/index.html b/usage/managing-tables/index.html
index bb0aa39870..0e62eac7fb 100644
--- a/usage/managing-tables/index.html
+++ b/usage/managing-tables/index.html
@@ -11,7 +11,7 @@
         <link rel="canonical" href="https://github.com/delta-io/delta-rs/usage/managing-tables/">
       
       
-        <link rel="prev" href="../querying-delta-tables/">
+        <link rel="prev" href="../merging-tables/">
       
       
         <link rel="next" href="../writing/">
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -763,6 +765,26 @@
                 
   
   
+  
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
     
   
   
@@ -858,7 +880,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -874,14 +896,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -960,10 +982,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -974,8 +996,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
@@ -1936,7 +1958,7 @@ <h2 id="optimizing-tables">Optimizing tables</h2>
       <nav class="md-footer__inner md-grid" aria-label="Footer" >
         
           
-          <a href="../querying-delta-tables/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Querying a table">
+          <a href="../merging-tables/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Merging a table">
             <div class="md-footer__button md-icon">
               
               <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
@@ -1946,7 +1968,7 @@ <h2 id="optimizing-tables">Optimizing tables</h2>
                 Previous
               </span>
               <div class="md-ellipsis">
-                Querying a table
+                Merging a table
               </div>
             </div>
           </a>
diff --git a/usage/merging-tables/index.html b/usage/merging-tables/index.html
new file mode 100644
index 0000000000..520a87de19
--- /dev/null
+++ b/usage/merging-tables/index.html
@@ -0,0 +1,2388 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+      
+      
+        <link rel="canonical" href="https://github.com/delta-io/delta-rs/usage/merging-tables/">
+      
+      
+        <link rel="prev" href="../querying-delta-tables/">
+      
+      
+        <link rel="next" href="../managing-tables/">
+      
+      
+      <link rel="icon" href="../../delta-rust-no-whitespace.svg">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.4.5">
+    
+    
+      
+        <title>Merging a table - Delta Lake Documentation</title>
+      
+    
+    
+      <link rel="stylesheet" href="../../assets/stylesheets/main.6a10b989.min.css">
+      
+        
+        <link rel="stylesheet" href="../../assets/stylesheets/palette.356b1318.min.css">
+      
+      
+
+
+    
+    
+      
+    
+    
+      
+        
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+      
+    
+    
+      <link rel="stylesheet" href="../../assets/_markdown_exec_ansi.css">
+    
+      <link rel="stylesheet" href="../../assets/_mkdocstrings.css">
+    
+    <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+    
+      
+
+    
+    
+    
+  </head>
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
+  
+    
+    
+      <script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#merging-a-table" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+    
+      
+
+  
+
+<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="../.." title="Delta Lake Documentation" class="md-header__button md-logo" aria-label="Delta Lake Documentation" data-md-component="logo">
+      
+  <img src="../../delta-rust-no-whitespace.svg" alt="logo">
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            Delta Lake Documentation
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              Merging a table
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      
+        <form class="md-header__option" data-md-component="palette">
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme)" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_1">
+    
+      <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_3" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m14.3 16-.7-2h-3.2l-.7 2H7.8L11 7h2l3.2 9h-1.9M20 8.69V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69m-9.15 3.96h2.3L12 9l-1.15 3.65Z"/></svg>
+      </label>
+    
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_2">
+    
+      <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
+      </label>
+    
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label="Switch to system preference"  type="radio" name="__palette" id="__palette_3">
+    
+      <label class="md-header__button md-icon" title="Switch to system preference" for="__palette_2" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
+      </label>
+    
+  
+</form>
+      
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+      </label>
+      <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
+        </button>
+      </nav>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        <a href="https://github.com/delta-io/delta-rs" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    delta-io/delta-rs
+  </div>
+</a>
+      </div>
+    
+  </nav>
+  
+    
+      
+<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
+  <div class="md-grid">
+    <ul class="md-tabs__list">
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../.." class="md-tabs__link">
+          
+  
+    
+  
+  Home
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+    
+  
+  
+    
+    
+      <li class="md-tabs__item md-tabs__item--active">
+        <a href="../installation/" class="md-tabs__link">
+          
+  
+    
+  
+  Usage
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../../api/delta_writer/" class="md-tabs__link">
+          
+  
+    
+  
+  API Reference
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+  
+    
+    
+      
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../../integrations/object-storage/adls/" class="md-tabs__link">
+          
+  
+    
+  
+  Integrations
+
+        </a>
+      </li>
+    
+  
+
+    
+  
+
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../../how-delta-lake-works/architecture-of-delta-table/" class="md-tabs__link">
+          
+  
+    
+  
+  How Delta Lake works
+
+        </a>
+      </li>
+    
+  
+
+      
+    </ul>
+  </div>
+</nav>
+    
+  
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+
+  
+
+
+<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="../.." title="Delta Lake Documentation" class="md-nav__button md-logo" aria-label="Delta Lake Documentation" data-md-component="logo">
+      
+  <img src="../../delta-rust-no-whitespace.svg" alt="logo">
+
+    </a>
+    Delta Lake Documentation
+  </label>
+  
+    <div class="md-nav__source">
+      <a href="https://github.com/delta-io/delta-rs" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    delta-io/delta-rs
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+          
+        
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../.." class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Home
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_1" id="__nav_1_label" tabindex="">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_1">
+            <span class="md-nav__icon md-icon"></span>
+            Home
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../why-use-delta-lake/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Why Use Delta Lake
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../delta-lake-big-data-small-data/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Delta Lake for big and small data
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../delta-lake-best-practices/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Best practices
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+    
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" checked>
+        
+          
+          <label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="">
+            
+  
+  <span class="md-ellipsis">
+    Usage
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_2">
+            <span class="md-nav__icon md-icon"></span>
+            Usage
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../installation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Installation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../overview/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Overview
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../create-delta-lake-table/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Creating a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../loading-table/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Loading a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../appending-overwriting-delta-lake-table/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Append/overwrite tables
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../constraints/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Adding a constraint
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../read-cdf/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Reading Change Data
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examining-table/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Examining a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../querying-delta-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Querying a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+    
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+      
+      
+        
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+      
+        
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#basic-structure-of-merge-command" class="md-nav__link">
+    Basic Structure of MERGE Command
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#update" class="md-nav__link">
+    Update
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#insert" class="md-nav__link">
+    Insert
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#delete" class="md-nav__link">
+    Delete
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#upsert" class="md-nav__link">
+    Upsert
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#upsert-with-delete" class="md-nav__link">
+    Upsert with Delete
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#multiple-matches" class="md-nav__link">
+    Multiple Matches
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#conditional-updates" class="md-nav__link">
+    Conditional Updates
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#notes" class="md-nav__link">
+    Notes
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../managing-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+          
+        
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../writing/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Writing a table
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
+            <span class="md-nav__icon md-icon"></span>
+            Writing a table
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../writing/writing-to-s3-with-locking-provider/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Writing to S3 with a locking provider
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../deleting-rows-from-delta-lake-table/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Deleting rows from a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+          
+        
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
+        
+          
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Optimize
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
+            <span class="md-nav__icon md-icon"></span>
+            Optimize
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../optimize/small-file-compaction-with-optimize/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Small file compaction
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../optimize/delta-lake-z-order/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Z Order
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+          
+        
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_3" >
+        
+          
+          <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="">
+            
+  
+  <span class="md-ellipsis">
+    API Reference
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3">
+            <span class="md-nav__icon md-icon"></span>
+            API Reference
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/delta_writer/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Writer
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+          
+        
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_3_2" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../../api/delta_table/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Table
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3_2" id="__nav_3_2_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_2">
+            <span class="md-nav__icon md-icon"></span>
+            Table
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/delta_table/metadata/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Metadata
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/delta_table/delta_table_merger/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TableMerger
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/delta_table/delta_table_optimizer/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TableOptimizer
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/delta_table/delta_table_alterer/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TableAlterer
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/schema/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Schema
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/storage/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Storage
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/catalog/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Catalog
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../api/exceptions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Exceptions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+          
+        
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4" >
+        
+          
+          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="">
+            
+  
+  <span class="md-ellipsis">
+    Integrations
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_4">
+            <span class="md-nav__icon md-icon"></span>
+            Integrations
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+          
+        
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4_1" >
+        
+          
+          <label class="md-nav__link" for="__nav_4_1" id="__nav_4_1_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Object Storage
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_4_1">
+            <span class="md-nav__icon md-icon"></span>
+            Object Storage
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/object-storage/adls/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Azure ADLS Storage Backend
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/object-storage/gcs/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    GCS Storage Backend
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/object-storage/hdfs/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    HDFS Storage Backend
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/object-storage/s3/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    AWS S3 Storage Backend
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/object-storage/s3-like/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    CloudFlare R2 & Minio
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/delta-lake-arrow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Arrow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/delta-lake-daft/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Daft
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/delta-lake-dagster/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Dagster
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/delta-lake-dask/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Dask
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/delta-lake-datafusion/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    DataFusion
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/delta-lake-pandas/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    pandas
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../integrations/delta-lake-polars/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Polars
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+    
+    
+    
+    
+      
+      
+    
+    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+      
+        
+        
+        
+          
+        
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
+        
+          
+          <label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="">
+            
+  
+  <span class="md-ellipsis">
+    How Delta Lake works
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_5">
+            <span class="md-nav__icon md-icon"></span>
+            How Delta Lake works
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../how-delta-lake-works/architecture-of-delta-table/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../how-delta-lake-works/delta-lake-acid-transactions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Transactions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../how-delta-lake-works/delta-lake-file-skipping/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    File skipping
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#basic-structure-of-merge-command" class="md-nav__link">
+    Basic Structure of MERGE Command
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#update" class="md-nav__link">
+    Update
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#insert" class="md-nav__link">
+    Insert
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#delete" class="md-nav__link">
+    Delete
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#upsert" class="md-nav__link">
+    Upsert
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#upsert-with-delete" class="md-nav__link">
+    Upsert with Delete
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#multiple-matches" class="md-nav__link">
+    Multiple Matches
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#conditional-updates" class="md-nav__link">
+    Conditional Updates
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#notes" class="md-nav__link">
+    Notes
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+                
+                  
+
+  
+  
+
+
+<h1 id="merging-a-table">Merging a Table</h1>
+<p>Delta Lake <code>MERGE</code> operations allow you to merge source data into a target table based on specific conditions. <code>MERGE</code> operations are great for making selective changes to your Delta table without having to rewrite the entire table.</p>
+<p>Use <a class="autorefs autorefs-internal" href="../../api/delta_table/#deltalake.DeltaTable.merge"><code>dt.merge()</code></a> with a single or multiple conditional statements to perform CRUD operations (Create, Read, Update, Delete) at scale. You can also use Delta Lake merge for efficient Change Data Capture (CDC), Slowly Changing Dimensions (SDC) operations and to ensure GDPR compliance.</p>
+<h2 id="basic-structure-of-merge-command">Basic Structure of <code>MERGE</code> Command</h2>
+<p>Let’s start by understanding the basic structure of a Delta Lake <code>MERGE</code> command in delta-rs.</p>
+<p>Use the <a href="https://delta-io.github.io/delta-rs/api/delta_table/delta_table_merger/">TableMerger API</a> to construct a <code>MERGE</code> command with one or multiple conditional clauses:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>    <span class="p">(</span>
+</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>        <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>                                       <span class="c1"># target data</span>
+</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a>            <span class="n">source</span><span class="o">=</span><span class="n">source_data</span><span class="p">,</span>                         <span class="c1"># source data</span>
+</span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a>            <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;target.x = source.x&quot;</span><span class="p">,</span>
+</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a>            <span class="n">source_alias</span><span class="o">=</span><span class="s2">&quot;source&quot;</span><span class="p">,</span>
+</span><span id="__span-0-6"><a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a>            <span class="n">target_alias</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)</span>
+</span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a>        <span class="o">.</span><span class="n">when_matched_update</span><span class="p">(</span>                           <span class="c1"># conditional statement</span>
+</span><span id="__span-0-8"><a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a>            <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-0-9"><a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a>        <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a>    <span class="p">)</span>
+</span></code></pre></div>
+<p>In the syntax above:</p>
+<ul>
+<li><code>dt</code> is your target Delta table</li>
+<li><code>source_data</code> is your source data</li>
+<li><code>source</code> and <code>target</code> are your respective aliases for your source and target datasets</li>
+<li><code>when_matched_update</code> is one of many possible conditional statements, see the sections below for more</li>
+<li><code>execute()</code> executes the MERGE operation with the specified settings</li>
+</ul>
+<p>Note that executing a <code>MERGE</code> operation automatically writes the changes to your Delta table in a single transaction.</p>
+<h2 id="update">Update</h2>
+<p>Use <a class="autorefs autorefs-internal" href="../../api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_update"><code>when_matched_update</code></a> to perform an UPDATE operation.</p>
+<p>You can define the rules for the update using the <code>updates</code> keyword. If a <code>predicate</code> is passed to <code>merge()</code> then only rows which evaluate to true will be updated.</p>
+<p>For example, let’s update the value of a column in the target table based on a matching row in the source table.</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="kn">from</span> <span class="nn">deltalake</span> <span class="kn">import</span> <span class="n">DeltaTable</span><span class="p">,</span> <span class="n">write_deltalake</span>
+</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="nn">pa</span>
+</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>
+</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="c1"># define target table</span>
+</span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="o">&gt;</span> <span class="n">target_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span>
+</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a><span class="o">&gt;</span> <span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">,</span> <span class="n">target_data</span><span class="p">)</span>
+</span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a><span class="o">&gt;</span> <span class="n">dt</span> <span class="o">=</span> <span class="n">DeltaTable</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">)</span>
+</span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="o">&gt;</span> <span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</span><span id="__span-1-9"><a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a>
+</span><span id="__span-1-10"><a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-1-11"><a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">4</span>
+</span><span id="__span-1-12"><a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a><span class="mi">1</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-1-13"><a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a><span class="mi">2</span>  <span class="mi">3</span>  <span class="mi">6</span>
+</span><span id="__span-1-14"><a id="__codelineno-1-14" name="__codelineno-1-14" href="#__codelineno-1-14"></a>
+</span><span id="__span-1-15"><a id="__codelineno-1-15" name="__codelineno-1-15" href="#__codelineno-1-15"></a><span class="c1"># define source table</span>
+</span><span id="__span-1-16"><a id="__codelineno-1-16" name="__codelineno-1-16" href="#__codelineno-1-16"></a><span class="o">&gt;</span> <span class="n">source_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">5</span><span class="p">,</span><span class="mi">8</span><span class="p">]})</span>
+</span><span id="__span-1-17"><a id="__codelineno-1-17" name="__codelineno-1-17" href="#__codelineno-1-17"></a><span class="o">&gt;</span> <span class="n">source_data</span>
+</span><span id="__span-1-18"><a id="__codelineno-1-18" name="__codelineno-1-18" href="#__codelineno-1-18"></a>
+</span><span id="__span-1-19"><a id="__codelineno-1-19" name="__codelineno-1-19" href="#__codelineno-1-19"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-1-20"><a id="__codelineno-1-20" name="__codelineno-1-20" href="#__codelineno-1-20"></a><span class="mi">0</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-1-21"><a id="__codelineno-1-21" name="__codelineno-1-21" href="#__codelineno-1-21"></a><span class="mi">1</span>  <span class="mi">3</span>  <span class="mi">8</span>
+</span><span id="__span-1-22"><a id="__codelineno-1-22" name="__codelineno-1-22" href="#__codelineno-1-22"></a>
+</span><span id="__span-1-23"><a id="__codelineno-1-23" name="__codelineno-1-23" href="#__codelineno-1-23"></a><span class="c1"># define merge logic</span>
+</span><span id="__span-1-24"><a id="__codelineno-1-24" name="__codelineno-1-24" href="#__codelineno-1-24"></a><span class="o">&gt;</span> <span class="p">(</span>
+</span><span id="__span-1-25"><a id="__codelineno-1-25" name="__codelineno-1-25" href="#__codelineno-1-25"></a><span class="o">&gt;</span>     <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>
+</span><span id="__span-1-26"><a id="__codelineno-1-26" name="__codelineno-1-26" href="#__codelineno-1-26"></a><span class="o">&gt;</span>         <span class="n">source</span><span class="o">=</span><span class="n">source_data</span><span class="p">,</span>
+</span><span id="__span-1-27"><a id="__codelineno-1-27" name="__codelineno-1-27" href="#__codelineno-1-27"></a><span class="o">&gt;</span>         <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;target.x = source.x&quot;</span><span class="p">,</span>
+</span><span id="__span-1-28"><a id="__codelineno-1-28" name="__codelineno-1-28" href="#__codelineno-1-28"></a><span class="o">&gt;</span>         <span class="n">source_alias</span><span class="o">=</span><span class="s2">&quot;source&quot;</span><span class="p">,</span>
+</span><span id="__span-1-29"><a id="__codelineno-1-29" name="__codelineno-1-29" href="#__codelineno-1-29"></a><span class="o">&gt;</span>         <span class="n">target_alias</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)</span>
+</span><span id="__span-1-30"><a id="__codelineno-1-30" name="__codelineno-1-30" href="#__codelineno-1-30"></a><span class="o">&gt;</span>     <span class="o">.</span><span class="n">when_matched_update</span><span class="p">(</span>
+</span><span id="__span-1-31"><a id="__codelineno-1-31" name="__codelineno-1-31" href="#__codelineno-1-31"></a><span class="o">&gt;</span>         <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-1-32"><a id="__codelineno-1-32" name="__codelineno-1-32" href="#__codelineno-1-32"></a><span class="o">&gt;</span>     <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-1-33"><a id="__codelineno-1-33" name="__codelineno-1-33" href="#__codelineno-1-33"></a><span class="o">&gt;</span> <span class="p">)</span>
+</span></code></pre></div>
+<p>First, we match rows for which the <code>x</code> values are the same using <code>predicate="target.x = source.x"</code>. We then update the <code>x</code> and <code>y</code> values of the matched row with the new (source) values using <code>updates={"x": "source.x", "y":"source.y"}</code>.</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="c1"># inspect result</span>
+</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="o">&gt;</span> <span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a>
+</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">4</span>
+</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a><span class="mi">1</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a><span class="mi">2</span>  <span class="mi">3</span>  <span class="mi">8</span>
+</span></code></pre></div>
+<p>The value of the <code>y</code> column has been correctly updated for the row that matches our predicate.</p>
+<p>You can also use <a class="autorefs autorefs-internal" href="../../api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_update_all"><code>when_matched_update_all</code></a> to update all source fields to target fields. In this case, source and target are required to have the same field names.</p>
+<h2 id="insert">Insert</h2>
+<p>Use <a class="autorefs autorefs-internal" href="../../api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_insert"><code>when_not_matched_insert</code></a> to perform an INSERT operation.</p>
+<p>For example, let’s say we start with the same target table:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="o">&gt;</span> <span class="n">target_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span>
+</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a><span class="o">&gt;</span> <span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">,</span> <span class="n">target_data</span><span class="p">)</span>
+</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a><span class="o">&gt;</span> <span class="n">dt</span> <span class="o">=</span> <span class="n">DeltaTable</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">)</span>
+</span><span id="__span-3-4"><a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a>
+</span><span id="__span-3-5"><a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-3-6"><a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">4</span>
+</span><span id="__span-3-7"><a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a><span class="mi">1</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-3-8"><a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a><span class="mi">2</span>  <span class="mi">3</span>  <span class="mi">6</span>
+</span></code></pre></div>
+<p>And we want to merge only new records from our source data, without duplication:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="o">&gt;</span> <span class="n">source_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">7</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">,</span><span class="mi">8</span><span class="p">]})</span>
+</span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a>
+</span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a><span class="mi">0</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-4-5"><a id="__codelineno-4-5" name="__codelineno-4-5" href="#__codelineno-4-5"></a><span class="mi">1</span>  <span class="mi">3</span>  <span class="mi">6</span>
+</span><span id="__span-4-6"><a id="__codelineno-4-6" name="__codelineno-4-6" href="#__codelineno-4-6"></a><span class="mi">2</span>  <span class="mi">7</span>  <span class="mi">8</span>
+</span></code></pre></div>
+<p>The <code>MERGE</code> syntax would be as follows:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-5-1"><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a><span class="p">(</span>
+</span><span id="__span-5-2"><a id="__codelineno-5-2" name="__codelineno-5-2" href="#__codelineno-5-2"></a>    <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>
+</span><span id="__span-5-3"><a id="__codelineno-5-3" name="__codelineno-5-3" href="#__codelineno-5-3"></a>        <span class="n">source</span><span class="o">=</span><span class="n">source_data</span><span class="p">,</span>
+</span><span id="__span-5-4"><a id="__codelineno-5-4" name="__codelineno-5-4" href="#__codelineno-5-4"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;target.x = source.x&quot;</span><span class="p">,</span>
+</span><span id="__span-5-5"><a id="__codelineno-5-5" name="__codelineno-5-5" href="#__codelineno-5-5"></a>        <span class="n">source_alias</span><span class="o">=</span><span class="s2">&quot;source&quot;</span><span class="p">,</span>
+</span><span id="__span-5-6"><a id="__codelineno-5-6" name="__codelineno-5-6" href="#__codelineno-5-6"></a>        <span class="n">target_alias</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)</span>
+</span><span id="__span-5-7"><a id="__codelineno-5-7" name="__codelineno-5-7" href="#__codelineno-5-7"></a>    <span class="o">.</span><span class="n">when_not_matched_insert</span><span class="p">(</span>
+</span><span id="__span-5-8"><a id="__codelineno-5-8" name="__codelineno-5-8" href="#__codelineno-5-8"></a>        <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-5-9"><a id="__codelineno-5-9" name="__codelineno-5-9" href="#__codelineno-5-9"></a>    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-5-10"><a id="__codelineno-5-10" name="__codelineno-5-10" href="#__codelineno-5-10"></a><span class="p">)</span>
+</span><span id="__span-5-11"><a id="__codelineno-5-11" name="__codelineno-5-11" href="#__codelineno-5-11"></a>
+</span><span id="__span-5-12"><a id="__codelineno-5-12" name="__codelineno-5-12" href="#__codelineno-5-12"></a><span class="o">&gt;</span> <span class="c1"># inspect result</span>
+</span><span id="__span-5-13"><a id="__codelineno-5-13" name="__codelineno-5-13" href="#__codelineno-5-13"></a><span class="o">&gt;</span> <span class="nb">print</span><span class="p">(</span><span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span>
+</span><span id="__span-5-14"><a id="__codelineno-5-14" name="__codelineno-5-14" href="#__codelineno-5-14"></a>
+</span><span id="__span-5-15"><a id="__codelineno-5-15" name="__codelineno-5-15" href="#__codelineno-5-15"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-5-16"><a id="__codelineno-5-16" name="__codelineno-5-16" href="#__codelineno-5-16"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">4</span>
+</span><span id="__span-5-17"><a id="__codelineno-5-17" name="__codelineno-5-17" href="#__codelineno-5-17"></a><span class="mi">1</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-5-18"><a id="__codelineno-5-18" name="__codelineno-5-18" href="#__codelineno-5-18"></a><span class="mi">2</span>  <span class="mi">3</span>  <span class="mi">6</span>
+</span><span id="__span-5-19"><a id="__codelineno-5-19" name="__codelineno-5-19" href="#__codelineno-5-19"></a><span class="mi">3</span>  <span class="mi">7</span>  <span class="mi">8</span>
+</span></code></pre></div>
+<p>The new row has been successfully added to the target dataset.</p>
+<p>You can also use <a class="autorefs autorefs-internal" href="../../api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_insert_all"><code>when_not_matched_insert_all</code></a> to insert a new row to the target table, updating all source fields to target fields. In this case, source and target are required to have the same field names.</p>
+<h2 id="delete">Delete</h2>
+<p>Use <a class="autorefs autorefs-internal" href="../../api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_delete"><code>when_matched_delete</code></a> to perform a DELETE operation.</p>
+<p>For example, given the following <code>target_data</code> and <code>source_data</code>:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a><span class="n">target_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span>
+</span><span id="__span-6-2"><a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">,</span> <span class="n">target_data</span><span class="p">)</span>
+</span><span id="__span-6-3"><a id="__codelineno-6-3" name="__codelineno-6-3" href="#__codelineno-6-3"></a><span class="n">dt</span> <span class="o">=</span> <span class="n">DeltaTable</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">)</span>
+</span><span id="__span-6-4"><a id="__codelineno-6-4" name="__codelineno-6-4" href="#__codelineno-6-4"></a><span class="n">source_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;deleted&quot;</span><span class="p">:</span> <span class="p">[</span><span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">]})</span>
+</span></code></pre></div>
+<p>You can delete the rows that match a predicate (in this case <code>"deleted" = True</code>) using:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a><span class="p">(</span>
+</span><span id="__span-7-2"><a id="__codelineno-7-2" name="__codelineno-7-2" href="#__codelineno-7-2"></a>    <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>
+</span><span id="__span-7-3"><a id="__codelineno-7-3" name="__codelineno-7-3" href="#__codelineno-7-3"></a>        <span class="n">source</span><span class="o">=</span><span class="n">source_data</span><span class="p">,</span>
+</span><span id="__span-7-4"><a id="__codelineno-7-4" name="__codelineno-7-4" href="#__codelineno-7-4"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;target.x = source.x&quot;</span><span class="p">,</span>
+</span><span id="__span-7-5"><a id="__codelineno-7-5" name="__codelineno-7-5" href="#__codelineno-7-5"></a>        <span class="n">source_alias</span><span class="o">=</span><span class="s2">&quot;source&quot;</span><span class="p">,</span>
+</span><span id="__span-7-6"><a id="__codelineno-7-6" name="__codelineno-7-6" href="#__codelineno-7-6"></a>        <span class="n">target_alias</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)</span>
+</span><span id="__span-7-7"><a id="__codelineno-7-7" name="__codelineno-7-7" href="#__codelineno-7-7"></a>    <span class="o">.</span><span class="n">when_matched_delete</span><span class="p">(</span>
+</span><span id="__span-7-8"><a id="__codelineno-7-8" name="__codelineno-7-8" href="#__codelineno-7-8"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;source.deleted = true&quot;</span><span class="p">)</span>
+</span><span id="__span-7-9"><a id="__codelineno-7-9" name="__codelineno-7-9" href="#__codelineno-7-9"></a>    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-7-10"><a id="__codelineno-7-10" name="__codelineno-7-10" href="#__codelineno-7-10"></a><span class="p">)</span>
+</span></code></pre></div>
+<p>This will result in:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-8-1"><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a><span class="o">&gt;</span> <span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</span><span id="__span-8-2"><a id="__codelineno-8-2" name="__codelineno-8-2" href="#__codelineno-8-2"></a>
+</span><span id="__span-8-3"><a id="__codelineno-8-3" name="__codelineno-8-3" href="#__codelineno-8-3"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-8-4"><a id="__codelineno-8-4" name="__codelineno-8-4" href="#__codelineno-8-4"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">4</span>
+</span><span id="__span-8-5"><a id="__codelineno-8-5" name="__codelineno-8-5" href="#__codelineno-8-5"></a><span class="mi">1</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span></code></pre></div>
+<p>The matched row has been successfully deleted.</p>
+<h2 id="upsert">Upsert</h2>
+<p>You can combine conditional statements to perform more complex operations.</p>
+<p>To perform an upsert operation, use <code>when_matched_update</code> and <code>when_not_matched_insert</code> in a single <code>merge()</code> clause.</p>
+<p>For example:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-9-1"><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a><span class="n">target_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span>
+</span><span id="__span-9-2"><a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">,</span> <span class="n">target_data</span><span class="p">)</span>
+</span><span id="__span-9-3"><a id="__codelineno-9-3" name="__codelineno-9-3" href="#__codelineno-9-3"></a><span class="n">dt</span> <span class="o">=</span> <span class="n">DeltaTable</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">)</span>
+</span><span id="__span-9-4"><a id="__codelineno-9-4" name="__codelineno-9-4" href="#__codelineno-9-4"></a><span class="n">source_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">5</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">11</span><span class="p">]})</span>
+</span><span id="__span-9-5"><a id="__codelineno-9-5" name="__codelineno-9-5" href="#__codelineno-9-5"></a>
+</span><span id="__span-9-6"><a id="__codelineno-9-6" name="__codelineno-9-6" href="#__codelineno-9-6"></a><span class="p">(</span>
+</span><span id="__span-9-7"><a id="__codelineno-9-7" name="__codelineno-9-7" href="#__codelineno-9-7"></a>    <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>
+</span><span id="__span-9-8"><a id="__codelineno-9-8" name="__codelineno-9-8" href="#__codelineno-9-8"></a>        <span class="n">source</span><span class="o">=</span><span class="n">source_data</span><span class="p">,</span>
+</span><span id="__span-9-9"><a id="__codelineno-9-9" name="__codelineno-9-9" href="#__codelineno-9-9"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;target.x = source.x&quot;</span><span class="p">,</span>
+</span><span id="__span-9-10"><a id="__codelineno-9-10" name="__codelineno-9-10" href="#__codelineno-9-10"></a>        <span class="n">source_alias</span><span class="o">=</span><span class="s2">&quot;source&quot;</span><span class="p">,</span>
+</span><span id="__span-9-11"><a id="__codelineno-9-11" name="__codelineno-9-11" href="#__codelineno-9-11"></a>        <span class="n">target_alias</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)</span>
+</span><span id="__span-9-12"><a id="__codelineno-9-12" name="__codelineno-9-12" href="#__codelineno-9-12"></a>    <span class="o">.</span><span class="n">when_matched_update</span><span class="p">(</span>
+</span><span id="__span-9-13"><a id="__codelineno-9-13" name="__codelineno-9-13" href="#__codelineno-9-13"></a>        <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-9-14"><a id="__codelineno-9-14" name="__codelineno-9-14" href="#__codelineno-9-14"></a>    <span class="o">.</span><span class="n">when_not_matched_insert</span><span class="p">(</span>
+</span><span id="__span-9-15"><a id="__codelineno-9-15" name="__codelineno-9-15" href="#__codelineno-9-15"></a>        <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-9-16"><a id="__codelineno-9-16" name="__codelineno-9-16" href="#__codelineno-9-16"></a>    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-9-17"><a id="__codelineno-9-17" name="__codelineno-9-17" href="#__codelineno-9-17"></a><span class="p">)</span>
+</span></code></pre></div>
+<p>This will give you the following output:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-10-1"><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a><span class="o">&gt;</span> <span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</span><span id="__span-10-2"><a id="__codelineno-10-2" name="__codelineno-10-2" href="#__codelineno-10-2"></a>
+</span><span id="__span-10-3"><a id="__codelineno-10-3" name="__codelineno-10-3" href="#__codelineno-10-3"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-10-4"><a id="__codelineno-10-4" name="__codelineno-10-4" href="#__codelineno-10-4"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">4</span>
+</span><span id="__span-10-5"><a id="__codelineno-10-5" name="__codelineno-10-5" href="#__codelineno-10-5"></a><span class="mi">1</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-10-6"><a id="__codelineno-10-6" name="__codelineno-10-6" href="#__codelineno-10-6"></a><span class="mi">2</span>  <span class="mi">3</span>  <span class="mi">8</span>
+</span><span id="__span-10-7"><a id="__codelineno-10-7" name="__codelineno-10-7" href="#__codelineno-10-7"></a><span class="mi">3</span>  <span class="mi">5</span>  <span class="mi">11</span>
+</span></code></pre></div>
+<h2 id="upsert-with-delete">Upsert with Delete</h2>
+<p>Use the <a class="autorefs autorefs-internal" href="../../api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_matched_delete"><code>when_matched_delete</code></a> or <a class="autorefs autorefs-internal" href="../../api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_by_source_delete"><code>when_not_matched_by_source_delete</code></a> methods to add a DELETE operation to your upsert. This is helpful if you want to delete stale records from the target data.</p>
+<p>For example, given the same <code>target_data</code> and <code>source_data</code> used in the section above:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-11-1"><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a><span class="p">(</span>
+</span><span id="__span-11-2"><a id="__codelineno-11-2" name="__codelineno-11-2" href="#__codelineno-11-2"></a>    <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>
+</span><span id="__span-11-3"><a id="__codelineno-11-3" name="__codelineno-11-3" href="#__codelineno-11-3"></a>        <span class="n">source</span><span class="o">=</span><span class="n">source_data</span><span class="p">,</span>
+</span><span id="__span-11-4"><a id="__codelineno-11-4" name="__codelineno-11-4" href="#__codelineno-11-4"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;target.x = source.x&quot;</span><span class="p">,</span>
+</span><span id="__span-11-5"><a id="__codelineno-11-5" name="__codelineno-11-5" href="#__codelineno-11-5"></a>        <span class="n">source_alias</span><span class="o">=</span><span class="s2">&quot;source&quot;</span><span class="p">,</span>
+</span><span id="__span-11-6"><a id="__codelineno-11-6" name="__codelineno-11-6" href="#__codelineno-11-6"></a>        <span class="n">target_alias</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)</span>
+</span><span id="__span-11-7"><a id="__codelineno-11-7" name="__codelineno-11-7" href="#__codelineno-11-7"></a>    <span class="o">.</span><span class="n">when_matched_update</span><span class="p">(</span>
+</span><span id="__span-11-8"><a id="__codelineno-11-8" name="__codelineno-11-8" href="#__codelineno-11-8"></a>        <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-11-9"><a id="__codelineno-11-9" name="__codelineno-11-9" href="#__codelineno-11-9"></a>    <span class="o">.</span><span class="n">when_not_matched_insert</span><span class="p">(</span>
+</span><span id="__span-11-10"><a id="__codelineno-11-10" name="__codelineno-11-10" href="#__codelineno-11-10"></a>        <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-11-11"><a id="__codelineno-11-11" name="__codelineno-11-11" href="#__codelineno-11-11"></a>    <span class="o">.</span><span class="n">when_not_matched_by_source_delete</span><span class="p">()</span>
+</span><span id="__span-11-12"><a id="__codelineno-11-12" name="__codelineno-11-12" href="#__codelineno-11-12"></a>    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-11-13"><a id="__codelineno-11-13" name="__codelineno-11-13" href="#__codelineno-11-13"></a><span class="p">)</span>
+</span></code></pre></div>
+<p>This will result in:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-12-1"><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a><span class="o">&gt;</span> <span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</span><span id="__span-12-2"><a id="__codelineno-12-2" name="__codelineno-12-2" href="#__codelineno-12-2"></a>
+</span><span id="__span-12-3"><a id="__codelineno-12-3" name="__codelineno-12-3" href="#__codelineno-12-3"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-12-4"><a id="__codelineno-12-4" name="__codelineno-12-4" href="#__codelineno-12-4"></a><span class="mi">0</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-12-5"><a id="__codelineno-12-5" name="__codelineno-12-5" href="#__codelineno-12-5"></a><span class="mi">1</span>  <span class="mi">3</span>  <span class="mi">8</span>
+</span><span id="__span-12-6"><a id="__codelineno-12-6" name="__codelineno-12-6" href="#__codelineno-12-6"></a><span class="mi">2</span>  <span class="mi">5</span>  <span class="mi">11</span>
+</span></code></pre></div>
+<p>The row containing old data no longer present in the source dataset has been successfully deleted.</p>
+<h2 id="multiple-matches">Multiple Matches</h2>
+<p>Note that when multiple match conditions are met, the first condition that matches is executed.</p>
+<p>For example, given the following <code>target_data</code> and <code>source_data</code>:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-13-1"><a id="__codelineno-13-1" name="__codelineno-13-1" href="#__codelineno-13-1"></a><span class="n">target_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span>
+</span><span id="__span-13-2"><a id="__codelineno-13-2" name="__codelineno-13-2" href="#__codelineno-13-2"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">,</span> <span class="n">target_data</span><span class="p">)</span>
+</span><span id="__span-13-3"><a id="__codelineno-13-3" name="__codelineno-13-3" href="#__codelineno-13-3"></a><span class="n">dt</span> <span class="o">=</span> <span class="n">DeltaTable</span><span class="p">(</span><span class="s2">&quot;tmp_table&quot;</span><span class="p">)</span>
+</span><span id="__span-13-4"><a id="__codelineno-13-4" name="__codelineno-13-4" href="#__codelineno-13-4"></a><span class="n">source_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">],</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">5</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">11</span><span class="p">]})</span>
+</span></code></pre></div>
+<p>Let’s perform a merge with <code>when_matched_delete</code> first, followed by <code>when_matched_update</code>:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-14-1"><a id="__codelineno-14-1" name="__codelineno-14-1" href="#__codelineno-14-1"></a><span class="p">(</span>
+</span><span id="__span-14-2"><a id="__codelineno-14-2" name="__codelineno-14-2" href="#__codelineno-14-2"></a>    <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>
+</span><span id="__span-14-3"><a id="__codelineno-14-3" name="__codelineno-14-3" href="#__codelineno-14-3"></a>        <span class="n">source</span><span class="o">=</span><span class="n">source_data</span><span class="p">,</span>
+</span><span id="__span-14-4"><a id="__codelineno-14-4" name="__codelineno-14-4" href="#__codelineno-14-4"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;target.x = source.x&quot;</span><span class="p">,</span>
+</span><span id="__span-14-5"><a id="__codelineno-14-5" name="__codelineno-14-5" href="#__codelineno-14-5"></a>        <span class="n">source_alias</span><span class="o">=</span><span class="s2">&quot;source&quot;</span><span class="p">,</span>
+</span><span id="__span-14-6"><a id="__codelineno-14-6" name="__codelineno-14-6" href="#__codelineno-14-6"></a>        <span class="n">target_alias</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)</span>
+</span><span id="__span-14-7"><a id="__codelineno-14-7" name="__codelineno-14-7" href="#__codelineno-14-7"></a>    <span class="o">.</span><span class="n">when_matched_delete</span><span class="p">(</span>
+</span><span id="__span-14-8"><a id="__codelineno-14-8" name="__codelineno-14-8" href="#__codelineno-14-8"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;source.x = target.x&quot;</span><span class="p">)</span>
+</span><span id="__span-14-9"><a id="__codelineno-14-9" name="__codelineno-14-9" href="#__codelineno-14-9"></a>    <span class="o">.</span><span class="n">when_matched_update</span><span class="p">(</span>
+</span><span id="__span-14-10"><a id="__codelineno-14-10" name="__codelineno-14-10" href="#__codelineno-14-10"></a>        <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-14-11"><a id="__codelineno-14-11" name="__codelineno-14-11" href="#__codelineno-14-11"></a>    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-14-12"><a id="__codelineno-14-12" name="__codelineno-14-12" href="#__codelineno-14-12"></a><span class="p">)</span>
+</span></code></pre></div>
+<p>This will result in:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-15-1"><a id="__codelineno-15-1" name="__codelineno-15-1" href="#__codelineno-15-1"></a><span class="o">&gt;</span> <span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</span><span id="__span-15-2"><a id="__codelineno-15-2" name="__codelineno-15-2" href="#__codelineno-15-2"></a>
+</span><span id="__span-15-3"><a id="__codelineno-15-3" name="__codelineno-15-3" href="#__codelineno-15-3"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-15-4"><a id="__codelineno-15-4" name="__codelineno-15-4" href="#__codelineno-15-4"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">4</span>
+</span></code></pre></div>
+<p>Let’s now perform the merge with the flipped order: <code>update</code> first, then <code>delete</code>:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-16-1"><a id="__codelineno-16-1" name="__codelineno-16-1" href="#__codelineno-16-1"></a><span class="p">(</span>
+</span><span id="__span-16-2"><a id="__codelineno-16-2" name="__codelineno-16-2" href="#__codelineno-16-2"></a>    <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>
+</span><span id="__span-16-3"><a id="__codelineno-16-3" name="__codelineno-16-3" href="#__codelineno-16-3"></a>        <span class="n">source</span><span class="o">=</span><span class="n">source_data</span><span class="p">,</span>
+</span><span id="__span-16-4"><a id="__codelineno-16-4" name="__codelineno-16-4" href="#__codelineno-16-4"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;target.x = source.x&quot;</span><span class="p">,</span>
+</span><span id="__span-16-5"><a id="__codelineno-16-5" name="__codelineno-16-5" href="#__codelineno-16-5"></a>        <span class="n">source_alias</span><span class="o">=</span><span class="s2">&quot;source&quot;</span><span class="p">,</span>
+</span><span id="__span-16-6"><a id="__codelineno-16-6" name="__codelineno-16-6" href="#__codelineno-16-6"></a>        <span class="n">target_alias</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)</span>
+</span><span id="__span-16-7"><a id="__codelineno-16-7" name="__codelineno-16-7" href="#__codelineno-16-7"></a>    <span class="o">.</span><span class="n">when_matched_update</span><span class="p">(</span>
+</span><span id="__span-16-8"><a id="__codelineno-16-8" name="__codelineno-16-8" href="#__codelineno-16-8"></a>        <span class="n">updates</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="s2">&quot;source.x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">:</span><span class="s2">&quot;source.y&quot;</span><span class="p">})</span>
+</span><span id="__span-16-9"><a id="__codelineno-16-9" name="__codelineno-16-9" href="#__codelineno-16-9"></a>    <span class="o">.</span><span class="n">when_matched_delete</span><span class="p">(</span>
+</span><span id="__span-16-10"><a id="__codelineno-16-10" name="__codelineno-16-10" href="#__codelineno-16-10"></a>        <span class="n">predicate</span><span class="o">=</span><span class="s2">&quot;source.x = target.x&quot;</span><span class="p">)</span>
+</span><span id="__span-16-11"><a id="__codelineno-16-11" name="__codelineno-16-11" href="#__codelineno-16-11"></a>    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-16-12"><a id="__codelineno-16-12" name="__codelineno-16-12" href="#__codelineno-16-12"></a><span class="p">)</span>
+</span></code></pre></div>
+<p>This will result in:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-17-1"><a id="__codelineno-17-1" name="__codelineno-17-1" href="#__codelineno-17-1"></a><span class="o">&gt;</span> <span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</span><span id="__span-17-2"><a id="__codelineno-17-2" name="__codelineno-17-2" href="#__codelineno-17-2"></a>
+</span><span id="__span-17-3"><a id="__codelineno-17-3" name="__codelineno-17-3" href="#__codelineno-17-3"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-17-4"><a id="__codelineno-17-4" name="__codelineno-17-4" href="#__codelineno-17-4"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">4</span>
+</span><span id="__span-17-5"><a id="__codelineno-17-5" name="__codelineno-17-5" href="#__codelineno-17-5"></a><span class="mi">1</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-17-6"><a id="__codelineno-17-6" name="__codelineno-17-6" href="#__codelineno-17-6"></a><span class="mi">2</span>  <span class="mi">3</span>  <span class="mi">8</span>
+</span></code></pre></div>
+<h2 id="conditional-updates">Conditional Updates</h2>
+<p>You can perform conditional updates for rows that have no match in the source data using <a class="autorefs autorefs-internal" href="../../api/delta_table/delta_table_merger/#deltalake.table.TableMerger.when_not_matched_by_source_update">when_not_matched_by_source_update</a>.</p>
+<p>For example, given the following target_data and source_data:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-18-1"><a id="__codelineno-18-1" name="__codelineno-18-1" href="#__codelineno-18-1"></a><span class="n">target_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;1 x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">&quot;1y&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span>
+</span><span id="__span-18-2"><a id="__codelineno-18-2" name="__codelineno-18-2" href="#__codelineno-18-2"></a><span class="n">write_deltalake</span><span class="p">(</span><span class="s2">&quot;tmp&quot;</span><span class="p">,</span> <span class="n">target_data</span><span class="p">)</span>
+</span><span id="__span-18-3"><a id="__codelineno-18-3" name="__codelineno-18-3" href="#__codelineno-18-3"></a><span class="n">dt</span> <span class="o">=</span> <span class="n">DeltaTable</span><span class="p">(</span><span class="s2">&quot;tmp&quot;</span><span class="p">)</span>
+</span><span id="__span-18-4"><a id="__codelineno-18-4" name="__codelineno-18-4" href="#__codelineno-18-4"></a><span class="n">source_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">&quot;x&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]})</span>
+</span></code></pre></div>
+<p>Set y = 0 for all rows that have no matches in the new source data, provided that the original y value is greater than 3:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-19-1"><a id="__codelineno-19-1" name="__codelineno-19-1" href="#__codelineno-19-1"></a><span class="p">(</span>
+</span><span id="__span-19-2"><a id="__codelineno-19-2" name="__codelineno-19-2" href="#__codelineno-19-2"></a>   <span class="n">dt</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span>
+</span><span id="__span-19-3"><a id="__codelineno-19-3" name="__codelineno-19-3" href="#__codelineno-19-3"></a>       <span class="n">source</span><span class="o">=</span><span class="n">new_data</span><span class="p">,</span>
+</span><span id="__span-19-4"><a id="__codelineno-19-4" name="__codelineno-19-4" href="#__codelineno-19-4"></a>       <span class="n">predicate</span><span class="o">=</span><span class="s1">&#39;target.x = source.x&#39;</span><span class="p">,</span>
+</span><span id="__span-19-5"><a id="__codelineno-19-5" name="__codelineno-19-5" href="#__codelineno-19-5"></a>       <span class="n">source_alias</span><span class="o">=</span><span class="s1">&#39;source&#39;</span><span class="p">,</span>
+</span><span id="__span-19-6"><a id="__codelineno-19-6" name="__codelineno-19-6" href="#__codelineno-19-6"></a>       <span class="n">target_alias</span><span class="o">=</span><span class="s1">&#39;target&#39;</span><span class="p">)</span>
+</span><span id="__span-19-7"><a id="__codelineno-19-7" name="__codelineno-19-7" href="#__codelineno-19-7"></a>   <span class="o">.</span><span class="n">when_not_matched_by_source_update</span><span class="p">(</span>
+</span><span id="__span-19-8"><a id="__codelineno-19-8" name="__codelineno-19-8" href="#__codelineno-19-8"></a>       <span class="n">predicate</span> <span class="o">=</span> <span class="s2">&quot;`1y` &gt; 3&quot;</span><span class="p">,</span>
+</span><span id="__span-19-9"><a id="__codelineno-19-9" name="__codelineno-19-9" href="#__codelineno-19-9"></a>       <span class="n">updates</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;`1y`&quot;</span><span class="p">:</span> <span class="s2">&quot;0&quot;</span><span class="p">})</span>
+</span><span id="__span-19-10"><a id="__codelineno-19-10" name="__codelineno-19-10" href="#__codelineno-19-10"></a>   <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+</span><span id="__span-19-11"><a id="__codelineno-19-11" name="__codelineno-19-11" href="#__codelineno-19-11"></a><span class="p">)</span>
+</span></code></pre></div>
+<p>This will result in:</p>
+<div class="language-python highlight"><pre><span></span><code><span id="__span-20-1"><a id="__codelineno-20-1" name="__codelineno-20-1" href="#__codelineno-20-1"></a><span class="o">&gt;</span> <span class="n">dt</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</span><span id="__span-20-2"><a id="__codelineno-20-2" name="__codelineno-20-2" href="#__codelineno-20-2"></a>   <span class="n">x</span>  <span class="n">y</span>
+</span><span id="__span-20-3"><a id="__codelineno-20-3" name="__codelineno-20-3" href="#__codelineno-20-3"></a><span class="mi">0</span>  <span class="mi">1</span>  <span class="mi">0</span>
+</span><span id="__span-20-4"><a id="__codelineno-20-4" name="__codelineno-20-4" href="#__codelineno-20-4"></a><span class="mi">1</span>  <span class="mi">2</span>  <span class="mi">5</span>
+</span><span id="__span-20-5"><a id="__codelineno-20-5" name="__codelineno-20-5" href="#__codelineno-20-5"></a><span class="mi">2</span>  <span class="mi">3</span>  <span class="mi">6</span>
+</span></code></pre></div>
+<h2 id="notes">Notes</h2>
+<ul>
+<li>Column names with special characters, such as numbers or spaces should be encapsulated in backticks: "target.<code>123column</code>" or "target.<code>my column</code>"</li>
+</ul>
+
+
+
+
+
+                
+              </article>
+            </div>
+          
+          
+  <script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var tab,labels=set.querySelector(".tabbed-labels");for(tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
+
+        </div>
+        
+      </main>
+      
+        <footer class="md-footer">
+  
+    
+      
+      <nav class="md-footer__inner md-grid" aria-label="Footer" >
+        
+          
+          <a href="../querying-delta-tables/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Querying a table">
+            <div class="md-footer__button md-icon">
+              
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
+            </div>
+            <div class="md-footer__title">
+              <span class="md-footer__direction">
+                Previous
+              </span>
+              <div class="md-ellipsis">
+                Querying a table
+              </div>
+            </div>
+          </a>
+        
+        
+          
+          <a href="../managing-tables/" class="md-footer__link md-footer__link--next" aria-label="Next: Managing a table">
+            <div class="md-footer__title">
+              <span class="md-footer__direction">
+                Next
+              </span>
+              <div class="md-ellipsis">
+                Managing a table
+              </div>
+            </div>
+            <div class="md-footer__button md-icon">
+              
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4Z"/></svg>
+            </div>
+          </a>
+        
+      </nav>
+    
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+  
+  
+</div>
+      
+        <div class="md-social">
+  
+    
+    
+    
+    
+    <a href="https://go.delta.io/slack" target="_blank" rel="noopener" title="Delta slack channel" class="md-social__link">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M94.12 315.1c0 25.9-21.16 47.06-47.06 47.06S0 341 0 315.1c0-25.9 21.16-47.06 47.06-47.06h47.06v47.06zm23.72 0c0-25.9 21.16-47.06 47.06-47.06s47.06 21.16 47.06 47.06v117.84c0 25.9-21.16 47.06-47.06 47.06s-47.06-21.16-47.06-47.06V315.1zm47.06-188.98c-25.9 0-47.06-21.16-47.06-47.06S139 32 164.9 32s47.06 21.16 47.06 47.06v47.06H164.9zm0 23.72c25.9 0 47.06 21.16 47.06 47.06s-21.16 47.06-47.06 47.06H47.06C21.16 243.96 0 222.8 0 196.9s21.16-47.06 47.06-47.06H164.9zm188.98 47.06c0-25.9 21.16-47.06 47.06-47.06 25.9 0 47.06 21.16 47.06 47.06s-21.16 47.06-47.06 47.06h-47.06V196.9zm-23.72 0c0 25.9-21.16 47.06-47.06 47.06-25.9 0-47.06-21.16-47.06-47.06V79.06c0-25.9 21.16-47.06 47.06-47.06 25.9 0 47.06 21.16 47.06 47.06V196.9zM283.1 385.88c25.9 0 47.06 21.16 47.06 47.06 0 25.9-21.16 47.06-47.06 47.06-25.9 0-47.06-21.16-47.06-47.06v-47.06h47.06zm0-23.72c-25.9 0-47.06-21.16-47.06-47.06 0-25.9 21.16-47.06 47.06-47.06h117.84c25.9 0 47.06 21.16 47.06 47.06 0 25.9-21.16 47.06-47.06 47.06H283.1z"/></svg>
+    </a>
+  
+</div>
+      
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    
+    
+    <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tracking", "navigation.instant", "navigation.expand", "navigation.tabs", "navigation.indexes", "navigation.tabs.sticky", "navigation.footer", "content.tabs.link", "content.code.annotation", "content.code.copy"], "search": "../../assets/javascripts/workers/search.f886a092.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
+    
+    
+      <script src="../../assets/javascripts/bundle.aecac24b.min.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file
diff --git a/usage/optimize/delta-lake-z-order/index.html b/usage/optimize/delta-lake-z-order/index.html
index 8bd3b2acf0..b17a647eb5 100644
--- a/usage/optimize/delta-lake-z-order/index.html
+++ b/usage/optimize/delta-lake-z-order/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_13" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_14" checked>
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/optimize/small-file-compaction-with-optimize/index.html b/usage/optimize/small-file-compaction-with-optimize/index.html
index 0c2c7707c7..f8f1e90ce7 100644
--- a/usage/optimize/small-file-compaction-with-optimize/index.html
+++ b/usage/optimize/small-file-compaction-with-optimize/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_13" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_14" checked>
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/overview/index.html b/usage/overview/index.html
index 39efbe769e..eb65655fc2 100644
--- a/usage/overview/index.html
+++ b/usage/overview/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -774,6 +776,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -814,7 +836,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -830,14 +852,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -916,10 +938,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -930,8 +952,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/querying-delta-tables/index.html b/usage/querying-delta-tables/index.html
index 5594d6da4d..aba9716d8a 100644
--- a/usage/querying-delta-tables/index.html
+++ b/usage/querying-delta-tables/index.html
@@ -14,7 +14,7 @@
         <link rel="prev" href="../examining-table/">
       
       
-        <link rel="next" href="../managing-tables/">
+        <link rel="next" href="../merging-tables/">
       
       
       <link rel="icon" href="../../delta-rust-no-whitespace.svg">
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -774,6 +776,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -814,7 +836,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -830,14 +852,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -916,10 +938,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -930,8 +952,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
@@ -1964,13 +1986,13 @@ <h1 id="querying-delta-tables">Querying Delta Tables</h1>
         
         
           
-          <a href="../managing-tables/" class="md-footer__link md-footer__link--next" aria-label="Next: Managing a table">
+          <a href="../merging-tables/" class="md-footer__link md-footer__link--next" aria-label="Next: Merging a table">
             <div class="md-footer__title">
               <span class="md-footer__direction">
                 Next
               </span>
               <div class="md-ellipsis">
-                Managing a table
+                Merging a table
               </div>
             </div>
             <div class="md-footer__button md-icon">
diff --git a/usage/read-cdf/index.html b/usage/read-cdf/index.html
index 5197084d64..fb75f43252 100644
--- a/usage/read-cdf/index.html
+++ b/usage/read-cdf/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -811,6 +813,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -851,7 +873,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -867,14 +889,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -953,10 +975,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -967,8 +989,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/writing/index.html b/usage/writing/index.html
index f59686118c..eff0947abb 100644
--- a/usage/writing/index.html
+++ b/usage/writing/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_11" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_12" checked>
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link md-nav__link--active" for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link md-nav__link--active" for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -906,10 +928,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -920,8 +942,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/usage/writing/writing-to-s3-with-locking-provider/index.html b/usage/writing/writing-to-s3-with-locking-provider/index.html
index e73d29f729..b59e53e566 100644
--- a/usage/writing/writing-to-s3-with-locking-provider/index.html
+++ b/usage/writing/writing-to-s3-with-locking-provider/index.html
@@ -543,6 +543,8 @@
         
       
         
+      
+        
       
         
       
@@ -764,6 +766,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../managing-tables/" class="md-nav__link">
         
@@ -804,7 +826,7 @@
         
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_11" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_12" checked>
         
           
           
@@ -820,14 +842,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -974,10 +996,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -988,8 +1010,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>
diff --git a/why-use-delta-lake/index.html b/why-use-delta-lake/index.html
index 1efd5be74f..4927587b1a 100644
--- a/why-use-delta-lake/index.html
+++ b/why-use-delta-lake/index.html
@@ -630,6 +630,8 @@
         
       
         
+      
+        
       
         
       
@@ -853,6 +855,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../usage/merging-tables/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Merging a table
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../usage/managing-tables/" class="md-nav__link">
         
@@ -893,7 +915,7 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_11" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_12" >
         
           
           
@@ -909,14 +931,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_2_11" id="__nav_2_11_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_11_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_11">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_12">
             <span class="md-nav__icon md-icon"></span>
             Writing a table
           </label>
@@ -995,10 +1017,10 @@
         
           
         
-        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_13" >
+        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_2_14" >
         
           
-          <label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_2_14" id="__nav_2_14_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1009,8 +1031,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_2_13">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_14_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_14">
             <span class="md-nav__icon md-icon"></span>
             Optimize
           </label>