diff --git a/scripts/reinvent/reinvent.py b/scripts/reinvent/reinvent.py
index c06cc14d..d7b47f3f 100644
--- a/scripts/reinvent/reinvent.py
+++ b/scripts/reinvent/reinvent.py
@@ -315,7 +315,8 @@ def create_env_fn():
             # Add data to the replay buffer
             reward = replay_data.get(("next", "reward"))
             replay_data.set("priority", reward)
-            experience_replay_buffer.extend(replay_data)
+            if len(replay_data) > 0:
+                experience_replay_buffer.extend(replay_data)
 
         # Log info
         if logger: