Fix error handling logic in try_setting_buffer_size (#5631) (#5641)

* Refs #22208. Fix error handling logic in `try_setting_buffer_size`.

Signed-off-by: Miguel Company <[email protected]>

* Refs #22756. Apply suggestion.

Signed-off-by: Miguel Company <[email protected]>

---------

Signed-off-by: Miguel Company <[email protected]>
(cherry picked from commit 7259840)

Co-authored-by: Miguel Company <[email protected]>

src/cpp/rtps/transport/asio_helpers.hpp

@@ -50,6 +50,8 @@ struct asio_helpers
     {
         asio::error_code ec;
 
+        assert(initial_buffer_value >= minimum_buffer_value);
+
         final_buffer_value = initial_buffer_value;
         while (final_buffer_value > minimum_buffer_value)
         {
@@ -70,9 +72,9 @@ struct asio_helpers
                     final_buffer_value = option.value();
                     continue;
                 }
-                // Could not determine the actual value, but the option was set successfully.
-                // Assume the option was set to the desired value.
-                return true;
+                // Could not determine the actual value, even though the option was set successfully.
+                // The current buffer size is not defined.
+                return false;
             }
 
             final_buffer_value /= 2;
@@ -85,13 +87,14 @@ struct asio_helpers
         if (!ec)
         {
             // Last attempt was successful. Get the actual value set.
+            int32_t max_value = static_cast<int32_t>(initial_buffer_value);
             BufferOptionType option;
             socket.get_option(option, ec);
-            if (!ec)
+            if (!ec && (option.value() >= value_to_set) && (option.value() <= max_value))
             {
                 final_buffer_value = option.value();
+                return true;
             }
-            return true;
         }
         return false;
     }