Added experimental disable_reuseaddr to bypass the issue

changelog.d/2819.fixed.md

@@ -0,0 +1 @@
+Added experimental disable_reuseaddr to bypass the issue

mirrord-schema.json

@@ -756,6 +756,14 @@
       "description": "mirrord Experimental features. This shouldn't be used unless someone from MetalBear/mirrord tells you to.",
       "type": "object",
       "properties": {
+        "disable_reuseaddr": {
+          "title": "_experimental_ disable_reuseaddr {#experimental-disable_reuseaddr}",
+          "description": "Disables the `SO_REUSEADDR` socket option on sockets that mirrord steals/mirrors. On macOS the application can use the same address many times but then we don't steal it correctly. This probably should be on by default but we want to gradually roll it out. <https://github.com/metalbear-co/mirrord/issues/2819> This option applies only on macOS.",
+          "type": [
+            "boolean",
+            "null"
+          ]
+        },
         "enable_exec_hooks_linux": {
           "title": "_experimental_ enable_exec_hooks_linux {#experimental-enable_exec_hooks_linux}",
           "description": "Enables exec hooks on Linux. Enable Linux hooks can fix issues when the application shares sockets with child commands (e.g Python web servers with reload), but the feature is not stable and may cause other issues.",

mirrord/config/src/experimental.rs

@@ -42,6 +42,16 @@ pub struct ExperimentalConfig {
     /// Enables `getifaddrs` hook that removes IPv6 interfaces from the list returned by libc.
     #[config(default = false)]
     pub hide_ipv6_interfaces: bool,
+
+    /// ### _experimental_ disable_reuseaddr {#experimental-disable_reuseaddr}
+    ///
+    /// Disables the `SO_REUSEADDR` socket option on sockets that mirrord steals/mirrors.
+    /// On macOS the application can use the same address many times but then we don't steal it
+    /// correctly. This probably should be on by default but we want to gradually roll it out.
+    /// <https://github.com/metalbear-co/mirrord/issues/2819>
+    /// This option applies only on macOS.
+    #[config(default = false)]
+    pub disable_reuseaddr: bool,
 }
 
 impl CollectAnalytics for &ExperimentalConfig {
@@ -51,5 +61,6 @@ impl CollectAnalytics for &ExperimentalConfig {
         analytics.add("trust_any_certificate", self.trust_any_certificate);
         analytics.add("enable_exec_hooks_linux", self.enable_exec_hooks_linux);
         analytics.add("hide_ipv6_interfaces", self.hide_ipv6_interfaces);
+        analytics.add("disable_reuseaddr", self.disable_reuseaddr);
     }
 }

mirrord/layer/src/socket/ops.rs

@@ -217,7 +217,6 @@ pub(super) fn bind(
     let requested_address = SocketAddr::try_from_raw(raw_address, address_length)?;
     let requested_port = requested_address.port();
     let incoming_config = crate::setup().incoming_config();
-
     let mut socket = {
         SOCKETS
             .lock()?
@@ -275,6 +274,23 @@ pub(super) fn bind(
         Err(HookError::AddressAlreadyBound(requested_address))?;
     }
 
+    #[cfg(target_os = "macos")]
+    {
+        let experimental = crate::setup().experimental();
+        if experimental.disable_reuseaddr {
+            let fd = unsafe { BorrowedFd::borrow_raw(sockfd) };
+            if let Err(e) =
+                nix::sys::socket::setsockopt(&fd, nix::sys::socket::sockopt::ReuseAddr, &false)
+            {
+                tracing::debug!(?e, "Failed to set SO_REUSEADDR to false");
+            }
+            if let Err(e) =
+                nix::sys::socket::setsockopt(&fd, nix::sys::socket::sockopt::ReusePort, &false)
+            {
+                tracing::debug!(?e, "Failed to set SE_REUSEPORT to false");
+            }
+        }
+    }
     // Try to bind a port from listen ports, if no configuration
     // try to bind the requested port, if not available get a random port
     // if there's configuration and binding fails with the requested port