Revert PR #1156 "Make DTLS fragment stay within MTU size range" becau…

…se it causes a memory leak

- Fixes #1340
- Reopents #1100

### Details

As perfectly documented in issue #1340, there is a leak due to the usage of `BIO_set_callback_ex()` whose fix is unclear and, anyway, I'm pretty sure that the usage of that OpenSSL utility makes the performance worse.

As a workaround for issue #1100 (which now must be reopened), in order to avoid "DTLS fragment exceeds MTU size if the certificate is large", users should not pass a big DTLS certificate to mediasoup.

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+### Next
+
+- `Revert ([PR #1156](https://github.com/versatica/mediasoup/pull/1156)) "Make DTLS fragment stay within MTU size range" because it causes a memory leak ([PR #XXXX](https://github.com/versatica/mediasoup/pull/XXXX)).
+
 ### 3.13.20
 
 - Add server side ICE consent checks to detect silent WebRTC disconnections ([PR #1332](https://github.com/versatica/mediasoup/pull/1332)).

worker/include/RTC/DtlsTransport.hpp

@@ -152,8 +152,6 @@ namespace RTC
 			return this->localRole;
 		}
 		void SendApplicationData(const uint8_t* data, size_t len);
-		// This method must be public since it's called within an OpenSSL callback.
-		void SendDtlsData(const uint8_t* data, size_t len);
 
 	private:
 		bool IsRunning() const
@@ -175,6 +173,7 @@ namespace RTC
 		}
 		void Reset();
 		bool CheckStatus(int returnCode);
+		void SendPendingOutgoingDtlsData();
 		bool SetTimeout();
 		bool ProcessHandshake();
 		bool CheckRemoteFingerprint();

worker/src/RTC/DtlsTransport.cpp

@@ -61,30 +61,6 @@ inline static unsigned int onSslDtlsTimer(SSL* /*ssl*/, unsigned int timerUs)
 	}
 }
 
-inline static long onSslBioOut(
-  BIO* bio,
-  int operationType,
-  const char* argp,
-  size_t len,
-  int /*argi*/,
-  long /*argl*/,
-  int ret,
-  size_t* /*processed*/)
-{
-	const long resultOfcallback = (operationType == BIO_CB_RETURN) ? static_cast<long>(ret) : 1;
-
-	if (operationType == BIO_CB_WRITE && argp && len > 0)
-	{
-		MS_DEBUG_DEV("%zu bytes of DTLS data ready to be sent", len);
-
-		auto* dtlsTransport = reinterpret_cast<RTC::DtlsTransport*>(BIO_get_callback_arg(bio));
-
-		dtlsTransport->SendDtlsData(reinterpret_cast<const uint8_t*>(argp), len);
-	}
-
-	return resultOfcallback;
-}
-
 namespace RTC
 {
 	/* Static. */
@@ -730,8 +706,6 @@ namespace RTC
 			goto error;
 		}
 
-		BIO_set_callback_ex(this->sslBioToNetwork, onSslBioOut);
-		BIO_set_callback_arg(this->sslBioToNetwork, reinterpret_cast<char*>(this));
 		SSL_set_bio(this->ssl, this->sslBioFromNetwork, this->sslBioToNetwork);
 
 		// Set the MTU so that we don't send packets that are too large with no fragmentation.
@@ -778,6 +752,7 @@ namespace RTC
 		{
 			// Send close alert to the peer.
 			SSL_shutdown(this->ssl);
+			SendPendingOutgoingDtlsData();
 		}
 
 		if (this->ssl)
@@ -970,6 +945,9 @@ namespace RTC
 		// Must call SSL_read() to process received DTLS data.
 		read = SSL_read(this->ssl, static_cast<void*>(DtlsTransport::sslReadBuffer), SslReadBufferSize);
 
+		// Send data if it's ready.
+		SendPendingOutgoingDtlsData();
+
 		// Check SSL status and return if it is bad/closed.
 		if (!CheckStatus(read))
 		{
@@ -1036,14 +1014,9 @@ namespace RTC
 			MS_WARN_TAG(
 			  dtls, "OpenSSL SSL_write() wrote less (%d bytes) than given data (%zu bytes)", written, len);
 		}
-	}
-
-	void DtlsTransport::SendDtlsData(const uint8_t* data, size_t len)
-	{
-		MS_TRACE();
 
-		// Notify the listener.
-		this->listener->OnDtlsTransportSendData(this, data, len);
+		// Send data.
+		SendPendingOutgoingDtlsData();
 	}
 
 	void DtlsTransport::Reset()
@@ -1062,8 +1035,9 @@ namespace RTC
 		// Stop the DTLS timer.
 		this->timer->Stop();
 
-		// NOTE: We need to reset the SSL instance so we need to "shutdown" it, but we
-		// don't want to send a Close Alert to the peer. However this is gonna happen.
+		// NOTE: We need to reset the SSL instance so we need to "shutdown" it, but
+		// we don't want to send a Close Alert to the peer, so just don't call
+		// SendPendingOutgoingDTLSData().
 		SSL_shutdown(this->ssl);
 
 		this->localRole.reset();
@@ -1083,7 +1057,7 @@ namespace RTC
 		}
 	}
 
-	inline bool DtlsTransport::CheckStatus(int returnCode)
+	bool DtlsTransport::CheckStatus(int returnCode)
 	{
 		MS_TRACE();
 
@@ -1200,7 +1174,37 @@ namespace RTC
 		}
 	}
 
-	inline bool DtlsTransport::SetTimeout()
+	void DtlsTransport::SendPendingOutgoingDtlsData()
+	{
+		MS_TRACE();
+
+		if (BIO_eof(this->sslBioToNetwork))
+		{
+			return;
+		}
+
+		int64_t read;
+		char* data{ nullptr };
+
+		read = BIO_get_mem_data(this->sslBioToNetwork, &data); // NOLINT
+
+		if (read <= 0)
+		{
+			return;
+		}
+
+		MS_DEBUG_DEV("%" PRIu64 " bytes of DTLS data ready to sent to the peer", read);
+
+		// Notify the listener.
+		this->listener->OnDtlsTransportSendData(
+		  this, reinterpret_cast<uint8_t*>(data), static_cast<size_t>(read));
+
+		// Clear the BIO buffer.
+		// NOTE: the (void) avoids the -Wunused-value warning.
+		(void)BIO_reset(this->sslBioToNetwork);
+	}
+
+	bool DtlsTransport::SetTimeout()
 	{
 		MS_TRACE();
 
@@ -1250,7 +1254,7 @@ namespace RTC
 		}
 	}
 
-	inline bool DtlsTransport::ProcessHandshake()
+	bool DtlsTransport::ProcessHandshake()
 	{
 		MS_TRACE();
 
@@ -1292,7 +1296,7 @@ namespace RTC
 		return false;
 	}
 
-	inline bool DtlsTransport::CheckRemoteFingerprint()
+	bool DtlsTransport::CheckRemoteFingerprint()
 	{
 		MS_TRACE();
 
@@ -1424,7 +1428,7 @@ namespace RTC
 		return true;
 	}
 
-	inline void DtlsTransport::ExtractSrtpKeys(RTC::SrtpSession::CryptoSuite srtpCryptoSuite)
+	void DtlsTransport::ExtractSrtpKeys(RTC::SrtpSession::CryptoSuite srtpCryptoSuite)
 	{
 		MS_TRACE();
 
@@ -1529,7 +1533,7 @@ namespace RTC
 		delete[] srtpRemoteMasterKey;
 	}
 
-	inline std::optional<RTC::SrtpSession::CryptoSuite> DtlsTransport::GetNegotiatedSrtpCryptoSuite()
+	std::optional<RTC::SrtpSession::CryptoSuite> DtlsTransport::GetNegotiatedSrtpCryptoSuite()
 	{
 		MS_TRACE();
 
@@ -1563,7 +1567,7 @@ namespace RTC
 		return negotiatedSrtpCryptoSuite;
 	}
 
-	inline void DtlsTransport::OnSslInfo(int where, int ret)
+	void DtlsTransport::OnSslInfo(int where, int ret)
 	{
 		MS_TRACE();
 
@@ -1650,7 +1654,7 @@ namespace RTC
 		// receipt of a close alert does not work (the flag is set after this callback).
 	}
 
-	inline void DtlsTransport::OnTimer(TimerHandle* /*timer*/)
+	void DtlsTransport::OnTimer(TimerHandle* /*timer*/)
 	{
 		MS_TRACE();
 
@@ -1670,6 +1674,9 @@ namespace RTC
 
 		if (ret == 1)
 		{
+			// If required, send DTLS data.
+			SendPendingOutgoingDtlsData();
+
 			// Set the DTLS timer again.
 			SetTimeout();
 		}