From 536e60a4ec97aed7c55740ec8513cbbc23c638e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Aled=C5=BA?= Date: Fri, 31 Jan 2025 18:47:48 +0100 Subject: [PATCH] WIP fixes --- .../peer_connection/configuration.ex | 12 ++-- lib/ex_webrtc/rtp_sender.ex | 58 ++++++++++--------- lib/ex_webrtc/rtp_transceiver.ex | 11 +++- 3 files changed, 47 insertions(+), 34 deletions(-) diff --git a/lib/ex_webrtc/peer_connection/configuration.ex b/lib/ex_webrtc/peer_connection/configuration.ex index 49cf012c..43b901b4 100644 --- a/lib/ex_webrtc/peer_connection/configuration.ex +++ b/lib/ex_webrtc/peer_connection/configuration.ex @@ -24,11 +24,6 @@ defmodule ExWebRTC.PeerConnection.Configuration do ] @default_video_codecs [ - %RTPCodecParameters{ - payload_type: 96, - mime_type: "video/VP8", - clock_rate: 90_000 - }, %RTPCodecParameters{ payload_type: 98, mime_type: "video/H264", @@ -40,6 +35,11 @@ defmodule ExWebRTC.PeerConnection.Configuration do profile_level_id: 0x42E01F } }, + %RTPCodecParameters{ + payload_type: 96, + mime_type: "video/VP8", + clock_rate: 90_000 + }, %RTPCodecParameters{ payload_type: 45, mime_type: "video/AV1", @@ -528,6 +528,8 @@ defmodule ExWebRTC.PeerConnection.Configuration do codecs |> Enum.find( # as of now, we ignore sdp_fmtp_line + # Once we start checking FMTP, rtp sender code + # needs to be adjusted to check it too. &(String.downcase(&1.mime_type) == String.downcase(sdp_codec.mime_type) and &1.payload_type == sdp_codec.payload_type and &1.clock_rate == sdp_codec.clock_rate and diff --git a/lib/ex_webrtc/rtp_sender.ex b/lib/ex_webrtc/rtp_sender.ex index 97540ae0..ea72628d 100644 --- a/lib/ex_webrtc/rtp_sender.ex +++ b/lib/ex_webrtc/rtp_sender.ex @@ -22,8 +22,6 @@ defmodule ExWebRTC.RTPSender do codecs: [RTPCodecParameters.t()], rtp_hdr_exts: %{Extmap.extension_id() => Extmap.t()}, mid: String.t() | nil, - pt: non_neg_integer() | nil, - rtx_pt: non_neg_integer() | nil, # ssrc and rtx_ssrc are always present, even if there is no track, # or transceiver direction is recvonly. # We preallocate them so they can be included in SDP when needed. @@ -85,11 +83,8 @@ defmodule ExWebRTC.RTPSender do # In other case, if PeerConnection negotiated multiple codecs, # user would have to pass RTP codec when sending RTP packets, # or assign payload type on their own. - {codec, rtx_codec} = get_default_codec(codecs) - # TODO: handle cases when codec == nil (no valid codecs after negotiation) - pt = if codec != nil, do: codec.payload_type, else: nil - rtx_pt = if rtx_codec != nil, do: rtx_codec.payload_type, else: nil + {codec, rtx_codec} = get_default_codec(codecs) %{ id: Utils.generate_id(), @@ -98,8 +93,6 @@ defmodule ExWebRTC.RTPSender do rtx_codec: rtx_codec, codecs: codecs, rtp_hdr_exts: rtp_hdr_exts, - pt: pt, - rtx_pt: rtx_pt, ssrc: ssrc, rtx_ssrc: rtx_ssrc, mid: mid, @@ -126,42 +119,39 @@ defmodule ExWebRTC.RTPSender do # Keep already selected codec if it is still supported. # Otherwise, clear it and wait until user sets it again. - codec = if sender.codec in codecs, do: sender.codec, else: nil + # TODO: handle cases when codec == nil (no valid codecs after negotiation) + codec = if supported?(codecs, sender.codec), do: sender.codec, else: nil rtx_codec = codec && find_associated_rtx_codec(codecs, codec) log_codec_change(sender, codec, codecs) log_rtx_codec_change(sender, rtx_codec, codecs) - # TODO: handle cases when codec == nil (no valid codecs after negotiation) - pt = if codec != nil, do: codec.payload_type, else: nil - rtx_pt = if rtx_codec != nil, do: rtx_codec.payload_type, else: nil - %{ sender | mid: mid, codec: codec, rtx_codec: rtx_codec, codecs: codecs, - rtp_hdr_exts: rtp_hdr_exts, - pt: pt, - rtx_pt: rtx_pt + rtp_hdr_exts: rtp_hdr_exts } end defp log_codec_change(%{codec: codec} = sender, nil, neg_codecs) when codec != nil do - Logger.debug(""" + Logger.warning(""" Unselecting RTP sender codec as it is no longer supported by the remote side. Call set_sender_codec again passing supported codec. - Codec: #{inspect(sender.codec)} - Currently negotiated codecs: #{inspect(neg_codecs)} + Codec: #{inspect(sender.codec, pretty: true)} + Currently negotiated codecs: #{inspect(neg_codecs, pretty: true)} """) + + raise "" end defp log_codec_change(_sender, _codec, _neg_codecs), do: :ok defp log_rtx_codec_change(%{rtx_codec: rtx_codec} = sender, nil, neg_codecs) when rtx_codec != nil do - Logger.debug(""" + Logger.warning(""" Unselecting RTP sender codec as it is no longer supported by the remote side. Call set_sender_codec again passing supported codec. Codec: #{inspect(sender.codec)} @@ -186,7 +176,13 @@ defmodule ExWebRTC.RTPSender do end ssrc_attrs = - get_ssrc_attrs(sender.pt, sender.rtx_pt, sender.ssrc, sender.rtx_ssrc, sender.track) + get_ssrc_attrs( + sender.codec.payload_type, + sender.rtx_codec.payload_type, + sender.ssrc, + sender.rtx_ssrc, + sender.track + ) msid_attrs ++ ssrc_attrs end @@ -251,7 +247,7 @@ defmodule ExWebRTC.RTPSender do @doc false @spec set_codec(sender(), RTPCodecParameters.t()) :: {:ok, sender()} | {:error, term()} def set_codec(sender, codec) do - if not rtx?(codec) and supported?(sender, codec) and same_clock_rate?(sender, codec) do + if not rtx?(codec) and supported?(sender.codecs, codec) and same_clock_rate?(sender, codec) do rtx_codec = find_associated_rtx_codec(sender.codecs, codec) sender = %{sender | codec: codec, rtx_codec: rtx_codec} {:ok, sender} @@ -261,7 +257,17 @@ defmodule ExWebRTC.RTPSender do end defp rtx?(codec), do: String.ends_with?(codec.mime_type, "rtx") - defp supported?(sender, codec), do: codec in sender.codecs + + defp supported?(neg_codecs, codec) do + # Once we start checking FMTP in Configuration.intersect_codecs, + # this code has to be adjusted. + codec = %{codec | sdp_fmtp_line: nil, rtcp_fbs: MapSet.new(codec.rtcp_fbs)} + + Enum.find(neg_codecs, fn s_codec -> + s_codec = %{s_codec | sdp_fmtp_line: nil, rtcp_fbs: MapSet.new(s_codec.rtcp_fbs)} + codec == s_codec + end) != nil + end # As long as report recorder is not initialized i.e. we have not sent any RTP packet, # allow for codec changes. Once we start sending RTP packets, require the same clock rate. @@ -271,12 +277,10 @@ defmodule ExWebRTC.RTPSender do @doc false @spec send_packet(sender(), ExRTP.Packet.t(), boolean()) :: {binary(), sender()} def send_packet(%{rtx_codec: nil} = sender, _packet, true) do - Logger.warning("Tried to retransmit packet but there is no selected RTX codec. Ignoring.") {<<>>, sender} end def send_packet(%{codec: nil} = sender, _packet, false) do - Logger.warning("Tried to send packet but there is no selected codec. Ignoring.") {<<>>, sender} end @@ -297,9 +301,9 @@ defmodule ExWebRTC.RTPSender do def do_send_packet(sender, packet, rtx?) do {pt, ssrc} = if rtx? do - {sender.rtx_pt, sender.rtx_ssrc} + {sender.rtx_codec.payload_type, sender.rtx_ssrc} else - {sender.pt, sender.ssrc} + {sender.codec.payload_type, sender.ssrc} end packet = %{packet | payload_type: pt, ssrc: ssrc} diff --git a/lib/ex_webrtc/rtp_transceiver.ex b/lib/ex_webrtc/rtp_transceiver.ex index e1707d79..7544e972 100644 --- a/lib/ex_webrtc/rtp_transceiver.ex +++ b/lib/ex_webrtc/rtp_transceiver.ex @@ -281,9 +281,16 @@ defmodule ExWebRTC.RTPTransceiver do @spec set_sender_codec(transceiver(), RTPCodecParameters.t()) :: {:ok, transceiver()} | {:error, term()} def set_sender_codec(transceiver, codec) do + dbg(transceiver) + dbg(codec) + case RTPSender.set_codec(transceiver.sender, codec) do - {:ok, sender} -> {:ok, %{transceiver | sender: sender}} - {:error, _reason} = error -> error + {:ok, sender} -> + dbg(sender) + {:ok, %{transceiver | sender: sender}} + + {:error, _reason} = error -> + error end end