Simulcast.

README.PROTOCOL

@@ -135,8 +135,17 @@ A peer must explicitly request the streams that it wants to receive.
 ```
 
 The field `request` is a dictionary that maps the labels of requested
-streams to a list containing either 'audio', 'video' or both.  An entry
-with an empty key `''` serves as default.
+streams to a list containing either 'audio', or one of 'video' or
+'video-low'.  The empty key `''` serves as default.  For example:
+
+```javascript
+{
+    type: 'request',
+    request: {
+        camera: ['audio', 'video-low'],
+        '': ['audio', 'video']
+    }
+}
 
 ## Pushing streams
 
@@ -157,16 +166,22 @@ A stream is created by the sender with the `offer` message:
 If a stream with the same id exists, then this is a renegotation;
 otherwise this message creates a new stream.  If the field `replace` is
 not empty, then this request additionally requests that an existing stream
-with the given id should be closed, and the new stream should replace it.
+with the given id should be closed, and the new stream should replace it;
+this is used most notably when changing the simulcast envelope.
 
-The field `label` is one of `camera`, `screenshare` or `video`, as in the
-`request` message.
+The field `label` is one of `camera`, `screenshare` or `video`, and will
+be matched against the keys sent by the receiver in their `request` message.
 
 The field `sdp` contains the raw SDP string (i.e. the `sdp` field of
 a JSEP session description).  Galène will interpret the `nack`,
 `nack pli`, `ccm fir` and `goog-remb` RTCP feedback types, and act
 accordingly.
 
+The sender may either send a single stream per media section in the SDP,
+or use rid-based simulcasting.  In the latter case, it should send two
+video streams, one with rid 'h' and high throughput, and one with rid 'l'
+and throughput limited to roughly 100kbit/s.
+
 The receiver may either abort the stream immediately (see below), or send
 an answer.
 

conn/conn.go

@@ -25,6 +25,7 @@ type UpTrack interface {
 	AddLocal(DownTrack) error
 	DelLocal(DownTrack) bool
 	Kind() webrtc.RTPCodecType
+	Label() string
 	Codec() webrtc.RTPCodecCapability
 	// get a recent packet.  Returns 0 if the packet is not in cache.
 	GetRTP(seqno uint16, result []byte) uint16
@@ -33,7 +34,6 @@ type UpTrack interface {
 
 // Type Down represents a connection in the server to client direction.
 type Down interface {
-	GetMaxBitrate(now uint64) uint64
 }
 
 // Type DownTrack represents a track in the server to client direction.
@@ -42,4 +42,5 @@ type DownTrack interface {
 	Accumulate(bytes uint32)
 	SetTimeOffset(ntp uint64, rtp uint32)
 	SetCname(string)
+	GetMaxBitrate() uint64
 }

diskwriter/diskwriter.go

@@ -600,7 +600,7 @@ func (conn *diskConn) initWriter(width, height uint32) error {
 	return nil
 }
 
-func (down *diskConn) GetMaxBitrate(now uint64) uint64 {
+func (t *diskTrack) GetMaxBitrate() uint64 {
 	return ^uint64(0)
 }
 

group/group.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/pion/ice/v2"
+	"github.com/pion/sdp/v3"
 	"github.com/pion/webrtc/v3"
 )
 
@@ -60,7 +61,8 @@ type ChatHistoryEntry struct {
 }
 
 const (
-	MinBitrate = 200000
+	LowBitrate = 100000
+	MinBitrate = 2 * LowBitrate
 )
 
 type Group struct {
@@ -252,6 +254,12 @@ func APIFromCodecs(codecs []webrtc.RTPCodecCapability) (*webrtc.API, error) {
 	if UDPMin > 0 && UDPMax > 0 {
 		s.SetEphemeralUDPPortRange(UDPMin, UDPMax)
 	}
+	m.RegisterHeaderExtension(
+		webrtc.RTPHeaderExtensionCapability{sdp.SDESMidURI},
+		webrtc.RTPCodecTypeVideo)
+	m.RegisterHeaderExtension(
+		webrtc.RTPHeaderExtensionCapability{sdp.SDESRTPStreamIDURI},
+		webrtc.RTPCodecTypeVideo)
 
 	return webrtc.NewAPI(
 		webrtc.WithSettingEngine(s),

rtpconn/rtpconn.go

@@ -77,15 +77,16 @@ type downTrackAtomics struct {
 }
 
 type rtpDownTrack struct {
-	track      *webrtc.TrackLocalStaticRTP
-	sender     *webrtc.RTPSender
-	remote     conn.UpTrack
-	ssrc       webrtc.SSRC
-	maxBitrate *bitrate
-	rate       *estimator.Estimator
-	stats      *receiverStats
-	atomics    *downTrackAtomics
-	cname      atomic.Value
+	track          *webrtc.TrackLocalStaticRTP
+	sender         *webrtc.RTPSender
+	remote         conn.UpTrack
+	ssrc           webrtc.SSRC
+	maxBitrate     *bitrate
+	maxREMBBitrate *bitrate
+	rate           *estimator.Estimator
+	stats          *receiverStats
+	atomics        *downTrackAtomics
+	cname          atomic.Value
 }
 
 func (down *rtpDownTrack) WriteRTP(packet *rtp.Packet) error {
@@ -140,7 +141,6 @@ type rtpDownConnection struct {
 	id                string
 	pc                *webrtc.PeerConnection
 	remote            conn.Up
-	maxREMBBitrate    *bitrate
 	iceCandidates     []*webrtc.ICECandidateInit
 	negotiationNeeded int
 
@@ -174,31 +174,22 @@ func newDownConn(c group.Client, id string, remote conn.Up) (*rtpDownConnection,
 		id:             id,
 		pc:             pc,
 		remote:         remote,
-		maxREMBBitrate: new(bitrate),
 	}
 
 	return conn, nil
 }
 
-func (down *rtpDownConnection) GetMaxBitrate(now uint64) uint64 {
-	rate := down.maxREMBBitrate.Get(now)
-	var trackRate uint64
-	tracks := down.getTracks()
-	for _, t := range tracks {
-		r := t.maxBitrate.Get(now)
-		if r == ^uint64(0) {
-			if t.track.Kind() == webrtc.RTPCodecTypeAudio {
-				r = 128 * 1024
-			} else {
-				r = 512 * 1024
-			}
-		}
-		trackRate += r
+func (t *rtpDownTrack) GetMaxBitrate() uint64 {
+	now := rtptime.Jiffies()
+	r := t.maxBitrate.Get(now)
+	if r == ^uint64(0) {
+		r = 512 * 1024
 	}
-	if trackRate < rate {
-		return trackRate
+	rr := t.maxREMBBitrate.Get(now)
+	if rr == 0 || r < rr {
+		return r
 	}
-	return rate
+	return rr
 }
 
 func (down *rtpDownConnection) addICECandidate(candidate *webrtc.ICECandidateInit) error {
@@ -311,6 +302,10 @@ func (up *rtpUpTrack) GetRTP(seqno uint16, result []byte) uint16 {
 	return up.cache.Get(seqno, result)
 }
 
+func (up *rtpUpTrack) Label() string {
+	return up.track.RID()
+}
+
 func (up *rtpUpTrack) Kind() webrtc.RTPCodecType {
 	return up.track.Kind()
 }
@@ -687,7 +682,7 @@ func rtcpUpListener(conn *rtpUpConnection, track *rtpUpTrack, r *webrtc.RTPRecei
 
 	for {
 		firstSR := false
-		n, _, err := r.Read(buf)
+		n, _, err := r.ReadSimulcast(buf, track.track.RID())
 		if err != nil {
 			if err != io.EOF && err != io.ErrClosedPipe {
 				log.Printf("Read RTCP: %v", err)
@@ -752,11 +747,11 @@ func rtcpUpListener(conn *rtpUpConnection, track *rtpUpTrack, r *webrtc.RTPRecei
 	}
 }
 
-func sendUpRTCP(conn *rtpUpConnection) error {
-	tracks := conn.getTracks()
+func sendUpRTCP(up *rtpUpConnection) error {
+	tracks := up.getTracks()
 
-	if len(conn.tracks) == 0 {
-		state := conn.pc.ConnectionState()
+	if len(up.tracks) == 0 {
+		state := up.pc.ConnectionState()
 		if state == webrtc.PeerConnectionStateClosed {
 			return io.ErrClosedPipe
 		}
@@ -765,7 +760,7 @@ func sendUpRTCP(conn *rtpUpConnection) error {
 
 	now := rtptime.Jiffies()
 
-	reports := make([]rtcp.ReceptionReport, 0, len(conn.tracks))
+	reports := make([]rtcp.ReceptionReport, 0, len(up.tracks))
 	for _, t := range tracks {
 		updateUpTrack(t)
 		stats := t.cache.GetStats(true)
@@ -810,37 +805,46 @@ func sendUpRTCP(conn *rtpUpConnection) error {
 		},
 	}
 
-	rate := ^uint64(0)
-
-	local := conn.getLocal()
-	for _, l := range local {
-		r := l.GetMaxBitrate(now)
-		if r < rate {
-			rate = r
-		}
-	}
-
-	if rate < group.MinBitrate {
-		rate = group.MinBitrate
-	}
-
 	var ssrcs []uint32
+	var rate uint64
 	for _, t := range tracks {
 		if !t.hasRtcpFb("goog-remb", "") {
 			continue
 		}
 		ssrcs = append(ssrcs, uint32(t.track.SSRC()))
+		var r uint64
+		if t.Kind() == webrtc.RTPCodecTypeAudio {
+			r = 100 * 1024
+		} else if t.Label() == "l" {
+			r = group.LowBitrate
+		} else {
+			local := t.getLocal()
+			r = ^uint64(0)
+			for _, down := range local {
+				rr := down.GetMaxBitrate()
+				if rr < group.MinBitrate {
+					rr = group.MinBitrate
+				}
+				if r > rr {
+					r = rr
+				}
+			}
+			if r == ^uint64(0) {
+				r = 512 * 1024
+			}
+		}
+		rate += r
 	}
 
-	if len(ssrcs) > 0 {
+	if rate < ^uint64(0) && len(ssrcs) > 0 {
 		packets = append(packets,
 			&rtcp.ReceiverEstimatedMaximumBitrate{
 				Bitrate: rate,
 				SSRCs:   ssrcs,
 			},
 		)
 	}
-	return conn.pc.WriteRTCP(packets)
+	return up.pc.WriteRTCP(packets)
 }
 
 func rtcpUpSender(conn *rtpUpConnection) {
@@ -1049,7 +1053,7 @@ func rtcpDownListener(conn *rtpDownConnection, track *rtpDownTrack, s *webrtc.RT
 					log.Printf("sendFIR: %v", err)
 				}
 			case *rtcp.ReceiverEstimatedMaximumBitrate:
-				conn.maxREMBBitrate.Set(p.Bitrate, jiffies)
+				track.maxREMBBitrate.Set(p.Bitrate, jiffies)
 			case *rtcp.ReceiverReport:
 				for _, r := range p.Reports {
 					if r.SSRC == uint32(track.ssrc) {

rtpconn/rtpreader.go

@@ -149,6 +149,16 @@ func readLoop(conn *rtpUpConnection, track *rtpUpTrack) {
 
 		kf, _ := isKeyframe(codec.MimeType, &packet)
 
+		if packet.Extension {
+			packet.Extension = false
+			packet.Extensions = nil
+			bytes, err = packet.MarshalTo(buf)
+			if err != nil {
+				log.Printf("%v", err)
+				continue
+			}
+		}
+
 		first, index := track.cache.Store(
 			packet.SequenceNumber, packet.Timestamp,
 			kf, packet.Marker, buf[:bytes],

rtpconn/rtpstats.go

@@ -47,7 +47,6 @@ func (c *webClient) GetStats() *stats.Client {
 	for _, down := range c.down {
 		conns := stats.Conn{
 			Id:         down.id,
-			MaxBitrate: down.GetMaxBitrate(jiffies),
 		}
 		for _, t := range down.tracks {
 			rate, _ := t.rate.Estimate()