queue up to 32 DATAGRAM frames to send (#4222)

connection.go

@@ -2359,7 +2359,7 @@ func (s *connection) SendDatagram(p []byte) error {
 	}
 	f.Data = make([]byte, len(p))
 	copy(f.Data, p)
-	return s.datagramQueue.AddAndWait(f)
+	return s.datagramQueue.Add(f)
 }
 
 func (s *connection) ReceiveDatagram(ctx context.Context) ([]byte, error) {

datagram_queue.go

@@ -4,14 +4,19 @@ import (
 	"context"
 	"sync"
 
-	"github.com/quic-go/quic-go/internal/protocol"
 	"github.com/quic-go/quic-go/internal/utils"
 	"github.com/quic-go/quic-go/internal/wire"
 )
 
+const (
+	maxDatagramSendQueueLen = 32
+	maxDatagramRcvQueueLen  = 128
+)
+
 type datagramQueue struct {
-	sendQueue chan *wire.DatagramFrame
+	sendMx    sync.Mutex
-	nextFrame *wire.DatagramFrame
+	sendQueue []*wire.DatagramFrame // TODO: this could be a ring buffer
+	sent      chan struct{}         // used to notify Add that a datagram was dequeued
 
 	rcvMx    sync.Mutex
 	rcvQueue [][]byte
@@ -22,60 +27,68 @@ type datagramQueue struct {
 
 	hasData func()
 
-	dequeued chan struct{}
-
 	logger utils.Logger
 }
 
 func newDatagramQueue(hasData func(), logger utils.Logger) *datagramQueue {
 	return &datagramQueue{
-		hasData:   hasData,
+		hasData: hasData,
-		sendQueue: make(chan *wire.DatagramFrame, 1),
+		rcvd:    make(chan struct{}, 1),
-		rcvd:      make(chan struct{}, 1),
+		sent:    make(chan struct{}, 1),
-		dequeued:  make(chan struct{}),
+		closed:  make(chan struct{}),
-		closed:    make(chan struct{}),
+		logger:  logger,
-		logger:    logger,
 	}
 }
 
-// AddAndWait queues a new DATAGRAM frame for sending.
+// Add queues a new DATAGRAM frame for sending.
-// It blocks until the frame has been dequeued.
+// Up to 32 DATAGRAM frames will be queued.
-func (h *datagramQueue) AddAndWait(f *wire.DatagramFrame) error {
+// Once that limit is reached, Add blocks until the queue size has reduced.
-	select {
+func (h *datagramQueue) Add(f *wire.DatagramFrame) error {
-	case h.sendQueue <- f:
+	h.sendMx.Lock()
-		h.hasData()
-	case <-h.closed:
-		return h.closeErr
-	}
 
-	select {
+	for {
-	case <-h.dequeued:
+		if len(h.sendQueue) < maxDatagramSendQueueLen {
-		return nil
+			h.sendQueue = append(h.sendQueue, f)
-	case <-h.closed:
+			h.sendMx.Unlock()
-		return h.closeErr
+			h.hasData()
+			return nil
+		}
+		select {
+		case <-h.sent: // drain the queue so we don't loop immediately
+		default:
+		}
+		h.sendMx.Unlock()
+		select {
+		case <-h.closed:
+			return h.closeErr
+		case <-h.sent:
+		}
+		h.sendMx.Lock()
 	}
 }
 
 // Peek gets the next DATAGRAM frame for sending.
 // If actually sent out, Pop needs to be called before the next call to Peek.
 func (h *datagramQueue) Peek() *wire.DatagramFrame {
-	if h.nextFrame != nil {
+	h.sendMx.Lock()
-		return h.nextFrame
+	defer h.sendMx.Unlock()
-	}
+	if len(h.sendQueue) == 0 {
-	select {
-	case h.nextFrame = <-h.sendQueue:
-		h.dequeued <- struct{}{}
-	default:
 		return nil
 	}
-	return h.nextFrame
+	return h.sendQueue[0]
 }
 
 func (h *datagramQueue) Pop() {
-	if h.nextFrame == nil {
+	h.sendMx.Lock()
+	defer h.sendMx.Unlock()
+	if len(h.sendQueue) == 0 {
 		panic("datagramQueue BUG: Pop called for nil frame")
 	}
-	h.nextFrame = nil
+	h.sendQueue = h.sendQueue[1:]
+	select {
+	case h.sent <- struct{}{}:
+	default:
+	}
 }
 
 // HandleDatagramFrame handles a received DATAGRAM frame.
@@ -84,7 +97,7 @@ func (h *datagramQueue) HandleDatagramFrame(f *wire.DatagramFrame) {
 	copy(data, f.Data)
 	var queued bool
 	h.rcvMx.Lock()
-	if len(h.rcvQueue) < protocol.DatagramRcvQueueLen {
+	if len(h.rcvQueue) < maxDatagramRcvQueueLen {
 		h.rcvQueue = append(h.rcvQueue, data)
 		queued = true
 		select {
@@ -94,7 +107,7 @@ func (h *datagramQueue) HandleDatagramFrame(f *wire.DatagramFrame) {
 	}
 	h.rcvMx.Unlock()
 	if !queued && h.logger.Debug() {
-		h.logger.Debugf("Discarding DATAGRAM frame (%d bytes payload)", len(f.Data))
+		h.logger.Debugf("Discarding received DATAGRAM frame (%d bytes payload)", len(f.Data))
 	}
 }
 

datagram_queue_test.go

@@ -3,6 +3,7 @@ package quic
 import (
 	"context"
 	"errors"
+	"time"
 
 	"github.com/quic-go/quic-go/internal/utils"
 	"github.com/quic-go/quic-go/internal/wire"
@@ -26,55 +27,65 @@ var _ = Describe("Datagram Queue", func() {
 		})
 
 		It("queues a datagram", func() {
-			done := make(chan struct{})
 			frame := &wire.DatagramFrame{Data: []byte("foobar")}
-			go func() {
+			Expect(queue.Add(frame)).To(Succeed())
-				defer GinkgoRecover()
+			Expect(queued).To(HaveLen(1))
-				defer close(done)
-				Expect(queue.AddAndWait(frame)).To(Succeed())
-			}()
-
-			Eventually(queued).Should(HaveLen(1))
-			Consistently(done).ShouldNot(BeClosed())
 			f := queue.Peek()
 			Expect(f.Data).To(Equal([]byte("foobar")))
-			Eventually(done).Should(BeClosed())
 			queue.Pop()
 			Expect(queue.Peek()).To(BeNil())
 		})
 
-		It("returns the same datagram multiple times, when Pop isn't called", func() {
+		It("blocks when the maximum number of datagrams have been queued", func() {
-			sent := make(chan struct{}, 1)
+			for i := 0; i < maxDatagramSendQueueLen; i++ {
+				Expect(queue.Add(&wire.DatagramFrame{Data: []byte{0}})).To(Succeed())
+			}
+			errChan := make(chan error, 1)
 			go func() {
 				defer GinkgoRecover()
-				Expect(queue.AddAndWait(&wire.DatagramFrame{Data: []byte("foo")})).To(Succeed())
+				errChan <- queue.Add(&wire.DatagramFrame{Data: []byte("foobar")})
-				sent <- struct{}{}
-				Expect(queue.AddAndWait(&wire.DatagramFrame{Data: []byte("bar")})).To(Succeed())
-				sent <- struct{}{}
 			}()
+			Consistently(errChan, 50*time.Millisecond).ShouldNot(Receive())
+			Expect(queue.Peek()).ToNot(BeNil())
+			Consistently(errChan, 50*time.Millisecond).ShouldNot(Receive())
+			queue.Pop()
+			Eventually(errChan).Should(Receive(BeNil()))
+			for i := 1; i < maxDatagramSendQueueLen; i++ {
+				queue.Pop()
+			}
+			f := queue.Peek()
+			Expect(f).ToNot(BeNil())
+			Expect(f.Data).To(Equal([]byte("foobar")))
+		})
 
-			Eventually(queued).Should(HaveLen(1))
+		It("returns the same datagram multiple times, when Pop isn't called", func() {
+			Expect(queue.Add(&wire.DatagramFrame{Data: []byte("foo")})).To(Succeed())
+			Expect(queue.Add(&wire.DatagramFrame{Data: []byte("bar")})).To(Succeed())
+
+			Eventually(queued).Should(HaveLen(2))
 			f := queue.Peek()
 			Expect(f.Data).To(Equal([]byte("foo")))
-			Eventually(sent).Should(Receive())
 			Expect(queue.Peek()).To(Equal(f))
 			Expect(queue.Peek()).To(Equal(f))
 			queue.Pop()
-			Eventually(func() *wire.DatagramFrame { f = queue.Peek(); return f }).ShouldNot(BeNil())
 			f = queue.Peek()
+			Expect(f).ToNot(BeNil())
 			Expect(f.Data).To(Equal([]byte("bar")))
 		})
 
 		It("closes", func() {
+			for i := 0; i < maxDatagramSendQueueLen; i++ {
+				Expect(queue.Add(&wire.DatagramFrame{Data: []byte("foo")})).To(Succeed())
+			}
 			errChan := make(chan error, 1)
 			go func() {
 				defer GinkgoRecover()
-				errChan <- queue.AddAndWait(&wire.DatagramFrame{Data: []byte("foobar")})
+				errChan <- queue.Add(&wire.DatagramFrame{Data: []byte("foo")})
 			}()
-
+			Consistently(errChan, 25*time.Millisecond).ShouldNot(Receive())
-			Consistently(errChan).ShouldNot(Receive())
+			testErr := errors.New("test error")
-			queue.CloseWithError(errors.New("test error"))
+			queue.CloseWithError(testErr)
-			Eventually(errChan).Should(Receive(MatchError("test error")))
+			Eventually(errChan).Should(Receive(MatchError(testErr)))
 		})
 	})
 

internal/protocol/params.go

@@ -129,9 +129,6 @@ const MaxPostHandshakeCryptoFrameSize = 1000
 // but must ensure that a maximum size ACK frame fits into one packet.
 const MaxAckFrameSize ByteCount = 1000
 
-// DatagramRcvQueueLen is the length of the receive queue for DATAGRAM frames (RFC 9221)
-const DatagramRcvQueueLen = 128
-
 // MaxNumAckRanges is the maximum number of ACK ranges that we send in an ACK frame.
 // It also serves as a limit for the packet history.
 // If at any point we keep track of more ranges, old ranges are discarded.

packet_packer_test.go

@@ -602,7 +602,7 @@ var _ = Describe("Packet packer", func() {
 				go func() {
 					defer GinkgoRecover()
 					defer close(done)
-					datagramQueue.AddAndWait(f)
+					datagramQueue.Add(f)
 				}()
 				// make sure the DATAGRAM has actually been queued
 				time.Sleep(scaleDuration(20 * time.Millisecond))
@@ -630,7 +630,7 @@ var _ = Describe("Packet packer", func() {
 				go func() {
 					defer GinkgoRecover()
 					defer close(done)
-					datagramQueue.AddAndWait(f)
+					datagramQueue.Add(f)
 				}()
 				// make sure the DATAGRAM has actually been queued
 				time.Sleep(scaleDuration(20 * time.Millisecond))
@@ -659,7 +659,7 @@ var _ = Describe("Packet packer", func() {
 				go func() {
 					defer GinkgoRecover()
 					defer close(done)
-					datagramQueue.AddAndWait(f)
+					datagramQueue.Add(f)
 				}()
 				// make sure the DATAGRAM has actually been queued
 				time.Sleep(scaleDuration(20 * time.Millisecond))