diff --git a/conn_id_manager.go b/conn_id_manager.go
index a2b268df..2fc10145 100644
--- a/conn_id_manager.go
+++ b/conn_id_manager.go
@@ -15,7 +15,7 @@ type connIDManager struct {
 	queue utils.NewConnectionIDList
 
 	activeSequenceNumber      uint64
-	retiredPriorTo            uint64
+	highestRetired            uint64
 	activeConnectionID        protocol.ConnectionID
 	activeStatelessResetToken *[16]byte
 
@@ -65,7 +65,7 @@ func (h *connIDManager) Add(f *wire.NewConnectionIDFrame) error {
 func (h *connIDManager) add(f *wire.NewConnectionIDFrame) error {
 	// If the NEW_CONNECTION_ID frame is reordered, such that its sequenece number
 	// was already retired, send the RETIRE_CONNECTION_ID frame immediately.
-	if f.SequenceNumber < h.retiredPriorTo {
+	if f.SequenceNumber < h.highestRetired {
 		h.queueControlFrame(&wire.RetireConnectionIDFrame{
 			SequenceNumber: f.SequenceNumber,
 		})
@@ -74,7 +74,7 @@ func (h *connIDManager) add(f *wire.NewConnectionIDFrame) error {
 
 	// Retire elements in the queue.
 	// Doesn't retire the active connection ID.
-	if f.RetirePriorTo > h.retiredPriorTo {
+	if f.RetirePriorTo > h.highestRetired {
 		var next *utils.NewConnectionIDElement
 		for el := h.queue.Front(); el != nil; el = next {
 			if el.Value.SequenceNumber >= f.RetirePriorTo {
@@ -86,7 +86,7 @@ func (h *connIDManager) add(f *wire.NewConnectionIDFrame) error {
 			})
 			h.queue.Remove(el)
 		}
-		h.retiredPriorTo = f.RetirePriorTo
+		h.highestRetired = f.RetirePriorTo
 	}
 
 	if f.SequenceNumber == h.activeSequenceNumber {
@@ -135,9 +135,11 @@ func (h *connIDManager) updateConnectionID() {
 	h.queueControlFrame(&wire.RetireConnectionIDFrame{
 		SequenceNumber: h.activeSequenceNumber,
 	})
+	h.highestRetired = utils.MaxUint64(h.highestRetired, h.activeSequenceNumber)
 	if h.activeStatelessResetToken != nil {
 		h.retireStatelessResetToken(*h.activeStatelessResetToken)
 	}
+
 	front := h.queue.Remove(h.queue.Front())
 	h.activeSequenceNumber = front.SequenceNumber
 	h.activeConnectionID = front.ConnectionID
diff --git a/conn_id_manager_test.go b/conn_id_manager_test.go
index 49af58d0..66047f10 100644
--- a/conn_id_manager_test.go
+++ b/conn_id_manager_test.go
@@ -15,7 +15,7 @@ var _ = Describe("Connection ID Manager", func() {
 		retiredTokens [][16]byte
 		removedTokens [][16]byte
 	)
-	initialConnID := protocol.ConnectionID{1, 1, 1, 1}
+	initialConnID := protocol.ConnectionID{0, 0, 0, 0}
 
 	BeforeEach(func() {
 		frameQueue = nil
@@ -243,6 +243,35 @@ var _ = Describe("Connection ID Manager", func() {
 		Expect(counter).To(BeNumerically("~", 50, 10))
 	})
 
+	It("retires delayed connection IDs that arrive after a higher connection ID was already retired", func() {
+		for s := uint8(10); s <= 10+protocol.MaxActiveConnectionIDs/2; s++ {
+			Expect(m.Add(&wire.NewConnectionIDFrame{
+				SequenceNumber:      uint64(s),
+				ConnectionID:        protocol.ConnectionID{s, s, s, s},
+				StatelessResetToken: [16]byte{s, s, s, s, s, s, s, s, s, s, s, s, s, s, s, s},
+			})).To(Succeed())
+		}
+		Expect(m.Get()).To(Equal(protocol.ConnectionID{10, 10, 10, 10}))
+		for {
+			m.SentPacket()
+			if m.Get().Equal(protocol.ConnectionID{11, 11, 11, 11}) {
+				break
+			}
+		}
+		// The active conn ID is now {11, 11, 11, 11}
+		Expect(m.queue.Front().Value.ConnectionID).To(Equal(protocol.ConnectionID{12, 12, 12, 12}))
+		// Add a delayed connection ID. It should just be ignored now.
+		frameQueue = nil
+		Expect(m.Add(&wire.NewConnectionIDFrame{
+			SequenceNumber:      uint64(5),
+			ConnectionID:        protocol.ConnectionID{5, 5, 5, 5},
+			StatelessResetToken: [16]byte{5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5},
+		})).To(Succeed())
+		Expect(m.queue.Front().Value.ConnectionID).To(Equal(protocol.ConnectionID{12, 12, 12, 12}))
+		Expect(frameQueue).To(HaveLen(1))
+		Expect(frameQueue[0].(*wire.RetireConnectionIDFrame).SequenceNumber).To(BeEquivalentTo(5))
+	})
+
 	It("only initiates subsequent updates when enough if enough connection IDs are queued", func() {
 		for i := uint8(1); i <= protocol.MaxActiveConnectionIDs/2; i++ {
 			Expect(m.Add(&wire.NewConnectionIDFrame{