diff --git a/ackhandler/received_packet_handler.go b/ackhandler/received_packet_handler.go
index fd3e5695..4f804e60 100644
--- a/ackhandler/received_packet_handler.go
+++ b/ackhandler/received_packet_handler.go
@@ -135,5 +135,7 @@ func (h *receivedPacketHandler) garbageCollectReceivedTimes() {
 	for i := h.lowestInReceivedTimes; i <= h.ignorePacketsBelow; i++ {
 		delete(h.receivedTimes, i)
 	}
-	h.lowestInReceivedTimes = h.ignorePacketsBelow
+	if h.ignorePacketsBelow > h.lowestInReceivedTimes {
+		h.lowestInReceivedTimes = h.ignorePacketsBelow + 1
+	}
 }
diff --git a/ackhandler/received_packet_handler_test.go b/ackhandler/received_packet_handler_test.go
index 956e4b71..c0b0413f 100644
--- a/ackhandler/received_packet_handler_test.go
+++ b/ackhandler/received_packet_handler_test.go
@@ -278,6 +278,7 @@ var _ = Describe("receivedPacketHandler", func() {
 			Expect(handler.receivedTimes).ToNot(HaveKey(protocol.PacketNumber(2)))
 			Expect(handler.receivedTimes).To(HaveKey(protocol.PacketNumber(3)))
 			Expect(handler.receivedTimes).To(HaveKey(protocol.PacketNumber(4)))
+			Expect(handler.lowestInReceivedTimes).To(Equal(protocol.PacketNumber(3)))
 		})
 
 		It("garbage collects the receivedTimes after receiving multiple StopWaitings", func() {
@@ -292,6 +293,7 @@ var _ = Describe("receivedPacketHandler", func() {
 			Expect(handler.receivedTimes).To(HaveLen(2)) // packets 8 and 9
 			Expect(handler.receivedTimes).To(HaveKey(protocol.PacketNumber(8)))
 			Expect(handler.receivedTimes).To(HaveKey(protocol.PacketNumber(9)))
+			Expect(handler.lowestInReceivedTimes).To(Equal(protocol.PacketNumber(8)))
 		})
 
 		It("garbage collects receivedTimes after receiving a StopWaiting, if there are missing packets", func() {
@@ -306,6 +308,7 @@ var _ = Describe("receivedPacketHandler", func() {
 			Expect(handler.receivedTimes).ToNot(HaveKey(protocol.PacketNumber(1)))
 			Expect(handler.receivedTimes).ToNot(HaveKey(protocol.PacketNumber(2)))
 			Expect(handler.receivedTimes).To(HaveKey(protocol.PacketNumber(4)))
+			Expect(handler.lowestInReceivedTimes).To(Equal(protocol.PacketNumber(4)))
 		})
 	})
 })