diff --git a/ackhandler/received_packet_handler.go b/ackhandler/received_packet_handler.go
index 6fef3a64..72110747 100644
--- a/ackhandler/received_packet_handler.go
+++ b/ackhandler/received_packet_handler.go
@@ -22,9 +22,11 @@ type receivedPacketHandler struct {
 	highestInOrderObserved        protocol.PacketNumber
 	highestInOrderObservedEntropy EntropyAccumulator
 	largestObserved               protocol.PacketNumber
-	packetHistory                 map[protocol.PacketNumber]packetHistoryEntry
 	currentAckFrame               *frames.AckFrame
 	stateChanged                  bool // has an ACK for this state already been sent? Will be set to false every time a new packet arrives, and to false every time an ACK is sent
+
+	packetHistory           map[protocol.PacketNumber]packetHistoryEntry
+	smallestInPacketHistory protocol.PacketNumber
 }
 
 // NewReceivedPacketHandler creates a new receivedPacketHandler
@@ -59,6 +61,9 @@ func (h *receivedPacketHandler) ReceivedPacket(packetNumber protocol.PacketNumbe
 		EntropyBit:   entropyBit,
 		TimeReceived: time.Now(),
 	}
+
+	h.garbageCollect()
+
 	return nil
 }
 
@@ -72,6 +77,8 @@ func (h *receivedPacketHandler) ReceivedStopWaiting(f *frames.StopWaitingFrame)
 	h.highestInOrderObserved = f.LeastUnacked - 1
 	h.highestInOrderObservedEntropy = EntropyAccumulator(f.Entropy)
 
+	h.garbageCollect()
+
 	return nil
 }
 
@@ -130,3 +137,10 @@ func (h *receivedPacketHandler) GetAckFrame(dequeue bool) (*frames.AckFrame, err
 	}
 	return h.currentAckFrame, nil
 }
+
+func (h *receivedPacketHandler) garbageCollect() {
+	for i := h.smallestInPacketHistory; i < h.highestInOrderObserved; i++ {
+		delete(h.packetHistory, i)
+	}
+	h.smallestInPacketHistory = h.highestInOrderObserved
+}
diff --git a/ackhandler/received_packet_handler_test.go b/ackhandler/received_packet_handler_test.go
index 6028d2ba..906030f7 100644
--- a/ackhandler/received_packet_handler_test.go
+++ b/ackhandler/received_packet_handler_test.go
@@ -25,13 +25,13 @@ var _ = Describe("receivedPacketHandler", func() {
 		It("handles a packet that arrives late", func() {
 			err := handler.ReceivedPacket(protocol.PacketNumber(1), false)
 			Expect(err).ToNot(HaveOccurred())
+			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(1)))
 			err = handler.ReceivedPacket(protocol.PacketNumber(3), false)
 			Expect(err).ToNot(HaveOccurred())
+			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(3)))
 			err = handler.ReceivedPacket(protocol.PacketNumber(2), false)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(1)))
 			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(2)))
-			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(3)))
 		})
 
 		It("rejects packets with packet number 0", func() {
@@ -334,4 +334,26 @@ var _ = Describe("receivedPacketHandler", func() {
 			Expect(ack.NackRanges).To(BeEmpty())
 		})
 	})
+
+	Context("Garbage Collector", func() {
+		It("only keeps packets with packet numbers higher than the highestInOrderObserved in packetHistory", func() {
+			handler.ReceivedPacket(1, true)
+			handler.ReceivedPacket(2, true)
+			handler.ReceivedPacket(4, true)
+			Expect(handler.packetHistory).ToNot(HaveKey(protocol.PacketNumber(1)))
+			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(2)))
+			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(4)))
+		})
+
+		It("garbage collects packetHistory after receiving a StopWaiting", func() {
+			handler.ReceivedPacket(1, true)
+			handler.ReceivedPacket(2, true)
+			handler.ReceivedPacket(4, true)
+			swf := frames.StopWaitingFrame{LeastUnacked: 4}
+			handler.ReceivedStopWaiting(&swf)
+			Expect(handler.packetHistory).ToNot(HaveKey(protocol.PacketNumber(1)))
+			Expect(handler.packetHistory).ToNot(HaveKey(protocol.PacketNumber(2)))
+			Expect(handler.packetHistory).To(HaveKey(protocol.PacketNumber(4)))
+		})
+	})
 })