diff --git a/mock_packet_handler_manager_test.go b/mock_packet_handler_manager_test.go
index 8d057df6..e129febf 100644
--- a/mock_packet_handler_manager_test.go
+++ b/mock_packet_handler_manager_test.go
@@ -110,18 +110,6 @@ func (mr *MockPacketHandlerManagerMockRecorder) Remove(arg0 interface{}) *gomock
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockPacketHandlerManager)(nil).Remove), arg0)
 }
 
-// RemoveResetToken mocks base method
-func (m *MockPacketHandlerManager) RemoveResetToken(arg0 [16]byte) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "RemoveResetToken", arg0)
-}
-
-// RemoveResetToken indicates an expected call of RemoveResetToken
-func (mr *MockPacketHandlerManagerMockRecorder) RemoveResetToken(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveResetToken", reflect.TypeOf((*MockPacketHandlerManager)(nil).RemoveResetToken), arg0)
-}
-
 // ReplaceWithClosed mocks base method
 func (m *MockPacketHandlerManager) ReplaceWithClosed(arg0 protocol.ConnectionID, arg1 packetHandler) {
 	m.ctrl.T.Helper()
@@ -146,6 +134,18 @@ func (mr *MockPacketHandlerManagerMockRecorder) Retire(arg0 interface{}) *gomock
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Retire", reflect.TypeOf((*MockPacketHandlerManager)(nil).Retire), arg0)
 }
 
+// RetireResetToken mocks base method
+func (m *MockPacketHandlerManager) RetireResetToken(arg0 [16]byte) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "RetireResetToken", arg0)
+}
+
+// RetireResetToken indicates an expected call of RetireResetToken
+func (mr *MockPacketHandlerManagerMockRecorder) RetireResetToken(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetireResetToken", reflect.TypeOf((*MockPacketHandlerManager)(nil).RetireResetToken), arg0)
+}
+
 // SetServer mocks base method
 func (m *MockPacketHandlerManager) SetServer(arg0 unknownPacketHandler) {
 	m.ctrl.T.Helper()
diff --git a/mock_session_runner_test.go b/mock_session_runner_test.go
index d9f5d3f4..e8308b20 100644
--- a/mock_session_runner_test.go
+++ b/mock_session_runner_test.go
@@ -58,18 +58,6 @@ func (mr *MockSessionRunnerMockRecorder) Remove(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockSessionRunner)(nil).Remove), arg0)
 }
 
-// RemoveResetToken mocks base method
-func (m *MockSessionRunner) RemoveResetToken(arg0 [16]byte) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "RemoveResetToken", arg0)
-}
-
-// RemoveResetToken indicates an expected call of RemoveResetToken
-func (mr *MockSessionRunnerMockRecorder) RemoveResetToken(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveResetToken", reflect.TypeOf((*MockSessionRunner)(nil).RemoveResetToken), arg0)
-}
-
 // ReplaceWithClosed mocks base method
 func (m *MockSessionRunner) ReplaceWithClosed(arg0 protocol.ConnectionID, arg1 packetHandler) {
 	m.ctrl.T.Helper()
@@ -93,3 +81,15 @@ func (mr *MockSessionRunnerMockRecorder) Retire(arg0 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Retire", reflect.TypeOf((*MockSessionRunner)(nil).Retire), arg0)
 }
+
+// RetireResetToken mocks base method
+func (m *MockSessionRunner) RetireResetToken(arg0 [16]byte) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "RetireResetToken", arg0)
+}
+
+// RetireResetToken indicates an expected call of RetireResetToken
+func (mr *MockSessionRunnerMockRecorder) RetireResetToken(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetireResetToken", reflect.TypeOf((*MockSessionRunner)(nil).RetireResetToken), arg0)
+}
diff --git a/packet_handler_map.go b/packet_handler_map.go
index a357d517..f9b22ba6 100644
--- a/packet_handler_map.go
+++ b/packet_handler_map.go
@@ -134,10 +134,12 @@ func (h *packetHandlerMap) AddResetToken(token [16]byte, handler packetHandler)
 	h.mutex.Unlock()
 }
 
-func (h *packetHandlerMap) RemoveResetToken(token [16]byte) {
-	h.mutex.Lock()
-	delete(h.resetTokens, token)
-	h.mutex.Unlock()
+func (h *packetHandlerMap) RetireResetToken(token [16]byte) {
+	time.AfterFunc(h.deleteRetiredSessionsAfter, func() {
+		h.mutex.Lock()
+		delete(h.resetTokens, token)
+		h.mutex.Unlock()
+	})
 }
 
 func (h *packetHandlerMap) SetServer(s unknownPacketHandler) {
diff --git a/packet_handler_map_test.go b/packet_handler_map_test.go
index f7471358..b639247a 100644
--- a/packet_handler_map_test.go
+++ b/packet_handler_map_test.go
@@ -249,22 +249,21 @@ var _ = Describe("Packet Handler Map", func() {
 				Eventually(destroyed).Should(BeClosed())
 			})
 
-			It("deletes reset tokens", func() {
+			It("retires reset tokens", func() {
 				handler.deleteRetiredSessionsAfter = scaleDuration(10 * time.Millisecond)
 				connID := protocol.ConnectionID{0xde, 0xad, 0xbe, 0xef, 0x42}
 				packetHandler := NewMockPacketHandler(mockCtrl)
 				handler.Add(connID, packetHandler)
 				token := [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
 				handler.AddResetToken(token, NewMockPacketHandler(mockCtrl))
-				handler.RemoveResetToken(token)
+				handler.RetireResetToken(token)
 				packetHandler.EXPECT().handlePacket(gomock.Any())
 				p := append([]byte{0x40} /* short header packet */, connID.Bytes()...)
 				p = append(p, make([]byte, 50)...)
 				p = append(p, token[:]...)
+
+				time.Sleep(scaleDuration(30 * time.Millisecond))
 				handler.handlePacket(nil, nil, p)
-				// destroy() would be called from a separate go routine
-				// make sure we give it enough time to be called to cause an error here
-				time.Sleep(scaleDuration(25 * time.Millisecond))
 			})
 
 			It("ignores packets too small to contain a stateless reset", func() {
diff --git a/session.go b/session.go
index 5f8b45e9..f101262b 100644
--- a/session.go
+++ b/session.go
@@ -77,7 +77,7 @@ type sessionRunner interface {
 	Remove(protocol.ConnectionID)
 	ReplaceWithClosed(protocol.ConnectionID, packetHandler)
 	AddResetToken([16]byte, packetHandler)
-	RemoveResetToken([16]byte)
+	RetireResetToken([16]byte)
 }
 
 type handshakeRunner struct {