diff --git a/packet_handler_map.go b/packet_handler_map.go
index 80f874b9..b723b54e 100644
--- a/packet_handler_map.go
+++ b/packet_handler_map.go
@@ -250,7 +250,7 @@ func (h *packetHandlerMap) CloseServer() {
 	wg.Wait()
 }
 
-// Destroy the underlying connection and wait until listen() has returned.
+// Destroy closes the underlying connection and waits until listen() has returned.
 // It does not close active sessions.
 func (h *packetHandlerMap) Destroy() error {
 	if err := h.conn.Close(); err != nil {
diff --git a/server.go b/server.go
index 40ff7c8b..5f81911e 100644
--- a/server.go
+++ b/server.go
@@ -274,25 +274,27 @@ func (s *baseServer) accept(ctx context.Context) (quicSession, error) {
 
 // Close the server
 func (s *baseServer) Close() error {
-	s.sessionHandler.CloseServer()
 	s.mutex.Lock()
-	defer s.mutex.Unlock()
 	if s.closed {
+		s.mutex.Unlock()
 		return nil
 	}
 	if s.serverError == nil {
 		s.serverError = errors.New("server closed")
 	}
-	var err error
 	// If the server was started with ListenAddr, we created the packet conn.
 	// We need to close it in order to make the go routine reading from that conn return.
-	if s.createdPacketConn {
-		err = s.sessionHandler.Destroy()
-	}
+	createdPacketConn := s.createdPacketConn
 	s.closed = true
 	close(s.errorChan)
+	s.mutex.Unlock()
+
 	<-s.running
-	return err
+	s.sessionHandler.CloseServer()
+	if createdPacketConn {
+		return s.sessionHandler.Destroy()
+	}
+	return nil
 }
 
 func (s *baseServer) setCloseError(e error) {