diff --git a/server.go b/server.go
index a8c6fd7e..930997e7 100644
--- a/server.go
+++ b/server.go
@@ -718,10 +718,6 @@ func (s *baseServer) handleInitialImpl(p receivedPacket, hdr *wire.Header) error
 			conn.closeWithTransportError(ConnectionRefused)
 		}
 	}()
-	if conn == nil {
-		p.buffer.Release()
-		return nil
-	}
 	return nil
 }