From 60a5fbe1fe11a0555c67118c7f9a2c8a91366025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Mon, 27 Nov 2023 13:06:23 -0500 Subject: [PATCH] Optimize search3, by removing `OFFSET` when paginating (#2655) * Optimize pagination, removing offset * For search, don't add `where` clause for empty queries * Revert "Replace `COUNT(DISTINCT primary_key)` with `COUNT(*)`" Genres are required as part of the count queries, so filter by genres work * Optimize search3 query, using order by id if it is a "" query. Also fix the optimizePagination query logic * Allow offset optimizer threshold to be configured --- conf/configuration.go | 2 ++ persistence/mediafile_repository.go | 2 +- persistence/sql_base_repository.go | 19 ++++++++++++++++++- persistence/sql_search.go | 23 +++++++++++++++++------ 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/conf/configuration.go b/conf/configuration.go index 13ea3632f..0614c1a29 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -96,6 +96,7 @@ type configOptions struct { DevSidebarPlaylists bool DevEnableBufferedScrobble bool DevShowArtistPage bool + DevOffsetOptimize int DevArtworkMaxRequests int DevArtworkThrottleBacklogLimit int DevArtworkThrottleBacklogTimeout time.Duration @@ -352,6 +353,7 @@ func init() { viper.SetDefault("devenablebufferedscrobble", true) viper.SetDefault("devsidebarplaylists", true) viper.SetDefault("devshowartistpage", true) + viper.SetDefault("devoffsetoptimize", 50000) viper.SetDefault("devartworkmaxrequests", number.Max(2, runtime.NumCPU()/3)) viper.SetDefault("devartworkthrottlebackloglimit", consts.RequestThrottleBacklogLimit) viper.SetDefault("devartworkthrottlebacklogtimeout", consts.RequestThrottleBacklogTimeout) diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index 9ebbf68b3..c0c1b4a56 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -91,7 +91,7 @@ func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) { func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) { sq := r.selectMediaFile(options...) res := model.MediaFiles{} - err := r.queryAll(sq, &res) + err := r.queryAll(sq, &res, options...) if err != nil { return nil, err } diff --git a/persistence/sql_base_repository.go b/persistence/sql_base_repository.go index d0ecd37fb..10899817d 100644 --- a/persistence/sql_base_repository.go +++ b/persistence/sql_base_repository.go @@ -10,6 +10,7 @@ import ( . "github.com/Masterminds/squirrel" "github.com/beego/beego/v2/client/orm" "github.com/google/uuid" + "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" @@ -157,7 +158,10 @@ func (r sqlRepository) queryOne(sq Sqlizer, response interface{}) error { return err } -func (r sqlRepository) queryAll(sq Sqlizer, response interface{}) error { +func (r sqlRepository) queryAll(sq SelectBuilder, response interface{}, options ...model.QueryOptions) error { + if len(options) > 0 && options[0].Offset > 0 { + sq = r.optimizePagination(sq, options[0]) + } query, args, err := sq.ToSql() if err != nil { return err @@ -172,6 +176,19 @@ func (r sqlRepository) queryAll(sq Sqlizer, response interface{}) error { return err } +// optimizePagination uses a less inefficient pagination, by not using OFFSET. +// See https://gist.github.com/ssokolow/262503 +func (r sqlRepository) optimizePagination(sq SelectBuilder, options model.QueryOptions) SelectBuilder { + if options.Offset > conf.Server.DevOffsetOptimize { + sq = sq.RemoveOffset() + oidSq := sq.RemoveColumns().Columns(r.tableName + ".oid") + oidSq = oidSq.Limit(uint64(options.Offset)) + oidSql, args, _ := oidSq.ToSql() + sq = sq.Where(r.tableName+".oid not in ("+oidSql+")", args...) + } + return sq +} + func (r sqlRepository) exists(existsQuery SelectBuilder) (bool, error) { existsQuery = existsQuery.Columns("count(*) as exist").From(r.tableName) var res struct{ Exist int64 } diff --git a/persistence/sql_search.go b/persistence/sql_search.go index 0ef0d623e..bd4295368 100644 --- a/persistence/sql_search.go +++ b/persistence/sql_search.go @@ -5,6 +5,7 @@ import ( . "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils" ) @@ -21,21 +22,31 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, } sq := r.newSelectWithAnnotation(r.tableName + ".id").Columns("*") - sq = sq.Limit(uint64(size)).Offset(uint64(offset)) - if len(orderBys) > 0 { - sq = sq.OrderBy(orderBys...) + filter := fullTextExpr(q) + if filter != nil { + sq = sq.Where(filter) + if len(orderBys) > 0 { + sq = sq.OrderBy(orderBys...) + } + } else { + // If the filter is empty, we sort by id. + // This is to speed up the results of `search3?query=""`, for OpenSubsonic + sq = sq.OrderBy("id") } - sq = sq.Where(fullTextExpr(q)) - err := r.queryAll(sq, results) + sq = sq.Limit(uint64(size)).Offset(uint64(offset)) + err := r.queryAll(sq, results, model.QueryOptions{Offset: offset}) return err } func fullTextExpr(value string) Sqlizer { + q := utils.SanitizeStrings(value) + if q == "" { + return nil + } var sep string if !conf.Server.SearchFullString { sep = " " } - q := utils.SanitizeStrings(value) parts := strings.Split(q, " ") filters := And{} for _, part := range parts {