From c2448d3880640b9a0a77d187f417db0a247f6c30 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 13 Jan 2020 15:28:55 -0500 Subject: [PATCH] Using squirrel to generalize SQL search --- go.mod | 1 + go.sum | 6 ++ persistence/db_sql/album_repository.go | 4 +- persistence/db_sql/artist_repository.go | 4 +- persistence/db_sql/checksum_repository.go | 2 +- persistence/db_sql/mediafile_repository.go | 26 ++++--- persistence/db_sql/playlist_repository.go | 4 +- persistence/db_sql/sql.go | 3 + persistence/db_sql/sql_repository.go | 36 +++++----- persistence/db_sql/sql_searcher.go | 82 ++++++++++++++++++++++ 10 files changed, 134 insertions(+), 34 deletions(-) create mode 100644 persistence/db_sql/sql_searcher.go diff --git a/go.mod b/go.mod index 140044038..24fe1ee07 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/DataDog/zstd v1.4.4 // indirect + github.com/Masterminds/squirrel v1.1.0 github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 // indirect github.com/asdine/storm v2.1.2+incompatible github.com/astaxie/beego v1.12.0 diff --git a/go.sum b/go.sum index 627a31aa1..81bb44052 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/squirrel v1.1.0 h1:baP1qLdoQCeTw3ifCdOq2dkYc6vGcmRdaociKLbEJXs= +github.com/Masterminds/squirrel v1.1.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA= github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM= github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM= github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM= @@ -79,6 +81,10 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= diff --git a/persistence/db_sql/album_repository.go b/persistence/db_sql/album_repository.go index 37d1c4360..51962befd 100644 --- a/persistence/db_sql/album_repository.go +++ b/persistence/db_sql/album_repository.go @@ -41,7 +41,9 @@ func NewAlbumRepository() domain.AlbumRepository { func (r *albumRepository) Put(a *domain.Album) error { ta := Album(*a) - return r.put(a.ID, &ta) + return WithTx(func(o orm.Ormer) error { + return r.put(o, a.ID, &ta) + }) } func (r *albumRepository) Get(id string) (*domain.Album, error) { diff --git a/persistence/db_sql/artist_repository.go b/persistence/db_sql/artist_repository.go index 43854f2d6..bdc3c282e 100644 --- a/persistence/db_sql/artist_repository.go +++ b/persistence/db_sql/artist_repository.go @@ -24,7 +24,9 @@ func NewArtistRepository() domain.ArtistRepository { func (r *artistRepository) Put(a *domain.Artist) error { ta := Artist(*a) - return r.put(a.ID, &ta) + return WithTx(func(o orm.Ormer) error { + return r.put(o, a.ID, &ta) + }) } func (r *artistRepository) Get(id string) (*domain.Artist, error) { diff --git a/persistence/db_sql/checksum_repository.go b/persistence/db_sql/checksum_repository.go index c00027a47..7d7b4be4f 100644 --- a/persistence/db_sql/checksum_repository.go +++ b/persistence/db_sql/checksum_repository.go @@ -62,7 +62,7 @@ func (r *checkSumRepository) SetData(newSums map[string]string) error { cks := Checksum{ID: k, Sum: v} checksums = append(checksums, cks) } - _, err = Db().InsertMulti(100, &checksums) + _, err = Db().InsertMulti(batchSize, &checksums) if err != nil { return err } diff --git a/persistence/db_sql/mediafile_repository.go b/persistence/db_sql/mediafile_repository.go index c88967bc3..6c1d40a3a 100644 --- a/persistence/db_sql/mediafile_repository.go +++ b/persistence/db_sql/mediafile_repository.go @@ -1,7 +1,6 @@ package db_sql import ( - "strings" "time" "github.com/astaxie/beego/orm" @@ -27,9 +26,9 @@ type MediaFile struct { BitRate int `` Genre string `` Compilation bool `` - PlayCount int `` + PlayCount int `orm:"index"` PlayDate time.Time `orm:"null"` - Rating int `` + Rating int `orm:"index"` Starred bool `orm:"index"` StarredAt time.Time `orm:"null"` CreatedAt time.Time `orm:"null"` @@ -48,7 +47,13 @@ func NewMediaFileRepository() domain.MediaFileRepository { func (r *mediaFileRepository) Put(m *domain.MediaFile) error { tm := MediaFile(*m) - return r.put(m.ID, &tm) + return WithTx(func(o orm.Ormer) error { + err := r.put(o, m.ID, &tm) + if err != nil { + return err + } + return r.searcher.Index(o, r.tableName, m.ID, m.Title) + }) } func (r *mediaFileRepository) Get(id string) (*domain.MediaFile, error) { @@ -97,19 +102,12 @@ func (r *mediaFileRepository) PurgeInactive(activeList domain.MediaFiles) ([]str } func (r *mediaFileRepository) Search(q string, offset int, size int) (domain.MediaFiles, error) { - parts := strings.Split(q, " ") - if len(parts) == 0 { + if len(q) <= 2 { return nil, nil } - qs := r.newQuery(Db(), domain.QueryOptions{Offset: offset, Size: size}) - cond := orm.NewCondition() - for _, part := range parts { - c := orm.NewCondition() - cond = cond.AndCond(c.Or("title__istartswith", part).Or("title__icontains", " "+part)) - } - qs = qs.SetCond(cond).OrderBy("-rating", "-starred", "-play_count") + var results []MediaFile - _, err := qs.All(&results) + err := r.searcher.Search(r.tableName, q, offset, size, &results, "rating desc", "starred desc", "play_count desc", "title") if err != nil { return nil, err } diff --git a/persistence/db_sql/playlist_repository.go b/persistence/db_sql/playlist_repository.go index 2cd4a1aba..06f3b68d2 100644 --- a/persistence/db_sql/playlist_repository.go +++ b/persistence/db_sql/playlist_repository.go @@ -30,7 +30,9 @@ func NewPlaylistRepository() domain.PlaylistRepository { func (r *playlistRepository) Put(p *domain.Playlist) error { tp := r.fromDomain(p) - return r.put(p.ID, &tp) + return WithTx(func(o orm.Ormer) error { + return r.put(o, p.ID, &tp) + }) } func (r *playlistRepository) Get(id string) (*domain.Playlist, error) { diff --git a/persistence/db_sql/sql.go b/persistence/db_sql/sql.go index f79b3fddc..bf43d48b7 100644 --- a/persistence/db_sql/sql.go +++ b/persistence/db_sql/sql.go @@ -11,6 +11,8 @@ import ( _ "github.com/mattn/go-sqlite3" ) +const batchSize = 100 + var once sync.Once func Db() orm.Ormer { @@ -66,6 +68,7 @@ func initORM(dbPath string) error { orm.RegisterModel(new(Checksum)) orm.RegisterModel(new(Property)) orm.RegisterModel(new(Playlist)) + orm.RegisterModel(new(Search)) err := orm.RegisterDataBase("default", "sqlite3", dbPath) if err != nil { panic(err) diff --git a/persistence/db_sql/sql_repository.go b/persistence/db_sql/sql_repository.go index 7d64f72b9..dabef1fea 100644 --- a/persistence/db_sql/sql_repository.go +++ b/persistence/db_sql/sql_repository.go @@ -9,6 +9,7 @@ import ( type sqlRepository struct { tableName string + searcher sqlSearcher } func (r *sqlRepository) newQuery(o orm.Ormer, options ...domain.QueryOptions) orm.QuerySeter { @@ -55,19 +56,17 @@ func (r *sqlRepository) GetAllIds() ([]string, error) { return result, nil } -func (r *sqlRepository) put(id string, a interface{}) error { - return WithTx(func(o orm.Ormer) error { - c, err := r.newQuery(o).Filter("id", id).Count() - if err != nil { - return err - } - if c == 0 { - _, err = o.Insert(a) - return err - } - _, err = o.Update(a) +func (r *sqlRepository) put(o orm.Ormer, id string, a interface{}) error { + c, err := r.newQuery(o).Filter("id", id).Count() + if err != nil { return err - }) + } + if c == 0 { + _, err = o.Insert(a) + return err + } + _, err = o.Update(a) + return err } func paginateSlice(slice []string, skip int, size int) []string { @@ -104,8 +103,13 @@ func difference(slice1 []string, slice2 []string) []string { } func (r *sqlRepository) DeleteAll() error { - _, err := r.newQuery(Db()).Filter("id__isnull", false).Delete() - return err + return WithTx(func(o orm.Ormer) error { + _, err := r.newQuery(Db()).Filter("id__isnull", false).Delete() + if err != nil { + return err + } + return r.searcher.DeleteAll(o, r.tableName) + }) } func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) { @@ -123,7 +127,7 @@ func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item in err = WithTx(func(o orm.Ormer) error { var offset int for { - var subset = paginateSlice(idsToDelete, offset, 100) + var subset = paginateSlice(idsToDelete, offset, batchSize) if len(subset) == 0 { break } @@ -134,7 +138,7 @@ func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item in return err } } - return nil + return r.searcher.Remove(o, r.tableName, idsToDelete) }) return idsToDelete, err } diff --git a/persistence/db_sql/sql_searcher.go b/persistence/db_sql/sql_searcher.go new file mode 100644 index 000000000..cc03ca8a9 --- /dev/null +++ b/persistence/db_sql/sql_searcher.go @@ -0,0 +1,82 @@ +package db_sql + +import ( + "strings" + + "github.com/Masterminds/squirrel" + "github.com/astaxie/beego/orm" + "github.com/cloudsonic/sonic-server/log" + "github.com/kennygrant/sanitize" +) + +type Search struct { + ID string `orm:"pk;column(id)"` + Table string `orm:"index"` + FullText string `orm:"type(text)"` +} + +type sqlSearcher struct{} + +func (s *sqlSearcher) Index(o orm.Ormer, table, id, text string) error { + item := Search{ID: id, Table: table} + err := o.Read(&item) + if err != nil && err != orm.ErrNoRows { + return err + } + sanitizedText := strings.TrimSpace(sanitize.Accents(strings.ToLower(text))) + item = Search{ID: id, Table: table, FullText: sanitizedText} + if err == orm.ErrNoRows { + _, err = o.Insert(&item) + } else { + _, err = o.Update(&item) + } + return err +} + +func (s *sqlSearcher) Remove(o orm.Ormer, table string, ids []string) error { + var offset int + for { + var subset = paginateSlice(ids, offset, batchSize) + if len(subset) == 0 { + break + } + log.Trace("Deleting searchable items", "table", table, "num", len(subset), "from", offset) + offset += len(subset) + _, err := o.QueryTable(&Search{}).Filter("table", table).Filter("id__in", subset).Delete() + if err != nil { + return err + } + } + return nil +} + +func (s *sqlSearcher) DeleteAll(o orm.Ormer, table string) error { + _, err := o.QueryTable(&Search{}).Filter("table", table).Delete() + return err +} + +func (s *sqlSearcher) Search(table string, q string, offset, size int, results interface{}, orderBys ...string) error { + q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*")))) + if len(q) <= 2 { + return nil + } + sq := squirrel.Select("*").From(table).OrderBy() + sq = sq.Limit(uint64(size)).Offset(uint64(offset)) + if len(orderBys) > 0 { + sq = sq.OrderBy(orderBys...) + } + sq = sq.Join("search").Where("search.id = " + table + ".id") + parts := strings.Split(q, " ") + for _, part := range parts { + sq = sq.Where(squirrel.Or{ + squirrel.Like{"full_text": part + "%"}, + squirrel.Like{"full_text": "%" + part + "%"}, + }) + } + sql, args, err := sq.ToSql() + if err != nil { + return err + } + _, err = Db().Raw(sql, args...).QueryRows(results) + return err +}