mirrors/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2025-04-03 20:47:35 +03:00
Code Issues Projects Releases Packages Wiki Activity

Using squirrel to generalize SQL search

Browse source
This commit is contained in:
Deluan 2020-01-13 15:28:55 -05:00 • committed by Deluan Quintão
parent d3af7e689d
commit c2448d3880
10 changed files with 134 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

1
go.mod
View file

@ -5,6 +5,7 @@ go 1.13
require (
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/DataDog/zstd v1.4.4 // indirect
github.com/Masterminds/squirrel v1.1.0
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 // indirect
github.com/asdine/storm v2.1.2+incompatible
github.com/astaxie/beego v1.12.0

6
go.sum
View file

@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE=
github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/squirrel v1.1.0 h1:baP1qLdoQCeTw3ifCdOq2dkYc6vGcmRdaociKLbEJXs=
github.com/Masterminds/squirrel v1.1.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA=
github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM=
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM=
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
@ -79,6 +81,10 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=

4
persistence/db_sql/album_repository.go
View file

@ -41,7 +41,9 @@ func NewAlbumRepository() domain.AlbumRepository {
func (r *albumRepository) Put(a *domain.Album) error {
ta := Album(*a)
return r.put(a.ID, &ta)
return WithTx(func(o orm.Ormer) error {
return r.put(o, a.ID, &ta)
})
}
func (r *albumRepository) Get(id string) (*domain.Album, error) {

4
persistence/db_sql/artist_repository.go
View file

@ -24,7 +24,9 @@ func NewArtistRepository() domain.ArtistRepository {
func (r *artistRepository) Put(a *domain.Artist) error {
ta := Artist(*a)
return r.put(a.ID, &ta)
return WithTx(func(o orm.Ormer) error {
return r.put(o, a.ID, &ta)
})
}
func (r *artistRepository) Get(id string) (*domain.Artist, error) {

2
persistence/db_sql/checksum_repository.go
View file

@ -62,7 +62,7 @@ func (r *checkSumRepository) SetData(newSums map[string]string) error {
cks := Checksum{ID: k, Sum: v}
checksums = append(checksums, cks)
}
_, err = Db().InsertMulti(100, &checksums)
_, err = Db().InsertMulti(batchSize, &checksums)
if err != nil {
return err
}

26
persistence/db_sql/mediafile_repository.go
View file

@ -1,7 +1,6 @@
package db_sql
import (
"strings"
"time"
"github.com/astaxie/beego/orm"
@ -27,9 +26,9 @@ type MediaFile struct {
BitRate int ``
Genre string ``
Compilation bool ``
PlayCount int ``
PlayCount int `orm:"index"`
PlayDate time.Time `orm:"null"`
Rating int ``
Rating int `orm:"index"`
Starred bool `orm:"index"`
StarredAt time.Time `orm:"null"`
CreatedAt time.Time `orm:"null"`
@ -48,7 +47,13 @@ func NewMediaFileRepository() domain.MediaFileRepository {
func (r *mediaFileRepository) Put(m *domain.MediaFile) error {
tm := MediaFile(*m)
return r.put(m.ID, &tm)
return WithTx(func(o orm.Ormer) error {
err := r.put(o, m.ID, &tm)
if err != nil {
return err
}
return r.searcher.Index(o, r.tableName, m.ID, m.Title)
})
}
func (r *mediaFileRepository) Get(id string) (*domain.MediaFile, error) {
@ -97,19 +102,12 @@ func (r *mediaFileRepository) PurgeInactive(activeList domain.MediaFiles) ([]str
}
func (r *mediaFileRepository) Search(q string, offset int, size int) (domain.MediaFiles, error) {
parts := strings.Split(q, " ")
if len(parts) == 0 {
if len(q) <= 2 {
return nil, nil
}
qs := r.newQuery(Db(), domain.QueryOptions{Offset: offset, Size: size})
cond := orm.NewCondition()
for _, part := range parts {
c := orm.NewCondition()
cond = cond.AndCond(c.Or("title__istartswith", part).Or("title__icontains", " "+part))
}
qs = qs.SetCond(cond).OrderBy("-rating", "-starred", "-play_count")
var results []MediaFile
_, err := qs.All(&results)
err := r.searcher.Search(r.tableName, q, offset, size, &results, "rating desc", "starred desc", "play_count desc", "title")
if err != nil {
return nil, err
}

4
persistence/db_sql/playlist_repository.go
View file

@ -30,7 +30,9 @@ func NewPlaylistRepository() domain.PlaylistRepository {
func (r *playlistRepository) Put(p *domain.Playlist) error {
tp := r.fromDomain(p)
return r.put(p.ID, &tp)
return WithTx(func(o orm.Ormer) error {
return r.put(o, p.ID, &tp)
})
}
func (r *playlistRepository) Get(id string) (*domain.Playlist, error) {

3
persistence/db_sql/sql.go
View file

@ -11,6 +11,8 @@ import (
_ "github.com/mattn/go-sqlite3"
)
const batchSize = 100
var once sync.Once
func Db() orm.Ormer {
@ -66,6 +68,7 @@ func initORM(dbPath string) error {
orm.RegisterModel(new(Checksum))
orm.RegisterModel(new(Property))
orm.RegisterModel(new(Playlist))
orm.RegisterModel(new(Search))
err := orm.RegisterDataBase("default", "sqlite3", dbPath)
if err != nil {
panic(err)

36
persistence/db_sql/sql_repository.go
View file

@ -9,6 +9,7 @@ import (
type sqlRepository struct {
tableName string
searcher sqlSearcher
}
func (r *sqlRepository) newQuery(o orm.Ormer, options ...domain.QueryOptions) orm.QuerySeter {
@ -55,19 +56,17 @@ func (r *sqlRepository) GetAllIds() ([]string, error) {
return result, nil
}
func (r *sqlRepository) put(id string, a interface{}) error {
return WithTx(func(o orm.Ormer) error {
c, err := r.newQuery(o).Filter("id", id).Count()
if err != nil {
return err
}
if c == 0 {
_, err = o.Insert(a)
return err
}
_, err = o.Update(a)
func (r *sqlRepository) put(o orm.Ormer, id string, a interface{}) error {
c, err := r.newQuery(o).Filter("id", id).Count()
if err != nil {
return err
})
}
if c == 0 {
_, err = o.Insert(a)
return err
}
_, err = o.Update(a)
return err
}
func paginateSlice(slice []string, skip int, size int) []string {
@ -104,8 +103,13 @@ func difference(slice1 []string, slice2 []string) []string {
}
func (r *sqlRepository) DeleteAll() error {
_, err := r.newQuery(Db()).Filter("id__isnull", false).Delete()
return err
return WithTx(func(o orm.Ormer) error {
_, err := r.newQuery(Db()).Filter("id__isnull", false).Delete()
if err != nil {
return err
}
return r.searcher.DeleteAll(o, r.tableName)
})
}
func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) {
@ -123,7 +127,7 @@ func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item in
err = WithTx(func(o orm.Ormer) error {
var offset int
for {
var subset = paginateSlice(idsToDelete, offset, 100)
var subset = paginateSlice(idsToDelete, offset, batchSize)
if len(subset) == 0 {
break
}
@ -134,7 +138,7 @@ func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item in
return err
}
}
return nil
return r.searcher.Remove(o, r.tableName, idsToDelete)
})
return idsToDelete, err
}

82
persistence/db_sql/sql_searcher.go Normal file
View file

@ -0,0 +1,82 @@
package db_sql
import (
"strings"
"github.com/Masterminds/squirrel"
"github.com/astaxie/beego/orm"
"github.com/cloudsonic/sonic-server/log"
"github.com/kennygrant/sanitize"
)
type Search struct {
ID string `orm:"pk;column(id)"`
Table string `orm:"index"`
FullText string `orm:"type(text)"`
}
type sqlSearcher struct{}
func (s *sqlSearcher) Index(o orm.Ormer, table, id, text string) error {
item := Search{ID: id, Table: table}
err := o.Read(&item)
if err != nil && err != orm.ErrNoRows {
return err
}
sanitizedText := strings.TrimSpace(sanitize.Accents(strings.ToLower(text)))
item = Search{ID: id, Table: table, FullText: sanitizedText}
if err == orm.ErrNoRows {
_, err = o.Insert(&item)
} else {
_, err = o.Update(&item)
}
return err
}
func (s *sqlSearcher) Remove(o orm.Ormer, table string, ids []string) error {
var offset int
for {
var subset = paginateSlice(ids, offset, batchSize)
if len(subset) == 0 {
break
}
log.Trace("Deleting searchable items", "table", table, "num", len(subset), "from", offset)
offset += len(subset)
_, err := o.QueryTable(&Search{}).Filter("table", table).Filter("id__in", subset).Delete()
if err != nil {
return err
}
}
return nil
}
func (s *sqlSearcher) DeleteAll(o orm.Ormer, table string) error {
_, err := o.QueryTable(&Search{}).Filter("table", table).Delete()
return err
}
func (s *sqlSearcher) Search(table string, q string, offset, size int, results interface{}, orderBys ...string) error {
q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))
if len(q) <= 2 {
return nil
}
sq := squirrel.Select("*").From(table).OrderBy()
sq = sq.Limit(uint64(size)).Offset(uint64(offset))
if len(orderBys) > 0 {
sq = sq.OrderBy(orderBys...)
}
sq = sq.Join("search").Where("search.id = " + table + ".id")
parts := strings.Split(q, " ")
for _, part := range parts {
sq = sq.Where(squirrel.Or{
squirrel.Like{"full_text": part + "%"},
squirrel.Like{"full_text": "%" + part + "%"},
})
}
sql, args, err := sq.ToSql()
if err != nil {
return err
}
_, err = Db().Raw(sql, args...).QueryRows(results)
return err
}