mirrors/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2025-04-06 14:07:36 +03:00
Code Issues Projects Releases Packages Wiki Activity

Implement Artist search with SQL, removed old search/indexer code

Browse source
This commit is contained in:
Deluan 2020-01-13 16:02:49 -05:00 • committed by Deluan Quintão
parent 3a9559a860
commit 614f4afe28
11 changed files with 45 additions and 171 deletions

10
api/wire_gen.go
View file

@ -7,9 +7,6 @@ package api
import (
"github.com/cloudsonic/sonic-server/itunesbridge"
"github.com/cloudsonic/sonic-server/persistence/db_ledis"
"github.com/deluan/gomate"
"github.com/deluan/gomate/ledis"
"github.com/google/wire"
)
@ -78,10 +75,5 @@ var allProviders = wire.NewSet(itunesbridge.NewItunesControl, NewSystemControlle
NewSearchingController,
NewUsersController,
NewMediaRetrievalController,
NewStreamController,
newDB, wire.FieldsOf(new(*Router), "Browser", "Cover", "ListGenerator", "Playlists", "Ratings", "Scrobbler", "Search"),
NewStreamController, wire.FieldsOf(new(*Router), "Browser", "Cover", "ListGenerator", "Playlists", "Ratings", "Scrobbler", "Search"),
)
func newDB() gomate.DB {
return ledis.NewEmbeddedDB(db_ledis.Db())
}

8
api/wire_injectors.go
View file

@ -4,9 +4,6 @@ package api
import (
"github.com/cloudsonic/sonic-server/itunesbridge"
"github.com/cloudsonic/sonic-server/persistence/db_ledis"
"github.com/deluan/gomate"
"github.com/deluan/gomate/ledis"
"github.com/google/wire"
)
@ -21,7 +18,6 @@ var allProviders = wire.NewSet(
NewUsersController,
NewMediaRetrievalController,
NewStreamController,
newDB,
wire.FieldsOf(new(*Router), "Browser", "Cover", "ListGenerator", "Playlists", "Ratings", "Scrobbler", "Search"),
)
@ -60,7 +56,3 @@ func initMediaRetrievalController(router *Router) *MediaRetrievalController {
func initStreamController(router *Router) *StreamController {
panic(wire.Build(allProviders))
}
func newDB() gomate.DB {
return ledis.NewEmbeddedDB(db_ledis.Db())
}

1
domain/artist.go
View file

@ -11,6 +11,7 @@ type ArtistRepository interface {
Put(m *Artist) error
Get(id string) (*Artist, error)
PurgeInactive(active Artists) ([]string, error)
Search(q string, offset int, size int) (Artists, error)
}
type Artists []Artist

100
engine/search.go
View file

@ -5,21 +5,10 @@ import (
"strings"
"github.com/cloudsonic/sonic-server/domain"
"github.com/cloudsonic/sonic-server/log"
"github.com/deluan/gomate"
"github.com/kennygrant/sanitize"
)
type Search interface {
ClearAll() error
IndexArtist(ar *domain.Artist) error
IndexAlbum(al *domain.Album) error
IndexMediaFile(mf *domain.MediaFile) error
RemoveArtist(ids ...string) error
RemoveAlbum(ids ...string) error
RemoveMediaFile(ids ...string) error
SearchArtist(ctx context.Context, q string, offset int, size int) (Entries, error)
SearchAlbum(ctx context.Context, q string, offset int, size int) (Entries, error)
SearchSong(ctx context.Context, q string, offset int, size int) (Entries, error)
@ -29,79 +18,22 @@ type search struct {
artistRepo domain.ArtistRepository
albumRepo domain.AlbumRepository
mfileRepo domain.MediaFileRepository
idxArtist gomate.Indexer
idxAlbum gomate.Indexer
idxSong gomate.Indexer
sArtist gomate.Searcher
sAlbum gomate.Searcher
sSong gomate.Searcher
}
func NewSearch(ar domain.ArtistRepository, alr domain.AlbumRepository, mr domain.MediaFileRepository, db gomate.DB) Search {
func NewSearch(ar domain.ArtistRepository, alr domain.AlbumRepository, mr domain.MediaFileRepository) Search {
s := &search{artistRepo: ar, albumRepo: alr, mfileRepo: mr}
s.idxArtist = gomate.NewIndexer(db, "gomate-artist-idx")
s.sArtist = gomate.NewSearcher(db, "gomate-artist-idx")
s.idxAlbum = gomate.NewIndexer(db, "gomate-album-idx")
s.sAlbum = gomate.NewSearcher(db, "gomate-album-idx")
s.idxSong = gomate.NewIndexer(db, "gomate-song-idx")
s.sSong = gomate.NewSearcher(db, "gomate-song-idx")
return s
}
func (s *search) ClearAll() error {
if err := s.idxArtist.Clear(); err != nil {
return err
}
if err := s.idxAlbum.Clear(); err != nil {
return err
}
if err := s.idxSong.Clear(); err != nil {
return err
}
return nil
}
func (s *search) IndexArtist(ar *domain.Artist) error {
return s.idxArtist.Index(ar.ID, sanitize.Accents(strings.ToLower(ar.Name)))
}
func (s *search) IndexAlbum(al *domain.Album) error {
return s.idxAlbum.Index(al.ID, sanitize.Accents(strings.ToLower(al.Name)))
}
func (s *search) IndexMediaFile(mf *domain.MediaFile) error {
return s.idxSong.Index(mf.ID, sanitize.Accents(strings.ToLower(mf.Title)))
}
func (s *search) RemoveArtist(ids ...string) error {
return s.idxArtist.Remove(ids...)
}
func (s *search) RemoveAlbum(ids ...string) error {
return s.idxAlbum.Remove(ids...)
}
func (s *search) RemoveMediaFile(ids ...string) error {
return s.idxSong.Remove(ids...)
}
func (s *search) SearchArtist(ctx context.Context, q string, offset int, size int) (Entries, error) {
q = sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*")))
min := offset
max := min + size - 1
resp, err := s.sArtist.Search(q, min, max)
resp, err := s.artistRepo.Search(q, offset, size)
if err != nil {
return nil, nil
}
res := make(Entries, 0, len(resp))
for _, id := range resp {
a, err := s.artistRepo.Get(id)
if criticalError(ctx, "Artist", id, err) {
return nil, err
}
if err == nil {
res = append(res, FromArtist(a))
}
for _, ar := range resp {
res = append(res, FromArtist(&ar))
}
return res, nil
}
@ -114,12 +46,7 @@ func (s *search) SearchAlbum(ctx context.Context, q string, offset int, size int
}
res := make(Entries, 0, len(resp))
for _, al := range resp {
if criticalError(ctx, "Album", al.ID, err) {
return nil, err
}
if err == nil {
res = append(res, FromAlbum(&al))
}
res = append(res, FromAlbum(&al))
}
return res, nil
}
@ -132,22 +59,7 @@ func (s *search) SearchSong(ctx context.Context, q string, offset int, size int)
}
res := make(Entries, 0, len(resp))
for _, mf := range resp {
if criticalError(ctx, "Song", mf.ID, err) {
return nil, err
}
if err == nil {
res = append(res, FromMediaFile(&mf))
}
res = append(res, FromMediaFile(&mf))
}
return res, nil
}
func criticalError(ctx context.Context, kind, id string, err error) bool {
switch {
case err != nil:
return true
case err == domain.ErrNotFound:
log.Warn(ctx, kind+"ID not in DB. Need a reindex?", "id", id)
}
return false
}

6
go.mod
View file

@ -4,13 +4,9 @@ go 1.13
require (
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/DataDog/zstd v1.4.4 // indirect
github.com/Masterminds/squirrel v1.1.0
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 // indirect
github.com/asdine/storm v2.1.2+incompatible
github.com/astaxie/beego v1.12.0
github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
github.com/deluan/gomate v0.0.0-20160327212459-3eb40643dd6f
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131
github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a // indirect
github.com/dhowden/tag v0.0.0-20170128231422-9edd38ca5d10
@ -33,8 +29,6 @@ require (
github.com/smartystreets/goconvey v1.6.4
github.com/stretchr/testify v1.4.0 // indirect
github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c
github.com/vmihailenco/msgpack v4.0.1+incompatible // indirect
go.etcd.io/bbolt v1.3.3 // indirect
golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect

12
go.sum
View file

@ -1,15 +1,9 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE=
github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/squirrel v1.1.0 h1:baP1qLdoQCeTw3ifCdOq2dkYc6vGcmRdaociKLbEJXs=
github.com/Masterminds/squirrel v1.1.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA=
github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM=
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM=
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
github.com/asdine/storm v2.1.2+incompatible h1:dczuIkyqwY2LrtXPz8ixMrU/OFgZp71kbKTHGrXYt/Q=
github.com/asdine/storm v2.1.2+incompatible/go.mod h1:RarYDc9hq1UPLImuiXK3BIWPJLdIygvV3PsInK0FbVQ=
github.com/astaxie/beego v1.12.0 h1:MRhVoeeye5N+Flul5PoVfD9CslfdoH+xqC/xvSQ5u2Y=
github.com/astaxie/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o=
github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
@ -28,8 +22,6 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deluan/gomate v0.0.0-20160327212459-3eb40643dd6f h1:jZxJHFEzOavX4cM1BacQGZAMmhgHERXD7Qxyi2NLYtE=
github.com/deluan/gomate v0.0.0-20160327212459-3eb40643dd6f/go.mod h1:10VOt8RwQ8an9cSC2r77s1jqTucTHZSGN2wz46v+7ZM=
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131 h1:siEGb+iB1Ea75U7BnkYVSqSRzE6QHlXCbqEXenxRmhQ=
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131/go.mod h1:eVWQJVQ67aMvYhpkDwaH2Goy2vo6v8JCMfGXfQ9sPtw=
github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a h1:7MucP9rMAsQRcRE1sGpvMZoTxFYZlDmfDvCH+z7H+90=
@ -125,11 +117,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c h1:3eGShk3EQf5gJCYW+WzA0TEJQd37HLOmlYF7N0YJwv0=
github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/vmihailenco/msgpack v4.0.1+incompatible h1:RMF1enSPeKTlXrXdOcqjFUElywVZjjC6pqse21bKbEU=
github.com/vmihailenco/msgpack v4.0.1+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

3
persistence/db_ledis/artist_repository.go
View file

@ -34,5 +34,8 @@ func (r *artistRepository) PurgeInactive(active domain.Artists) ([]string, error
return e.(domain.Artist).ID
})
}
func (r *artistRepository) Search(q string, offset int, size int) (domain.Artists, error) {
return nil, errors.New("not implemented")
}
var _ domain.ArtistRepository = (*artistRepository)(nil)

28
persistence/db_sql/artist_repository.go
View file

@ -25,7 +25,11 @@ func NewArtistRepository() domain.ArtistRepository {
func (r *artistRepository) Put(a *domain.Artist) error {
ta := Artist(*a)
return WithTx(func(o orm.Ormer) error {
return r.put(o, a.ID, &ta)
err := r.put(o, a.ID, &ta)
if err != nil {
return err
}
return r.searcher.Index(o, r.tableName, a.ID, a.Name)
})
}
@ -48,5 +52,27 @@ func (r *artistRepository) PurgeInactive(activeList domain.Artists) ([]string, e
})
}
func (r *artistRepository) Search(q string, offset int, size int) (domain.Artists, error) {
if len(q) <= 2 {
return nil, nil
}
var results []Artist
err := r.searcher.Search(r.tableName, q, offset, size, &results, "name")
if err != nil {
return nil, err
}
return r.toArtists(results), nil
}
func (r *artistRepository) toArtists(all []Artist) domain.Artists {
result := make(domain.Artists, len(all))
for i, a := range all {
result[i] = domain.Artist(a)
}
return result
}
var _ domain.ArtistRepository = (*artistRepository)(nil)
var _ = domain.Artist(Artist{})

26
scanner/importer.go
View file

@ -9,7 +9,6 @@ import (
"github.com/cloudsonic/sonic-server/conf"
"github.com/cloudsonic/sonic-server/domain"
"github.com/cloudsonic/sonic-server/engine"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/utils"
)
@ -33,12 +32,11 @@ type Importer struct {
idxRepo domain.ArtistIndexRepository
plsRepo domain.PlaylistRepository
propertyRepo domain.PropertyRepository
search engine.Search
lastScan time.Time
lastCheck time.Time
}
func NewImporter(mediaFolder string, scanner Scanner, mfRepo domain.MediaFileRepository, albumRepo domain.AlbumRepository, artistRepo domain.ArtistRepository, idxRepo domain.ArtistIndexRepository, plsRepo domain.PlaylistRepository, propertyRepo domain.PropertyRepository, search engine.Search) *Importer {
func NewImporter(mediaFolder string, scanner Scanner, mfRepo domain.MediaFileRepository, albumRepo domain.AlbumRepository, artistRepo domain.ArtistRepository, idxRepo domain.ArtistIndexRepository, plsRepo domain.PlaylistRepository, propertyRepo domain.PropertyRepository) *Importer {
return &Importer{
scanner: scanner,
mediaFolder: mediaFolder,
@ -48,7 +46,6 @@ func NewImporter(mediaFolder string, scanner Scanner, mfRepo domain.MediaFileRep
idxRepo: idxRepo,
plsRepo: plsRepo,
propertyRepo: propertyRepo,
search: search,
}
}
@ -137,20 +134,14 @@ func (i *Importer) importLibrary() (err error) {
i.importArtistIndex()
log.Debug("Purging old data")
if deleted, err := i.mfRepo.PurgeInactive(mfs); err != nil {
if _, err := i.mfRepo.PurgeInactive(mfs); err != nil {
log.Error(err)
} else {
i.search.RemoveMediaFile(deleted...)
}
if deleted, err := i.albumRepo.PurgeInactive(als); err != nil {
if _, err := i.albumRepo.PurgeInactive(als); err != nil {
log.Error(err)
} else {
i.search.RemoveAlbum(deleted...)
}
if deleted, err := i.artistRepo.PurgeInactive(ars); err != nil {
if _, err := i.artistRepo.PurgeInactive(ars); err != nil {
log.Error("Deleting inactive artists", err)
} else {
i.search.RemoveArtist(deleted...)
}
if _, err := i.plsRepo.PurgeInactive(pls); err != nil {
log.Error(err)
@ -198,9 +189,6 @@ func (i *Importer) importMediaFiles() (domain.MediaFiles, int) {
if err := i.mfRepo.Put(mf); err != nil {
log.Error(err)
}
if err := i.search.IndexMediaFile(mf); err != nil {
log.Error("Error indexing artist", err)
}
updates++
if !i.lastScan.IsZero() {
log.Debug(fmt.Sprintf(`-- Updated Track: "%s"`, mf.Title))
@ -230,9 +218,6 @@ func (i *Importer) importAlbums() (domain.Albums, int) {
if err := i.albumRepo.Put(al); err != nil {
log.Error(err)
}
if err := i.search.IndexAlbum(al); err != nil {
log.Error("Error indexing artist", err)
}
updates++
if !i.lastScan.IsZero() {
log.Debug(fmt.Sprintf(`-- Updated Album: "%s" from "%s"`, al.Name, al.Artist))
@ -250,9 +235,6 @@ func (i *Importer) importArtists() domain.Artists {
if err := i.artistRepo.Put(ar); err != nil {
log.Error(err)
}
if err := i.search.IndexArtist(ar); err != nil {
log.Error("Error indexing artist", err)
}
}
return ars
}

15
wire_gen.go
View file

@ -14,8 +14,6 @@ import (
"github.com/cloudsonic/sonic-server/persistence/db_ledis"
"github.com/cloudsonic/sonic-server/persistence/db_sql"
"github.com/cloudsonic/sonic-server/scanner"
"github.com/deluan/gomate"
"github.com/deluan/gomate/ledis"
"github.com/google/wire"
)
@ -31,9 +29,7 @@ func CreateApp(musicFolder string, p persistence.ProviderIdentifier) *App {
artistIndexRepository := provider.ArtistIndexRepository
playlistRepository := provider.PlaylistRepository
propertyRepository := provider.PropertyRepository
db := newDB()
search := engine.NewSearch(artistRepository, albumRepository, mediaFileRepository, db)
importer := scanner.NewImporter(musicFolder, itunesScanner, mediaFileRepository, albumRepository, artistRepository, artistIndexRepository, playlistRepository, propertyRepository, search)
importer := scanner.NewImporter(musicFolder, itunesScanner, mediaFileRepository, albumRepository, artistRepository, artistIndexRepository, playlistRepository, propertyRepository)
app := NewApp(importer)
return app
}
@ -55,8 +51,7 @@ func CreateSubsonicAPIRouter(p persistence.ProviderIdentifier) *api.Router {
playlists := engine.NewPlaylists(itunesControl, playlistRepository, mediaFileRepository)
ratings := engine.NewRatings(itunesControl, mediaFileRepository, albumRepository, artistRepository)
scrobbler := engine.NewScrobbler(itunesControl, mediaFileRepository, nowPlayingRepository)
db := newDB()
search := engine.NewSearch(artistRepository, albumRepository, mediaFileRepository, db)
search := engine.NewSearch(artistRepository, albumRepository, mediaFileRepository)
router := api.NewRouter(browser, cover, listGenerator, playlists, ratings, scrobbler, search)
return router
}
@ -123,7 +118,7 @@ type Provider struct {
PropertyRepository domain.PropertyRepository
}
var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner.Set, newDB, api.NewRouter, wire.FieldsOf(new(*Provider), "AlbumRepository", "ArtistRepository", "CheckSumRepository",
var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner.Set, api.NewRouter, wire.FieldsOf(new(*Provider), "AlbumRepository", "ArtistRepository", "CheckSumRepository",
"ArtistIndexRepository", "MediaFileRepository", "MediaFolderRepository", "NowPlayingRepository",
"PlaylistRepository", "PropertyRepository"), createPersistenceProvider,
)
@ -136,7 +131,3 @@ func createPersistenceProvider(provider persistence.ProviderIdentifier) *Provide
return createLedisDBProvider()
}
}
func newDB() gomate.DB {
return ledis.NewEmbeddedDB(db_ledis.Db())
}

7
wire_injectors.go
View file

@ -11,8 +11,6 @@ import (
"github.com/cloudsonic/sonic-server/persistence/db_ledis"
"github.com/cloudsonic/sonic-server/persistence/db_sql"
"github.com/cloudsonic/sonic-server/scanner"
"github.com/deluan/gomate"
"github.com/deluan/gomate/ledis"
"github.com/google/wire"
)
@ -32,7 +30,6 @@ var allProviders = wire.NewSet(
itunesbridge.NewItunesControl,
engine.Set,
scanner.Set,
newDB,
api.NewRouter,
wire.FieldsOf(new(*Provider), "AlbumRepository", "ArtistRepository", "CheckSumRepository",
"ArtistIndexRepository", "MediaFileRepository", "MediaFolderRepository", "NowPlayingRepository",
@ -73,7 +70,3 @@ func createLedisDBProvider() *Provider {
wire.Struct(new(Provider), "*"),
))
}
func newDB() gomate.DB {
return ledis.NewEmbeddedDB(db_ledis.Db())
}