mirrors/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2025-04-03 20:47:35 +03:00
Code Issues Projects Releases Packages Wiki Activity

Fine tune search functionality

Browse source
This commit is contained in:
Deluan 2020-04-19 22:40:24 -04:00
parent db02f5f07f
commit 80c8d85cb9
11 changed files with 67 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

19
db/migration/20200419222708_reindex_to_change_full_text_search.go Normal file
View file

@ -0,0 +1,19 @@
package migration
import (
"database/sql"
"github.com/pressly/goose"
)
func init() {
goose.AddMigration(Up20200419222708, Down20200419222708)
}
func Up20200419222708(tx *sql.Tx) error {
notice(tx, "A full rescan will be performed to change the search behaviour")
return forceFullRescan(tx)
}
func Down20200419222708(tx *sql.Tx) error {
return nil
}

2
persistence/album_repository.go
View file

@ -148,7 +148,7 @@ func (r *albumRepository) Refresh(ids ...string) error {
toInsert++
al.CreatedAt = time.Now()
}
al.FullText = r.getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists)
al.FullText = getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists)
_, err := r.put(al.ID, al.Album)
if err != nil {
return err

2
persistence/album_repository_test.go
View file

@ -20,7 +20,7 @@ var _ = Describe("AlbumRepository", func() {
Describe("Get", func() {
It("returns an existent album", func() {
Expect(repo.Get("3")).To(Equal(&albumRadioactivity))
Expect(repo.Get("103")).To(Equal(&albumRadioactivity))
})
It("returns ErrNotFound when the album does not exist", func() {
_, err := repo.Get("666")

2
persistence/artist_repository.go
View file

@ -56,7 +56,7 @@ func (r *artistRepository) getIndexKey(a *model.Artist) string {
}
func (r *artistRepository) Put(a *model.Artist) error {
a.FullText = r.getFullText(a.Name)
a.FullText = getFullText(a.Name)
_, err := r.put(a.ID, a)
return err
}

2
persistence/mediafile_repository.go
View file

@ -41,7 +41,7 @@ func (r mediaFileRepository) Exists(id string) (bool, error) {
}
func (r mediaFileRepository) Put(m *model.MediaFile) error {
m.FullText = r.getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist)
m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist)
_, err := r.put(m.ID, m)
return err
}

4
persistence/mediafile_repository_test.go
View file

@ -21,7 +21,7 @@ var _ = Describe("MediaRepository", func() {
})
It("gets mediafile from the DB", func() {
Expect(mr.Get("4")).To(Equal(&songAntenna))
Expect(mr.Get("1004")).To(Equal(&songAntenna))
})
It("returns ErrNotFound", func() {
@ -39,7 +39,7 @@ var _ = Describe("MediaRepository", func() {
})
It("find mediafiles by album", func() {
Expect(mr.FindByAlbum("3")).To(Equal(model.MediaFiles{
Expect(mr.FindByAlbum("103")).To(Equal(model.MediaFiles{
songRadioactivity,
songAntenna,
}))

22
persistence/persistence_suite_test.go
View file

@ -31,8 +31,8 @@ func TestPersistence(t *testing.T) {
}
var (
artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: "kraftwerk"}
artistBeatles = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: "beatles the"}
artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: " kraftwerk"}
artistBeatles = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: " beatles the"}
testArtists = model.Artists{
artistKraftwerk,
artistBeatles,
@ -40,9 +40,9 @@ var (
)
var (
albumSgtPeppers = model.Album{ID: "1", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: "beatles peppers sgt the"}
albumAbbeyRoad = model.Album{ID: "2", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: "abbey beatles road the"}
albumRadioactivity = model.Album{ID: "3", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: "kraftwerk radioactivity"}
albumSgtPeppers = model.Album{ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the"}
albumAbbeyRoad = model.Album{ID: "102", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the"}
albumRadioactivity = model.Album{ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity"}
testAlbums = model.Albums{
albumSgtPeppers,
albumAbbeyRoad,
@ -51,10 +51,10 @@ var (
)
var (
songDayInALife = model.MediaFile{ID: "1", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "1", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: "a beatles day in life peppers sgt the"}
songComeTogether = model.MediaFile{ID: "2", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "2", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: "abbey beatles come road the together"}
songRadioactivity = model.MediaFile{ID: "3", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: "kraftwerk radioactivity"}
songAntenna = model.MediaFile{ID: "4", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: "antenna kraftwerk"}
songDayInALife = model.MediaFile{ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"}
songComeTogether = model.MediaFile{ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"}
songRadioactivity = model.MediaFile{ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"}
songAntenna = model.MediaFile{ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: " antenna kraftwerk"}
testSongs = model.MediaFiles{
songDayInALife,
songComeTogether,
@ -70,9 +70,9 @@ var (
Comment: "No Comments",
Owner: "userid",
Public: true,
Tracks: model.MediaFiles{{ID: "1"}, {ID: "3"}},
Tracks: model.MediaFiles{{ID: "1001"}, {ID: "1003"}},
}
plsCool = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "4"}}}
plsCool = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "1004"}}}
testPlaylists = model.Playlists{plsBest, plsCool}
)

10
persistence/playlist_repository_test.go
View file

@ -63,19 +63,19 @@ var _ = Describe("PlaylistRepository", func() {
Describe("Put/Exists/Delete", func() {
var newPls model.Playlist
BeforeEach(func() {
newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "4"}, {ID: "3"}}}
newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "1004"}, {ID: "1003"}}}
})
It("saves the playlist to the DB", func() {
Expect(repo.Put(&newPls)).To(BeNil())
})
It("adds repeated songs to a playlist and keeps the order", func() {
newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "4"})
newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "1004"})
Expect(repo.Put(&newPls)).To(BeNil())
saved, _ := repo.Get("22")
Expect(saved.Tracks).To(HaveLen(3))
Expect(saved.Tracks[0].ID).To(Equal("4"))
Expect(saved.Tracks[1].ID).To(Equal("3"))
Expect(saved.Tracks[2].ID).To(Equal("4"))
Expect(saved.Tracks[0].ID).To(Equal("1004"))
Expect(saved.Tracks[1].ID).To(Equal("1003"))
Expect(saved.Tracks[2].ID).To(Equal("1004"))
})
It("returns the newly created playlist", func() {
Expect(repo.Exists("22")).To(BeTrue())

9
persistence/sql_restful.go
View file

@ -7,7 +7,6 @@ import (
. "github.com/Masterminds/squirrel"
"github.com/deluan/navidrome/model"
"github.com/deluan/rest"
"github.com/kennygrant/sanitize"
)
type filterFunc = func(field string, value interface{}) Sqlizer
@ -59,15 +58,11 @@ func booleanFilter(field string, value interface{}) Sqlizer {
}
func fullTextFilter(field string, value interface{}) Sqlizer {
q := value.(string)
q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))
q := sanitizeStrings(value.(string))
parts := strings.Split(q, " ")
filters := And{}
for _, part := range parts {
filters = append(filters, Or{
Like{"full_text": part + "%"},
Like{"full_text": "%" + part + "%"},
})
filters = append(filters, Like{"full_text": "% " + part + "%"})
}
return filters
}

23
persistence/sql_search.go
View file

@ -1,6 +1,7 @@
package persistence
import (
"regexp"
"sort"
"strings"
@ -8,7 +9,14 @@ import (
"github.com/kennygrant/sanitize"
)
func (r sqlRepository) getFullText(text ...string) string {
var quotesRegex = regexp.MustCompile("[“”‘’'\"]")
func getFullText(text ...string) string {
fullText := sanitizeStrings(text...)
return " " + fullText
}
func sanitizeStrings(text ...string) string {
sanitizedText := strings.Builder{}
for _, txt := range text {
sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ")
@ -19,14 +27,18 @@ func (r sqlRepository) getFullText(text ...string) string {
}
var fullText []string
for w := range words {
fullText = append(fullText, w)
w = quotesRegex.ReplaceAllString(w, "")
if w != "" {
fullText = append(fullText, w)
}
}
sort.Strings(fullText)
return strings.Join(fullText, " ")
}
func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, orderBys ...string) error {
q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))
q = strings.TrimSuffix(q, "*")
q = sanitizeStrings(q)
if len(q) < 2 {
return nil
}
@ -37,10 +49,7 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{},
}
parts := strings.Split(q, " ")
for _, part := range parts {
sq = sq.Where(Or{
Like{"full_text": part + "%"},
Like{"full_text": "%" + part + "%"},
})
sq = sq.Where(Like{"full_text": "% " + part + "%"})
}
err := r.queryAll(sq, results)
return err

14
persistence/sql_search_test.go
View file

@ -6,23 +6,25 @@ import (
)
var _ = Describe("sqlRepository", func() {
var sqlRepository = &sqlRepository{}
Describe("getFullText", func() {
It("returns all lowercase chars", func() {
Expect(sqlRepository.getFullText("Some Text")).To(Equal("some text"))
Expect(getFullText("Some Text")).To(Equal(" some text"))
})
It("removes accents", func() {
Expect(sqlRepository.getFullText("Quintão")).To(Equal("quintao"))
Expect(getFullText("Quintão")).To(Equal(" quintao"))
})
It("remove extra spaces", func() {
Expect(sqlRepository.getFullText(" some text ")).To(Equal("some text"))
Expect(getFullText(" some text ")).To(Equal(" some text"))
})
It("remove duplicated words", func() {
Expect(sqlRepository.getFullText("legião urbana urbana legiÃo")).To(Equal("legiao urbana"))
Expect(getFullText("legião urbana urbana legiÃo")).To(Equal(" legiao urbana"))
})
It("remove symbols", func() {
Expect(getFullText("Tom’s Diner ' “40” ‘A’")).To(Equal(" 40 a diner toms"))
})
})
})