Fine tune search functionality

db/migration/20200419222708_reindex_to_change_full_text_search.go

@@ -0,0 +1,19 @@
+package migration
+
+import (
+	"database/sql"
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20200419222708, Down20200419222708)
+}
+
+func Up20200419222708(tx *sql.Tx) error {
+	notice(tx, "A full rescan will be performed to change the search behaviour")
+	return forceFullRescan(tx)
+}
+
+func Down20200419222708(tx *sql.Tx) error {
+	return nil
+}

persistence/album_repository.go

@@ -148,7 +148,7 @@ func (r *albumRepository) Refresh(ids ...string) error {
 			toInsert++
 			al.CreatedAt = time.Now()
 		}
-		al.FullText = r.getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists)
+		al.FullText = getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists)
 		_, err := r.put(al.ID, al.Album)
 		if err != nil {
 			return err

persistence/album_repository_test.go

@@ -20,7 +20,7 @@ var _ = Describe("AlbumRepository", func() {
 
 	Describe("Get", func() {
 		It("returns an existent album", func() {
-			Expect(repo.Get("3")).To(Equal(&albumRadioactivity))
+			Expect(repo.Get("103")).To(Equal(&albumRadioactivity))
 		})
 		It("returns ErrNotFound when the album does not exist", func() {
 			_, err := repo.Get("666")

persistence/artist_repository.go

@@ -56,7 +56,7 @@ func (r *artistRepository) getIndexKey(a *model.Artist) string {
 }
 
 func (r *artistRepository) Put(a *model.Artist) error {
-	a.FullText = r.getFullText(a.Name)
+	a.FullText = getFullText(a.Name)
 	_, err := r.put(a.ID, a)
 	return err
 }

persistence/mediafile_repository.go

@@ -41,7 +41,7 @@ func (r mediaFileRepository) Exists(id string) (bool, error) {
 }
 
 func (r mediaFileRepository) Put(m *model.MediaFile) error {
-	m.FullText = r.getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist)
+	m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist)
 	_, err := r.put(m.ID, m)
 	return err
 }

persistence/mediafile_repository_test.go

@@ -21,7 +21,7 @@ var _ = Describe("MediaRepository", func() {
 	})
 
 	It("gets mediafile from the DB", func() {
-		Expect(mr.Get("4")).To(Equal(&songAntenna))
+		Expect(mr.Get("1004")).To(Equal(&songAntenna))
 	})
 
 	It("returns ErrNotFound", func() {
@@ -39,7 +39,7 @@ var _ = Describe("MediaRepository", func() {
 	})
 
 	It("find mediafiles by album", func() {
-		Expect(mr.FindByAlbum("3")).To(Equal(model.MediaFiles{
+		Expect(mr.FindByAlbum("103")).To(Equal(model.MediaFiles{
 			songRadioactivity,
 			songAntenna,
 		}))

persistence/persistence_suite_test.go

@@ -31,8 +31,8 @@ func TestPersistence(t *testing.T) {
 }
 
 var (
-	artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: "kraftwerk"}
+	artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: " kraftwerk"}
-	artistBeatles   = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: "beatles the"}
+	artistBeatles   = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: " beatles the"}
 	testArtists     = model.Artists{
 		artistKraftwerk,
 		artistBeatles,
@@ -40,9 +40,9 @@ var (
 )
 
 var (
-	albumSgtPeppers    = model.Album{ID: "1", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: "beatles peppers sgt the"}
+	albumSgtPeppers    = model.Album{ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the"}
-	albumAbbeyRoad     = model.Album{ID: "2", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: "abbey beatles road the"}
+	albumAbbeyRoad     = model.Album{ID: "102", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the"}
-	albumRadioactivity = model.Album{ID: "3", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: "kraftwerk radioactivity"}
+	albumRadioactivity = model.Album{ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity"}
 	testAlbums         = model.Albums{
 		albumSgtPeppers,
 		albumAbbeyRoad,
@@ -51,10 +51,10 @@ var (
 )
 
 var (
-	songDayInALife    = model.MediaFile{ID: "1", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "1", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: "a beatles day in life peppers sgt the"}
+	songDayInALife    = model.MediaFile{ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"}
-	songComeTogether  = model.MediaFile{ID: "2", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "2", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: "abbey beatles come road the together"}
+	songComeTogether  = model.MediaFile{ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"}
-	songRadioactivity = model.MediaFile{ID: "3", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: "kraftwerk radioactivity"}
+	songRadioactivity = model.MediaFile{ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"}
-	songAntenna       = model.MediaFile{ID: "4", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: "antenna kraftwerk"}
+	songAntenna       = model.MediaFile{ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: " antenna kraftwerk"}
 	testSongs         = model.MediaFiles{
 		songDayInALife,
 		songComeTogether,
@@ -70,9 +70,9 @@ var (
 		Comment: "No Comments",
 		Owner:   "userid",
 		Public:  true,
-		Tracks:  model.MediaFiles{{ID: "1"}, {ID: "3"}},
+		Tracks:  model.MediaFiles{{ID: "1001"}, {ID: "1003"}},
 	}
-	plsCool       = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "4"}}}
+	plsCool       = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "1004"}}}
 	testPlaylists = model.Playlists{plsBest, plsCool}
 )
 

persistence/playlist_repository_test.go

@@ -63,19 +63,19 @@ var _ = Describe("PlaylistRepository", func() {
 	Describe("Put/Exists/Delete", func() {
 		var newPls model.Playlist
 		BeforeEach(func() {
-			newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "4"}, {ID: "3"}}}
+			newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "1004"}, {ID: "1003"}}}
 		})
 		It("saves the playlist to the DB", func() {
 			Expect(repo.Put(&newPls)).To(BeNil())
 		})
 		It("adds repeated songs to a playlist and keeps the order", func() {
-			newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "4"})
+			newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "1004"})
 			Expect(repo.Put(&newPls)).To(BeNil())
 			saved, _ := repo.Get("22")
 			Expect(saved.Tracks).To(HaveLen(3))
-			Expect(saved.Tracks[0].ID).To(Equal("4"))
+			Expect(saved.Tracks[0].ID).To(Equal("1004"))
-			Expect(saved.Tracks[1].ID).To(Equal("3"))
+			Expect(saved.Tracks[1].ID).To(Equal("1003"))
-			Expect(saved.Tracks[2].ID).To(Equal("4"))
+			Expect(saved.Tracks[2].ID).To(Equal("1004"))
 		})
 		It("returns the newly created playlist", func() {
 			Expect(repo.Exists("22")).To(BeTrue())

persistence/sql_restful.go

@@ -7,7 +7,6 @@ import (
 	. "github.com/Masterminds/squirrel"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/rest"
-	"github.com/kennygrant/sanitize"
 )
 
 type filterFunc = func(field string, value interface{}) Sqlizer
@@ -59,15 +58,11 @@ func booleanFilter(field string, value interface{}) Sqlizer {
 }
 
 func fullTextFilter(field string, value interface{}) Sqlizer {
-	q := value.(string)
+	q := sanitizeStrings(value.(string))
-	q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))
 	parts := strings.Split(q, " ")
 	filters := And{}
 	for _, part := range parts {
-		filters = append(filters, Or{
+		filters = append(filters, Like{"full_text": "% " + part + "%"})
-			Like{"full_text": part + "%"},
-			Like{"full_text": "%" + part + "%"},
-		})
 	}
 	return filters
 }

persistence/sql_search.go

@@ -1,6 +1,7 @@
 package persistence
 
 import (
+	"regexp"
 	"sort"
 	"strings"
 
@@ -8,7 +9,14 @@ import (
 	"github.com/kennygrant/sanitize"
 )
 
-func (r sqlRepository) getFullText(text ...string) string {
+var quotesRegex = regexp.MustCompile("[“”‘’'\"]")
+
+func getFullText(text ...string) string {
+	fullText := sanitizeStrings(text...)
+	return " " + fullText
+}
+
+func sanitizeStrings(text ...string) string {
 	sanitizedText := strings.Builder{}
 	for _, txt := range text {
 		sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ")
@@ -19,14 +27,18 @@ func (r sqlRepository) getFullText(text ...string) string {
 	}
 	var fullText []string
 	for w := range words {
-		fullText = append(fullText, w)
+		w = quotesRegex.ReplaceAllString(w, "")
+		if w != "" {
+			fullText = append(fullText, w)
+		}
 	}
 	sort.Strings(fullText)
 	return strings.Join(fullText, " ")
 }
 
 func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, orderBys ...string) error {
-	q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))
+	q = strings.TrimSuffix(q, "*")
+	q = sanitizeStrings(q)
 	if len(q) < 2 {
 		return nil
 	}
@@ -37,10 +49,7 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{},
 	}
 	parts := strings.Split(q, " ")
 	for _, part := range parts {
-		sq = sq.Where(Or{
+		sq = sq.Where(Like{"full_text": "% " + part + "%"})
-			Like{"full_text": part + "%"},
-			Like{"full_text": "%" + part + "%"},
-		})
 	}
 	err := r.queryAll(sq, results)
 	return err

persistence/sql_search_test.go

@@ -6,23 +6,25 @@ import (
 )
 
 var _ = Describe("sqlRepository", func() {
-	var sqlRepository = &sqlRepository{}
-
 	Describe("getFullText", func() {
 		It("returns all lowercase chars", func() {
-			Expect(sqlRepository.getFullText("Some Text")).To(Equal("some text"))
+			Expect(getFullText("Some Text")).To(Equal(" some text"))
 		})
 
 		It("removes accents", func() {
-			Expect(sqlRepository.getFullText("Quintão")).To(Equal("quintao"))
+			Expect(getFullText("Quintão")).To(Equal(" quintao"))
 		})
 
 		It("remove extra spaces", func() {
-			Expect(sqlRepository.getFullText(" some  text  ")).To(Equal("some text"))
+			Expect(getFullText(" some  text  ")).To(Equal(" some text"))
 		})
 
 		It("remove duplicated words", func() {
-			Expect(sqlRepository.getFullText("legião urbana urbana legiÃo")).To(Equal("legiao urbana"))
+			Expect(getFullText("legião urbana urbana legiÃo")).To(Equal(" legiao urbana"))
+		})
+
+		It("remove symbols", func() {
+			Expect(getFullText("Tom’s Diner ' “40” ‘A’")).To(Equal(" 40 a diner toms"))
 		})
 	})
 })