From 7cd3a8ba67bf7c3914cd712f49fbbfce9add4272 Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 15 Jul 2021 19:53:40 -0400 Subject: [PATCH] Add genre tables, read multiple-genres from tags --- conf/configuration.go | 5 +- .../20210715151153_add_genre_tables.go | 64 ++++++++++++++ model/genres.go | 2 + persistence/genre_repository.go | 41 ++++++++- scanner/mapping.go | 4 +- scanner/metadata/ffmpeg.go | 2 +- scanner/metadata/ffmpeg_test.go | 4 +- scanner/metadata/metadata.go | 84 +++++++++++-------- scanner/metadata/metadata_test.go | 13 +++ scanner/metadata/taglib.go | 2 +- scanner/metadata/taglib_test.go | 2 +- server/subsonic/helpers.go | 6 +- utils/strings.go | 29 +++++-- 13 files changed, 205 insertions(+), 53 deletions(-) create mode 100644 db/migration/20210715151153_add_genre_tables.go diff --git a/conf/configuration.go b/conf/configuration.go index 7b0fe833b..0b8b7bb7c 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -74,7 +74,8 @@ type configOptions struct { } type scannerOptions struct { - Extractor string + Extractor string + GenreSeparators string } type lastfmOptions struct { @@ -214,6 +215,8 @@ func init() { viper.SetDefault("reverseproxywhitelist", "") viper.SetDefault("scanner.extractor", "taglib") + viper.SetDefault("scanner.genreseparators", ";/") + viper.SetDefault("agents", "lastfm,spotify") viper.SetDefault("lastfm.enabled", true) viper.SetDefault("lastfm.language", "en") diff --git a/db/migration/20210715151153_add_genre_tables.go b/db/migration/20210715151153_add_genre_tables.go new file mode 100644 index 000000000..0ce20bdd3 --- /dev/null +++ b/db/migration/20210715151153_add_genre_tables.go @@ -0,0 +1,64 @@ +package migrations + +import ( + "database/sql" + + "github.com/pressly/goose" +) + +func init() { + goose.AddMigration(upAddGenreTables, downAddGenreTables) +} + +func upAddGenreTables(tx *sql.Tx) error { + _, err := tx.Exec(` +create table if not exists genre +( + id varchar not null primary key, + name varchar not null, + constraint genre_name_ux + unique (name) +); + +create table if not exists album_genres +( + album_id varchar default null not null + references album + on delete cascade, + genre_id varchar default null not null + references genre + on delete cascade, + constraint album_genre_ux + unique (album_id, genre_id) +); + +create table if not exists media_file_genres +( + media_file_id varchar default null not null + references media_file + on delete cascade, + genre_id varchar default null not null + references genre + on delete cascade, + constraint media_file_genre_ux + unique (media_file_id, genre_id) +); + +create table if not exists artist_genres +( + artist_id varchar default null not null + references artist + on delete cascade, + genre_id varchar default null not null + references genre + on delete cascade, + constraint artist_genre_ux + unique (artist_id, genre_id) +); +`) + return err +} + +func downAddGenreTables(tx *sql.Tx) error { + return nil +} diff --git a/model/genres.go b/model/genres.go index 44f153ac8..d8300b31c 100644 --- a/model/genres.go +++ b/model/genres.go @@ -1,6 +1,7 @@ package model type Genre struct { + ID string `json:"id" orm:"column(id)"` Name string SongCount int AlbumCount int @@ -10,4 +11,5 @@ type Genres []Genre type GenreRepository interface { GetAll() (Genres, error) + Put(m *Genre) error } diff --git a/persistence/genre_repository.go b/persistence/genre_repository.go index 312f36d8b..0adb4cdb5 100644 --- a/persistence/genre_repository.go +++ b/persistence/genre_repository.go @@ -3,6 +3,8 @@ package persistence import ( "context" + "github.com/deluan/rest" + . "github.com/Masterminds/squirrel" "github.com/astaxie/beego/orm" "github.com/navidrome/navidrome/model" @@ -10,20 +12,55 @@ import ( type genreRepository struct { sqlRepository + sqlRestful } func NewGenreRepository(ctx context.Context, o orm.Ormer) model.GenreRepository { r := &genreRepository{} r.ctx = ctx r.ormer = o - r.tableName = "media_file" + r.tableName = "genre" return r } -func (r genreRepository) GetAll() (model.Genres, error) { +func (r *genreRepository) GetAll() (model.Genres, error) { sq := Select("genre as name", "count(distinct album_id) as album_count", "count(distinct id) as song_count"). From("media_file").GroupBy("genre") res := model.Genres{} err := r.queryAll(sq, &res) return res, err } + +func (r *genreRepository) Put(m *model.Genre) error { + _, err := r.put(m.ID, m) + return err +} + +func (r *genreRepository) Count(options ...rest.QueryOptions) (int64, error) { + return r.count(Select(), r.parseRestOptions(options...)) +} + +func (r *genreRepository) Read(id string) (interface{}, error) { + sel := r.newSelect().Columns("*").Where(Eq{"id": id}) + var res model.Genre + err := r.queryOne(sel, &res) + return &res, err +} + +func (r *genreRepository) ReadAll(options ...rest.QueryOptions) (interface{}, error) { + sel := r.newSelect(r.parseRestOptions(options...)).Columns("*") + res := model.Genres{} + err := r.queryAll(sel, &res) + return res, err +} + +func (r *genreRepository) EntityName() string { + return r.tableName +} + +func (r *genreRepository) NewInstance() interface{} { + return &model.Genre{} +} + +var _ model.GenreRepository = (*genreRepository)(nil) +var _ model.ResourceRepository = (*genreRepository)(nil) diff --git a/scanner/mapping.go b/scanner/mapping.go index 2cbab955f..860a3328b 100644 --- a/scanner/mapping.go +++ b/scanner/mapping.go @@ -36,7 +36,9 @@ func (s *mediaFileMapper) toMediaFile(md *metadata.Tags) model.MediaFile { mf.Artist = s.mapArtistName(md) mf.AlbumArtistID = s.albumArtistID(md) mf.AlbumArtist = s.mapAlbumArtistName(md) - mf.Genre = md.Genre() + if len(md.Genres()) > 0 { + mf.Genre = md.Genres()[0] + } mf.Compilation = md.Compilation() mf.Year = md.Year() mf.TrackNumber, _ = md.TrackNumber() diff --git a/scanner/metadata/ffmpeg.go b/scanner/metadata/ffmpeg.go index 395c62dbf..603ac10a5 100644 --- a/scanner/metadata/ffmpeg.go +++ b/scanner/metadata/ffmpeg.go @@ -85,7 +85,7 @@ func (e *ffmpegExtractor) extractMetadata(filePath, info string) (*Tags, error) return nil, errors.New("not a media file") } - tags := NewTag(filePath, parsedTags, map[string][]string{ + tags := NewTags(filePath, parsedTags, map[string][]string{ "disc": {"tpa"}, "has_picture": {"metadata_block_picture"}, }) diff --git a/scanner/metadata/ffmpeg_test.go b/scanner/metadata/ffmpeg_test.go index 3bd058622..203fd5425 100644 --- a/scanner/metadata/ffmpeg_test.go +++ b/scanner/metadata/ffmpeg_test.go @@ -23,7 +23,7 @@ var _ = Describe("ffmpegExtractor", func() { Expect(m.Artist()).To(Equal("Artist")) Expect(m.AlbumArtist()).To(Equal("Album Artist")) Expect(m.Compilation()).To(BeTrue()) - Expect(m.Genre()).To(Equal("Rock")) + Expect(m.Genres()).To(Equal("Rock")) Expect(m.Year()).To(Equal(2014)) n, t := m.TrackNumber() Expect(n).To(Equal(2)) @@ -187,7 +187,7 @@ Input #0, flac, from '/Users/deluan/Downloads/06. Back In Black.flac': md, _ := e.extractMetadata("tests/fixtures/test.mp3", output) Expect(md.Title()).To(Equal("Back In Black")) Expect(md.Album()).To(Equal("Back In Black")) - Expect(md.Genre()).To(Equal("Hard Rock")) + Expect(md.Genres()).To(ConsistOf("Hard Rock")) n, t := md.TrackNumber() Expect(n).To(Equal(6)) Expect(t).To(Equal(10)) diff --git a/scanner/metadata/metadata.go b/scanner/metadata/metadata.go index 762553007..bbfd72831 100644 --- a/scanner/metadata/metadata.go +++ b/scanner/metadata/metadata.go @@ -13,6 +13,7 @@ import ( "github.com/google/uuid" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/utils" ) type Extractor interface { @@ -43,7 +44,7 @@ type Tags struct { custom map[string][]string } -func NewTag(filePath string, tags, custom map[string][]string) *Tags { +func NewTags(filePath string, tags, custom map[string][]string) *Tags { fileInfo, err := os.Stat(filePath) if err != nil { log.Warn("Error stating file. Skipping", "filePath", filePath, err) @@ -61,25 +62,29 @@ func NewTag(filePath string, tags, custom map[string][]string) *Tags { // Common tags -func (t *Tags) Title() string { return t.getTag("title", "sort_name", "titlesort") } -func (t *Tags) Album() string { return t.getTag("album", "sort_album", "albumsort") } -func (t *Tags) Artist() string { return t.getTag("artist", "sort_artist", "artistsort") } -func (t *Tags) AlbumArtist() string { return t.getTag("album_artist", "album artist", "albumartist") } +func (t *Tags) Title() string { return t.getFirstTagValue("title", "sort_name", "titlesort") } +func (t *Tags) Album() string { return t.getFirstTagValue("album", "sort_album", "albumsort") } +func (t *Tags) Artist() string { return t.getFirstTagValue("artist", "sort_artist", "artistsort") } +func (t *Tags) AlbumArtist() string { + return t.getFirstTagValue("album_artist", "album artist", "albumartist") +} func (t *Tags) SortTitle() string { return t.getSortTag("", "title", "name") } func (t *Tags) SortAlbum() string { return t.getSortTag("", "album") } func (t *Tags) SortArtist() string { return t.getSortTag("", "artist") } func (t *Tags) SortAlbumArtist() string { return t.getSortTag("tso2", "albumartist", "album_artist") } -func (t *Tags) Genre() string { return t.getTag("genre") } +func (t *Tags) Genres() []string { return t.getAllTagValues("genre") } func (t *Tags) Year() int { return t.getYear("date") } -func (t *Tags) Comment() string { return t.getTag("comment") } -func (t *Tags) Lyrics() string { return t.getTag("lyrics", "lyrics-eng") } +func (t *Tags) Comment() string { return t.getFirstTagValue("comment") } +func (t *Tags) Lyrics() string { return t.getFirstTagValue("lyrics", "lyrics-eng") } func (t *Tags) Compilation() bool { return t.getBool("tcmp", "compilation") } func (t *Tags) TrackNumber() (int, int) { return t.getTuple("track", "tracknumber") } func (t *Tags) DiscNumber() (int, int) { return t.getTuple("disc", "discnumber") } -func (t *Tags) DiscSubtitle() string { return t.getTag("tsst", "discsubtitle", "setsubtitle") } -func (t *Tags) CatalogNum() string { return t.getTag("catalognumber") } -func (t *Tags) Bpm() int { return (int)(math.Round(t.getFloat("tbpm", "bpm", "fbpm"))) } -func (t *Tags) HasPicture() bool { return t.getTag("has_picture") != "" } +func (t *Tags) DiscSubtitle() string { + return t.getFirstTagValue("tsst", "discsubtitle", "setsubtitle") +} +func (t *Tags) CatalogNum() string { return t.getFirstTagValue("catalognumber") } +func (t *Tags) Bpm() int { return (int)(math.Round(t.getFloat("tbpm", "bpm", "fbpm"))) } +func (t *Tags) HasPicture() bool { return t.getFirstTagValue("has_picture") != "" } // MusicBrainz Identifiers @@ -92,10 +97,10 @@ func (t *Tags) MbzAlbumArtistID() string { return t.getMbzID("musicbrainz_albumartistid", "musicbrainz album artist id") } func (t *Tags) MbzAlbumType() string { - return t.getTag("musicbrainz_albumtype", "musicbrainz album type") + return t.getFirstTagValue("musicbrainz_albumtype", "musicbrainz album type") } func (t *Tags) MbzAlbumComment() string { - return t.getTag("musicbrainz_albumcomment", "musicbrainz album comment") + return t.getFirstTagValue("musicbrainz_albumcomment", "musicbrainz album comment") } // File properties @@ -107,8 +112,8 @@ func (t *Tags) Size() int64 { return t.fileInfo.Size() } func (t *Tags) FilePath() string { return t.filePath } func (t *Tags) Suffix() string { return t.suffix } -func (t *Tags) getTags(tags ...string) []string { - allTags := append(tags, t.custom[tags[0]]...) +func (t *Tags) getTags(tagNames ...string) []string { + allTags := append(tagNames, t.custom[tagNames[0]]...) for _, tag := range allTags { if v, ok := t.tags[tag]; ok { return v @@ -117,30 +122,41 @@ func (t *Tags) getTags(tags ...string) []string { return nil } -func (t *Tags) getTag(tags ...string) string { - ts := t.getTags(tags...) +func (t *Tags) getFirstTagValue(tagNames ...string) string { + ts := t.getTags(tagNames...) if len(ts) > 0 { return ts[0] } return "" } -func (t *Tags) getSortTag(originalTag string, tags ...string) string { +func (t *Tags) getAllTagValues(tagNames ...string) []string { + tagNames = append(tagNames, t.custom[tagNames[0]]...) + var values []string + for _, tag := range tagNames { + if v, ok := t.tags[tag]; ok { + values = append(values, v...) + } + } + return utils.UniqueStrings(values) +} + +func (t *Tags) getSortTag(originalTag string, tagNamess ...string) string { formats := []string{"sort%s", "sort_%s", "sort-%s", "%ssort", "%s_sort", "%s-sort"} all := []string{originalTag} - for _, tag := range tags { + for _, tag := range tagNamess { for _, format := range formats { name := fmt.Sprintf(format, tag) all = append(all, name) } } - return t.getTag(all...) + return t.getFirstTagValue(all...) } var dateRegex = regexp.MustCompile(`([12]\d\d\d)`) -func (t *Tags) getYear(tags ...string) int { - tag := t.getTag(tags...) +func (t *Tags) getYear(tagNames ...string) int { + tag := t.getFirstTagValue(tagNames...) if tag == "" { return 0 } @@ -153,8 +169,8 @@ func (t *Tags) getYear(tags ...string) int { return year } -func (t *Tags) getBool(tags ...string) bool { - tag := t.getTag(tags...) +func (t *Tags) getBool(tagNames ...string) bool { + tag := t.getFirstTagValue(tagNames...) if tag == "" { return false } @@ -162,8 +178,8 @@ func (t *Tags) getBool(tags ...string) bool { return i == 1 } -func (t *Tags) getTuple(tags ...string) (int, int) { - tag := t.getTag(tags...) +func (t *Tags) getTuple(tagNames ...string) (int, int) { + tag := t.getFirstTagValue(tagNames...) if tag == "" { return 0, 0 } @@ -173,28 +189,28 @@ func (t *Tags) getTuple(tags ...string) (int, int) { if len(tuple) > 1 { t2, _ = strconv.Atoi(tuple[1]) } else { - t2tag := t.getTag(tags[0] + "total") + t2tag := t.getFirstTagValue(tagNames[0] + "total") t2, _ = strconv.Atoi(t2tag) } return t1, t2 } -func (t *Tags) getMbzID(tags ...string) string { - tag := t.getTag(tags...) +func (t *Tags) getMbzID(tagNames ...string) string { + tag := t.getFirstTagValue(tagNames...) if _, err := uuid.Parse(tag); err != nil { return "" } return tag } -func (t *Tags) getInt(tags ...string) int { - tag := t.getTag(tags...) +func (t *Tags) getInt(tagNames ...string) int { + tag := t.getFirstTagValue(tagNames...) i, _ := strconv.Atoi(tag) return i } -func (t *Tags) getFloat(tags ...string) float64 { - var tag = t.getTag(tags...) +func (t *Tags) getFloat(tagNames ...string) float64 { + var tag = t.getFirstTagValue(tagNames...) var value, err = strconv.ParseFloat(tag, 64) if err != nil { return 0 diff --git a/scanner/metadata/metadata_test.go b/scanner/metadata/metadata_test.go index d502fd20c..d62115e6b 100644 --- a/scanner/metadata/metadata_test.go +++ b/scanner/metadata/metadata_test.go @@ -60,4 +60,17 @@ var _ = Describe("Tags", func() { Expect(md.MbzAlbumArtistID()).To(Equal("")) }) }) + + Describe("getAllTagValues", func() { + It("returns values from all tag names", func() { + md := &Tags{} + md.tags = map[string][]string{ + "genre": {"Rock", "Pop"}, + "_genre": {"New Wave", "Rock"}, + } + md.custom = map[string][]string{"genre": {"_genre"}} + + Expect(md.Genres()).To(ConsistOf("Rock", "Pop", "New Wave")) + }) + }) }) diff --git a/scanner/metadata/taglib.go b/scanner/metadata/taglib.go index 7932857db..f959af82d 100644 --- a/scanner/metadata/taglib.go +++ b/scanner/metadata/taglib.go @@ -31,7 +31,7 @@ func (e *taglibExtractor) extractMetadata(filePath string) (*Tags, error) { } } - tags := NewTag(filePath, parsedTags, map[string][]string{ + tags := NewTags(filePath, parsedTags, map[string][]string{ "title": {"_track", "titlesort"}, "album": {"_album", "albumsort"}, "artist": {"_artist", "artistsort"}, diff --git a/scanner/metadata/taglib_test.go b/scanner/metadata/taglib_test.go index 31beef7a9..f949c89a7 100644 --- a/scanner/metadata/taglib_test.go +++ b/scanner/metadata/taglib_test.go @@ -19,7 +19,7 @@ var _ = Describe("taglibExtractor", func() { Expect(m.Artist()).To(Equal("Artist")) Expect(m.AlbumArtist()).To(Equal("Album Artist")) Expect(m.Compilation()).To(BeTrue()) - Expect(m.Genre()).To(Equal("Rock")) + Expect(m.Genres()).To(ConsistOf("Rock")) Expect(m.Year()).To(Equal(2014)) n, t := m.TrackNumber() Expect(n).To(Equal(2)) diff --git a/server/subsonic/helpers.go b/server/subsonic/helpers.go index 2f9979437..6b2d1a07f 100644 --- a/server/subsonic/helpers.go +++ b/server/subsonic/helpers.go @@ -110,7 +110,11 @@ func toArtistID3(ctx context.Context, a model.Artist) responses.ArtistID3 { func toGenres(genres model.Genres) *responses.Genres { response := make([]responses.Genre, len(genres)) for i, g := range genres { - response[i] = responses.Genre(g) + response[i] = responses.Genre{ + Name: g.Name, + SongCount: g.SongCount, + AlbumCount: g.AlbumCount, + } } return &responses.Genres{Genre: response} } diff --git a/utils/strings.go b/utils/strings.go index 6f5295cb5..43babf81f 100644 --- a/utils/strings.go +++ b/utils/strings.go @@ -17,8 +17,8 @@ func NoArticle(name string) string { return name } -func StringInSlice(a string, list []string) bool { - for _, b := range list { +func StringInSlice(a string, slice []string) bool { + for _, b := range slice { if b == a { return true } @@ -26,17 +26,28 @@ func StringInSlice(a string, list []string) bool { return false } -func InsertString(array []string, value string, index int) []string { - return append(array[:index], append([]string{value}, array[index:]...)...) +func InsertString(slice []string, value string, index int) []string { + return append(slice[:index], append([]string{value}, slice[index:]...)...) } -func RemoveString(array []string, index int) []string { - return append(array[:index], array[index+1:]...) +func RemoveString(slice []string, index int) []string { + return append(slice[:index], slice[index+1:]...) } -func MoveString(array []string, srcIndex int, dstIndex int) []string { - value := array[srcIndex] - return InsertString(RemoveString(array, srcIndex), value, dstIndex) +func UniqueStrings(slice []string) []string { + var unique []string + for _, s := range slice { + if StringInSlice(s, unique) { + continue + } + unique = append(unique, s) + } + return unique +} + +func MoveString(slice []string, srcIndex int, dstIndex int) []string { + value := slice[srcIndex] + return InsertString(RemoveString(slice, srcIndex), value, dstIndex) } func BreakUpStringSlice(items []string, chunkSize int) [][]string {