diff --git a/db/migration/20231219003407_add_folder_table.go b/db/migration/20231219003407_add_folder_table.go index d2cd5aa49..d0ffda02e 100644 --- a/db/migration/20231219003407_add_folder_table.go +++ b/db/migration/20231219003407_add_folder_table.go @@ -34,15 +34,36 @@ alter table media_file add column pid varchar default id not null; alter table media_file add column album_pid varchar default album_id not null; +alter table media_file + add column tags JSONB default '{}' not null; -create index if not exists media_file_folder_id_index +create index if not exists media_file_folder_id_ix on media_file (folder_id); -create index if not exists media_file_pid_index +create unique index if not exists media_file_pid_ix on media_file (pid); -create index if not exists media_file_album_pid_index +create index if not exists media_file_album_pid_ix on media_file (album_pid); -- FIXME Needs to process current media_file.paths, creating folders as needed + +create table if not exists tag( + id varchar not null primary key, + name varchar default '' not null, + value varchar default '' not null, + constraint tags_name_value_ux + unique (name, value) +); + +create table if not exists item_tags( + item_id varchar not null, + item_type varchar not null, + tag_name varchar not null, + tag_id varchar not null, + constraint item_tags_ux + unique (item_id, item_type, tag_id) +); + +create index if not exists item_tag_name_ix on item_tags(item_id, tag_name) `) return err diff --git a/model/datastore.go b/model/datastore.go index 57944034f..859243444 100644 --- a/model/datastore.go +++ b/model/datastore.go @@ -26,6 +26,7 @@ type DataStore interface { Artist(ctx context.Context) ArtistRepository MediaFile(ctx context.Context) MediaFileRepository Genre(ctx context.Context) GenreRepository + Tag(ctx context.Context) TagRepository Playlist(ctx context.Context) PlaylistRepository PlayQueue(ctx context.Context) PlayQueueRepository Transcoding(ctx context.Context) TranscodingRepository diff --git a/model/mediafile.go b/model/mediafile.go index 78f6793f5..d7dcf2b8e 100644 --- a/model/mediafile.go +++ b/model/mediafile.go @@ -22,7 +22,7 @@ type MediaFile struct { ID string `structs:"id" json:"id"` PID string `structs:"pid" json:"pid"` - LibraryID string `structs:"library_id" json:"libraryId"` + LibraryID int `structs:"library_id" json:"libraryId"` FolderID string `structs:"folder_id" json:"folderId"` Path string `structs:"path" json:"path"` Title string `structs:"title" json:"title"` @@ -76,6 +76,8 @@ type MediaFile struct { RgTrackGain float64 `structs:"rg_track_gain" json:"rgTrackGain"` RgTrackPeak float64 `structs:"rg_track_peak" json:"rgTrackPeak"` + Tags Tags `structs:"tags" json:"tags,omitempty"` // All tags from the original file + CreatedAt time.Time `structs:"created_at" json:"createdAt"` // Time this entry was created in the DB UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"` // Time of file last update (mtime) } diff --git a/model/tag.go b/model/tag.go new file mode 100644 index 000000000..b740d5bda --- /dev/null +++ b/model/tag.go @@ -0,0 +1,59 @@ +package model + +import ( + "crypto/md5" + "fmt" + "strings" + + "github.com/navidrome/navidrome/consts" +) + +type Tag struct { + ID string + Name string + Value string +} + +type FlattenedTags []Tag + +func (t Tag) String() string { + return fmt.Sprintf("%s=%s", t.Name, t.Value) +} + +type Tags map[string][]string + +func (t Tags) Values(name string) []string { + return t[name] +} + +func (t Tags) Flatten(name string) FlattenedTags { + var tags FlattenedTags + for _, v := range t[name] { + tags = append(tags, NewTag(name, v)) + } + return tags +} + +func (t Tags) FlattenAll() FlattenedTags { + var tags FlattenedTags + for name, values := range t { + for _, v := range values { + tags = append(tags, NewTag(name, v)) + } + } + return tags +} + +func NewTag(name, value string) Tag { + name = strings.ToLower(name) + id := fmt.Sprintf("%x", md5.Sum([]byte(name+consts.Zwsp+strings.ToLower(value)))) + return Tag{ + ID: id, + Name: name, + Value: value, + } +} + +type TagRepository interface { + Add(...Tag) error +} diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 3ef3528ae..add6bc199 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -167,7 +167,7 @@ func (r *albumRepository) Get(id string) (*model.Album, error) { } func (r *albumRepository) Put(m *model.Album) error { - _, err := r.put(m.ID, &dbAlbum{Album: m}) + _, err := r.put("pid", m.PID, &dbAlbum{Album: m}) if err != nil { return err } diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go index 74b44d2f5..28af52271 100644 --- a/persistence/artist_repository.go +++ b/persistence/artist_repository.go @@ -95,7 +95,7 @@ func (r *artistRepository) Exists(id string) (bool, error) { func (r *artistRepository) Put(a *model.Artist, colsToUpdate ...string) error { a.FullText = getFullText(a.Name, a.SortArtistName) dba := &dbArtist{Artist: a} - _, err := r.put(dba.ID, dba, colsToUpdate...) + _, err := r.put("id", dba.ID, dba, colsToUpdate...) if err != nil { return err } diff --git a/persistence/folder_repository.go b/persistence/folder_repository.go index 5226dfff2..98682160c 100644 --- a/persistence/folder_repository.go +++ b/persistence/folder_repository.go @@ -55,7 +55,7 @@ func (r folderRepository) GetLastUpdates(lib model.Library) (map[string]time.Tim func (r folderRepository) Put(lib model.Library, path string) error { folder := model.NewFolder(lib, path) - _, err := r.put(folder.ID, folder) + _, err := r.put("id", folder.ID, folder) return err } diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index ec940c940..f5f254a6a 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -2,6 +2,8 @@ package persistence import ( "context" + "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -21,7 +23,51 @@ type mediaFileRepository struct { sqlRestful } -func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepository { +type dbMediaFile struct { + *model.MediaFile `structs:",flatten"` + Tags string `structs:"-" json:"tags"` +} + +func (m *dbMediaFile) PostScan() error { + if m.Tags == "" { + m.MediaFile.Tags = make(map[string][]string) + return nil + } + err := json.Unmarshal([]byte(m.Tags), &m.MediaFile.Tags) + if err != nil { + return err + } + // Map genres from tags + for _, g := range m.MediaFile.Tags.Flatten("genre") { + m.MediaFile.Genres = append(m.MediaFile.Genres, model.Genre{Name: g.Value, ID: g.ID}) + } + return nil +} + +func (m *dbMediaFile) PostMapArgs(args map[string]any) error { + if len(m.MediaFile.Tags) == 0 { + args["tags"] = "{}" + return nil + } + b, err := json.Marshal(m.MediaFile.Tags) + if err != nil { + return err + } + args["tags"] = string(b) + return nil +} + +type dbMediaFiles []dbMediaFile + +func (m *dbMediaFiles) toModels() model.MediaFiles { + res := make(model.MediaFiles, len(*m)) + for i, mf := range *m { + res[i] = *mf.MediaFile + } + return res +} + +func NewMediaFileRepository(ctx context.Context, db dbx.Builder) model.MediaFileRepository { r := &mediaFileRepository{} r.ctx = ctx r.db = db @@ -50,7 +96,8 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepos func (r *mediaFileRepository) CountAll(options ...model.QueryOptions) (int64, error) { sql := r.newSelectWithAnnotation("media_file.id") - sql = r.withGenres(sql) // Required for filtering by genre + // FIXME Genres + //sql = r.withGenres(sql) // Required for filtering by genre return r.count(sql, options...) } @@ -59,73 +106,89 @@ func (r *mediaFileRepository) Exists(id string) (bool, error) { } func (r *mediaFileRepository) Put(m *model.MediaFile) error { + if m.ID == "" || m.PID == "" { + return errors.New("id and pid are required") + } m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist, m.SortTitle, m.SortAlbumName, m.SortArtistName, m.SortAlbumArtistName, m.DiscSubtitle) - _, err := r.put(m.ID, m) + _, err := r.put("pid", m.PID, &dbMediaFile{MediaFile: m}) if err != nil { return err } - return r.updateGenres(m.ID, m.Genres) + + return r.updateTags(m.ID, m.Tags) + + // FIXME Genres + //if err != nil { + // return err + //} + //return r.updateGenres(m.ID, r.tableName, m.Genres) } func (r *mediaFileRepository) selectMediaFile(options ...model.QueryOptions) SelectBuilder { sql := r.newSelectWithAnnotation("media_file.id", options...).Columns("media_file.*") sql = r.withBookmark(sql, "media_file.id") - if len(options) > 0 && options[0].Filters != nil { - s, _, _ := options[0].Filters.ToSql() - // If there's any reference of genre in the filter, joins with genre - if strings.Contains(s, "genre") { - sql = r.withGenres(sql) - // If there's no filter on genre_id, group the results by media_file.id - if !strings.Contains(s, "genre_id") { - sql = sql.GroupBy("media_file.id") - } - } - } + // FIXME Genres + //if len(options) > 0 && options[0].Filters != nil { + // s, _, _ := options[0].Filters.ToSql() + // // If there's any reference of genre in the filter, joins with genre + // if strings.Contains(s, "genre") { + // sql = r.withGenres(sql) + // // If there's no filter on genre_id, group the results by media_file.id + // if !strings.Contains(s, "genre_id") { + // sql = sql.GroupBy("media_file.id") + // } + // } + //} return sql } func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) { sel := r.selectMediaFile().Where(Eq{"media_file.id": id}) - var res model.MediaFiles + var res dbMediaFiles if err := r.queryAll(sel, &res); err != nil { return nil, err } if len(res) == 0 { return nil, model.ErrNotFound } - err := loadAllGenres(r, res) - return &res[0], err + // FIXME Genres + //err := r.loadMediaFileGenres(&res) + return res[0].MediaFile, nil } func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) { sq := r.selectMediaFile(options...) - res := model.MediaFiles{} + var res dbMediaFiles err := r.queryAll(sq, &res, options...) if err != nil { return nil, err } - err = loadAllGenres(r, res) - return res, err + // FIXME Genres + //err = r.loadMediaFileGenres(&rows) + return res.toModels(), nil } func (r *mediaFileRepository) GetByFolder(folderID string) (model.MediaFiles, error) { sq := r.newSelect().Columns("*").Where(Eq{"folder_id": folderID}) - res := model.MediaFiles{} + var res dbMediaFiles err := r.queryAll(sq, &res) - return res, err + if err != nil { + return nil, err + } + return res.toModels(), nil } func (r *mediaFileRepository) FindByPath(path string) (*model.MediaFile, error) { sel := r.newSelect().Columns("*").Where(Like{"path": path}) - var res model.MediaFiles + var res dbMediaFiles if err := r.queryAll(sel, &res); err != nil { return nil, err } if len(res) == 0 { return nil, model.ErrNotFound } - return &res[0], nil + return res[0].MediaFile, nil } func cleanPath(path string) string { @@ -151,9 +214,9 @@ func (r *mediaFileRepository) FindAllByPath(path string) (model.MediaFiles, erro sel := r.newSelect().Columns("*", "item NOT GLOB '*"+string(os.PathSeparator)+"*' AS isLast"). Where(Eq{"isLast": 1}).FromSelect(sel0, "sel0") - res := model.MediaFiles{} + res := dbMediaFiles{} err := r.queryAll(sel, &res) - return res, err + return res.toModels(), err } // FindPathsRecursively returns a list of all subfolders of basePath, recursively @@ -202,13 +265,14 @@ func (r *mediaFileRepository) removeNonAlbumArtistIds() error { } func (r *mediaFileRepository) Search(q string, offset int, size int) (model.MediaFiles, error) { - results := model.MediaFiles{} + results := dbMediaFiles{} err := r.doSearch(q, offset, size, &results, "title") if err != nil { return nil, err } - err = loadAllGenres(r, results) - return results, err + // FIXME Genres + //err = r.loadMediaFileGenres(&results) + return results.toModels(), err } func (r *mediaFileRepository) Count(options ...rest.QueryOptions) (int64, error) { diff --git a/persistence/persistence.go b/persistence/persistence.go index db6b057ce..a9e74841a 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -43,6 +43,10 @@ func (s *SQLStore) Genre(ctx context.Context) model.GenreRepository { return NewGenreRepository(ctx, s.getDBXBuilder()) } +func (s *SQLStore) Tag(ctx context.Context) model.TagRepository { + return NewTagRepository(ctx, s.getDBXBuilder()) +} + func (s *SQLStore) PlayQueue(ctx context.Context) model.PlayQueueRepository { return NewPlayQueueRepository(ctx, s.getDBXBuilder()) } diff --git a/persistence/player_repository.go b/persistence/player_repository.go index ea28e2c40..ab372e54b 100644 --- a/persistence/player_repository.go +++ b/persistence/player_repository.go @@ -27,7 +27,7 @@ func NewPlayerRepository(ctx context.Context, db dbx.Builder) model.PlayerReposi } func (r *playerRepository) Put(p *model.Player) error { - _, err := r.put(p.ID, p) + _, err := r.put("id", p.ID, p) return err } @@ -102,7 +102,7 @@ func (r *playerRepository) Save(entity interface{}) (string, error) { if !r.isPermitted(t) { return "", rest.ErrPermissionDenied } - id, err := r.put(t.ID, t) + id, err := r.put("id", t.ID, t) if errors.Is(err, model.ErrNotFound) { return "", rest.ErrNotFound } @@ -115,7 +115,7 @@ func (r *playerRepository) Update(id string, entity interface{}, cols ...string) if !r.isPermitted(t) { return rest.ErrPermissionDenied } - _, err := r.put(id, t, cols...) + _, err := r.put("id", id, t, cols...) if errors.Is(err, model.ErrNotFound) { return rest.ErrNotFound } diff --git a/persistence/playlist_repository.go b/persistence/playlist_repository.go index feafc844c..b7755fb83 100644 --- a/persistence/playlist_repository.go +++ b/persistence/playlist_repository.go @@ -118,7 +118,7 @@ func (r *playlistRepository) Put(p *model.Playlist) error { } pls.UpdatedAt = time.Now() - id, err := r.put(pls.ID, pls) + id, err := r.put("id", pls.ID, pls) if err != nil { return err } @@ -417,7 +417,7 @@ func (r *playlistRepository) Update(id string, entity interface{}, cols ...strin } pls.ID = id pls.UpdatedAt = time.Now() - _, err = r.put(id, pls, append(cols, "updatedAt")...) + _, err = r.put("id", id, pls, append(cols, "updatedAt")...) if errors.Is(err, model.ErrNotFound) { return rest.ErrNotFound } diff --git a/persistence/playqueue_repository.go b/persistence/playqueue_repository.go index fa21f184b..3b6538689 100644 --- a/persistence/playqueue_repository.go +++ b/persistence/playqueue_repository.go @@ -47,7 +47,7 @@ func (r *playQueueRepository) Store(q *model.PlayQueue) error { pq.CreatedAt = time.Now() } pq.UpdatedAt = time.Now() - _, err = r.put(pq.ID, pq) + _, err = r.put("id", pq.ID, pq) if err != nil { log.Error(r.ctx, "Error saving playqueue", "user", u.UserName, err) return err diff --git a/persistence/share_repository.go b/persistence/share_repository.go index 2547bcfa5..3c1419ebe 100644 --- a/persistence/share_repository.go +++ b/persistence/share_repository.go @@ -138,7 +138,7 @@ func (r *shareRepository) Update(id string, entity interface{}, cols ...string) s.ID = id s.UpdatedAt = time.Now() cols = append(cols, "updated_at") - _, err := r.put(id, s, cols...) + _, err := r.put("id", id, s, cols...) if errors.Is(err, model.ErrNotFound) { return rest.ErrNotFound } @@ -154,7 +154,7 @@ func (r *shareRepository) Save(entity interface{}) (string, error) { } s.CreatedAt = time.Now() s.UpdatedAt = time.Now() - id, err := r.put(s.ID, s) + id, err := r.put("id", s.ID, s) if errors.Is(err, model.ErrNotFound) { return "", rest.ErrNotFound } diff --git a/persistence/sql_base_repository.go b/persistence/sql_base_repository.go index 3bb050e89..433d1476c 100644 --- a/persistence/sql_base_repository.go +++ b/persistence/sql_base_repository.go @@ -241,10 +241,10 @@ func (r sqlRepository) count(countQuery SelectBuilder, options ...model.QueryOpt return res.Count, err } -func (r sqlRepository) put(id string, m interface{}, colsToUpdate ...string) (newId string, err error) { +func (r sqlRepository) put(idCol string, idVal string, m interface{}, colsToUpdate ...string) (newId string, err error) { values, _ := toSQLArgs(m) // If there's an ID, try to update first - if id != "" { + if idVal != "" { updateValues := map[string]interface{}{} // This is a map of the columns that need to be updated, if specified @@ -259,23 +259,23 @@ func (r sqlRepository) put(id string, m interface{}, colsToUpdate ...string) (ne } delete(updateValues, "created_at") - update := Update(r.tableName).Where(Eq{"id": id}).SetMap(updateValues) + update := Update(r.tableName).Where(Eq{idCol: idVal}).SetMap(updateValues) count, err := r.executeSQL(update) if err != nil { return "", err } if count > 0 { - return id, nil + return idVal, nil } } // If it does not have an ID OR the ID was not found (when it is a new record with predefined id) - if id == "" { - id = uuid.NewString() - values["id"] = id + if idVal == "" { + idVal = uuid.NewString() + values[idCol] = idVal } insert := Insert(r.tableName).SetMap(values) _, err = r.executeSQL(insert) - return id, err + return idVal, err } func (r sqlRepository) delete(cond Sqlizer) error { diff --git a/persistence/tag_repository.go b/persistence/tag_repository.go new file mode 100644 index 000000000..a327b8092 --- /dev/null +++ b/persistence/tag_repository.go @@ -0,0 +1,52 @@ +package persistence + +import ( + "context" + + . "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils/slice" + "github.com/pocketbase/dbx" +) + +type tagRepository struct { + sqlRepository +} + +func NewTagRepository(ctx context.Context, db dbx.Builder) model.TagRepository { + r := &tagRepository{} + r.ctx = ctx + r.db = db + r.tableName = "tag" + return r +} + +func (r *tagRepository) Add(tags ...model.Tag) error { + return slice.RangeByChunks(tags, 200, func(chunk []model.Tag) error { + sq := Insert(r.tableName).Columns("id", "name", "value"). + Suffix("on conflict (id) do nothing") + for _, t := range chunk { + sq = sq.Values(t.ID, t.Name, t.Value) + } + _, err := r.executeSQL(sq) + return err + }) +} + +func (r *sqlRepository) updateTags(itemID string, tags model.Tags) error { + sqd := Delete("item_tags").Where(Eq{"item_id": itemID, "item_type": r.tableName}) + _, err := r.executeSQL(sqd) + if err != nil { + return err + } + sqi := Insert("item_tags").Columns("item_id", "item_type", "tag_name", "tag_id"). + Suffix("on conflict (item_id, item_type, tag_id) do nothing") + for name, values := range tags { + for _, value := range values { + tag := model.NewTag(name, value) + sqi = sqi.Values(itemID, r.tableName, tag.Name, tag.ID) + } + } + _, err = r.executeSQL(sqi) + return err +} diff --git a/persistence/transcoding_repository.go b/persistence/transcoding_repository.go index 17cd0a019..723f988fc 100644 --- a/persistence/transcoding_repository.go +++ b/persistence/transcoding_repository.go @@ -42,7 +42,7 @@ func (r *transcodingRepository) FindByFormat(format string) (*model.Transcoding, } func (r *transcodingRepository) Put(t *model.Transcoding) error { - _, err := r.put(t.ID, t) + _, err := r.put("id", t.ID, t) return err } @@ -71,7 +71,7 @@ func (r *transcodingRepository) NewInstance() interface{} { func (r *transcodingRepository) Save(entity interface{}) (string, error) { t := entity.(*model.Transcoding) - id, err := r.put(t.ID, t) + id, err := r.put("id", t.ID, t) if errors.Is(err, model.ErrNotFound) { return "", rest.ErrNotFound } @@ -81,7 +81,7 @@ func (r *transcodingRepository) Save(entity interface{}) (string, error) { func (r *transcodingRepository) Update(id string, entity interface{}, cols ...string) error { t := entity.(*model.Transcoding) t.ID = id - _, err := r.put(id, t) + _, err := r.put("id", id, t) if errors.Is(err, model.ErrNotFound) { return rest.ErrNotFound } diff --git a/scanner/metadata/metadata.go b/scanner/metadata/metadata.go index 63a3bd789..0d4a3d9d5 100644 --- a/scanner/metadata/metadata.go +++ b/scanner/metadata/metadata.go @@ -231,7 +231,7 @@ func (t Tags) BirthTime() time.Time { return time.Now() } -// ReplayGain Properties +// ReplayGain Properties TODO: check rg_* tags func (t Tags) RGAlbumGain() float64 { return t.getGainValue("replaygain_album_gain") } func (t Tags) RGAlbumPeak() float64 { return t.getPeakValue("replaygain_album_peak") } @@ -385,3 +385,68 @@ func (t Tags) getFloat(tagNames ...string) float64 { } return value } + +// We exclude all tags that are already first-class citizens in the model +var excludedTags = map[string]struct{}{ + "duration": {}, + "bitrate": {}, + "channels": {}, + "bpm": {}, + "has_picture": {}, + "title": {}, + "album": {}, + "artist": {}, + "artists": {}, + "albumartist": {}, + "albumartists": {}, + "track": {}, + "tracknumber": {}, + "tracktotal": {}, + "totaltracks": {}, + "disc": {}, + "discnumber": {}, + "disctotal": {}, + "totaldiscs": {}, + "lyrics": {}, + "year": {}, + "date": {}, + "originaldate": {}, + "releasedate": {}, + "comment": {}, +} + +// Also exclude any tag that starts with one of these prefixes +var excludedPrefixes = []string{ + "musicbrainz", + "replaygain", + "sort", +} + +func isExcludedTag(tagName string) bool { + if _, ok := excludedTags[tagName]; ok { + return true + } + for _, prefix := range excludedPrefixes { + if strings.HasPrefix(tagName, prefix) { + return true + } + } + return false +} + +func (t Tags) ModelTags() model.Tags { + models := model.Tags{} + for tagName, values := range t.tags { + if isExcludedTag(tagName) { + continue + } + + for _, value := range values { + if value == "" { + continue + } + models[tagName] = append(models[tagName], value) + } + } + return models +} diff --git a/scanner/metadata/taglib/taglib.go b/scanner/metadata/taglib/taglib.go index 20403189f..db8234737 100644 --- a/scanner/metadata/taglib/taglib.go +++ b/scanner/metadata/taglib/taglib.go @@ -50,6 +50,7 @@ func (e *Extractor) extractMetadata(filePath string) (metadata.ParsedTags, error if duration := float64(millis) / 1000.0; duration > 0 { tags["duration"] = []string{strconv.FormatFloat(duration, 'f', 2, 32)} } + delete(tags, "lengthinmilliseconds") } // Adjust some ID3 tags parseTIPL(tags) diff --git a/scanner2/folder.go b/scanner2/folder.go index 2f6a8ce1b..798fdd6d6 100644 --- a/scanner2/folder.go +++ b/scanner2/folder.go @@ -20,6 +20,7 @@ type folderEntry struct { tracks model.MediaFiles albums model.Albums artists model.Artists + tags model.FlattenedTags missingTracks model.MediaFiles } diff --git a/scanner2/mapping.go b/scanner2/mapping.go index ed0f8aa5b..9bbf95b22 100644 --- a/scanner2/mapping.go +++ b/scanner2/mapping.go @@ -16,12 +16,12 @@ import ( ) type mediaFileMapper struct { - rootFolder string + entry *folderEntry } func newMediaFileMapper(entry *folderEntry) *mediaFileMapper { return &mediaFileMapper{ - rootFolder: entry.path, + entry: entry, } } @@ -74,6 +74,10 @@ func (s mediaFileMapper) toMediaFile(md metadata.Tags) model.MediaFile { mf.Bpm = md.Bpm() mf.CreatedAt = md.BirthTime() mf.UpdatedAt = md.ModificationTime() + mf.Tags = md.ModelTags() + mf.FolderID = s.entry.id + mf.LibraryID = s.entry.scanCtx.lib.ID + mf.PID = mf.ID return *mf } @@ -85,7 +89,7 @@ func sanitizeFieldForSorting(originalValue string) string { func (s mediaFileMapper) mapTrackTitle(md metadata.Tags) string { if md.Title() == "" { - s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator)) + s := strings.TrimPrefix(md.FilePath(), s.entry.path+string(os.PathSeparator)) e := filepath.Ext(s) return strings.TrimSuffix(s, e) } diff --git a/scanner2/persiste_changes.go b/scanner2/persiste_changes.go new file mode 100644 index 000000000..7f8a9f5d1 --- /dev/null +++ b/scanner2/persiste_changes.go @@ -0,0 +1,47 @@ +package scanner2 + +import ( + "context" + + "github.com/google/go-pipeline/pkg/pipeline" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils/slice" +) + +func persistChanges(ctx context.Context) pipeline.StageFn[*folderEntry] { + return func(entry *folderEntry) (*folderEntry, error) { + err := entry.scanCtx.ds.WithTx(func(tx model.DataStore) error { + // Save all tags to DB + err := slice.RangeByChunks(entry.tags, 100, func(chunk []model.Tag) error { + err := tx.Tag(ctx).Add(chunk...) + if err != nil { + log.Error(ctx, "Scanner: Error adding tags to DB", "folder", entry.path, err) + return err + } + return nil + }) + if err != nil { + return err + } + + // Save all tracks to DB + err = slice.RangeByChunks(entry.tracks, 100, func(chunk []model.MediaFile) error { + for i := range chunk { + track := chunk[i] + err = tx.MediaFile(ctx).Put(&track) + if err != nil { + log.Error(ctx, "Scanner: Error adding mediafile to DB", "folder", entry.path, "track", track, err) + return err + } + } + return nil + }) + return err + }) + if err != nil { + log.Error(ctx, "Scanner: Error persisting changes to DB", "folder", entry.path, err) + } + return entry, err + } +} diff --git a/scanner2/persistent_ids.go b/scanner2/persistent_ids.go index 98415c5e8..4c57df300 100644 --- a/scanner2/persistent_ids.go +++ b/scanner2/persistent_ids.go @@ -1,33 +1 @@ package scanner2 - -import ( - "crypto/md5" - "fmt" - - "github.com/navidrome/navidrome/scanner/metadata" - . "github.com/navidrome/navidrome/utils/gg" -) - -func artistPID(md metadata.Tags) string { - key := FirstOr(md.Artist(), "M"+md.MbzArtistID()) - return fmt.Sprintf("%x", md5.Sum([]byte(key))) -} - -func albumArtistPID(md metadata.Tags) string { - key := FirstOr(md.AlbumArtist(), "M"+md.MbzAlbumArtistID()) - return fmt.Sprintf("%x", md5.Sum([]byte(key))) -} - -func albumPID(md metadata.Tags) string { - var key string - if md.MbzAlbumID() != "" { - key = "M" + md.MbzAlbumID() - } else { - key = fmt.Sprintf("%s%s%t", albumArtistPID(md), md.Album(), md.Compilation()) - } - return fmt.Sprintf("%x", md5.Sum([]byte(key))) -} - -func trackPID(md metadata.Tags) string { - return fmt.Sprintf("%s%x", albumPID(md), md5.Sum([]byte(md.FilePath()))) -} diff --git a/scanner2/process_folders.go b/scanner2/process_folders.go index 7699046f8..f9b461fce 100644 --- a/scanner2/process_folders.go +++ b/scanner2/process_folders.go @@ -50,35 +50,41 @@ func processFolder(ctx context.Context) pipeline.StageFn[*folderEntry] { // Remaining dbTracks are tracks that were not found in the folder, so they should be marked as missing entry.missingTracks = maps.Values(dbTracks) - entry.tracks, err = loadTagsFromFiles(ctx, entry, filesToImport) - if err != nil { - log.Warn(ctx, "Scanner: Error loading tags from files. Skipping", "folder", entry.path, err) - return entry, nil - } + if len(filesToImport) > 0 { + entry.tracks, entry.tags, err = loadTagsFromFiles(ctx, entry, filesToImport) + if err != nil { + log.Warn(ctx, "Scanner: Error loading tags from files. Skipping", "folder", entry.path, err) + return entry, nil + } - entry.albums = loadAlbumsFromTags(ctx, entry) - entry.artists = loadArtistsFromTags(ctx, entry) + entry.albums = loadAlbumsFromTags(ctx, entry) + entry.artists = loadArtistsFromTags(ctx, entry) + } return entry, nil } } -func loadTagsFromFiles(ctx context.Context, entry *folderEntry, toImport []string) (model.MediaFiles, error) { +func loadTagsFromFiles(ctx context.Context, entry *folderEntry, toImport []string) (model.MediaFiles, model.FlattenedTags, error) { tracks := model.MediaFiles{} + uniqueTags := make(map[string]model.Tag) mapper := newMediaFileMapper(entry) err := slice.RangeByChunks(toImport, filesBatchSize, func(chunk []string) error { - allTags, err := metadata.Extract(toImport...) + allFileTags, err := metadata.Extract(toImport...) if err != nil { log.Warn(ctx, "Scanner: Error extracting tags from files. Skipping", "folder", entry.path, err) return err } - for _, tags := range allTags { - track := mapper.toMediaFile(tags) + for _, fileTags := range allFileTags { + track := mapper.toMediaFile(fileTags) tracks = append(tracks, track) + for _, t := range track.Tags.FlattenAll() { + uniqueTags[t.ID] = t + } } return nil }) - return tracks, err + return tracks, maps.Values(uniqueTags), err } func loadAlbumsFromTags(ctx context.Context, entry *folderEntry) model.Albums { diff --git a/scanner2/scanner2.go b/scanner2/scanner2.go index 933ad71f7..bf4a28427 100644 --- a/scanner2/scanner2.go +++ b/scanner2/scanner2.go @@ -34,6 +34,7 @@ func (s *scanner2) RescanAll(requestCtx context.Context, fullRescan bool) error err = s.runPipeline( pipeline.NewProducer(produceFolders(ctx, s.ds, libs, fullRescan), pipeline.Name("read folders from disk")), pipeline.NewStage(processFolder(ctx), pipeline.Name("process folder")), + pipeline.NewStage(persistChanges(ctx), pipeline.Name("persist changes")), pipeline.NewStage(logFolder(ctx), pipeline.Name("log results")), ) diff --git a/tests/mock_persistence.go b/tests/mock_persistence.go index ea4b7ec27..ff8d39f11 100644 --- a/tests/mock_persistence.go +++ b/tests/mock_persistence.go @@ -51,6 +51,10 @@ func (db *MockDataStore) Folder(context.Context) model.FolderRepository { return struct{ model.FolderRepository }{} } +func (db *MockDataStore) Tag(context.Context) model.TagRepository { + return struct{ model.TagRepository }{} +} + func (db *MockDataStore) Genre(context.Context) model.GenreRepository { if db.MockedGenre == nil { db.MockedGenre = &MockedGenreRepo{}