WIP: Persisting tracks and tags to DB

db/migration/20231219003407_add_folder_table.go

@@ -34,15 +34,36 @@ alter table media_file
     add column pid varchar default id not null;
 alter table media_file 
     add column album_pid varchar default album_id not null;
+alter table media_file 
+    add column tags JSONB default '{}' not null;
 
-create index if not exists media_file_folder_id_index
+create index if not exists media_file_folder_id_ix
  	on media_file (folder_id);
-create index if not exists media_file_pid_index
+create unique index if not exists media_file_pid_ix
 	on media_file (pid);
-create index if not exists media_file_album_pid_index
+create index if not exists media_file_album_pid_ix
 	on media_file (album_pid);
 
 -- FIXME Needs to process current media_file.paths, creating folders as needed
+
+create table if not exists tag(
+  	id varchar not null primary key,
+  	name varchar default '' not null,
+  	value varchar default '' not null,
+  	constraint tags_name_value_ux
+		unique (name, value)
+);
+
+create table if not exists item_tags(
+    item_id varchar not null,
+    item_type varchar not null,
+    tag_name varchar not null,
+    tag_id varchar not null,
+  	constraint item_tags_ux
+    	unique (item_id, item_type, tag_id)
+);
+
+create index if not exists item_tag_name_ix on item_tags(item_id, tag_name)
 `)
 
 	return err

model/datastore.go

@@ -26,6 +26,7 @@ type DataStore interface {
 	Artist(ctx context.Context) ArtistRepository
 	MediaFile(ctx context.Context) MediaFileRepository
 	Genre(ctx context.Context) GenreRepository
+	Tag(ctx context.Context) TagRepository
 	Playlist(ctx context.Context) PlaylistRepository
 	PlayQueue(ctx context.Context) PlayQueueRepository
 	Transcoding(ctx context.Context) TranscodingRepository

model/mediafile.go

@@ -22,7 +22,7 @@ type MediaFile struct {
 
 	ID                   string  `structs:"id" json:"id"`
 	PID                  string  `structs:"pid" json:"pid"`
-	LibraryID            string  `structs:"library_id" json:"libraryId"`
+	LibraryID            int     `structs:"library_id" json:"libraryId"`
 	FolderID             string  `structs:"folder_id" json:"folderId"`
 	Path                 string  `structs:"path" json:"path"`
 	Title                string  `structs:"title" json:"title"`
@@ -76,6 +76,8 @@ type MediaFile struct {
 	RgTrackGain          float64 `structs:"rg_track_gain" json:"rgTrackGain"`
 	RgTrackPeak          float64 `structs:"rg_track_peak" json:"rgTrackPeak"`
 
+	Tags Tags `structs:"tags" json:"tags,omitempty"` // All tags from the original file
+
 	CreatedAt time.Time `structs:"created_at" json:"createdAt"` // Time this entry was created in the DB
 	UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"` // Time of file last update (mtime)
 }

model/tag.go

@@ -0,0 +1,59 @@
+package model
+
+import (
+	"crypto/md5"
+	"fmt"
+	"strings"
+
+	"github.com/navidrome/navidrome/consts"
+)
+
+type Tag struct {
+	ID    string
+	Name  string
+	Value string
+}
+
+type FlattenedTags []Tag
+
+func (t Tag) String() string {
+	return fmt.Sprintf("%s=%s", t.Name, t.Value)
+}
+
+type Tags map[string][]string
+
+func (t Tags) Values(name string) []string {
+	return t[name]
+}
+
+func (t Tags) Flatten(name string) FlattenedTags {
+	var tags FlattenedTags
+	for _, v := range t[name] {
+		tags = append(tags, NewTag(name, v))
+	}
+	return tags
+}
+
+func (t Tags) FlattenAll() FlattenedTags {
+	var tags FlattenedTags
+	for name, values := range t {
+		for _, v := range values {
+			tags = append(tags, NewTag(name, v))
+		}
+	}
+	return tags
+}
+
+func NewTag(name, value string) Tag {
+	name = strings.ToLower(name)
+	id := fmt.Sprintf("%x", md5.Sum([]byte(name+consts.Zwsp+strings.ToLower(value))))
+	return Tag{
+		ID:    id,
+		Name:  name,
+		Value: value,
+	}
+}
+
+type TagRepository interface {
+	Add(...Tag) error
+}

persistence/album_repository.go

@@ -167,7 +167,7 @@ func (r *albumRepository) Get(id string) (*model.Album, error) {
 }
 
 func (r *albumRepository) Put(m *model.Album) error {
-	_, err := r.put(m.ID, &dbAlbum{Album: m})
+	_, err := r.put("pid", m.PID, &dbAlbum{Album: m})
 	if err != nil {
 		return err
 	}

persistence/artist_repository.go

@@ -95,7 +95,7 @@ func (r *artistRepository) Exists(id string) (bool, error) {
 func (r *artistRepository) Put(a *model.Artist, colsToUpdate ...string) error {
 	a.FullText = getFullText(a.Name, a.SortArtistName)
 	dba := &dbArtist{Artist: a}
-	_, err := r.put(dba.ID, dba, colsToUpdate...)
+	_, err := r.put("id", dba.ID, dba, colsToUpdate...)
 	if err != nil {
 		return err
 	}

persistence/folder_repository.go

@@ -55,7 +55,7 @@ func (r folderRepository) GetLastUpdates(lib model.Library) (map[string]time.Tim
 
 func (r folderRepository) Put(lib model.Library, path string) error {
 	folder := model.NewFolder(lib, path)
-	_, err := r.put(folder.ID, folder)
+	_, err := r.put("id", folder.ID, folder)
 	return err
 }
 

persistence/mediafile_repository.go

@@ -2,6 +2,8 @@ package persistence
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -21,7 +23,51 @@ type mediaFileRepository struct {
 	sqlRestful
 }
 
-func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepository {
+type dbMediaFile struct {
+	*model.MediaFile `structs:",flatten"`
+	Tags             string `structs:"-" json:"tags"`
+}
+
+func (m *dbMediaFile) PostScan() error {
+	if m.Tags == "" {
+		m.MediaFile.Tags = make(map[string][]string)
+		return nil
+	}
+	err := json.Unmarshal([]byte(m.Tags), &m.MediaFile.Tags)
+	if err != nil {
+		return err
+	}
+	// Map genres from tags
+	for _, g := range m.MediaFile.Tags.Flatten("genre") {
+		m.MediaFile.Genres = append(m.MediaFile.Genres, model.Genre{Name: g.Value, ID: g.ID})
+	}
+	return nil
+}
+
+func (m *dbMediaFile) PostMapArgs(args map[string]any) error {
+	if len(m.MediaFile.Tags) == 0 {
+		args["tags"] = "{}"
+		return nil
+	}
+	b, err := json.Marshal(m.MediaFile.Tags)
+	if err != nil {
+		return err
+	}
+	args["tags"] = string(b)
+	return nil
+}
+
+type dbMediaFiles []dbMediaFile
+
+func (m *dbMediaFiles) toModels() model.MediaFiles {
+	res := make(model.MediaFiles, len(*m))
+	for i, mf := range *m {
+		res[i] = *mf.MediaFile
+	}
+	return res
+}
+
+func NewMediaFileRepository(ctx context.Context, db dbx.Builder) model.MediaFileRepository {
 	r := &mediaFileRepository{}
 	r.ctx = ctx
 	r.db = db
@@ -50,7 +96,8 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepos
 
 func (r *mediaFileRepository) CountAll(options ...model.QueryOptions) (int64, error) {
 	sql := r.newSelectWithAnnotation("media_file.id")
-	sql = r.withGenres(sql) // Required for filtering by genre
+	// FIXME Genres
+	//sql = r.withGenres(sql) // Required for filtering by genre
 	return r.count(sql, options...)
 }
 
@@ -59,73 +106,89 @@ func (r *mediaFileRepository) Exists(id string) (bool, error) {
 }
 
 func (r *mediaFileRepository) Put(m *model.MediaFile) error {
+	if m.ID == "" || m.PID == "" {
+		return errors.New("id and pid are required")
+	}
 	m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist,
 		m.SortTitle, m.SortAlbumName, m.SortArtistName, m.SortAlbumArtistName, m.DiscSubtitle)
-	_, err := r.put(m.ID, m)
+	_, err := r.put("pid", m.PID, &dbMediaFile{MediaFile: m})
 	if err != nil {
 		return err
 	}
-	return r.updateGenres(m.ID, m.Genres)
+
+	return r.updateTags(m.ID, m.Tags)
+
+	// FIXME Genres
+	//if err != nil {
+	//	return err
+	//}
+	//return r.updateGenres(m.ID, r.tableName, m.Genres)
 }
 
 func (r *mediaFileRepository) selectMediaFile(options ...model.QueryOptions) SelectBuilder {
 	sql := r.newSelectWithAnnotation("media_file.id", options...).Columns("media_file.*")
 	sql = r.withBookmark(sql, "media_file.id")
-	if len(options) > 0 && options[0].Filters != nil {
+	// FIXME Genres
-		s, _, _ := options[0].Filters.ToSql()
+	//if len(options) > 0 && options[0].Filters != nil {
-		// If there's any reference of genre in the filter, joins with genre
+	//	s, _, _ := options[0].Filters.ToSql()
-		if strings.Contains(s, "genre") {
+	//	// If there's any reference of genre in the filter, joins with genre
-			sql = r.withGenres(sql)
+	//	if strings.Contains(s, "genre") {
-			// If there's no filter on genre_id, group the results by media_file.id
+	//		sql = r.withGenres(sql)
-			if !strings.Contains(s, "genre_id") {
+	//		// If there's no filter on genre_id, group the results by media_file.id
-				sql = sql.GroupBy("media_file.id")
+	//		if !strings.Contains(s, "genre_id") {
-			}
+	//			sql = sql.GroupBy("media_file.id")
-		}
+	//		}
-	}
+	//	}
+	//}
 	return sql
 }
 
 func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) {
 	sel := r.selectMediaFile().Where(Eq{"media_file.id": id})
-	var res model.MediaFiles
+	var res dbMediaFiles
 	if err := r.queryAll(sel, &res); err != nil {
 		return nil, err
 	}
 	if len(res) == 0 {
 		return nil, model.ErrNotFound
 	}
-	err := loadAllGenres(r, res)
+	// FIXME Genres
-	return &res[0], err
+	//err := r.loadMediaFileGenres(&res)
+	return res[0].MediaFile, nil
 }
 
 func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) {
 	sq := r.selectMediaFile(options...)
-	res := model.MediaFiles{}
+	var res dbMediaFiles
 	err := r.queryAll(sq, &res, options...)
 	if err != nil {
 		return nil, err
 	}
-	err = loadAllGenres(r, res)
+	// FIXME Genres
-	return res, err
+	//err = r.loadMediaFileGenres(&rows)
+	return res.toModels(), nil
 }
 
 func (r *mediaFileRepository) GetByFolder(folderID string) (model.MediaFiles, error) {
 	sq := r.newSelect().Columns("*").Where(Eq{"folder_id": folderID})
-	res := model.MediaFiles{}
+	var res dbMediaFiles
 	err := r.queryAll(sq, &res)
-	return res, err
+	if err != nil {
+		return nil, err
+	}
+	return res.toModels(), nil
 }
 
 func (r *mediaFileRepository) FindByPath(path string) (*model.MediaFile, error) {
 	sel := r.newSelect().Columns("*").Where(Like{"path": path})
-	var res model.MediaFiles
+	var res dbMediaFiles
 	if err := r.queryAll(sel, &res); err != nil {
 		return nil, err
 	}
 	if len(res) == 0 {
 		return nil, model.ErrNotFound
 	}
-	return &res[0], nil
+	return res[0].MediaFile, nil
 }
 
 func cleanPath(path string) string {
@@ -151,9 +214,9 @@ func (r *mediaFileRepository) FindAllByPath(path string) (model.MediaFiles, erro
 	sel := r.newSelect().Columns("*", "item NOT GLOB '*"+string(os.PathSeparator)+"*' AS isLast").
 		Where(Eq{"isLast": 1}).FromSelect(sel0, "sel0")
 
-	res := model.MediaFiles{}
+	res := dbMediaFiles{}
 	err := r.queryAll(sel, &res)
-	return res, err
+	return res.toModels(), err
 }
 
 // FindPathsRecursively returns a list of all subfolders of basePath, recursively
@@ -202,13 +265,14 @@ func (r *mediaFileRepository) removeNonAlbumArtistIds() error {
 }
 
 func (r *mediaFileRepository) Search(q string, offset int, size int) (model.MediaFiles, error) {
-	results := model.MediaFiles{}
+	results := dbMediaFiles{}
 	err := r.doSearch(q, offset, size, &results, "title")
 	if err != nil {
 		return nil, err
 	}
-	err = loadAllGenres(r, results)
+	// FIXME Genres
-	return results, err
+	//err = r.loadMediaFileGenres(&results)
+	return results.toModels(), err
 }
 
 func (r *mediaFileRepository) Count(options ...rest.QueryOptions) (int64, error) {

persistence/persistence.go

@@ -43,6 +43,10 @@ func (s *SQLStore) Genre(ctx context.Context) model.GenreRepository {
 	return NewGenreRepository(ctx, s.getDBXBuilder())
 }
 
+func (s *SQLStore) Tag(ctx context.Context) model.TagRepository {
+	return NewTagRepository(ctx, s.getDBXBuilder())
+}
+
 func (s *SQLStore) PlayQueue(ctx context.Context) model.PlayQueueRepository {
 	return NewPlayQueueRepository(ctx, s.getDBXBuilder())
 }

persistence/player_repository.go

@@ -27,7 +27,7 @@ func NewPlayerRepository(ctx context.Context, db dbx.Builder) model.PlayerReposi
 }
 
 func (r *playerRepository) Put(p *model.Player) error {
-	_, err := r.put(p.ID, p)
+	_, err := r.put("id", p.ID, p)
 	return err
 }
 
@@ -102,7 +102,7 @@ func (r *playerRepository) Save(entity interface{}) (string, error) {
 	if !r.isPermitted(t) {
 		return "", rest.ErrPermissionDenied
 	}
-	id, err := r.put(t.ID, t)
+	id, err := r.put("id", t.ID, t)
 	if errors.Is(err, model.ErrNotFound) {
 		return "", rest.ErrNotFound
 	}
@@ -115,7 +115,7 @@ func (r *playerRepository) Update(id string, entity interface{}, cols ...string)
 	if !r.isPermitted(t) {
 		return rest.ErrPermissionDenied
 	}
-	_, err := r.put(id, t, cols...)
+	_, err := r.put("id", id, t, cols...)
 	if errors.Is(err, model.ErrNotFound) {
 		return rest.ErrNotFound
 	}

persistence/playlist_repository.go

@@ -118,7 +118,7 @@ func (r *playlistRepository) Put(p *model.Playlist) error {
 	}
 	pls.UpdatedAt = time.Now()
 
-	id, err := r.put(pls.ID, pls)
+	id, err := r.put("id", pls.ID, pls)
 	if err != nil {
 		return err
 	}
@@ -417,7 +417,7 @@ func (r *playlistRepository) Update(id string, entity interface{}, cols ...strin
 	}
 	pls.ID = id
 	pls.UpdatedAt = time.Now()
-	_, err = r.put(id, pls, append(cols, "updatedAt")...)
+	_, err = r.put("id", id, pls, append(cols, "updatedAt")...)
 	if errors.Is(err, model.ErrNotFound) {
 		return rest.ErrNotFound
 	}

persistence/playqueue_repository.go

@@ -47,7 +47,7 @@ func (r *playQueueRepository) Store(q *model.PlayQueue) error {
 		pq.CreatedAt = time.Now()
 	}
 	pq.UpdatedAt = time.Now()
-	_, err = r.put(pq.ID, pq)
+	_, err = r.put("id", pq.ID, pq)
 	if err != nil {
 		log.Error(r.ctx, "Error saving playqueue", "user", u.UserName, err)
 		return err

persistence/share_repository.go

@@ -138,7 +138,7 @@ func (r *shareRepository) Update(id string, entity interface{}, cols ...string)
 	s.ID = id
 	s.UpdatedAt = time.Now()
 	cols = append(cols, "updated_at")
-	_, err := r.put(id, s, cols...)
+	_, err := r.put("id", id, s, cols...)
 	if errors.Is(err, model.ErrNotFound) {
 		return rest.ErrNotFound
 	}
@@ -154,7 +154,7 @@ func (r *shareRepository) Save(entity interface{}) (string, error) {
 	}
 	s.CreatedAt = time.Now()
 	s.UpdatedAt = time.Now()
-	id, err := r.put(s.ID, s)
+	id, err := r.put("id", s.ID, s)
 	if errors.Is(err, model.ErrNotFound) {
 		return "", rest.ErrNotFound
 	}

persistence/sql_base_repository.go

@@ -241,10 +241,10 @@ func (r sqlRepository) count(countQuery SelectBuilder, options ...model.QueryOpt
 	return res.Count, err
 }
 
-func (r sqlRepository) put(id string, m interface{}, colsToUpdate ...string) (newId string, err error) {
+func (r sqlRepository) put(idCol string, idVal string, m interface{}, colsToUpdate ...string) (newId string, err error) {
 	values, _ := toSQLArgs(m)
 	// If there's an ID, try to update first
-	if id != "" {
+	if idVal != "" {
 		updateValues := map[string]interface{}{}
 
 		// This is a map of the columns that need to be updated, if specified
@@ -259,23 +259,23 @@ func (r sqlRepository) put(id string, m interface{}, colsToUpdate ...string) (ne
 		}
 
 		delete(updateValues, "created_at")
-		update := Update(r.tableName).Where(Eq{"id": id}).SetMap(updateValues)
+		update := Update(r.tableName).Where(Eq{idCol: idVal}).SetMap(updateValues)
 		count, err := r.executeSQL(update)
 		if err != nil {
 			return "", err
 		}
 		if count > 0 {
-			return id, nil
+			return idVal, nil
 		}
 	}
 	// If it does not have an ID OR the ID was not found (when it is a new record with predefined id)
-	if id == "" {
+	if idVal == "" {
-		id = uuid.NewString()
+		idVal = uuid.NewString()
-		values["id"] = id
+		values[idCol] = idVal
 	}
 	insert := Insert(r.tableName).SetMap(values)
 	_, err = r.executeSQL(insert)
-	return id, err
+	return idVal, err
 }
 
 func (r sqlRepository) delete(cond Sqlizer) error {

persistence/tag_repository.go

@@ -0,0 +1,52 @@
+package persistence
+
+import (
+	"context"
+
+	. "github.com/Masterminds/squirrel"
+	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/utils/slice"
+	"github.com/pocketbase/dbx"
+)
+
+type tagRepository struct {
+	sqlRepository
+}
+
+func NewTagRepository(ctx context.Context, db dbx.Builder) model.TagRepository {
+	r := &tagRepository{}
+	r.ctx = ctx
+	r.db = db
+	r.tableName = "tag"
+	return r
+}
+
+func (r *tagRepository) Add(tags ...model.Tag) error {
+	return slice.RangeByChunks(tags, 200, func(chunk []model.Tag) error {
+		sq := Insert(r.tableName).Columns("id", "name", "value").
+			Suffix("on conflict (id) do nothing")
+		for _, t := range chunk {
+			sq = sq.Values(t.ID, t.Name, t.Value)
+		}
+		_, err := r.executeSQL(sq)
+		return err
+	})
+}
+
+func (r *sqlRepository) updateTags(itemID string, tags model.Tags) error {
+	sqd := Delete("item_tags").Where(Eq{"item_id": itemID, "item_type": r.tableName})
+	_, err := r.executeSQL(sqd)
+	if err != nil {
+		return err
+	}
+	sqi := Insert("item_tags").Columns("item_id", "item_type", "tag_name", "tag_id").
+		Suffix("on conflict (item_id, item_type, tag_id) do nothing")
+	for name, values := range tags {
+		for _, value := range values {
+			tag := model.NewTag(name, value)
+			sqi = sqi.Values(itemID, r.tableName, tag.Name, tag.ID)
+		}
+	}
+	_, err = r.executeSQL(sqi)
+	return err
+}

persistence/transcoding_repository.go

@@ -42,7 +42,7 @@ func (r *transcodingRepository) FindByFormat(format string) (*model.Transcoding,
 }
 
 func (r *transcodingRepository) Put(t *model.Transcoding) error {
-	_, err := r.put(t.ID, t)
+	_, err := r.put("id", t.ID, t)
 	return err
 }
 
@@ -71,7 +71,7 @@ func (r *transcodingRepository) NewInstance() interface{} {
 
 func (r *transcodingRepository) Save(entity interface{}) (string, error) {
 	t := entity.(*model.Transcoding)
-	id, err := r.put(t.ID, t)
+	id, err := r.put("id", t.ID, t)
 	if errors.Is(err, model.ErrNotFound) {
 		return "", rest.ErrNotFound
 	}
@@ -81,7 +81,7 @@ func (r *transcodingRepository) Save(entity interface{}) (string, error) {
 func (r *transcodingRepository) Update(id string, entity interface{}, cols ...string) error {
 	t := entity.(*model.Transcoding)
 	t.ID = id
-	_, err := r.put(id, t)
+	_, err := r.put("id", id, t)
 	if errors.Is(err, model.ErrNotFound) {
 		return rest.ErrNotFound
 	}

scanner/metadata/metadata.go

@@ -231,7 +231,7 @@ func (t Tags) BirthTime() time.Time {
 	return time.Now()
 }
 
-// ReplayGain Properties
+// ReplayGain Properties TODO: check rg_* tags
 
 func (t Tags) RGAlbumGain() float64 { return t.getGainValue("replaygain_album_gain") }
 func (t Tags) RGAlbumPeak() float64 { return t.getPeakValue("replaygain_album_peak") }
@@ -385,3 +385,68 @@ func (t Tags) getFloat(tagNames ...string) float64 {
 	}
 	return value
 }
+
+// We exclude all tags that are already first-class citizens in the model
+var excludedTags = map[string]struct{}{
+	"duration":     {},
+	"bitrate":      {},
+	"channels":     {},
+	"bpm":          {},
+	"has_picture":  {},
+	"title":        {},
+	"album":        {},
+	"artist":       {},
+	"artists":      {},
+	"albumartist":  {},
+	"albumartists": {},
+	"track":        {},
+	"tracknumber":  {},
+	"tracktotal":   {},
+	"totaltracks":  {},
+	"disc":         {},
+	"discnumber":   {},
+	"disctotal":    {},
+	"totaldiscs":   {},
+	"lyrics":       {},
+	"year":         {},
+	"date":         {},
+	"originaldate": {},
+	"releasedate":  {},
+	"comment":      {},
+}
+
+// Also exclude any tag that starts with one of these prefixes
+var excludedPrefixes = []string{
+	"musicbrainz",
+	"replaygain",
+	"sort",
+}
+
+func isExcludedTag(tagName string) bool {
+	if _, ok := excludedTags[tagName]; ok {
+		return true
+	}
+	for _, prefix := range excludedPrefixes {
+		if strings.HasPrefix(tagName, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func (t Tags) ModelTags() model.Tags {
+	models := model.Tags{}
+	for tagName, values := range t.tags {
+		if isExcludedTag(tagName) {
+			continue
+		}
+
+		for _, value := range values {
+			if value == "" {
+				continue
+			}
+			models[tagName] = append(models[tagName], value)
+		}
+	}
+	return models
+}

scanner/metadata/taglib/taglib.go

@@ -50,6 +50,7 @@ func (e *Extractor) extractMetadata(filePath string) (metadata.ParsedTags, error
 		if duration := float64(millis) / 1000.0; duration > 0 {
 			tags["duration"] = []string{strconv.FormatFloat(duration, 'f', 2, 32)}
 		}
+		delete(tags, "lengthinmilliseconds")
 	}
 	// Adjust some ID3 tags
 	parseTIPL(tags)

scanner2/folder.go

@@ -20,6 +20,7 @@ type folderEntry struct {
 	tracks          model.MediaFiles
 	albums          model.Albums
 	artists         model.Artists
+	tags            model.FlattenedTags
 	missingTracks   model.MediaFiles
 }
 

scanner2/mapping.go

@@ -16,12 +16,12 @@ import (
 )
 
 type mediaFileMapper struct {
-	rootFolder string
+	entry *folderEntry
 }
 
 func newMediaFileMapper(entry *folderEntry) *mediaFileMapper {
 	return &mediaFileMapper{
-		rootFolder: entry.path,
+		entry: entry,
 	}
 }
 
@@ -74,6 +74,10 @@ func (s mediaFileMapper) toMediaFile(md metadata.Tags) model.MediaFile {
 	mf.Bpm = md.Bpm()
 	mf.CreatedAt = md.BirthTime()
 	mf.UpdatedAt = md.ModificationTime()
+	mf.Tags = md.ModelTags()
+	mf.FolderID = s.entry.id
+	mf.LibraryID = s.entry.scanCtx.lib.ID
+	mf.PID = mf.ID
 
 	return *mf
 }
@@ -85,7 +89,7 @@ func sanitizeFieldForSorting(originalValue string) string {
 
 func (s mediaFileMapper) mapTrackTitle(md metadata.Tags) string {
 	if md.Title() == "" {
-		s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
+		s := strings.TrimPrefix(md.FilePath(), s.entry.path+string(os.PathSeparator))
 		e := filepath.Ext(s)
 		return strings.TrimSuffix(s, e)
 	}

scanner2/persiste_changes.go

@@ -0,0 +1,47 @@
+package scanner2
+
+import (
+	"context"
+
+	"github.com/google/go-pipeline/pkg/pipeline"
+	"github.com/navidrome/navidrome/log"
+	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/utils/slice"
+)
+
+func persistChanges(ctx context.Context) pipeline.StageFn[*folderEntry] {
+	return func(entry *folderEntry) (*folderEntry, error) {
+		err := entry.scanCtx.ds.WithTx(func(tx model.DataStore) error {
+			// Save all tags to DB
+			err := slice.RangeByChunks(entry.tags, 100, func(chunk []model.Tag) error {
+				err := tx.Tag(ctx).Add(chunk...)
+				if err != nil {
+					log.Error(ctx, "Scanner: Error adding tags to DB", "folder", entry.path, err)
+					return err
+				}
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+
+			// Save all tracks to DB
+			err = slice.RangeByChunks(entry.tracks, 100, func(chunk []model.MediaFile) error {
+				for i := range chunk {
+					track := chunk[i]
+					err = tx.MediaFile(ctx).Put(&track)
+					if err != nil {
+						log.Error(ctx, "Scanner: Error adding mediafile to DB", "folder", entry.path, "track", track, err)
+						return err
+					}
+				}
+				return nil
+			})
+			return err
+		})
+		if err != nil {
+			log.Error(ctx, "Scanner: Error persisting changes to DB", "folder", entry.path, err)
+		}
+		return entry, err
+	}
+}

scanner2/persistent_ids.go

@@ -1,33 +1 @@
 package scanner2
-
-import (
-	"crypto/md5"
-	"fmt"
-
-	"github.com/navidrome/navidrome/scanner/metadata"
-	. "github.com/navidrome/navidrome/utils/gg"
-)
-
-func artistPID(md metadata.Tags) string {
-	key := FirstOr(md.Artist(), "M"+md.MbzArtistID())
-	return fmt.Sprintf("%x", md5.Sum([]byte(key)))
-}
-
-func albumArtistPID(md metadata.Tags) string {
-	key := FirstOr(md.AlbumArtist(), "M"+md.MbzAlbumArtistID())
-	return fmt.Sprintf("%x", md5.Sum([]byte(key)))
-}
-
-func albumPID(md metadata.Tags) string {
-	var key string
-	if md.MbzAlbumID() != "" {
-		key = "M" + md.MbzAlbumID()
-	} else {
-		key = fmt.Sprintf("%s%s%t", albumArtistPID(md), md.Album(), md.Compilation())
-	}
-	return fmt.Sprintf("%x", md5.Sum([]byte(key)))
-}
-
-func trackPID(md metadata.Tags) string {
-	return fmt.Sprintf("%s%x", albumPID(md), md5.Sum([]byte(md.FilePath())))
-}

scanner2/process_folders.go

@@ -50,35 +50,41 @@ func processFolder(ctx context.Context) pipeline.StageFn[*folderEntry] {
 		// Remaining dbTracks are tracks that were not found in the folder, so they should be marked as missing
 		entry.missingTracks = maps.Values(dbTracks)
 
-		entry.tracks, err = loadTagsFromFiles(ctx, entry, filesToImport)
+		if len(filesToImport) > 0 {
-		if err != nil {
+			entry.tracks, entry.tags, err = loadTagsFromFiles(ctx, entry, filesToImport)
-			log.Warn(ctx, "Scanner: Error loading tags from files. Skipping", "folder", entry.path, err)
+			if err != nil {
-			return entry, nil
+				log.Warn(ctx, "Scanner: Error loading tags from files. Skipping", "folder", entry.path, err)
-		}
+				return entry, nil
+			}
 
-		entry.albums = loadAlbumsFromTags(ctx, entry)
+			entry.albums = loadAlbumsFromTags(ctx, entry)
-		entry.artists = loadArtistsFromTags(ctx, entry)
+			entry.artists = loadArtistsFromTags(ctx, entry)
+		}
 
 		return entry, nil
 	}
 }
 
-func loadTagsFromFiles(ctx context.Context, entry *folderEntry, toImport []string) (model.MediaFiles, error) {
+func loadTagsFromFiles(ctx context.Context, entry *folderEntry, toImport []string) (model.MediaFiles, model.FlattenedTags, error) {
 	tracks := model.MediaFiles{}
+	uniqueTags := make(map[string]model.Tag)
 	mapper := newMediaFileMapper(entry)
 	err := slice.RangeByChunks(toImport, filesBatchSize, func(chunk []string) error {
-		allTags, err := metadata.Extract(toImport...)
+		allFileTags, err := metadata.Extract(toImport...)
 		if err != nil {
 			log.Warn(ctx, "Scanner: Error extracting tags from files. Skipping", "folder", entry.path, err)
 			return err
 		}
-		for _, tags := range allTags {
+		for _, fileTags := range allFileTags {
-			track := mapper.toMediaFile(tags)
+			track := mapper.toMediaFile(fileTags)
 			tracks = append(tracks, track)
+			for _, t := range track.Tags.FlattenAll() {
+				uniqueTags[t.ID] = t
+			}
 		}
 		return nil
 	})
-	return tracks, err
+	return tracks, maps.Values(uniqueTags), err
 }
 
 func loadAlbumsFromTags(ctx context.Context, entry *folderEntry) model.Albums {

scanner2/scanner2.go

@@ -34,6 +34,7 @@ func (s *scanner2) RescanAll(requestCtx context.Context, fullRescan bool) error
 	err = s.runPipeline(
 		pipeline.NewProducer(produceFolders(ctx, s.ds, libs, fullRescan), pipeline.Name("read folders from disk")),
 		pipeline.NewStage(processFolder(ctx), pipeline.Name("process folder")),
+		pipeline.NewStage(persistChanges(ctx), pipeline.Name("persist changes")),
 		pipeline.NewStage(logFolder(ctx), pipeline.Name("log results")),
 	)
 

tests/mock_persistence.go

@@ -51,6 +51,10 @@ func (db *MockDataStore) Folder(context.Context) model.FolderRepository {
 	return struct{ model.FolderRepository }{}
 }
 
+func (db *MockDataStore) Tag(context.Context) model.TagRepository {
+	return struct{ model.TagRepository }{}
+}
+
 func (db *MockDataStore) Genre(context.Context) model.GenreRepository {
 	if db.MockedGenre == nil {
 		db.MockedGenre = &MockedGenreRepo{}