Associate main entities with library

scanner/refresher.go

@@ -12,7 +12,6 @@ import (
 	"github.com/navidrome/navidrome/core/artwork"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
-	. "github.com/navidrome/navidrome/utils/gg"
 	"github.com/navidrome/navidrome/utils/slice"
 	"golang.org/x/exp/maps"
 )
@@ -24,15 +23,17 @@ import (
 // The actual mappings happen in MediaFiles.ToAlbum() and Albums.ToAlbumArtist()
 type refresher struct {
 	ds          model.DataStore
+	lib         model.Library
 	album       map[string]struct{}
 	artist      map[string]struct{}
 	dirMap      dirMap
 	cacheWarmer artwork.CacheWarmer
 }
 
-func newRefresher(ds model.DataStore, cw artwork.CacheWarmer, dirMap dirMap) *refresher {
+func newRefresher(ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, dirMap dirMap) *refresher {
 	return &refresher{
 		ds:          ds,
+		lib:         lib,
 		album:       map[string]struct{}{},
 		artist:      map[string]struct{}{},
 		dirMap:      dirMap,
@@ -101,6 +102,7 @@ func (r *refresher) refreshAlbums(ctx context.Context, ids ...string) error {
 		if updatedAt.After(a.UpdatedAt) {
 			a.UpdatedAt = updatedAt
 		}
+		a.LibraryID = r.lib.ID
 		err := repo.Put(&a)
 		if err != nil {
 			return err
@@ -135,18 +137,25 @@ func (r *refresher) refreshArtists(ctx context.Context, ids ...string) error {
 	}
 
 	repo := r.ds.Artist(ctx)
+	libRepo := r.ds.Library(ctx)
 	grouped := slice.Group(albums, func(al model.Album) string { return al.AlbumArtistID })
 	for _, group := range grouped {
 		a := model.Albums(group).ToAlbumArtist()
 
-		// Force a external metadata lookup on next access
-		a.ExternalInfoUpdatedAt = P(time.Time{})
+		// Force an external metadata lookup on next access
+		a.ExternalInfoUpdatedAt = &time.Time{}
 
 		// Do not remove old metadata
 		err := repo.Put(&a, "album_count", "genres", "external_info_updated_at", "mbz_artist_id", "name", "order_artist_name", "size", "sort_artist_name", "song_count")
 		if err != nil {
 			return err
 		}
+
+		// Link the artist to the current library being scanned
+		err = libRepo.AddArtist(r.lib.ID, a.ID)
+		if err != nil {
+			return err
+		}
 		r.cacheWarmer.PreCache(a.CoverArtID())
 	}
 	return nil

scanner/scanner.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strconv"
 	"sync"
 	"time"
 
@@ -36,13 +35,15 @@ var (
 
 type FolderScanner interface {
 	// Scan process finds any changes after `lastModifiedSince` and returns the number of changes found
-	Scan(ctx context.Context, lastModifiedSince time.Time, progress chan uint32) (int64, error)
+	Scan(ctx context.Context, fullRescan bool, progress chan uint32) (int64, error)
 }
 
 var isScanning sync.Mutex
 
 type scanner struct {
+	once        sync.Once
 	folders     map[string]FolderScanner
+	libs        map[string]model.Library
 	status      map[string]*scanStatus
 	lock        *sync.RWMutex
 	ds          model.DataStore
@@ -65,6 +66,7 @@ func GetInstance(ds model.DataStore, playlists core.Playlists, cacheWarmer artwo
 			pls:         playlists,
 			broker:      broker,
 			folders:     map[string]FolderScanner{},
+			libs:        map[string]model.Library{},
 			status:      map[string]*scanStatus{},
 			lock:        &sync.RWMutex{},
 			cacheWarmer: cacheWarmer,
@@ -78,21 +80,25 @@ func (s *scanner) rescan(ctx context.Context, library string, fullRescan bool) e
 	folderScanner := s.folders[library]
 	start := time.Now()
 
+	lib, ok := s.libs[library]
+	if !ok {
+		log.Error(ctx, "Folder not a valid library path", "folder", library)
+		return fmt.Errorf("folder %s not a valid library path", library)
+	}
+
 	s.setStatusStart(library)
 	defer s.setStatusEnd(library, start)
 
-	lastModifiedSince := time.Time{}
-	if !fullRescan {
-		lastModifiedSince = s.getLastModifiedSince(ctx, library)
-		log.Debug("Scanning folder", "folder", library, "lastModifiedSince", lastModifiedSince)
-	} else {
+	if fullRescan {
 		log.Debug("Scanning folder (full scan)", "folder", library)
+	} else {
+		log.Debug("Scanning folder", "folder", library, "lastScan", lib.LastScanAt)
 	}
 
 	progress, cancel := s.startProgressTracker(library)
 	defer cancel()
 
-	changeCount, err := folderScanner.Scan(ctx, lastModifiedSince, progress)
+	changeCount, err := folderScanner.Scan(ctx, fullRescan, progress)
 	if err != nil {
 		log.Error("Error scanning Library", "folder", library, err)
 	}
@@ -104,11 +110,12 @@ func (s *scanner) rescan(ctx context.Context, library string, fullRescan bool) e
 		s.broker.SendMessage(context.Background(), &events.RefreshResource{})
 	}
 
-	s.updateLastModifiedSince(library, start)
+	s.updateLastModifiedSince(ctx, library, start)
 	return err
 }
 
 func (s *scanner) startProgressTracker(library string) (chan uint32, context.CancelFunc) {
+	// Must be a new context (not the one passed to the scan method) to allow broadcasting the scan status to all clients
 	ctx, cancel := context.WithCancel(context.Background())
 	progress := make(chan uint32, 100)
 	go func() {
@@ -182,6 +189,8 @@ func (s *scanner) setStatusEnd(folder string, lastUpdate time.Time) {
 
 func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error {
 	ctx = context.WithoutCancel(ctx)
+	s.once.Do(s.loadFolders)
+
 	if !isScanning.TryLock() {
 		log.Debug(ctx, "Scanner already running, ignoring request for rescan.")
 		return ErrAlreadyScanning
@@ -203,6 +212,7 @@ func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error {
 }
 
 func (s *scanner) Status(library string) (*StatusInfo, error) {
+	s.once.Do(s.loadFolders)
 	status, ok := s.getStatus(library)
 	if !ok {
 		return nil, errors.New("library not found")
@@ -216,40 +226,32 @@ func (s *scanner) Status(library string) (*StatusInfo, error) {
 	}, nil
 }
 
-func (s *scanner) getLastModifiedSince(ctx context.Context, folder string) time.Time {
-	ms, err := s.ds.Property(ctx).Get(model.PropLastScan + "-" + folder)
-	if err != nil {
-		return time.Time{}
-	}
-	if ms == "" {
-		return time.Time{}
-	}
-	i, _ := strconv.ParseInt(ms, 10, 64)
-	return time.Unix(0, i*int64(time.Millisecond))
-}
-
-func (s *scanner) updateLastModifiedSince(folder string, t time.Time) {
-	millis := t.UnixNano() / int64(time.Millisecond)
-	if err := s.ds.Property(context.TODO()).Put(model.PropLastScan+"-"+folder, fmt.Sprint(millis)); err != nil {
+func (s *scanner) updateLastModifiedSince(ctx context.Context, folder string, t time.Time) {
+	lib := s.libs[folder]
+	id := lib.ID
+	if err := s.ds.Library(ctx).UpdateLastScan(id, t); err != nil {
 		log.Error("Error updating DB after scan", err)
 	}
+	lib.LastScanAt = t
+	s.libs[folder] = lib
 }
 
 func (s *scanner) loadFolders() {
 	ctx := context.TODO()
-	fs, _ := s.ds.Library(ctx).GetAll()
-	for _, f := range fs {
-		log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path)
-		s.folders[f.Path] = s.newScanner(f)
-		s.status[f.Path] = &scanStatus{
+	libs, _ := s.ds.Library(ctx).GetAll()
+	for _, lib := range libs {
+		log.Info("Configuring Media Folder", "name", lib.Name, "path", lib.Path)
+		s.folders[lib.Path] = s.newScanner(lib)
+		s.libs[lib.Path] = lib
+		s.status[lib.Path] = &scanStatus{
 			active:      false,
 			fileCount:   0,
 			folderCount: 0,
-			lastUpdate:  s.getLastModifiedSince(ctx, f.Path),
+			lastUpdate:  lib.LastScanAt,
 		}
 	}
 }
 
 func (s *scanner) newScanner(f model.Library) FolderScanner {
-	return NewTagScanner(f.Path, s.ds, s.pls, s.cacheWarmer)
+	return NewTagScanner(f, s.ds, s.pls, s.cacheWarmer)
 }

scanner/tag_scanner.go

@@ -23,7 +23,7 @@ import (
 )
 
 type TagScanner struct {
-	rootFolder  string
+	lib         model.Library
 	ds          model.DataStore
 	plsSync     *playlistImporter
 	cnt         *counters
@@ -31,10 +31,10 @@ type TagScanner struct {
 	cacheWarmer artwork.CacheWarmer
 }
 
-func NewTagScanner(rootFolder string, ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.CacheWarmer) FolderScanner {
+func NewTagScanner(lib model.Library, ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.CacheWarmer) FolderScanner {
 	s := &TagScanner{
-		rootFolder:  rootFolder,
-		plsSync:     newPlaylistImporter(ds, playlists, cacheWarmer, rootFolder),
+		lib:         lib,
+		plsSync:     newPlaylistImporter(ds, playlists, cacheWarmer, lib.Path),
 		ds:          ds,
 		cacheWarmer: cacheWarmer,
 	}
@@ -75,20 +75,20 @@ const (
 // - If the playlist is not in the DB, import it, setting sync = true
 // - If the playlist is in the DB and sync == true, import it, or else skip it
 // Delete all empty albums, delete all empty artists, clean-up playlists
-func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, progress chan uint32) (int64, error) {
+func (s *TagScanner) Scan(ctx context.Context, fullScan bool, progress chan uint32) (int64, error) {
 	ctx = auth.WithAdminUser(ctx, s.ds)
 	start := time.Now()
 
-	// Special case: if lastModifiedSince is zero, re-import all files
-	fullScan := lastModifiedSince.IsZero()
+	// Special case: if LastScanAt is zero, re-import all files
+	fullScan = fullScan || s.lib.LastScanAt.IsZero()
 
 	// If the media folder is empty (no music and no subfolders), abort to avoid deleting all data from DB
-	empty, err := isDirEmpty(ctx, s.rootFolder)
+	empty, err := isDirEmpty(ctx, s.lib.Path)
 	if err != nil {
 		return 0, err
 	}
 	if empty && !fullScan {
-		log.Error(ctx, "Media Folder is empty. Aborting scan.", "folder", s.rootFolder)
+		log.Error(ctx, "Media Folder is empty. Aborting scan.", "folder", s.lib.Path)
 		return 0, nil
 	}
 
@@ -101,38 +101,36 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog
 	var changedDirs []string
 	s.cnt = &counters{}
 	genres := newCachedGenreRepository(ctx, s.ds.Genre(ctx))
-	s.mapper = NewMediaFileMapper(s.rootFolder, genres)
-	refresher := newRefresher(s.ds, s.cacheWarmer, allFSDirs)
+	s.mapper = NewMediaFileMapper(s.lib.Path, genres)
+	refresher := newRefresher(s.ds, s.cacheWarmer, s.lib, allFSDirs)
 
-	log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
-	foldersFound, walkerError := walkDirTree(ctx, s.rootFolder)
+	log.Trace(ctx, "Loading directory tree from music folder", "folder", s.lib.Path)
+	foldersFound, walkerError := walkDirTree(ctx, s.lib.Path)
 
-	for {
-		folderStats, more := <-foldersFound
-		if !more {
-			break
-		}
-		progress <- folderStats.AudioFilesCount
-		allFSDirs[folderStats.Path] = folderStats
+	go func() {
+		for folderStats := range foldersFound {
+			progress <- folderStats.AudioFilesCount
+			allFSDirs[folderStats.Path] = folderStats
 
-		if s.folderHasChanged(folderStats, allDBDirs, lastModifiedSince) {
-			changedDirs = append(changedDirs, folderStats.Path)
-			log.Debug("Processing changed folder", "dir", folderStats.Path)
-			err := s.processChangedDir(ctx, refresher, fullScan, folderStats.Path)
-			if err != nil {
-				log.Error("Error updating folder in the DB", "dir", folderStats.Path, err)
+			if s.folderHasChanged(folderStats, allDBDirs, s.lib.LastScanAt) || fullScan {
+				changedDirs = append(changedDirs, folderStats.Path)
+				log.Debug("Processing changed folder", "dir", folderStats.Path)
+				err := s.processChangedDir(ctx, refresher, fullScan, folderStats.Path)
+				if err != nil {
+					log.Error("Error updating folder in the DB", "dir", folderStats.Path, err)
+				}
 			}
 		}
-	}
+	}()
 
-	if err := <-walkerError; err != nil {
+	for err := range walkerError {
 		log.Error("Scan was interrupted by error. See errors above", err)
 		return 0, err
 	}
 
 	deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
 	if len(deletedDirs)+len(changedDirs) == 0 {
-		log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start))
+		log.Debug(ctx, "No changes found in Music Folder", "folder", s.lib.Path, "elapsed", time.Since(start))
 		return 0, nil
 	}
 
@@ -162,8 +160,8 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog
 		log.Debug("Playlist auto-import is disabled")
 	}
 
-	err = s.ds.GC(log.NewContext(ctx), s.rootFolder)
-	log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
+	err = s.ds.GC(log.NewContext(ctx), s.lib.Path)
+	log.Info("Finished processing Music Folder", "folder", s.lib.Path, "elapsed", time.Since(start),
 		"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", s.cnt.playlists)
 
 	return s.cnt.total(), err
@@ -179,10 +177,10 @@ func isDirEmpty(ctx context.Context, dir string) (bool, error) {
 
 func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) {
 	start := time.Now()
-	log.Trace(ctx, "Loading directory tree from database", "folder", s.rootFolder)
+	log.Trace(ctx, "Loading directory tree from database", "folder", s.lib.Path)
 
 	repo := s.ds.MediaFile(ctx)
-	dirs, err := repo.FindPathsRecursively(s.rootFolder)
+	dirs, err := repo.FindPathsRecursively(s.lib.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -368,6 +366,7 @@ func (s *TagScanner) addOrUpdateTracksInDB(
 			if t, ok := currentTracks[n.Path]; ok {
 				n.Annotations = t.Annotations
 			}
+			n.LibraryID = s.lib.ID
 			err := s.ds.MediaFile(ctx).Put(&n)
 			if err != nil {
 				return 0, err

scanner/walk_dir_tree.go

@@ -26,7 +26,7 @@ type (
 	}
 )
 
-func walkDirTree(ctx context.Context, rootFolder string) (<-chan dirStats, chan error) {
+func walkDirTree(ctx context.Context, rootFolder string) (<-chan dirStats, <-chan error) {
 	results := make(chan dirStats)
 	errC := make(chan error)
 	go func() {