diff --git a/conf/configuration.go b/conf/configuration.go index f7e905f54..130c11dbe 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -40,7 +40,6 @@ type configOptions struct { // DevFlags. These are used to enable/disable debugging and incomplete features DevLogSourceLine bool DevAutoCreateAdminPassword string - DevOldScanner bool } var Server = &configOptions{} diff --git a/scanner/change_detector.go b/scanner/change_detector.go deleted file mode 100644 index 2ec943290..000000000 --- a/scanner/change_detector.go +++ /dev/null @@ -1,128 +0,0 @@ -package scanner - -import ( - "context" - "io/ioutil" - "os" - "path/filepath" - "time" - - "github.com/deluan/navidrome/log" -) - -type dirInfo struct { - mdate time.Time - maybe bool -} -type dirInfoMap map[string]dirInfo - -type changeDetector struct { - rootFolder string - dirMap dirInfoMap -} - -func newChangeDetector(rootFolder string) *changeDetector { - return &changeDetector{ - rootFolder: rootFolder, - dirMap: dirInfoMap{}, - } -} - -func (s *changeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) { - start := time.Now() - newMap := make(dirInfoMap) - err = s.loadMap(ctx, newMap, s.rootFolder, lastModifiedSince, false) - if err != nil { - return - } - changed, deleted, err = s.checkForUpdates(lastModifiedSince, newMap) - if err != nil { - return - } - elapsed := time.Since(start) - - log.Trace(ctx, "Folder analysis complete", "total", len(newMap), "changed", len(changed), "deleted", len(deleted), "elapsed", elapsed) - s.dirMap = newMap - return -} - -func (s *changeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) { - dirInfo, err := os.Stat(dirPath) - if err != nil { - log.Error(ctx, "Error stating dir", "path", dirPath, err) - return - } - lastUpdated = dirInfo.ModTime() - - files, err := ioutil.ReadDir(dirPath) - if err != nil { - log.Error(ctx, "Error reading dir", "path", dirPath, err) - return - } - for _, f := range files { - isDir, err := isDirOrSymlinkToDir(dirPath, f) - // Skip invalid symlinks - if err != nil { - continue - } - if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) { - children = append(children, filepath.Join(dirPath, f.Name())) - } else { - if f.ModTime().After(lastUpdated) { - lastUpdated = f.ModTime() - } - } - } - return -} - -func (s *changeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error { - children, lastUpdated, err := s.loadDir(ctx, path) - if err != nil { - return err - } - maybe = maybe || lastUpdated.After(since) - for _, c := range children { - err := s.loadMap(ctx, dirMap, c, since, maybe) - if err != nil { - return err - } - } - - dir := s.getRelativePath(path) - dirMap[dir] = dirInfo{mdate: lastUpdated, maybe: maybe} - - return nil -} - -func (s *changeDetector) getRelativePath(subFolder string) string { - dir, _ := filepath.Rel(s.rootFolder, subFolder) - if dir == "" { - dir = "." - } - return dir -} - -func (s *changeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) { - for dir, newEntry := range newMap { - lastUpdated := newEntry.mdate - oldLastUpdated := lastModifiedSince - if oldEntry, ok := s.dirMap[dir]; ok { - oldLastUpdated = oldEntry.mdate - } else { - if newEntry.maybe { - oldLastUpdated = time.Time{} - } - } - - if lastUpdated.After(oldLastUpdated) { - changed = append(changed, dir) - } - } - for dir := range s.dirMap { - if _, ok := newMap[dir]; !ok { - deleted = append(deleted, dir) - } - } - return -} diff --git a/scanner/change_detector_test.go b/scanner/change_detector_test.go deleted file mode 100644 index e12889acb..000000000 --- a/scanner/change_detector_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package scanner - -import ( - "context" - "io/ioutil" - "os" - "path/filepath" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("changeDetector", func() { - var testFolder string - var scanner *changeDetector - - lastModifiedSince := time.Time{} - - BeforeEach(func() { - testFolder, _ = ioutil.TempDir("", "navidrome_tests") - err := os.MkdirAll(testFolder, 0777) - if err != nil { - panic(err) - } - scanner = newChangeDetector(testFolder) - }) - - It("detects changes recursively", func() { - // Scan empty folder - changed, deleted, err := scanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(BeEmpty()) - Expect(changed).To(ConsistOf(".")) - - // Add one subfolder - lastModifiedSince = nowWithDelay() - err = os.MkdirAll(filepath.Join(testFolder, "a"), 0777) - if err != nil { - panic(err) - } - changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(BeEmpty()) - Expect(changed).To(ConsistOf(".", P("a"))) - - // Add more subfolders - lastModifiedSince = nowWithDelay() - err = os.MkdirAll(filepath.Join(testFolder, "a", "b", "c"), 0777) - if err != nil { - panic(err) - } - changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(BeEmpty()) - Expect(changed).To(ConsistOf(P("a"), P("a/b"), P("a/b/c"))) - - // Scan with no changes - lastModifiedSince = nowWithDelay() - changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(BeEmpty()) - Expect(changed).To(BeEmpty()) - - // New file in subfolder - lastModifiedSince = nowWithDelay() - _, err = os.Create(filepath.Join(testFolder, "a", "b", "empty.txt")) - if err != nil { - panic(err) - } - changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(BeEmpty()) - Expect(changed).To(ConsistOf(P("a/b"))) - - // Delete file in subfolder - lastModifiedSince = nowWithDelay() - err = os.Remove(filepath.Join(testFolder, "a", "b", "empty.txt")) - if err != nil { - panic(err) - } - changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(BeEmpty()) - Expect(changed).To(ConsistOf(P("a/b"))) - - // Delete subfolder - lastModifiedSince = nowWithDelay() - err = os.Remove(filepath.Join(testFolder, "a", "b", "c")) - if err != nil { - panic(err) - } - changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(ConsistOf(P("a/b/c"))) - Expect(changed).To(ConsistOf(P("a/b"))) - - // Only returns changes after lastModifiedSince - lastModifiedSince = nowWithDelay() - newScanner := newChangeDetector(testFolder) - changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(BeEmpty()) - Expect(changed).To(BeEmpty()) - Expect(changed).To(BeEmpty()) - - f, _ := os.Create(filepath.Join(testFolder, "a", "b", "new.txt")) - _ = f.Close() - changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince) - Expect(err).To(BeNil()) - Expect(deleted).To(BeEmpty()) - Expect(changed).To(ConsistOf(P("a/b"))) - }) - - Describe("isDirOrSymlinkToDir", func() { - It("returns true for normal dirs", func() { - dir, _ := os.Stat("tests/fixtures") - Expect(isDirOrSymlinkToDir("tests", dir)).To(BeTrue()) - }) - It("returns true for symlinks to dirs", func() { - dir, _ := os.Stat("tests/fixtures/symlink2dir") - Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeTrue()) - }) - It("returns false for files", func() { - dir, _ := os.Stat("tests/fixtures/test.mp3") - Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse()) - }) - It("returns false for symlinks to files", func() { - dir, _ := os.Stat("tests/fixtures/symlink") - Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse()) - }) - }) - - Describe("isDirIgnored", func() { - baseDir := filepath.Join("tests", "fixtures") - It("returns false for normal dirs", func() { - dir, _ := os.Stat(filepath.Join(baseDir, "empty_folder")) - Expect(isDirIgnored(baseDir, dir)).To(BeFalse()) - }) - It("returns true when folder contains .ndignore file", func() { - dir, _ := os.Stat(filepath.Join(baseDir, "ignored_folder")) - Expect(isDirIgnored(baseDir, dir)).To(BeTrue()) - }) - }) -}) - -// I hate time-based tests.... -func nowWithDelay() time.Time { - now := time.Now() - time.Sleep(50 * time.Millisecond) - return now -} diff --git a/scanner/load_tree_test.go b/scanner/load_tree_test.go new file mode 100644 index 000000000..fb16b1c3c --- /dev/null +++ b/scanner/load_tree_test.go @@ -0,0 +1,42 @@ +package scanner + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("load_tree", func() { + Describe("isDirOrSymlinkToDir", func() { + It("returns true for normal dirs", func() { + dir, _ := os.Stat("tests/fixtures") + Expect(isDirOrSymlinkToDir("tests", dir)).To(BeTrue()) + }) + It("returns true for symlinks to dirs", func() { + dir, _ := os.Stat("tests/fixtures/symlink2dir") + Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeTrue()) + }) + It("returns false for files", func() { + dir, _ := os.Stat("tests/fixtures/test.mp3") + Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse()) + }) + It("returns false for symlinks to files", func() { + dir, _ := os.Stat("tests/fixtures/symlink") + Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse()) + }) + }) + + Describe("isDirIgnored", func() { + baseDir := filepath.Join("tests", "fixtures") + It("returns false for normal dirs", func() { + dir, _ := os.Stat(filepath.Join(baseDir, "empty_folder")) + Expect(isDirIgnored(baseDir, dir)).To(BeFalse()) + }) + It("returns true when folder contains .ndignore file", func() { + dir, _ := os.Stat(filepath.Join(baseDir, "ignored_folder")) + Expect(isDirIgnored(baseDir, dir)).To(BeTrue()) + }) + }) +}) diff --git a/scanner/mapping_test.go b/scanner/mapping_test.go new file mode 100644 index 000000000..6b5d9dab8 --- /dev/null +++ b/scanner/mapping_test.go @@ -0,0 +1,21 @@ +package scanner + +import ( + "github.com/deluan/navidrome/conf" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("mapping", func() { + Describe("sanitizeFieldForSorting", func() { + BeforeEach(func() { + conf.Server.IgnoredArticles = "The" + }) + It("sanitize accents", func() { + Expect(sanitizeFieldForSorting("Céu")).To(Equal("Ceu")) + }) + It("removes articles", func() { + Expect(sanitizeFieldForSorting("The Beatles")).To(Equal("Beatles")) + }) + }) +}) diff --git a/scanner/scanner.go b/scanner/scanner.go index f9af24455..9d3b1ca0b 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -7,7 +7,6 @@ import ( "strconv" "time" - "github.com/deluan/navidrome/conf" "github.com/deluan/navidrome/log" "github.com/deluan/navidrome/model" ) @@ -87,10 +86,7 @@ func (s *Scanner) loadFolders() { } func (s *Scanner) newScanner(f model.MediaFolder) FolderScanner { - if conf.Server.DevOldScanner { - return NewTagScanner(f.Path, s.ds) - } - return NewTagScanner2(f.Path, s.ds) + return NewTagScanner(f.Path, s.ds) } type Status int diff --git a/scanner/scanner_suite_test.go b/scanner/scanner_suite_test.go index 3f2cf7db2..8e4d14907 100644 --- a/scanner/scanner_suite_test.go +++ b/scanner/scanner_suite_test.go @@ -1,7 +1,6 @@ package scanner import ( - "path/filepath" "testing" "github.com/deluan/navidrome/log" @@ -16,7 +15,3 @@ func TestScanner(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Scanner Suite") } - -func P(path string) string { - return filepath.FromSlash(path) -} diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index 3eb6fb016..7372f4ca1 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -6,38 +6,32 @@ import ( "path/filepath" "sort" "strings" - "sync" "time" "github.com/deluan/navidrome/log" "github.com/deluan/navidrome/model" + "github.com/deluan/navidrome/model/request" "github.com/deluan/navidrome/utils" ) type TagScanner struct { rootFolder string ds model.DataStore - detector *changeDetector mapper *mediaFileMapper - firstRun sync.Once + plsSync *playlistSync + cnt *counters } func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner { return &TagScanner{ rootFolder: rootFolder, - ds: ds, - detector: newChangeDetector(rootFolder), mapper: newMediaFileMapper(rootFolder), - firstRun: sync.Once{}, + plsSync: newPlaylistSync(ds), + ds: ds, } } -const batchSize = 5 - type ( - artistMap map[string]struct{} - albumMap map[string]struct{} - counters struct { added int64 updated int64 @@ -46,113 +40,181 @@ type ( ) const ( - // filesBatchSize used for extract file metadata + // filesBatchSize used for batching file metadata extraction filesBatchSize = 100 ) -// Scan algorithm overview: -// For each changed folder: Get all files from DB that starts with the folder, scan each file: +// Scanner algorithm overview: +// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer) +// Load all directories from the DB +// Compare both collections to find changed folders (based on lastModifiedSince) and deleted folders +// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively) +// For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file: // if file in folder is newer, update the one in DB -// if file in folder does not exists in DB, add -// for each file in the DB that is not found in the folder, delete from DB -// For each deleted folder: delete all files from DB that starts with the folder path -// Only on first run, check if any folder under each changed folder is missing. -// if it is, delete everything under it +// if file in folder does not exists in DB, add it +// for each file in the DB that is not found in the folder, delete it from DB // Create new albums/artists, update counters: // collect all albumIDs and artistIDs from previous steps // refresh the collected albums and artists with the metadata from the mediafiles -// Delete all empty albums, delete all empty Artists +// For each changed folder, process playlists: +// If the playlist is not in the DB, import it, setting sync = true +// If the playlist is in the DB and sync == true, import it, or else skip it +// Delete all empty albums, delete all empty artists, clean-up playlists func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error { - start := time.Now() - log.Trace(ctx, "Looking for changes in music folder", "folder", s.rootFolder) + ctx = s.withAdminUser(ctx) - changed, deleted, err := s.detector.Scan(ctx, lastModifiedSince) + start := time.Now() + allFSDirs, err := s.getDirTree(ctx) if err != nil { return err } - if len(changed)+len(deleted) == 0 { + allDBDirs, err := s.getDBDirTree(ctx) + if err != nil { + return err + } + + changedDirs := s.getChangedDirs(ctx, allFSDirs, allDBDirs, lastModifiedSince) + deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs) + + if len(changedDirs)+len(deletedDirs) == 0 { log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder) return nil } if log.CurrentLevel() >= log.LevelTrace { - log.Info(ctx, "Folder changes found", "numChanged", len(changed), "numDeleted", len(deleted), - "changed", strings.Join(changed, ";"), "deleted", strings.Join(deleted, ";")) + log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs), + "changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";")) } else { - log.Info(ctx, "Folder changes found", "numChanged", len(changed), "numDeleted", len(deleted)) + log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs)) } - sort.Strings(changed) - sort.Strings(deleted) + s.cnt = &counters{} - updatedArtists := artistMap{} - updatedAlbums := albumMap{} - cnt := &counters{} - - for _, c := range changed { - err := s.processChangedDir(ctx, c, updatedArtists, updatedAlbums, cnt) + for _, dir := range deletedDirs { + err := s.processDeletedDir(ctx, dir) if err != nil { - return err + log.Error("Error removing deleted folder from DB", "path", dir, err) } - // TODO Search for playlists and import (with `sync` on) } - for _, c := range deleted { - err := s.processDeletedDir(ctx, c, updatedArtists, updatedAlbums, cnt) + for _, dir := range changedDirs { + err := s.processChangedDir(ctx, dir) if err != nil { - return err + log.Error("Error updating folder in the DB", "path", dir, err) } - // TODO "Un-sync" all playlists synched from a deleted folder } - err = s.flushAlbums(ctx, updatedAlbums) - if err != nil { - return err + // Now that all mediafiles are imported/updated, search for and import playlists + u, _ := request.UserFrom(ctx) + plsCount := 0 + for _, dir := range changedDirs { + info := allFSDirs[dir] + if info.hasPlaylist { + if !u.IsAdmin { + log.Warn("Playlists will not be imported, as there are no admin users yet, "+ + "Please create an admin user first, and then update the playlists for them to be imported", "dir", dir) + } else { + plsCount = s.plsSync.processPlaylists(ctx, dir) + } + } } - err = s.flushArtists(ctx, updatedArtists) - if err != nil { - return err - } - - s.firstRun.Do(func() { - s.removeDeletedFolders(context.TODO(), changed, cnt) - }) - - err = s.ds.GC(log.NewContext(context.TODO())) - log.Info("Finished Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start), - "added", cnt.added, "updated", cnt.updated, "deleted", cnt.deleted) + err = s.ds.GC(log.NewContext(ctx)) + log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start), + "added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount) return err } -func (s *TagScanner) flushAlbums(ctx context.Context, updatedAlbums albumMap) error { - if len(updatedAlbums) == 0 { - return nil - } - var ids []string - for id := range updatedAlbums { - ids = append(ids, id) - delete(updatedAlbums, id) - } - return s.ds.Album(ctx).Refresh(ids...) -} - -func (s *TagScanner) flushArtists(ctx context.Context, updatedArtists artistMap) error { - if len(updatedArtists) == 0 { - return nil - } - var ids []string - for id := range updatedArtists { - ids = append(ids, id) - delete(updatedArtists, id) - } - return s.ds.Artist(ctx).Refresh(ids...) -} - -func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedArtists artistMap, updatedAlbums albumMap, cnt *counters) error { - dir = filepath.Join(s.rootFolder, dir) +func (s *TagScanner) getDirTree(ctx context.Context) (dirMap, error) { start := time.Now() + log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder) + dirs, err := loadDirTree(ctx, s.rootFolder) + if err != nil { + return nil, err + } + log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start)) + return dirs, nil +} + +func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) { + start := time.Now() + log.Trace(ctx, "Loading directory tree from database", "folder", s.rootFolder) + + repo := s.ds.MediaFile(ctx) + dirs, err := repo.FindPathsRecursively(s.rootFolder) + if err != nil { + return nil, err + } + resp := map[string]struct{}{} + for _, d := range dirs { + resp[filepath.Clean(d)] = struct{}{} + } + + log.Debug("Directory tree loaded from DB", "total", len(resp), "elapsed", time.Since(start)) + return resp, nil +} + +func (s *TagScanner) getChangedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}, lastModified time.Time) []string { + start := time.Now() + log.Trace(ctx, "Checking for changed folders") + var changed []string + + for d, info := range fsDirs { + _, inDB := dbDirs[d] + if (!inDB && (info.hasAudioFiles)) || info.modTime.After(lastModified) { + changed = append(changed, d) + } + } + + sort.Strings(changed) + log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start)) + return changed +} + +func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string { + start := time.Now() + log.Trace(ctx, "Checking for deleted folders") + var deleted []string + + for d := range dbDirs { + if _, ok := fsDirs[d]; !ok { + deleted = append(deleted, d) + } + } + + sort.Strings(deleted) + log.Debug(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start)) + return deleted +} + +func (s *TagScanner) processDeletedDir(ctx context.Context, dir string) error { + start := time.Now() + buffer := newRefreshBuffer(ctx, s.ds) + + mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir) + if err != nil { + return err + } + + c, err := s.ds.MediaFile(ctx).DeleteByPath(dir) + if err != nil { + return err + } + s.cnt.deleted += c + + for _, t := range mfs { + buffer.accumulate(t) + } + + err = buffer.flush() + log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start)) + return err +} + +func (s *TagScanner) processChangedDir(ctx context.Context, dir string) error { + start := time.Now() + buffer := newRefreshBuffer(ctx, s.ds) // Load folder's current tracks from DB into a map currentTracks := map[string]model.MediaFile{} @@ -165,7 +227,7 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA } // Load tracks FileInfo from the folder - files, err := LoadAllAudioFiles(dir) + files, err := loadAllAudioFiles(dir) if err != nil { return err } @@ -175,159 +237,102 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA return nil } - // If track from folder is newer than the one in DB, select for update/insert in DB and delete from the current tracks - log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files)) + orphanTracks := map[string]model.MediaFile{} + for k, v := range currentTracks { + orphanTracks[k] = v + } + + // If track from folder is newer than the one in DB, select for update/insert in DB + log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files)) var filesToUpdate []string for filePath, info := range files { c, ok := currentTracks[filePath] if !ok { filesToUpdate = append(filesToUpdate, filePath) - cnt.added++ + s.cnt.added++ } if ok && info.ModTime().After(c.UpdatedAt) { filesToUpdate = append(filesToUpdate, filePath) - cnt.updated++ + s.cnt.updated++ } - delete(currentTracks, filePath) - // Force a refresh of the album and artist, to cater for cover art files. Ideally we would only do this - // if there are any image file in the folder (TODO) - err = s.updateAlbum(ctx, c.AlbumID, updatedAlbums) - if err != nil { - return err - } - err = s.updateArtist(ctx, c.AlbumArtistID, updatedArtists) - if err != nil { - return err - } + // Force a refresh of the album and artist, to cater for cover art files + buffer.accumulate(c) + + // Only leaves in orphanTracks the ones not found in the folder. After this loop any remaining orphanTracks + // are considered gone from the music folder and will be deleted from DB + delete(orphanTracks, filePath) } numUpdatedTracks := 0 numPurgedTracks := 0 if len(filesToUpdate) > 0 { - // Break the file list in chunks to avoid calling ffmpeg with too many parameters - chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize) - for _, chunk := range chunks { - // Load tracks Metadata from the folder - newTracks, err := s.loadTracks(chunk) - if err != nil { - return err - } - - // If track from folder is newer than the one in DB, update/insert in DB - log.Trace("Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk)) - for i := range newTracks { - n := newTracks[i] - err := s.ds.MediaFile(ctx).Put(&n) - if err != nil { - return err - } - err = s.updateAlbum(ctx, n.AlbumID, updatedAlbums) - if err != nil { - return err - } - err = s.updateArtist(ctx, n.AlbumArtistID, updatedArtists) - if err != nil { - return err - } - numUpdatedTracks++ - } - } - } - - if len(currentTracks) > 0 { - log.Trace("Deleting dangling tracks from DB", "dir", dir, "numTracks", len(currentTracks)) - // Remaining tracks from DB that are not in the folder are deleted - for _, ct := range currentTracks { - numPurgedTracks++ - err = s.updateAlbum(ctx, ct.AlbumID, updatedAlbums) - if err != nil { - return err - } - err = s.updateArtist(ctx, ct.AlbumArtistID, updatedArtists) - if err != nil { - return err - } - if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil { - return err - } - cnt.deleted++ - } - } - - log.Info("Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, "purged", numPurgedTracks, "elapsed", time.Since(start)) - return nil -} - -func (s *TagScanner) updateAlbum(ctx context.Context, albumId string, updatedAlbums albumMap) error { - updatedAlbums[albumId] = struct{}{} - if len(updatedAlbums) >= batchSize { - err := s.flushAlbums(ctx, updatedAlbums) - if err != nil { - return err - } - } - return nil -} - -func (s *TagScanner) updateArtist(ctx context.Context, artistId string, updatedArtists artistMap) error { - updatedArtists[artistId] = struct{}{} - if len(updatedArtists) >= batchSize { - err := s.flushArtists(ctx, updatedArtists) - if err != nil { - return err - } - } - return nil -} - -func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedArtists artistMap, updatedAlbums albumMap, cnt *counters) error { - dir = filepath.Join(s.rootFolder, dir) - start := time.Now() - - mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir) - if err != nil { - return err - } - for _, t := range mfs { - err = s.updateAlbum(ctx, t.AlbumID, updatedAlbums) - if err != nil { - return err - } - err = s.updateArtist(ctx, t.AlbumArtistID, updatedArtists) + numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, currentTracks, filesToUpdate, buffer) if err != nil { return err } } - log.Info("Finished processing deleted folder", "dir", dir, "purged", len(mfs), "elapsed", time.Since(start)) - c, err := s.ds.MediaFile(ctx).DeleteByPath(dir) - cnt.deleted += c + if len(orphanTracks) > 0 { + numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, orphanTracks, buffer) + if err != nil { + return err + } + } + + err = buffer.flush() + log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, + "purged", numPurgedTracks, "elapsed", time.Since(start)) return err } -func (s *TagScanner) removeDeletedFolders(ctx context.Context, changed []string, cnt *counters) { - for _, dir := range changed { - fullPath := filepath.Join(s.rootFolder, dir) - paths, err := s.ds.MediaFile(ctx).FindPathsRecursively(fullPath) +func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refreshBuffer) (int, error) { + numPurgedTracks := 0 + + log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete)) + // Remaining tracks from DB that are not in the folder are deleted + for _, ct := range tracksToDelete { + numPurgedTracks++ + buffer.accumulate(ct) + if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil { + return 0, err + } + s.cnt.deleted++ + } + return numPurgedTracks, nil +} + +func (s *TagScanner) addOrUpdateTracksInDB(ctx context.Context, dir string, currentTracks map[string]model.MediaFile, filesToUpdate []string, buffer *refreshBuffer) (int, error) { + numUpdatedTracks := 0 + + log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate)) + // Break the file list in chunks to avoid calling ffmpeg with too many parameters + chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize) + for _, chunk := range chunks { + // Load tracks Metadata from the folder + newTracks, err := s.loadTracks(chunk) if err != nil { - log.Error(ctx, "Error reading paths from DB", "path", dir, err) - return + return 0, err } - // If a path is unreadable, remove from the DB - for _, path := range paths { - if readable, err := utils.IsDirReadable(path); !readable { - log.Info(ctx, "Path unavailable. Removing tracks from DB", "path", path, err) - c, err := s.ds.MediaFile(ctx).DeleteByPath(path) - if err != nil { - log.Error(ctx, "Error removing MediaFiles from DB", "path", path, err) - } - cnt.deleted += c + // If track from folder is newer than the one in DB, update/insert in DB + log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk)) + for i := range newTracks { + n := newTracks[i] + // Keep current annotations if the track is in the DB + if t, ok := currentTracks[n.Path]; ok { + n.Annotations = t.Annotations } + err := s.ds.MediaFile(ctx).Put(&n) + if err != nil { + return 0, err + } + buffer.accumulate(n) + numUpdatedTracks++ } } + return numUpdatedTracks, nil } func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) { @@ -344,7 +349,18 @@ func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) { return mfs, nil } -func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) { +func (s *TagScanner) withAdminUser(ctx context.Context) context.Context { + u, err := s.ds.User(ctx).FindFirstAdmin() + if err != nil { + log.Warn(ctx, "No admin user found!", err) + u = &model.User{} + } + + ctx = request.WithUsername(ctx, u.UserName) + return request.WithUser(ctx, *u) +} + +func loadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) { dir, err := os.Open(dirPath) if err != nil { return nil, err diff --git a/scanner/tag_scanner_2.go b/scanner/tag_scanner_2.go deleted file mode 100644 index 61a1a78da..000000000 --- a/scanner/tag_scanner_2.go +++ /dev/null @@ -1,347 +0,0 @@ -package scanner - -import ( - "context" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/deluan/navidrome/log" - "github.com/deluan/navidrome/model" - "github.com/deluan/navidrome/model/request" - "github.com/deluan/navidrome/utils" -) - -type TagScanner2 struct { - rootFolder string - ds model.DataStore - mapper *mediaFileMapper - plsSync *playlistSync - cnt *counters -} - -func NewTagScanner2(rootFolder string, ds model.DataStore) *TagScanner2 { - return &TagScanner2{ - rootFolder: rootFolder, - mapper: newMediaFileMapper(rootFolder), - plsSync: newPlaylistSync(ds), - ds: ds, - } -} - -// Scan algorithm overview: -// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer) -// Load all directories from the DB -// Compare both collections to find changed folders (based on lastModifiedSince) and deleted folders -// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively) -// For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file: -// if file in folder is newer, update the one in DB -// if file in folder does not exists in DB, add it -// for each file in the DB that is not found in the folder, delete it from DB -// Create new albums/artists, update counters: -// collect all albumIDs and artistIDs from previous steps -// refresh the collected albums and artists with the metadata from the mediafiles -// For each changed folder, process playlists: -// If the playlist is not in the DB, import it, setting sync = true -// If the playlist is in the DB and sync == true, import it, or else skip it -// Delete all empty albums, delete all empty artists, clean-up playlists -func (s *TagScanner2) Scan(ctx context.Context, lastModifiedSince time.Time) error { - ctx = s.withAdminUser(ctx) - - start := time.Now() - allFSDirs, err := s.getDirTree(ctx) - if err != nil { - return err - } - - allDBDirs, err := s.getDBDirTree(ctx) - if err != nil { - return err - } - - changedDirs := s.getChangedDirs(ctx, allFSDirs, allDBDirs, lastModifiedSince) - deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs) - - if len(changedDirs)+len(deletedDirs) == 0 { - log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder) - return nil - } - - if log.CurrentLevel() >= log.LevelTrace { - log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs), - "changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";")) - } else { - log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs)) - } - - s.cnt = &counters{} - - for _, dir := range deletedDirs { - err := s.processDeletedDir(ctx, dir) - if err != nil { - log.Error("Error removing deleted folder from DB", "path", dir, err) - } - } - for _, dir := range changedDirs { - err := s.processChangedDir(ctx, dir) - if err != nil { - log.Error("Error updating folder in the DB", "path", dir, err) - } - } - - // Now that all mediafiles are imported/updated, search for and import playlists - u, _ := request.UserFrom(ctx) - plsCount := 0 - for _, dir := range changedDirs { - info := allFSDirs[dir] - if info.hasPlaylist { - if !u.IsAdmin { - log.Warn("Playlists will not be imported, as there are no admin users yet, "+ - "Please create an admin user first, and then update the playlists for them to be imported", "dir", dir) - } else { - plsCount = s.plsSync.processPlaylists(ctx, dir) - } - } - } - - err = s.ds.GC(log.NewContext(ctx)) - log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start), - "added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount) - - return err -} - -func (s *TagScanner2) getDirTree(ctx context.Context) (dirMap, error) { - start := time.Now() - log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder) - dirs, err := loadDirTree(ctx, s.rootFolder) - if err != nil { - return nil, err - } - log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start)) - return dirs, nil -} - -func (s *TagScanner2) getDBDirTree(ctx context.Context) (map[string]struct{}, error) { - start := time.Now() - log.Trace(ctx, "Loading directory tree from database", "folder", s.rootFolder) - - repo := s.ds.MediaFile(ctx) - dirs, err := repo.FindPathsRecursively(s.rootFolder) - if err != nil { - return nil, err - } - resp := map[string]struct{}{} - for _, d := range dirs { - resp[filepath.Clean(d)] = struct{}{} - } - - log.Debug("Directory tree loaded from DB", "total", len(resp), "elapsed", time.Since(start)) - return resp, nil -} - -func (s *TagScanner2) getChangedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}, lastModified time.Time) []string { - start := time.Now() - log.Trace(ctx, "Checking for changed folders") - var changed []string - - for d, info := range fsDirs { - _, inDB := dbDirs[d] - if (!inDB && (info.hasAudioFiles)) || info.modTime.After(lastModified) { - changed = append(changed, d) - } - } - - sort.Strings(changed) - log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start)) - return changed -} - -func (s *TagScanner2) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string { - start := time.Now() - log.Trace(ctx, "Checking for deleted folders") - var deleted []string - - for d := range dbDirs { - if _, ok := fsDirs[d]; !ok { - deleted = append(deleted, d) - } - } - - sort.Strings(deleted) - log.Debug(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start)) - return deleted -} - -func (s *TagScanner2) processDeletedDir(ctx context.Context, dir string) error { - start := time.Now() - buffer := newRefreshBuffer(ctx, s.ds) - - mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir) - if err != nil { - return err - } - - c, err := s.ds.MediaFile(ctx).DeleteByPath(dir) - if err != nil { - return err - } - s.cnt.deleted += c - - for _, t := range mfs { - buffer.accumulate(t) - } - - err = buffer.flush() - log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start)) - return err -} - -func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error { - start := time.Now() - buffer := newRefreshBuffer(ctx, s.ds) - - // Load folder's current tracks from DB into a map - currentTracks := map[string]model.MediaFile{} - ct, err := s.ds.MediaFile(ctx).FindAllByPath(dir) - if err != nil { - return err - } - for _, t := range ct { - currentTracks[t.Path] = t - } - - // Load tracks FileInfo from the folder - files, err := LoadAllAudioFiles(dir) - if err != nil { - return err - } - - // If no files to process, return - if len(files)+len(currentTracks) == 0 { - return nil - } - - orphanTracks := map[string]model.MediaFile{} - for k, v := range currentTracks { - orphanTracks[k] = v - } - - // If track from folder is newer than the one in DB, select for update/insert in DB - log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files)) - var filesToUpdate []string - for filePath, info := range files { - c, ok := currentTracks[filePath] - if !ok { - filesToUpdate = append(filesToUpdate, filePath) - s.cnt.added++ - } - if ok && info.ModTime().After(c.UpdatedAt) { - filesToUpdate = append(filesToUpdate, filePath) - s.cnt.updated++ - } - - // Force a refresh of the album and artist, to cater for cover art files - buffer.accumulate(c) - - // Only leaves in orphanTracks the ones not found in the folder. After this loop any remaining orphanTracks - // are considered gone from the music folder and will be deleted from DB - delete(orphanTracks, filePath) - } - - numUpdatedTracks := 0 - numPurgedTracks := 0 - - if len(filesToUpdate) > 0 { - numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, currentTracks, filesToUpdate, buffer) - if err != nil { - return err - } - } - - if len(orphanTracks) > 0 { - numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, orphanTracks, buffer) - if err != nil { - return err - } - } - - err = buffer.flush() - log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, - "purged", numPurgedTracks, "elapsed", time.Since(start)) - return err -} - -func (s *TagScanner2) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refreshBuffer) (int, error) { - numPurgedTracks := 0 - - log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete)) - // Remaining tracks from DB that are not in the folder are deleted - for _, ct := range tracksToDelete { - numPurgedTracks++ - buffer.accumulate(ct) - if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil { - return 0, err - } - s.cnt.deleted++ - } - return numPurgedTracks, nil -} - -func (s *TagScanner2) addOrUpdateTracksInDB(ctx context.Context, dir string, currentTracks map[string]model.MediaFile, filesToUpdate []string, buffer *refreshBuffer) (int, error) { - numUpdatedTracks := 0 - - log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate)) - // Break the file list in chunks to avoid calling ffmpeg with too many parameters - chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize) - for _, chunk := range chunks { - // Load tracks Metadata from the folder - newTracks, err := s.loadTracks(chunk) - if err != nil { - return 0, err - } - - // If track from folder is newer than the one in DB, update/insert in DB - log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk)) - for i := range newTracks { - n := newTracks[i] - // Keep current annotations if the track is in the DB - if t, ok := currentTracks[n.Path]; ok { - n.Annotations = t.Annotations - } - err := s.ds.MediaFile(ctx).Put(&n) - if err != nil { - return 0, err - } - buffer.accumulate(n) - numUpdatedTracks++ - } - } - return numUpdatedTracks, nil -} - -func (s *TagScanner2) loadTracks(filePaths []string) (model.MediaFiles, error) { - mds, err := ExtractAllMetadata(filePaths) - if err != nil { - return nil, err - } - - var mfs model.MediaFiles - for _, md := range mds { - mf := s.mapper.toMediaFile(md) - mfs = append(mfs, mf) - } - return mfs, nil -} - -func (s *TagScanner2) withAdminUser(ctx context.Context) context.Context { - u, err := s.ds.User(ctx).FindFirstAdmin() - if err != nil { - log.Warn(ctx, "No admin user found!", err) - u = &model.User{} - } - - ctx = request.WithUsername(ctx, u.UserName) - return request.WithUser(ctx, *u) -} diff --git a/scanner/tag_scanner_test.go b/scanner/tag_scanner_test.go index a3f30b9d2..9de918987 100644 --- a/scanner/tag_scanner_test.go +++ b/scanner/tag_scanner_test.go @@ -1,27 +1,14 @@ package scanner import ( - "github.com/deluan/navidrome/conf" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("TagScanner", func() { - Describe("sanitizeFieldForSorting", func() { - BeforeEach(func() { - conf.Server.IgnoredArticles = "The" - }) - It("sanitize accents", func() { - Expect(sanitizeFieldForSorting("Céu")).To(Equal("Ceu")) - }) - It("removes articles", func() { - Expect(sanitizeFieldForSorting("The Beatles")).To(Equal("Beatles")) - }) - }) - - Describe("LoadAllAudioFiles", func() { + Describe("loadAllAudioFiles", func() { It("return all audio files from the folder", func() { - files, err := LoadAllAudioFiles("tests/fixtures") + files, err := loadAllAudioFiles("tests/fixtures") Expect(err).ToNot(HaveOccurred()) Expect(files).To(HaveLen(3)) Expect(files).To(HaveKey("tests/fixtures/test.ogg")) @@ -30,12 +17,12 @@ var _ = Describe("TagScanner", func() { Expect(files).ToNot(HaveKey("tests/fixtures/playlist.m3u")) }) It("returns error if path does not exist", func() { - _, err := LoadAllAudioFiles("./INVALID/PATH") + _, err := loadAllAudioFiles("./INVALID/PATH") Expect(err).To(HaveOccurred()) }) It("returns empty map if there are no audio files in path", func() { - Expect(LoadAllAudioFiles("tests/fixtures/empty_folder")).To(BeEmpty()) + Expect(loadAllAudioFiles("tests/fixtures/empty_folder")).To(BeEmpty()) }) }) })