mirrors/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2025-04-04 04:57:37 +03:00
Code Issues Projects Releases Packages Wiki Activity

Remove old scanner

Browse source
This commit is contained in:
Deluan 2020-07-27 11:12:39 -04:00
parent 4e4fcb2304
commit f992b5663f
10 changed files with 291 additions and 862 deletions
Download patch file Download diff file Expand all files Collapse all files

1
conf/configuration.go
View file

@ -40,7 +40,6 @@ type configOptions struct {
// DevFlags. These are used to enable/disable debugging and incomplete features
DevLogSourceLine bool
DevAutoCreateAdminPassword string
DevOldScanner bool
}
var Server = &configOptions{}

128
scanner/change_detector.go
View file

@ -1,128 +0,0 @@
package scanner
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/deluan/navidrome/log"
)
type dirInfo struct {
mdate time.Time
maybe bool
}
type dirInfoMap map[string]dirInfo
type changeDetector struct {
rootFolder string
dirMap dirInfoMap
}
func newChangeDetector(rootFolder string) *changeDetector {
return &changeDetector{
rootFolder: rootFolder,
dirMap: dirInfoMap{},
}
}
func (s *changeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) {
start := time.Now()
newMap := make(dirInfoMap)
err = s.loadMap(ctx, newMap, s.rootFolder, lastModifiedSince, false)
if err != nil {
return
}
changed, deleted, err = s.checkForUpdates(lastModifiedSince, newMap)
if err != nil {
return
}
elapsed := time.Since(start)
log.Trace(ctx, "Folder analysis complete", "total", len(newMap), "changed", len(changed), "deleted", len(deleted), "elapsed", elapsed)
s.dirMap = newMap
return
}
func (s *changeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) {
dirInfo, err := os.Stat(dirPath)
if err != nil {
log.Error(ctx, "Error stating dir", "path", dirPath, err)
return
}
lastUpdated = dirInfo.ModTime()
files, err := ioutil.ReadDir(dirPath)
if err != nil {
log.Error(ctx, "Error reading dir", "path", dirPath, err)
return
}
for _, f := range files {
isDir, err := isDirOrSymlinkToDir(dirPath, f)
// Skip invalid symlinks
if err != nil {
continue
}
if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) {
children = append(children, filepath.Join(dirPath, f.Name()))
} else {
if f.ModTime().After(lastUpdated) {
lastUpdated = f.ModTime()
}
}
}
return
}
func (s *changeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error {
children, lastUpdated, err := s.loadDir(ctx, path)
if err != nil {
return err
}
maybe = maybe || lastUpdated.After(since)
for _, c := range children {
err := s.loadMap(ctx, dirMap, c, since, maybe)
if err != nil {
return err
}
}
dir := s.getRelativePath(path)
dirMap[dir] = dirInfo{mdate: lastUpdated, maybe: maybe}
return nil
}
func (s *changeDetector) getRelativePath(subFolder string) string {
dir, _ := filepath.Rel(s.rootFolder, subFolder)
if dir == "" {
dir = "."
}
return dir
}
func (s *changeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) {
for dir, newEntry := range newMap {
lastUpdated := newEntry.mdate
oldLastUpdated := lastModifiedSince
if oldEntry, ok := s.dirMap[dir]; ok {
oldLastUpdated = oldEntry.mdate
} else {
if newEntry.maybe {
oldLastUpdated = time.Time{}
}
}
if lastUpdated.After(oldLastUpdated) {
changed = append(changed, dir)
}
}
for dir := range s.dirMap {
if _, ok := newMap[dir]; !ok {
deleted = append(deleted, dir)
}
}
return
}

152
scanner/change_detector_test.go
View file

@ -1,152 +0,0 @@
package scanner
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("changeDetector", func() {
var testFolder string
var scanner *changeDetector
lastModifiedSince := time.Time{}
BeforeEach(func() {
testFolder, _ = ioutil.TempDir("", "navidrome_tests")
err := os.MkdirAll(testFolder, 0777)
if err != nil {
panic(err)
}
scanner = newChangeDetector(testFolder)
})
It("detects changes recursively", func() {
// Scan empty folder
changed, deleted, err := scanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf("."))
// Add one subfolder
lastModifiedSince = nowWithDelay()
err = os.MkdirAll(filepath.Join(testFolder, "a"), 0777)
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf(".", P("a")))
// Add more subfolders
lastModifiedSince = nowWithDelay()
err = os.MkdirAll(filepath.Join(testFolder, "a", "b", "c"), 0777)
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf(P("a"), P("a/b"), P("a/b/c")))
// Scan with no changes
lastModifiedSince = nowWithDelay()
changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(BeEmpty())
// New file in subfolder
lastModifiedSince = nowWithDelay()
_, err = os.Create(filepath.Join(testFolder, "a", "b", "empty.txt"))
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf(P("a/b")))
// Delete file in subfolder
lastModifiedSince = nowWithDelay()
err = os.Remove(filepath.Join(testFolder, "a", "b", "empty.txt"))
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf(P("a/b")))
// Delete subfolder
lastModifiedSince = nowWithDelay()
err = os.Remove(filepath.Join(testFolder, "a", "b", "c"))
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(ConsistOf(P("a/b/c")))
Expect(changed).To(ConsistOf(P("a/b")))
// Only returns changes after lastModifiedSince
lastModifiedSince = nowWithDelay()
newScanner := newChangeDetector(testFolder)
changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(BeEmpty())
Expect(changed).To(BeEmpty())
f, _ := os.Create(filepath.Join(testFolder, "a", "b", "new.txt"))
_ = f.Close()
changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince)
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf(P("a/b")))
})
Describe("isDirOrSymlinkToDir", func() {
It("returns true for normal dirs", func() {
dir, _ := os.Stat("tests/fixtures")
Expect(isDirOrSymlinkToDir("tests", dir)).To(BeTrue())
})
It("returns true for symlinks to dirs", func() {
dir, _ := os.Stat("tests/fixtures/symlink2dir")
Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeTrue())
})
It("returns false for files", func() {
dir, _ := os.Stat("tests/fixtures/test.mp3")
Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse())
})
It("returns false for symlinks to files", func() {
dir, _ := os.Stat("tests/fixtures/symlink")
Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse())
})
})
Describe("isDirIgnored", func() {
baseDir := filepath.Join("tests", "fixtures")
It("returns false for normal dirs", func() {
dir, _ := os.Stat(filepath.Join(baseDir, "empty_folder"))
Expect(isDirIgnored(baseDir, dir)).To(BeFalse())
})
It("returns true when folder contains .ndignore file", func() {
dir, _ := os.Stat(filepath.Join(baseDir, "ignored_folder"))
Expect(isDirIgnored(baseDir, dir)).To(BeTrue())
})
})
})
// I hate time-based tests....
func nowWithDelay() time.Time {
now := time.Now()
time.Sleep(50 * time.Millisecond)
return now
}

42
scanner/load_tree_test.go Normal file
View file

@ -0,0 +1,42 @@
package scanner
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("load_tree", func() {
Describe("isDirOrSymlinkToDir", func() {
It("returns true for normal dirs", func() {
dir, _ := os.Stat("tests/fixtures")
Expect(isDirOrSymlinkToDir("tests", dir)).To(BeTrue())
})
It("returns true for symlinks to dirs", func() {
dir, _ := os.Stat("tests/fixtures/symlink2dir")
Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeTrue())
})
It("returns false for files", func() {
dir, _ := os.Stat("tests/fixtures/test.mp3")
Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse())
})
It("returns false for symlinks to files", func() {
dir, _ := os.Stat("tests/fixtures/symlink")
Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse())
})
})
Describe("isDirIgnored", func() {
baseDir := filepath.Join("tests", "fixtures")
It("returns false for normal dirs", func() {
dir, _ := os.Stat(filepath.Join(baseDir, "empty_folder"))
Expect(isDirIgnored(baseDir, dir)).To(BeFalse())
})
It("returns true when folder contains .ndignore file", func() {
dir, _ := os.Stat(filepath.Join(baseDir, "ignored_folder"))
Expect(isDirIgnored(baseDir, dir)).To(BeTrue())
})
})
})

21
scanner/mapping_test.go Normal file
View file

@ -0,0 +1,21 @@
package scanner
import (
"github.com/deluan/navidrome/conf"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("mapping", func() {
Describe("sanitizeFieldForSorting", func() {
BeforeEach(func() {
conf.Server.IgnoredArticles = "The"
})
It("sanitize accents", func() {
Expect(sanitizeFieldForSorting("Céu")).To(Equal("Ceu"))
})
It("removes articles", func() {
Expect(sanitizeFieldForSorting("The Beatles")).To(Equal("Beatles"))
})
})
})

6
scanner/scanner.go
View file

@ -7,7 +7,6 @@ import (
"strconv"
"time"
"github.com/deluan/navidrome/conf"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
)
@ -87,10 +86,7 @@ func (s *Scanner) loadFolders() {
}
func (s *Scanner) newScanner(f model.MediaFolder) FolderScanner {
if conf.Server.DevOldScanner {
return NewTagScanner(f.Path, s.ds)
}
return NewTagScanner2(f.Path, s.ds)
return NewTagScanner(f.Path, s.ds)
}
type Status int

5
scanner/scanner_suite_test.go
View file

@ -1,7 +1,6 @@
package scanner
import (
"path/filepath"
"testing"
"github.com/deluan/navidrome/log"
@ -16,7 +15,3 @@ func TestScanner(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Scanner Suite")
}
func P(path string) string {
return filepath.FromSlash(path)
}

430
scanner/tag_scanner.go
View file

@ -6,38 +6,32 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
"github.com/deluan/navidrome/model/request"
"github.com/deluan/navidrome/utils"
)
type TagScanner struct {
rootFolder string
ds model.DataStore
detector *changeDetector
mapper *mediaFileMapper
firstRun sync.Once
plsSync *playlistSync
cnt *counters
}
func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
return &TagScanner{
rootFolder: rootFolder,
ds: ds,
detector: newChangeDetector(rootFolder),
mapper: newMediaFileMapper(rootFolder),
firstRun: sync.Once{},
plsSync: newPlaylistSync(ds),
ds: ds,
}
}
const batchSize = 5
type (
artistMap map[string]struct{}
albumMap map[string]struct{}
counters struct {
added int64
updated int64
@ -46,113 +40,181 @@ type (
)
const (
// filesBatchSize used for extract file metadata
// filesBatchSize used for batching file metadata extraction
filesBatchSize = 100
)
// Scan algorithm overview:
// For each changed folder: Get all files from DB that starts with the folder, scan each file:
// Scanner algorithm overview:
// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer)
// Load all directories from the DB
// Compare both collections to find changed folders (based on lastModifiedSince) and deleted folders
// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
// For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file:
// if file in folder is newer, update the one in DB
// if file in folder does not exists in DB, add
// for each file in the DB that is not found in the folder, delete from DB
// For each deleted folder: delete all files from DB that starts with the folder path
// Only on first run, check if any folder under each changed folder is missing.
// if it is, delete everything under it
// if file in folder does not exists in DB, add it
// for each file in the DB that is not found in the folder, delete it from DB
// Create new albums/artists, update counters:
// collect all albumIDs and artistIDs from previous steps
// refresh the collected albums and artists with the metadata from the mediafiles
// Delete all empty albums, delete all empty Artists
// For each changed folder, process playlists:
// If the playlist is not in the DB, import it, setting sync = true
// If the playlist is in the DB and sync == true, import it, or else skip it
// Delete all empty albums, delete all empty artists, clean-up playlists
func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error {
start := time.Now()
log.Trace(ctx, "Looking for changes in music folder", "folder", s.rootFolder)
ctx = s.withAdminUser(ctx)
changed, deleted, err := s.detector.Scan(ctx, lastModifiedSince)
start := time.Now()
allFSDirs, err := s.getDirTree(ctx)
if err != nil {
return err
}
if len(changed)+len(deleted) == 0 {
allDBDirs, err := s.getDBDirTree(ctx)
if err != nil {
return err
}
changedDirs := s.getChangedDirs(ctx, allFSDirs, allDBDirs, lastModifiedSince)
deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
if len(changedDirs)+len(deletedDirs) == 0 {
log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
return nil
}
if log.CurrentLevel() >= log.LevelTrace {
log.Info(ctx, "Folder changes found", "numChanged", len(changed), "numDeleted", len(deleted),
"changed", strings.Join(changed, ";"), "deleted", strings.Join(deleted, ";"))
log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs),
"changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";"))
} else {
log.Info(ctx, "Folder changes found", "numChanged", len(changed), "numDeleted", len(deleted))
log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
}
sort.Strings(changed)
sort.Strings(deleted)
s.cnt = &counters{}
updatedArtists := artistMap{}
updatedAlbums := albumMap{}
cnt := &counters{}
for _, c := range changed {
err := s.processChangedDir(ctx, c, updatedArtists, updatedAlbums, cnt)
for _, dir := range deletedDirs {
err := s.processDeletedDir(ctx, dir)
if err != nil {
return err
log.Error("Error removing deleted folder from DB", "path", dir, err)
}
// TODO Search for playlists and import (with `sync` on)
}
for _, c := range deleted {
err := s.processDeletedDir(ctx, c, updatedArtists, updatedAlbums, cnt)
for _, dir := range changedDirs {
err := s.processChangedDir(ctx, dir)
if err != nil {
return err
log.Error("Error updating folder in the DB", "path", dir, err)
}
// TODO "Un-sync" all playlists synched from a deleted folder
}
err = s.flushAlbums(ctx, updatedAlbums)
if err != nil {
return err
// Now that all mediafiles are imported/updated, search for and import playlists
u, _ := request.UserFrom(ctx)
plsCount := 0
for _, dir := range changedDirs {
info := allFSDirs[dir]
if info.hasPlaylist {
if !u.IsAdmin {
log.Warn("Playlists will not be imported, as there are no admin users yet, "+
"Please create an admin user first, and then update the playlists for them to be imported", "dir", dir)
} else {
plsCount = s.plsSync.processPlaylists(ctx, dir)
}
}
}
err = s.flushArtists(ctx, updatedArtists)
if err != nil {
return err
}
s.firstRun.Do(func() {
s.removeDeletedFolders(context.TODO(), changed, cnt)
})
err = s.ds.GC(log.NewContext(context.TODO()))
log.Info("Finished Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
"added", cnt.added, "updated", cnt.updated, "deleted", cnt.deleted)
err = s.ds.GC(log.NewContext(ctx))
log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount)
return err
}
func (s *TagScanner) flushAlbums(ctx context.Context, updatedAlbums albumMap) error {
if len(updatedAlbums) == 0 {
return nil
}
var ids []string
for id := range updatedAlbums {
ids = append(ids, id)
delete(updatedAlbums, id)
}
return s.ds.Album(ctx).Refresh(ids...)
}
func (s *TagScanner) flushArtists(ctx context.Context, updatedArtists artistMap) error {
if len(updatedArtists) == 0 {
return nil
}
var ids []string
for id := range updatedArtists {
ids = append(ids, id)
delete(updatedArtists, id)
}
return s.ds.Artist(ctx).Refresh(ids...)
}
func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedArtists artistMap, updatedAlbums albumMap, cnt *counters) error {
dir = filepath.Join(s.rootFolder, dir)
func (s *TagScanner) getDirTree(ctx context.Context) (dirMap, error) {
start := time.Now()
log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
dirs, err := loadDirTree(ctx, s.rootFolder)
if err != nil {
return nil, err
}
log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start))
return dirs, nil
}
func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) {
start := time.Now()
log.Trace(ctx, "Loading directory tree from database", "folder", s.rootFolder)
repo := s.ds.MediaFile(ctx)
dirs, err := repo.FindPathsRecursively(s.rootFolder)
if err != nil {
return nil, err
}
resp := map[string]struct{}{}
for _, d := range dirs {
resp[filepath.Clean(d)] = struct{}{}
}
log.Debug("Directory tree loaded from DB", "total", len(resp), "elapsed", time.Since(start))
return resp, nil
}
func (s *TagScanner) getChangedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}, lastModified time.Time) []string {
start := time.Now()
log.Trace(ctx, "Checking for changed folders")
var changed []string
for d, info := range fsDirs {
_, inDB := dbDirs[d]
if (!inDB && (info.hasAudioFiles)) || info.modTime.After(lastModified) {
changed = append(changed, d)
}
}
sort.Strings(changed)
log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start))
return changed
}
func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string {
start := time.Now()
log.Trace(ctx, "Checking for deleted folders")
var deleted []string
for d := range dbDirs {
if _, ok := fsDirs[d]; !ok {
deleted = append(deleted, d)
}
}
sort.Strings(deleted)
log.Debug(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start))
return deleted
}
func (s *TagScanner) processDeletedDir(ctx context.Context, dir string) error {
start := time.Now()
buffer := newRefreshBuffer(ctx, s.ds)
mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
if err != nil {
return err
}
c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
if err != nil {
return err
}
s.cnt.deleted += c
for _, t := range mfs {
buffer.accumulate(t)
}
err = buffer.flush()
log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start))
return err
}
func (s *TagScanner) processChangedDir(ctx context.Context, dir string) error {
start := time.Now()
buffer := newRefreshBuffer(ctx, s.ds)
// Load folder's current tracks from DB into a map
currentTracks := map[string]model.MediaFile{}
@ -165,7 +227,7 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA
}
// Load tracks FileInfo from the folder
files, err := LoadAllAudioFiles(dir)
files, err := loadAllAudioFiles(dir)
if err != nil {
return err
}
@ -175,159 +237,102 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA
return nil
}
// If track from folder is newer than the one in DB, select for update/insert in DB and delete from the current tracks
log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
orphanTracks := map[string]model.MediaFile{}
for k, v := range currentTracks {
orphanTracks[k] = v
}
// If track from folder is newer than the one in DB, select for update/insert in DB
log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
var filesToUpdate []string
for filePath, info := range files {
c, ok := currentTracks[filePath]
if !ok {
filesToUpdate = append(filesToUpdate, filePath)
cnt.added++
s.cnt.added++
}
if ok && info.ModTime().After(c.UpdatedAt) {
filesToUpdate = append(filesToUpdate, filePath)
cnt.updated++
s.cnt.updated++
}
delete(currentTracks, filePath)
// Force a refresh of the album and artist, to cater for cover art files. Ideally we would only do this
// if there are any image file in the folder (TODO)
err = s.updateAlbum(ctx, c.AlbumID, updatedAlbums)
if err != nil {
return err
}
err = s.updateArtist(ctx, c.AlbumArtistID, updatedArtists)
if err != nil {
return err
}
// Force a refresh of the album and artist, to cater for cover art files
buffer.accumulate(c)
// Only leaves in orphanTracks the ones not found in the folder. After this loop any remaining orphanTracks
// are considered gone from the music folder and will be deleted from DB
delete(orphanTracks, filePath)
}
numUpdatedTracks := 0
numPurgedTracks := 0
if len(filesToUpdate) > 0 {
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
for _, chunk := range chunks {
// Load tracks Metadata from the folder
newTracks, err := s.loadTracks(chunk)
if err != nil {
return err
}
// If track from folder is newer than the one in DB, update/insert in DB
log.Trace("Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
for i := range newTracks {
n := newTracks[i]
err := s.ds.MediaFile(ctx).Put(&n)
if err != nil {
return err
}
err = s.updateAlbum(ctx, n.AlbumID, updatedAlbums)
if err != nil {
return err
}
err = s.updateArtist(ctx, n.AlbumArtistID, updatedArtists)
if err != nil {
return err
}
numUpdatedTracks++
}
}
}
if len(currentTracks) > 0 {
log.Trace("Deleting dangling tracks from DB", "dir", dir, "numTracks", len(currentTracks))
// Remaining tracks from DB that are not in the folder are deleted
for _, ct := range currentTracks {
numPurgedTracks++
err = s.updateAlbum(ctx, ct.AlbumID, updatedAlbums)
if err != nil {
return err
}
err = s.updateArtist(ctx, ct.AlbumArtistID, updatedArtists)
if err != nil {
return err
}
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
return err
}
cnt.deleted++
}
}
log.Info("Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, "purged", numPurgedTracks, "elapsed", time.Since(start))
return nil
}
func (s *TagScanner) updateAlbum(ctx context.Context, albumId string, updatedAlbums albumMap) error {
updatedAlbums[albumId] = struct{}{}
if len(updatedAlbums) >= batchSize {
err := s.flushAlbums(ctx, updatedAlbums)
if err != nil {
return err
}
}
return nil
}
func (s *TagScanner) updateArtist(ctx context.Context, artistId string, updatedArtists artistMap) error {
updatedArtists[artistId] = struct{}{}
if len(updatedArtists) >= batchSize {
err := s.flushArtists(ctx, updatedArtists)
if err != nil {
return err
}
}
return nil
}
func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedArtists artistMap, updatedAlbums albumMap, cnt *counters) error {
dir = filepath.Join(s.rootFolder, dir)
start := time.Now()
mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
if err != nil {
return err
}
for _, t := range mfs {
err = s.updateAlbum(ctx, t.AlbumID, updatedAlbums)
if err != nil {
return err
}
err = s.updateArtist(ctx, t.AlbumArtistID, updatedArtists)
numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, currentTracks, filesToUpdate, buffer)
if err != nil {
return err
}
}
log.Info("Finished processing deleted folder", "dir", dir, "purged", len(mfs), "elapsed", time.Since(start))
c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
cnt.deleted += c
if len(orphanTracks) > 0 {
numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, orphanTracks, buffer)
if err != nil {
return err
}
}
err = buffer.flush()
log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks,
"purged", numPurgedTracks, "elapsed", time.Since(start))
return err
}
func (s *TagScanner) removeDeletedFolders(ctx context.Context, changed []string, cnt *counters) {
for _, dir := range changed {
fullPath := filepath.Join(s.rootFolder, dir)
paths, err := s.ds.MediaFile(ctx).FindPathsRecursively(fullPath)
func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refreshBuffer) (int, error) {
numPurgedTracks := 0
log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete))
// Remaining tracks from DB that are not in the folder are deleted
for _, ct := range tracksToDelete {
numPurgedTracks++
buffer.accumulate(ct)
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
return 0, err
}
s.cnt.deleted++
}
return numPurgedTracks, nil
}
func (s *TagScanner) addOrUpdateTracksInDB(ctx context.Context, dir string, currentTracks map[string]model.MediaFile, filesToUpdate []string, buffer *refreshBuffer) (int, error) {
numUpdatedTracks := 0
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
for _, chunk := range chunks {
// Load tracks Metadata from the folder
newTracks, err := s.loadTracks(chunk)
if err != nil {
log.Error(ctx, "Error reading paths from DB", "path", dir, err)
return
return 0, err
}
// If a path is unreadable, remove from the DB
for _, path := range paths {
if readable, err := utils.IsDirReadable(path); !readable {
log.Info(ctx, "Path unavailable. Removing tracks from DB", "path", path, err)
c, err := s.ds.MediaFile(ctx).DeleteByPath(path)
if err != nil {
log.Error(ctx, "Error removing MediaFiles from DB", "path", path, err)
}
cnt.deleted += c
// If track from folder is newer than the one in DB, update/insert in DB
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
for i := range newTracks {
n := newTracks[i]
// Keep current annotations if the track is in the DB
if t, ok := currentTracks[n.Path]; ok {
n.Annotations = t.Annotations
}
err := s.ds.MediaFile(ctx).Put(&n)
if err != nil {
return 0, err
}
buffer.accumulate(n)
numUpdatedTracks++
}
}
return numUpdatedTracks, nil
}
func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
@ -344,7 +349,18 @@ func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
return mfs, nil
}
func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) {
func (s *TagScanner) withAdminUser(ctx context.Context) context.Context {
u, err := s.ds.User(ctx).FindFirstAdmin()
if err != nil {
log.Warn(ctx, "No admin user found!", err)
u = &model.User{}
}
ctx = request.WithUsername(ctx, u.UserName)
return request.WithUser(ctx, *u)
}
func loadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) {
dir, err := os.Open(dirPath)
if err != nil {
return nil, err

347
scanner/tag_scanner_2.go
View file

@ -1,347 +0,0 @@
package scanner
import (
"context"
"path/filepath"
"sort"
"strings"
"time"
"github.com/deluan/navidrome/log"
"github.com/deluan/navidrome/model"
"github.com/deluan/navidrome/model/request"
"github.com/deluan/navidrome/utils"
)
type TagScanner2 struct {
rootFolder string
ds model.DataStore
mapper *mediaFileMapper
plsSync *playlistSync
cnt *counters
}
func NewTagScanner2(rootFolder string, ds model.DataStore) *TagScanner2 {
return &TagScanner2{
rootFolder: rootFolder,
mapper: newMediaFileMapper(rootFolder),
plsSync: newPlaylistSync(ds),
ds: ds,
}
}
// Scan algorithm overview:
// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer)
// Load all directories from the DB
// Compare both collections to find changed folders (based on lastModifiedSince) and deleted folders
// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
// For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file:
// if file in folder is newer, update the one in DB
// if file in folder does not exists in DB, add it
// for each file in the DB that is not found in the folder, delete it from DB
// Create new albums/artists, update counters:
// collect all albumIDs and artistIDs from previous steps
// refresh the collected albums and artists with the metadata from the mediafiles
// For each changed folder, process playlists:
// If the playlist is not in the DB, import it, setting sync = true
// If the playlist is in the DB and sync == true, import it, or else skip it
// Delete all empty albums, delete all empty artists, clean-up playlists
func (s *TagScanner2) Scan(ctx context.Context, lastModifiedSince time.Time) error {
ctx = s.withAdminUser(ctx)
start := time.Now()
allFSDirs, err := s.getDirTree(ctx)
if err != nil {
return err
}
allDBDirs, err := s.getDBDirTree(ctx)
if err != nil {
return err
}
changedDirs := s.getChangedDirs(ctx, allFSDirs, allDBDirs, lastModifiedSince)
deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
if len(changedDirs)+len(deletedDirs) == 0 {
log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
return nil
}
if log.CurrentLevel() >= log.LevelTrace {
log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs),
"changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";"))
} else {
log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
}
s.cnt = &counters{}
for _, dir := range deletedDirs {
err := s.processDeletedDir(ctx, dir)
if err != nil {
log.Error("Error removing deleted folder from DB", "path", dir, err)
}
}
for _, dir := range changedDirs {
err := s.processChangedDir(ctx, dir)
if err != nil {
log.Error("Error updating folder in the DB", "path", dir, err)
}
}
// Now that all mediafiles are imported/updated, search for and import playlists
u, _ := request.UserFrom(ctx)
plsCount := 0
for _, dir := range changedDirs {
info := allFSDirs[dir]
if info.hasPlaylist {
if !u.IsAdmin {
log.Warn("Playlists will not be imported, as there are no admin users yet, "+
"Please create an admin user first, and then update the playlists for them to be imported", "dir", dir)
} else {
plsCount = s.plsSync.processPlaylists(ctx, dir)
}
}
}
err = s.ds.GC(log.NewContext(ctx))
log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount)
return err
}
func (s *TagScanner2) getDirTree(ctx context.Context) (dirMap, error) {
start := time.Now()
log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
dirs, err := loadDirTree(ctx, s.rootFolder)
if err != nil {
return nil, err
}
log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start))
return dirs, nil
}
func (s *TagScanner2) getDBDirTree(ctx context.Context) (map[string]struct{}, error) {
start := time.Now()
log.Trace(ctx, "Loading directory tree from database", "folder", s.rootFolder)
repo := s.ds.MediaFile(ctx)
dirs, err := repo.FindPathsRecursively(s.rootFolder)
if err != nil {
return nil, err
}
resp := map[string]struct{}{}
for _, d := range dirs {
resp[filepath.Clean(d)] = struct{}{}
}
log.Debug("Directory tree loaded from DB", "total", len(resp), "elapsed", time.Since(start))
return resp, nil
}
func (s *TagScanner2) getChangedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}, lastModified time.Time) []string {
start := time.Now()
log.Trace(ctx, "Checking for changed folders")
var changed []string
for d, info := range fsDirs {
_, inDB := dbDirs[d]
if (!inDB && (info.hasAudioFiles)) || info.modTime.After(lastModified) {
changed = append(changed, d)
}
}
sort.Strings(changed)
log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start))
return changed
}
func (s *TagScanner2) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string {
start := time.Now()
log.Trace(ctx, "Checking for deleted folders")
var deleted []string
for d := range dbDirs {
if _, ok := fsDirs[d]; !ok {
deleted = append(deleted, d)
}
}
sort.Strings(deleted)
log.Debug(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start))
return deleted
}
func (s *TagScanner2) processDeletedDir(ctx context.Context, dir string) error {
start := time.Now()
buffer := newRefreshBuffer(ctx, s.ds)
mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
if err != nil {
return err
}
c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
if err != nil {
return err
}
s.cnt.deleted += c
for _, t := range mfs {
buffer.accumulate(t)
}
err = buffer.flush()
log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start))
return err
}
func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error {
start := time.Now()
buffer := newRefreshBuffer(ctx, s.ds)
// Load folder's current tracks from DB into a map
currentTracks := map[string]model.MediaFile{}
ct, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
if err != nil {
return err
}
for _, t := range ct {
currentTracks[t.Path] = t
}
// Load tracks FileInfo from the folder
files, err := LoadAllAudioFiles(dir)
if err != nil {
return err
}
// If no files to process, return
if len(files)+len(currentTracks) == 0 {
return nil
}
orphanTracks := map[string]model.MediaFile{}
for k, v := range currentTracks {
orphanTracks[k] = v
}
// If track from folder is newer than the one in DB, select for update/insert in DB
log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
var filesToUpdate []string
for filePath, info := range files {
c, ok := currentTracks[filePath]
if !ok {
filesToUpdate = append(filesToUpdate, filePath)
s.cnt.added++
}
if ok && info.ModTime().After(c.UpdatedAt) {
filesToUpdate = append(filesToUpdate, filePath)
s.cnt.updated++
}
// Force a refresh of the album and artist, to cater for cover art files
buffer.accumulate(c)
// Only leaves in orphanTracks the ones not found in the folder. After this loop any remaining orphanTracks
// are considered gone from the music folder and will be deleted from DB
delete(orphanTracks, filePath)
}
numUpdatedTracks := 0
numPurgedTracks := 0
if len(filesToUpdate) > 0 {
numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, currentTracks, filesToUpdate, buffer)
if err != nil {
return err
}
}
if len(orphanTracks) > 0 {
numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, orphanTracks, buffer)
if err != nil {
return err
}
}
err = buffer.flush()
log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks,
"purged", numPurgedTracks, "elapsed", time.Since(start))
return err
}
func (s *TagScanner2) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refreshBuffer) (int, error) {
numPurgedTracks := 0
log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete))
// Remaining tracks from DB that are not in the folder are deleted
for _, ct := range tracksToDelete {
numPurgedTracks++
buffer.accumulate(ct)
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
return 0, err
}
s.cnt.deleted++
}
return numPurgedTracks, nil
}
func (s *TagScanner2) addOrUpdateTracksInDB(ctx context.Context, dir string, currentTracks map[string]model.MediaFile, filesToUpdate []string, buffer *refreshBuffer) (int, error) {
numUpdatedTracks := 0
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
for _, chunk := range chunks {
// Load tracks Metadata from the folder
newTracks, err := s.loadTracks(chunk)
if err != nil {
return 0, err
}
// If track from folder is newer than the one in DB, update/insert in DB
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
for i := range newTracks {
n := newTracks[i]
// Keep current annotations if the track is in the DB
if t, ok := currentTracks[n.Path]; ok {
n.Annotations = t.Annotations
}
err := s.ds.MediaFile(ctx).Put(&n)
if err != nil {
return 0, err
}
buffer.accumulate(n)
numUpdatedTracks++
}
}
return numUpdatedTracks, nil
}
func (s *TagScanner2) loadTracks(filePaths []string) (model.MediaFiles, error) {
mds, err := ExtractAllMetadata(filePaths)
if err != nil {
return nil, err
}
var mfs model.MediaFiles
for _, md := range mds {
mf := s.mapper.toMediaFile(md)
mfs = append(mfs, mf)
}
return mfs, nil
}
func (s *TagScanner2) withAdminUser(ctx context.Context) context.Context {
u, err := s.ds.User(ctx).FindFirstAdmin()
if err != nil {
log.Warn(ctx, "No admin user found!", err)
u = &model.User{}
}
ctx = request.WithUsername(ctx, u.UserName)
return request.WithUser(ctx, *u)
}

21
scanner/tag_scanner_test.go
View file

@ -1,27 +1,14 @@
package scanner
import (
"github.com/deluan/navidrome/conf"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("TagScanner", func() {
Describe("sanitizeFieldForSorting", func() {
BeforeEach(func() {
conf.Server.IgnoredArticles = "The"
})
It("sanitize accents", func() {
Expect(sanitizeFieldForSorting("Céu")).To(Equal("Ceu"))
})
It("removes articles", func() {
Expect(sanitizeFieldForSorting("The Beatles")).To(Equal("Beatles"))
})
})
Describe("LoadAllAudioFiles", func() {
Describe("loadAllAudioFiles", func() {
It("return all audio files from the folder", func() {
files, err := LoadAllAudioFiles("tests/fixtures")
files, err := loadAllAudioFiles("tests/fixtures")
Expect(err).ToNot(HaveOccurred())
Expect(files).To(HaveLen(3))
Expect(files).To(HaveKey("tests/fixtures/test.ogg"))
@ -30,12 +17,12 @@ var _ = Describe("TagScanner", func() {
Expect(files).ToNot(HaveKey("tests/fixtures/playlist.m3u"))
})
It("returns error if path does not exist", func() {
_, err := LoadAllAudioFiles("./INVALID/PATH")
_, err := loadAllAudioFiles("./INVALID/PATH")
Expect(err).To(HaveOccurred())
})
It("returns empty map if there are no audio files in path", func() {
Expect(LoadAllAudioFiles("tests/fixtures/empty_folder")).To(BeEmpty())
Expect(loadAllAudioFiles("tests/fixtures/empty_folder")).To(BeEmpty())
})
})
})