WIP

scanner/playlist_sync.go

@@ -24,8 +24,8 @@ func newPlaylistSync(ds model.DataStore) *playlistSync {
 	return &playlistSync{ds: ds}
 }
 
-func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int {
+func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int64 {
-	count := 0
+	var count int64
 	files, err := ioutil.ReadDir(dir)
 	if err != nil {
 		log.Error(ctx, "Error reading files", "dir", dir, err)

scanner/tag_scanner.go

@@ -36,26 +36,30 @@ func NewTagScanner(rootFolder string, ds model.DataStore, cacheWarmer core.Cache
 	}
 }
 
-type counters struct {
+type (
-	added   int64
+	counters struct {
-	updated int64
+		added     int64
-	deleted int64
+		updated   int64
-}
+		deleted   int64
+		playlists int64
+	}
+	dirMap map[string]dirStats
+)
 
 const (
 	// filesBatchSize used for batching file metadata extraction
 	filesBatchSize = 100
 )
 
-// Scanner algorithm overview:
+// TagScanner algorithm overview:
-// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer)
 // Load all directories from the DB
-// Compare both collections to find changed folders (based on lastModifiedSince) and deleted folders
+// Traverse the music folder, collecting each subfolder's ModTime (self or any non-dir children, whichever is newer)
-// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
 // For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file:
 //	    if file in folder is newer, update the one in DB
 //      if file in folder does not exists in DB, add it
 // 	    for each file in the DB that is not found in the folder, delete it from DB
+// Compare directories in the fs with the ones in the DB to find deleted folders
+// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
 // Create new albums/artists, update counters:
 //      collect all albumIDs and artistIDs from previous steps
 //	    refresh the collected albums and artists with the metadata from the mediafiles
@@ -65,60 +69,59 @@ const (
 // Delete all empty albums, delete all empty artists, clean-up playlists
 func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error {
 	ctx = s.withAdminUser(ctx)
-
 	start := time.Now()
-	allFSDirs, err := s.getDirTree(ctx)
-	if err != nil {
-		return err
-	}
 
 	allDBDirs, err := s.getDBDirTree(ctx)
 	if err != nil {
 		return err
 	}
 
-	changedDirs := s.getChangedDirs(ctx, allFSDirs, allDBDirs, lastModifiedSince)
+	allFSDirs := dirMap{}
-	deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
+	var changedDirs []string
+	s.cnt = &counters{}
 
-	if len(changedDirs)+len(deletedDirs) == 0 {
+	foldersFound := s.getRootFolderWalker(ctx)
+	for {
+		folderStats, more := <-foldersFound
+		if !more {
+			break
+		}
+		allFSDirs[folderStats.Path] = folderStats
+
+		if s.isChangedDirs(ctx, folderStats, allDBDirs, lastModifiedSince) {
+			changedDirs = append(changedDirs, folderStats.Path)
+			err := s.processChangedDir(ctx, folderStats.Path)
+			if err != nil {
+				log.Error("Error updating folder in the DB", "path", folderStats.Path, err)
+			}
+		}
+	}
+
+	deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
+	if len(deletedDirs)+len(changedDirs) == 0 {
 		log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
 		return nil
 	}
 
-	if log.CurrentLevel() >= log.LevelTrace {
-		log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs),
-			"changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";"))
-	} else {
-		log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
-	}
-
-	s.cnt = &counters{}
-
 	for _, dir := range deletedDirs {
 		err := s.processDeletedDir(ctx, dir)
 		if err != nil {
 			log.Error("Error removing deleted folder from DB", "path", dir, err)
 		}
 	}
-	for _, dir := range changedDirs {
-		err := s.processChangedDir(ctx, dir)
-		if err != nil {
-			log.Error("Error updating folder in the DB", "path", dir, err)
-		}
-	}
 
-	plsCount := 0
+	s.cnt.playlists = 0
 	if conf.Server.AutoImportPlaylists {
-		// Now that all mediafiles are imported/updated, search for and import playlists
+		// Now that all mediafiles are imported/updated, search for and import/update playlists
 		u, _ := request.UserFrom(ctx)
 		for _, dir := range changedDirs {
 			info := allFSDirs[dir]
-			if info.hasPlaylist {
+			if info.HasPlaylist {
 				if !u.IsAdmin {
 					log.Warn("Playlists will not be imported, as there are no admin users yet, "+
 						"Please create an admin user first, and then update the playlists for them to be imported", "dir", dir)
 				} else {
-					plsCount = s.plsSync.processPlaylists(ctx, dir)
+					s.cnt.playlists = s.plsSync.processPlaylists(ctx, dir)
 				}
 			}
 		}
@@ -128,20 +131,22 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
 
 	err = s.ds.GC(log.NewContext(ctx), s.rootFolder)
 	log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
-		"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount)
+		"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", s.cnt.playlists)
 
 	return err
 }
 
-func (s *TagScanner) getDirTree(ctx context.Context) (dirMap, error) {
+func (s *TagScanner) getRootFolderWalker(ctx context.Context) walkResults {
 	start := time.Now()
 	log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
-	dirs, err := loadDirTree(ctx, s.rootFolder)
+	results := make(chan dirStats, 5000)
-	if err != nil {
+	go func() {
-		return nil, err
+		if err := walkDirTree(ctx, s.rootFolder, results); err != nil {
-	}
+			log.Error("Scan was interrupted by error", err)
-	log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start))
+		}
-	return dirs, nil
+		log.Debug("Finished reading directories from filesystem", "total", "elapsed", time.Since(start))
+	}()
+	return results
 }
 
 func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) {
@@ -162,21 +167,10 @@ func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, err
 	return resp, nil
 }
 
-func (s *TagScanner) getChangedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}, lastModified time.Time) []string {
+func (s *TagScanner) isChangedDirs(ctx context.Context, folder dirStats, dbDirs map[string]struct{}, lastModified time.Time) bool {
-	start := time.Now()
+	_, inDB := dbDirs[folder.Path]
-	log.Trace(ctx, "Checking for changed folders")
+	// If is a new folder with at least one song OR it was modified after lastModified
-	var changed []string
+	return (!inDB && (folder.AudioFilesCount > 0)) || folder.ModTime.After(lastModified)
-
-	for d, info := range fsDirs {
-		_, inDB := dbDirs[d]
-		if (!inDB && (info.hasAudioFiles)) || info.modTime.After(lastModified) {
-			changed = append(changed, d)
-		}
-	}
-
-	sort.Strings(changed)
-	log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start))
-	return changed
 }
 
 func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string {

scanner/walk_dir_tree.go

@@ -14,49 +14,53 @@ import (
 )
 
 type (
-	dirMapValue struct {
+	dirStats struct {
-		modTime       time.Time
+		Path            string
-		hasImages     bool
+		ModTime         time.Time
-		hasPlaylist   bool
+		HasImages       bool
-		hasAudioFiles bool
+		HasPlaylist     bool
+		AudioFilesCount int64
 	}
-	dirMap = map[string]dirMapValue
+	walkResults = chan dirStats
 )
 
-func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) {
+func walkDirTree(ctx context.Context, rootFolder string, results walkResults) error {
-	newMap := make(dirMap)
+	err := walkFolder(ctx, rootFolder, rootFolder, results)
-	err := loadMap(ctx, rootFolder, rootFolder, newMap)
 	if err != nil {
 		log.Error(ctx, "Error loading directory tree", err)
 	}
-	return newMap, err
+	close(results)
+	return err
 }
 
-func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error {
+func walkFolder(ctx context.Context, rootPath string, currentFolder string, results walkResults) error {
-	children, dirMapValue, err := loadDir(ctx, currentFolder)
+	children, stats, err := loadDir(ctx, currentFolder)
 	if err != nil {
 		return err
 	}
 	for _, c := range children {
-		err := loadMap(ctx, rootPath, c, dirMap)
+		err := walkFolder(ctx, rootPath, c, results)
 		if err != nil {
 			return err
 		}
 	}
 
 	dir := filepath.Clean(currentFolder)
-	dirMap[dir] = dirMapValue
+	log.Trace(ctx, "Found directory", "dir", dir, "audioCount", stats.AudioFilesCount,
+		"hasImages", stats.HasImages, "HasPlaylist", stats.HasPlaylist)
+	stats.Path = dir
+	results <- stats
 
 	return nil
 }
 
-func loadDir(ctx context.Context, dirPath string) (children []string, info dirMapValue, err error) {
+func loadDir(ctx context.Context, dirPath string) (children []string, stats dirStats, err error) {
 	dirInfo, err := os.Stat(dirPath)
 	if err != nil {
 		log.Error(ctx, "Error stating dir", "path", dirPath, err)
 		return
 	}
-	info.modTime = dirInfo.ModTime()
+	stats.ModTime = dirInfo.ModTime()
 
 	files, err := ioutil.ReadDir(dirPath)
 	if err != nil {
@@ -67,17 +71,21 @@ func loadDir(ctx context.Context, dirPath string) (children []string, info dirMa
 		isDir, err := isDirOrSymlinkToDir(dirPath, f)
 		// Skip invalid symlinks
 		if err != nil {
+			log.Error(ctx, "Invalid symlink", "dir", dirPath)
 			continue
 		}
 		if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) {
 			children = append(children, filepath.Join(dirPath, f.Name()))
 		} else {
-			if f.ModTime().After(info.modTime) {
+			if f.ModTime().After(stats.ModTime) {
-				info.modTime = f.ModTime()
+				stats.ModTime = f.ModTime()
+			}
+			if utils.IsAudioFile(f.Name()) {
+				stats.AudioFilesCount++
+			} else {
+				stats.HasPlaylist = stats.HasPlaylist || utils.IsPlaylist(f.Name())
+				stats.HasImages = stats.HasImages || utils.IsImageFile(f.Name())
 			}
-			info.hasImages = info.hasImages || utils.IsImageFile(f.Name())
-			info.hasPlaylist = info.hasPlaylist || utils.IsPlaylist(f.Name())
-			info.hasAudioFiles = info.hasAudioFiles || utils.IsAudioFile(f.Name())
 		}
 	}
 	return

scanner/walk_dir_tree_test.go

@@ -1,14 +1,46 @@
 package scanner
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
 )
 
 var _ = Describe("load_tree", func() {
+
+	Describe("walkDirTree", func() {
+		It("reads all info correctly", func() {
+			var collected = dirMap{}
+			results := make(walkResults, 5000)
+			var err error
+			go func() {
+				err = walkDirTree(context.TODO(), "tests/fixtures", results)
+			}()
+
+			for {
+				stats, more := <-results
+				if !more {
+					break
+				}
+				collected[stats.Path] = stats
+			}
+
+			Expect(err).To(BeNil())
+			Expect(collected["tests/fixtures"]).To(MatchFields(IgnoreExtras, Fields{
+				"HasImages":       BeTrue(),
+				"HasPlaylist":     BeFalse(),
+				"AudioFilesCount": BeNumerically("==", 4),
+			}))
+			Expect(collected["tests/fixtures/playlists"].HasPlaylist).To(BeTrue())
+			Expect(collected).To(HaveKey("tests/fixtures/symlink2dir"))
+			Expect(collected).To(HaveKey("tests/fixtures/empty_folder"))
+		})
+	})
+
 	Describe("isDirOrSymlinkToDir", func() {
 		It("returns true for normal dirs", func() {
 			dir, _ := os.Stat("tests/fixtures")

tests/fixtures/symlink2dir

@@ -1 +1 @@
-../
+empty_folder