From 2f394623c8fbd0296965f84a5dba4373943973c1 Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 29 Oct 2020 11:31:45 -0400 Subject: [PATCH] WIP --- scanner/playlist_sync.go | 4 +- scanner/tag_scanner.go | 112 +++++++++--------- scanner/{load_tree.go => walk_dir_tree.go} | 50 ++++---- ...oad_tree_test.go => walk_dir_tree_test.go} | 32 +++++ tests/fixtures/symlink2dir | 2 +- 5 files changed, 117 insertions(+), 83 deletions(-) rename scanner/{load_tree.go => walk_dir_tree.go} (68%) rename scanner/{load_tree_test.go => walk_dir_tree_test.go} (63%) diff --git a/scanner/playlist_sync.go b/scanner/playlist_sync.go index f2a8e56a7..1e3d51af2 100644 --- a/scanner/playlist_sync.go +++ b/scanner/playlist_sync.go @@ -24,8 +24,8 @@ func newPlaylistSync(ds model.DataStore) *playlistSync { return &playlistSync{ds: ds} } -func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int { - count := 0 +func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int64 { + var count int64 files, err := ioutil.ReadDir(dir) if err != nil { log.Error(ctx, "Error reading files", "dir", dir, err) diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index 5d8448f2d..876b30418 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -36,26 +36,30 @@ func NewTagScanner(rootFolder string, ds model.DataStore, cacheWarmer core.Cache } } -type counters struct { - added int64 - updated int64 - deleted int64 -} +type ( + counters struct { + added int64 + updated int64 + deleted int64 + playlists int64 + } + dirMap map[string]dirStats +) const ( // filesBatchSize used for batching file metadata extraction filesBatchSize = 100 ) -// Scanner algorithm overview: -// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer) +// TagScanner algorithm overview: // Load all directories from the DB -// Compare both collections to find changed folders (based on lastModifiedSince) and deleted folders -// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively) +// Traverse the music folder, collecting each subfolder's ModTime (self or any non-dir children, whichever is newer) // For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file: // if file in folder is newer, update the one in DB // if file in folder does not exists in DB, add it // for each file in the DB that is not found in the folder, delete it from DB +// Compare directories in the fs with the ones in the DB to find deleted folders +// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively) // Create new albums/artists, update counters: // collect all albumIDs and artistIDs from previous steps // refresh the collected albums and artists with the metadata from the mediafiles @@ -65,60 +69,59 @@ const ( // Delete all empty albums, delete all empty artists, clean-up playlists func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error { ctx = s.withAdminUser(ctx) - start := time.Now() - allFSDirs, err := s.getDirTree(ctx) - if err != nil { - return err - } allDBDirs, err := s.getDBDirTree(ctx) if err != nil { return err } - changedDirs := s.getChangedDirs(ctx, allFSDirs, allDBDirs, lastModifiedSince) - deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs) + allFSDirs := dirMap{} + var changedDirs []string + s.cnt = &counters{} - if len(changedDirs)+len(deletedDirs) == 0 { + foldersFound := s.getRootFolderWalker(ctx) + for { + folderStats, more := <-foldersFound + if !more { + break + } + allFSDirs[folderStats.Path] = folderStats + + if s.isChangedDirs(ctx, folderStats, allDBDirs, lastModifiedSince) { + changedDirs = append(changedDirs, folderStats.Path) + err := s.processChangedDir(ctx, folderStats.Path) + if err != nil { + log.Error("Error updating folder in the DB", "path", folderStats.Path, err) + } + } + } + + deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs) + if len(deletedDirs)+len(changedDirs) == 0 { log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder) return nil } - if log.CurrentLevel() >= log.LevelTrace { - log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs), - "changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";")) - } else { - log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs)) - } - - s.cnt = &counters{} - for _, dir := range deletedDirs { err := s.processDeletedDir(ctx, dir) if err != nil { log.Error("Error removing deleted folder from DB", "path", dir, err) } } - for _, dir := range changedDirs { - err := s.processChangedDir(ctx, dir) - if err != nil { - log.Error("Error updating folder in the DB", "path", dir, err) - } - } - plsCount := 0 + s.cnt.playlists = 0 if conf.Server.AutoImportPlaylists { - // Now that all mediafiles are imported/updated, search for and import playlists + // Now that all mediafiles are imported/updated, search for and import/update playlists u, _ := request.UserFrom(ctx) for _, dir := range changedDirs { info := allFSDirs[dir] - if info.hasPlaylist { + if info.HasPlaylist { if !u.IsAdmin { log.Warn("Playlists will not be imported, as there are no admin users yet, "+ "Please create an admin user first, and then update the playlists for them to be imported", "dir", dir) } else { - plsCount = s.plsSync.processPlaylists(ctx, dir) + s.cnt.playlists = s.plsSync.processPlaylists(ctx, dir) } } } @@ -128,20 +131,22 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro err = s.ds.GC(log.NewContext(ctx), s.rootFolder) log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start), - "added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount) + "added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", s.cnt.playlists) return err } -func (s *TagScanner) getDirTree(ctx context.Context) (dirMap, error) { +func (s *TagScanner) getRootFolderWalker(ctx context.Context) walkResults { start := time.Now() log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder) - dirs, err := loadDirTree(ctx, s.rootFolder) - if err != nil { - return nil, err - } - log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start)) - return dirs, nil + results := make(chan dirStats, 5000) + go func() { + if err := walkDirTree(ctx, s.rootFolder, results); err != nil { + log.Error("Scan was interrupted by error", err) + } + log.Debug("Finished reading directories from filesystem", "total", "elapsed", time.Since(start)) + }() + return results } func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) { @@ -162,21 +167,10 @@ func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, err return resp, nil } -func (s *TagScanner) getChangedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}, lastModified time.Time) []string { - start := time.Now() - log.Trace(ctx, "Checking for changed folders") - var changed []string - - for d, info := range fsDirs { - _, inDB := dbDirs[d] - if (!inDB && (info.hasAudioFiles)) || info.modTime.After(lastModified) { - changed = append(changed, d) - } - } - - sort.Strings(changed) - log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start)) - return changed +func (s *TagScanner) isChangedDirs(ctx context.Context, folder dirStats, dbDirs map[string]struct{}, lastModified time.Time) bool { + _, inDB := dbDirs[folder.Path] + // If is a new folder with at least one song OR it was modified after lastModified + return (!inDB && (folder.AudioFilesCount > 0)) || folder.ModTime.After(lastModified) } func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string { diff --git a/scanner/load_tree.go b/scanner/walk_dir_tree.go similarity index 68% rename from scanner/load_tree.go rename to scanner/walk_dir_tree.go index b5befb84c..256255f44 100644 --- a/scanner/load_tree.go +++ b/scanner/walk_dir_tree.go @@ -14,49 +14,53 @@ import ( ) type ( - dirMapValue struct { - modTime time.Time - hasImages bool - hasPlaylist bool - hasAudioFiles bool + dirStats struct { + Path string + ModTime time.Time + HasImages bool + HasPlaylist bool + AudioFilesCount int64 } - dirMap = map[string]dirMapValue + walkResults = chan dirStats ) -func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) { - newMap := make(dirMap) - err := loadMap(ctx, rootFolder, rootFolder, newMap) +func walkDirTree(ctx context.Context, rootFolder string, results walkResults) error { + err := walkFolder(ctx, rootFolder, rootFolder, results) if err != nil { log.Error(ctx, "Error loading directory tree", err) } - return newMap, err + close(results) + return err } -func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error { - children, dirMapValue, err := loadDir(ctx, currentFolder) +func walkFolder(ctx context.Context, rootPath string, currentFolder string, results walkResults) error { + children, stats, err := loadDir(ctx, currentFolder) if err != nil { return err } for _, c := range children { - err := loadMap(ctx, rootPath, c, dirMap) + err := walkFolder(ctx, rootPath, c, results) if err != nil { return err } } dir := filepath.Clean(currentFolder) - dirMap[dir] = dirMapValue + log.Trace(ctx, "Found directory", "dir", dir, "audioCount", stats.AudioFilesCount, + "hasImages", stats.HasImages, "HasPlaylist", stats.HasPlaylist) + stats.Path = dir + results <- stats return nil } -func loadDir(ctx context.Context, dirPath string) (children []string, info dirMapValue, err error) { +func loadDir(ctx context.Context, dirPath string) (children []string, stats dirStats, err error) { dirInfo, err := os.Stat(dirPath) if err != nil { log.Error(ctx, "Error stating dir", "path", dirPath, err) return } - info.modTime = dirInfo.ModTime() + stats.ModTime = dirInfo.ModTime() files, err := ioutil.ReadDir(dirPath) if err != nil { @@ -67,17 +71,21 @@ func loadDir(ctx context.Context, dirPath string) (children []string, info dirMa isDir, err := isDirOrSymlinkToDir(dirPath, f) // Skip invalid symlinks if err != nil { + log.Error(ctx, "Invalid symlink", "dir", dirPath) continue } if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) { children = append(children, filepath.Join(dirPath, f.Name())) } else { - if f.ModTime().After(info.modTime) { - info.modTime = f.ModTime() + if f.ModTime().After(stats.ModTime) { + stats.ModTime = f.ModTime() + } + if utils.IsAudioFile(f.Name()) { + stats.AudioFilesCount++ + } else { + stats.HasPlaylist = stats.HasPlaylist || utils.IsPlaylist(f.Name()) + stats.HasImages = stats.HasImages || utils.IsImageFile(f.Name()) } - info.hasImages = info.hasImages || utils.IsImageFile(f.Name()) - info.hasPlaylist = info.hasPlaylist || utils.IsPlaylist(f.Name()) - info.hasAudioFiles = info.hasAudioFiles || utils.IsAudioFile(f.Name()) } } return diff --git a/scanner/load_tree_test.go b/scanner/walk_dir_tree_test.go similarity index 63% rename from scanner/load_tree_test.go rename to scanner/walk_dir_tree_test.go index e645e6f9a..47c58d0ed 100644 --- a/scanner/load_tree_test.go +++ b/scanner/walk_dir_tree_test.go @@ -1,14 +1,46 @@ package scanner import ( + "context" "os" "path/filepath" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" ) var _ = Describe("load_tree", func() { + + Describe("walkDirTree", func() { + It("reads all info correctly", func() { + var collected = dirMap{} + results := make(walkResults, 5000) + var err error + go func() { + err = walkDirTree(context.TODO(), "tests/fixtures", results) + }() + + for { + stats, more := <-results + if !more { + break + } + collected[stats.Path] = stats + } + + Expect(err).To(BeNil()) + Expect(collected["tests/fixtures"]).To(MatchFields(IgnoreExtras, Fields{ + "HasImages": BeTrue(), + "HasPlaylist": BeFalse(), + "AudioFilesCount": BeNumerically("==", 4), + })) + Expect(collected["tests/fixtures/playlists"].HasPlaylist).To(BeTrue()) + Expect(collected).To(HaveKey("tests/fixtures/symlink2dir")) + Expect(collected).To(HaveKey("tests/fixtures/empty_folder")) + }) + }) + Describe("isDirOrSymlinkToDir", func() { It("returns true for normal dirs", func() { dir, _ := os.Stat("tests/fixtures") diff --git a/tests/fixtures/symlink2dir b/tests/fixtures/symlink2dir index b870225aa..d7b73dc42 120000 --- a/tests/fixtures/symlink2dir +++ b/tests/fixtures/symlink2dir @@ -1 +1 @@ -../ \ No newline at end of file +empty_folder \ No newline at end of file