diff --git a/model/album.go b/model/album.go index d5bf574c5..5e3f1b8b8 100644 --- a/model/album.go +++ b/model/album.go @@ -12,6 +12,7 @@ type Album struct { Annotations `structs:"-"` ID string `structs:"id" json:"id"` + LibraryID int `structs:"library_id" json:"libraryId"` Name string `structs:"name" json:"name"` EmbedArtPath string `structs:"embed_art_path" json:"embedArtPath"` ArtistID string `structs:"artist_id" json:"artistId"` diff --git a/model/library.go b/model/library.go index 72ef14569..dc37cd505 100644 --- a/model/library.go +++ b/model/library.go @@ -25,5 +25,8 @@ type Libraries []Library type LibraryRepository interface { Get(id int) (*Library, error) Put(*Library) error + StoreMusicFolder() error + AddArtist(id int, artistID string) error + UpdateLastScan(id int, t time.Time) error GetAll(...QueryOptions) (Libraries, error) } diff --git a/model/mediafile.go b/model/mediafile.go index 79a3bc0df..c1e4f87fb 100644 --- a/model/mediafile.go +++ b/model/mediafile.go @@ -21,6 +21,7 @@ type MediaFile struct { Bookmarkable `structs:"-"` ID string `structs:"id" json:"id"` + LibraryID int `structs:"library_id" json:"libraryId"` Path string `structs:"path" json:"path"` Title string `structs:"title" json:"title"` Album string `structs:"album" json:"album"` diff --git a/model/properties.go b/model/properties.go index 1247edec7..06bb9ebec 100644 --- a/model/properties.go +++ b/model/properties.go @@ -1,10 +1,5 @@ package model -const ( - // TODO Move other prop keys to here - PropLastScan = "LastScan" -) - type PropertyRepository interface { Put(id string, value string) error Get(id string) (string, error) diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index 5cc278daa..34949c4fe 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -100,7 +100,7 @@ var _ = Describe("AlbumRepository", func() { conf.Server.AlbumPlayCountMode = consts.AlbumPlayCountModeAbsolute id := uuid.NewString() - Expect(repo.Put(&model.Album{ID: id, Name: "name", SongCount: songCount})).To(Succeed()) + Expect(repo.Put(&model.Album{LibraryID: 1, ID: id, Name: "name", SongCount: songCount})).To(Succeed()) for i := 0; i < playCount; i++ { Expect(repo.IncPlayCount(id, time.Now())).To(Succeed()) } @@ -123,7 +123,7 @@ var _ = Describe("AlbumRepository", func() { conf.Server.AlbumPlayCountMode = consts.AlbumPlayCountModeNormalized id := uuid.NewString() - Expect(repo.Put(&model.Album{ID: id, Name: "name", SongCount: songCount})).To(Succeed()) + Expect(repo.Put(&model.Album{LibraryID: 1, ID: id, Name: "name", SongCount: songCount})).To(Succeed()) for i := 0; i < playCount; i++ { Expect(repo.IncPlayCount(id, time.Now())).To(Succeed()) } diff --git a/persistence/library_repository.go b/persistence/library_repository.go index eef2ef875..e5009eef9 100644 --- a/persistence/library_repository.go +++ b/persistence/library_repository.go @@ -5,13 +5,13 @@ import ( "time" . "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/model" "github.com/pocketbase/dbx" ) type libraryRepository struct { sqlRepository - sqlRestful } func NewLibraryRepository(ctx context.Context, db dbx.Builder) model.LibraryRepository { @@ -31,19 +31,44 @@ func (r *libraryRepository) Get(id int) (*model.Library, error) { func (r *libraryRepository) Put(l *model.Library) error { cols := map[string]any{ - "name": l.Name, - "path": l.Path, - "remote_path": l.RemotePath, - "last_scan_at": l.LastScanAt, - "updated_at": time.Now(), + "name": l.Name, + "path": l.Path, + "remote_path": l.RemotePath, + "updated_at": time.Now(), } if l.ID != 0 { cols["id"] = l.ID } sq := Insert(r.tableName).SetMap(cols). - Suffix(`ON CONFLICT(id) DO UPDATE set name = excluded.name, path = excluded.path, - remote_path = excluded.remote_path, last_scan_at = excluded.last_scan_at`) + Suffix(`on conflict(id) do update set name = excluded.name, path = excluded.path, + remote_path = excluded.remote_path, updated_at = excluded.updated_at`) + _, err := r.executeSQL(sq) + return err +} + +const hardCodedMusicFolderID = 1 + +// TODO Remove this method when we have a proper UI to add libraries +func (r *libraryRepository) StoreMusicFolder() error { + sq := Update(r.tableName).Set("path", conf.Server.MusicFolder).Set("updated_at", time.Now()). + Where(Eq{"id": hardCodedMusicFolderID}) + _, err := r.executeSQL(sq) + return err +} + +func (r *libraryRepository) AddArtist(id int, artistID string) error { + sq := Insert("library_artist").Columns("library_id", "artist_id").Values(id, artistID). + Suffix(`on conflict(library_id, artist_id) do nothing`) + _, err := r.executeSQL(sq) + if err != nil { + return err + } + return nil +} + +func (r *libraryRepository) UpdateLastScan(id int, t time.Time) error { + sq := Update(r.tableName).Set("last_scan_at", t).Where(Eq{"id": id}) _, err := r.executeSQL(sq) return err } diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.go index 4310787ad..f85275e69 100644 --- a/persistence/mediafile_repository_test.go +++ b/persistence/mediafile_repository_test.go @@ -41,8 +41,8 @@ var _ = Describe("MediaRepository", func() { }) It("finds tracks by path when using wildcards chars", func() { - Expect(mr.Put(&model.MediaFile{ID: "7001", Path: P("/Find:By'Path/_/123.mp3")})).To(BeNil()) - Expect(mr.Put(&model.MediaFile{ID: "7002", Path: P("/Find:By'Path/1/123.mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: "7001", Path: P("/Find:By'Path/_/123.mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: "7002", Path: P("/Find:By'Path/1/123.mp3")})).To(BeNil()) found, err := mr.FindAllByPath(P("/Find:By'Path/_/")) Expect(err).To(BeNil()) @@ -51,8 +51,8 @@ var _ = Describe("MediaRepository", func() { }) It("finds tracks by path when using UTF8 chars", func() { - Expect(mr.Put(&model.MediaFile{ID: "7010", Path: P("/Пётр Ильич Чайковский/123.mp3")})).To(BeNil()) - Expect(mr.Put(&model.MediaFile{ID: "7011", Path: P("/Пётр Ильич Чайковский/222.mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: "7010", Path: P("/Пётр Ильич Чайковский/123.mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: "7011", Path: P("/Пётр Ильич Чайковский/222.mp3")})).To(BeNil()) found, err := mr.FindAllByPath(P("/Пётр Ильич Чайковский/")) Expect(err).To(BeNil()) @@ -60,8 +60,8 @@ var _ = Describe("MediaRepository", func() { }) It("finds tracks by path case sensitively", func() { - Expect(mr.Put(&model.MediaFile{ID: "7003", Path: P("/Casesensitive/file1.mp3")})).To(BeNil()) - Expect(mr.Put(&model.MediaFile{ID: "7004", Path: P("/casesensitive/file2.mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: "7003", Path: P("/Casesensitive/file1.mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: "7004", Path: P("/casesensitive/file2.mp3")})).To(BeNil()) found, err := mr.FindAllByPath(P("/Casesensitive")) Expect(err).To(BeNil()) @@ -76,7 +76,7 @@ var _ = Describe("MediaRepository", func() { It("delete tracks by id", func() { id := uuid.NewString() - Expect(mr.Put(&model.MediaFile{ID: id})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id})).To(BeNil()) Expect(mr.Delete(id)).To(BeNil()) @@ -86,15 +86,15 @@ var _ = Describe("MediaRepository", func() { It("delete tracks by path", func() { id1 := "6001" - Expect(mr.Put(&model.MediaFile{ID: id1, Path: P("/abc/123/" + id1 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id1, Path: P("/abc/123/" + id1 + ".mp3")})).To(BeNil()) id2 := "6002" - Expect(mr.Put(&model.MediaFile{ID: id2, Path: P("/abc/123/" + id2 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id2, Path: P("/abc/123/" + id2 + ".mp3")})).To(BeNil()) id3 := "6003" - Expect(mr.Put(&model.MediaFile{ID: id3, Path: P("/ab_/" + id3 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id3, Path: P("/ab_/" + id3 + ".mp3")})).To(BeNil()) id4 := "6004" - Expect(mr.Put(&model.MediaFile{ID: id4, Path: P("/abc/" + id4 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id4, Path: P("/abc/" + id4 + ".mp3")})).To(BeNil()) id5 := "6005" - Expect(mr.Put(&model.MediaFile{ID: id5, Path: P("/Ab_/" + id5 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id5, Path: P("/Ab_/" + id5 + ".mp3")})).To(BeNil()) Expect(mr.DeleteByPath(P("/ab_"))).To(Equal(int64(1))) @@ -108,11 +108,11 @@ var _ = Describe("MediaRepository", func() { It("delete tracks by path containing UTF8 chars", func() { id1 := "6011" - Expect(mr.Put(&model.MediaFile{ID: id1, Path: P("/Legião Urbana/" + id1 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id1, Path: P("/Legião Urbana/" + id1 + ".mp3")})).To(BeNil()) id2 := "6012" - Expect(mr.Put(&model.MediaFile{ID: id2, Path: P("/Legião Urbana/" + id2 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id2, Path: P("/Legião Urbana/" + id2 + ".mp3")})).To(BeNil()) id3 := "6003" - Expect(mr.Put(&model.MediaFile{ID: id3, Path: P("/Legião Urbana/" + id3 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id3, Path: P("/Legião Urbana/" + id3 + ".mp3")})).To(BeNil()) Expect(mr.FindAllByPath(P("/Legião Urbana"))).To(HaveLen(3)) Expect(mr.DeleteByPath(P("/Legião Urbana"))).To(Equal(int64(3))) @@ -121,11 +121,11 @@ var _ = Describe("MediaRepository", func() { It("only deletes tracks that match exact path", func() { id1 := "6021" - Expect(mr.Put(&model.MediaFile{ID: id1, Path: P("/music/overlap/Ella Fitzgerald/" + id1 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id1, Path: P("/music/overlap/Ella Fitzgerald/" + id1 + ".mp3")})).To(BeNil()) id2 := "6022" - Expect(mr.Put(&model.MediaFile{ID: id2, Path: P("/music/overlap/Ella Fitzgerald/" + id2 + ".mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id2, Path: P("/music/overlap/Ella Fitzgerald/" + id2 + ".mp3")})).To(BeNil()) id3 := "6023" - Expect(mr.Put(&model.MediaFile{ID: id3, Path: P("/music/overlap/Ella Fitzgerald & Louis Armstrong - They Can't Take That Away From Me.mp3")})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id3, Path: P("/music/overlap/Ella Fitzgerald & Louis Armstrong - They Can't Take That Away From Me.mp3")})).To(BeNil()) Expect(mr.FindAllByPath(P("/music/overlap/Ella Fitzgerald"))).To(HaveLen(2)) Expect(mr.DeleteByPath(P("/music/overlap/Ella Fitzgerald"))).To(Equal(int64(2))) @@ -146,7 +146,7 @@ var _ = Describe("MediaRepository", func() { Context("Annotations", func() { It("increments play count when the tracks does not have annotations", func() { id := "incplay.firsttime" - Expect(mr.Put(&model.MediaFile{ID: id})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id})).To(BeNil()) playDate := time.Now() Expect(mr.IncPlayCount(id, playDate)).To(BeNil()) @@ -159,7 +159,7 @@ var _ = Describe("MediaRepository", func() { It("preserves play date if and only if provided date is older", func() { id := "incplay.playdate" - Expect(mr.Put(&model.MediaFile{ID: id})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id})).To(BeNil()) playDate := time.Now() Expect(mr.IncPlayCount(id, playDate)).To(BeNil()) mf, err := mr.Get(id) @@ -184,7 +184,7 @@ var _ = Describe("MediaRepository", func() { It("increments play count on newly starred items", func() { id := "star.incplay" - Expect(mr.Put(&model.MediaFile{ID: id})).To(BeNil()) + Expect(mr.Put(&model.MediaFile{LibraryID: 1, ID: id})).To(BeNil()) Expect(mr.SetStar(true, id)).To(BeNil()) playDate := time.Now() Expect(mr.IncPlayCount(id, playDate)).To(BeNil()) diff --git a/persistence/persistence_suite_test.go b/persistence/persistence_suite_test.go index 831b93a5d..c9ba6e0d6 100644 --- a/persistence/persistence_suite_test.go +++ b/persistence/persistence_suite_test.go @@ -49,9 +49,9 @@ var ( ) var ( - albumSgtPeppers = model.Album{ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", OrderAlbumName: "sgt peppers", AlbumArtistID: "3", Genre: "Rock", Genres: model.Genres{genreRock}, EmbedArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the", Discs: model.Discs{}} - albumAbbeyRoad = model.Album{ID: "102", Name: "Abbey Road", Artist: "The Beatles", OrderAlbumName: "abbey road", AlbumArtistID: "3", Genre: "Rock", Genres: model.Genres{genreRock}, EmbedArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the", Discs: model.Discs{}} - albumRadioactivity = model.Album{ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", OrderAlbumName: "radioactivity", AlbumArtistID: "2", Genre: "Electronic", Genres: model.Genres{genreElectronic, genreRock}, EmbedArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity", Discs: model.Discs{}} + albumSgtPeppers = model.Album{LibraryID: 1, ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", OrderAlbumName: "sgt peppers", AlbumArtistID: "3", Genre: "Rock", Genres: model.Genres{genreRock}, EmbedArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the", Discs: model.Discs{}} + albumAbbeyRoad = model.Album{LibraryID: 1, ID: "102", Name: "Abbey Road", Artist: "The Beatles", OrderAlbumName: "abbey road", AlbumArtistID: "3", Genre: "Rock", Genres: model.Genres{genreRock}, EmbedArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the", Discs: model.Discs{}} + albumRadioactivity = model.Album{LibraryID: 1, ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", OrderAlbumName: "radioactivity", AlbumArtistID: "2", Genre: "Electronic", Genres: model.Genres{genreElectronic, genreRock}, EmbedArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity", Discs: model.Discs{}} testAlbums = model.Albums{ albumSgtPeppers, albumAbbeyRoad, @@ -60,10 +60,10 @@ var ( ) var ( - songDayInALife = model.MediaFile{ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Genres: model.Genres{genreRock}, Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"} - songComeTogether = model.MediaFile{ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Genres: model.Genres{genreRock}, Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"} - songRadioactivity = model.MediaFile{ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Genres: model.Genres{genreElectronic}, Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"} - songAntenna = model.MediaFile{ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", + songDayInALife = model.MediaFile{LibraryID: 1, ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Genres: model.Genres{genreRock}, Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"} + songComeTogether = model.MediaFile{LibraryID: 1, ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Genres: model.Genres{genreRock}, Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"} + songRadioactivity = model.MediaFile{LibraryID: 1, ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Genres: model.Genres{genreElectronic}, Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"} + songAntenna = model.MediaFile{LibraryID: 1, ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Genre: "Electronic", Genres: model.Genres{genreElectronic, genreRock}, Path: P("/kraft/radio/antenna.mp3"), FullText: " antenna kraftwerk", RgAlbumGain: 1.0, RgAlbumPeak: 2.0, RgTrackGain: 3.0, RgTrackPeak: 4.0, diff --git a/scanner/refresher.go b/scanner/refresher.go index 16ced354c..62d48bc73 100644 --- a/scanner/refresher.go +++ b/scanner/refresher.go @@ -12,7 +12,6 @@ import ( "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" - . "github.com/navidrome/navidrome/utils/gg" "github.com/navidrome/navidrome/utils/slice" "golang.org/x/exp/maps" ) @@ -24,15 +23,17 @@ import ( // The actual mappings happen in MediaFiles.ToAlbum() and Albums.ToAlbumArtist() type refresher struct { ds model.DataStore + lib model.Library album map[string]struct{} artist map[string]struct{} dirMap dirMap cacheWarmer artwork.CacheWarmer } -func newRefresher(ds model.DataStore, cw artwork.CacheWarmer, dirMap dirMap) *refresher { +func newRefresher(ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, dirMap dirMap) *refresher { return &refresher{ ds: ds, + lib: lib, album: map[string]struct{}{}, artist: map[string]struct{}{}, dirMap: dirMap, @@ -101,6 +102,7 @@ func (r *refresher) refreshAlbums(ctx context.Context, ids ...string) error { if updatedAt.After(a.UpdatedAt) { a.UpdatedAt = updatedAt } + a.LibraryID = r.lib.ID err := repo.Put(&a) if err != nil { return err @@ -135,18 +137,25 @@ func (r *refresher) refreshArtists(ctx context.Context, ids ...string) error { } repo := r.ds.Artist(ctx) + libRepo := r.ds.Library(ctx) grouped := slice.Group(albums, func(al model.Album) string { return al.AlbumArtistID }) for _, group := range grouped { a := model.Albums(group).ToAlbumArtist() - // Force a external metadata lookup on next access - a.ExternalInfoUpdatedAt = P(time.Time{}) + // Force an external metadata lookup on next access + a.ExternalInfoUpdatedAt = &time.Time{} // Do not remove old metadata err := repo.Put(&a, "album_count", "genres", "external_info_updated_at", "mbz_artist_id", "name", "order_artist_name", "size", "sort_artist_name", "song_count") if err != nil { return err } + + // Link the artist to the current library being scanned + err = libRepo.AddArtist(r.lib.ID, a.ID) + if err != nil { + return err + } r.cacheWarmer.PreCache(a.CoverArtID()) } return nil diff --git a/scanner/scanner.go b/scanner/scanner.go index 3fd3b4deb..fbf12cf20 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strconv" "sync" "time" @@ -36,13 +35,15 @@ var ( type FolderScanner interface { // Scan process finds any changes after `lastModifiedSince` and returns the number of changes found - Scan(ctx context.Context, lastModifiedSince time.Time, progress chan uint32) (int64, error) + Scan(ctx context.Context, fullRescan bool, progress chan uint32) (int64, error) } var isScanning sync.Mutex type scanner struct { + once sync.Once folders map[string]FolderScanner + libs map[string]model.Library status map[string]*scanStatus lock *sync.RWMutex ds model.DataStore @@ -65,6 +66,7 @@ func GetInstance(ds model.DataStore, playlists core.Playlists, cacheWarmer artwo pls: playlists, broker: broker, folders: map[string]FolderScanner{}, + libs: map[string]model.Library{}, status: map[string]*scanStatus{}, lock: &sync.RWMutex{}, cacheWarmer: cacheWarmer, @@ -78,21 +80,25 @@ func (s *scanner) rescan(ctx context.Context, library string, fullRescan bool) e folderScanner := s.folders[library] start := time.Now() + lib, ok := s.libs[library] + if !ok { + log.Error(ctx, "Folder not a valid library path", "folder", library) + return fmt.Errorf("folder %s not a valid library path", library) + } + s.setStatusStart(library) defer s.setStatusEnd(library, start) - lastModifiedSince := time.Time{} - if !fullRescan { - lastModifiedSince = s.getLastModifiedSince(ctx, library) - log.Debug("Scanning folder", "folder", library, "lastModifiedSince", lastModifiedSince) - } else { + if fullRescan { log.Debug("Scanning folder (full scan)", "folder", library) + } else { + log.Debug("Scanning folder", "folder", library, "lastScan", lib.LastScanAt) } progress, cancel := s.startProgressTracker(library) defer cancel() - changeCount, err := folderScanner.Scan(ctx, lastModifiedSince, progress) + changeCount, err := folderScanner.Scan(ctx, fullRescan, progress) if err != nil { log.Error("Error scanning Library", "folder", library, err) } @@ -104,11 +110,12 @@ func (s *scanner) rescan(ctx context.Context, library string, fullRescan bool) e s.broker.SendMessage(context.Background(), &events.RefreshResource{}) } - s.updateLastModifiedSince(library, start) + s.updateLastModifiedSince(ctx, library, start) return err } func (s *scanner) startProgressTracker(library string) (chan uint32, context.CancelFunc) { + // Must be a new context (not the one passed to the scan method) to allow broadcasting the scan status to all clients ctx, cancel := context.WithCancel(context.Background()) progress := make(chan uint32, 100) go func() { @@ -182,6 +189,8 @@ func (s *scanner) setStatusEnd(folder string, lastUpdate time.Time) { func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error { ctx = context.WithoutCancel(ctx) + s.once.Do(s.loadFolders) + if !isScanning.TryLock() { log.Debug(ctx, "Scanner already running, ignoring request for rescan.") return ErrAlreadyScanning @@ -203,6 +212,7 @@ func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error { } func (s *scanner) Status(library string) (*StatusInfo, error) { + s.once.Do(s.loadFolders) status, ok := s.getStatus(library) if !ok { return nil, errors.New("library not found") @@ -216,40 +226,32 @@ func (s *scanner) Status(library string) (*StatusInfo, error) { }, nil } -func (s *scanner) getLastModifiedSince(ctx context.Context, folder string) time.Time { - ms, err := s.ds.Property(ctx).Get(model.PropLastScan + "-" + folder) - if err != nil { - return time.Time{} - } - if ms == "" { - return time.Time{} - } - i, _ := strconv.ParseInt(ms, 10, 64) - return time.Unix(0, i*int64(time.Millisecond)) -} - -func (s *scanner) updateLastModifiedSince(folder string, t time.Time) { - millis := t.UnixNano() / int64(time.Millisecond) - if err := s.ds.Property(context.TODO()).Put(model.PropLastScan+"-"+folder, fmt.Sprint(millis)); err != nil { +func (s *scanner) updateLastModifiedSince(ctx context.Context, folder string, t time.Time) { + lib := s.libs[folder] + id := lib.ID + if err := s.ds.Library(ctx).UpdateLastScan(id, t); err != nil { log.Error("Error updating DB after scan", err) } + lib.LastScanAt = t + s.libs[folder] = lib } func (s *scanner) loadFolders() { ctx := context.TODO() - fs, _ := s.ds.Library(ctx).GetAll() - for _, f := range fs { - log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path) - s.folders[f.Path] = s.newScanner(f) - s.status[f.Path] = &scanStatus{ + libs, _ := s.ds.Library(ctx).GetAll() + for _, lib := range libs { + log.Info("Configuring Media Folder", "name", lib.Name, "path", lib.Path) + s.folders[lib.Path] = s.newScanner(lib) + s.libs[lib.Path] = lib + s.status[lib.Path] = &scanStatus{ active: false, fileCount: 0, folderCount: 0, - lastUpdate: s.getLastModifiedSince(ctx, f.Path), + lastUpdate: lib.LastScanAt, } } } func (s *scanner) newScanner(f model.Library) FolderScanner { - return NewTagScanner(f.Path, s.ds, s.pls, s.cacheWarmer) + return NewTagScanner(f, s.ds, s.pls, s.cacheWarmer) } diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index fc01ce8a3..9bb5be4e9 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -23,7 +23,7 @@ import ( ) type TagScanner struct { - rootFolder string + lib model.Library ds model.DataStore plsSync *playlistImporter cnt *counters @@ -31,10 +31,10 @@ type TagScanner struct { cacheWarmer artwork.CacheWarmer } -func NewTagScanner(rootFolder string, ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.CacheWarmer) FolderScanner { +func NewTagScanner(lib model.Library, ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.CacheWarmer) FolderScanner { s := &TagScanner{ - rootFolder: rootFolder, - plsSync: newPlaylistImporter(ds, playlists, cacheWarmer, rootFolder), + lib: lib, + plsSync: newPlaylistImporter(ds, playlists, cacheWarmer, lib.Path), ds: ds, cacheWarmer: cacheWarmer, } @@ -75,20 +75,20 @@ const ( // - If the playlist is not in the DB, import it, setting sync = true // - If the playlist is in the DB and sync == true, import it, or else skip it // Delete all empty albums, delete all empty artists, clean-up playlists -func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, progress chan uint32) (int64, error) { +func (s *TagScanner) Scan(ctx context.Context, fullScan bool, progress chan uint32) (int64, error) { ctx = auth.WithAdminUser(ctx, s.ds) start := time.Now() - // Special case: if lastModifiedSince is zero, re-import all files - fullScan := lastModifiedSince.IsZero() + // Special case: if LastScanAt is zero, re-import all files + fullScan = fullScan || s.lib.LastScanAt.IsZero() // If the media folder is empty (no music and no subfolders), abort to avoid deleting all data from DB - empty, err := isDirEmpty(ctx, s.rootFolder) + empty, err := isDirEmpty(ctx, s.lib.Path) if err != nil { return 0, err } if empty && !fullScan { - log.Error(ctx, "Media Folder is empty. Aborting scan.", "folder", s.rootFolder) + log.Error(ctx, "Media Folder is empty. Aborting scan.", "folder", s.lib.Path) return 0, nil } @@ -101,38 +101,36 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog var changedDirs []string s.cnt = &counters{} genres := newCachedGenreRepository(ctx, s.ds.Genre(ctx)) - s.mapper = NewMediaFileMapper(s.rootFolder, genres) - refresher := newRefresher(s.ds, s.cacheWarmer, allFSDirs) + s.mapper = NewMediaFileMapper(s.lib.Path, genres) + refresher := newRefresher(s.ds, s.cacheWarmer, s.lib, allFSDirs) - log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder) - foldersFound, walkerError := walkDirTree(ctx, s.rootFolder) + log.Trace(ctx, "Loading directory tree from music folder", "folder", s.lib.Path) + foldersFound, walkerError := walkDirTree(ctx, s.lib.Path) - for { - folderStats, more := <-foldersFound - if !more { - break - } - progress <- folderStats.AudioFilesCount - allFSDirs[folderStats.Path] = folderStats + go func() { + for folderStats := range foldersFound { + progress <- folderStats.AudioFilesCount + allFSDirs[folderStats.Path] = folderStats - if s.folderHasChanged(folderStats, allDBDirs, lastModifiedSince) { - changedDirs = append(changedDirs, folderStats.Path) - log.Debug("Processing changed folder", "dir", folderStats.Path) - err := s.processChangedDir(ctx, refresher, fullScan, folderStats.Path) - if err != nil { - log.Error("Error updating folder in the DB", "dir", folderStats.Path, err) + if s.folderHasChanged(folderStats, allDBDirs, s.lib.LastScanAt) || fullScan { + changedDirs = append(changedDirs, folderStats.Path) + log.Debug("Processing changed folder", "dir", folderStats.Path) + err := s.processChangedDir(ctx, refresher, fullScan, folderStats.Path) + if err != nil { + log.Error("Error updating folder in the DB", "dir", folderStats.Path, err) + } } } - } + }() - if err := <-walkerError; err != nil { + for err := range walkerError { log.Error("Scan was interrupted by error. See errors above", err) return 0, err } deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs) if len(deletedDirs)+len(changedDirs) == 0 { - log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start)) + log.Debug(ctx, "No changes found in Music Folder", "folder", s.lib.Path, "elapsed", time.Since(start)) return 0, nil } @@ -162,8 +160,8 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog log.Debug("Playlist auto-import is disabled") } - err = s.ds.GC(log.NewContext(ctx), s.rootFolder) - log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start), + err = s.ds.GC(log.NewContext(ctx), s.lib.Path) + log.Info("Finished processing Music Folder", "folder", s.lib.Path, "elapsed", time.Since(start), "added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", s.cnt.playlists) return s.cnt.total(), err @@ -179,10 +177,10 @@ func isDirEmpty(ctx context.Context, dir string) (bool, error) { func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) { start := time.Now() - log.Trace(ctx, "Loading directory tree from database", "folder", s.rootFolder) + log.Trace(ctx, "Loading directory tree from database", "folder", s.lib.Path) repo := s.ds.MediaFile(ctx) - dirs, err := repo.FindPathsRecursively(s.rootFolder) + dirs, err := repo.FindPathsRecursively(s.lib.Path) if err != nil { return nil, err } @@ -368,6 +366,7 @@ func (s *TagScanner) addOrUpdateTracksInDB( if t, ok := currentTracks[n.Path]; ok { n.Annotations = t.Annotations } + n.LibraryID = s.lib.ID err := s.ds.MediaFile(ctx).Put(&n) if err != nil { return 0, err diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 7157e74ca..59a622848 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -26,7 +26,7 @@ type ( } ) -func walkDirTree(ctx context.Context, rootFolder string) (<-chan dirStats, chan error) { +func walkDirTree(ctx context.Context, rootFolder string) (<-chan dirStats, <-chan error) { results := make(chan dirStats) errC := make(chan error) go func() { diff --git a/server/initial_setup.go b/server/initial_setup.go index af2430b10..b5533c6b6 100644 --- a/server/initial_setup.go +++ b/server/initial_setup.go @@ -15,12 +15,13 @@ import ( ) func initialSetup(ds model.DataStore) { + ctx := context.TODO() _ = ds.WithTx(func(tx model.DataStore) error { - if err := createOrUpdateMusicFolder(ds); err != nil { + if err := ds.Library(ctx).StoreMusicFolder(); err != nil { return err } - properties := ds.Property(context.TODO()) + properties := ds.Property(ctx) _, err := properties.Get(consts.InitialSetupFlagKey) if err == nil { return nil @@ -116,12 +117,3 @@ func checkExternalCredentials() { } } } - -func createOrUpdateMusicFolder(ds model.DataStore) error { - lib := model.Library{ID: 1, Name: "Music Library", Path: conf.Server.MusicFolder} - err := ds.Library(context.TODO()).Put(&lib) - if err != nil { - log.Error("Could not access Library table", err) - } - return err -} diff --git a/server/subsonic/browsing.go b/server/subsonic/browsing.go index a60c662d8..effa9b2cf 100644 --- a/server/subsonic/browsing.go +++ b/server/subsonic/browsing.go @@ -4,7 +4,6 @@ import ( "context" "errors" "net/http" - "strconv" "time" "github.com/navidrome/navidrome/conf" @@ -30,22 +29,14 @@ func (api *Router) GetMusicFolders(r *http.Request) (*responses.Subsonic, error) func (api *Router) getArtistIndex(r *http.Request, libId int, ifModifiedSince time.Time) (*responses.Indexes, error) { ctx := r.Context() - folder, err := api.ds.Library(ctx).Get(libId) + lib, err := api.ds.Library(ctx).Get(libId) if err != nil { log.Error(ctx, "Error retrieving Library", "id", libId, err) return nil, err } - l, err := api.ds.Property(ctx).DefaultGet(model.PropLastScan+"-"+folder.Path, "-1") - if err != nil { - log.Error(ctx, "Error retrieving LastScan property", err) - return nil, err - } - var indexes model.ArtistIndexes - ms, _ := strconv.ParseInt(l, 10, 64) - lastModified := time.UnixMilli(ms) - if lastModified.After(ifModifiedSince) { + if lib.LastScanAt.After(ifModifiedSince) { indexes, err = api.ds.Artist(ctx).GetIndex() if err != nil { log.Error(ctx, "Error retrieving Indexes", err) @@ -55,7 +46,7 @@ func (api *Router) getArtistIndex(r *http.Request, libId int, ifModifiedSince ti res := &responses.Indexes{ IgnoredArticles: conf.Server.IgnoredArticles, - LastModified: lastModified.UnixMilli(), + LastModified: lib.LastScanAt.UnixMilli(), } res.Index = make([]responses.Index, len(indexes))